From f215e02bf85f68d3a6106c2a1f4f7f063f819064 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 11 Apr 2024 10:17:27 +0200 Subject: Adding upstream version 7.0.14-dfsg. Signed-off-by: Daniel Baumann --- src/VBox/VMM/VMMAll/APICAll.cpp | 3655 ++++ src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp | 111 + src/VBox/VMM/VMMAll/CPUMAllCpuId.cpp | 1599 ++ src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp | 6505 +++++++ src/VBox/VMM/VMMAll/CPUMAllRegs.cpp | 3039 ++++ src/VBox/VMM/VMMAll/DBGFAll.cpp | 592 + src/VBox/VMM/VMMAll/DBGFAllBp.cpp | 604 + src/VBox/VMM/VMMAll/DBGFAllTracer.cpp | 715 + src/VBox/VMM/VMMAll/EMAll.cpp | 1018 ++ src/VBox/VMM/VMMAll/GCMAll.cpp | 245 + src/VBox/VMM/VMMAll/GIMAll.cpp | 506 + src/VBox/VMM/VMMAll/GIMAllHv.cpp | 1495 ++ src/VBox/VMM/VMMAll/GIMAllKvm.cpp | 450 + src/VBox/VMM/VMMAll/HMAll.cpp | 904 + src/VBox/VMM/VMMAll/HMSVMAll.cpp | 553 + src/VBox/VMM/VMMAll/HMVMXAll.cpp | 1346 ++ src/VBox/VMM/VMMAll/IEMAll.cpp | 11599 ++++++++++++ src/VBox/VMM/VMMAll/IEMAllAImpl.asm | 6458 +++++++ src/VBox/VMM/VMMAll/IEMAllAImplC.cpp | 17407 +++++++++++++++++++ src/VBox/VMM/VMMAll/IEMAllCImpl.cpp | 9671 +++++++++++ src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h | 1773 ++ src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp | 1622 ++ src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp | 10000 +++++++++++ src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h | 143 + .../VMM/VMMAll/IEMAllInstructionsInterpretOnly.cpp | 1678 ++ .../VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h | 11860 +++++++++++++ src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py | 3953 +++++ .../VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h | 2187 +++ .../VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h | 1549 ++ .../VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h | 13975 +++++++++++++++ .../VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h | 5642 ++++++ .../VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h | 2107 +++ .../VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h | 1004 ++ src/VBox/VMM/VMMAll/IOMAll.cpp | 618 + src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp | 1320 ++ src/VBox/VMM/VMMAll/MMAll.cpp | 160 + src/VBox/VMM/VMMAll/Makefile.kup | 0 src/VBox/VMM/VMMAll/NEMAll.cpp | 136 + src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h | 3038 ++++ src/VBox/VMM/VMMAll/PDMAll.cpp | 428 + src/VBox/VMM/VMMAll/PDMAllCritSect.cpp | 1208 ++ src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp | 118 + src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp | 2134 +++ src/VBox/VMM/VMMAll/PDMAllIommu.cpp | 468 + src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp | 139 + src/VBox/VMM/VMMAll/PDMAllQueue.cpp | 313 + src/VBox/VMM/VMMAll/PDMAllTask.cpp | 124 + src/VBox/VMM/VMMAll/PGMAll.cpp | 4077 +++++ src/VBox/VMM/VMMAll/PGMAllBth.h | 5268 ++++++ src/VBox/VMM/VMMAll/PGMAllGst.h | 545 + src/VBox/VMM/VMMAll/PGMAllGstSlatEpt.cpp.h | 408 + src/VBox/VMM/VMMAll/PGMAllHandler.cpp | 2001 +++ src/VBox/VMM/VMMAll/PGMAllPhys.cpp | 4151 +++++ src/VBox/VMM/VMMAll/PGMAllPool.cpp | 5898 +++++++ src/VBox/VMM/VMMAll/PGMAllShw.h | 647 + src/VBox/VMM/VMMAll/SELMAll.cpp | 392 + src/VBox/VMM/VMMAll/TMAll.cpp | 2850 +++ src/VBox/VMM/VMMAll/TMAllCpu.cpp | 661 + src/VBox/VMM/VMMAll/TMAllReal.cpp | 63 + src/VBox/VMM/VMMAll/TMAllVirtual.cpp | 1154 ++ src/VBox/VMM/VMMAll/TRPMAll.cpp | 438 + src/VBox/VMM/VMMAll/VMAll.cpp | 444 + src/VBox/VMM/VMMAll/VMMAll.cpp | 296 + src/VBox/VMM/VMMAll/VMMAllA.asm | 93 + src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h | 12003 +++++++++++++ 65 files changed, 177558 insertions(+) create mode 100644 src/VBox/VMM/VMMAll/APICAll.cpp create mode 100644 src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp create mode 100644 src/VBox/VMM/VMMAll/CPUMAllCpuId.cpp create mode 100644 src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp create mode 100644 src/VBox/VMM/VMMAll/CPUMAllRegs.cpp create mode 100644 src/VBox/VMM/VMMAll/DBGFAll.cpp create mode 100644 src/VBox/VMM/VMMAll/DBGFAllBp.cpp create mode 100644 src/VBox/VMM/VMMAll/DBGFAllTracer.cpp create mode 100644 src/VBox/VMM/VMMAll/EMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/GCMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/GIMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/GIMAllHv.cpp create mode 100644 src/VBox/VMM/VMMAll/GIMAllKvm.cpp create mode 100644 src/VBox/VMM/VMMAll/HMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/HMSVMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/HMVMXAll.cpp create mode 100644 src/VBox/VMM/VMMAll/IEMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/IEMAllAImpl.asm create mode 100644 src/VBox/VMM/VMMAll/IEMAllAImplC.cpp create mode 100644 src/VBox/VMM/VMMAll/IEMAllCImpl.cpp create mode 100644 src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h create mode 100644 src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp create mode 100644 src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsInterpretOnly.cpp create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h create mode 100755 src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h create mode 100644 src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h create mode 100644 src/VBox/VMM/VMMAll/IOMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp create mode 100644 src/VBox/VMM/VMMAll/MMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/Makefile.kup create mode 100644 src/VBox/VMM/VMMAll/NEMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h create mode 100644 src/VBox/VMM/VMMAll/PDMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/PDMAllCritSect.cpp create mode 100644 src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp create mode 100644 src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp create mode 100644 src/VBox/VMM/VMMAll/PDMAllIommu.cpp create mode 100644 src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp create mode 100644 src/VBox/VMM/VMMAll/PDMAllQueue.cpp create mode 100644 src/VBox/VMM/VMMAll/PDMAllTask.cpp create mode 100644 src/VBox/VMM/VMMAll/PGMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/PGMAllBth.h create mode 100644 src/VBox/VMM/VMMAll/PGMAllGst.h create mode 100644 src/VBox/VMM/VMMAll/PGMAllGstSlatEpt.cpp.h create mode 100644 src/VBox/VMM/VMMAll/PGMAllHandler.cpp create mode 100644 src/VBox/VMM/VMMAll/PGMAllPhys.cpp create mode 100644 src/VBox/VMM/VMMAll/PGMAllPool.cpp create mode 100644 src/VBox/VMM/VMMAll/PGMAllShw.h create mode 100644 src/VBox/VMM/VMMAll/SELMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/TMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/TMAllCpu.cpp create mode 100644 src/VBox/VMM/VMMAll/TMAllReal.cpp create mode 100644 src/VBox/VMM/VMMAll/TMAllVirtual.cpp create mode 100644 src/VBox/VMM/VMMAll/TRPMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/VMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/VMMAll.cpp create mode 100644 src/VBox/VMM/VMMAll/VMMAllA.asm create mode 100644 src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h (limited to 'src/VBox/VMM/VMMAll') diff --git a/src/VBox/VMM/VMMAll/APICAll.cpp b/src/VBox/VMM/VMMAll/APICAll.cpp new file mode 100644 index 00000000..6041a843 --- /dev/null +++ b/src/VBox/VMM/VMMAll/APICAll.cpp @@ -0,0 +1,3655 @@ +/* $Id: APICAll.cpp $ */ +/** @file + * APIC - Advanced Programmable Interrupt Controller - All Contexts. + */ + +/* + * Copyright (C) 2016-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_DEV_APIC +#define VMCPU_INCL_CPUM_GST_CTX /* for macOS hack */ +#include "APICInternal.h" +#include +#include +#include +#include +#include +#include +#ifdef IN_RING0 +# include +#endif + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +static void apicSetInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType); +static void apicStopTimer(PVMCPUCC pVCpu); + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 +/** An ordered array of valid LVT masks. */ +static const uint32_t g_au32LvtValidMasks[] = +{ + XAPIC_LVT_TIMER_VALID, + XAPIC_LVT_THERMAL_VALID, + XAPIC_LVT_PERF_VALID, + XAPIC_LVT_LINT_VALID, /* LINT0 */ + XAPIC_LVT_LINT_VALID, /* LINT1 */ + XAPIC_LVT_ERROR_VALID +}; +#endif + +#if 0 +/** @todo CMCI */ +static const uint32_t g_au32LvtExtValidMask[] = +{ + XAPIC_LVT_CMCI_VALID +}; +#endif + + +/** + * Checks if a vector is set in an APIC 256-bit sparse register. + * + * @returns true if the specified vector is set, false otherwise. + * @param pApicReg The APIC 256-bit spare register. + * @param uVector The vector to check if set. + */ +DECLINLINE(bool) apicTestVectorInReg(const volatile XAPIC256BITREG *pApicReg, uint8_t uVector) +{ + const volatile uint8_t *pbBitmap = (const volatile uint8_t *)&pApicReg->u[0]; + return ASMBitTest(pbBitmap, (XAPIC_REG256_VECTOR_OFF(uVector) << 3) + XAPIC_REG256_VECTOR_BIT(uVector)); +} + + +/** + * Sets the vector in an APIC 256-bit sparse register. + * + * @param pApicReg The APIC 256-bit spare register. + * @param uVector The vector to set. + */ +DECLINLINE(void) apicSetVectorInReg(volatile XAPIC256BITREG *pApicReg, uint8_t uVector) +{ + volatile uint8_t *pbBitmap = (volatile uint8_t *)&pApicReg->u[0]; + ASMAtomicBitSet(pbBitmap, (XAPIC_REG256_VECTOR_OFF(uVector) << 3) + XAPIC_REG256_VECTOR_BIT(uVector)); +} + + +/** + * Clears the vector in an APIC 256-bit sparse register. + * + * @param pApicReg The APIC 256-bit spare register. + * @param uVector The vector to clear. + */ +DECLINLINE(void) apicClearVectorInReg(volatile XAPIC256BITREG *pApicReg, uint8_t uVector) +{ + volatile uint8_t *pbBitmap = (volatile uint8_t *)&pApicReg->u[0]; + ASMAtomicBitClear(pbBitmap, (XAPIC_REG256_VECTOR_OFF(uVector) << 3) + XAPIC_REG256_VECTOR_BIT(uVector)); +} + + +#if 0 /* unused */ +/** + * Checks if a vector is set in an APIC Pending-Interrupt Bitmap (PIB). + * + * @returns true if the specified vector is set, false otherwise. + * @param pvPib Opaque pointer to the PIB. + * @param uVector The vector to check if set. + */ +DECLINLINE(bool) apicTestVectorInPib(volatile void *pvPib, uint8_t uVector) +{ + return ASMBitTest(pvPib, uVector); +} +#endif /* unused */ + + +/** + * Atomically sets the PIB notification bit. + * + * @returns non-zero if the bit was already set, 0 otherwise. + * @param pApicPib Pointer to the PIB. + */ +DECLINLINE(uint32_t) apicSetNotificationBitInPib(PAPICPIB pApicPib) +{ + return ASMAtomicXchgU32(&pApicPib->fOutstandingNotification, RT_BIT_32(31)); +} + + +/** + * Atomically tests and clears the PIB notification bit. + * + * @returns non-zero if the bit was already set, 0 otherwise. + * @param pApicPib Pointer to the PIB. + */ +DECLINLINE(uint32_t) apicClearNotificationBitInPib(PAPICPIB pApicPib) +{ + return ASMAtomicXchgU32(&pApicPib->fOutstandingNotification, UINT32_C(0)); +} + + +/** + * Sets the vector in an APIC Pending-Interrupt Bitmap (PIB). + * + * @param pvPib Opaque pointer to the PIB. + * @param uVector The vector to set. + */ +DECLINLINE(void) apicSetVectorInPib(volatile void *pvPib, uint8_t uVector) +{ + ASMAtomicBitSet(pvPib, uVector); +} + +#if 0 /* unused */ +/** + * Clears the vector in an APIC Pending-Interrupt Bitmap (PIB). + * + * @param pvPib Opaque pointer to the PIB. + * @param uVector The vector to clear. + */ +DECLINLINE(void) apicClearVectorInPib(volatile void *pvPib, uint8_t uVector) +{ + ASMAtomicBitClear(pvPib, uVector); +} +#endif /* unused */ + +#if 0 /* unused */ +/** + * Atomically OR's a fragment (32 vectors) into an APIC 256-bit sparse + * register. + * + * @param pApicReg The APIC 256-bit spare register. + * @param idxFragment The index of the 32-bit fragment in @a + * pApicReg. + * @param u32Fragment The 32-bit vector fragment to OR. + */ +DECLINLINE(void) apicOrVectorsToReg(volatile XAPIC256BITREG *pApicReg, size_t idxFragment, uint32_t u32Fragment) +{ + Assert(idxFragment < RT_ELEMENTS(pApicReg->u)); + ASMAtomicOrU32(&pApicReg->u[idxFragment].u32Reg, u32Fragment); +} +#endif /* unused */ + + +#if 0 /* unused */ +/** + * Atomically AND's a fragment (32 vectors) into an APIC + * 256-bit sparse register. + * + * @param pApicReg The APIC 256-bit spare register. + * @param idxFragment The index of the 32-bit fragment in @a + * pApicReg. + * @param u32Fragment The 32-bit vector fragment to AND. + */ +DECLINLINE(void) apicAndVectorsToReg(volatile XAPIC256BITREG *pApicReg, size_t idxFragment, uint32_t u32Fragment) +{ + Assert(idxFragment < RT_ELEMENTS(pApicReg->u)); + ASMAtomicAndU32(&pApicReg->u[idxFragment].u32Reg, u32Fragment); +} +#endif /* unused */ + + +/** + * Reports and returns appropriate error code for invalid MSR accesses. + * + * @returns VERR_CPUM_RAISE_GP_0 + * + * @param pVCpu The cross context virtual CPU structure. + * @param u32Reg The MSR being accessed. + * @param enmAccess The invalid-access type. + */ +static int apicMsrAccessError(PVMCPUCC pVCpu, uint32_t u32Reg, APICMSRACCESS enmAccess) +{ + static struct + { + const char *pszBefore; /* The error message before printing the MSR index */ + const char *pszAfter; /* The error message after printing the MSR index */ + } const s_aAccess[] = + { + /* enmAccess pszBefore pszAfter */ + /* 0 */ { "read MSR", " while not in x2APIC mode" }, + /* 1 */ { "write MSR", " while not in x2APIC mode" }, + /* 2 */ { "read reserved/unknown MSR", "" }, + /* 3 */ { "write reserved/unknown MSR", "" }, + /* 4 */ { "read write-only MSR", "" }, + /* 5 */ { "write read-only MSR", "" }, + /* 6 */ { "read reserved bits of MSR", "" }, + /* 7 */ { "write reserved bits of MSR", "" }, + /* 8 */ { "write an invalid value to MSR", "" }, + /* 9 */ { "write MSR", " disallowed by configuration" }, + /* 10 */ { "read MSR", " disallowed by configuration" }, + }; + AssertCompile(RT_ELEMENTS(s_aAccess) == APICMSRACCESS_COUNT); + + size_t const i = enmAccess; + Assert(i < RT_ELEMENTS(s_aAccess)); + if (pVCpu->apic.s.cLogMaxAccessError++ < 5) + LogRel(("APIC%u: Attempt to %s (%#x)%s -> #GP(0)\n", pVCpu->idCpu, s_aAccess[i].pszBefore, u32Reg, s_aAccess[i].pszAfter)); + return VERR_CPUM_RAISE_GP_0; +} + + +/** + * Gets the descriptive APIC mode. + * + * @returns The name. + * @param enmMode The xAPIC mode. + */ +const char *apicGetModeName(APICMODE enmMode) +{ + switch (enmMode) + { + case APICMODE_DISABLED: return "Disabled"; + case APICMODE_XAPIC: return "xAPIC"; + case APICMODE_X2APIC: return "x2APIC"; + default: break; + } + return "Invalid"; +} + + +/** + * Gets the descriptive destination format name. + * + * @returns The destination format name. + * @param enmDestFormat The destination format. + */ +const char *apicGetDestFormatName(XAPICDESTFORMAT enmDestFormat) +{ + switch (enmDestFormat) + { + case XAPICDESTFORMAT_FLAT: return "Flat"; + case XAPICDESTFORMAT_CLUSTER: return "Cluster"; + default: break; + } + return "Invalid"; +} + + +/** + * Gets the descriptive delivery mode name. + * + * @returns The delivery mode name. + * @param enmDeliveryMode The delivery mode. + */ +const char *apicGetDeliveryModeName(XAPICDELIVERYMODE enmDeliveryMode) +{ + switch (enmDeliveryMode) + { + case XAPICDELIVERYMODE_FIXED: return "Fixed"; + case XAPICDELIVERYMODE_LOWEST_PRIO: return "Lowest-priority"; + case XAPICDELIVERYMODE_SMI: return "SMI"; + case XAPICDELIVERYMODE_NMI: return "NMI"; + case XAPICDELIVERYMODE_INIT: return "INIT"; + case XAPICDELIVERYMODE_STARTUP: return "SIPI"; + case XAPICDELIVERYMODE_EXTINT: return "ExtINT"; + default: break; + } + return "Invalid"; +} + + +/** + * Gets the descriptive destination mode name. + * + * @returns The destination mode name. + * @param enmDestMode The destination mode. + */ +const char *apicGetDestModeName(XAPICDESTMODE enmDestMode) +{ + switch (enmDestMode) + { + case XAPICDESTMODE_PHYSICAL: return "Physical"; + case XAPICDESTMODE_LOGICAL: return "Logical"; + default: break; + } + return "Invalid"; +} + + +/** + * Gets the descriptive trigger mode name. + * + * @returns The trigger mode name. + * @param enmTriggerMode The trigger mode. + */ +const char *apicGetTriggerModeName(XAPICTRIGGERMODE enmTriggerMode) +{ + switch (enmTriggerMode) + { + case XAPICTRIGGERMODE_EDGE: return "Edge"; + case XAPICTRIGGERMODE_LEVEL: return "Level"; + default: break; + } + return "Invalid"; +} + + +/** + * Gets the destination shorthand name. + * + * @returns The destination shorthand name. + * @param enmDestShorthand The destination shorthand. + */ +const char *apicGetDestShorthandName(XAPICDESTSHORTHAND enmDestShorthand) +{ + switch (enmDestShorthand) + { + case XAPICDESTSHORTHAND_NONE: return "None"; + case XAPICDESTSHORTHAND_SELF: return "Self"; + case XAPIDDESTSHORTHAND_ALL_INCL_SELF: return "All including self"; + case XAPICDESTSHORTHAND_ALL_EXCL_SELF: return "All excluding self"; + default: break; + } + return "Invalid"; +} + + +/** + * Gets the timer mode name. + * + * @returns The timer mode name. + * @param enmTimerMode The timer mode. + */ +const char *apicGetTimerModeName(XAPICTIMERMODE enmTimerMode) +{ + switch (enmTimerMode) + { + case XAPICTIMERMODE_ONESHOT: return "One-shot"; + case XAPICTIMERMODE_PERIODIC: return "Periodic"; + case XAPICTIMERMODE_TSC_DEADLINE: return "TSC deadline"; + default: break; + } + return "Invalid"; +} + + +/** + * Gets the APIC mode given the base MSR value. + * + * @returns The APIC mode. + * @param uApicBaseMsr The APIC Base MSR value. + */ +APICMODE apicGetMode(uint64_t uApicBaseMsr) +{ + uint32_t const uMode = (uApicBaseMsr >> 10) & UINT64_C(3); + APICMODE const enmMode = (APICMODE)uMode; +#ifdef VBOX_STRICT + /* Paranoia. */ + switch (uMode) + { + case APICMODE_DISABLED: + case APICMODE_INVALID: + case APICMODE_XAPIC: + case APICMODE_X2APIC: + break; + default: + AssertMsgFailed(("Invalid mode")); + } +#endif + return enmMode; +} + + +/** + * Returns whether the APIC is hardware enabled or not. + * + * @returns true if enabled, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) APICIsEnabled(PCVMCPUCC pVCpu) +{ + PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + return RT_BOOL(pApicCpu->uApicBaseMsr & MSR_IA32_APICBASE_EN); +} + + +/** + * Finds the most significant set bit in an APIC 256-bit sparse register. + * + * @returns @a rcNotFound if no bit was set, 0-255 otherwise. + * @param pReg The APIC 256-bit sparse register. + * @param rcNotFound What to return when no bit is set. + */ +static int apicGetHighestSetBitInReg(volatile const XAPIC256BITREG *pReg, int rcNotFound) +{ + ssize_t const cFragments = RT_ELEMENTS(pReg->u); + unsigned const uFragmentShift = 5; + AssertCompile(1 << uFragmentShift == sizeof(pReg->u[0].u32Reg) * 8); + for (ssize_t i = cFragments - 1; i >= 0; i--) + { + uint32_t const uFragment = pReg->u[i].u32Reg; + if (uFragment) + { + unsigned idxSetBit = ASMBitLastSetU32(uFragment); + --idxSetBit; + idxSetBit |= i << uFragmentShift; + return idxSetBit; + } + } + return rcNotFound; +} + + +/** + * Reads a 32-bit register at a specified offset. + * + * @returns The value at the specified offset. + * @param pXApicPage The xAPIC page. + * @param offReg The offset of the register being read. + */ +DECLINLINE(uint32_t) apicReadRaw32(PCXAPICPAGE pXApicPage, uint16_t offReg) +{ + Assert(offReg < sizeof(*pXApicPage) - sizeof(uint32_t)); + uint8_t const *pbXApic = (const uint8_t *)pXApicPage; + uint32_t const uValue = *(const uint32_t *)(pbXApic + offReg); + return uValue; +} + + +/** + * Writes a 32-bit register at a specified offset. + * + * @param pXApicPage The xAPIC page. + * @param offReg The offset of the register being written. + * @param uReg The value of the register. + */ +DECLINLINE(void) apicWriteRaw32(PXAPICPAGE pXApicPage, uint16_t offReg, uint32_t uReg) +{ + Assert(offReg < sizeof(*pXApicPage) - sizeof(uint32_t)); + uint8_t *pbXApic = (uint8_t *)pXApicPage; + *(uint32_t *)(pbXApic + offReg) = uReg; +} + + +/** + * Sets an error in the internal ESR of the specified APIC. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uError The error. + * @thread Any. + */ +DECLINLINE(void) apicSetError(PVMCPUCC pVCpu, uint32_t uError) +{ + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + ASMAtomicOrU32(&pApicCpu->uEsrInternal, uError); +} + + +/** + * Clears all errors in the internal ESR. + * + * @returns The value of the internal ESR before clearing. + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(uint32_t) apicClearAllErrors(PVMCPUCC pVCpu) +{ + VMCPU_ASSERT_EMT(pVCpu); + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + return ASMAtomicXchgU32(&pApicCpu->uEsrInternal, 0); +} + + +/** + * Signals the guest if a pending interrupt is ready to be serviced. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void apicSignalNextPendingIntr(PVMCPUCC pVCpu) +{ + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu); + if (pXApicPage->svr.u.fApicSoftwareEnable) + { + int const irrv = apicGetHighestSetBitInReg(&pXApicPage->irr, -1 /* rcNotFound */); + if (irrv >= 0) + { + Assert(irrv <= (int)UINT8_MAX); + uint8_t const uVector = irrv; + int const isrv = apicGetHighestSetBitInReg(&pXApicPage->isr, 0 /* rcNotFound */); + Assert(isrv <= (int)UINT8_MAX); + uint8_t const uIsrVec = isrv; + + /* uIsrVect reflects the highest interrupt vector currently serviced (i.e. in ISR), + * or zero if there's none. We want to report a pending interrupt only if IRR > ISR but + * regardless of TPR. Hence we can't look at the PPR value, since that also reflects TPR. + * NB: The APIC emulation will know when ISR changes, but not necessarily when TPR does. + */ + if (XAPIC_PPR_GET_PP(uVector) > XAPIC_PPR_GET_PP(uIsrVec)) + { + Log2(("APIC%u: apicSignalNextPendingIntr: Signalling pending interrupt. uVector=%#x\n", pVCpu->idCpu, uVector)); + apicSetInterruptFF(pVCpu, PDMAPICIRQ_HARDWARE); + } + else + Log2(("APIC%u: apicSignalNextPendingIntr: Nothing to signal yet. uVector=%#x uIsrVec=%#x\n", pVCpu->idCpu, uVector, uIsrVec)); + } + } + else + { + Log2(("APIC%u: apicSignalNextPendingIntr: APIC software-disabled, clearing pending interrupt\n", pVCpu->idCpu)); + apicClearInterruptFF(pVCpu, PDMAPICIRQ_HARDWARE); + } +} + + +/** + * Sets the Spurious-Interrupt Vector Register (SVR). + * + * @returns VINF_SUCCESS or VERR_CPUM_RAISE_GP_0. + * @param pVCpu The cross context virtual CPU structure. + * @param uSvr The SVR value. + */ +static int apicSetSvr(PVMCPUCC pVCpu, uint32_t uSvr) +{ + VMCPU_ASSERT_EMT(pVCpu); + + uint32_t uValidMask = XAPIC_SVR_VALID; + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + if (pXApicPage->version.u.fEoiBroadcastSupression) + uValidMask |= XAPIC_SVR_SUPRESS_EOI_BROADCAST; + + if ( XAPIC_IN_X2APIC_MODE(pVCpu) + && (uSvr & ~uValidMask)) + return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_SVR, APICMSRACCESS_WRITE_RSVD_BITS); + + Log2(("APIC%u: apicSetSvr: uSvr=%#RX32\n", pVCpu->idCpu, uSvr)); + apicWriteRaw32(pXApicPage, XAPIC_OFF_SVR, uSvr); + if (!pXApicPage->svr.u.fApicSoftwareEnable) + { + /** @todo CMCI. */ + pXApicPage->lvt_timer.u.u1Mask = 1; +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + pXApicPage->lvt_thermal.u.u1Mask = 1; +#endif + pXApicPage->lvt_perf.u.u1Mask = 1; + pXApicPage->lvt_lint0.u.u1Mask = 1; + pXApicPage->lvt_lint1.u.u1Mask = 1; + pXApicPage->lvt_error.u.u1Mask = 1; + } + + apicSignalNextPendingIntr(pVCpu); + return VINF_SUCCESS; +} + + +/** + * Sends an interrupt to one or more APICs. + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure, can be + * NULL if the source of the interrupt is not an + * APIC (for e.g. a bus). + * @param uVector The interrupt vector. + * @param enmTriggerMode The trigger mode. + * @param enmDeliveryMode The delivery mode. + * @param pDestCpuSet The destination CPU set. + * @param pfIntrAccepted Where to store whether this interrupt was + * accepted by the target APIC(s) or not. + * Optional, can be NULL. + * @param uSrcTag The interrupt source tag (debugging). + * @param rcRZ The return code if the operation cannot be + * performed in the current context. + */ +static VBOXSTRICTRC apicSendIntr(PVMCC pVM, PVMCPUCC pVCpu, uint8_t uVector, XAPICTRIGGERMODE enmTriggerMode, + XAPICDELIVERYMODE enmDeliveryMode, PCVMCPUSET pDestCpuSet, bool *pfIntrAccepted, + uint32_t uSrcTag, int rcRZ) +{ + AssertCompile(sizeof(pVM->apic.s) <= sizeof(pVM->apic.padding)); + AssertCompile(sizeof(pVCpu->apic.s) <= sizeof(pVCpu->apic.padding)); +#ifdef IN_RING0 + AssertCompile(sizeof(pVM->apicr0.s) <= sizeof(pVM->apicr0.padding)); +#endif + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + VMCPUID const cCpus = pVM->cCpus; + bool fAccepted = false; + switch (enmDeliveryMode) + { + case XAPICDELIVERYMODE_FIXED: + { + for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++) + if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu)) + { + PVMCPUCC pItVCpu = pVM->CTX_SUFF(apCpus)[idCpu]; + if (APICIsEnabled(pItVCpu)) + fAccepted = apicPostInterrupt(pItVCpu, uVector, enmTriggerMode, uSrcTag); + } + break; + } + + case XAPICDELIVERYMODE_LOWEST_PRIO: + { + VMCPUID const idCpu = VMCPUSET_FIND_FIRST_PRESENT(pDestCpuSet); + AssertMsgBreak(idCpu < pVM->cCpus, ("APIC: apicSendIntr: No CPU found for lowest-priority delivery mode! idCpu=%u\n", idCpu)); + PVMCPUCC pVCpuDst = pVM->CTX_SUFF(apCpus)[idCpu]; + if (APICIsEnabled(pVCpuDst)) + fAccepted = apicPostInterrupt(pVCpuDst, uVector, enmTriggerMode, uSrcTag); + else + AssertMsgFailed(("APIC: apicSendIntr: Target APIC not enabled in lowest-priority delivery mode! idCpu=%u\n", idCpu)); + break; + } + + case XAPICDELIVERYMODE_SMI: + { + for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++) + if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu)) + { + Log2(("APIC: apicSendIntr: Raising SMI on VCPU%u\n", idCpu)); + apicSetInterruptFF(pVM->CTX_SUFF(apCpus)[idCpu], PDMAPICIRQ_SMI); + fAccepted = true; + } + break; + } + + case XAPICDELIVERYMODE_NMI: + { + for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++) + if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu)) + { + PVMCPUCC pItVCpu = pVM->CTX_SUFF(apCpus)[idCpu]; + if (APICIsEnabled(pItVCpu)) + { + Log2(("APIC: apicSendIntr: Raising NMI on VCPU%u\n", idCpu)); + apicSetInterruptFF(pItVCpu, PDMAPICIRQ_NMI); + fAccepted = true; + } + } + break; + } + + case XAPICDELIVERYMODE_INIT: + { +#ifdef IN_RING3 + for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++) + if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu)) + { + Log2(("APIC: apicSendIntr: Issuing INIT to VCPU%u\n", idCpu)); + VMMR3SendInitIpi(pVM, idCpu); + fAccepted = true; + } +#else + /* We need to return to ring-3 to deliver the INIT. */ + rcStrict = rcRZ; + fAccepted = true; +#endif + break; + } + + case XAPICDELIVERYMODE_STARTUP: + { +#ifdef IN_RING3 + for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++) + if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu)) + { + Log2(("APIC: apicSendIntr: Issuing SIPI to VCPU%u\n", idCpu)); + VMMR3SendStartupIpi(pVM, idCpu, uVector); + fAccepted = true; + } +#else + /* We need to return to ring-3 to deliver the SIPI. */ + rcStrict = rcRZ; + fAccepted = true; + Log2(("APIC: apicSendIntr: SIPI issued, returning to RZ. rc=%Rrc\n", rcRZ)); +#endif + break; + } + + case XAPICDELIVERYMODE_EXTINT: + { + for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++) + if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu)) + { + Log2(("APIC: apicSendIntr: Raising EXTINT on VCPU%u\n", idCpu)); + apicSetInterruptFF(pVM->CTX_SUFF(apCpus)[idCpu], PDMAPICIRQ_EXTINT); + fAccepted = true; + } + break; + } + + default: + { + AssertMsgFailed(("APIC: apicSendIntr: Unsupported delivery mode %#x (%s)\n", enmDeliveryMode, + apicGetDeliveryModeName(enmDeliveryMode))); + break; + } + } + + /* + * If an illegal vector is programmed, set the 'send illegal vector' error here if the + * interrupt is being sent by an APIC. + * + * The 'receive illegal vector' will be set on the target APIC when the interrupt + * gets generated, see apicPostInterrupt(). + * + * See Intel spec. 10.5.3 "Error Handling". + */ + if ( rcStrict != rcRZ + && pVCpu) + { + /* + * Flag only errors when the delivery mode is fixed and not others. + * + * Ubuntu 10.04-3 amd64 live CD with 2 VCPUs gets upset as it sends an SIPI to the + * 2nd VCPU with vector 6 and checks the ESR for no errors, see @bugref{8245#c86}. + */ + /** @todo The spec says this for LVT, but not explcitly for ICR-lo + * but it probably is true. */ + if (enmDeliveryMode == XAPICDELIVERYMODE_FIXED) + { + if (RT_UNLIKELY(uVector <= XAPIC_ILLEGAL_VECTOR_END)) + apicSetError(pVCpu, XAPIC_ESR_SEND_ILLEGAL_VECTOR); + } + } + + if (pfIntrAccepted) + *pfIntrAccepted = fAccepted; + + return rcStrict; +} + + +/** + * Checks if this APIC belongs to a logical destination. + * + * @returns true if the APIC belongs to the logical + * destination, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param fDest The destination mask. + * + * @thread Any. + */ +static bool apicIsLogicalDest(PVMCPUCC pVCpu, uint32_t fDest) +{ + if (XAPIC_IN_X2APIC_MODE(pVCpu)) + { + /* + * Flat logical mode is not supported in x2APIC mode. + * In clustered logical mode, the 32-bit logical ID in the LDR is interpreted as follows: + * - High 16 bits is the cluster ID. + * - Low 16 bits: each bit represents a unique APIC within the cluster. + */ + PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpu); + uint32_t const u32Ldr = pX2ApicPage->ldr.u32LogicalApicId; + if (X2APIC_LDR_GET_CLUSTER_ID(u32Ldr) == (fDest & X2APIC_LDR_CLUSTER_ID)) + return RT_BOOL(u32Ldr & fDest & X2APIC_LDR_LOGICAL_ID); + return false; + } + +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + /* + * In both flat and clustered logical mode, a destination mask of all set bits indicates a broadcast. + * See AMD spec. 16.6.1 "Receiving System and IPI Interrupts". + */ + Assert(!XAPIC_IN_X2APIC_MODE(pVCpu)); + if ((fDest & XAPIC_LDR_FLAT_LOGICAL_ID) == XAPIC_LDR_FLAT_LOGICAL_ID) + return true; + + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu); + XAPICDESTFORMAT enmDestFormat = (XAPICDESTFORMAT)pXApicPage->dfr.u.u4Model; + if (enmDestFormat == XAPICDESTFORMAT_FLAT) + { + /* The destination mask is interpreted as a bitmap of 8 unique logical APIC IDs. */ + uint8_t const u8Ldr = pXApicPage->ldr.u.u8LogicalApicId; + return RT_BOOL(u8Ldr & fDest & XAPIC_LDR_FLAT_LOGICAL_ID); + } + + /* + * In clustered logical mode, the 8-bit logical ID in the LDR is interpreted as follows: + * - High 4 bits is the cluster ID. + * - Low 4 bits: each bit represents a unique APIC within the cluster. + */ + Assert(enmDestFormat == XAPICDESTFORMAT_CLUSTER); + uint8_t const u8Ldr = pXApicPage->ldr.u.u8LogicalApicId; + if (XAPIC_LDR_CLUSTERED_GET_CLUSTER_ID(u8Ldr) == (fDest & XAPIC_LDR_CLUSTERED_CLUSTER_ID)) + return RT_BOOL(u8Ldr & fDest & XAPIC_LDR_CLUSTERED_LOGICAL_ID); + return false; +#else +# error "Implement Pentium and P6 family APIC architectures" +#endif +} + + +/** + * Figures out the set of destination CPUs for a given destination mode, format + * and delivery mode setting. + * + * @param pVM The cross context VM structure. + * @param fDestMask The destination mask. + * @param fBroadcastMask The broadcast mask. + * @param enmDestMode The destination mode. + * @param enmDeliveryMode The delivery mode. + * @param pDestCpuSet The destination CPU set to update. + */ +static void apicGetDestCpuSet(PVMCC pVM, uint32_t fDestMask, uint32_t fBroadcastMask, XAPICDESTMODE enmDestMode, + XAPICDELIVERYMODE enmDeliveryMode, PVMCPUSET pDestCpuSet) +{ + VMCPUSET_EMPTY(pDestCpuSet); + + /* + * Physical destination mode only supports either a broadcast or a single target. + * - Broadcast with lowest-priority delivery mode is not supported[1], we deliver it + * as a regular broadcast like in fixed delivery mode. + * - For a single target, lowest-priority delivery mode makes no sense. We deliver + * to the target like in fixed delivery mode. + * + * [1] See Intel spec. 10.6.2.1 "Physical Destination Mode". + */ + if ( enmDestMode == XAPICDESTMODE_PHYSICAL + && enmDeliveryMode == XAPICDELIVERYMODE_LOWEST_PRIO) + { + AssertMsgFailed(("APIC: Lowest-priority delivery using physical destination mode!")); + enmDeliveryMode = XAPICDELIVERYMODE_FIXED; + } + + uint32_t const cCpus = pVM->cCpus; + if (enmDeliveryMode == XAPICDELIVERYMODE_LOWEST_PRIO) + { + Assert(enmDestMode == XAPICDESTMODE_LOGICAL); +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + VMCPUID idCpuLowestTpr = NIL_VMCPUID; + uint8_t u8LowestTpr = UINT8_C(0xff); + for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++) + { + PVMCPUCC pVCpuDst = pVM->CTX_SUFF(apCpus)[idCpu]; + if (apicIsLogicalDest(pVCpuDst, fDestMask)) + { + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpuDst); + uint8_t const u8Tpr = pXApicPage->tpr.u8Tpr; /* PAV */ + + /* + * If there is a tie for lowest priority, the local APIC with the highest ID is chosen. + * Hence the use of "<=" in the check below. + * See AMD spec. 16.6.2 "Lowest Priority Messages and Arbitration". + */ + if (u8Tpr <= u8LowestTpr) + { + u8LowestTpr = u8Tpr; + idCpuLowestTpr = idCpu; + } + } + } + if (idCpuLowestTpr != NIL_VMCPUID) + VMCPUSET_ADD(pDestCpuSet, idCpuLowestTpr); +#else +# error "Implement Pentium and P6 family APIC architectures" +#endif + return; + } + + /* + * x2APIC: + * - In both physical and logical destination mode, a destination mask of 0xffffffff implies a broadcast[1]. + * xAPIC: + * - In physical destination mode, a destination mask of 0xff implies a broadcast[2]. + * - In both flat and clustered logical mode, a destination mask of 0xff implies a broadcast[3]. + * + * [1] See Intel spec. 10.12.9 "ICR Operation in x2APIC Mode". + * [2] See Intel spec. 10.6.2.1 "Physical Destination Mode". + * [2] See AMD spec. 16.6.1 "Receiving System and IPI Interrupts". + */ + if ((fDestMask & fBroadcastMask) == fBroadcastMask) + { + VMCPUSET_FILL(pDestCpuSet); + return; + } + + if (enmDestMode == XAPICDESTMODE_PHYSICAL) + { + /* The destination mask is interpreted as the physical APIC ID of a single target. */ +#if 1 + /* Since our physical APIC ID is read-only to software, set the corresponding bit in the CPU set. */ + if (RT_LIKELY(fDestMask < cCpus)) + VMCPUSET_ADD(pDestCpuSet, fDestMask); +#else + /* The physical APIC ID may not match our VCPU ID, search through the list of targets. */ + for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++) + { + PVMCPUCC pVCpuDst = &pVM->aCpus[idCpu]; + if (XAPIC_IN_X2APIC_MODE(pVCpuDst)) + { + PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpuDst); + if (pX2ApicPage->id.u32ApicId == fDestMask) + VMCPUSET_ADD(pDestCpuSet, pVCpuDst->idCpu); + } + else + { + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpuDst); + if (pXApicPage->id.u8ApicId == (uint8_t)fDestMask) + VMCPUSET_ADD(pDestCpuSet, pVCpuDst->idCpu); + } + } +#endif + } + else + { + Assert(enmDestMode == XAPICDESTMODE_LOGICAL); + + /* A destination mask of all 0's implies no target APICs (since it's interpreted as a bitmap or partial bitmap). */ + if (RT_UNLIKELY(!fDestMask)) + return; + + /* The destination mask is interpreted as a bitmap of software-programmable logical APIC ID of the target APICs. */ + for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++) + { + PVMCPUCC pVCpuDst = pVM->CTX_SUFF(apCpus)[idCpu]; + if (apicIsLogicalDest(pVCpuDst, fDestMask)) + VMCPUSET_ADD(pDestCpuSet, pVCpuDst->idCpu); + } + } +} + + +/** + * Sends an Interprocessor Interrupt (IPI) using values from the Interrupt + * Command Register (ICR). + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param rcRZ The return code if the operation cannot be + * performed in the current context. + */ +DECLINLINE(VBOXSTRICTRC) apicSendIpi(PVMCPUCC pVCpu, int rcRZ) +{ + VMCPU_ASSERT_EMT(pVCpu); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + XAPICDELIVERYMODE const enmDeliveryMode = (XAPICDELIVERYMODE)pXApicPage->icr_lo.u.u3DeliveryMode; + XAPICDESTMODE const enmDestMode = (XAPICDESTMODE)pXApicPage->icr_lo.u.u1DestMode; + XAPICINITLEVEL const enmInitLevel = (XAPICINITLEVEL)pXApicPage->icr_lo.u.u1Level; + XAPICTRIGGERMODE const enmTriggerMode = (XAPICTRIGGERMODE)pXApicPage->icr_lo.u.u1TriggerMode; + XAPICDESTSHORTHAND const enmDestShorthand = (XAPICDESTSHORTHAND)pXApicPage->icr_lo.u.u2DestShorthand; + uint8_t const uVector = pXApicPage->icr_lo.u.u8Vector; + + PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu); + uint32_t const fDest = XAPIC_IN_X2APIC_MODE(pVCpu) ? pX2ApicPage->icr_hi.u32IcrHi : pXApicPage->icr_hi.u.u8Dest; + Log5(("apicSendIpi: delivery=%u mode=%u init=%u trigger=%u short=%u vector=%#x fDest=%#x\n", + enmDeliveryMode, enmDestMode, enmInitLevel, enmTriggerMode, enmDestShorthand, uVector, fDest)); + +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + /* + * INIT Level De-assert is not support on Pentium 4 and Xeon processors. + * Apparently, this also applies to NMI, SMI, lowest-priority and fixed delivery modes, + * see @bugref{8245#c116}. + * + * See AMD spec. 16.5 "Interprocessor Interrupts (IPI)" for a table of valid ICR combinations. + */ + if ( enmTriggerMode == XAPICTRIGGERMODE_LEVEL + && enmInitLevel == XAPICINITLEVEL_DEASSERT + && ( enmDeliveryMode == XAPICDELIVERYMODE_FIXED + || enmDeliveryMode == XAPICDELIVERYMODE_LOWEST_PRIO + || enmDeliveryMode == XAPICDELIVERYMODE_SMI + || enmDeliveryMode == XAPICDELIVERYMODE_NMI + || enmDeliveryMode == XAPICDELIVERYMODE_INIT)) + { + Log2(("APIC%u: %s level de-assert unsupported, ignoring!\n", pVCpu->idCpu, apicGetDeliveryModeName(enmDeliveryMode))); + return VINF_SUCCESS; + } +#else +# error "Implement Pentium and P6 family APIC architectures" +#endif + + /* + * The destination and delivery modes are ignored/by-passed when a destination shorthand is specified. + * See Intel spec. 10.6.2.3 "Broadcast/Self Delivery Mode". + */ + VMCPUSET DestCpuSet; + switch (enmDestShorthand) + { + case XAPICDESTSHORTHAND_NONE: + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + uint32_t const fBroadcastMask = XAPIC_IN_X2APIC_MODE(pVCpu) ? X2APIC_ID_BROADCAST_MASK : XAPIC_ID_BROADCAST_MASK; + apicGetDestCpuSet(pVM, fDest, fBroadcastMask, enmDestMode, enmDeliveryMode, &DestCpuSet); + break; + } + + case XAPICDESTSHORTHAND_SELF: + { + VMCPUSET_EMPTY(&DestCpuSet); + VMCPUSET_ADD(&DestCpuSet, pVCpu->idCpu); + break; + } + + case XAPIDDESTSHORTHAND_ALL_INCL_SELF: + { + VMCPUSET_FILL(&DestCpuSet); + break; + } + + case XAPICDESTSHORTHAND_ALL_EXCL_SELF: + { + VMCPUSET_FILL(&DestCpuSet); + VMCPUSET_DEL(&DestCpuSet, pVCpu->idCpu); + break; + } + } + + return apicSendIntr(pVCpu->CTX_SUFF(pVM), pVCpu, uVector, enmTriggerMode, enmDeliveryMode, &DestCpuSet, + NULL /* pfIntrAccepted */, 0 /* uSrcTag */, rcRZ); +} + + +/** + * Sets the Interrupt Command Register (ICR) high dword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uIcrHi The ICR high dword. + */ +static VBOXSTRICTRC apicSetIcrHi(PVMCPUCC pVCpu, uint32_t uIcrHi) +{ + VMCPU_ASSERT_EMT(pVCpu); + Assert(!XAPIC_IN_X2APIC_MODE(pVCpu)); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + pXApicPage->icr_hi.all.u32IcrHi = uIcrHi & XAPIC_ICR_HI_DEST; + STAM_COUNTER_INC(&pVCpu->apic.s.StatIcrHiWrite); + Log2(("APIC%u: apicSetIcrHi: uIcrHi=%#RX32\n", pVCpu->idCpu, pXApicPage->icr_hi.all.u32IcrHi)); + + return VINF_SUCCESS; +} + + +/** + * Sets the Interrupt Command Register (ICR) low dword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uIcrLo The ICR low dword. + * @param rcRZ The return code if the operation cannot be performed + * in the current context. + * @param fUpdateStat Whether to update the ICR low write statistics + * counter. + */ +static VBOXSTRICTRC apicSetIcrLo(PVMCPUCC pVCpu, uint32_t uIcrLo, int rcRZ, bool fUpdateStat) +{ + VMCPU_ASSERT_EMT(pVCpu); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + pXApicPage->icr_lo.all.u32IcrLo = uIcrLo & XAPIC_ICR_LO_WR_VALID; + Log2(("APIC%u: apicSetIcrLo: uIcrLo=%#RX32\n", pVCpu->idCpu, pXApicPage->icr_lo.all.u32IcrLo)); + + if (fUpdateStat) + STAM_COUNTER_INC(&pVCpu->apic.s.StatIcrLoWrite); + RT_NOREF(fUpdateStat); + + return apicSendIpi(pVCpu, rcRZ); +} + + +/** + * Sets the Interrupt Command Register (ICR). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param u64Icr The ICR (High and Low combined). + * @param rcRZ The return code if the operation cannot be performed + * in the current context. + * + * @remarks This function is used by both x2APIC interface and the Hyper-V + * interface, see APICHvSetIcr. The Hyper-V spec isn't clear what + * happens when invalid bits are set. For the time being, it will + * \#GP like a regular x2APIC access. + */ +static VBOXSTRICTRC apicSetIcr(PVMCPUCC pVCpu, uint64_t u64Icr, int rcRZ) +{ + VMCPU_ASSERT_EMT(pVCpu); + + /* Validate. */ + uint32_t const uLo = RT_LO_U32(u64Icr); + if (RT_LIKELY(!(uLo & ~XAPIC_ICR_LO_WR_VALID))) + { + /* Update high dword first, then update the low dword which sends the IPI. */ + PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu); + pX2ApicPage->icr_hi.u32IcrHi = RT_HI_U32(u64Icr); + STAM_COUNTER_INC(&pVCpu->apic.s.StatIcrFullWrite); + return apicSetIcrLo(pVCpu, uLo, rcRZ, false /* fUpdateStat */); + } + return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_ICR, APICMSRACCESS_WRITE_RSVD_BITS); +} + + +/** + * Sets the Error Status Register (ESR). + * + * @returns VINF_SUCCESS or VERR_CPUM_RAISE_GP_0. + * @param pVCpu The cross context virtual CPU structure. + * @param uEsr The ESR value. + */ +static int apicSetEsr(PVMCPUCC pVCpu, uint32_t uEsr) +{ + VMCPU_ASSERT_EMT(pVCpu); + + Log2(("APIC%u: apicSetEsr: uEsr=%#RX32\n", pVCpu->idCpu, uEsr)); + + if ( XAPIC_IN_X2APIC_MODE(pVCpu) + && (uEsr & ~XAPIC_ESR_WO_VALID)) + return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_ESR, APICMSRACCESS_WRITE_RSVD_BITS); + + /* + * Writes to the ESR causes the internal state to be updated in the register, + * clearing the original state. See AMD spec. 16.4.6 "APIC Error Interrupts". + */ + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + pXApicPage->esr.all.u32Errors = apicClearAllErrors(pVCpu); + return VINF_SUCCESS; +} + + +/** + * Updates the Processor Priority Register (PPR). + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void apicUpdatePpr(PVMCPUCC pVCpu) +{ + VMCPU_ASSERT_EMT(pVCpu); + + /* See Intel spec 10.8.3.1 "Task and Processor Priorities". */ + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + uint8_t const uIsrv = apicGetHighestSetBitInReg(&pXApicPage->isr, 0 /* rcNotFound */); + uint8_t uPpr; + if (XAPIC_TPR_GET_TP(pXApicPage->tpr.u8Tpr) >= XAPIC_PPR_GET_PP(uIsrv)) + uPpr = pXApicPage->tpr.u8Tpr; + else + uPpr = XAPIC_PPR_GET_PP(uIsrv); + pXApicPage->ppr.u8Ppr = uPpr; +} + + +/** + * Gets the Processor Priority Register (PPR). + * + * @returns The PPR value. + * @param pVCpu The cross context virtual CPU structure. + */ +static uint8_t apicGetPpr(PVMCPUCC pVCpu) +{ + VMCPU_ASSERT_EMT(pVCpu); + STAM_COUNTER_INC(&pVCpu->apic.s.StatTprRead); + + /* + * With virtualized APIC registers or with TPR virtualization, the hardware may + * update ISR/TPR transparently. We thus re-calculate the PPR which may be out of sync. + * See Intel spec. 29.2.2 "Virtual-Interrupt Delivery". + * + * In all other instances, whenever the TPR or ISR changes, we need to update the PPR + * as well (e.g. like we do manually in apicR3InitIpi and by calling apicUpdatePpr). + */ + PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM)); + if (pApic->fVirtApicRegsEnabled) /** @todo re-think this */ + apicUpdatePpr(pVCpu); + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu); + return pXApicPage->ppr.u8Ppr; +} + + +/** + * Sets the Task Priority Register (TPR). + * + * @returns VINF_SUCCESS or VERR_CPUM_RAISE_GP_0. + * @param pVCpu The cross context virtual CPU structure. + * @param uTpr The TPR value. + * @param fForceX2ApicBehaviour Pretend the APIC is in x2APIC mode during + * this write. + */ +static int apicSetTprEx(PVMCPUCC pVCpu, uint32_t uTpr, bool fForceX2ApicBehaviour) +{ + VMCPU_ASSERT_EMT(pVCpu); + + Log2(("APIC%u: apicSetTprEx: uTpr=%#RX32\n", pVCpu->idCpu, uTpr)); + STAM_COUNTER_INC(&pVCpu->apic.s.StatTprWrite); + + bool const fX2ApicMode = XAPIC_IN_X2APIC_MODE(pVCpu) || fForceX2ApicBehaviour; + if ( fX2ApicMode + && (uTpr & ~XAPIC_TPR_VALID)) + return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_TPR, APICMSRACCESS_WRITE_RSVD_BITS); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + pXApicPage->tpr.u8Tpr = uTpr; + apicUpdatePpr(pVCpu); + apicSignalNextPendingIntr(pVCpu); + return VINF_SUCCESS; +} + + +/** + * Sets the End-Of-Interrupt (EOI) register. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uEoi The EOI value. + * @param fForceX2ApicBehaviour Pretend the APIC is in x2APIC mode during + * this write. + */ +static VBOXSTRICTRC apicSetEoi(PVMCPUCC pVCpu, uint32_t uEoi, bool fForceX2ApicBehaviour) +{ + VMCPU_ASSERT_EMT(pVCpu); + + Log2(("APIC%u: apicSetEoi: uEoi=%#RX32\n", pVCpu->idCpu, uEoi)); + STAM_COUNTER_INC(&pVCpu->apic.s.StatEoiWrite); + + bool const fX2ApicMode = XAPIC_IN_X2APIC_MODE(pVCpu) || fForceX2ApicBehaviour; + if ( fX2ApicMode + && (uEoi & ~XAPIC_EOI_WO_VALID)) + return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_EOI, APICMSRACCESS_WRITE_RSVD_BITS); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + int isrv = apicGetHighestSetBitInReg(&pXApicPage->isr, -1 /* rcNotFound */); + if (isrv >= 0) + { + /* + * Broadcast the EOI to the I/O APIC(s). + * + * We'll handle the EOI broadcast first as there is tiny chance we get rescheduled to + * ring-3 due to contention on the I/O APIC lock. This way we don't mess with the rest + * of the APIC state and simply restart the EOI write operation from ring-3. + */ + Assert(isrv <= (int)UINT8_MAX); + uint8_t const uVector = isrv; + bool const fLevelTriggered = apicTestVectorInReg(&pXApicPage->tmr, uVector); + if (fLevelTriggered) + { + PDMIoApicBroadcastEoi(pVCpu->CTX_SUFF(pVM), uVector); + + /* + * Clear the vector from the TMR. + * + * The broadcast to I/O APIC can re-trigger new interrupts to arrive via the bus. However, + * APICUpdatePendingInterrupts() which updates TMR can only be done from EMT which we + * currently are on, so no possibility of concurrent updates. + */ + apicClearVectorInReg(&pXApicPage->tmr, uVector); + + /* + * Clear the remote IRR bit for level-triggered, fixed mode LINT0 interrupt. + * The LINT1 pin does not support level-triggered interrupts. + * See Intel spec. 10.5.1 "Local Vector Table". + */ + uint32_t const uLvtLint0 = pXApicPage->lvt_lint0.all.u32LvtLint0; + if ( XAPIC_LVT_GET_REMOTE_IRR(uLvtLint0) + && XAPIC_LVT_GET_VECTOR(uLvtLint0) == uVector + && XAPIC_LVT_GET_DELIVERY_MODE(uLvtLint0) == XAPICDELIVERYMODE_FIXED) + { + ASMAtomicAndU32((volatile uint32_t *)&pXApicPage->lvt_lint0.all.u32LvtLint0, ~XAPIC_LVT_REMOTE_IRR); + Log2(("APIC%u: apicSetEoi: Cleared remote-IRR for LINT0. uVector=%#x\n", pVCpu->idCpu, uVector)); + } + + Log2(("APIC%u: apicSetEoi: Cleared level triggered interrupt from TMR. uVector=%#x\n", pVCpu->idCpu, uVector)); + } + + /* + * Mark interrupt as serviced, update the PPR and signal pending interrupts. + */ + Log2(("APIC%u: apicSetEoi: Clearing interrupt from ISR. uVector=%#x\n", pVCpu->idCpu, uVector)); + apicClearVectorInReg(&pXApicPage->isr, uVector); + apicUpdatePpr(pVCpu); + apicSignalNextPendingIntr(pVCpu); + } + else + { +#ifdef DEBUG_ramshankar + /** @todo Figure out if this is done intentionally by guests or is a bug + * in our emulation. Happened with Win10 SMP VM during reboot after + * installation of guest additions with 3D support. */ + AssertMsgFailed(("APIC%u: apicSetEoi: Failed to find any ISR bit\n", pVCpu->idCpu)); +#endif + } + + return VINF_SUCCESS; +} + + +/** + * Sets the Logical Destination Register (LDR). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uLdr The LDR value. + * + * @remarks LDR is read-only in x2APIC mode. + */ +static VBOXSTRICTRC apicSetLdr(PVMCPUCC pVCpu, uint32_t uLdr) +{ + VMCPU_ASSERT_EMT(pVCpu); + PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM)); + Assert(!XAPIC_IN_X2APIC_MODE(pVCpu) || pApic->fHyperVCompatMode); RT_NOREF_PV(pApic); + + Log2(("APIC%u: apicSetLdr: uLdr=%#RX32\n", pVCpu->idCpu, uLdr)); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + apicWriteRaw32(pXApicPage, XAPIC_OFF_LDR, uLdr & XAPIC_LDR_VALID); + STAM_COUNTER_INC(&pVCpu->apic.s.StatLdrWrite); + return VINF_SUCCESS; +} + + +/** + * Sets the Destination Format Register (DFR). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uDfr The DFR value. + * + * @remarks DFR is not available in x2APIC mode. + */ +static VBOXSTRICTRC apicSetDfr(PVMCPUCC pVCpu, uint32_t uDfr) +{ + VMCPU_ASSERT_EMT(pVCpu); + Assert(!XAPIC_IN_X2APIC_MODE(pVCpu)); + + uDfr &= XAPIC_DFR_VALID; + uDfr |= XAPIC_DFR_RSVD_MB1; + + Log2(("APIC%u: apicSetDfr: uDfr=%#RX32\n", pVCpu->idCpu, uDfr)); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + apicWriteRaw32(pXApicPage, XAPIC_OFF_DFR, uDfr); + STAM_COUNTER_INC(&pVCpu->apic.s.StatDfrWrite); + return VINF_SUCCESS; +} + + +/** + * Sets the Timer Divide Configuration Register (DCR). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uTimerDcr The timer DCR value. + */ +static VBOXSTRICTRC apicSetTimerDcr(PVMCPUCC pVCpu, uint32_t uTimerDcr) +{ + VMCPU_ASSERT_EMT(pVCpu); + if ( XAPIC_IN_X2APIC_MODE(pVCpu) + && (uTimerDcr & ~XAPIC_TIMER_DCR_VALID)) + return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_TIMER_DCR, APICMSRACCESS_WRITE_RSVD_BITS); + + Log2(("APIC%u: apicSetTimerDcr: uTimerDcr=%#RX32\n", pVCpu->idCpu, uTimerDcr)); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + apicWriteRaw32(pXApicPage, XAPIC_OFF_TIMER_DCR, uTimerDcr); + STAM_COUNTER_INC(&pVCpu->apic.s.StatDcrWrite); + return VINF_SUCCESS; +} + + +/** + * Gets the timer's Current Count Register (CCR). + * + * @returns VBox status code. + * @param pDevIns The device instance. + * @param pVCpu The cross context virtual CPU structure. + * @param rcBusy The busy return code for the timer critical section. + * @param puValue Where to store the LVT timer CCR. + */ +static VBOXSTRICTRC apicGetTimerCcr(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, int rcBusy, uint32_t *puValue) +{ + VMCPU_ASSERT_EMT(pVCpu); + Assert(puValue); + + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu); + *puValue = 0; + + /* In TSC-deadline mode, CCR returns 0, see Intel spec. 10.5.4.1 "TSC-Deadline Mode". */ + if (pXApicPage->lvt_timer.u.u2TimerMode == XAPIC_TIMER_MODE_TSC_DEADLINE) + return VINF_SUCCESS; + + /* If the initial-count register is 0, CCR returns 0 as it cannot exceed the ICR. */ + uint32_t const uInitialCount = pXApicPage->timer_icr.u32InitialCount; + if (!uInitialCount) + return VINF_SUCCESS; + + /* + * Reading the virtual-sync clock requires locking its timer because it's not + * a simple atomic operation, see tmVirtualSyncGetEx(). + * + * We also need to lock before reading the timer CCR, see apicR3TimerCallback(). + */ + PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + TMTIMERHANDLE hTimer = pApicCpu->hTimer; + + VBOXSTRICTRC rc = PDMDevHlpTimerLockClock(pDevIns, hTimer, rcBusy); + if (rc == VINF_SUCCESS) + { + /* If the current-count register is 0, it implies the timer expired. */ + uint32_t const uCurrentCount = pXApicPage->timer_ccr.u32CurrentCount; + if (uCurrentCount) + { + uint64_t const cTicksElapsed = PDMDevHlpTimerGet(pDevIns, hTimer) - pApicCpu->u64TimerInitial; + PDMDevHlpTimerUnlockClock(pDevIns, hTimer); + uint8_t const uTimerShift = apicGetTimerShift(pXApicPage); + uint64_t const uDelta = cTicksElapsed >> uTimerShift; + if (uInitialCount > uDelta) + *puValue = uInitialCount - uDelta; + } + else + PDMDevHlpTimerUnlockClock(pDevIns, hTimer); + } + return rc; +} + + +/** + * Sets the timer's Initial-Count Register (ICR). + * + * @returns Strict VBox status code. + * @param pDevIns The device instance. + * @param pVCpu The cross context virtual CPU structure. + * @param rcBusy The busy return code for the timer critical section. + * @param uInitialCount The timer ICR. + */ +static VBOXSTRICTRC apicSetTimerIcr(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, int rcBusy, uint32_t uInitialCount) +{ + VMCPU_ASSERT_EMT(pVCpu); + + PAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM)); + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + + Log2(("APIC%u: apicSetTimerIcr: uInitialCount=%#RX32\n", pVCpu->idCpu, uInitialCount)); + STAM_COUNTER_INC(&pApicCpu->StatTimerIcrWrite); + + /* In TSC-deadline mode, timer ICR writes are ignored, see Intel spec. 10.5.4.1 "TSC-Deadline Mode". */ + if ( pApic->fSupportsTscDeadline + && pXApicPage->lvt_timer.u.u2TimerMode == XAPIC_TIMER_MODE_TSC_DEADLINE) + return VINF_SUCCESS; + + /* + * The timer CCR may be modified by apicR3TimerCallback() in parallel, + * so obtain the lock -before- updating it here to be consistent with the + * timer ICR. We rely on CCR being consistent in apicGetTimerCcr(). + */ + TMTIMERHANDLE hTimer = pApicCpu->hTimer; + VBOXSTRICTRC rc = PDMDevHlpTimerLockClock(pDevIns, hTimer, rcBusy); + if (rc == VINF_SUCCESS) + { + pXApicPage->timer_icr.u32InitialCount = uInitialCount; + pXApicPage->timer_ccr.u32CurrentCount = uInitialCount; + if (uInitialCount) + apicStartTimer(pVCpu, uInitialCount); + else + apicStopTimer(pVCpu); + PDMDevHlpTimerUnlockClock(pDevIns, hTimer); + } + return rc; +} + + +/** + * Sets an LVT entry. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param offLvt The LVT entry offset in the xAPIC page. + * @param uLvt The LVT value to set. + */ +static VBOXSTRICTRC apicSetLvtEntry(PVMCPUCC pVCpu, uint16_t offLvt, uint32_t uLvt) +{ + VMCPU_ASSERT_EMT(pVCpu); + +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + AssertMsg( offLvt == XAPIC_OFF_LVT_TIMER + || offLvt == XAPIC_OFF_LVT_THERMAL + || offLvt == XAPIC_OFF_LVT_PERF + || offLvt == XAPIC_OFF_LVT_LINT0 + || offLvt == XAPIC_OFF_LVT_LINT1 + || offLvt == XAPIC_OFF_LVT_ERROR, + ("APIC%u: apicSetLvtEntry: invalid offset, offLvt=%#RX16, uLvt=%#RX32\n", pVCpu->idCpu, offLvt, uLvt)); + + /* + * If TSC-deadline mode isn't support, ignore the bit in xAPIC mode + * and raise #GP(0) in x2APIC mode. + */ + PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM)); + if (offLvt == XAPIC_OFF_LVT_TIMER) + { + STAM_COUNTER_INC(&pVCpu->apic.s.StatLvtTimerWrite); + if ( !pApic->fSupportsTscDeadline + && (uLvt & XAPIC_LVT_TIMER_TSCDEADLINE)) + { + if (XAPIC_IN_X2APIC_MODE(pVCpu)) + return apicMsrAccessError(pVCpu, XAPIC_GET_X2APIC_MSR(offLvt), APICMSRACCESS_WRITE_RSVD_BITS); + uLvt &= ~XAPIC_LVT_TIMER_TSCDEADLINE; + /** @todo TSC-deadline timer mode transition */ + } + } + + /* + * Validate rest of the LVT bits. + */ + uint16_t const idxLvt = (offLvt - XAPIC_OFF_LVT_START) >> 4; + AssertReturn(idxLvt < RT_ELEMENTS(g_au32LvtValidMasks), VERR_OUT_OF_RANGE); + + /* + * For x2APIC, disallow setting of invalid/reserved bits. + * For xAPIC, mask out invalid/reserved bits (i.e. ignore them). + */ + if ( XAPIC_IN_X2APIC_MODE(pVCpu) + && (uLvt & ~g_au32LvtValidMasks[idxLvt])) + return apicMsrAccessError(pVCpu, XAPIC_GET_X2APIC_MSR(offLvt), APICMSRACCESS_WRITE_RSVD_BITS); + + uLvt &= g_au32LvtValidMasks[idxLvt]; + + /* + * In the software-disabled state, LVT mask-bit must remain set and attempts to clear the mask + * bit must be ignored. See Intel spec. 10.4.7.2 "Local APIC State After It Has Been Software Disabled". + */ + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + if (!pXApicPage->svr.u.fApicSoftwareEnable) + uLvt |= XAPIC_LVT_MASK; + + /* + * It is unclear whether we should signal a 'send illegal vector' error here and ignore updating + * the LVT entry when the delivery mode is 'fixed'[1] or update it in addition to signalling the + * error or not signal the error at all. For now, we'll allow setting illegal vectors into the LVT + * but set the 'send illegal vector' error here. The 'receive illegal vector' error will be set if + * the interrupt for the vector happens to be generated, see apicPostInterrupt(). + * + * [1] See Intel spec. 10.5.2 "Valid Interrupt Vectors". + */ + if (RT_UNLIKELY( XAPIC_LVT_GET_VECTOR(uLvt) <= XAPIC_ILLEGAL_VECTOR_END + && XAPIC_LVT_GET_DELIVERY_MODE(uLvt) == XAPICDELIVERYMODE_FIXED)) + apicSetError(pVCpu, XAPIC_ESR_SEND_ILLEGAL_VECTOR); + + Log2(("APIC%u: apicSetLvtEntry: offLvt=%#RX16 uLvt=%#RX32\n", pVCpu->idCpu, offLvt, uLvt)); + + apicWriteRaw32(pXApicPage, offLvt, uLvt); + return VINF_SUCCESS; +#else +# error "Implement Pentium and P6 family APIC architectures" +#endif /* XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 */ +} + + +#if 0 +/** + * Sets an LVT entry in the extended LVT range. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param offLvt The LVT entry offset in the xAPIC page. + * @param uValue The LVT value to set. + */ +static int apicSetLvtExtEntry(PVMCPUCC pVCpu, uint16_t offLvt, uint32_t uLvt) +{ + VMCPU_ASSERT_EMT(pVCpu); + AssertMsg(offLvt == XAPIC_OFF_CMCI, ("APIC%u: apicSetLvt1Entry: invalid offset %#RX16\n", pVCpu->idCpu, offLvt)); + + /** @todo support CMCI. */ + return VERR_NOT_IMPLEMENTED; +} +#endif + + +/** + * Hints TM about the APIC timer frequency. + * + * @param pDevIns The device instance. + * @param pApicCpu The APIC CPU state. + * @param uInitialCount The new initial count. + * @param uTimerShift The new timer shift. + * @thread Any. + */ +void apicHintTimerFreq(PPDMDEVINS pDevIns, PAPICCPU pApicCpu, uint32_t uInitialCount, uint8_t uTimerShift) +{ + Assert(pApicCpu); + + if ( pApicCpu->uHintedTimerInitialCount != uInitialCount + || pApicCpu->uHintedTimerShift != uTimerShift) + { + uint32_t uHz; + if (uInitialCount) + { + uint64_t cTicksPerPeriod = (uint64_t)uInitialCount << uTimerShift; + uHz = PDMDevHlpTimerGetFreq(pDevIns, pApicCpu->hTimer) / cTicksPerPeriod; + } + else + uHz = 0; + + PDMDevHlpTimerSetFrequencyHint(pDevIns, pApicCpu->hTimer, uHz); + pApicCpu->uHintedTimerInitialCount = uInitialCount; + pApicCpu->uHintedTimerShift = uTimerShift; + } +} + + +/** + * Gets the Interrupt Command Register (ICR), without performing any interface + * checks. + * + * @returns The ICR value. + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(uint64_t) apicGetIcrNoCheck(PVMCPUCC pVCpu) +{ + PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpu); + uint64_t const uHi = pX2ApicPage->icr_hi.u32IcrHi; + uint64_t const uLo = pX2ApicPage->icr_lo.all.u32IcrLo; + uint64_t const uIcr = RT_MAKE_U64(uLo, uHi); + return uIcr; +} + + +/** + * Reads an APIC register. + * + * @returns VBox status code. + * @param pDevIns The device instance. + * @param pVCpu The cross context virtual CPU structure. + * @param offReg The offset of the register being read. + * @param puValue Where to store the register value. + */ +DECLINLINE(VBOXSTRICTRC) apicReadRegister(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, uint16_t offReg, uint32_t *puValue) +{ + VMCPU_ASSERT_EMT(pVCpu); + Assert(offReg <= XAPIC_OFF_MAX_VALID); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + uint32_t uValue = 0; + VBOXSTRICTRC rc = VINF_SUCCESS; + switch (offReg) + { + case XAPIC_OFF_ID: + case XAPIC_OFF_VERSION: + case XAPIC_OFF_TPR: + case XAPIC_OFF_EOI: + case XAPIC_OFF_RRD: + case XAPIC_OFF_LDR: + case XAPIC_OFF_DFR: + case XAPIC_OFF_SVR: + case XAPIC_OFF_ISR0: case XAPIC_OFF_ISR1: case XAPIC_OFF_ISR2: case XAPIC_OFF_ISR3: + case XAPIC_OFF_ISR4: case XAPIC_OFF_ISR5: case XAPIC_OFF_ISR6: case XAPIC_OFF_ISR7: + case XAPIC_OFF_TMR0: case XAPIC_OFF_TMR1: case XAPIC_OFF_TMR2: case XAPIC_OFF_TMR3: + case XAPIC_OFF_TMR4: case XAPIC_OFF_TMR5: case XAPIC_OFF_TMR6: case XAPIC_OFF_TMR7: + case XAPIC_OFF_IRR0: case XAPIC_OFF_IRR1: case XAPIC_OFF_IRR2: case XAPIC_OFF_IRR3: + case XAPIC_OFF_IRR4: case XAPIC_OFF_IRR5: case XAPIC_OFF_IRR6: case XAPIC_OFF_IRR7: + case XAPIC_OFF_ESR: + case XAPIC_OFF_ICR_LO: + case XAPIC_OFF_ICR_HI: + case XAPIC_OFF_LVT_TIMER: +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + case XAPIC_OFF_LVT_THERMAL: +#endif + case XAPIC_OFF_LVT_PERF: + case XAPIC_OFF_LVT_LINT0: + case XAPIC_OFF_LVT_LINT1: + case XAPIC_OFF_LVT_ERROR: + case XAPIC_OFF_TIMER_ICR: + case XAPIC_OFF_TIMER_DCR: + { + Assert( !XAPIC_IN_X2APIC_MODE(pVCpu) + || ( offReg != XAPIC_OFF_DFR + && offReg != XAPIC_OFF_ICR_HI + && offReg != XAPIC_OFF_EOI)); + uValue = apicReadRaw32(pXApicPage, offReg); + Log2(("APIC%u: apicReadRegister: offReg=%#x uValue=%#x\n", pVCpu->idCpu, offReg, uValue)); + break; + } + + case XAPIC_OFF_PPR: + { + uValue = apicGetPpr(pVCpu); + break; + } + + case XAPIC_OFF_TIMER_CCR: + { + Assert(!XAPIC_IN_X2APIC_MODE(pVCpu)); + rc = apicGetTimerCcr(pDevIns, pVCpu, VINF_IOM_R3_MMIO_READ, &uValue); + break; + } + + case XAPIC_OFF_APR: + { +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + /* Unsupported on Pentium 4 and Xeon CPUs, invalid in x2APIC mode. */ + Assert(!XAPIC_IN_X2APIC_MODE(pVCpu)); +#else +# error "Implement Pentium and P6 family APIC architectures" +#endif + break; + } + + default: + { + Assert(!XAPIC_IN_X2APIC_MODE(pVCpu)); + rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS, "VCPU[%u]: offReg=%#RX16\n", pVCpu->idCpu, offReg); + apicSetError(pVCpu, XAPIC_ESR_ILLEGAL_REG_ADDRESS); + break; + } + } + + *puValue = uValue; + return rc; +} + + +/** + * Writes an APIC register. + * + * @returns Strict VBox status code. + * @param pDevIns The device instance. + * @param pVCpu The cross context virtual CPU structure. + * @param offReg The offset of the register being written. + * @param uValue The register value. + */ +DECLINLINE(VBOXSTRICTRC) apicWriteRegister(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, uint16_t offReg, uint32_t uValue) +{ + VMCPU_ASSERT_EMT(pVCpu); + Assert(offReg <= XAPIC_OFF_MAX_VALID); + Assert(!XAPIC_IN_X2APIC_MODE(pVCpu)); + + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + switch (offReg) + { + case XAPIC_OFF_TPR: + { + rcStrict = apicSetTprEx(pVCpu, uValue, false /* fForceX2ApicBehaviour */); + break; + } + + case XAPIC_OFF_LVT_TIMER: +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + case XAPIC_OFF_LVT_THERMAL: +#endif + case XAPIC_OFF_LVT_PERF: + case XAPIC_OFF_LVT_LINT0: + case XAPIC_OFF_LVT_LINT1: + case XAPIC_OFF_LVT_ERROR: + { + rcStrict = apicSetLvtEntry(pVCpu, offReg, uValue); + break; + } + + case XAPIC_OFF_TIMER_ICR: + { + rcStrict = apicSetTimerIcr(pDevIns, pVCpu, VINF_IOM_R3_MMIO_WRITE, uValue); + break; + } + + case XAPIC_OFF_EOI: + { + rcStrict = apicSetEoi(pVCpu, uValue, false /* fForceX2ApicBehaviour */); + break; + } + + case XAPIC_OFF_LDR: + { + rcStrict = apicSetLdr(pVCpu, uValue); + break; + } + + case XAPIC_OFF_DFR: + { + rcStrict = apicSetDfr(pVCpu, uValue); + break; + } + + case XAPIC_OFF_SVR: + { + rcStrict = apicSetSvr(pVCpu, uValue); + break; + } + + case XAPIC_OFF_ICR_LO: + { + rcStrict = apicSetIcrLo(pVCpu, uValue, VINF_IOM_R3_MMIO_WRITE, true /* fUpdateStat */); + break; + } + + case XAPIC_OFF_ICR_HI: + { + rcStrict = apicSetIcrHi(pVCpu, uValue); + break; + } + + case XAPIC_OFF_TIMER_DCR: + { + rcStrict = apicSetTimerDcr(pVCpu, uValue); + break; + } + + case XAPIC_OFF_ESR: + { + rcStrict = apicSetEsr(pVCpu, uValue); + break; + } + + case XAPIC_OFF_APR: + case XAPIC_OFF_RRD: + { +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + /* Unsupported on Pentium 4 and Xeon CPUs but writes do -not- set an illegal register access error. */ +#else +# error "Implement Pentium and P6 family APIC architectures" +#endif + break; + } + + /* Read-only, write ignored: */ + case XAPIC_OFF_VERSION: + case XAPIC_OFF_ID: + break; + + /* Unavailable/reserved in xAPIC mode: */ + case X2APIC_OFF_SELF_IPI: + /* Read-only registers: */ + case XAPIC_OFF_PPR: + case XAPIC_OFF_ISR0: case XAPIC_OFF_ISR1: case XAPIC_OFF_ISR2: case XAPIC_OFF_ISR3: + case XAPIC_OFF_ISR4: case XAPIC_OFF_ISR5: case XAPIC_OFF_ISR6: case XAPIC_OFF_ISR7: + case XAPIC_OFF_TMR0: case XAPIC_OFF_TMR1: case XAPIC_OFF_TMR2: case XAPIC_OFF_TMR3: + case XAPIC_OFF_TMR4: case XAPIC_OFF_TMR5: case XAPIC_OFF_TMR6: case XAPIC_OFF_TMR7: + case XAPIC_OFF_IRR0: case XAPIC_OFF_IRR1: case XAPIC_OFF_IRR2: case XAPIC_OFF_IRR3: + case XAPIC_OFF_IRR4: case XAPIC_OFF_IRR5: case XAPIC_OFF_IRR6: case XAPIC_OFF_IRR7: + case XAPIC_OFF_TIMER_CCR: + default: + { + rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS, "APIC%u: offReg=%#RX16\n", pVCpu->idCpu, offReg); + apicSetError(pVCpu, XAPIC_ESR_ILLEGAL_REG_ADDRESS); + break; + } + } + + return rcStrict; +} + + +/** + * Reads an APIC MSR. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param u32Reg The MSR being read. + * @param pu64Value Where to store the read value. + */ +VMM_INT_DECL(VBOXSTRICTRC) APICReadMsr(PVMCPUCC pVCpu, uint32_t u32Reg, uint64_t *pu64Value) +{ + /* + * Validate. + */ + VMCPU_ASSERT_EMT(pVCpu); + Assert(u32Reg >= MSR_IA32_X2APIC_ID && u32Reg <= MSR_IA32_X2APIC_SELF_IPI); + Assert(pu64Value); + + /* + * Is the APIC enabled? + */ + PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM)); + if (APICIsEnabled(pVCpu)) + { /* likely */ } + else + return apicMsrAccessError(pVCpu, u32Reg, pApic->enmMaxMode == PDMAPICMODE_NONE ? + APICMSRACCESS_READ_DISALLOWED_CONFIG : APICMSRACCESS_READ_RSVD_OR_UNKNOWN); + +#ifndef IN_RING3 + if (pApic->CTXALLMID(f,Enabled)) + { /* likely */} + else + return VINF_CPUM_R3_MSR_READ; +#endif + + STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMsrRead)); + + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + if (RT_LIKELY( XAPIC_IN_X2APIC_MODE(pVCpu) + || pApic->fHyperVCompatMode)) + { + switch (u32Reg) + { + /* Special handling for x2APIC: */ + case MSR_IA32_X2APIC_ICR: + { + *pu64Value = apicGetIcrNoCheck(pVCpu); + break; + } + + /* Special handling, compatible with xAPIC: */ + case MSR_IA32_X2APIC_TIMER_CCR: + { + uint32_t uValue; + rcStrict = apicGetTimerCcr(VMCPU_TO_DEVINS(pVCpu), pVCpu, VINF_CPUM_R3_MSR_READ, &uValue); + *pu64Value = uValue; + break; + } + + /* Special handling, compatible with xAPIC: */ + case MSR_IA32_X2APIC_PPR: + { + *pu64Value = apicGetPpr(pVCpu); + break; + } + + /* Raw read, compatible with xAPIC: */ + case MSR_IA32_X2APIC_ID: + { + STAM_COUNTER_INC(&pVCpu->apic.s.StatIdMsrRead); + /* Horrible macOS hack (sample rdmsr addres: 0008:ffffff801686f21a). */ + if ( !pApic->fMacOSWorkaround + || pVCpu->cpum.GstCtx.cs.Sel != 8 + || pVCpu->cpum.GstCtx.rip < UINT64_C(0xffffff8000000000)) + { /* likely */ } + else + { + PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpu); + uint32_t const idApic = pX2ApicPage->id.u32ApicId; + *pu64Value = (idApic << 24) | idApic; + Log(("APIC: Applying macOS hack to MSR_IA32_X2APIC_ID: %#RX64\n", *pu64Value)); + break; + } + RT_FALL_THRU(); + } + case MSR_IA32_X2APIC_VERSION: + case MSR_IA32_X2APIC_TPR: + case MSR_IA32_X2APIC_LDR: + case MSR_IA32_X2APIC_SVR: + case MSR_IA32_X2APIC_ISR0: case MSR_IA32_X2APIC_ISR1: case MSR_IA32_X2APIC_ISR2: case MSR_IA32_X2APIC_ISR3: + case MSR_IA32_X2APIC_ISR4: case MSR_IA32_X2APIC_ISR5: case MSR_IA32_X2APIC_ISR6: case MSR_IA32_X2APIC_ISR7: + case MSR_IA32_X2APIC_TMR0: case MSR_IA32_X2APIC_TMR1: case MSR_IA32_X2APIC_TMR2: case MSR_IA32_X2APIC_TMR3: + case MSR_IA32_X2APIC_TMR4: case MSR_IA32_X2APIC_TMR5: case MSR_IA32_X2APIC_TMR6: case MSR_IA32_X2APIC_TMR7: + case MSR_IA32_X2APIC_IRR0: case MSR_IA32_X2APIC_IRR1: case MSR_IA32_X2APIC_IRR2: case MSR_IA32_X2APIC_IRR3: + case MSR_IA32_X2APIC_IRR4: case MSR_IA32_X2APIC_IRR5: case MSR_IA32_X2APIC_IRR6: case MSR_IA32_X2APIC_IRR7: + case MSR_IA32_X2APIC_ESR: + case MSR_IA32_X2APIC_LVT_TIMER: + case MSR_IA32_X2APIC_LVT_THERMAL: + case MSR_IA32_X2APIC_LVT_PERF: + case MSR_IA32_X2APIC_LVT_LINT0: + case MSR_IA32_X2APIC_LVT_LINT1: + case MSR_IA32_X2APIC_LVT_ERROR: + case MSR_IA32_X2APIC_TIMER_ICR: + case MSR_IA32_X2APIC_TIMER_DCR: + { + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + uint16_t const offReg = X2APIC_GET_XAPIC_OFF(u32Reg); + *pu64Value = apicReadRaw32(pXApicPage, offReg); + break; + } + + /* Write-only MSRs: */ + case MSR_IA32_X2APIC_SELF_IPI: + case MSR_IA32_X2APIC_EOI: + { + rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_READ_WRITE_ONLY); + break; + } + + /* + * Windows guest using Hyper-V x2APIC MSR compatibility mode tries to read the "high" + * LDR bits, which is quite absurd (as it's a 32-bit register) using this invalid MSR + * index (0x80E), see @bugref{8382#c175}. + */ + case MSR_IA32_X2APIC_LDR + 1: + { + if (pApic->fHyperVCompatMode) + *pu64Value = 0; + else + rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_READ_RSVD_OR_UNKNOWN); + break; + } + + /* Reserved MSRs: */ + case MSR_IA32_X2APIC_LVT_CMCI: + default: + { + rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_READ_RSVD_OR_UNKNOWN); + break; + } + } + } + else + rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_INVALID_READ_MODE); + + return rcStrict; +} + + +/** + * Writes an APIC MSR. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param u32Reg The MSR being written. + * @param u64Value The value to write. + */ +VMM_INT_DECL(VBOXSTRICTRC) APICWriteMsr(PVMCPUCC pVCpu, uint32_t u32Reg, uint64_t u64Value) +{ + /* + * Validate. + */ + VMCPU_ASSERT_EMT(pVCpu); + Assert(u32Reg >= MSR_IA32_X2APIC_ID && u32Reg <= MSR_IA32_X2APIC_SELF_IPI); + + /* + * Is the APIC enabled? + */ + PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM)); + if (APICIsEnabled(pVCpu)) + { /* likely */ } + else + return apicMsrAccessError(pVCpu, u32Reg, pApic->enmMaxMode == PDMAPICMODE_NONE ? + APICMSRACCESS_WRITE_DISALLOWED_CONFIG : APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN); + +#ifndef IN_RING3 + if (pApic->CTXALLMID(f,Enabled)) + { /* likely */ } + else + return VINF_CPUM_R3_MSR_WRITE; +#endif + + STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMsrWrite)); + + /* + * In x2APIC mode, we need to raise #GP(0) for writes to reserved bits, unlike MMIO + * accesses where they are ignored. Hence, we need to validate each register before + * invoking the generic/xAPIC write functions. + * + * Bits 63:32 of all registers except the ICR are reserved, we'll handle this common + * case first and handle validating the remaining bits on a per-register basis. + * See Intel spec. 10.12.1.2 "x2APIC Register Address Space". + */ + if ( u32Reg != MSR_IA32_X2APIC_ICR + && RT_HI_U32(u64Value)) + return apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_RSVD_BITS); + + uint32_t u32Value = RT_LO_U32(u64Value); + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + if (RT_LIKELY( XAPIC_IN_X2APIC_MODE(pVCpu) + || pApic->fHyperVCompatMode)) + { + switch (u32Reg) + { + case MSR_IA32_X2APIC_TPR: + { + rcStrict = apicSetTprEx(pVCpu, u32Value, false /* fForceX2ApicBehaviour */); + break; + } + + case MSR_IA32_X2APIC_ICR: + { + rcStrict = apicSetIcr(pVCpu, u64Value, VINF_CPUM_R3_MSR_WRITE); + break; + } + + case MSR_IA32_X2APIC_SVR: + { + rcStrict = apicSetSvr(pVCpu, u32Value); + break; + } + + case MSR_IA32_X2APIC_ESR: + { + rcStrict = apicSetEsr(pVCpu, u32Value); + break; + } + + case MSR_IA32_X2APIC_TIMER_DCR: + { + rcStrict = apicSetTimerDcr(pVCpu, u32Value); + break; + } + + case MSR_IA32_X2APIC_LVT_TIMER: + case MSR_IA32_X2APIC_LVT_THERMAL: + case MSR_IA32_X2APIC_LVT_PERF: + case MSR_IA32_X2APIC_LVT_LINT0: + case MSR_IA32_X2APIC_LVT_LINT1: + case MSR_IA32_X2APIC_LVT_ERROR: + { + rcStrict = apicSetLvtEntry(pVCpu, X2APIC_GET_XAPIC_OFF(u32Reg), u32Value); + break; + } + + case MSR_IA32_X2APIC_TIMER_ICR: + { + rcStrict = apicSetTimerIcr(VMCPU_TO_DEVINS(pVCpu), pVCpu, VINF_CPUM_R3_MSR_WRITE, u32Value); + break; + } + + /* Write-only MSRs: */ + case MSR_IA32_X2APIC_SELF_IPI: + { + uint8_t const uVector = XAPIC_SELF_IPI_GET_VECTOR(u32Value); + apicPostInterrupt(pVCpu, uVector, XAPICTRIGGERMODE_EDGE, 0 /* uSrcTag */); + rcStrict = VINF_SUCCESS; + break; + } + + case MSR_IA32_X2APIC_EOI: + { + rcStrict = apicSetEoi(pVCpu, u32Value, false /* fForceX2ApicBehaviour */); + break; + } + + /* + * Windows guest using Hyper-V x2APIC MSR compatibility mode tries to write the "high" + * LDR bits, which is quite absurd (as it's a 32-bit register) using this invalid MSR + * index (0x80E). The write value was 0xffffffff on a Windows 8.1 64-bit guest. We can + * safely ignore this nonsense, See @bugref{8382#c7}. + */ + case MSR_IA32_X2APIC_LDR + 1: + { + if (pApic->fHyperVCompatMode) + rcStrict = VINF_SUCCESS; + else + rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN); + break; + } + + /* Special-treament (read-only normally, but not with Hyper-V) */ + case MSR_IA32_X2APIC_LDR: + { + if (pApic->fHyperVCompatMode) + { + rcStrict = apicSetLdr(pVCpu, u32Value); + break; + } + } + RT_FALL_THRU(); + /* Read-only MSRs: */ + case MSR_IA32_X2APIC_ID: + case MSR_IA32_X2APIC_VERSION: + case MSR_IA32_X2APIC_PPR: + case MSR_IA32_X2APIC_ISR0: case MSR_IA32_X2APIC_ISR1: case MSR_IA32_X2APIC_ISR2: case MSR_IA32_X2APIC_ISR3: + case MSR_IA32_X2APIC_ISR4: case MSR_IA32_X2APIC_ISR5: case MSR_IA32_X2APIC_ISR6: case MSR_IA32_X2APIC_ISR7: + case MSR_IA32_X2APIC_TMR0: case MSR_IA32_X2APIC_TMR1: case MSR_IA32_X2APIC_TMR2: case MSR_IA32_X2APIC_TMR3: + case MSR_IA32_X2APIC_TMR4: case MSR_IA32_X2APIC_TMR5: case MSR_IA32_X2APIC_TMR6: case MSR_IA32_X2APIC_TMR7: + case MSR_IA32_X2APIC_IRR0: case MSR_IA32_X2APIC_IRR1: case MSR_IA32_X2APIC_IRR2: case MSR_IA32_X2APIC_IRR3: + case MSR_IA32_X2APIC_IRR4: case MSR_IA32_X2APIC_IRR5: case MSR_IA32_X2APIC_IRR6: case MSR_IA32_X2APIC_IRR7: + case MSR_IA32_X2APIC_TIMER_CCR: + { + rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_READ_ONLY); + break; + } + + /* Reserved MSRs: */ + case MSR_IA32_X2APIC_LVT_CMCI: + default: + { + rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN); + break; + } + } + } + else + rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_INVALID_WRITE_MODE); + + return rcStrict; +} + + +/** + * Resets the APIC base MSR. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void apicResetBaseMsr(PVMCPUCC pVCpu) +{ + /* + * Initialize the APIC base MSR. The APIC enable-bit is set upon power-up or reset[1]. + * + * A Reset (in xAPIC and x2APIC mode) brings up the local APIC in xAPIC mode. + * An INIT IPI does -not- cause a transition between xAPIC and x2APIC mode[2]. + * + * [1] See AMD spec. 14.1.3 "Processor Initialization State" + * [2] See Intel spec. 10.12.5.1 "x2APIC States". + */ + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + + /* Construct. */ + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + PAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM)); + uint64_t uApicBaseMsr = MSR_IA32_APICBASE_ADDR; + if (pVCpu->idCpu == 0) + uApicBaseMsr |= MSR_IA32_APICBASE_BSP; + + /* If the VM was configured with no APIC, don't enable xAPIC mode, obviously. */ + if (pApic->enmMaxMode != PDMAPICMODE_NONE) + { + uApicBaseMsr |= MSR_IA32_APICBASE_EN; + + /* + * While coming out of a reset the APIC is enabled and in xAPIC mode. If software had previously + * disabled the APIC (which results in the CPUID bit being cleared as well) we re-enable it here. + * See Intel spec. 10.12.5.1 "x2APIC States". + */ + if (CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, true /*fVisible*/) == false) + LogRel(("APIC%u: Resetting mode to xAPIC\n", pVCpu->idCpu)); + } + + /* Commit. */ + ASMAtomicWriteU64(&pApicCpu->uApicBaseMsr, uApicBaseMsr); +} + + +/** + * Initializes per-VCPU APIC to the state following an INIT reset + * ("Wait-for-SIPI" state). + * + * @param pVCpu The cross context virtual CPU structure. + */ +void apicInitIpi(PVMCPUCC pVCpu) +{ + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + + /* + * See Intel spec. 10.4.7.3 "Local APIC State After an INIT Reset (Wait-for-SIPI State)" + * and AMD spec 16.3.2 "APIC Registers". + * + * The reason we don't simply zero out the entire APIC page and only set the non-zero members + * is because there are some registers that are not touched by the INIT IPI (e.g. version) + * operation and this function is only a subset of the reset operation. + */ + RT_ZERO(pXApicPage->irr); + RT_ZERO(pXApicPage->irr); + RT_ZERO(pXApicPage->isr); + RT_ZERO(pXApicPage->tmr); + RT_ZERO(pXApicPage->icr_hi); + RT_ZERO(pXApicPage->icr_lo); + RT_ZERO(pXApicPage->ldr); + RT_ZERO(pXApicPage->tpr); + RT_ZERO(pXApicPage->ppr); + RT_ZERO(pXApicPage->timer_icr); + RT_ZERO(pXApicPage->timer_ccr); + RT_ZERO(pXApicPage->timer_dcr); + + pXApicPage->dfr.u.u4Model = XAPICDESTFORMAT_FLAT; + pXApicPage->dfr.u.u28ReservedMb1 = UINT32_C(0xfffffff); + + /** @todo CMCI. */ + + RT_ZERO(pXApicPage->lvt_timer); + pXApicPage->lvt_timer.u.u1Mask = 1; + +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + RT_ZERO(pXApicPage->lvt_thermal); + pXApicPage->lvt_thermal.u.u1Mask = 1; +#endif + + RT_ZERO(pXApicPage->lvt_perf); + pXApicPage->lvt_perf.u.u1Mask = 1; + + RT_ZERO(pXApicPage->lvt_lint0); + pXApicPage->lvt_lint0.u.u1Mask = 1; + + RT_ZERO(pXApicPage->lvt_lint1); + pXApicPage->lvt_lint1.u.u1Mask = 1; + + RT_ZERO(pXApicPage->lvt_error); + pXApicPage->lvt_error.u.u1Mask = 1; + + RT_ZERO(pXApicPage->svr); + pXApicPage->svr.u.u8SpuriousVector = 0xff; + + /* The self-IPI register is reset to 0. See Intel spec. 10.12.5.1 "x2APIC States" */ + PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu); + RT_ZERO(pX2ApicPage->self_ipi); + + /* Clear the pending-interrupt bitmaps. */ + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + RT_BZERO(&pApicCpu->ApicPibLevel, sizeof(APICPIB)); + RT_BZERO(pApicCpu->CTX_SUFF(pvApicPib), sizeof(APICPIB)); + + /* Clear the interrupt line states for LINT0 and LINT1 pins. */ + pApicCpu->fActiveLint0 = false; + pApicCpu->fActiveLint1 = false; +} + + +/** + * Initializes per-VCPU APIC to the state following a power-up or hardware + * reset. + * + * @param pVCpu The cross context virtual CPU structure. + * @param fResetApicBaseMsr Whether to reset the APIC base MSR. + */ +void apicResetCpu(PVMCPUCC pVCpu, bool fResetApicBaseMsr) +{ + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + + LogFlow(("APIC%u: apicR3ResetCpu: fResetApicBaseMsr=%RTbool\n", pVCpu->idCpu, fResetApicBaseMsr)); + +#ifdef VBOX_STRICT + /* Verify that the initial APIC ID reported via CPUID matches our VMCPU ID assumption. */ + uint32_t uEax, uEbx, uEcx, uEdx; + uEax = uEbx = uEcx = uEdx = UINT32_MAX; + CPUMGetGuestCpuId(pVCpu, 1, 0, -1 /*f64BitMode*/, &uEax, &uEbx, &uEcx, &uEdx); + Assert(((uEbx >> 24) & 0xff) == pVCpu->idCpu); +#endif + + /* + * The state following a power-up or reset is a superset of the INIT state. + * See Intel spec. 10.4.7.3 "Local APIC State After an INIT Reset ('Wait-for-SIPI' State)" + */ + apicInitIpi(pVCpu); + + /* + * The APIC version register is read-only, so just initialize it here. + * It is not clear from the specs, where exactly it is initialized. + * The version determines the number of LVT entries and size of the APIC ID (8 bits for P4). + */ + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); +#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 + pXApicPage->version.u.u8MaxLvtEntry = XAPIC_MAX_LVT_ENTRIES_P4 - 1; + pXApicPage->version.u.u8Version = XAPIC_HARDWARE_VERSION_P4; + AssertCompile(sizeof(pXApicPage->id.u8ApicId) >= XAPIC_APIC_ID_BIT_COUNT_P4 / 8); +#else +# error "Implement Pentium and P6 family APIC architectures" +#endif + + /** @todo It isn't clear in the spec. where exactly the default base address + * is (re)initialized, atm we do it here in Reset. */ + if (fResetApicBaseMsr) + apicResetBaseMsr(pVCpu); + + /* + * Initialize the APIC ID register to xAPIC format. + */ + ASMMemZero32(&pXApicPage->id, sizeof(pXApicPage->id)); + pXApicPage->id.u8ApicId = pVCpu->idCpu; +} + + +/** + * Sets the APIC base MSR. + * + * @returns VBox status code - no informational ones, esp. not + * VINF_CPUM_R3_MSR_WRITE. Only the following two: + * @retval VINF_SUCCESS + * @retval VERR_CPUM_RAISE_GP_0 + * + * @param pVCpu The cross context virtual CPU structure. + * @param u64BaseMsr The value to set. + */ +VMM_INT_DECL(int) APICSetBaseMsr(PVMCPUCC pVCpu, uint64_t u64BaseMsr) +{ + Assert(pVCpu); + + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + PAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM)); + APICMODE enmOldMode = apicGetMode(pApicCpu->uApicBaseMsr); + APICMODE enmNewMode = apicGetMode(u64BaseMsr); + uint64_t uBaseMsr = pApicCpu->uApicBaseMsr; + + Log2(("APIC%u: ApicSetBaseMsr: u64BaseMsr=%#RX64 enmNewMode=%s enmOldMode=%s\n", pVCpu->idCpu, u64BaseMsr, + apicGetModeName(enmNewMode), apicGetModeName(enmOldMode))); + + /* + * We do not support re-mapping the APIC base address because: + * - We'll have to manage all the mappings ourselves in the APIC (reference counting based unmapping etc.) + * i.e. we can only unmap the MMIO region if no other APIC is mapped on that location. + * - It's unclear how/if IOM can fallback to handling regions as regular memory (if the MMIO + * region remains mapped but doesn't belong to the called VCPU's APIC). + */ + /** @todo Handle per-VCPU APIC base relocation. */ + if (MSR_IA32_APICBASE_GET_ADDR(uBaseMsr) != MSR_IA32_APICBASE_ADDR) + { + if (pVCpu->apic.s.cLogMaxSetApicBaseAddr++ < 5) + LogRel(("APIC%u: Attempt to relocate base to %#RGp, unsupported -> #GP(0)\n", pVCpu->idCpu, + MSR_IA32_APICBASE_GET_ADDR(uBaseMsr))); + return VERR_CPUM_RAISE_GP_0; + } + + /* Don't allow enabling xAPIC/x2APIC if the VM is configured with the APIC disabled. */ + if (pApic->enmMaxMode == PDMAPICMODE_NONE) + { + LogRel(("APIC%u: Disallowing APIC base MSR write as the VM is configured with APIC disabled!\n", pVCpu->idCpu)); + return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_DISALLOWED_CONFIG); + } + + /* + * Act on state transition. + */ + if (enmNewMode != enmOldMode) + { + switch (enmNewMode) + { + case APICMODE_DISABLED: + { + /* + * The APIC state needs to be reset (especially the APIC ID as x2APIC APIC ID bit layout + * is different). We can start with a clean slate identical to the state after a power-up/reset. + * + * See Intel spec. 10.4.3 "Enabling or Disabling the Local APIC". + * + * We'll also manually manage the APIC base MSR here. We want a single-point of commit + * at the end of this function rather than updating it in apicR3ResetCpu. This means we also + * need to update the CPUID leaf ourselves. + */ + apicResetCpu(pVCpu, false /* fResetApicBaseMsr */); + uBaseMsr &= ~(MSR_IA32_APICBASE_EN | MSR_IA32_APICBASE_EXTD); + CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, false /*fVisible*/); + LogRel(("APIC%u: Switched mode to disabled\n", pVCpu->idCpu)); + break; + } + + case APICMODE_XAPIC: + { + if (enmOldMode != APICMODE_DISABLED) + { + LogRel(("APIC%u: Can only transition to xAPIC state from disabled state\n", pVCpu->idCpu)); + return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID); + } + + uBaseMsr |= MSR_IA32_APICBASE_EN; + CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, true /*fVisible*/); + LogRel(("APIC%u: Switched mode to xAPIC\n", pVCpu->idCpu)); + break; + } + + case APICMODE_X2APIC: + { + if (pApic->enmMaxMode != PDMAPICMODE_X2APIC) + { + LogRel(("APIC%u: Disallowing transition to x2APIC mode as the VM is configured with the x2APIC disabled!\n", + pVCpu->idCpu)); + return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID); + } + + if (enmOldMode != APICMODE_XAPIC) + { + LogRel(("APIC%u: Can only transition to x2APIC state from xAPIC state\n", pVCpu->idCpu)); + return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID); + } + + uBaseMsr |= MSR_IA32_APICBASE_EN | MSR_IA32_APICBASE_EXTD; + + /* + * The APIC ID needs updating when entering x2APIC mode. + * Software written APIC ID in xAPIC mode isn't preserved. + * The APIC ID becomes read-only to software in x2APIC mode. + * + * See Intel spec. 10.12.5.1 "x2APIC States". + */ + PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu); + ASMMemZero32(&pX2ApicPage->id, sizeof(pX2ApicPage->id)); + pX2ApicPage->id.u32ApicId = pVCpu->idCpu; + + /* + * LDR initialization occurs when entering x2APIC mode. + * See Intel spec. 10.12.10.2 "Deriving Logical x2APIC ID from the Local x2APIC ID". + */ + pX2ApicPage->ldr.u32LogicalApicId = ((pX2ApicPage->id.u32ApicId & UINT32_C(0xffff0)) << 16) + | (UINT32_C(1) << pX2ApicPage->id.u32ApicId & UINT32_C(0xf)); + + LogRel(("APIC%u: Switched mode to x2APIC\n", pVCpu->idCpu)); + break; + } + + case APICMODE_INVALID: + default: + { + Log(("APIC%u: Invalid state transition attempted\n", pVCpu->idCpu)); + return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID); + } + } + } + + ASMAtomicWriteU64(&pApicCpu->uApicBaseMsr, uBaseMsr); + return VINF_SUCCESS; +} + + +/** + * Gets the APIC base MSR (no checks are performed wrt APIC hardware or its + * state). + * + * @returns The base MSR value. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(uint64_t) APICGetBaseMsrNoCheck(PCVMCPUCC pVCpu) +{ + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + return pApicCpu->uApicBaseMsr; +} + + +/** + * Gets the APIC base MSR. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pu64Value Where to store the MSR value. + */ +VMM_INT_DECL(VBOXSTRICTRC) APICGetBaseMsr(PVMCPUCC pVCpu, uint64_t *pu64Value) +{ + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + + PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM)); + if (pApic->enmMaxMode != PDMAPICMODE_NONE) + { + *pu64Value = APICGetBaseMsrNoCheck(pVCpu); + return VINF_SUCCESS; + } + + if (pVCpu->apic.s.cLogMaxGetApicBaseAddr++ < 5) + LogRel(("APIC%u: Reading APIC base MSR (%#x) when there is no APIC -> #GP(0)\n", pVCpu->idCpu, MSR_IA32_APICBASE)); + return VERR_CPUM_RAISE_GP_0; +} + + +/** + * Sets the TPR (Task Priority Register). + * + * @retval VINF_SUCCESS + * @retval VERR_CPUM_RAISE_GP_0 + * @retval VERR_PDM_NO_APIC_INSTANCE + * + * @param pVCpu The cross context virtual CPU structure. + * @param u8Tpr The TPR value to set. + */ +VMMDECL(int) APICSetTpr(PVMCPUCC pVCpu, uint8_t u8Tpr) +{ + if (APICIsEnabled(pVCpu)) + return apicSetTprEx(pVCpu, u8Tpr, false /* fForceX2ApicBehaviour */); + return VERR_PDM_NO_APIC_INSTANCE; +} + + +/** + * Gets the highest priority pending interrupt. + * + * @returns true if any interrupt is pending, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param pu8PendingIntr Where to store the interrupt vector if the + * interrupt is pending (optional, can be NULL). + */ +static bool apicGetHighestPendingInterrupt(PCVMCPUCC pVCpu, uint8_t *pu8PendingIntr) +{ + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu); + int const irrv = apicGetHighestSetBitInReg(&pXApicPage->irr, -1); + if (irrv >= 0) + { + Assert(irrv <= (int)UINT8_MAX); + if (pu8PendingIntr) + *pu8PendingIntr = (uint8_t)irrv; + return true; + } + return false; +} + + +/** + * Gets the APIC TPR (Task Priority Register). + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pu8Tpr Where to store the TPR. + * @param pfPending Where to store whether there is a pending interrupt + * (optional, can be NULL). + * @param pu8PendingIntr Where to store the highest-priority pending + * interrupt (optional, can be NULL). + */ +VMMDECL(int) APICGetTpr(PCVMCPUCC pVCpu, uint8_t *pu8Tpr, bool *pfPending, uint8_t *pu8PendingIntr) +{ + VMCPU_ASSERT_EMT(pVCpu); + if (APICIsEnabled(pVCpu)) + { + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu); + if (pfPending) + { + /* + * Just return whatever the highest pending interrupt is in the IRR. + * The caller is responsible for figuring out if it's masked by the TPR etc. + */ + *pfPending = apicGetHighestPendingInterrupt(pVCpu, pu8PendingIntr); + } + + *pu8Tpr = pXApicPage->tpr.u8Tpr; + return VINF_SUCCESS; + } + + *pu8Tpr = 0; + return VERR_PDM_NO_APIC_INSTANCE; +} + + +/** + * Gets the APIC timer frequency. + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pu64Value Where to store the timer frequency. + */ +VMM_INT_DECL(int) APICGetTimerFreq(PVMCC pVM, uint64_t *pu64Value) +{ + /* + * Validate. + */ + Assert(pVM); + AssertPtrReturn(pu64Value, VERR_INVALID_PARAMETER); + + PVMCPUCC pVCpu = pVM->CTX_SUFF(apCpus)[0]; + if (APICIsEnabled(pVCpu)) + { + PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + *pu64Value = PDMDevHlpTimerGetFreq(VMCPU_TO_DEVINS(pVCpu), pApicCpu->hTimer); + return VINF_SUCCESS; + } + return VERR_PDM_NO_APIC_INSTANCE; +} + + +/** + * Delivers an interrupt message via the system bus. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param uDest The destination mask. + * @param uDestMode The destination mode. + * @param uDeliveryMode The delivery mode. + * @param uVector The interrupt vector. + * @param uPolarity The interrupt line polarity. + * @param uTriggerMode The trigger mode. + * @param uSrcTag The interrupt source tag (debugging). + */ +VMM_INT_DECL(int) APICBusDeliver(PVMCC pVM, uint8_t uDest, uint8_t uDestMode, uint8_t uDeliveryMode, uint8_t uVector, + uint8_t uPolarity, uint8_t uTriggerMode, uint32_t uSrcTag) +{ + NOREF(uPolarity); + + /* + * If the APIC isn't enabled, do nothing and pretend success. + */ + if (APICIsEnabled(pVM->CTX_SUFF(apCpus)[0])) + { /* likely */ } + else + return VINF_SUCCESS; + + /* + * The destination field (mask) in the IO APIC redirectable table entry is 8-bits. + * Hence, the broadcast mask is 0xff. + * See IO APIC spec. 3.2.4. "IOREDTBL[23:0] - I/O Redirectable Table Registers". + */ + XAPICTRIGGERMODE enmTriggerMode = (XAPICTRIGGERMODE)uTriggerMode; + XAPICDELIVERYMODE enmDeliveryMode = (XAPICDELIVERYMODE)uDeliveryMode; + XAPICDESTMODE enmDestMode = (XAPICDESTMODE)uDestMode; + uint32_t fDestMask = uDest; + uint32_t fBroadcastMask = UINT32_C(0xff); + + Log2(("APIC: apicBusDeliver: fDestMask=%#x enmDestMode=%s enmTriggerMode=%s enmDeliveryMode=%s uVector=%#x uSrcTag=%#x\n", + fDestMask, apicGetDestModeName(enmDestMode), apicGetTriggerModeName(enmTriggerMode), + apicGetDeliveryModeName(enmDeliveryMode), uVector, uSrcTag)); + + bool fIntrAccepted; + VMCPUSET DestCpuSet; + apicGetDestCpuSet(pVM, fDestMask, fBroadcastMask, enmDestMode, enmDeliveryMode, &DestCpuSet); + VBOXSTRICTRC rcStrict = apicSendIntr(pVM, NULL /* pVCpu */, uVector, enmTriggerMode, enmDeliveryMode, &DestCpuSet, + &fIntrAccepted, uSrcTag, VINF_SUCCESS /* rcRZ */); + if (fIntrAccepted) + return VBOXSTRICTRC_VAL(rcStrict); + return VERR_APIC_INTR_DISCARDED; +} + + +/** + * Assert/de-assert the local APIC's LINT0/LINT1 interrupt pins. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param u8Pin The interrupt pin (0 for LINT0 or 1 for LINT1). + * @param u8Level The level (0 for low or 1 for high). + * @param rcRZ The return code if the operation cannot be performed in + * the current context. + * + * @note All callers totally ignores the status code! + */ +VMM_INT_DECL(VBOXSTRICTRC) APICLocalInterrupt(PVMCPUCC pVCpu, uint8_t u8Pin, uint8_t u8Level, int rcRZ) +{ + AssertReturn(u8Pin <= 1, VERR_INVALID_PARAMETER); + AssertReturn(u8Level <= 1, VERR_INVALID_PARAMETER); + + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + + /* If the APIC is enabled, the interrupt is subject to LVT programming. */ + if (APICIsEnabled(pVCpu)) + { + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu); + + /* Pick the LVT entry corresponding to the interrupt pin. */ + static const uint16_t s_au16LvtOffsets[] = + { + XAPIC_OFF_LVT_LINT0, + XAPIC_OFF_LVT_LINT1 + }; + Assert(u8Pin < RT_ELEMENTS(s_au16LvtOffsets)); + uint16_t const offLvt = s_au16LvtOffsets[u8Pin]; + uint32_t const uLvt = apicReadRaw32(pXApicPage, offLvt); + + /* If software hasn't masked the interrupt in the LVT entry, proceed interrupt processing. */ + if (!XAPIC_LVT_IS_MASKED(uLvt)) + { + XAPICDELIVERYMODE const enmDeliveryMode = XAPIC_LVT_GET_DELIVERY_MODE(uLvt); + XAPICTRIGGERMODE enmTriggerMode = XAPIC_LVT_GET_TRIGGER_MODE(uLvt); + + switch (enmDeliveryMode) + { + case XAPICDELIVERYMODE_INIT: + { + /** @todo won't work in R0/RC because callers don't care about rcRZ. */ + AssertMsgFailed(("INIT through LINT0/LINT1 is not yet supported\n")); + } + RT_FALL_THRU(); + case XAPICDELIVERYMODE_FIXED: + { + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + uint8_t const uVector = XAPIC_LVT_GET_VECTOR(uLvt); + bool fActive = RT_BOOL(u8Level & 1); + bool volatile *pfActiveLine = u8Pin == 0 ? &pApicCpu->fActiveLint0 : &pApicCpu->fActiveLint1; + /** @todo Polarity is busted elsewhere, we need to fix that + * first. See @bugref{8386#c7}. */ +#if 0 + uint8_t const u8Polarity = XAPIC_LVT_GET_POLARITY(uLvt); + fActive ^= u8Polarity; */ +#endif + if (!fActive) + { + ASMAtomicCmpXchgBool(pfActiveLine, false, true); + break; + } + + /* Level-sensitive interrupts are not supported for LINT1. See Intel spec. 10.5.1 "Local Vector Table". */ + if (offLvt == XAPIC_OFF_LVT_LINT1) + enmTriggerMode = XAPICTRIGGERMODE_EDGE; + /** @todo figure out what "If the local APIC is not used in conjunction with an I/O APIC and fixed + delivery mode is selected; the Pentium 4, Intel Xeon, and P6 family processors will always + use level-sensitive triggering, regardless if edge-sensitive triggering is selected." + means. */ + + bool fSendIntr; + if (enmTriggerMode == XAPICTRIGGERMODE_EDGE) + { + /* Recognize and send the interrupt only on an edge transition. */ + fSendIntr = ASMAtomicCmpXchgBool(pfActiveLine, true, false); + } + else + { + /* For level-triggered interrupts, redundant interrupts are not a problem. */ + Assert(enmTriggerMode == XAPICTRIGGERMODE_LEVEL); + ASMAtomicCmpXchgBool(pfActiveLine, true, false); + + /* Only when the remote IRR isn't set, set it and send the interrupt. */ + if (!(pXApicPage->lvt_lint0.all.u32LvtLint0 & XAPIC_LVT_REMOTE_IRR)) + { + Assert(offLvt == XAPIC_OFF_LVT_LINT0); + ASMAtomicOrU32((volatile uint32_t *)&pXApicPage->lvt_lint0.all.u32LvtLint0, XAPIC_LVT_REMOTE_IRR); + fSendIntr = true; + } + else + fSendIntr = false; + } + + if (fSendIntr) + { + VMCPUSET DestCpuSet; + VMCPUSET_EMPTY(&DestCpuSet); + VMCPUSET_ADD(&DestCpuSet, pVCpu->idCpu); + rcStrict = apicSendIntr(pVCpu->CTX_SUFF(pVM), pVCpu, uVector, enmTriggerMode, enmDeliveryMode, + &DestCpuSet, NULL /* pfIntrAccepted */, 0 /* uSrcTag */, rcRZ); + } + break; + } + + case XAPICDELIVERYMODE_SMI: + case XAPICDELIVERYMODE_NMI: + { + VMCPUSET DestCpuSet; + VMCPUSET_EMPTY(&DestCpuSet); + VMCPUSET_ADD(&DestCpuSet, pVCpu->idCpu); + uint8_t const uVector = XAPIC_LVT_GET_VECTOR(uLvt); + rcStrict = apicSendIntr(pVCpu->CTX_SUFF(pVM), pVCpu, uVector, enmTriggerMode, enmDeliveryMode, &DestCpuSet, + NULL /* pfIntrAccepted */, 0 /* uSrcTag */, rcRZ); + break; + } + + case XAPICDELIVERYMODE_EXTINT: + { + Log2(("APIC%u: apicLocalInterrupt: %s ExtINT through LINT%u\n", pVCpu->idCpu, + u8Level ? "Raising" : "Lowering", u8Pin)); + if (u8Level) + apicSetInterruptFF(pVCpu, PDMAPICIRQ_EXTINT); + else + apicClearInterruptFF(pVCpu, PDMAPICIRQ_EXTINT); + break; + } + + /* Reserved/unknown delivery modes: */ + case XAPICDELIVERYMODE_LOWEST_PRIO: + case XAPICDELIVERYMODE_STARTUP: + default: + { + AssertMsgFailed(("APIC%u: LocalInterrupt: Invalid delivery mode %#x (%s) on LINT%d\n", pVCpu->idCpu, + enmDeliveryMode, apicGetDeliveryModeName(enmDeliveryMode), u8Pin)); + rcStrict = VERR_INTERNAL_ERROR_3; + break; + } + } + } + } + else + { + /* The APIC is hardware disabled. The CPU behaves as though there is no on-chip APIC. */ + if (u8Pin == 0) + { + /* LINT0 behaves as an external interrupt pin. */ + Log2(("APIC%u: apicLocalInterrupt: APIC hardware-disabled, %s INTR\n", pVCpu->idCpu, + u8Level ? "raising" : "lowering")); + if (u8Level) + apicSetInterruptFF(pVCpu, PDMAPICIRQ_EXTINT); + else + apicClearInterruptFF(pVCpu, PDMAPICIRQ_EXTINT); + } + else + { + /* LINT1 behaves as NMI. */ + Log2(("APIC%u: apicLocalInterrupt: APIC hardware-disabled, raising NMI\n", pVCpu->idCpu)); + apicSetInterruptFF(pVCpu, PDMAPICIRQ_NMI); + } + } + + return rcStrict; +} + + +/** + * Gets the next highest-priority interrupt from the APIC, marking it as an + * "in-service" interrupt. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pu8Vector Where to store the vector. + * @param puSrcTag Where to store the interrupt source tag (debugging). + */ +VMM_INT_DECL(int) APICGetInterrupt(PVMCPUCC pVCpu, uint8_t *pu8Vector, uint32_t *puSrcTag) +{ + VMCPU_ASSERT_EMT(pVCpu); + Assert(pu8Vector); + + LogFlow(("APIC%u: apicGetInterrupt:\n", pVCpu->idCpu)); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + bool const fApicHwEnabled = APICIsEnabled(pVCpu); + if ( fApicHwEnabled + && pXApicPage->svr.u.fApicSoftwareEnable) + { + int const irrv = apicGetHighestSetBitInReg(&pXApicPage->irr, -1); + if (RT_LIKELY(irrv >= 0)) + { + Assert(irrv <= (int)UINT8_MAX); + uint8_t const uVector = irrv; + + /* + * This can happen if the APIC receives an interrupt when the CPU has interrupts + * disabled but the TPR is raised by the guest before re-enabling interrupts. + */ + uint8_t const uTpr = pXApicPage->tpr.u8Tpr; + if ( uTpr > 0 + && XAPIC_TPR_GET_TP(uVector) <= XAPIC_TPR_GET_TP(uTpr)) + { + Log2(("APIC%u: apicGetInterrupt: Interrupt masked. uVector=%#x uTpr=%#x SpuriousVector=%#x\n", pVCpu->idCpu, + uVector, uTpr, pXApicPage->svr.u.u8SpuriousVector)); + *pu8Vector = uVector; + *puSrcTag = 0; + STAM_COUNTER_INC(&pVCpu->apic.s.StatMaskedByTpr); + return VERR_APIC_INTR_MASKED_BY_TPR; + } + + /* + * The PPR should be up-to-date at this point through apicSetEoi(). + * We're on EMT so no parallel updates possible. + * Subject the pending vector to PPR prioritization. + */ + uint8_t const uPpr = pXApicPage->ppr.u8Ppr; + if ( !uPpr + || XAPIC_PPR_GET_PP(uVector) > XAPIC_PPR_GET_PP(uPpr)) + { + apicClearVectorInReg(&pXApicPage->irr, uVector); + apicSetVectorInReg(&pXApicPage->isr, uVector); + apicUpdatePpr(pVCpu); + apicSignalNextPendingIntr(pVCpu); + + /* Retrieve the interrupt source tag associated with this interrupt. */ + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + AssertCompile(RT_ELEMENTS(pApicCpu->auSrcTags) > UINT8_MAX); + *puSrcTag = pApicCpu->auSrcTags[uVector]; + pApicCpu->auSrcTags[uVector] = 0; + + Log2(("APIC%u: apicGetInterrupt: Valid Interrupt. uVector=%#x uSrcTag=%#x\n", pVCpu->idCpu, uVector, *puSrcTag)); + *pu8Vector = uVector; + return VINF_SUCCESS; + } + + STAM_COUNTER_INC(&pVCpu->apic.s.StatMaskedByPpr); + Log2(("APIC%u: apicGetInterrupt: Interrupt's priority is not higher than the PPR. uVector=%#x PPR=%#x\n", + pVCpu->idCpu, uVector, uPpr)); + } + else + Log2(("APIC%u: apicGetInterrupt: No pending bits in IRR\n", pVCpu->idCpu)); + } + else + Log2(("APIC%u: apicGetInterrupt: APIC %s disabled\n", pVCpu->idCpu, !fApicHwEnabled ? "hardware" : "software")); + + *pu8Vector = 0; + *puSrcTag = 0; + return VERR_APIC_INTR_NOT_PENDING; +} + + +/** + * @callback_method_impl{FNIOMMMIONEWREAD} + */ +DECLCALLBACK(VBOXSTRICTRC) apicReadMmio(PPDMDEVINS pDevIns, void *pvUser, RTGCPHYS off, void *pv, unsigned cb) +{ + NOREF(pvUser); + Assert(!(off & 0xf)); + Assert(cb == 4); RT_NOREF_PV(cb); + + PVMCPUCC pVCpu = PDMDevHlpGetVMCPU(pDevIns); + uint16_t offReg = off & 0xff0; + uint32_t uValue = 0; + + STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMmioRead)); + + VBOXSTRICTRC rc = VBOXSTRICTRC_VAL(apicReadRegister(pDevIns, pVCpu, offReg, &uValue)); + *(uint32_t *)pv = uValue; + + Log2(("APIC%u: apicReadMmio: offReg=%#RX16 uValue=%#RX32\n", pVCpu->idCpu, offReg, uValue)); + return rc; +} + + +/** + * @callback_method_impl{FNIOMMMIONEWWRITE} + */ +DECLCALLBACK(VBOXSTRICTRC) apicWriteMmio(PPDMDEVINS pDevIns, void *pvUser, RTGCPHYS off, void const *pv, unsigned cb) +{ + NOREF(pvUser); + Assert(!(off & 0xf)); + Assert(cb == 4); RT_NOREF_PV(cb); + + PVMCPUCC pVCpu = PDMDevHlpGetVMCPU(pDevIns); + uint16_t offReg = off & 0xff0; + uint32_t uValue = *(uint32_t *)pv; + + STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMmioWrite)); + + Log2(("APIC%u: apicWriteMmio: offReg=%#RX16 uValue=%#RX32\n", pVCpu->idCpu, offReg, uValue)); + + return apicWriteRegister(pDevIns, pVCpu, offReg, uValue); +} + + +/** + * Sets the interrupt pending force-flag and pokes the EMT if required. + * + * @param pVCpu The cross context virtual CPU structure. + * @param enmType The IRQ type. + */ +static void apicSetInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType) +{ +#ifdef IN_RING3 + /* IRQ state should be loaded as-is by "LoadExec". Changes can be made from LoadDone. */ + Assert(pVCpu->pVMR3->enmVMState != VMSTATE_LOADING || PDMR3HasLoadedState(pVCpu->pVMR3)); +#endif + + switch (enmType) + { + case PDMAPICIRQ_HARDWARE: + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC); + break; + case PDMAPICIRQ_UPDATE_PENDING: VMCPU_FF_SET(pVCpu, VMCPU_FF_UPDATE_APIC); break; + case PDMAPICIRQ_NMI: VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI); break; + case PDMAPICIRQ_SMI: VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI); break; + case PDMAPICIRQ_EXTINT: VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC); break; + default: + AssertMsgFailed(("enmType=%d\n", enmType)); + break; + } + + /* + * We need to wake up the target CPU if we're not on EMT. + */ + /** @todo r=bird: Why do we skip this waking up for PDMAPICIRQ_HARDWARE? */ + /** @todo r=bird: We could just use RTThreadNativeSelf() here, couldn't we? */ +#if defined(IN_RING0) + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPUID idCpu = pVCpu->idCpu; + if ( enmType != PDMAPICIRQ_HARDWARE + && VMMGetCpuId(pVM) != idCpu) + { + switch (VMCPU_GET_STATE(pVCpu)) + { + case VMCPUSTATE_STARTED_EXEC: + Log7Func(("idCpu=%u VMCPUSTATE_STARTED_EXEC\n", idCpu)); + GVMMR0SchedPokeNoGVMNoLock(pVM, idCpu); + break; + + case VMCPUSTATE_STARTED_HALTED: + Log7Func(("idCpu=%u VMCPUSTATE_STARTED_HALTED\n", idCpu)); + GVMMR0SchedWakeUpNoGVMNoLock(pVM, idCpu); + break; + + default: + Log7Func(("idCpu=%u enmState=%d\n", idCpu, pVCpu->enmState)); + break; /* nothing to do in other states. */ + } + } +#elif defined(IN_RING3) + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPUID idCpu = pVCpu->idCpu; + if ( enmType != PDMAPICIRQ_HARDWARE + && VMMGetCpuId(pVM) != idCpu) + { + Log7Func(("idCpu=%u enmState=%d\n", idCpu, pVCpu->enmState)); + VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM | VMNOTIFYFF_FLAGS_POKE); + } +#endif +} + + +/** + * Clears the interrupt pending force-flag. + * + * @param pVCpu The cross context virtual CPU structure. + * @param enmType The IRQ type. + */ +void apicClearInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType) +{ +#ifdef IN_RING3 + /* IRQ state should be loaded as-is by "LoadExec". Changes can be made from LoadDone. */ + Assert(pVCpu->pVMR3->enmVMState != VMSTATE_LOADING || PDMR3HasLoadedState(pVCpu->pVMR3)); +#endif + + /* NMI/SMI can't be cleared. */ + switch (enmType) + { + case PDMAPICIRQ_HARDWARE: VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC); break; + case PDMAPICIRQ_EXTINT: VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC); break; + default: + AssertMsgFailed(("enmType=%d\n", enmType)); + break; + } +} + + +/** + * Posts an interrupt to a target APIC. + * + * This function handles interrupts received from the system bus or + * interrupts generated locally from the LVT or via a self IPI. + * + * Don't use this function to try and deliver ExtINT style interrupts. + * + * @returns true if the interrupt was accepted, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param uVector The vector of the interrupt to be posted. + * @param enmTriggerMode The trigger mode of the interrupt. + * @param uSrcTag The interrupt source tag (debugging). + * + * @thread Any. + */ +bool apicPostInterrupt(PVMCPUCC pVCpu, uint8_t uVector, XAPICTRIGGERMODE enmTriggerMode, uint32_t uSrcTag) +{ + Assert(pVCpu); + Assert(uVector > XAPIC_ILLEGAL_VECTOR_END); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PCAPIC pApic = VM_TO_APIC(pVM); + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + bool fAccepted = true; + + STAM_PROFILE_START(&pApicCpu->StatPostIntr, a); + STAM_REL_COUNTER_INC(&pApicCpu->StatPostIntrCnt); + STAM_REL_COUNTER_INC(&pApicCpu->aStatVectors[uVector]); + + /* + * Only post valid interrupt vectors. + * See Intel spec. 10.5.2 "Valid Interrupt Vectors". + */ + if (RT_LIKELY(uVector > XAPIC_ILLEGAL_VECTOR_END)) + { + /* + * If the interrupt is already pending in the IRR we can skip the + * potential expensive operation of poking the guest EMT out of execution. + */ + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu); + if (!apicTestVectorInReg(&pXApicPage->irr, uVector)) /* PAV */ + { + /* Update the interrupt source tag (debugging). */ + if (!pApicCpu->auSrcTags[uVector]) + pApicCpu->auSrcTags[uVector] = uSrcTag; + else + pApicCpu->auSrcTags[uVector] |= RT_BIT_32(31); + + Log2(("APIC: apicPostInterrupt: SrcCpu=%u TargetCpu=%u uVector=%#x %s\n", + VMMGetCpuId(pVM), pVCpu->idCpu, uVector, enmTriggerMode == XAPICTRIGGERMODE_EDGE ? "edge" : "lvl")); + if (enmTriggerMode == XAPICTRIGGERMODE_EDGE) + { + if (pApic->fPostedIntrsEnabled) + { /** @todo posted-interrupt call to hardware */ } + else + { + apicSetVectorInPib(pApicCpu->CTX_SUFF(pvApicPib), uVector); + uint32_t const fAlreadySet = apicSetNotificationBitInPib((PAPICPIB)pApicCpu->CTX_SUFF(pvApicPib)); + if (!fAlreadySet) + { + Log2(("APIC: apicPostInterrupt: Setting UPDATE_APIC FF for edge-triggered intr. uVector=%#x\n", uVector)); + apicSetInterruptFF(pVCpu, PDMAPICIRQ_UPDATE_PENDING); + } + } + } + else + { + /* + * Level-triggered interrupts requires updating of the TMR and thus cannot be + * delivered asynchronously. + */ + apicSetVectorInPib(&pApicCpu->ApicPibLevel, uVector); + uint32_t const fAlreadySet = apicSetNotificationBitInPib(&pApicCpu->ApicPibLevel); + if (!fAlreadySet) + { + Log2(("APIC: apicPostInterrupt: Setting UPDATE_APIC FF for level-triggered intr. uVector=%#x\n", uVector)); + apicSetInterruptFF(pVCpu, PDMAPICIRQ_UPDATE_PENDING); + } + } + } + else + { + Log2(("APIC: apicPostInterrupt: SrcCpu=%u TargetCpu=%u. Vector %#x Already in IRR, skipping\n", VMMGetCpuId(pVM), + pVCpu->idCpu, uVector)); + STAM_COUNTER_INC(&pApicCpu->StatPostIntrAlreadyPending); + } + } + else + { + fAccepted = false; + apicSetError(pVCpu, XAPIC_ESR_RECV_ILLEGAL_VECTOR); + } + + STAM_PROFILE_STOP(&pApicCpu->StatPostIntr, a); + return fAccepted; +} + + +/** + * Starts the APIC timer. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uInitialCount The timer's Initial-Count Register (ICR), must be > + * 0. + * @thread Any. + */ +void apicStartTimer(PVMCPUCC pVCpu, uint32_t uInitialCount) +{ + Assert(pVCpu); + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + PPDMDEVINS pDevIns = VMCPU_TO_DEVINS(pVCpu); + Assert(PDMDevHlpTimerIsLockOwner(pDevIns, pApicCpu->hTimer)); + Assert(uInitialCount > 0); + + PCXAPICPAGE pXApicPage = APICCPU_TO_CXAPICPAGE(pApicCpu); + uint8_t const uTimerShift = apicGetTimerShift(pXApicPage); + uint64_t const cTicksToNext = (uint64_t)uInitialCount << uTimerShift; + + Log2(("APIC%u: apicStartTimer: uInitialCount=%#RX32 uTimerShift=%u cTicksToNext=%RU64\n", pVCpu->idCpu, uInitialCount, + uTimerShift, cTicksToNext)); + + /* + * The assumption here is that the timer doesn't tick during this call + * and thus setting a relative time to fire next is accurate. The advantage + * however is updating u64TimerInitial 'atomically' while setting the next + * tick. + */ + PDMDevHlpTimerSetRelative(pDevIns, pApicCpu->hTimer, cTicksToNext, &pApicCpu->u64TimerInitial); + apicHintTimerFreq(pDevIns, pApicCpu, uInitialCount, uTimerShift); +} + + +/** + * Stops the APIC timer. + * + * @param pVCpu The cross context virtual CPU structure. + * @thread Any. + */ +static void apicStopTimer(PVMCPUCC pVCpu) +{ + Assert(pVCpu); + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + PPDMDEVINS pDevIns = VMCPU_TO_DEVINS(pVCpu); + Assert(PDMDevHlpTimerIsLockOwner(pDevIns, pApicCpu->hTimer)); + + Log2(("APIC%u: apicStopTimer\n", pVCpu->idCpu)); + + PDMDevHlpTimerStop(pDevIns, pApicCpu->hTimer); /* This will reset the hint, no need to explicitly call TMTimerSetFrequencyHint(). */ + pApicCpu->uHintedTimerInitialCount = 0; + pApicCpu->uHintedTimerShift = 0; +} + + +/** + * Queues a pending interrupt as in-service. + * + * This function should only be needed without virtualized APIC + * registers. With virtualized APIC registers, it's sufficient to keep + * the interrupts pending in the IRR as the hardware takes care of + * virtual interrupt delivery. + * + * @returns true if the interrupt was queued to in-service interrupts, + * false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param u8PendingIntr The pending interrupt to queue as + * in-service. + * + * @remarks This assumes the caller has done the necessary checks and + * is ready to take actually service the interrupt (TPR, + * interrupt shadow etc.) + */ +VMM_INT_DECL(bool) APICQueueInterruptToService(PVMCPUCC pVCpu, uint8_t u8PendingIntr) +{ + VMCPU_ASSERT_EMT(pVCpu); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PAPIC pApic = VM_TO_APIC(pVM); + Assert(!pApic->fVirtApicRegsEnabled); + NOREF(pApic); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + bool const fIsPending = apicTestVectorInReg(&pXApicPage->irr, u8PendingIntr); + if (fIsPending) + { + apicClearVectorInReg(&pXApicPage->irr, u8PendingIntr); + apicSetVectorInReg(&pXApicPage->isr, u8PendingIntr); + apicUpdatePpr(pVCpu); + return true; + } + return false; +} + + +/** + * De-queues a pending interrupt from in-service. + * + * This undoes APICQueueInterruptToService() for premature VM-exits before event + * injection. + * + * @param pVCpu The cross context virtual CPU structure. + * @param u8PendingIntr The pending interrupt to de-queue from + * in-service. + */ +VMM_INT_DECL(void) APICDequeueInterruptFromService(PVMCPUCC pVCpu, uint8_t u8PendingIntr) +{ + VMCPU_ASSERT_EMT(pVCpu); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PAPIC pApic = VM_TO_APIC(pVM); + Assert(!pApic->fVirtApicRegsEnabled); + NOREF(pApic); + + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + bool const fInService = apicTestVectorInReg(&pXApicPage->isr, u8PendingIntr); + if (fInService) + { + apicClearVectorInReg(&pXApicPage->isr, u8PendingIntr); + apicSetVectorInReg(&pXApicPage->irr, u8PendingIntr); + apicUpdatePpr(pVCpu); + } +} + + +/** + * Updates pending interrupts from the pending-interrupt bitmaps to the IRR. + * + * @param pVCpu The cross context virtual CPU structure. + * + * @note NEM/win is ASSUMING the an up to date TPR is not required here. + */ +VMMDECL(void) APICUpdatePendingInterrupts(PVMCPUCC pVCpu) +{ + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + + PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu); + bool fHasPendingIntrs = false; + + Log3(("APIC%u: APICUpdatePendingInterrupts:\n", pVCpu->idCpu)); + STAM_PROFILE_START(&pApicCpu->StatUpdatePendingIntrs, a); + + /* Update edge-triggered pending interrupts. */ + PAPICPIB pPib = (PAPICPIB)pApicCpu->CTX_SUFF(pvApicPib); + for (;;) + { + uint32_t const fAlreadySet = apicClearNotificationBitInPib((PAPICPIB)pApicCpu->CTX_SUFF(pvApicPib)); + if (!fAlreadySet) + break; + + AssertCompile(RT_ELEMENTS(pXApicPage->irr.u) == 2 * RT_ELEMENTS(pPib->au64VectorBitmap)); + for (size_t idxPib = 0, idxReg = 0; idxPib < RT_ELEMENTS(pPib->au64VectorBitmap); idxPib++, idxReg += 2) + { + uint64_t const u64Fragment = ASMAtomicXchgU64(&pPib->au64VectorBitmap[idxPib], 0); + if (u64Fragment) + { + uint32_t const u32FragmentLo = RT_LO_U32(u64Fragment); + uint32_t const u32FragmentHi = RT_HI_U32(u64Fragment); + Log6Func(("edge[%u/%u]: %'016RX64: irr=%08RX32'%08RX32 |; tmr=%08RX32'%08RX32 &~\n", idxPib, idxReg, u64Fragment, + pXApicPage->irr.u[idxReg].u32Reg, pXApicPage->irr.u[idxReg + 1].u32Reg, + pXApicPage->tmr.u[idxReg].u32Reg, pXApicPage->tmr.u[idxReg + 1].u32Reg)); + + pXApicPage->irr.u[idxReg].u32Reg |= u32FragmentLo; + pXApicPage->irr.u[idxReg + 1].u32Reg |= u32FragmentHi; + + pXApicPage->tmr.u[idxReg].u32Reg &= ~u32FragmentLo; + pXApicPage->tmr.u[idxReg + 1].u32Reg &= ~u32FragmentHi; + fHasPendingIntrs = true; + } + } + } + + /* Update level-triggered pending interrupts. */ + pPib = (PAPICPIB)&pApicCpu->ApicPibLevel; + for (;;) + { + uint32_t const fAlreadySet = apicClearNotificationBitInPib((PAPICPIB)&pApicCpu->ApicPibLevel); + if (!fAlreadySet) + break; + + AssertCompile(RT_ELEMENTS(pXApicPage->irr.u) == 2 * RT_ELEMENTS(pPib->au64VectorBitmap)); + for (size_t idxPib = 0, idxReg = 0; idxPib < RT_ELEMENTS(pPib->au64VectorBitmap); idxPib++, idxReg += 2) + { + uint64_t const u64Fragment = ASMAtomicXchgU64(&pPib->au64VectorBitmap[idxPib], 0); + if (u64Fragment) + { + Log6Func(("level[%u/%u]: %'016RX64: irr=%08RX32'%08RX32 |; tmr=%08RX32'%08RX32 |\n", idxPib, idxReg, u64Fragment, + pXApicPage->irr.u[idxReg].u32Reg, pXApicPage->irr.u[idxReg + 1].u32Reg, + pXApicPage->tmr.u[idxReg].u32Reg, pXApicPage->tmr.u[idxReg + 1].u32Reg)); + uint32_t const u32FragmentLo = RT_LO_U32(u64Fragment); + uint32_t const u32FragmentHi = RT_HI_U32(u64Fragment); + + pXApicPage->irr.u[idxReg].u32Reg |= u32FragmentLo; + pXApicPage->irr.u[idxReg + 1].u32Reg |= u32FragmentHi; + + pXApicPage->tmr.u[idxReg].u32Reg |= u32FragmentLo; + pXApicPage->tmr.u[idxReg + 1].u32Reg |= u32FragmentHi; + fHasPendingIntrs = true; + } + } + } + + STAM_PROFILE_STOP(&pApicCpu->StatUpdatePendingIntrs, a); + Log3(("APIC%u: APICUpdatePendingInterrupts: fHasPendingIntrs=%RTbool\n", pVCpu->idCpu, fHasPendingIntrs)); + + if ( fHasPendingIntrs + && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC)) + apicSignalNextPendingIntr(pVCpu); +} + + +/** + * Gets the highest priority pending interrupt. + * + * @returns true if any interrupt is pending, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param pu8PendingIntr Where to store the interrupt vector if the + * interrupt is pending. + */ +VMM_INT_DECL(bool) APICGetHighestPendingInterrupt(PVMCPUCC pVCpu, uint8_t *pu8PendingIntr) +{ + VMCPU_ASSERT_EMT(pVCpu); + return apicGetHighestPendingInterrupt(pVCpu, pu8PendingIntr); +} + + +/** + * Posts an interrupt to a target APIC, Hyper-V interface. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uVector The vector of the interrupt to be posted. + * @param fAutoEoi Whether this interrupt has automatic EOI + * treatment. + * @param enmTriggerMode The trigger mode of the interrupt. + * + * @thread Any. + */ +VMM_INT_DECL(void) APICHvSendInterrupt(PVMCPUCC pVCpu, uint8_t uVector, bool fAutoEoi, XAPICTRIGGERMODE enmTriggerMode) +{ + Assert(pVCpu); + Assert(!fAutoEoi); /** @todo AutoEOI. */ + RT_NOREF(fAutoEoi); + apicPostInterrupt(pVCpu, uVector, enmTriggerMode, 0 /* uSrcTag */); +} + + +/** + * Sets the Task Priority Register (TPR), Hyper-V interface. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uTpr The TPR value to set. + * + * @remarks Validates like in x2APIC mode. + */ +VMM_INT_DECL(VBOXSTRICTRC) APICHvSetTpr(PVMCPUCC pVCpu, uint8_t uTpr) +{ + Assert(pVCpu); + VMCPU_ASSERT_EMT(pVCpu); + return apicSetTprEx(pVCpu, uTpr, true /* fForceX2ApicBehaviour */); +} + + +/** + * Gets the Task Priority Register (TPR), Hyper-V interface. + * + * @returns The TPR value. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(uint8_t) APICHvGetTpr(PVMCPUCC pVCpu) +{ + Assert(pVCpu); + VMCPU_ASSERT_EMT(pVCpu); + + /* + * The APIC could be operating in xAPIC mode and thus we should not use the apicReadMsr() + * interface which validates the APIC mode and will throw a #GP(0) if not in x2APIC mode. + * We could use the apicReadRegister() MMIO interface, but why bother getting the PDMDEVINS + * pointer, so just directly read the APIC page. + */ + PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu); + return apicReadRaw32(pXApicPage, XAPIC_OFF_TPR); +} + + +/** + * Sets the Interrupt Command Register (ICR), Hyper-V interface. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uIcr The ICR value to set. + */ +VMM_INT_DECL(VBOXSTRICTRC) APICHvSetIcr(PVMCPUCC pVCpu, uint64_t uIcr) +{ + Assert(pVCpu); + VMCPU_ASSERT_EMT(pVCpu); + return apicSetIcr(pVCpu, uIcr, VINF_CPUM_R3_MSR_WRITE); +} + + +/** + * Gets the Interrupt Command Register (ICR), Hyper-V interface. + * + * @returns The ICR value. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(uint64_t) APICHvGetIcr(PVMCPUCC pVCpu) +{ + Assert(pVCpu); + VMCPU_ASSERT_EMT(pVCpu); + return apicGetIcrNoCheck(pVCpu); +} + + +/** + * Sets the End-Of-Interrupt (EOI) register, Hyper-V interface. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uEoi The EOI value. + */ +VMM_INT_DECL(VBOXSTRICTRC) APICHvSetEoi(PVMCPUCC pVCpu, uint32_t uEoi) +{ + Assert(pVCpu); + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + return apicSetEoi(pVCpu, uEoi, true /* fForceX2ApicBehaviour */); +} + + +/** + * Gets the APIC page pointers for the specified VCPU. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pHCPhys Where to store the host-context physical address. + * @param pR0Ptr Where to store the ring-0 address. + * @param pR3Ptr Where to store the ring-3 address (optional). + */ +VMM_INT_DECL(int) APICGetApicPageForCpu(PCVMCPUCC pVCpu, PRTHCPHYS pHCPhys, PRTR0PTR pR0Ptr, PRTR3PTR pR3Ptr) +{ + AssertReturn(pVCpu, VERR_INVALID_PARAMETER); + AssertReturn(pHCPhys, VERR_INVALID_PARAMETER); + AssertReturn(pR0Ptr, VERR_INVALID_PARAMETER); + + Assert(PDMHasApic(pVCpu->CTX_SUFF(pVM))); + + PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu); + *pHCPhys = pApicCpu->HCPhysApicPage; + *pR0Ptr = pApicCpu->pvApicPageR0; + if (pR3Ptr) + *pR3Ptr = pApicCpu->pvApicPageR3; + return VINF_SUCCESS; +} + +#ifndef IN_RING3 + +/** + * @callback_method_impl{PDMDEVREGR0,pfnConstruct} + */ +static DECLCALLBACK(int) apicRZConstruct(PPDMDEVINS pDevIns) +{ + PDMDEV_CHECK_VERSIONS_RETURN(pDevIns); + PAPICDEV pThis = PDMDEVINS_2_DATA(pDevIns, PAPICDEV); + PVMCC pVM = PDMDevHlpGetVM(pDevIns); + + pVM->apicr0.s.pDevInsR0 = pDevIns; + + int rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns)); + AssertRCReturn(rc, rc); + + rc = PDMDevHlpApicSetUpContext(pDevIns); + AssertRCReturn(rc, rc); + + rc = PDMDevHlpMmioSetUpContext(pDevIns, pThis->hMmio, apicWriteMmio, apicReadMmio, NULL /*pvUser*/); + AssertRCReturn(rc, rc); + + return VINF_SUCCESS; +} +#endif /* !IN_RING3 */ + +/** + * APIC device registration structure. + */ +const PDMDEVREG g_DeviceAPIC = +{ + /* .u32Version = */ PDM_DEVREG_VERSION, + /* .uReserved0 = */ 0, + /* .szName = */ "apic", + /* .fFlags = */ PDM_DEVREG_FLAGS_DEFAULT_BITS | PDM_DEVREG_FLAGS_RZ | PDM_DEVREG_FLAGS_NEW_STYLE + | PDM_DEVREG_FLAGS_REQUIRE_R0 | PDM_DEVREG_FLAGS_REQUIRE_RC, + /* .fClass = */ PDM_DEVREG_CLASS_PIC, + /* .cMaxInstances = */ 1, + /* .uSharedVersion = */ 42, + /* .cbInstanceShared = */ sizeof(APICDEV), + /* .cbInstanceCC = */ 0, + /* .cbInstanceRC = */ 0, + /* .cMaxPciDevices = */ 0, + /* .cMaxMsixVectors = */ 0, + /* .pszDescription = */ "Advanced Programmable Interrupt Controller", +#if defined(IN_RING3) + /* .szRCMod = */ "VMMRC.rc", + /* .szR0Mod = */ "VMMR0.r0", + /* .pfnConstruct = */ apicR3Construct, + /* .pfnDestruct = */ apicR3Destruct, + /* .pfnRelocate = */ apicR3Relocate, + /* .pfnMemSetup = */ NULL, + /* .pfnPowerOn = */ NULL, + /* .pfnReset = */ apicR3Reset, + /* .pfnSuspend = */ NULL, + /* .pfnResume = */ NULL, + /* .pfnAttach = */ NULL, + /* .pfnDetach = */ NULL, + /* .pfnQueryInterface = */ NULL, + /* .pfnInitComplete = */ apicR3InitComplete, + /* .pfnPowerOff = */ NULL, + /* .pfnSoftReset = */ NULL, + /* .pfnReserved0 = */ NULL, + /* .pfnReserved1 = */ NULL, + /* .pfnReserved2 = */ NULL, + /* .pfnReserved3 = */ NULL, + /* .pfnReserved4 = */ NULL, + /* .pfnReserved5 = */ NULL, + /* .pfnReserved6 = */ NULL, + /* .pfnReserved7 = */ NULL, +#elif defined(IN_RING0) + /* .pfnEarlyConstruct = */ NULL, + /* .pfnConstruct = */ apicRZConstruct, + /* .pfnDestruct = */ NULL, + /* .pfnFinalDestruct = */ NULL, + /* .pfnRequest = */ NULL, + /* .pfnReserved0 = */ NULL, + /* .pfnReserved1 = */ NULL, + /* .pfnReserved2 = */ NULL, + /* .pfnReserved3 = */ NULL, + /* .pfnReserved4 = */ NULL, + /* .pfnReserved5 = */ NULL, + /* .pfnReserved6 = */ NULL, + /* .pfnReserved7 = */ NULL, +#elif defined(IN_RC) + /* .pfnConstruct = */ apicRZConstruct, + /* .pfnReserved0 = */ NULL, + /* .pfnReserved1 = */ NULL, + /* .pfnReserved2 = */ NULL, + /* .pfnReserved3 = */ NULL, + /* .pfnReserved4 = */ NULL, + /* .pfnReserved5 = */ NULL, + /* .pfnReserved6 = */ NULL, + /* .pfnReserved7 = */ NULL, +#else +# error "Not in IN_RING3, IN_RING0 or IN_RC!" +#endif + /* .u32VersionEnd = */ PDM_DEVREG_VERSION +}; + diff --git a/src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp b/src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp new file mode 100644 index 00000000..827236a5 --- /dev/null +++ b/src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp @@ -0,0 +1,111 @@ +/* $Id: AllPdbTypeHack.cpp $ */ +/** @file + * Debug info hack for the VM and VMCPU structures. + */ + +/* + * Copyright (C) 2016-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include +#include "../include/PDMInternal.h" +#include +#include "../include/CFGMInternal.h" +#include "../include/CPUMInternal.h" +#include "../include/MMInternal.h" +#include "../include/PGMInternal.h" +#include "../include/SELMInternal.h" +#include "../include/TRPMInternal.h" +#include "../include/TMInternal.h" +#include "../include/IOMInternal.h" +#ifdef IN_RING3 +# include "../include/SSMInternal.h" +#endif +#include "../include/HMInternal.h" +#include "../include/VMMInternal.h" +#include "../include/DBGFInternal.h" +#include "../include/GIMInternal.h" +#include "../include/APICInternal.h" +#include "../include/STAMInternal.h" +#include "../include/VMInternal.h" +#include "../include/EMInternal.h" +#include "../include/IEMInternal.h" +#include "../include/NEMInternal.h" +#include "../VMMR0/GMMR0Internal.h" +#include "../VMMR0/GVMMR0Internal.h" +#include +#ifdef IN_RING3 +# include +#endif +#include + + +extern "C" { + +/* Global pointer variables as an alternative to the parameter list. Just to ensure the precense of the types. */ +PVM g_PdbTypeHack1 = NULL; +PVMCPU g_PdbTypeHack2 = NULL; +PPDMCRITSECT g_PdbTypeHack3 = NULL; +PPDMCRITSECTRW g_PdbTypeHack4 = NULL; +PPDMDEVINS g_PdbTypeHack5 = NULL; +PPDMDRVINS g_PdbTypeHack6 = NULL; +PPDMUSBINS g_PdbTypeHack7 = NULL; +PCVMCPU g_PdbTypeHack8 = NULL; +CTX_SUFF(PVM) g_PdbTypeHack9 = NULL; +CTX_SUFF(PVMCPU) g_PdbTypeHack10 = NULL; + +DECLEXPORT(uint32_t) PdbTypeHack(PVM pVM, PVMCPU pVCpu, PPDMCRITSECT pCs1, PPDMCRITSECTRW pCs2); +} + +DECLEXPORT(uint32_t) PdbTypeHack(PVM pVM, PVMCPU pVCpu, PPDMCRITSECT pCs1, PPDMCRITSECTRW pCs2) +{ + /* Just some dummy operations accessing each type. Probably not necessary, but + helps making sure we've included all we need to get at the internal stuff.. */ + return pVM->fGlobalForcedActions + | (pVM == g_PdbTypeHack1) + | (pVM == g_PdbTypeHack9) + | pVCpu->fLocalForcedActions + | (pVCpu == g_PdbTypeHack2) + | (pVCpu == g_PdbTypeHack8) + | (pVCpu == g_PdbTypeHack10) + | pCs1->s.Core.fFlags + | (pCs1 == g_PdbTypeHack3) + | pCs2->s.Core.fFlags + | (pCs2 == g_PdbTypeHack4) + | g_PdbTypeHack5->Internal.s.idxR0Device + | (g_PdbTypeHack5 != NULL) + | (uint32_t)g_PdbTypeHack6->Internal.s.fDetaching + | (g_PdbTypeHack6 != NULL) + | (uint32_t)g_PdbTypeHack7->Internal.s.fVMSuspended + | (g_PdbTypeHack7 != NULL); +} + diff --git a/src/VBox/VMM/VMMAll/CPUMAllCpuId.cpp b/src/VBox/VMM/VMMAll/CPUMAllCpuId.cpp new file mode 100644 index 00000000..48a309b4 --- /dev/null +++ b/src/VBox/VMM/VMMAll/CPUMAllCpuId.cpp @@ -0,0 +1,1599 @@ +/* $Id: CPUMAllCpuId.cpp $ */ +/** @file + * CPUM - CPU ID part, common bits. + */ + +/* + * Copyright (C) 2013-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_CPUM +#include +#include +#include +#include "CPUMInternal.h" +#include +#include + +#include +#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64) +# include +#endif +#include +#include +#include +#include + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +/** + * The intel pentium family. + */ +static const CPUMMICROARCH g_aenmIntelFamily06[] = +{ + /* [ 0(0x00)] = */ kCpumMicroarch_Intel_P6, /* Pentium Pro A-step (says sandpile.org). */ + /* [ 1(0x01)] = */ kCpumMicroarch_Intel_P6, /* Pentium Pro */ + /* [ 2(0x02)] = */ kCpumMicroarch_Intel_Unknown, + /* [ 3(0x03)] = */ kCpumMicroarch_Intel_P6_II, /* PII Klamath */ + /* [ 4(0x04)] = */ kCpumMicroarch_Intel_Unknown, + /* [ 5(0x05)] = */ kCpumMicroarch_Intel_P6_II, /* PII Deschutes */ + /* [ 6(0x06)] = */ kCpumMicroarch_Intel_P6_II, /* Celeron Mendocino. */ + /* [ 7(0x07)] = */ kCpumMicroarch_Intel_P6_III, /* PIII Katmai. */ + /* [ 8(0x08)] = */ kCpumMicroarch_Intel_P6_III, /* PIII Coppermine (includes Celeron). */ + /* [ 9(0x09)] = */ kCpumMicroarch_Intel_P6_M_Banias, /* Pentium/Celeron M Banias. */ + /* [10(0x0a)] = */ kCpumMicroarch_Intel_P6_III, /* PIII Xeon */ + /* [11(0x0b)] = */ kCpumMicroarch_Intel_P6_III, /* PIII Tualatin (includes Celeron). */ + /* [12(0x0c)] = */ kCpumMicroarch_Intel_Unknown, + /* [13(0x0d)] = */ kCpumMicroarch_Intel_P6_M_Dothan, /* Pentium/Celeron M Dothan. */ + /* [14(0x0e)] = */ kCpumMicroarch_Intel_Core_Yonah, /* Core Yonah (Enhanced Pentium M). */ + /* [15(0x0f)] = */ kCpumMicroarch_Intel_Core2_Merom, /* Merom */ + /* [16(0x10)] = */ kCpumMicroarch_Intel_Unknown, + /* [17(0x11)] = */ kCpumMicroarch_Intel_Unknown, + /* [18(0x12)] = */ kCpumMicroarch_Intel_Unknown, + /* [19(0x13)] = */ kCpumMicroarch_Intel_Unknown, + /* [20(0x14)] = */ kCpumMicroarch_Intel_Unknown, + /* [21(0x15)] = */ kCpumMicroarch_Intel_P6_M_Dothan, /* Tolapai - System-on-a-chip. */ + /* [22(0x16)] = */ kCpumMicroarch_Intel_Core2_Merom, + /* [23(0x17)] = */ kCpumMicroarch_Intel_Core2_Penryn, + /* [24(0x18)] = */ kCpumMicroarch_Intel_Unknown, + /* [25(0x19)] = */ kCpumMicroarch_Intel_Unknown, + /* [26(0x1a)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Nehalem-EP */ + /* [27(0x1b)] = */ kCpumMicroarch_Intel_Unknown, + /* [28(0x1c)] = */ kCpumMicroarch_Intel_Atom_Bonnell, /* Diamonville, Pineview, */ + /* [29(0x1d)] = */ kCpumMicroarch_Intel_Core2_Penryn, + /* [30(0x1e)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Clarksfield, Lynnfield, Jasper Forest. */ + /* [31(0x1f)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Only listed by sandpile.org. 2 cores ABD/HVD, whatever that means. */ + /* [32(0x20)] = */ kCpumMicroarch_Intel_Unknown, + /* [33(0x21)] = */ kCpumMicroarch_Intel_Unknown, + /* [34(0x22)] = */ kCpumMicroarch_Intel_Unknown, + /* [35(0x23)] = */ kCpumMicroarch_Intel_Unknown, + /* [36(0x24)] = */ kCpumMicroarch_Intel_Unknown, + /* [37(0x25)] = */ kCpumMicroarch_Intel_Core7_Westmere, /* Arrandale, Clarksdale. */ + /* [38(0x26)] = */ kCpumMicroarch_Intel_Atom_Lincroft, + /* [39(0x27)] = */ kCpumMicroarch_Intel_Atom_Saltwell, + /* [40(0x28)] = */ kCpumMicroarch_Intel_Unknown, + /* [41(0x29)] = */ kCpumMicroarch_Intel_Unknown, + /* [42(0x2a)] = */ kCpumMicroarch_Intel_Core7_SandyBridge, + /* [43(0x2b)] = */ kCpumMicroarch_Intel_Unknown, + /* [44(0x2c)] = */ kCpumMicroarch_Intel_Core7_Westmere, /* Gulftown, Westmere-EP. */ + /* [45(0x2d)] = */ kCpumMicroarch_Intel_Core7_SandyBridge, /* SandyBridge-E, SandyBridge-EN, SandyBridge-EP. */ + /* [46(0x2e)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Beckton (Xeon). */ + /* [47(0x2f)] = */ kCpumMicroarch_Intel_Core7_Westmere, /* Westmere-EX. */ + /* [48(0x30)] = */ kCpumMicroarch_Intel_Unknown, + /* [49(0x31)] = */ kCpumMicroarch_Intel_Unknown, + /* [50(0x32)] = */ kCpumMicroarch_Intel_Unknown, + /* [51(0x33)] = */ kCpumMicroarch_Intel_Unknown, + /* [52(0x34)] = */ kCpumMicroarch_Intel_Unknown, + /* [53(0x35)] = */ kCpumMicroarch_Intel_Atom_Saltwell, /* ?? */ + /* [54(0x36)] = */ kCpumMicroarch_Intel_Atom_Saltwell, /* Cedarview, ++ */ + /* [55(0x37)] = */ kCpumMicroarch_Intel_Atom_Silvermont, + /* [56(0x38)] = */ kCpumMicroarch_Intel_Unknown, + /* [57(0x39)] = */ kCpumMicroarch_Intel_Unknown, + /* [58(0x3a)] = */ kCpumMicroarch_Intel_Core7_IvyBridge, + /* [59(0x3b)] = */ kCpumMicroarch_Intel_Unknown, + /* [60(0x3c)] = */ kCpumMicroarch_Intel_Core7_Haswell, + /* [61(0x3d)] = */ kCpumMicroarch_Intel_Core7_Broadwell, + /* [62(0x3e)] = */ kCpumMicroarch_Intel_Core7_IvyBridge, + /* [63(0x3f)] = */ kCpumMicroarch_Intel_Core7_Haswell, + /* [64(0x40)] = */ kCpumMicroarch_Intel_Unknown, + /* [65(0x41)] = */ kCpumMicroarch_Intel_Unknown, + /* [66(0x42)] = */ kCpumMicroarch_Intel_Unknown, + /* [67(0x43)] = */ kCpumMicroarch_Intel_Unknown, + /* [68(0x44)] = */ kCpumMicroarch_Intel_Unknown, + /* [69(0x45)] = */ kCpumMicroarch_Intel_Core7_Haswell, + /* [70(0x46)] = */ kCpumMicroarch_Intel_Core7_Haswell, + /* [71(0x47)] = */ kCpumMicroarch_Intel_Core7_Broadwell, /* i7-5775C */ + /* [72(0x48)] = */ kCpumMicroarch_Intel_Unknown, + /* [73(0x49)] = */ kCpumMicroarch_Intel_Unknown, + /* [74(0x4a)] = */ kCpumMicroarch_Intel_Atom_Silvermont, + /* [75(0x4b)] = */ kCpumMicroarch_Intel_Unknown, + /* [76(0x4c)] = */ kCpumMicroarch_Intel_Atom_Airmount, + /* [77(0x4d)] = */ kCpumMicroarch_Intel_Atom_Silvermont, + /* [78(0x4e)] = */ kCpumMicroarch_Intel_Core7_Skylake, + /* [79(0x4f)] = */ kCpumMicroarch_Intel_Core7_Broadwell, /* Broadwell-E */ + /* [80(0x50)] = */ kCpumMicroarch_Intel_Unknown, + /* [81(0x51)] = */ kCpumMicroarch_Intel_Unknown, + /* [82(0x52)] = */ kCpumMicroarch_Intel_Unknown, + /* [83(0x53)] = */ kCpumMicroarch_Intel_Unknown, + /* [84(0x54)] = */ kCpumMicroarch_Intel_Unknown, + /* [85(0x55)] = */ kCpumMicroarch_Intel_Core7_Skylake, /* server cpu; skylake <= 4, cascade lake > 5 */ + /* [86(0x56)] = */ kCpumMicroarch_Intel_Core7_Broadwell, /* Xeon D-1540, Broadwell-DE */ + /* [87(0x57)] = */ kCpumMicroarch_Intel_Phi_KnightsLanding, + /* [88(0x58)] = */ kCpumMicroarch_Intel_Unknown, + /* [89(0x59)] = */ kCpumMicroarch_Intel_Unknown, + /* [90(0x5a)] = */ kCpumMicroarch_Intel_Atom_Silvermont, /* Moorefield */ + /* [91(0x5b)] = */ kCpumMicroarch_Intel_Unknown, + /* [92(0x5c)] = */ kCpumMicroarch_Intel_Atom_Goldmont, /* Apollo Lake */ + /* [93(0x5d)] = */ kCpumMicroarch_Intel_Atom_Silvermont, /* x3-C3230 */ + /* [94(0x5e)] = */ kCpumMicroarch_Intel_Core7_Skylake, /* i7-6700K */ + /* [95(0x5f)] = */ kCpumMicroarch_Intel_Atom_Goldmont, /* Denverton */ + /* [96(0x60)] = */ kCpumMicroarch_Intel_Unknown, + /* [97(0x61)] = */ kCpumMicroarch_Intel_Unknown, + /* [98(0x62)] = */ kCpumMicroarch_Intel_Unknown, + /* [99(0x63)] = */ kCpumMicroarch_Intel_Unknown, + /*[100(0x64)] = */ kCpumMicroarch_Intel_Unknown, + /*[101(0x65)] = */ kCpumMicroarch_Intel_Atom_Silvermont, /* SoFIA */ + /*[102(0x66)] = */ kCpumMicroarch_Intel_Core7_CannonLake, /* unconfirmed */ + /*[103(0x67)] = */ kCpumMicroarch_Intel_Unknown, + /*[104(0x68)] = */ kCpumMicroarch_Intel_Unknown, + /*[105(0x69)] = */ kCpumMicroarch_Intel_Unknown, + /*[106(0x6a)] = */ kCpumMicroarch_Intel_Core7_IceLake, /* unconfirmed server */ + /*[107(0x6b)] = */ kCpumMicroarch_Intel_Unknown, + /*[108(0x6c)] = */ kCpumMicroarch_Intel_Core7_IceLake, /* unconfirmed server */ + /*[109(0x6d)] = */ kCpumMicroarch_Intel_Unknown, + /*[110(0x6e)] = */ kCpumMicroarch_Intel_Atom_Airmount, /* or silvermount? */ + /*[111(0x6f)] = */ kCpumMicroarch_Intel_Unknown, + /*[112(0x70)] = */ kCpumMicroarch_Intel_Unknown, + /*[113(0x71)] = */ kCpumMicroarch_Intel_Unknown, + /*[114(0x72)] = */ kCpumMicroarch_Intel_Unknown, + /*[115(0x73)] = */ kCpumMicroarch_Intel_Unknown, + /*[116(0x74)] = */ kCpumMicroarch_Intel_Unknown, + /*[117(0x75)] = */ kCpumMicroarch_Intel_Atom_Airmount, /* or silvermount? */ + /*[118(0x76)] = */ kCpumMicroarch_Intel_Unknown, + /*[119(0x77)] = */ kCpumMicroarch_Intel_Unknown, + /*[120(0x78)] = */ kCpumMicroarch_Intel_Unknown, + /*[121(0x79)] = */ kCpumMicroarch_Intel_Unknown, + /*[122(0x7a)] = */ kCpumMicroarch_Intel_Atom_GoldmontPlus, + /*[123(0x7b)] = */ kCpumMicroarch_Intel_Unknown, + /*[124(0x7c)] = */ kCpumMicroarch_Intel_Unknown, + /*[125(0x7d)] = */ kCpumMicroarch_Intel_Core7_IceLake, /* unconfirmed */ + /*[126(0x7e)] = */ kCpumMicroarch_Intel_Core7_IceLake, /* unconfirmed */ + /*[127(0x7f)] = */ kCpumMicroarch_Intel_Unknown, + /*[128(0x80)] = */ kCpumMicroarch_Intel_Unknown, + /*[129(0x81)] = */ kCpumMicroarch_Intel_Unknown, + /*[130(0x82)] = */ kCpumMicroarch_Intel_Unknown, + /*[131(0x83)] = */ kCpumMicroarch_Intel_Unknown, + /*[132(0x84)] = */ kCpumMicroarch_Intel_Unknown, + /*[133(0x85)] = */ kCpumMicroarch_Intel_Phi_KnightsMill, + /*[134(0x86)] = */ kCpumMicroarch_Intel_Unknown, + /*[135(0x87)] = */ kCpumMicroarch_Intel_Unknown, + /*[136(0x88)] = */ kCpumMicroarch_Intel_Unknown, + /*[137(0x89)] = */ kCpumMicroarch_Intel_Unknown, + /*[138(0x8a)] = */ kCpumMicroarch_Intel_Unknown, + /*[139(0x8b)] = */ kCpumMicroarch_Intel_Unknown, + /*[140(0x8c)] = */ kCpumMicroarch_Intel_Core7_TigerLake, /* 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz (bird) */ + /*[141(0x8d)] = */ kCpumMicroarch_Intel_Core7_TigerLake, /* unconfirmed */ + /*[142(0x8e)] = */ kCpumMicroarch_Intel_Core7_KabyLake, /* Stepping >= 0xB is Whiskey Lake, 0xA is CoffeeLake. */ + /*[143(0x8f)] = */ kCpumMicroarch_Intel_Core7_SapphireRapids, + /*[144(0x90)] = */ kCpumMicroarch_Intel_Unknown, + /*[145(0x91)] = */ kCpumMicroarch_Intel_Unknown, + /*[146(0x92)] = */ kCpumMicroarch_Intel_Unknown, + /*[147(0x93)] = */ kCpumMicroarch_Intel_Unknown, + /*[148(0x94)] = */ kCpumMicroarch_Intel_Unknown, + /*[149(0x95)] = */ kCpumMicroarch_Intel_Unknown, + /*[150(0x96)] = */ kCpumMicroarch_Intel_Unknown, + /*[151(0x97)] = */ kCpumMicroarch_Intel_Core7_AlderLake, /* unconfirmed, unreleased */ + /*[152(0x98)] = */ kCpumMicroarch_Intel_Unknown, + /*[153(0x99)] = */ kCpumMicroarch_Intel_Unknown, + /*[154(0x9a)] = */ kCpumMicroarch_Intel_Core7_AlderLake, /* unconfirmed, unreleased */ + /*[155(0x9b)] = */ kCpumMicroarch_Intel_Unknown, + /*[156(0x9c)] = */ kCpumMicroarch_Intel_Unknown, + /*[157(0x9d)] = */ kCpumMicroarch_Intel_Unknown, + /*[158(0x9e)] = */ kCpumMicroarch_Intel_Core7_KabyLake, /* Stepping >= 0xB is Whiskey Lake, 0xA is CoffeeLake. */ + /*[159(0x9f)] = */ kCpumMicroarch_Intel_Unknown, + /*[160(0xa0)] = */ kCpumMicroarch_Intel_Unknown, + /*[161(0xa1)] = */ kCpumMicroarch_Intel_Unknown, + /*[162(0xa2)] = */ kCpumMicroarch_Intel_Unknown, + /*[163(0xa3)] = */ kCpumMicroarch_Intel_Unknown, + /*[164(0xa4)] = */ kCpumMicroarch_Intel_Unknown, + /*[165(0xa5)] = */ kCpumMicroarch_Intel_Core7_CometLake, /* unconfirmed */ + /*[166(0xa6)] = */ kCpumMicroarch_Intel_Unknown, + /*[167(0xa7)] = */ kCpumMicroarch_Intel_Core7_CypressCove, /* 14nm backport, unconfirmed */ +}; +AssertCompile(RT_ELEMENTS(g_aenmIntelFamily06) == 0xa7+1); + + +/** + * Figures out the (sub-)micro architecture given a bit of CPUID info. + * + * @returns Micro architecture. + * @param enmVendor The CPU vendor. + * @param bFamily The CPU family. + * @param bModel The CPU model. + * @param bStepping The CPU stepping. + */ +VMMDECL(CPUMMICROARCH) CPUMCpuIdDetermineX86MicroarchEx(CPUMCPUVENDOR enmVendor, uint8_t bFamily, + uint8_t bModel, uint8_t bStepping) +{ + if (enmVendor == CPUMCPUVENDOR_AMD) + { + switch (bFamily) + { + case 0x02: return kCpumMicroarch_AMD_Am286; /* Not really kosher... */ + case 0x03: return kCpumMicroarch_AMD_Am386; + case 0x23: return kCpumMicroarch_AMD_Am386; /* SX*/ + case 0x04: return bModel < 14 ? kCpumMicroarch_AMD_Am486 : kCpumMicroarch_AMD_Am486Enh; + case 0x05: return bModel < 6 ? kCpumMicroarch_AMD_K5 : kCpumMicroarch_AMD_K6; /* Genode LX is 0x0a, lump it with K6. */ + case 0x06: + switch (bModel) + { + case 0: return kCpumMicroarch_AMD_K7_Palomino; + case 1: return kCpumMicroarch_AMD_K7_Palomino; + case 2: return kCpumMicroarch_AMD_K7_Palomino; + case 3: return kCpumMicroarch_AMD_K7_Spitfire; + case 4: return kCpumMicroarch_AMD_K7_Thunderbird; + case 6: return kCpumMicroarch_AMD_K7_Palomino; + case 7: return kCpumMicroarch_AMD_K7_Morgan; + case 8: return kCpumMicroarch_AMD_K7_Thoroughbred; + case 10: return kCpumMicroarch_AMD_K7_Barton; /* Thorton too. */ + } + return kCpumMicroarch_AMD_K7_Unknown; + case 0x0f: + /* + * This family is a friggin mess. Trying my best to make some + * sense out of it. Too much happened in the 0x0f family to + * lump it all together as K8 (130nm->90nm->65nm, AMD-V, ++). + * + * Emperical CPUID.01h.EAX evidence from revision guides, wikipedia, + * cpu-world.com, and other places: + * - 130nm: + * - ClawHammer: F7A/SH-CG, F5A/-CG, F4A/-CG, F50/-B0, F48/-C0, F58/-C0, + * - SledgeHammer: F50/SH-B0, F48/-C0, F58/-C0, F4A/-CG, F5A/-CG, F7A/-CG, F51/-B3 + * - Newcastle: FC0/DH-CG (erratum #180: FE0/DH-CG), FF0/DH-CG + * - Dublin: FC0/-CG, FF0/-CG, F82/CH-CG, F4A/-CG, F48/SH-C0, + * - Odessa: FC0/DH-CG (erratum #180: FE0/DH-CG) + * - Paris: FF0/DH-CG, FC0/DH-CG (erratum #180: FE0/DH-CG), + * - 90nm: + * - Winchester: 10FF0/DH-D0, 20FF0/DH-E3. + * - Oakville: 10FC0/DH-D0. + * - Georgetown: 10FC0/DH-D0. + * - Sonora: 10FC0/DH-D0. + * - Venus: 20F71/SH-E4 + * - Troy: 20F51/SH-E4 + * - Athens: 20F51/SH-E4 + * - San Diego: 20F71/SH-E4. + * - Lancaster: 20F42/SH-E5 + * - Newark: 20F42/SH-E5. + * - Albany: 20FC2/DH-E6. + * - Roma: 20FC2/DH-E6. + * - Venice: 20FF0/DH-E3, 20FC2/DH-E6, 20FF2/DH-E6. + * - Palermo: 10FC0/DH-D0, 20FF0/DH-E3, 20FC0/DH-E3, 20FC2/DH-E6, 20FF2/DH-E6 + * - 90nm introducing Dual core: + * - Denmark: 20F30/JH-E1, 20F32/JH-E6 + * - Italy: 20F10/JH-E1, 20F12/JH-E6 + * - Egypt: 20F10/JH-E1, 20F12/JH-E6 + * - Toledo: 20F32/JH-E6, 30F72/DH-E6 (single code variant). + * - Manchester: 20FB1/BH-E4, 30FF2/BH-E4. + * - 90nm 2nd gen opteron ++, AMD-V introduced (might be missing in some cheaper models): + * - Santa Ana: 40F32/JH-F2, /-F3 + * - Santa Rosa: 40F12/JH-F2, 40F13/JH-F3 + * - Windsor: 40F32/JH-F2, 40F33/JH-F3, C0F13/JH-F3, 40FB2/BH-F2, ??20FB1/BH-E4??. + * - Manila: 50FF2/DH-F2, 40FF2/DH-F2 + * - Orleans: 40FF2/DH-F2, 50FF2/DH-F2, 50FF3/DH-F3. + * - Keene: 40FC2/DH-F2. + * - Richmond: 40FC2/DH-F2 + * - Taylor: 40F82/BH-F2 + * - Trinidad: 40F82/BH-F2 + * + * - 65nm: + * - Brisbane: 60FB1/BH-G1, 60FB2/BH-G2. + * - Tyler: 60F81/BH-G1, 60F82/BH-G2. + * - Sparta: 70FF1/DH-G1, 70FF2/DH-G2. + * - Lima: 70FF1/DH-G1, 70FF2/DH-G2. + * - Sherman: /-G1, 70FC2/DH-G2. + * - Huron: 70FF2/DH-G2. + */ + if (bModel < 0x10) + return kCpumMicroarch_AMD_K8_130nm; + if (bModel >= 0x60 && bModel < 0x80) + return kCpumMicroarch_AMD_K8_65nm; + if (bModel >= 0x40) + return kCpumMicroarch_AMD_K8_90nm_AMDV; + switch (bModel) + { + case 0x21: + case 0x23: + case 0x2b: + case 0x2f: + case 0x37: + case 0x3f: + return kCpumMicroarch_AMD_K8_90nm_DualCore; + } + return kCpumMicroarch_AMD_K8_90nm; + case 0x10: + return kCpumMicroarch_AMD_K10; + case 0x11: + return kCpumMicroarch_AMD_K10_Lion; + case 0x12: + return kCpumMicroarch_AMD_K10_Llano; + case 0x14: + return kCpumMicroarch_AMD_Bobcat; + case 0x15: + switch (bModel) + { + case 0x00: return kCpumMicroarch_AMD_15h_Bulldozer; /* Any? prerelease? */ + case 0x01: return kCpumMicroarch_AMD_15h_Bulldozer; /* Opteron 4200, FX-81xx. */ + case 0x02: return kCpumMicroarch_AMD_15h_Piledriver; /* Opteron 4300, FX-83xx. */ + case 0x10: return kCpumMicroarch_AMD_15h_Piledriver; /* A10-5800K for e.g. */ + case 0x11: /* ?? */ + case 0x12: /* ?? */ + case 0x13: return kCpumMicroarch_AMD_15h_Piledriver; /* A10-6800K for e.g. */ + } + return kCpumMicroarch_AMD_15h_Unknown; + case 0x16: + return kCpumMicroarch_AMD_Jaguar; + case 0x17: + return kCpumMicroarch_AMD_Zen_Ryzen; + } + return kCpumMicroarch_AMD_Unknown; + } + + if (enmVendor == CPUMCPUVENDOR_INTEL) + { + switch (bFamily) + { + case 3: + return kCpumMicroarch_Intel_80386; + case 4: + return kCpumMicroarch_Intel_80486; + case 5: + return kCpumMicroarch_Intel_P5; + case 6: + if (bModel < RT_ELEMENTS(g_aenmIntelFamily06)) + { + CPUMMICROARCH enmMicroArch = g_aenmIntelFamily06[bModel]; + if (enmMicroArch == kCpumMicroarch_Intel_Core7_KabyLake) + { + if (bStepping >= 0xa && bStepping <= 0xc) + enmMicroArch = kCpumMicroarch_Intel_Core7_CoffeeLake; + else if (bStepping >= 0xc) + enmMicroArch = kCpumMicroarch_Intel_Core7_WhiskeyLake; + } + else if ( enmMicroArch == kCpumMicroarch_Intel_Core7_Skylake + && bModel == 0x55 + && bStepping >= 5) + enmMicroArch = kCpumMicroarch_Intel_Core7_CascadeLake; + return enmMicroArch; + } + return kCpumMicroarch_Intel_Atom_Unknown; + case 15: + switch (bModel) + { + case 0: return kCpumMicroarch_Intel_NB_Willamette; + case 1: return kCpumMicroarch_Intel_NB_Willamette; + case 2: return kCpumMicroarch_Intel_NB_Northwood; + case 3: return kCpumMicroarch_Intel_NB_Prescott; + case 4: return kCpumMicroarch_Intel_NB_Prescott2M; /* ?? */ + case 5: return kCpumMicroarch_Intel_NB_Unknown; /*??*/ + case 6: return kCpumMicroarch_Intel_NB_CedarMill; + case 7: return kCpumMicroarch_Intel_NB_Gallatin; + default: return kCpumMicroarch_Intel_NB_Unknown; + } + break; + /* The following are not kosher but kind of follow intuitively from 6, 5 & 4. */ + case 0: + return kCpumMicroarch_Intel_8086; + case 1: + return kCpumMicroarch_Intel_80186; + case 2: + return kCpumMicroarch_Intel_80286; + } + return kCpumMicroarch_Intel_Unknown; + } + + if (enmVendor == CPUMCPUVENDOR_VIA) + { + switch (bFamily) + { + case 5: + switch (bModel) + { + case 1: return kCpumMicroarch_Centaur_C6; + case 4: return kCpumMicroarch_Centaur_C6; + case 8: return kCpumMicroarch_Centaur_C2; + case 9: return kCpumMicroarch_Centaur_C3; + } + break; + + case 6: + switch (bModel) + { + case 5: return kCpumMicroarch_VIA_C3_M2; + case 6: return kCpumMicroarch_VIA_C3_C5A; + case 7: return bStepping < 8 ? kCpumMicroarch_VIA_C3_C5B : kCpumMicroarch_VIA_C3_C5C; + case 8: return kCpumMicroarch_VIA_C3_C5N; + case 9: return bStepping < 8 ? kCpumMicroarch_VIA_C3_C5XL : kCpumMicroarch_VIA_C3_C5P; + case 10: return kCpumMicroarch_VIA_C7_C5J; + case 15: return kCpumMicroarch_VIA_Isaiah; + } + break; + } + return kCpumMicroarch_VIA_Unknown; + } + + if (enmVendor == CPUMCPUVENDOR_SHANGHAI) + { + switch (bFamily) + { + case 6: + case 7: + return kCpumMicroarch_Shanghai_Wudaokou; + default: + break; + } + return kCpumMicroarch_Shanghai_Unknown; + } + + if (enmVendor == CPUMCPUVENDOR_CYRIX) + { + switch (bFamily) + { + case 4: + switch (bModel) + { + case 9: return kCpumMicroarch_Cyrix_5x86; + } + break; + + case 5: + switch (bModel) + { + case 2: return kCpumMicroarch_Cyrix_M1; + case 4: return kCpumMicroarch_Cyrix_MediaGX; + case 5: return kCpumMicroarch_Cyrix_MediaGXm; + } + break; + + case 6: + switch (bModel) + { + case 0: return kCpumMicroarch_Cyrix_M2; + } + break; + + } + return kCpumMicroarch_Cyrix_Unknown; + } + + if (enmVendor == CPUMCPUVENDOR_HYGON) + { + switch (bFamily) + { + case 0x18: + return kCpumMicroarch_Hygon_Dhyana; + default: + break; + } + return kCpumMicroarch_Hygon_Unknown; + } + + return kCpumMicroarch_Unknown; +} + + +/** + * Translates a microarchitecture enum value to the corresponding string + * constant. + * + * @returns Read-only string constant (omits "kCpumMicroarch_" prefix). Returns + * NULL if the value is invalid. + * + * @param enmMicroarch The enum value to convert. + */ +VMMDECL(const char *) CPUMMicroarchName(CPUMMICROARCH enmMicroarch) +{ + switch (enmMicroarch) + { +#define CASE_RET_STR(enmValue) case enmValue: return #enmValue + (sizeof("kCpumMicroarch_") - 1) + CASE_RET_STR(kCpumMicroarch_Intel_8086); + CASE_RET_STR(kCpumMicroarch_Intel_80186); + CASE_RET_STR(kCpumMicroarch_Intel_80286); + CASE_RET_STR(kCpumMicroarch_Intel_80386); + CASE_RET_STR(kCpumMicroarch_Intel_80486); + CASE_RET_STR(kCpumMicroarch_Intel_P5); + + CASE_RET_STR(kCpumMicroarch_Intel_P6); + CASE_RET_STR(kCpumMicroarch_Intel_P6_II); + CASE_RET_STR(kCpumMicroarch_Intel_P6_III); + + CASE_RET_STR(kCpumMicroarch_Intel_P6_M_Banias); + CASE_RET_STR(kCpumMicroarch_Intel_P6_M_Dothan); + CASE_RET_STR(kCpumMicroarch_Intel_Core_Yonah); + + CASE_RET_STR(kCpumMicroarch_Intel_Core2_Merom); + CASE_RET_STR(kCpumMicroarch_Intel_Core2_Penryn); + + CASE_RET_STR(kCpumMicroarch_Intel_Core7_Nehalem); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_Westmere); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_SandyBridge); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_IvyBridge); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_Haswell); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_Broadwell); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_Skylake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_KabyLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_CoffeeLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_WhiskeyLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_CascadeLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_CannonLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_CometLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_IceLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_RocketLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_TigerLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_AlderLake); + CASE_RET_STR(kCpumMicroarch_Intel_Core7_SapphireRapids); + + CASE_RET_STR(kCpumMicroarch_Intel_Atom_Bonnell); + CASE_RET_STR(kCpumMicroarch_Intel_Atom_Lincroft); + CASE_RET_STR(kCpumMicroarch_Intel_Atom_Saltwell); + CASE_RET_STR(kCpumMicroarch_Intel_Atom_Silvermont); + CASE_RET_STR(kCpumMicroarch_Intel_Atom_Airmount); + CASE_RET_STR(kCpumMicroarch_Intel_Atom_Goldmont); + CASE_RET_STR(kCpumMicroarch_Intel_Atom_GoldmontPlus); + CASE_RET_STR(kCpumMicroarch_Intel_Atom_Unknown); + + CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsFerry); + CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsCorner); + CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsLanding); + CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsHill); + CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsMill); + + CASE_RET_STR(kCpumMicroarch_Intel_NB_Willamette); + CASE_RET_STR(kCpumMicroarch_Intel_NB_Northwood); + CASE_RET_STR(kCpumMicroarch_Intel_NB_Prescott); + CASE_RET_STR(kCpumMicroarch_Intel_NB_Prescott2M); + CASE_RET_STR(kCpumMicroarch_Intel_NB_CedarMill); + CASE_RET_STR(kCpumMicroarch_Intel_NB_Gallatin); + CASE_RET_STR(kCpumMicroarch_Intel_NB_Unknown); + + CASE_RET_STR(kCpumMicroarch_Intel_Unknown); + + CASE_RET_STR(kCpumMicroarch_AMD_Am286); + CASE_RET_STR(kCpumMicroarch_AMD_Am386); + CASE_RET_STR(kCpumMicroarch_AMD_Am486); + CASE_RET_STR(kCpumMicroarch_AMD_Am486Enh); + CASE_RET_STR(kCpumMicroarch_AMD_K5); + CASE_RET_STR(kCpumMicroarch_AMD_K6); + + CASE_RET_STR(kCpumMicroarch_AMD_K7_Palomino); + CASE_RET_STR(kCpumMicroarch_AMD_K7_Spitfire); + CASE_RET_STR(kCpumMicroarch_AMD_K7_Thunderbird); + CASE_RET_STR(kCpumMicroarch_AMD_K7_Morgan); + CASE_RET_STR(kCpumMicroarch_AMD_K7_Thoroughbred); + CASE_RET_STR(kCpumMicroarch_AMD_K7_Barton); + CASE_RET_STR(kCpumMicroarch_AMD_K7_Unknown); + + CASE_RET_STR(kCpumMicroarch_AMD_K8_130nm); + CASE_RET_STR(kCpumMicroarch_AMD_K8_90nm); + CASE_RET_STR(kCpumMicroarch_AMD_K8_90nm_DualCore); + CASE_RET_STR(kCpumMicroarch_AMD_K8_90nm_AMDV); + CASE_RET_STR(kCpumMicroarch_AMD_K8_65nm); + + CASE_RET_STR(kCpumMicroarch_AMD_K10); + CASE_RET_STR(kCpumMicroarch_AMD_K10_Lion); + CASE_RET_STR(kCpumMicroarch_AMD_K10_Llano); + CASE_RET_STR(kCpumMicroarch_AMD_Bobcat); + CASE_RET_STR(kCpumMicroarch_AMD_Jaguar); + + CASE_RET_STR(kCpumMicroarch_AMD_15h_Bulldozer); + CASE_RET_STR(kCpumMicroarch_AMD_15h_Piledriver); + CASE_RET_STR(kCpumMicroarch_AMD_15h_Steamroller); + CASE_RET_STR(kCpumMicroarch_AMD_15h_Excavator); + CASE_RET_STR(kCpumMicroarch_AMD_15h_Unknown); + + CASE_RET_STR(kCpumMicroarch_AMD_16h_First); + + CASE_RET_STR(kCpumMicroarch_AMD_Zen_Ryzen); + + CASE_RET_STR(kCpumMicroarch_AMD_Unknown); + + CASE_RET_STR(kCpumMicroarch_Hygon_Dhyana); + CASE_RET_STR(kCpumMicroarch_Hygon_Unknown); + + CASE_RET_STR(kCpumMicroarch_Centaur_C6); + CASE_RET_STR(kCpumMicroarch_Centaur_C2); + CASE_RET_STR(kCpumMicroarch_Centaur_C3); + CASE_RET_STR(kCpumMicroarch_VIA_C3_M2); + CASE_RET_STR(kCpumMicroarch_VIA_C3_C5A); + CASE_RET_STR(kCpumMicroarch_VIA_C3_C5B); + CASE_RET_STR(kCpumMicroarch_VIA_C3_C5C); + CASE_RET_STR(kCpumMicroarch_VIA_C3_C5N); + CASE_RET_STR(kCpumMicroarch_VIA_C3_C5XL); + CASE_RET_STR(kCpumMicroarch_VIA_C3_C5P); + CASE_RET_STR(kCpumMicroarch_VIA_C7_C5J); + CASE_RET_STR(kCpumMicroarch_VIA_Isaiah); + CASE_RET_STR(kCpumMicroarch_VIA_Unknown); + + CASE_RET_STR(kCpumMicroarch_Shanghai_Wudaokou); + CASE_RET_STR(kCpumMicroarch_Shanghai_Unknown); + + CASE_RET_STR(kCpumMicroarch_Cyrix_5x86); + CASE_RET_STR(kCpumMicroarch_Cyrix_M1); + CASE_RET_STR(kCpumMicroarch_Cyrix_MediaGX); + CASE_RET_STR(kCpumMicroarch_Cyrix_MediaGXm); + CASE_RET_STR(kCpumMicroarch_Cyrix_M2); + CASE_RET_STR(kCpumMicroarch_Cyrix_Unknown); + + CASE_RET_STR(kCpumMicroarch_NEC_V20); + CASE_RET_STR(kCpumMicroarch_NEC_V30); + + CASE_RET_STR(kCpumMicroarch_Unknown); + +#undef CASE_RET_STR + case kCpumMicroarch_Invalid: + case kCpumMicroarch_Intel_End: + case kCpumMicroarch_Intel_Core2_End: + case kCpumMicroarch_Intel_Core7_End: + case kCpumMicroarch_Intel_Atom_End: + case kCpumMicroarch_Intel_P6_Core_Atom_End: + case kCpumMicroarch_Intel_Phi_End: + case kCpumMicroarch_Intel_NB_End: + case kCpumMicroarch_AMD_K7_End: + case kCpumMicroarch_AMD_K8_End: + case kCpumMicroarch_AMD_15h_End: + case kCpumMicroarch_AMD_16h_End: + case kCpumMicroarch_AMD_Zen_End: + case kCpumMicroarch_AMD_End: + case kCpumMicroarch_Hygon_End: + case kCpumMicroarch_VIA_End: + case kCpumMicroarch_Shanghai_End: + case kCpumMicroarch_Cyrix_End: + case kCpumMicroarch_NEC_End: + case kCpumMicroarch_32BitHack: + break; + /* no default! */ + } + + return NULL; +} + + +/** + * Gets a matching leaf in the CPUID leaf array. + * + * @returns Pointer to the matching leaf, or NULL if not found. + * @param paLeaves The CPUID leaves to search. This is sorted. + * @param cLeaves The number of leaves in the array. + * @param uLeaf The leaf to locate. + * @param uSubLeaf The subleaf to locate. Pass 0 if no sub-leaves. + */ +PCPUMCPUIDLEAF cpumCpuIdGetLeafInt(PCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf, uint32_t uSubLeaf) +{ + /* Lazy bird does linear lookup here since this is only used for the + occational CPUID overrides. */ + for (uint32_t i = 0; i < cLeaves; i++) + if ( paLeaves[i].uLeaf == uLeaf + && paLeaves[i].uSubLeaf == (uSubLeaf & paLeaves[i].fSubLeafMask)) + return &paLeaves[i]; + return NULL; +} + + +/** + * Ensures that the CPUID leaf array can hold one more leaf. + * + * @returns Pointer to the CPUID leaf array (*ppaLeaves) on success. NULL on + * failure. + * @param pVM The cross context VM structure. If NULL, use + * the process heap, otherwise the VM's hyper heap. + * @param ppaLeaves Pointer to the variable holding the array pointer + * (input/output). + * @param cLeaves The current array size. + * + * @remarks This function will automatically update the R0 and RC pointers when + * using the hyper heap, which means @a ppaLeaves and @a cLeaves must + * be the corresponding VM's CPUID arrays (which is asserted). + */ +PCPUMCPUIDLEAF cpumCpuIdEnsureSpace(PVM pVM, PCPUMCPUIDLEAF *ppaLeaves, uint32_t cLeaves) +{ + /* + * If pVM is not specified, we're on the regular heap and can waste a + * little space to speed things up. + */ + uint32_t cAllocated; + if (!pVM) + { + cAllocated = RT_ALIGN(cLeaves, 16); + if (cLeaves + 1 > cAllocated) + { + void *pvNew = RTMemRealloc(*ppaLeaves, (cAllocated + 16) * sizeof(**ppaLeaves)); + if (pvNew) + *ppaLeaves = (PCPUMCPUIDLEAF)pvNew; + else + { + RTMemFree(*ppaLeaves); + *ppaLeaves = NULL; + } + } + } + /* + * Otherwise, we're on the hyper heap and are probably just inserting + * one or two leaves and should conserve space. + */ + else + { +#ifdef IN_VBOX_CPU_REPORT + AssertReleaseFailed(); +#else +# ifdef IN_RING3 + Assert(ppaLeaves == &pVM->cpum.s.GuestInfo.paCpuIdLeavesR3); + Assert(*ppaLeaves == pVM->cpum.s.GuestInfo.aCpuIdLeaves); + Assert(cLeaves == pVM->cpum.s.GuestInfo.cCpuIdLeaves); + + if (cLeaves + 1 <= RT_ELEMENTS(pVM->cpum.s.GuestInfo.aCpuIdLeaves)) + { } + else +# endif + { + *ppaLeaves = NULL; + LogRel(("CPUM: cpumR3CpuIdEnsureSpace: Out of CPUID space!\n")); + } +#endif + } + return *ppaLeaves; +} + + +#ifdef VBOX_STRICT +/** + * Checks that we've updated the CPUID leaves array correctly. + * + * This is a no-op in non-strict builds. + * + * @param paLeaves The leaves array. + * @param cLeaves The number of leaves. + */ +void cpumCpuIdAssertOrder(PCPUMCPUIDLEAF paLeaves, uint32_t cLeaves) +{ + for (uint32_t i = 1; i < cLeaves; i++) + if (paLeaves[i].uLeaf != paLeaves[i - 1].uLeaf) + AssertMsg(paLeaves[i].uLeaf > paLeaves[i - 1].uLeaf, ("%#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i - 1].uLeaf)); + else + { + AssertMsg(paLeaves[i].uSubLeaf > paLeaves[i - 1].uSubLeaf, + ("%#x: %#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, paLeaves[i - 1].uSubLeaf)); + AssertMsg(paLeaves[i].fSubLeafMask == paLeaves[i - 1].fSubLeafMask, + ("%#x/%#x: %#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, paLeaves[i].fSubLeafMask, paLeaves[i - 1].fSubLeafMask)); + AssertMsg(paLeaves[i].fFlags == paLeaves[i - 1].fFlags, + ("%#x/%#x: %#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, paLeaves[i].fFlags, paLeaves[i - 1].fFlags)); + } +} +#endif + +#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64) + +/** + * Append a CPUID leaf or sub-leaf. + * + * ASSUMES linear insertion order, so we'll won't need to do any searching or + * replace anything. Use cpumR3CpuIdInsert() for those cases. + * + * @returns VINF_SUCCESS or VERR_NO_MEMORY. On error, *ppaLeaves is freed, so + * the caller need do no more work. + * @param ppaLeaves Pointer to the pointer to the array of sorted + * CPUID leaves and sub-leaves. + * @param pcLeaves Where we keep the leaf count for *ppaLeaves. + * @param uLeaf The leaf we're adding. + * @param uSubLeaf The sub-leaf number. + * @param fSubLeafMask The sub-leaf mask. + * @param uEax The EAX value. + * @param uEbx The EBX value. + * @param uEcx The ECX value. + * @param uEdx The EDX value. + * @param fFlags The flags. + */ +static int cpumCollectCpuIdInfoAddOne(PCPUMCPUIDLEAF *ppaLeaves, uint32_t *pcLeaves, + uint32_t uLeaf, uint32_t uSubLeaf, uint32_t fSubLeafMask, + uint32_t uEax, uint32_t uEbx, uint32_t uEcx, uint32_t uEdx, uint32_t fFlags) +{ + if (!cpumCpuIdEnsureSpace(NULL /* pVM */, ppaLeaves, *pcLeaves)) + return VERR_NO_MEMORY; + + PCPUMCPUIDLEAF pNew = &(*ppaLeaves)[*pcLeaves]; + Assert( *pcLeaves == 0 + || pNew[-1].uLeaf < uLeaf + || (pNew[-1].uLeaf == uLeaf && pNew[-1].uSubLeaf < uSubLeaf) ); + + pNew->uLeaf = uLeaf; + pNew->uSubLeaf = uSubLeaf; + pNew->fSubLeafMask = fSubLeafMask; + pNew->uEax = uEax; + pNew->uEbx = uEbx; + pNew->uEcx = uEcx; + pNew->uEdx = uEdx; + pNew->fFlags = fFlags; + + *pcLeaves += 1; + return VINF_SUCCESS; +} + + +/** + * Checks if ECX make a difference when reading a given CPUID leaf. + * + * @returns @c true if it does, @c false if it doesn't. + * @param uLeaf The leaf we're reading. + * @param pcSubLeaves Number of sub-leaves accessible via ECX. + * @param pfFinalEcxUnchanged Whether ECX is passed thru when going beyond the + * final sub-leaf (for leaf 0xb only). + */ +static bool cpumIsEcxRelevantForCpuIdLeaf(uint32_t uLeaf, uint32_t *pcSubLeaves, bool *pfFinalEcxUnchanged) +{ + *pfFinalEcxUnchanged = false; + + uint32_t auCur[4]; + uint32_t auPrev[4]; + ASMCpuIdExSlow(uLeaf, 0, 0, 0, &auPrev[0], &auPrev[1], &auPrev[2], &auPrev[3]); + + /* Look for sub-leaves. */ + uint32_t uSubLeaf = 1; + for (;;) + { + ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &auCur[0], &auCur[1], &auCur[2], &auCur[3]); + if (memcmp(auCur, auPrev, sizeof(auCur))) + break; + + /* Advance / give up. */ + uSubLeaf++; + if (uSubLeaf >= 64) + { + *pcSubLeaves = 1; + return false; + } + } + + /* Count sub-leaves. */ + uint32_t cMinLeaves = uLeaf == 0xd ? 64 : 0; + uint32_t cRepeats = 0; + uSubLeaf = 0; + for (;;) + { + ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &auCur[0], &auCur[1], &auCur[2], &auCur[3]); + + /* Figuring out when to stop isn't entirely straight forward as we need + to cover undocumented behavior up to a point and implementation shortcuts. */ + + /* 1. Look for more than 4 repeating value sets. */ + if ( auCur[0] == auPrev[0] + && auCur[1] == auPrev[1] + && ( auCur[2] == auPrev[2] + || ( auCur[2] == uSubLeaf + && auPrev[2] == uSubLeaf - 1) ) + && auCur[3] == auPrev[3]) + { + if ( uLeaf != 0xd + || uSubLeaf >= 64 + || ( auCur[0] == 0 + && auCur[1] == 0 + && auCur[2] == 0 + && auCur[3] == 0 + && auPrev[2] == 0) ) + cRepeats++; + if (cRepeats > 4 && uSubLeaf >= cMinLeaves) + break; + } + else + cRepeats = 0; + + /* 2. Look for zero values. */ + if ( auCur[0] == 0 + && auCur[1] == 0 + && (auCur[2] == 0 || auCur[2] == uSubLeaf) + && (auCur[3] == 0 || uLeaf == 0xb /* edx is fixed */) + && uSubLeaf >= cMinLeaves) + { + cRepeats = 0; + break; + } + + /* 3. Leaf 0xb level type 0 check. */ + if ( uLeaf == 0xb + && (auCur[2] & 0xff00) == 0 + && (auPrev[2] & 0xff00) == 0) + { + cRepeats = 0; + break; + } + + /* 99. Give up. */ + if (uSubLeaf >= 128) + { +# ifndef IN_VBOX_CPU_REPORT + /* Ok, limit it according to the documentation if possible just to + avoid annoying users with these detection issues. */ + uint32_t cDocLimit = UINT32_MAX; + if (uLeaf == 0x4) + cDocLimit = 4; + else if (uLeaf == 0x7) + cDocLimit = 1; + else if (uLeaf == 0xd) + cDocLimit = 63; + else if (uLeaf == 0xf) + cDocLimit = 2; + if (cDocLimit != UINT32_MAX) + { + *pfFinalEcxUnchanged = auCur[2] == uSubLeaf && uLeaf == 0xb; + *pcSubLeaves = cDocLimit + 3; + return true; + } +# endif + *pcSubLeaves = UINT32_MAX; + return true; + } + + /* Advance. */ + uSubLeaf++; + memcpy(auPrev, auCur, sizeof(auCur)); + } + + /* Standard exit. */ + *pfFinalEcxUnchanged = auCur[2] == uSubLeaf && uLeaf == 0xb; + *pcSubLeaves = uSubLeaf + 1 - cRepeats; + if (*pcSubLeaves == 0) + *pcSubLeaves = 1; + return true; +} + + +/** + * Collects CPUID leaves and sub-leaves, returning a sorted array of them. + * + * @returns VBox status code. + * @param ppaLeaves Where to return the array pointer on success. + * Use RTMemFree to release. + * @param pcLeaves Where to return the size of the array on + * success. + */ +VMMDECL(int) CPUMCpuIdCollectLeavesX86(PCPUMCPUIDLEAF *ppaLeaves, uint32_t *pcLeaves) +{ + *ppaLeaves = NULL; + *pcLeaves = 0; + + /* + * Try out various candidates. This must be sorted! + */ + static struct { uint32_t uMsr; bool fSpecial; } const s_aCandidates[] = + { + { UINT32_C(0x00000000), false }, + { UINT32_C(0x10000000), false }, + { UINT32_C(0x20000000), false }, + { UINT32_C(0x30000000), false }, + { UINT32_C(0x40000000), false }, + { UINT32_C(0x50000000), false }, + { UINT32_C(0x60000000), false }, + { UINT32_C(0x70000000), false }, + { UINT32_C(0x80000000), false }, + { UINT32_C(0x80860000), false }, + { UINT32_C(0x8ffffffe), true }, + { UINT32_C(0x8fffffff), true }, + { UINT32_C(0x90000000), false }, + { UINT32_C(0xa0000000), false }, + { UINT32_C(0xb0000000), false }, + { UINT32_C(0xc0000000), false }, + { UINT32_C(0xd0000000), false }, + { UINT32_C(0xe0000000), false }, + { UINT32_C(0xf0000000), false }, + }; + + for (uint32_t iOuter = 0; iOuter < RT_ELEMENTS(s_aCandidates); iOuter++) + { + uint32_t uLeaf = s_aCandidates[iOuter].uMsr; + uint32_t uEax, uEbx, uEcx, uEdx; + ASMCpuIdExSlow(uLeaf, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx); + + /* + * Does EAX look like a typical leaf count value? + */ + if ( uEax > uLeaf + && uEax - uLeaf < UINT32_C(0xff)) /* Adjust 0xff limit when exceeded by real HW. */ + { + /* Yes, dump them. */ + uint32_t cLeaves = uEax - uLeaf + 1; + while (cLeaves-- > 0) + { + ASMCpuIdExSlow(uLeaf, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx); + + uint32_t fFlags = 0; + + /* There are currently three known leaves containing an APIC ID + that needs EMT specific attention */ + if (uLeaf == 1) + fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC_ID; + else if (uLeaf == 0xb && uEcx != 0) + fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC_ID; + else if ( uLeaf == UINT32_C(0x8000001e) + && ( uEax + || uEbx + || uEdx + || RTX86IsAmdCpu((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx) + || RTX86IsHygonCpu((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)) ) + fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC_ID; + + /* The APIC bit is per-VCpu and needs flagging. */ + if (uLeaf == 1) + fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC; + else if ( uLeaf == UINT32_C(0x80000001) + && ( (uEdx & X86_CPUID_AMD_FEATURE_EDX_APIC) + || RTX86IsAmdCpu((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx) + || RTX86IsHygonCpu((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)) ) + fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC; + + /* Check three times here to reduce the chance of CPU migration + resulting in false positives with things like the APIC ID. */ + uint32_t cSubLeaves; + bool fFinalEcxUnchanged; + if ( cpumIsEcxRelevantForCpuIdLeaf(uLeaf, &cSubLeaves, &fFinalEcxUnchanged) + && cpumIsEcxRelevantForCpuIdLeaf(uLeaf, &cSubLeaves, &fFinalEcxUnchanged) + && cpumIsEcxRelevantForCpuIdLeaf(uLeaf, &cSubLeaves, &fFinalEcxUnchanged)) + { + if (cSubLeaves > (uLeaf == 0xd ? 68U : 16U)) + { + /* This shouldn't happen. But in case it does, file all + relevant details in the release log. */ + LogRel(("CPUM: VERR_CPUM_TOO_MANY_CPUID_SUBLEAVES! uLeaf=%#x cSubLeaves=%#x\n", uLeaf, cSubLeaves)); + LogRel(("------------------ dump of problematic sub-leaves -----------------\n")); + for (uint32_t uSubLeaf = 0; uSubLeaf < 128; uSubLeaf++) + { + uint32_t auTmp[4]; + ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &auTmp[0], &auTmp[1], &auTmp[2], &auTmp[3]); + LogRel(("CPUM: %#010x, %#010x => %#010x %#010x %#010x %#010x\n", + uLeaf, uSubLeaf, auTmp[0], auTmp[1], auTmp[2], auTmp[3])); + } + LogRel(("----------------- dump of what we've found so far -----------------\n")); + for (uint32_t i = 0 ; i < *pcLeaves; i++) + LogRel(("CPUM: %#010x, %#010x/%#010x => %#010x %#010x %#010x %#010x\n", + (*ppaLeaves)[i].uLeaf, (*ppaLeaves)[i].uSubLeaf, (*ppaLeaves)[i].fSubLeafMask, + (*ppaLeaves)[i].uEax, (*ppaLeaves)[i].uEbx, (*ppaLeaves)[i].uEcx, (*ppaLeaves)[i].uEdx)); + LogRel(("\nPlease create a defect on virtualbox.org and attach this log file!\n\n")); + return VERR_CPUM_TOO_MANY_CPUID_SUBLEAVES; + } + + if (fFinalEcxUnchanged) + fFlags |= CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES; + + for (uint32_t uSubLeaf = 0; uSubLeaf < cSubLeaves; uSubLeaf++) + { + ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &uEax, &uEbx, &uEcx, &uEdx); + int rc = cpumCollectCpuIdInfoAddOne(ppaLeaves, pcLeaves, + uLeaf, uSubLeaf, UINT32_MAX, uEax, uEbx, uEcx, uEdx, fFlags); + if (RT_FAILURE(rc)) + return rc; + } + } + else + { + int rc = cpumCollectCpuIdInfoAddOne(ppaLeaves, pcLeaves, uLeaf, 0, 0, uEax, uEbx, uEcx, uEdx, fFlags); + if (RT_FAILURE(rc)) + return rc; + } + + /* next */ + uLeaf++; + } + } + /* + * Special CPUIDs needs special handling as they don't follow the + * leaf count principle used above. + */ + else if (s_aCandidates[iOuter].fSpecial) + { + bool fKeep = false; + if (uLeaf == 0x8ffffffe && uEax == UINT32_C(0x00494544)) + fKeep = true; + else if ( uLeaf == 0x8fffffff + && RT_C_IS_PRINT(RT_BYTE1(uEax)) + && RT_C_IS_PRINT(RT_BYTE2(uEax)) + && RT_C_IS_PRINT(RT_BYTE3(uEax)) + && RT_C_IS_PRINT(RT_BYTE4(uEax)) + && RT_C_IS_PRINT(RT_BYTE1(uEbx)) + && RT_C_IS_PRINT(RT_BYTE2(uEbx)) + && RT_C_IS_PRINT(RT_BYTE3(uEbx)) + && RT_C_IS_PRINT(RT_BYTE4(uEbx)) + && RT_C_IS_PRINT(RT_BYTE1(uEcx)) + && RT_C_IS_PRINT(RT_BYTE2(uEcx)) + && RT_C_IS_PRINT(RT_BYTE3(uEcx)) + && RT_C_IS_PRINT(RT_BYTE4(uEcx)) + && RT_C_IS_PRINT(RT_BYTE1(uEdx)) + && RT_C_IS_PRINT(RT_BYTE2(uEdx)) + && RT_C_IS_PRINT(RT_BYTE3(uEdx)) + && RT_C_IS_PRINT(RT_BYTE4(uEdx)) ) + fKeep = true; + if (fKeep) + { + int rc = cpumCollectCpuIdInfoAddOne(ppaLeaves, pcLeaves, uLeaf, 0, 0, uEax, uEbx, uEcx, uEdx, 0); + if (RT_FAILURE(rc)) + return rc; + } + } + } + +# ifdef VBOX_STRICT + cpumCpuIdAssertOrder(*ppaLeaves, *pcLeaves); +# endif + return VINF_SUCCESS; +} +#endif /* RT_ARCH_X86 || RT_ARCH_AMD64 */ + + +/** + * Detect the CPU vendor give n the + * + * @returns The vendor. + * @param uEAX EAX from CPUID(0). + * @param uEBX EBX from CPUID(0). + * @param uECX ECX from CPUID(0). + * @param uEDX EDX from CPUID(0). + */ +VMMDECL(CPUMCPUVENDOR) CPUMCpuIdDetectX86VendorEx(uint32_t uEAX, uint32_t uEBX, uint32_t uECX, uint32_t uEDX) +{ + if (RTX86IsValidStdRange(uEAX)) + { + if (RTX86IsAmdCpu(uEBX, uECX, uEDX)) + return CPUMCPUVENDOR_AMD; + + if (RTX86IsIntelCpu(uEBX, uECX, uEDX)) + return CPUMCPUVENDOR_INTEL; + + if (RTX86IsViaCentaurCpu(uEBX, uECX, uEDX)) + return CPUMCPUVENDOR_VIA; + + if (RTX86IsShanghaiCpu(uEBX, uECX, uEDX)) + return CPUMCPUVENDOR_SHANGHAI; + + if ( uEBX == UINT32_C(0x69727943) /* CyrixInstead */ + && uECX == UINT32_C(0x64616574) + && uEDX == UINT32_C(0x736E4978)) + return CPUMCPUVENDOR_CYRIX; + + if (RTX86IsHygonCpu(uEBX, uECX, uEDX)) + return CPUMCPUVENDOR_HYGON; + + /* "Geode by NSC", example: family 5, model 9. */ + + /** @todo detect the other buggers... */ + } + + return CPUMCPUVENDOR_UNKNOWN; +} + + +/** + * Translates a CPU vendor enum value into the corresponding string constant. + * + * The named can be prefixed with 'CPUMCPUVENDOR_' to construct a valid enum + * value name. This can be useful when generating code. + * + * @returns Read only name string. + * @param enmVendor The CPU vendor value. + */ +VMMDECL(const char *) CPUMCpuVendorName(CPUMCPUVENDOR enmVendor) +{ + switch (enmVendor) + { + case CPUMCPUVENDOR_INTEL: return "INTEL"; + case CPUMCPUVENDOR_AMD: return "AMD"; + case CPUMCPUVENDOR_VIA: return "VIA"; + case CPUMCPUVENDOR_CYRIX: return "CYRIX"; + case CPUMCPUVENDOR_SHANGHAI: return "SHANGHAI"; + case CPUMCPUVENDOR_HYGON: return "HYGON"; + case CPUMCPUVENDOR_UNKNOWN: return "UNKNOWN"; + + case CPUMCPUVENDOR_INVALID: + case CPUMCPUVENDOR_32BIT_HACK: + break; + } + return "Invalid-cpu-vendor"; +} + + +static PCCPUMCPUIDLEAF cpumCpuIdFindLeaf(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf) +{ + /* Could do binary search, doing linear now because I'm lazy. */ + PCCPUMCPUIDLEAF pLeaf = paLeaves; + while (cLeaves-- > 0) + { + if (pLeaf->uLeaf == uLeaf) + return pLeaf; + pLeaf++; + } + return NULL; +} + + +static PCCPUMCPUIDLEAF cpumCpuIdFindLeafEx(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf, uint32_t uSubLeaf) +{ + PCCPUMCPUIDLEAF pLeaf = cpumCpuIdFindLeaf(paLeaves, cLeaves, uLeaf); + if ( !pLeaf + || pLeaf->uSubLeaf != (uSubLeaf & pLeaf->fSubLeafMask)) + return pLeaf; + + /* Linear sub-leaf search. Lazy as usual. */ + cLeaves -= pLeaf - paLeaves; + while ( cLeaves-- > 0 + && pLeaf->uLeaf == uLeaf) + { + if (pLeaf->uSubLeaf == (uSubLeaf & pLeaf->fSubLeafMask)) + return pLeaf; + pLeaf++; + } + + return NULL; +} + + +static void cpumExplodeVmxFeatures(PCVMXMSRS pVmxMsrs, PCPUMFEATURES pFeatures) +{ + Assert(pVmxMsrs); + Assert(pFeatures); + Assert(pFeatures->fVmx); + + /* Basic information. */ + bool const fVmxTrueMsrs = RT_BOOL(pVmxMsrs->u64Basic & VMX_BF_BASIC_TRUE_CTLS_MASK); + { + uint64_t const u64Basic = pVmxMsrs->u64Basic; + pFeatures->fVmxInsOutInfo = RT_BF_GET(u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS); + } + + /* Pin-based VM-execution controls. */ + { + uint32_t const fPinCtls = fVmxTrueMsrs ? pVmxMsrs->TruePinCtls.n.allowed1 : pVmxMsrs->PinCtls.n.allowed1; + pFeatures->fVmxExtIntExit = RT_BOOL(fPinCtls & VMX_PIN_CTLS_EXT_INT_EXIT); + pFeatures->fVmxNmiExit = RT_BOOL(fPinCtls & VMX_PIN_CTLS_NMI_EXIT); + pFeatures->fVmxVirtNmi = RT_BOOL(fPinCtls & VMX_PIN_CTLS_VIRT_NMI); + pFeatures->fVmxPreemptTimer = RT_BOOL(fPinCtls & VMX_PIN_CTLS_PREEMPT_TIMER); + pFeatures->fVmxPostedInt = RT_BOOL(fPinCtls & VMX_PIN_CTLS_POSTED_INT); + } + + /* Processor-based VM-execution controls. */ + { + uint32_t const fProcCtls = fVmxTrueMsrs ? pVmxMsrs->TrueProcCtls.n.allowed1 : pVmxMsrs->ProcCtls.n.allowed1; + pFeatures->fVmxIntWindowExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT); + pFeatures->fVmxTscOffsetting = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING); + pFeatures->fVmxHltExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_HLT_EXIT); + pFeatures->fVmxInvlpgExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_INVLPG_EXIT); + pFeatures->fVmxMwaitExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MWAIT_EXIT); + pFeatures->fVmxRdpmcExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_RDPMC_EXIT); + pFeatures->fVmxRdtscExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_RDTSC_EXIT); + pFeatures->fVmxCr3LoadExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR3_LOAD_EXIT); + pFeatures->fVmxCr3StoreExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR3_STORE_EXIT); + pFeatures->fVmxTertiaryExecCtls = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS); + pFeatures->fVmxCr8LoadExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR8_LOAD_EXIT); + pFeatures->fVmxCr8StoreExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR8_STORE_EXIT); + pFeatures->fVmxUseTprShadow = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); + pFeatures->fVmxNmiWindowExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT); + pFeatures->fVmxMovDRxExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT); + pFeatures->fVmxUncondIoExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT); + pFeatures->fVmxUseIoBitmaps = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS); + pFeatures->fVmxMonitorTrapFlag = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG); + pFeatures->fVmxUseMsrBitmaps = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS); + pFeatures->fVmxMonitorExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MONITOR_EXIT); + pFeatures->fVmxPauseExit = RT_BOOL(fProcCtls & VMX_PROC_CTLS_PAUSE_EXIT); + pFeatures->fVmxSecondaryExecCtls = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS); + } + + /* Secondary processor-based VM-execution controls. */ + { + uint32_t const fProcCtls2 = pFeatures->fVmxSecondaryExecCtls ? pVmxMsrs->ProcCtls2.n.allowed1 : 0; + pFeatures->fVmxVirtApicAccess = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS); + pFeatures->fVmxEpt = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_EPT); + pFeatures->fVmxDescTableExit = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT); + pFeatures->fVmxRdtscp = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_RDTSCP); + pFeatures->fVmxVirtX2ApicMode = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE); + pFeatures->fVmxVpid = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VPID); + pFeatures->fVmxWbinvdExit = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_WBINVD_EXIT); + pFeatures->fVmxUnrestrictedGuest = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST); + pFeatures->fVmxApicRegVirt = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT); + pFeatures->fVmxVirtIntDelivery = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY); + pFeatures->fVmxPauseLoopExit = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT); + pFeatures->fVmxRdrandExit = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT); + pFeatures->fVmxInvpcid = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_INVPCID); + pFeatures->fVmxVmFunc = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VMFUNC); + pFeatures->fVmxVmcsShadowing = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING); + pFeatures->fVmxRdseedExit = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_RDSEED_EXIT); + pFeatures->fVmxPml = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_PML); + pFeatures->fVmxEptXcptVe = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_EPT_XCPT_VE); + pFeatures->fVmxConcealVmxFromPt = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_CONCEAL_VMX_FROM_PT); + pFeatures->fVmxXsavesXrstors = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_XSAVES_XRSTORS); + pFeatures->fVmxModeBasedExecuteEpt = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_MODE_BASED_EPT_PERM); + pFeatures->fVmxSppEpt = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_SPP_EPT); + pFeatures->fVmxPtEpt = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_PT_EPT); + pFeatures->fVmxUseTscScaling = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_TSC_SCALING); + pFeatures->fVmxUserWaitPause = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_USER_WAIT_PAUSE); + pFeatures->fVmxEnclvExit = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_ENCLV_EXIT); + } + + /* Tertiary processor-based VM-execution controls. */ + { + uint64_t const fProcCtls3 = pFeatures->fVmxTertiaryExecCtls ? pVmxMsrs->u64ProcCtls3 : 0; + pFeatures->fVmxLoadIwKeyExit = RT_BOOL(fProcCtls3 & VMX_PROC_CTLS3_LOADIWKEY_EXIT); + } + + /* VM-exit controls. */ + { + uint32_t const fExitCtls = fVmxTrueMsrs ? pVmxMsrs->TrueExitCtls.n.allowed1 : pVmxMsrs->ExitCtls.n.allowed1; + pFeatures->fVmxExitSaveDebugCtls = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_DEBUG); + pFeatures->fVmxHostAddrSpaceSize = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE); + pFeatures->fVmxExitAckExtInt = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT); + pFeatures->fVmxExitSavePatMsr = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_PAT_MSR); + pFeatures->fVmxExitLoadPatMsr = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_LOAD_PAT_MSR); + pFeatures->fVmxExitSaveEferMsr = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_EFER_MSR); + pFeatures->fVmxExitLoadEferMsr = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR); + pFeatures->fVmxSavePreemptTimer = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER); + pFeatures->fVmxSecondaryExitCtls = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_USE_SECONDARY_CTLS); + } + + /* VM-entry controls. */ + { + uint32_t const fEntryCtls = fVmxTrueMsrs ? pVmxMsrs->TrueEntryCtls.n.allowed1 : pVmxMsrs->EntryCtls.n.allowed1; + pFeatures->fVmxEntryLoadDebugCtls = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG); + pFeatures->fVmxIa32eModeGuest = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST); + pFeatures->fVmxEntryLoadEferMsr = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR); + pFeatures->fVmxEntryLoadPatMsr = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR); + } + + /* Miscellaneous data. */ + { + uint32_t const fMiscData = pVmxMsrs->u64Misc; + pFeatures->fVmxExitSaveEferLma = RT_BOOL(fMiscData & VMX_MISC_EXIT_SAVE_EFER_LMA); + pFeatures->fVmxPt = RT_BOOL(fMiscData & VMX_MISC_INTEL_PT); + pFeatures->fVmxVmwriteAll = RT_BOOL(fMiscData & VMX_MISC_VMWRITE_ALL); + pFeatures->fVmxEntryInjectSoftInt = RT_BOOL(fMiscData & VMX_MISC_ENTRY_INJECT_SOFT_INT); + } +} + + +int cpumCpuIdExplodeFeaturesX86(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, PCCPUMMSRS pMsrs, PCPUMFEATURES pFeatures) +{ + Assert(pMsrs); + RT_ZERO(*pFeatures); + if (cLeaves >= 2) + { + AssertLogRelReturn(paLeaves[0].uLeaf == 0, VERR_CPUM_IPE_1); + AssertLogRelReturn(paLeaves[1].uLeaf == 1, VERR_CPUM_IPE_1); + PCCPUMCPUIDLEAF const pStd0Leaf = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 0, 0); + AssertLogRelReturn(pStd0Leaf, VERR_CPUM_IPE_1); + PCCPUMCPUIDLEAF const pStd1Leaf = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 1, 0); + AssertLogRelReturn(pStd1Leaf, VERR_CPUM_IPE_1); + + pFeatures->enmCpuVendor = CPUMCpuIdDetectX86VendorEx(pStd0Leaf->uEax, + pStd0Leaf->uEbx, + pStd0Leaf->uEcx, + pStd0Leaf->uEdx); + pFeatures->uFamily = RTX86GetCpuFamily(pStd1Leaf->uEax); + pFeatures->uModel = RTX86GetCpuModel(pStd1Leaf->uEax, pFeatures->enmCpuVendor == CPUMCPUVENDOR_INTEL); + pFeatures->uStepping = RTX86GetCpuStepping(pStd1Leaf->uEax); + pFeatures->enmMicroarch = CPUMCpuIdDetermineX86MicroarchEx((CPUMCPUVENDOR)pFeatures->enmCpuVendor, + pFeatures->uFamily, + pFeatures->uModel, + pFeatures->uStepping); + + PCCPUMCPUIDLEAF const pExtLeaf8 = cpumCpuIdFindLeaf(paLeaves, cLeaves, 0x80000008); + if (pExtLeaf8) + { + pFeatures->cMaxPhysAddrWidth = pExtLeaf8->uEax & 0xff; + pFeatures->cMaxLinearAddrWidth = (pExtLeaf8->uEax >> 8) & 0xff; + } + else if (pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PSE36) + { + pFeatures->cMaxPhysAddrWidth = 36; + pFeatures->cMaxLinearAddrWidth = 36; + } + else + { + pFeatures->cMaxPhysAddrWidth = 32; + pFeatures->cMaxLinearAddrWidth = 32; + } + + /* Standard features. */ + pFeatures->fMsr = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_MSR); + pFeatures->fApic = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_APIC); + pFeatures->fX2Apic = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_X2APIC); + pFeatures->fPse = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PSE); + pFeatures->fPse36 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PSE36); + pFeatures->fPae = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PAE); + pFeatures->fPge = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PGE); + pFeatures->fPat = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PAT); + pFeatures->fFxSaveRstor = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_FXSR); + pFeatures->fXSaveRstor = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_XSAVE); + pFeatures->fOpSysXSaveRstor = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_OSXSAVE); + pFeatures->fMmx = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_MMX); + pFeatures->fSse = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_SSE); + pFeatures->fSse2 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_SSE2); + pFeatures->fSse3 = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSE3); + pFeatures->fSsse3 = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSSE3); + pFeatures->fSse41 = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSE4_1); + pFeatures->fSse42 = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSE4_2); + pFeatures->fAesNi = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_AES); + pFeatures->fAvx = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_AVX); + pFeatures->fTsc = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_TSC); + pFeatures->fSysEnter = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_SEP); + pFeatures->fHypervisorPresent = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_HVP); + pFeatures->fMonitorMWait = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_MONITOR); + pFeatures->fMovCmpXchg16b = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_CX16); + pFeatures->fClFlush = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_CLFSH); + pFeatures->fPcid = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_PCID); + pFeatures->fPopCnt = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_POPCNT); + pFeatures->fRdRand = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_RDRAND); + pFeatures->fVmx = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_VMX); + pFeatures->fPclMul = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_PCLMUL); + pFeatures->fMovBe = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_MOVBE); + if (pFeatures->fVmx) + cpumExplodeVmxFeatures(&pMsrs->hwvirt.vmx, pFeatures); + + /* Structured extended features. */ + PCCPUMCPUIDLEAF const pSxfLeaf0 = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 7, 0); + if (pSxfLeaf0) + { + pFeatures->fFsGsBase = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_FSGSBASE); + pFeatures->fAvx2 = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_AVX2); + pFeatures->fAvx512Foundation = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_AVX512F); + pFeatures->fClFlushOpt = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_CLFLUSHOPT); + pFeatures->fInvpcid = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_INVPCID); + pFeatures->fBmi1 = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_BMI1); + pFeatures->fBmi2 = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_BMI2); + pFeatures->fRdSeed = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_RDSEED); + pFeatures->fHle = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_HLE); + pFeatures->fRtm = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_RTM); + + pFeatures->fIbpb = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_IBRS_IBPB); + pFeatures->fIbrs = pFeatures->fIbpb; + pFeatures->fStibp = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_STIBP); + pFeatures->fFlushCmd = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD); + pFeatures->fArchCap = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_ARCHCAP); + pFeatures->fMdsClear = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR); + } + + /* MWAIT/MONITOR leaf. */ + PCCPUMCPUIDLEAF const pMWaitLeaf = cpumCpuIdFindLeaf(paLeaves, cLeaves, 5); + if (pMWaitLeaf) + pFeatures->fMWaitExtensions = (pMWaitLeaf->uEcx & (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0)) + == (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0); + + /* Extended features. */ + PCCPUMCPUIDLEAF const pExtLeaf = cpumCpuIdFindLeaf(paLeaves, cLeaves, 0x80000001); + if (pExtLeaf) + { + pFeatures->fLongMode = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE); + pFeatures->fSysCall = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_SYSCALL); + pFeatures->fNoExecute = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_NX); + pFeatures->fLahfSahf = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_EXT_FEATURE_ECX_LAHF_SAHF); + pFeatures->fRdTscP = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_RDTSCP); + pFeatures->fMovCr8In32Bit = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_CMPL); + pFeatures->f3DNow = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_3DNOW); + pFeatures->f3DNowPrefetch = (pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF) + || (pExtLeaf->uEdx & ( X86_CPUID_EXT_FEATURE_EDX_LONG_MODE + | X86_CPUID_AMD_FEATURE_EDX_3DNOW)); + pFeatures->fAbm = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_ABM); + } + + /* VMX (VMXON, VMCS region and related data structures) physical address width (depends on long-mode). */ + pFeatures->cVmxMaxPhysAddrWidth = pFeatures->fLongMode ? pFeatures->cMaxPhysAddrWidth : 32; + + if ( pExtLeaf + && ( pFeatures->enmCpuVendor == CPUMCPUVENDOR_AMD + || pFeatures->enmCpuVendor == CPUMCPUVENDOR_HYGON)) + { + /* AMD features. */ + pFeatures->fMsr |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_MSR); + pFeatures->fApic |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_APIC); + pFeatures->fPse |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PSE); + pFeatures->fPse36 |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PSE36); + pFeatures->fPae |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PAE); + pFeatures->fPge |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PGE); + pFeatures->fPat |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PAT); + pFeatures->fFxSaveRstor |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_FXSR); + pFeatures->fMmx |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_MMX); + pFeatures->fTsc |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_TSC); + pFeatures->fIbpb |= pExtLeaf8 && (pExtLeaf8->uEbx & X86_CPUID_AMD_EFEID_EBX_IBPB); + pFeatures->fAmdMmxExts = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_AXMMX); + pFeatures->fXop = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_XOP); + pFeatures->fTbm = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_TBM); + pFeatures->fSvm = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_SVM); + if (pFeatures->fSvm) + { + PCCPUMCPUIDLEAF pSvmLeaf = cpumCpuIdFindLeaf(paLeaves, cLeaves, 0x8000000a); + AssertLogRelReturn(pSvmLeaf, VERR_CPUM_IPE_1); + pFeatures->fSvmNestedPaging = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING); + pFeatures->fSvmLbrVirt = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT); + pFeatures->fSvmSvmLock = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_SVM_LOCK); + pFeatures->fSvmNextRipSave = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE); + pFeatures->fSvmTscRateMsr = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_TSC_RATE_MSR); + pFeatures->fSvmVmcbClean = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN); + pFeatures->fSvmFlusbByAsid = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID); + pFeatures->fSvmDecodeAssists = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS); + pFeatures->fSvmPauseFilter = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER); + pFeatures->fSvmPauseFilterThreshold = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD); + pFeatures->fSvmAvic = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_AVIC); + pFeatures->fSvmVirtVmsaveVmload = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD); + pFeatures->fSvmVGif = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_VGIF); + pFeatures->fSvmGmet = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_GMET); + pFeatures->fSvmSSSCheck = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_SSSCHECK); + pFeatures->fSvmSpecCtrl = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_SPEC_CTRL); + pFeatures->fSvmHostMceOverride = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_HOST_MCE_OVERRIDE); + pFeatures->fSvmTlbiCtl = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_TLBICTL); + pFeatures->uSvmMaxAsid = pSvmLeaf->uEbx; + } + } + + /* + * Quirks. + */ + pFeatures->fLeakyFxSR = pExtLeaf + && (pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_FFXSR) + && ( ( pFeatures->enmCpuVendor == CPUMCPUVENDOR_AMD + && pFeatures->uFamily >= 6 /* K7 and up */) + || pFeatures->enmCpuVendor == CPUMCPUVENDOR_HYGON); + + /* + * Max extended (/FPU) state. + */ + pFeatures->cbMaxExtendedState = pFeatures->fFxSaveRstor ? sizeof(X86FXSTATE) : sizeof(X86FPUSTATE); + if (pFeatures->fXSaveRstor) + { + PCCPUMCPUIDLEAF const pXStateLeaf0 = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 13, 0); + if (pXStateLeaf0) + { + if ( pXStateLeaf0->uEcx >= sizeof(X86FXSTATE) + && pXStateLeaf0->uEcx <= CPUM_MAX_XSAVE_AREA_SIZE + && RT_ALIGN_32(pXStateLeaf0->uEcx, 8) == pXStateLeaf0->uEcx + && pXStateLeaf0->uEbx >= sizeof(X86FXSTATE) + && pXStateLeaf0->uEbx <= pXStateLeaf0->uEcx + && RT_ALIGN_32(pXStateLeaf0->uEbx, 8) == pXStateLeaf0->uEbx) + { + pFeatures->cbMaxExtendedState = pXStateLeaf0->uEcx; + + /* (paranoia:) */ + PCCPUMCPUIDLEAF const pXStateLeaf1 = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 13, 1); + if ( pXStateLeaf1 + && pXStateLeaf1->uEbx > pFeatures->cbMaxExtendedState + && pXStateLeaf1->uEbx <= CPUM_MAX_XSAVE_AREA_SIZE + && (pXStateLeaf1->uEcx || pXStateLeaf1->uEdx) ) + pFeatures->cbMaxExtendedState = pXStateLeaf1->uEbx; + } + else + AssertLogRelMsgFailedStmt(("Unexpected max/cur XSAVE area sizes: %#x/%#x\n", pXStateLeaf0->uEcx, pXStateLeaf0->uEbx), + pFeatures->fXSaveRstor = 0); + } + else + AssertLogRelMsgFailedStmt(("Expected leaf eax=0xd/ecx=0 with the XSAVE/XRSTOR feature!\n"), + pFeatures->fXSaveRstor = 0); + } + } + else + AssertLogRelReturn(cLeaves == 0, VERR_CPUM_IPE_1); + return VINF_SUCCESS; +} + diff --git a/src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp b/src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp new file mode 100644 index 00000000..29a4e52d --- /dev/null +++ b/src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp @@ -0,0 +1,6505 @@ +/* $Id: CPUMAllMsrs.cpp $ */ +/** @file + * CPUM - CPU MSR Registers. + */ + +/* + * Copyright (C) 2013-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_CPUM +#include +#include +#include +#include +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +# include +#endif +#include +#include +#include "CPUMInternal.h" +#include +#include + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** + * Validates the CPUMMSRRANGE::offCpumCpu value and declares a local variable + * pointing to it. + * + * ASSUMES sizeof(a_Type) is a power of two and that the member is aligned + * correctly. + */ +#define CPUM_MSR_ASSERT_CPUMCPU_OFFSET_RETURN(a_pVCpu, a_pRange, a_Type, a_VarName) \ + AssertMsgReturn( (a_pRange)->offCpumCpu >= 8 \ + && (a_pRange)->offCpumCpu < sizeof(CPUMCPU) \ + && !((a_pRange)->offCpumCpu & (RT_MIN(sizeof(a_Type), 8) - 1)) \ + , ("offCpumCpu=%#x %s\n", (a_pRange)->offCpumCpu, (a_pRange)->szName), \ + VERR_CPUM_MSR_BAD_CPUMCPU_OFFSET); \ + a_Type *a_VarName = (a_Type *)((uintptr_t)&(a_pVCpu)->cpum.s + (a_pRange)->offCpumCpu) + + +/********************************************************************************************************************************* +* Structures and Typedefs * +*********************************************************************************************************************************/ + +/** + * Implements reading one or more MSRs. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VINF_CPUM_R3_MSR_READ if the MSR read could not be serviced in the + * current context (raw-mode or ring-0). + * @retval VERR_CPUM_RAISE_GP_0 on failure (invalid MSR). + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR we're reading. + * @param pRange The MSR range descriptor. + * @param puValue Where to return the value. + */ +typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNCPUMRDMSR,(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)); +/** Pointer to a RDMSR worker for a specific MSR or range of MSRs. */ +typedef FNCPUMRDMSR *PFNCPUMRDMSR; + + +/** + * Implements writing one or more MSRs. + * + * @retval VINF_SUCCESS on success. + * @retval VINF_CPUM_R3_MSR_WRITE if the MSR write could not be serviced in the + * current context (raw-mode or ring-0). + * @retval VERR_CPUM_RAISE_GP_0 on failure. + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR we're writing. + * @param pRange The MSR range descriptor. + * @param uValue The value to set, ignored bits masked. + * @param uRawValue The raw value with the ignored bits not masked. + */ +typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNCPUMWRMSR,(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, + uint64_t uValue, uint64_t uRawValue)); +/** Pointer to a WRMSR worker for a specific MSR or range of MSRs. */ +typedef FNCPUMWRMSR *PFNCPUMWRMSR; + + + +/* + * Generic functions. + * Generic functions. + * Generic functions. + */ + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_FixedValue(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IgnoreWrite(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + Log(("CPUM: Ignoring WRMSR %#x (%s), %#llx\n", idMsr, pRange->szName, uValue)); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_WriteOnly(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(puValue); + return VERR_CPUM_RAISE_GP_0; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_ReadOnly(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + Assert(pRange->fWrGpMask == UINT64_MAX); + return VERR_CPUM_RAISE_GP_0; +} + + + + +/* + * IA32 + * IA32 + * IA32 + */ + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32P5McAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = 0; /** @todo implement machine check injection. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32P5McAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement machine check injection. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32P5McType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = 0; /** @todo implement machine check injection. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32P5McType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement machine check injection. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32TimestampCounter(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = TMCpuTickGet(pVCpu); +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + *puValue = CPUMApplyNestedGuestTscOffset(pVCpu, *puValue); +#endif + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32TimestampCounter(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + TMCpuTickSet(pVCpu->CTX_SUFF(pVM), pVCpu, uValue); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PlatformId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + uint64_t uValue = pRange->uValue; + if (uValue & 0x1f00) + { + /* Max allowed bus ratio present. */ + /** @todo Implement scaled BUS frequency. */ + } + + *puValue = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ApicBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + return APICGetBaseMsr(pVCpu, puValue); +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ApicBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + return APICSetBaseMsr(pVCpu, uValue); +} + + +/** + * Gets IA32_FEATURE_CONTROL value for IEM, NEM and cpumMsrRd_Ia32FeatureControl. + * + * @returns IA32_FEATURE_CONTROL value. + * @param pVCpu The cross context per CPU structure. + */ +VMM_INT_DECL(uint64_t) CPUMGetGuestIa32FeatCtrl(PCVMCPUCC pVCpu) +{ + uint64_t uFeatCtrlMsr = MSR_IA32_FEATURE_CONTROL_LOCK; + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + uFeatCtrlMsr |= MSR_IA32_FEATURE_CONTROL_VMXON; + return uFeatCtrlMsr; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32FeatureControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = CPUMGetGuestIa32FeatCtrl(pVCpu); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FeatureControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + return VERR_CPUM_RAISE_GP_0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32BiosSignId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo fake microcode update. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32BiosSignId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /* Normally, zero is written to Ia32BiosSignId before reading it in order + to select the signature instead of the BBL_CR_D3 behaviour. The GP mask + of the database entry should take care of most illegal writes for now, so + just ignore all writes atm. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32BiosUpdateTrigger(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + + /* Microcode updates cannot be loaded in VMX non-root mode. */ + if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest)) + return VINF_SUCCESS; + + /** @todo Fake bios update trigger better. The value is the address to an + * update package, I think. We should probably GP if it's invalid. */ + return VINF_SUCCESS; +} + + +/** + * Get MSR_IA32_SMM_MONITOR_CTL value for IEM and cpumMsrRd_Ia32SmmMonitorCtl. + * + * @returns The MSR_IA32_SMM_MONITOR_CTL value. + * @param pVCpu The cross context per CPU structure. + */ +VMM_INT_DECL(uint64_t) CPUMGetGuestIa32SmmMonitorCtl(PCVMCPUCC pVCpu) +{ + /* We do not support dual-monitor treatment for SMI and SMM. */ + /** @todo SMM. */ + RT_NOREF(pVCpu); + return 0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SmmMonitorCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = CPUMGetGuestIa32SmmMonitorCtl(pVCpu); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SmmMonitorCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo SMM. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PmcN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo check CPUID leaf 0ah. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PmcN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo check CPUID leaf 0ah. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MonitorFilterLineSize(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo return 0x1000 if we try emulate mwait 100% correctly. */ + *puValue = 0x40; /** @todo Change to CPU cache line size. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MonitorFilterLineSize(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo should remember writes, though it's supposedly something only a BIOS + * would write so, it's not extremely important. */ + return VINF_SUCCESS; +} + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MPerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Read MPERF: Adjust against previously written MPERF value. Is TSC + * what we want? */ + *puValue = TMCpuTickGet(pVCpu); +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + *puValue = CPUMApplyNestedGuestTscOffset(pVCpu, *puValue); +#endif + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MPerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Write MPERF: Calc adjustment. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32APerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Read APERF: Adjust against previously written MPERF value. Is TSC + * what we want? */ + *puValue = TMCpuTickGet(pVCpu); +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + *puValue = CPUMApplyNestedGuestTscOffset(pVCpu, *puValue); +#endif + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32APerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Write APERF: Calc adjustment. */ + return VINF_SUCCESS; +} + + +/** + * Get fixed IA32_MTRR_CAP value for NEM and cpumMsrRd_Ia32MtrrCap. + * + * @returns Fixed IA32_MTRR_CAP value. + * @param pVCpu The cross context per CPU structure. + */ +VMM_INT_DECL(uint64_t) CPUMGetGuestIa32MtrrCap(PCVMCPU pVCpu) +{ + RT_NOREF_PV(pVCpu); + + /* This is currently a bit weird. :-) */ + uint8_t const cVariableRangeRegs = 0; + bool const fSystemManagementRangeRegisters = false; + bool const fFixedRangeRegisters = false; + bool const fWriteCombiningType = false; + return cVariableRangeRegs + | (fFixedRangeRegisters ? RT_BIT_64(8) : 0) + | (fWriteCombiningType ? RT_BIT_64(10) : 0) + | (fSystemManagementRangeRegisters ? RT_BIT_64(11) : 0); +} + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = CPUMGetGuestIa32MtrrCap(pVCpu); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrPhysBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Implement variable MTRR storage. */ + Assert(pRange->uValue == (idMsr - 0x200) / 2); + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrPhysBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + /* + * Validate the value. + */ + Assert(pRange->uValue == (idMsr - 0x200) / 2); + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(uRawValue); RT_NOREF_PV(pRange); + + uint8_t uType = uValue & 0xff; + if ((uType >= 7) || (uType == 2) || (uType == 3)) + { + Log(("CPUM: Invalid type set writing MTRR PhysBase MSR %#x: %#llx (%#llx)\n", idMsr, uValue, uType)); + return VERR_CPUM_RAISE_GP_0; + } + + uint64_t fInvPhysMask = ~(RT_BIT_64(pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.cMaxPhysAddrWidth) - 1U); + if (fInvPhysMask & uValue) + { + Log(("CPUM: Invalid physical address bits set writing MTRR PhysBase MSR %#x: %#llx (%#llx)\n", + idMsr, uValue, uValue & fInvPhysMask)); + return VERR_CPUM_RAISE_GP_0; + } + + /* + * Store it. + */ + /** @todo Implement variable MTRR storage. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrPhysMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Implement variable MTRR storage. */ + Assert(pRange->uValue == (idMsr - 0x200) / 2); + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrPhysMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + /* + * Validate the value. + */ + Assert(pRange->uValue == (idMsr - 0x200) / 2); + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(uRawValue); RT_NOREF_PV(pRange); + + uint64_t fInvPhysMask = ~(RT_BIT_64(pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.cMaxPhysAddrWidth) - 1U); + if (fInvPhysMask & uValue) + { + Log(("CPUM: Invalid physical address bits set writing MTRR PhysMask MSR %#x: %#llx (%#llx)\n", + idMsr, uValue, uValue & fInvPhysMask)); + return VERR_CPUM_RAISE_GP_0; + } + + /* + * Store it. + */ + /** @todo Implement variable MTRR storage. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrFixed(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + CPUM_MSR_ASSERT_CPUMCPU_OFFSET_RETURN(pVCpu, pRange, uint64_t, puFixedMtrr); + *puValue = *puFixedMtrr; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrFixed(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + CPUM_MSR_ASSERT_CPUMCPU_OFFSET_RETURN(pVCpu, pRange, uint64_t, puFixedMtrr); + RT_NOREF_PV(idMsr); RT_NOREF_PV(uRawValue); + + for (uint32_t cShift = 0; cShift < 63; cShift += 8) + { + uint8_t uType = (uint8_t)(uValue >> cShift); + if ((uType >= 7) || (uType == 2) || (uType == 3)) + { + Log(("CPUM: Invalid MTRR type at %u:%u in fixed range (%#x/%s): %#llx (%#llx)\n", + cShift + 7, cShift, idMsr, pRange->szName, uValue, uType)); + return VERR_CPUM_RAISE_GP_0; + } + } + *puFixedMtrr = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrDefType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.GuestMsrs.msr.MtrrDefType; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrDefType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + + uint8_t uType = uValue & 0xff; + if ((uType >= 7) || (uType == 2) || (uType == 3)) + { + Log(("CPUM: Invalid MTRR default type value on %s: %#llx (%#llx)\n", pRange->szName, uValue, uType)); + return VERR_CPUM_RAISE_GP_0; + } + + pVCpu->cpum.s.GuestMsrs.msr.MtrrDefType = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32Pat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.msrPAT; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32Pat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + if (CPUMIsPatMsrValid(uValue)) + { + pVCpu->cpum.s.Guest.msrPAT = uValue; + return VINF_SUCCESS; + } + return VERR_CPUM_RAISE_GP_0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SysEnterCs(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.SysEnter.cs; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SysEnterCs(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + + /* Note! We used to mask this by 0xffff, but turns out real HW doesn't and + there are generally 32-bit working bits backing this register. */ + pVCpu->cpum.s.Guest.SysEnter.cs = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SysEnterEsp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.SysEnter.esp; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SysEnterEsp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + if (X86_IS_CANONICAL(uValue)) + { + pVCpu->cpum.s.Guest.SysEnter.esp = uValue; + return VINF_SUCCESS; + } + Log(("CPUM: IA32_SYSENTER_ESP not canonical! %#llx\n", uValue)); + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + return VERR_CPUM_RAISE_GP_0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SysEnterEip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.SysEnter.eip; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SysEnterEip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + if (X86_IS_CANONICAL(uValue)) + { + pVCpu->cpum.s.Guest.SysEnter.eip = uValue; + return VINF_SUCCESS; + } + LogRel(("CPUM: IA32_SYSENTER_EIP not canonical! %#llx\n", uValue)); + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + return VERR_CPUM_RAISE_GP_0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McgCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); +#if 0 /** @todo implement machine checks. */ + *puValue = pRange->uValue & (RT_BIT_64(8) | 0); +#else + *puValue = 0; +#endif + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McgStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement machine checks. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McgStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement machine checks. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McgCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement machine checks. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McgCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement machine checks. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32DebugCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement IA32_DEBUGCTL. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32DebugCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement IA32_DEBUGCTL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SmrrPhysBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement intel SMM. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SmrrPhysBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement intel SMM. */ + return VERR_CPUM_RAISE_GP_0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SmrrPhysMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement intel SMM. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SmrrPhysMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement intel SMM. */ + return VERR_CPUM_RAISE_GP_0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PlatformDcaCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement intel direct cache access (DCA)?? */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PlatformDcaCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement intel direct cache access (DCA)?? */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32CpuDcaCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement intel direct cache access (DCA)?? */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32Dca0Cap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement intel direct cache access (DCA)?? */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32Dca0Cap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement intel direct cache access (DCA)?? */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement IA32_PERFEVTSEL0+. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement IA32_PERFEVTSEL0+. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); + uint64_t uValue = pRange->uValue; + + /* Always provide the max bus ratio for now. XNU expects it. */ + uValue &= ~((UINT64_C(0x1f) << 40) | RT_BIT_64(46)); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM); + uint64_t uTscHz = TMCpuTicksPerSecond(pVM); + uint8_t uTscRatio = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz); + if (uTscRatio > 0x1f) + uTscRatio = 0x1f; + uValue |= (uint64_t)uTscRatio << 40; + + *puValue = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /* Pentium4 allows writing, but all bits are ignored. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement IA32_PERFCTL. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement IA32_PERFCTL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32FixedCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement IA32_FIXED_CTRn (fixed performance counters). */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FixedCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement IA32_FIXED_CTRn (fixed performance counters). */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfCapabilities(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfCapabilities(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32FixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PebsEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PebsEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ClockModulation(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement IA32_CLOCK_MODULATION. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ClockModulation(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement IA32_CLOCK_MODULATION. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ThermInterrupt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement IA32_THERM_INTERRUPT. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ThermInterrupt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement IA32_THERM_STATUS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ThermStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement IA32_THERM_STATUS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ThermStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement IA32_THERM_INTERRUPT. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32Therm2Ctl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement IA32_THERM2_CTL. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32Therm2Ctl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement IA32_THERM2_CTL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MiscEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.GuestMsrs.msr.MiscEnable; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MiscEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); +#ifdef LOG_ENABLED + uint64_t const uOld = pVCpu->cpum.s.GuestMsrs.msr.MiscEnable; +#endif + + /* Unsupported bits are generally ignored and stripped by the MSR range + entry that got us here. So, we just need to preserve fixed bits. */ + pVCpu->cpum.s.GuestMsrs.msr.MiscEnable = uValue + | MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL + | MSR_IA32_MISC_ENABLE_BTS_UNAVAIL; + + Log(("CPUM: IA32_MISC_ENABLE; old=%#llx written=%#llx => %#llx\n", + uOld, uValue, pVCpu->cpum.s.GuestMsrs.msr.MiscEnable)); + + /** @todo Wire IA32_MISC_ENABLE bit 22 to our NT 4 CPUID trick. */ + /** @todo Wire up MSR_IA32_MISC_ENABLE_XD_DISABLE. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McCtlStatusAddrMiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(pRange); + + /** @todo Implement machine check exception injection. */ + switch (idMsr & 3) + { + case 0: + case 1: + *puValue = 0; + break; + + /* The ADDR and MISC registers aren't accessible since the + corresponding STATUS bits are zero. */ + case 2: + Log(("CPUM: Reading IA32_MCi_ADDR %#x -> #GP\n", idMsr)); + return VERR_CPUM_RAISE_GP_0; + case 3: + Log(("CPUM: Reading IA32_MCi_MISC %#x -> #GP\n", idMsr)); + return VERR_CPUM_RAISE_GP_0; + } + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McCtlStatusAddrMiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + switch (idMsr & 3) + { + case 0: + /* Ignore writes to the CTL register. */ + break; + + case 1: + /* According to specs, the STATUS register can only be written to + with the value 0. VBoxCpuReport thinks different for a + Pentium M Dothan, but implementing according to specs now. */ + if (uValue != 0) + { + Log(("CPUM: Writing non-zero value (%#llx) to IA32_MCi_STATUS %#x -> #GP\n", uValue, idMsr)); + return VERR_CPUM_RAISE_GP_0; + } + break; + + /* Specs states that ADDR and MISC can be cleared by writing zeros. + Writing 1s will GP. Need to figure out how this relates to the + ADDRV and MISCV status flags. If writing is independent of those + bits, we need to know whether the CPU really implements them since + that is exposed by writing 0 to them. + Implementing the solution with the fewer GPs for now. */ + case 2: + if (uValue != 0) + { + Log(("CPUM: Writing non-zero value (%#llx) to IA32_MCi_ADDR %#x -> #GP\n", uValue, idMsr)); + return VERR_CPUM_RAISE_GP_0; + } + break; + case 3: + if (uValue != 0) + { + Log(("CPUM: Writing non-zero value (%#llx) to IA32_MCi_MISC %#x -> #GP\n", uValue, idMsr)); + return VERR_CPUM_RAISE_GP_0; + } + break; + } + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McNCtl2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Implement machine check exception injection. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McNCtl2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Implement machine check exception injection. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32DsArea(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement IA32_DS_AREA. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32DsArea(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32TscDeadline(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement TSC deadline timer. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32TscDeadline(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement TSC deadline timer. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32X2ApicN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pRange); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest) + && CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.s.Guest, VMX_PROC_CTLS2_VIRT_X2APIC_MODE)) + { + VBOXSTRICTRC rcStrict = IEMExecVmxVirtApicAccessMsr(pVCpu, idMsr, puValue, false /* fWrite */); + if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR) + return VINF_SUCCESS; + if (rcStrict == VERR_OUT_OF_RANGE) + return VERR_CPUM_RAISE_GP_0; + Assert(rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE); + } +#endif + return APICReadMsr(pVCpu, idMsr, puValue); +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32X2ApicN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest) + && CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.s.Guest, VMX_PROC_CTLS2_VIRT_X2APIC_MODE)) + { + VBOXSTRICTRC rcStrict = IEMExecVmxVirtApicAccessMsr(pVCpu, idMsr, &uValue, true /* fWrite */); + if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR) + return VINF_SUCCESS; + if (rcStrict == VERR_OUT_OF_RANGE) + return VERR_CPUM_RAISE_GP_0; + Assert(rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE); + } +#endif + return APICWriteMsr(pVCpu, idMsr, uValue); +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32DebugInterface(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo IA32_DEBUG_INTERFACE (no docs) */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32DebugInterface(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo IA32_DEBUG_INTERFACE (no docs) */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxBasic(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Basic; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxPinbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.PinCtls.u; + else + *puValue = 0; + return VINF_SUCCESS; +} + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxProcbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.ProcCtls.u; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxExitCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.ExitCtls.u; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxEntryCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.EntryCtls.u; + else + *puValue = 0; + return VINF_SUCCESS; +} + + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxMisc(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Misc; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr0Fixed0(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr0Fixed0; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr0Fixed1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr0Fixed1; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr4Fixed0(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr4Fixed0; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr4Fixed1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr4Fixed1; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxVmcsEnum(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64VmcsEnum; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxProcBasedCtls2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.ProcCtls2.u; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** + * Get fixed IA32_VMX_EPT_VPID_CAP value for PGM and cpumMsrRd_Ia32VmxEptVpidCap. + * + * @returns Fixed IA32_VMX_EPT_VPID_CAP value. + * @param pVCpu The cross context per CPU structure. + */ +VMM_INT_DECL(uint64_t) CPUMGetGuestIa32VmxEptVpidCap(PCVMCPUCC pVCpu) +{ + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + return pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64EptVpidCaps; + return 0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxEptVpidCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = CPUMGetGuestIa32VmxEptVpidCap(pVCpu); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTruePinbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.TruePinCtls.u; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTrueProcbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.TrueProcCtls.u; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTrueExitCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.TrueExitCtls.u; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTrueEntryCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.TrueEntryCtls.u; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxVmFunc(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx) + *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64VmFunc; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SpecCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SpecCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + + /* NB: The STIBP bit can be set even when IBRS is present, regardless of whether STIBP is actually implemented. */ + if (uValue & ~(MSR_IA32_SPEC_CTRL_F_IBRS | MSR_IA32_SPEC_CTRL_F_STIBP)) + { + Log(("CPUM: Invalid IA32_SPEC_CTRL bits (trying to write %#llx)\n", uValue)); + return VERR_CPUM_RAISE_GP_0; + } + + pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PredCmd(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ArchCapabilities(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.GuestMsrs.msr.ArchCaps; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FlushCmd(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + if ((uValue & ~MSR_IA32_FLUSH_CMD_F_L1D) == 0) + return VINF_SUCCESS; + Log(("CPUM: Invalid MSR_IA32_FLUSH_CMD_ bits (trying to write %#llx)\n", uValue)); + return VERR_CPUM_RAISE_GP_0; +} + + + +/* + * AMD64 + * AMD64 + * AMD64 + */ + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64Efer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.msrEFER; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64Efer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + uint64_t uValidatedEfer; + uint64_t const uOldEfer = pVCpu->cpum.s.Guest.msrEFER; + int rc = CPUMIsGuestEferMsrWriteValid(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.s.Guest.cr0, uOldEfer, uValue, &uValidatedEfer); + if (RT_FAILURE(rc)) + return VERR_CPUM_RAISE_GP_0; + + CPUMSetGuestEferMsrNoChecks(pVCpu, uOldEfer, uValidatedEfer); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64SyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.msrSTAR; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64SyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + pVCpu->cpum.s.Guest.msrSTAR = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64LongSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.msrLSTAR; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64LongSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + if (!X86_IS_CANONICAL(uValue)) + { + Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue)); + return VERR_CPUM_RAISE_GP_0; + } + pVCpu->cpum.s.Guest.msrLSTAR = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64CompSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.msrCSTAR; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64CompSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + if (!X86_IS_CANONICAL(uValue)) + { + Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue)); + return VERR_CPUM_RAISE_GP_0; + } + pVCpu->cpum.s.Guest.msrCSTAR = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64SyscallFlagMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.msrSFMASK; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64SyscallFlagMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + pVCpu->cpum.s.Guest.msrSFMASK = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64FsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.fs.u64Base; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64FsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + pVCpu->cpum.s.Guest.fs.u64Base = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64GsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.gs.u64Base; + return VINF_SUCCESS; +} + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64GsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + pVCpu->cpum.s.Guest.gs.u64Base = uValue; + return VINF_SUCCESS; +} + + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64KernelGsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.msrKERNELGSBASE; + return VINF_SUCCESS; +} + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64KernelGsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + pVCpu->cpum.s.Guest.msrKERNELGSBASE = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64TscAux(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.GuestMsrs.msr.TscAux; + return VINF_SUCCESS; +} + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64TscAux(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + pVCpu->cpum.s.GuestMsrs.msr.TscAux = uValue; + return VINF_SUCCESS; +} + + +/* + * Intel specific + * Intel specific + * Intel specific + */ + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelEblCrPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo recalc clock frequency ratio? */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelEblCrPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Write EBL_CR_POWERON: Remember written bits. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7CoreThreadCount(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + + /* Note! According to cpuid_set_info in XNU (10.7.0), Westmere CPU only + have a 4-bit core count. */ + uint16_t cCores = pVCpu->CTX_SUFF(pVM)->cCpus; + uint16_t cThreads = cCores; /** @todo hyper-threading. */ + *puValue = RT_MAKE_U32(cThreads, cCores); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP4EbcHardPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo P4 hard power on config */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP4EbcHardPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo P4 hard power on config */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP4EbcSoftPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo P4 soft power on config */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP4EbcSoftPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo P4 soft power on config */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP4EbcFrequencyId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); + + uint64_t uValue; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM); + if (pVM->cpum.s.GuestFeatures.uModel >= 2) + { + if (uScalableBusHz <= CPUM_SBUSFREQ_100MHZ && pVM->cpum.s.GuestFeatures.uModel <= 2) + { + uScalableBusHz = CPUM_SBUSFREQ_100MHZ; + uValue = 0; + } + else if (uScalableBusHz <= CPUM_SBUSFREQ_133MHZ) + { + uScalableBusHz = CPUM_SBUSFREQ_133MHZ; + uValue = 1; + } + else if (uScalableBusHz <= CPUM_SBUSFREQ_167MHZ) + { + uScalableBusHz = CPUM_SBUSFREQ_167MHZ; + uValue = 3; + } + else if (uScalableBusHz <= CPUM_SBUSFREQ_200MHZ) + { + uScalableBusHz = CPUM_SBUSFREQ_200MHZ; + uValue = 2; + } + else if (uScalableBusHz <= CPUM_SBUSFREQ_267MHZ && pVM->cpum.s.GuestFeatures.uModel > 2) + { + uScalableBusHz = CPUM_SBUSFREQ_267MHZ; + uValue = 0; + } + else + { + uScalableBusHz = CPUM_SBUSFREQ_333MHZ; + uValue = 6; + } + uValue <<= 16; + + uint64_t uTscHz = TMCpuTicksPerSecond(pVM); + uint8_t uTscRatio = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz); + uValue |= (uint32_t)uTscRatio << 24; + + uValue |= pRange->uValue & ~UINT64_C(0xff0f0000); + } + else + { + /* Probably more stuff here, but intel doesn't want to tell us. */ + uValue = pRange->uValue; + uValue &= ~(RT_BIT_64(21) | RT_BIT_64(22) | RT_BIT_64(23)); /* 100 MHz is only documented value */ + } + + *puValue = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP4EbcFrequencyId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo P4 bus frequency config */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP6FsbFrequency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); + + /* Convert the scalable bus frequency to the encoding in the intel manual (for core+). */ + uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVCpu->CTX_SUFF(pVM)); + if (uScalableBusHz <= CPUM_SBUSFREQ_100MHZ) + *puValue = 5; + else if (uScalableBusHz <= CPUM_SBUSFREQ_133MHZ) + *puValue = 1; + else if (uScalableBusHz <= CPUM_SBUSFREQ_167MHZ) + *puValue = 3; + else if (uScalableBusHz <= CPUM_SBUSFREQ_200MHZ) + *puValue = 2; + else if (uScalableBusHz <= CPUM_SBUSFREQ_267MHZ) + *puValue = 0; + else if (uScalableBusHz <= CPUM_SBUSFREQ_333MHZ) + *puValue = 4; + else /*if (uScalableBusHz <= CPUM_SBUSFREQ_400MHZ)*/ + *puValue = 6; + + *puValue |= pRange->uValue & ~UINT64_C(0x7); + + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelPlatformInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + + /* Just indicate a fixed TSC, no turbo boost, no programmable anything. */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM); + uint64_t uTscHz = TMCpuTicksPerSecond(pVM); + uint8_t uTscRatio = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz); + uint64_t uValue = ((uint32_t)uTscRatio << 8) /* TSC invariant frequency. */ + | ((uint64_t)uTscRatio << 40); /* The max turbo frequency. */ + + /* Ivy bridge has a minimum operating ratio as well. */ + if (true) /** @todo detect sandy bridge. */ + uValue |= (uint64_t)uTscRatio << 48; + + *puValue = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelFlexRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); + + uint64_t uValue = pRange->uValue & ~UINT64_C(0x1ff00); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM); + uint64_t uTscHz = TMCpuTicksPerSecond(pVM); + uint8_t uTscRatio = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz); + uValue |= (uint32_t)uTscRatio << 8; + + *puValue = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelFlexRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement writing MSR_FLEX_RATIO. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelPkgCStConfigControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelPkgCStConfigControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + + if (pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl & RT_BIT_64(15)) + { + Log(("CPUM: WRMSR %#x (%s), %#llx: Write protected -> #GP\n", idMsr, pRange->szName, uValue)); + return VERR_CPUM_RAISE_GP_0; + } +#if 0 /** @todo check what real (old) hardware does. */ + if ((uValue & 7) >= 5) + { + Log(("CPUM: WRMSR %#x (%s), %#llx: Invalid limit (%d) -> #GP\n", idMsr, pRange->szName, uValue, (uint32_t)(uValue & 7))); + return VERR_CPUM_RAISE_GP_0; + } +#endif + pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelPmgIoCaptureBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement I/O mwait wakeup. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelPmgIoCaptureBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement I/O mwait wakeup. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchFromToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement last branch records. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchFromToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement last branch records. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchFromN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement last branch records. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchFromN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + /** @todo implement last branch records. */ + /** @todo Probing indicates that bit 63 is settable on SandyBridge, at least + * if the rest of the bits are zero. Automatic sign extending? + * Investigate! */ + if (!X86_IS_CANONICAL(uValue)) + { + Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue)); + return VERR_CPUM_RAISE_GP_0; + } + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement last branch records. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement last branch records. */ + /** @todo Probing indicates that bit 63 is settable on SandyBridge, at least + * if the rest of the bits are zero. Automatic sign extending? + * Investigate! */ + if (!X86_IS_CANONICAL(uValue)) + { + Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue)); + return VERR_CPUM_RAISE_GP_0; + } + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchTos(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement last branch records. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchTos(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement last branch records. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelBblCrCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelBblCrCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelBblCrCtl3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelBblCrCtl3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7TemperatureTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7TemperatureTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7MsrOffCoreResponseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo machine check. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7MsrOffCoreResponseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo machine check. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7MiscPwrMgmt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7MiscPwrMgmt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP6CrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); + int rc = CPUMGetGuestCRx(pVCpu, pRange->uValue, puValue); + AssertRC(rc); + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP6CrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /* This CRx interface differs from the MOV CRx, GReg interface in that + #GP(0) isn't raised if unsupported bits are written to. Instead they + are simply ignored and masked off. (Pentium M Dothan) */ + /** @todo Implement MSR_P6_CRx writing. Too much effort for very little, if + * any, gain. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCpuId1FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement CPUID masking. */ + *puValue = UINT64_MAX; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCpuId1FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement CPUID masking. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCpuId1FeatureMaskEax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement CPUID masking. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCpuId1FeatureMaskEax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement CPUID masking. */ + return VINF_SUCCESS; +} + + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCpuId80000001FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement CPUID masking. */ + *puValue = UINT64_MAX; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCpuId80000001FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement CPUID masking. */ + return VINF_SUCCESS; +} + + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyAesNiCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement AES-NI. */ + *puValue = 3; /* Bit 0 is lock bit, bit 1 disables AES-NI. That's what they say. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyAesNiCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement AES-NI. */ + return VERR_CPUM_RAISE_GP_0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7TurboRatioLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo implement intel C states. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7TurboRatioLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement intel C states. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7LbrSelect(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement last-branch-records. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7LbrSelect(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement last-branch-records. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyErrorControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement memory error injection (MSR_ERROR_CONTROL). */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyErrorControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement memory error injection (MSR_ERROR_CONTROL). */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7VirtualLegacyWireCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo implement memory VLW? */ + *puValue = pRange->uValue; + /* Note: A20M is known to be bit 1 as this was disclosed in spec update + AAJ49/AAK51/????, which documents the inversion of this bit. The + Sandy bridge CPU here has value 0x74, so it probably doesn't have a BIOS + that correct things. Some guesses at the other bits: + bit 2 = INTR + bit 4 = SMI + bit 5 = INIT + bit 6 = NMI */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7PowerCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7PowerCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel power management */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyPebsNumAlt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyPebsNumAlt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7PebsLdLat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7PebsLdLat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7PkgCnResidencyN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7CoreCnResidencyN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyVrCurrentConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyVrCurrentConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyVrMiscConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyVrMiscConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyRaplPowerUnit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo intel RAPL. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyRaplPowerUnit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /* Note! This is documented as read only and except for a Silvermont sample has + always been classified as read only. This is just here to make it compile. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyPkgCnIrtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyPkgCnIrtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel power management. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyPkgC2Residency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyPkgC2Residency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /* Note! This is documented as read only and except for a Silvermont sample has + always been classified as read only. This is just here to make it compile. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel RAPL. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPkgPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel RAPL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgEnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgPerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgPowerInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel RAPL. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplDramPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel RAPL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramEnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramPerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramPowerInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel RAPL. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp0PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel RAPL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0EnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel RAPL. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp0Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel RAPL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0PerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp1PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel RAPL. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp1PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel RAPL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp1EnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp1Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel RAPL. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp1Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel RAPL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpNominal(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo intel power management. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpLevel1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo intel power management. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpLevel2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo intel power management. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7IvyConfigTdpControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel power management. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyTurboActivationRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo intel power management. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7IvyTurboActivationRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo intel power management. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo uncore msrs. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo uncore msrs. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo uncore msrs. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo uncore msrs. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo uncore msrs. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo uncore msrs. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfFixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo uncore msrs. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfFixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo uncore msrs. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfFixedCtr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo uncore msrs. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfFixedCtr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo uncore msrs. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncCBoxConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo uncore msrs. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncArbPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo uncore msrs. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncArbPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo uncore msrs. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncArbPerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo uncore msrs. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncArbPerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo uncore msrs. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SmiCount(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + + /* + * 31:0 is SMI count (read only), 63:32 reserved. + * Since we don't do SMI, the count is always zero. + */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore2EmttmCrTablesN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo implement enhanced multi thread termal monitoring? */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore2EmttmCrTablesN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement enhanced multi thread termal monitoring? */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore2SmmCStMiscInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo SMM & C-states? */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore2SmmCStMiscInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo SMM & C-states? */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore1ExtConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Core1&2 EXT_CONFIG (whatever that is)? */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore1ExtConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Core1&2 EXT_CONFIG (whatever that is)? */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore1DtsCalControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Core1&2(?) DTS_CAL_CTRL (whatever that is)? */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore1DtsCalControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Core1&2(?) DTS_CAL_CTRL (whatever that is)? */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore2PeciControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Core2+ platform environment control interface control register? */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore2PeciControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Core2+ platform environment control interface control register? */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelAtSilvCoreC1Recidency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = 0; + return VINF_SUCCESS; +} + + +/* + * Multiple vendor P6 MSRs. + * Multiple vendor P6 MSRs. + * Multiple vendor P6 MSRs. + * + * These MSRs were introduced with the P6 but not elevated to architectural + * MSRs, despite other vendors implementing them. + */ + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastBranchFromIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /* AMD seems to just record RIP, while intel claims to record RIP+CS.BASE + if I read the docs correctly, thus the need for separate functions. */ + /** @todo implement last branch records. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastBranchToIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement last branch records. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastIntFromIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement last exception records. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_P6LastIntFromIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement last exception records. */ + /* Note! On many CPUs, the high bit of the 0x000001dd register is always writable, even when the result is + a non-cannonical address. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastIntToIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo implement last exception records. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_P6LastIntToIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo implement last exception records. */ + return VINF_SUCCESS; +} + + + +/* + * AMD specific + * AMD specific + * AMD specific + */ + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hTscRate(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Implement TscRateMsr */ + *puValue = RT_MAKE_U64(0, 1); /* 1.0 = reset value. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hTscRate(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Implement TscRateMsr */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hLwpCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */ + /* Note: Only listes in BKDG for Family 15H. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hLwpCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hLwpCbAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */ + /* Note: Only listes in BKDG for Family 15H. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hLwpCbAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hMc4MiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo machine check. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hMc4MiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo machine check. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8PerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD performance events. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8PerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD performance events. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8PerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD performance events. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8PerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD performance events. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SysCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD SYS_CFG */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SysCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SYS_CFG */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8HwCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD HW_CFG */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8HwCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD HW_CFG */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IorrBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IorrMask/IorrBase */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IorrBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IorrMask/IorrBase */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IorrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IorrMask/IorrBase */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IorrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IorrMask/IorrBase */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8TopOfMemN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = 0; + /** @todo return 4GB - RamHoleSize here for TOPMEM. Figure out what to return + * for TOPMEM2. */ + //if (pRange->uValue == 0) + // *puValue = _4G - RamHoleSize; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8TopOfMemN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD TOPMEM and TOPMEM2/TOM2. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8NbCfg1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD NB_CFG1 */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8NbCfg1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD NB_CFG1 */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8McXcptRedir(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo machine check. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8McXcptRedir(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo machine check. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuNameN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); + PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), pRange->uValue / 2 + 0x80000001); + if (pLeaf) + { + if (!(pRange->uValue & 1)) + *puValue = RT_MAKE_U64(pLeaf->uEax, pLeaf->uEbx); + else + *puValue = RT_MAKE_U64(pLeaf->uEcx, pLeaf->uEdx); + } + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuNameN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Remember guest programmed CPU name. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8HwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD HTC. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8HwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD HTC. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD STC. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD STC. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8FidVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD FIDVID_CTL. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8FidVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD FIDVID_CTL. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8FidVidStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD FIDVID_STATUS. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8McCtlMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD MC. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8McCtlMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD MC. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmiOnIoTrapN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SMM/SMI and I/O trap. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmiOnIoTrapN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SMM/SMI and I/O trap. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmiOnIoTrapCtlSts(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SMM/SMI and I/O trap. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmiOnIoTrapCtlSts(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SMM/SMI and I/O trap. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IntPendingMessage(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Interrupt pending message. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IntPendingMessage(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Interrupt pending message. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmiTriggerIoCycle(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SMM/SMI and trigger I/O cycle. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmiTriggerIoCycle(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SMM/SMI and trigger I/O cycle. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hMmioCfgBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD MMIO Configuration base address. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hMmioCfgBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD MMIO Configuration base address. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hTrapCtlMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD 0xc0010059. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hTrapCtlMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD 0xc0010059. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateCurLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD P-states. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD P-states. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hPStateControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD P-states. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD P-states. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hPStateStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD P-states. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD P-states. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hPStateN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD P-states. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCofVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD P-states. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCofVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD P-states. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCofVidStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD P-states. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCofVidStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /* Note! Writing 0 seems to not GP, not sure if it does anything to the value... */ + /** @todo AMD P-states. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCStateIoBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD C-states. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCStateIoBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD C-states. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCpuWatchdogTimer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD machine checks. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCpuWatchdogTimer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD machine checks. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SMM. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SMM. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SMM. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SMM. */ + return VINF_SUCCESS; +} + + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SMM. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SMM. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8VmCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + PVM pVM = pVCpu->CTX_SUFF(pVM); + if (pVM->cpum.s.GuestFeatures.fSvm) + *puValue = MSR_K8_VM_CR_LOCK; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8VmCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + PVM pVM = pVCpu->CTX_SUFF(pVM); + if (pVM->cpum.s.GuestFeatures.fSvm) + { + /* Silently ignore writes to LOCK and SVM_DISABLE bit when the LOCK bit is set (see cpumMsrRd_AmdK8VmCr). */ + if (uValue & (MSR_K8_VM_CR_DPD | MSR_K8_VM_CR_R_INIT | MSR_K8_VM_CR_DIS_A20M)) + return VERR_CPUM_RAISE_GP_0; + return VINF_SUCCESS; + } + return VERR_CPUM_RAISE_GP_0; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IgnNe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IGNNE\# control. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IgnNe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IGNNE\# control. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SMM. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SMM. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8VmHSavePa(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + *puValue = pVCpu->cpum.s.Guest.hwvirt.svm.uMsrHSavePa; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8VmHSavePa(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue); + if (uValue & UINT64_C(0xfff)) + { + Log(("CPUM: Invalid setting of low 12 bits set writing host-state save area MSR %#x: %#llx\n", idMsr, uValue)); + return VERR_CPUM_RAISE_GP_0; + } + + uint64_t fInvPhysMask = ~(RT_BIT_64(pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.cMaxPhysAddrWidth) - 1U); + if (fInvPhysMask & uValue) + { + Log(("CPUM: Invalid physical address bits set writing host-state save area MSR %#x: %#llx (%#llx)\n", + idMsr, uValue, uValue & fInvPhysMask)); + return VERR_CPUM_RAISE_GP_0; + } + + pVCpu->cpum.s.Guest.hwvirt.svm.uMsrHSavePa = uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hVmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SVM. */ + *puValue = 0; /* RAZ */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hVmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SVM. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hSmmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SMM. */ + *puValue = 0; /* RAZ */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hSmmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SMM. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hLocalSmiStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD SMM/SMI. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hLocalSmiStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD SMM/SMI. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hOsVisWrkIdLength(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo AMD OS visible workaround. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hOsVisWrkIdLength(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD OS visible workaround. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hOsVisWrkStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD OS visible workaround. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hOsVisWrkStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD OS visible workaround. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam16hL2IPerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD L2I performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam16hL2IPerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD L2I performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam16hL2IPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD L2I performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam16hL2IPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD L2I performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hNorthbridgePerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD Northbridge performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hNorthbridgePerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD Northbridge performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hNorthbridgePerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD Northbridge performance counters. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hNorthbridgePerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD Northbridge performance counters. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7MicrocodeCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo Undocumented register only seen mentioned in fam15h erratum \#608. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7MicrocodeCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo Undocumented register only seen mentioned in fam15h erratum \#608. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7ClusterIdMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo Undocumented register only seen mentioned in fam16h BKDG r3.00 when + * describing EBL_CR_POWERON. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7ClusterIdMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo Undocumented register only seen mentioned in fam16h BKDG r3.00 when + * describing EBL_CR_POWERON. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlStd07hEbax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + bool fIgnored; + PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeafEx(pVCpu->CTX_SUFF(pVM), 0x00000007, 0, &fIgnored); + if (pLeaf) + *puValue = RT_MAKE_U64(pLeaf->uEbx, pLeaf->uEax); + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlStd07hEbax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Changing CPUID leaf 7/0. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlStd06hEcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), 0x00000006); + if (pLeaf) + *puValue = pLeaf->uEcx; + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlStd06hEcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Changing CPUID leaf 6. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlStd01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), 0x00000001); + if (pLeaf) + *puValue = RT_MAKE_U64(pLeaf->uEdx, pLeaf->uEcx); + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlStd01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Changing CPUID leaf 0x80000001. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlExt01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), 0x80000001); + if (pLeaf) + *puValue = RT_MAKE_U64(pLeaf->uEdx, pLeaf->uEcx); + else + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlExt01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Changing CPUID leaf 0x80000001. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8PatchLevel(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); + /** @todo Fake AMD microcode patching. */ + *puValue = pRange->uValue; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8PatchLoader(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Fake AMD microcode patching. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DebugStatusMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DebugStatusMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BHTraceBaseMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BHTraceBaseMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BHTracePtrMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BHTracePtrMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BHTraceLimitMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BHTraceLimitMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7HardwareDebugToolCfgMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7HardwareDebugToolCfgMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7FastFlushCountMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7FastFlushCountMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo undocumented */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7NodeId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD node ID and bios scratch. */ + *puValue = 0; /* nodeid = 0; nodes-per-cpu = 1 */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7NodeId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD node ID and bios scratch. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DrXAddrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD DRx address masking (range breakpoints). */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DrXAddrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD DRx address masking (range breakpoints). */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7Dr0DataMatchMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD undocument debugging features. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7Dr0DataMatchMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD undocument debugging features. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7Dr0DataMaskMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD undocument debugging features. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7Dr0DataMaskMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD undocument debugging features. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7LoadStoreCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD load-store config. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7LoadStoreCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD load-store config. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7InstrCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD instruction cache config. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7InstrCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD instruction cache config. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DataCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD data cache config. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DataCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD data cache config. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BusUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD bus unit config. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BusUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo AMD bus unit config. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DebugCtl2Maybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo Undocument AMD debug control register \#2. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DebugCtl2Maybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older + * cpus. Need to be explored and verify K7 presence. */ + /** @todo Undocument AMD debug control register \#2. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hFpuCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD FPU config. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hFpuCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD FPU config. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hDecoderCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD decoder config. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hDecoderCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD decoder config. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hBusUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /* Note! 10h and 16h */ + /** @todo AMD bus unit config. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hBusUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /* Note! 10h and 16h */ + /** @todo AMD bus unit config. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hCombUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD unit config. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hCombUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD unit config. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hCombUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD unit config 2. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hCombUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD unit config 2. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hCombUnitCfg3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD combined unit config 3. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hCombUnitCfg3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD combined unit config 3. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hExecUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD execution unit config. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hExecUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD execution unit config. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hLoadStoreCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD load-store config 2. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hLoadStoreCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD load-store config 2. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsFetchCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsFetchCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsFetchLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsFetchLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsFetchPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsFetchPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpExecCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpExecCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpRip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpRip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + if (!X86_IS_CANONICAL(uValue)) + { + Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue)); + return VERR_CPUM_RAISE_GP_0; + } + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpData(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpData(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpData2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpData2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpData3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpData3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsDcLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsDcLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + if (!X86_IS_CANONICAL(uValue)) + { + Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue)); + return VERR_CPUM_RAISE_GP_0; + } + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsDcPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsDcPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam14hIbsBrTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); + /** @todo AMD IBS. */ + *puValue = 0; + return VINF_SUCCESS; +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam14hIbsBrTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue); + /** @todo AMD IBS. */ + if (!X86_IS_CANONICAL(uValue)) + { + Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue)); + return VERR_CPUM_RAISE_GP_0; + } + return VINF_SUCCESS; +} + + + +/* + * GIM MSRs. + * GIM MSRs. + * GIM MSRs. + */ + + +/** @callback_method_impl{FNCPUMRDMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Gim(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ +#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) + /* Raise #GP(0) like a physical CPU would since the nested-hypervisor hasn't intercept these MSRs. */ + if ( CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.s.Guest) + || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest)) + return VERR_CPUM_RAISE_GP_0; +#endif + return GIMReadMsr(pVCpu, idMsr, pRange, puValue); +} + + +/** @callback_method_impl{FNCPUMWRMSR} */ +static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Gim(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ +#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) + /* Raise #GP(0) like a physical CPU would since the nested-hypervisor hasn't intercept these MSRs. */ + if ( CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.s.Guest) + || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest)) + return VERR_CPUM_RAISE_GP_0; +#endif + return GIMWriteMsr(pVCpu, idMsr, pRange, uValue, uRawValue); +} + + +/** + * MSR read function table. + */ +static const struct READMSRCLANG11WEIRDNOTHROW { PFNCPUMRDMSR pfnRdMsr; } g_aCpumRdMsrFns[kCpumMsrRdFn_End] = +{ + { NULL }, /* Invalid */ + { cpumMsrRd_FixedValue }, + { NULL }, /* Alias */ + { cpumMsrRd_WriteOnly }, + { cpumMsrRd_Ia32P5McAddr }, + { cpumMsrRd_Ia32P5McType }, + { cpumMsrRd_Ia32TimestampCounter }, + { cpumMsrRd_Ia32PlatformId }, + { cpumMsrRd_Ia32ApicBase }, + { cpumMsrRd_Ia32FeatureControl }, + { cpumMsrRd_Ia32BiosSignId }, + { cpumMsrRd_Ia32SmmMonitorCtl }, + { cpumMsrRd_Ia32PmcN }, + { cpumMsrRd_Ia32MonitorFilterLineSize }, + { cpumMsrRd_Ia32MPerf }, + { cpumMsrRd_Ia32APerf }, + { cpumMsrRd_Ia32MtrrCap }, + { cpumMsrRd_Ia32MtrrPhysBaseN }, + { cpumMsrRd_Ia32MtrrPhysMaskN }, + { cpumMsrRd_Ia32MtrrFixed }, + { cpumMsrRd_Ia32MtrrDefType }, + { cpumMsrRd_Ia32Pat }, + { cpumMsrRd_Ia32SysEnterCs }, + { cpumMsrRd_Ia32SysEnterEsp }, + { cpumMsrRd_Ia32SysEnterEip }, + { cpumMsrRd_Ia32McgCap }, + { cpumMsrRd_Ia32McgStatus }, + { cpumMsrRd_Ia32McgCtl }, + { cpumMsrRd_Ia32DebugCtl }, + { cpumMsrRd_Ia32SmrrPhysBase }, + { cpumMsrRd_Ia32SmrrPhysMask }, + { cpumMsrRd_Ia32PlatformDcaCap }, + { cpumMsrRd_Ia32CpuDcaCap }, + { cpumMsrRd_Ia32Dca0Cap }, + { cpumMsrRd_Ia32PerfEvtSelN }, + { cpumMsrRd_Ia32PerfStatus }, + { cpumMsrRd_Ia32PerfCtl }, + { cpumMsrRd_Ia32FixedCtrN }, + { cpumMsrRd_Ia32PerfCapabilities }, + { cpumMsrRd_Ia32FixedCtrCtrl }, + { cpumMsrRd_Ia32PerfGlobalStatus }, + { cpumMsrRd_Ia32PerfGlobalCtrl }, + { cpumMsrRd_Ia32PerfGlobalOvfCtrl }, + { cpumMsrRd_Ia32PebsEnable }, + { cpumMsrRd_Ia32ClockModulation }, + { cpumMsrRd_Ia32ThermInterrupt }, + { cpumMsrRd_Ia32ThermStatus }, + { cpumMsrRd_Ia32Therm2Ctl }, + { cpumMsrRd_Ia32MiscEnable }, + { cpumMsrRd_Ia32McCtlStatusAddrMiscN }, + { cpumMsrRd_Ia32McNCtl2 }, + { cpumMsrRd_Ia32DsArea }, + { cpumMsrRd_Ia32TscDeadline }, + { cpumMsrRd_Ia32X2ApicN }, + { cpumMsrRd_Ia32DebugInterface }, + { cpumMsrRd_Ia32VmxBasic }, + { cpumMsrRd_Ia32VmxPinbasedCtls }, + { cpumMsrRd_Ia32VmxProcbasedCtls }, + { cpumMsrRd_Ia32VmxExitCtls }, + { cpumMsrRd_Ia32VmxEntryCtls }, + { cpumMsrRd_Ia32VmxMisc }, + { cpumMsrRd_Ia32VmxCr0Fixed0 }, + { cpumMsrRd_Ia32VmxCr0Fixed1 }, + { cpumMsrRd_Ia32VmxCr4Fixed0 }, + { cpumMsrRd_Ia32VmxCr4Fixed1 }, + { cpumMsrRd_Ia32VmxVmcsEnum }, + { cpumMsrRd_Ia32VmxProcBasedCtls2 }, + { cpumMsrRd_Ia32VmxEptVpidCap }, + { cpumMsrRd_Ia32VmxTruePinbasedCtls }, + { cpumMsrRd_Ia32VmxTrueProcbasedCtls }, + { cpumMsrRd_Ia32VmxTrueExitCtls }, + { cpumMsrRd_Ia32VmxTrueEntryCtls }, + { cpumMsrRd_Ia32VmxVmFunc }, + { cpumMsrRd_Ia32SpecCtrl }, + { cpumMsrRd_Ia32ArchCapabilities }, + + { cpumMsrRd_Amd64Efer }, + { cpumMsrRd_Amd64SyscallTarget }, + { cpumMsrRd_Amd64LongSyscallTarget }, + { cpumMsrRd_Amd64CompSyscallTarget }, + { cpumMsrRd_Amd64SyscallFlagMask }, + { cpumMsrRd_Amd64FsBase }, + { cpumMsrRd_Amd64GsBase }, + { cpumMsrRd_Amd64KernelGsBase }, + { cpumMsrRd_Amd64TscAux }, + + { cpumMsrRd_IntelEblCrPowerOn }, + { cpumMsrRd_IntelI7CoreThreadCount }, + { cpumMsrRd_IntelP4EbcHardPowerOn }, + { cpumMsrRd_IntelP4EbcSoftPowerOn }, + { cpumMsrRd_IntelP4EbcFrequencyId }, + { cpumMsrRd_IntelP6FsbFrequency }, + { cpumMsrRd_IntelPlatformInfo }, + { cpumMsrRd_IntelFlexRatio }, + { cpumMsrRd_IntelPkgCStConfigControl }, + { cpumMsrRd_IntelPmgIoCaptureBase }, + { cpumMsrRd_IntelLastBranchFromToN }, + { cpumMsrRd_IntelLastBranchFromN }, + { cpumMsrRd_IntelLastBranchToN }, + { cpumMsrRd_IntelLastBranchTos }, + { cpumMsrRd_IntelBblCrCtl }, + { cpumMsrRd_IntelBblCrCtl3 }, + { cpumMsrRd_IntelI7TemperatureTarget }, + { cpumMsrRd_IntelI7MsrOffCoreResponseN }, + { cpumMsrRd_IntelI7MiscPwrMgmt }, + { cpumMsrRd_IntelP6CrN }, + { cpumMsrRd_IntelCpuId1FeatureMaskEcdx }, + { cpumMsrRd_IntelCpuId1FeatureMaskEax }, + { cpumMsrRd_IntelCpuId80000001FeatureMaskEcdx }, + { cpumMsrRd_IntelI7SandyAesNiCtl }, + { cpumMsrRd_IntelI7TurboRatioLimit }, + { cpumMsrRd_IntelI7LbrSelect }, + { cpumMsrRd_IntelI7SandyErrorControl }, + { cpumMsrRd_IntelI7VirtualLegacyWireCap }, + { cpumMsrRd_IntelI7PowerCtl }, + { cpumMsrRd_IntelI7SandyPebsNumAlt }, + { cpumMsrRd_IntelI7PebsLdLat }, + { cpumMsrRd_IntelI7PkgCnResidencyN }, + { cpumMsrRd_IntelI7CoreCnResidencyN }, + { cpumMsrRd_IntelI7SandyVrCurrentConfig }, + { cpumMsrRd_IntelI7SandyVrMiscConfig }, + { cpumMsrRd_IntelI7SandyRaplPowerUnit }, + { cpumMsrRd_IntelI7SandyPkgCnIrtlN }, + { cpumMsrRd_IntelI7SandyPkgC2Residency }, + { cpumMsrRd_IntelI7RaplPkgPowerLimit }, + { cpumMsrRd_IntelI7RaplPkgEnergyStatus }, + { cpumMsrRd_IntelI7RaplPkgPerfStatus }, + { cpumMsrRd_IntelI7RaplPkgPowerInfo }, + { cpumMsrRd_IntelI7RaplDramPowerLimit }, + { cpumMsrRd_IntelI7RaplDramEnergyStatus }, + { cpumMsrRd_IntelI7RaplDramPerfStatus }, + { cpumMsrRd_IntelI7RaplDramPowerInfo }, + { cpumMsrRd_IntelI7RaplPp0PowerLimit }, + { cpumMsrRd_IntelI7RaplPp0EnergyStatus }, + { cpumMsrRd_IntelI7RaplPp0Policy }, + { cpumMsrRd_IntelI7RaplPp0PerfStatus }, + { cpumMsrRd_IntelI7RaplPp1PowerLimit }, + { cpumMsrRd_IntelI7RaplPp1EnergyStatus }, + { cpumMsrRd_IntelI7RaplPp1Policy }, + { cpumMsrRd_IntelI7IvyConfigTdpNominal }, + { cpumMsrRd_IntelI7IvyConfigTdpLevel1 }, + { cpumMsrRd_IntelI7IvyConfigTdpLevel2 }, + { cpumMsrRd_IntelI7IvyConfigTdpControl }, + { cpumMsrRd_IntelI7IvyTurboActivationRatio }, + { cpumMsrRd_IntelI7UncPerfGlobalCtrl }, + { cpumMsrRd_IntelI7UncPerfGlobalStatus }, + { cpumMsrRd_IntelI7UncPerfGlobalOvfCtrl }, + { cpumMsrRd_IntelI7UncPerfFixedCtrCtrl }, + { cpumMsrRd_IntelI7UncPerfFixedCtr }, + { cpumMsrRd_IntelI7UncCBoxConfig }, + { cpumMsrRd_IntelI7UncArbPerfCtrN }, + { cpumMsrRd_IntelI7UncArbPerfEvtSelN }, + { cpumMsrRd_IntelI7SmiCount }, + { cpumMsrRd_IntelCore2EmttmCrTablesN }, + { cpumMsrRd_IntelCore2SmmCStMiscInfo }, + { cpumMsrRd_IntelCore1ExtConfig }, + { cpumMsrRd_IntelCore1DtsCalControl }, + { cpumMsrRd_IntelCore2PeciControl }, + { cpumMsrRd_IntelAtSilvCoreC1Recidency }, + + { cpumMsrRd_P6LastBranchFromIp }, + { cpumMsrRd_P6LastBranchToIp }, + { cpumMsrRd_P6LastIntFromIp }, + { cpumMsrRd_P6LastIntToIp }, + + { cpumMsrRd_AmdFam15hTscRate }, + { cpumMsrRd_AmdFam15hLwpCfg }, + { cpumMsrRd_AmdFam15hLwpCbAddr }, + { cpumMsrRd_AmdFam10hMc4MiscN }, + { cpumMsrRd_AmdK8PerfCtlN }, + { cpumMsrRd_AmdK8PerfCtrN }, + { cpumMsrRd_AmdK8SysCfg }, + { cpumMsrRd_AmdK8HwCr }, + { cpumMsrRd_AmdK8IorrBaseN }, + { cpumMsrRd_AmdK8IorrMaskN }, + { cpumMsrRd_AmdK8TopOfMemN }, + { cpumMsrRd_AmdK8NbCfg1 }, + { cpumMsrRd_AmdK8McXcptRedir }, + { cpumMsrRd_AmdK8CpuNameN }, + { cpumMsrRd_AmdK8HwThermalCtrl }, + { cpumMsrRd_AmdK8SwThermalCtrl }, + { cpumMsrRd_AmdK8FidVidControl }, + { cpumMsrRd_AmdK8FidVidStatus }, + { cpumMsrRd_AmdK8McCtlMaskN }, + { cpumMsrRd_AmdK8SmiOnIoTrapN }, + { cpumMsrRd_AmdK8SmiOnIoTrapCtlSts }, + { cpumMsrRd_AmdK8IntPendingMessage }, + { cpumMsrRd_AmdK8SmiTriggerIoCycle }, + { cpumMsrRd_AmdFam10hMmioCfgBaseAddr }, + { cpumMsrRd_AmdFam10hTrapCtlMaybe }, + { cpumMsrRd_AmdFam10hPStateCurLimit }, + { cpumMsrRd_AmdFam10hPStateControl }, + { cpumMsrRd_AmdFam10hPStateStatus }, + { cpumMsrRd_AmdFam10hPStateN }, + { cpumMsrRd_AmdFam10hCofVidControl }, + { cpumMsrRd_AmdFam10hCofVidStatus }, + { cpumMsrRd_AmdFam10hCStateIoBaseAddr }, + { cpumMsrRd_AmdFam10hCpuWatchdogTimer }, + { cpumMsrRd_AmdK8SmmBase }, + { cpumMsrRd_AmdK8SmmAddr }, + { cpumMsrRd_AmdK8SmmMask }, + { cpumMsrRd_AmdK8VmCr }, + { cpumMsrRd_AmdK8IgnNe }, + { cpumMsrRd_AmdK8SmmCtl }, + { cpumMsrRd_AmdK8VmHSavePa }, + { cpumMsrRd_AmdFam10hVmLockKey }, + { cpumMsrRd_AmdFam10hSmmLockKey }, + { cpumMsrRd_AmdFam10hLocalSmiStatus }, + { cpumMsrRd_AmdFam10hOsVisWrkIdLength }, + { cpumMsrRd_AmdFam10hOsVisWrkStatus }, + { cpumMsrRd_AmdFam16hL2IPerfCtlN }, + { cpumMsrRd_AmdFam16hL2IPerfCtrN }, + { cpumMsrRd_AmdFam15hNorthbridgePerfCtlN }, + { cpumMsrRd_AmdFam15hNorthbridgePerfCtrN }, + { cpumMsrRd_AmdK7MicrocodeCtl }, + { cpumMsrRd_AmdK7ClusterIdMaybe }, + { cpumMsrRd_AmdK8CpuIdCtlStd07hEbax }, + { cpumMsrRd_AmdK8CpuIdCtlStd06hEcx }, + { cpumMsrRd_AmdK8CpuIdCtlStd01hEdcx }, + { cpumMsrRd_AmdK8CpuIdCtlExt01hEdcx }, + { cpumMsrRd_AmdK8PatchLevel }, + { cpumMsrRd_AmdK7DebugStatusMaybe }, + { cpumMsrRd_AmdK7BHTraceBaseMaybe }, + { cpumMsrRd_AmdK7BHTracePtrMaybe }, + { cpumMsrRd_AmdK7BHTraceLimitMaybe }, + { cpumMsrRd_AmdK7HardwareDebugToolCfgMaybe }, + { cpumMsrRd_AmdK7FastFlushCountMaybe }, + { cpumMsrRd_AmdK7NodeId }, + { cpumMsrRd_AmdK7DrXAddrMaskN }, + { cpumMsrRd_AmdK7Dr0DataMatchMaybe }, + { cpumMsrRd_AmdK7Dr0DataMaskMaybe }, + { cpumMsrRd_AmdK7LoadStoreCfg }, + { cpumMsrRd_AmdK7InstrCacheCfg }, + { cpumMsrRd_AmdK7DataCacheCfg }, + { cpumMsrRd_AmdK7BusUnitCfg }, + { cpumMsrRd_AmdK7DebugCtl2Maybe }, + { cpumMsrRd_AmdFam15hFpuCfg }, + { cpumMsrRd_AmdFam15hDecoderCfg }, + { cpumMsrRd_AmdFam10hBusUnitCfg2 }, + { cpumMsrRd_AmdFam15hCombUnitCfg }, + { cpumMsrRd_AmdFam15hCombUnitCfg2 }, + { cpumMsrRd_AmdFam15hCombUnitCfg3 }, + { cpumMsrRd_AmdFam15hExecUnitCfg }, + { cpumMsrRd_AmdFam15hLoadStoreCfg2 }, + { cpumMsrRd_AmdFam10hIbsFetchCtl }, + { cpumMsrRd_AmdFam10hIbsFetchLinAddr }, + { cpumMsrRd_AmdFam10hIbsFetchPhysAddr }, + { cpumMsrRd_AmdFam10hIbsOpExecCtl }, + { cpumMsrRd_AmdFam10hIbsOpRip }, + { cpumMsrRd_AmdFam10hIbsOpData }, + { cpumMsrRd_AmdFam10hIbsOpData2 }, + { cpumMsrRd_AmdFam10hIbsOpData3 }, + { cpumMsrRd_AmdFam10hIbsDcLinAddr }, + { cpumMsrRd_AmdFam10hIbsDcPhysAddr }, + { cpumMsrRd_AmdFam10hIbsCtl }, + { cpumMsrRd_AmdFam14hIbsBrTarget }, + + { cpumMsrRd_Gim }, +}; + + +/** + * MSR write function table. + */ +static const struct WRITEMSRCLANG11WEIRDNOTHROW { PFNCPUMWRMSR pfnWrMsr; } g_aCpumWrMsrFns[kCpumMsrWrFn_End] = +{ + { NULL }, /* Invalid */ + { cpumMsrWr_IgnoreWrite }, + { cpumMsrWr_ReadOnly }, + { NULL }, /* Alias */ + { cpumMsrWr_Ia32P5McAddr }, + { cpumMsrWr_Ia32P5McType }, + { cpumMsrWr_Ia32TimestampCounter }, + { cpumMsrWr_Ia32ApicBase }, + { cpumMsrWr_Ia32FeatureControl }, + { cpumMsrWr_Ia32BiosSignId }, + { cpumMsrWr_Ia32BiosUpdateTrigger }, + { cpumMsrWr_Ia32SmmMonitorCtl }, + { cpumMsrWr_Ia32PmcN }, + { cpumMsrWr_Ia32MonitorFilterLineSize }, + { cpumMsrWr_Ia32MPerf }, + { cpumMsrWr_Ia32APerf }, + { cpumMsrWr_Ia32MtrrPhysBaseN }, + { cpumMsrWr_Ia32MtrrPhysMaskN }, + { cpumMsrWr_Ia32MtrrFixed }, + { cpumMsrWr_Ia32MtrrDefType }, + { cpumMsrWr_Ia32Pat }, + { cpumMsrWr_Ia32SysEnterCs }, + { cpumMsrWr_Ia32SysEnterEsp }, + { cpumMsrWr_Ia32SysEnterEip }, + { cpumMsrWr_Ia32McgStatus }, + { cpumMsrWr_Ia32McgCtl }, + { cpumMsrWr_Ia32DebugCtl }, + { cpumMsrWr_Ia32SmrrPhysBase }, + { cpumMsrWr_Ia32SmrrPhysMask }, + { cpumMsrWr_Ia32PlatformDcaCap }, + { cpumMsrWr_Ia32Dca0Cap }, + { cpumMsrWr_Ia32PerfEvtSelN }, + { cpumMsrWr_Ia32PerfStatus }, + { cpumMsrWr_Ia32PerfCtl }, + { cpumMsrWr_Ia32FixedCtrN }, + { cpumMsrWr_Ia32PerfCapabilities }, + { cpumMsrWr_Ia32FixedCtrCtrl }, + { cpumMsrWr_Ia32PerfGlobalStatus }, + { cpumMsrWr_Ia32PerfGlobalCtrl }, + { cpumMsrWr_Ia32PerfGlobalOvfCtrl }, + { cpumMsrWr_Ia32PebsEnable }, + { cpumMsrWr_Ia32ClockModulation }, + { cpumMsrWr_Ia32ThermInterrupt }, + { cpumMsrWr_Ia32ThermStatus }, + { cpumMsrWr_Ia32Therm2Ctl }, + { cpumMsrWr_Ia32MiscEnable }, + { cpumMsrWr_Ia32McCtlStatusAddrMiscN }, + { cpumMsrWr_Ia32McNCtl2 }, + { cpumMsrWr_Ia32DsArea }, + { cpumMsrWr_Ia32TscDeadline }, + { cpumMsrWr_Ia32X2ApicN }, + { cpumMsrWr_Ia32DebugInterface }, + { cpumMsrWr_Ia32SpecCtrl }, + { cpumMsrWr_Ia32PredCmd }, + { cpumMsrWr_Ia32FlushCmd }, + + { cpumMsrWr_Amd64Efer }, + { cpumMsrWr_Amd64SyscallTarget }, + { cpumMsrWr_Amd64LongSyscallTarget }, + { cpumMsrWr_Amd64CompSyscallTarget }, + { cpumMsrWr_Amd64SyscallFlagMask }, + { cpumMsrWr_Amd64FsBase }, + { cpumMsrWr_Amd64GsBase }, + { cpumMsrWr_Amd64KernelGsBase }, + { cpumMsrWr_Amd64TscAux }, + + { cpumMsrWr_IntelEblCrPowerOn }, + { cpumMsrWr_IntelP4EbcHardPowerOn }, + { cpumMsrWr_IntelP4EbcSoftPowerOn }, + { cpumMsrWr_IntelP4EbcFrequencyId }, + { cpumMsrWr_IntelFlexRatio }, + { cpumMsrWr_IntelPkgCStConfigControl }, + { cpumMsrWr_IntelPmgIoCaptureBase }, + { cpumMsrWr_IntelLastBranchFromToN }, + { cpumMsrWr_IntelLastBranchFromN }, + { cpumMsrWr_IntelLastBranchToN }, + { cpumMsrWr_IntelLastBranchTos }, + { cpumMsrWr_IntelBblCrCtl }, + { cpumMsrWr_IntelBblCrCtl3 }, + { cpumMsrWr_IntelI7TemperatureTarget }, + { cpumMsrWr_IntelI7MsrOffCoreResponseN }, + { cpumMsrWr_IntelI7MiscPwrMgmt }, + { cpumMsrWr_IntelP6CrN }, + { cpumMsrWr_IntelCpuId1FeatureMaskEcdx }, + { cpumMsrWr_IntelCpuId1FeatureMaskEax }, + { cpumMsrWr_IntelCpuId80000001FeatureMaskEcdx }, + { cpumMsrWr_IntelI7SandyAesNiCtl }, + { cpumMsrWr_IntelI7TurboRatioLimit }, + { cpumMsrWr_IntelI7LbrSelect }, + { cpumMsrWr_IntelI7SandyErrorControl }, + { cpumMsrWr_IntelI7PowerCtl }, + { cpumMsrWr_IntelI7SandyPebsNumAlt }, + { cpumMsrWr_IntelI7PebsLdLat }, + { cpumMsrWr_IntelI7SandyVrCurrentConfig }, + { cpumMsrWr_IntelI7SandyVrMiscConfig }, + { cpumMsrWr_IntelI7SandyRaplPowerUnit }, + { cpumMsrWr_IntelI7SandyPkgCnIrtlN }, + { cpumMsrWr_IntelI7SandyPkgC2Residency }, + { cpumMsrWr_IntelI7RaplPkgPowerLimit }, + { cpumMsrWr_IntelI7RaplDramPowerLimit }, + { cpumMsrWr_IntelI7RaplPp0PowerLimit }, + { cpumMsrWr_IntelI7RaplPp0Policy }, + { cpumMsrWr_IntelI7RaplPp1PowerLimit }, + { cpumMsrWr_IntelI7RaplPp1Policy }, + { cpumMsrWr_IntelI7IvyConfigTdpControl }, + { cpumMsrWr_IntelI7IvyTurboActivationRatio }, + { cpumMsrWr_IntelI7UncPerfGlobalCtrl }, + { cpumMsrWr_IntelI7UncPerfGlobalStatus }, + { cpumMsrWr_IntelI7UncPerfGlobalOvfCtrl }, + { cpumMsrWr_IntelI7UncPerfFixedCtrCtrl }, + { cpumMsrWr_IntelI7UncPerfFixedCtr }, + { cpumMsrWr_IntelI7UncArbPerfCtrN }, + { cpumMsrWr_IntelI7UncArbPerfEvtSelN }, + { cpumMsrWr_IntelCore2EmttmCrTablesN }, + { cpumMsrWr_IntelCore2SmmCStMiscInfo }, + { cpumMsrWr_IntelCore1ExtConfig }, + { cpumMsrWr_IntelCore1DtsCalControl }, + { cpumMsrWr_IntelCore2PeciControl }, + + { cpumMsrWr_P6LastIntFromIp }, + { cpumMsrWr_P6LastIntToIp }, + + { cpumMsrWr_AmdFam15hTscRate }, + { cpumMsrWr_AmdFam15hLwpCfg }, + { cpumMsrWr_AmdFam15hLwpCbAddr }, + { cpumMsrWr_AmdFam10hMc4MiscN }, + { cpumMsrWr_AmdK8PerfCtlN }, + { cpumMsrWr_AmdK8PerfCtrN }, + { cpumMsrWr_AmdK8SysCfg }, + { cpumMsrWr_AmdK8HwCr }, + { cpumMsrWr_AmdK8IorrBaseN }, + { cpumMsrWr_AmdK8IorrMaskN }, + { cpumMsrWr_AmdK8TopOfMemN }, + { cpumMsrWr_AmdK8NbCfg1 }, + { cpumMsrWr_AmdK8McXcptRedir }, + { cpumMsrWr_AmdK8CpuNameN }, + { cpumMsrWr_AmdK8HwThermalCtrl }, + { cpumMsrWr_AmdK8SwThermalCtrl }, + { cpumMsrWr_AmdK8FidVidControl }, + { cpumMsrWr_AmdK8McCtlMaskN }, + { cpumMsrWr_AmdK8SmiOnIoTrapN }, + { cpumMsrWr_AmdK8SmiOnIoTrapCtlSts }, + { cpumMsrWr_AmdK8IntPendingMessage }, + { cpumMsrWr_AmdK8SmiTriggerIoCycle }, + { cpumMsrWr_AmdFam10hMmioCfgBaseAddr }, + { cpumMsrWr_AmdFam10hTrapCtlMaybe }, + { cpumMsrWr_AmdFam10hPStateControl }, + { cpumMsrWr_AmdFam10hPStateStatus }, + { cpumMsrWr_AmdFam10hPStateN }, + { cpumMsrWr_AmdFam10hCofVidControl }, + { cpumMsrWr_AmdFam10hCofVidStatus }, + { cpumMsrWr_AmdFam10hCStateIoBaseAddr }, + { cpumMsrWr_AmdFam10hCpuWatchdogTimer }, + { cpumMsrWr_AmdK8SmmBase }, + { cpumMsrWr_AmdK8SmmAddr }, + { cpumMsrWr_AmdK8SmmMask }, + { cpumMsrWr_AmdK8VmCr }, + { cpumMsrWr_AmdK8IgnNe }, + { cpumMsrWr_AmdK8SmmCtl }, + { cpumMsrWr_AmdK8VmHSavePa }, + { cpumMsrWr_AmdFam10hVmLockKey }, + { cpumMsrWr_AmdFam10hSmmLockKey }, + { cpumMsrWr_AmdFam10hLocalSmiStatus }, + { cpumMsrWr_AmdFam10hOsVisWrkIdLength }, + { cpumMsrWr_AmdFam10hOsVisWrkStatus }, + { cpumMsrWr_AmdFam16hL2IPerfCtlN }, + { cpumMsrWr_AmdFam16hL2IPerfCtrN }, + { cpumMsrWr_AmdFam15hNorthbridgePerfCtlN }, + { cpumMsrWr_AmdFam15hNorthbridgePerfCtrN }, + { cpumMsrWr_AmdK7MicrocodeCtl }, + { cpumMsrWr_AmdK7ClusterIdMaybe }, + { cpumMsrWr_AmdK8CpuIdCtlStd07hEbax }, + { cpumMsrWr_AmdK8CpuIdCtlStd06hEcx }, + { cpumMsrWr_AmdK8CpuIdCtlStd01hEdcx }, + { cpumMsrWr_AmdK8CpuIdCtlExt01hEdcx }, + { cpumMsrWr_AmdK8PatchLoader }, + { cpumMsrWr_AmdK7DebugStatusMaybe }, + { cpumMsrWr_AmdK7BHTraceBaseMaybe }, + { cpumMsrWr_AmdK7BHTracePtrMaybe }, + { cpumMsrWr_AmdK7BHTraceLimitMaybe }, + { cpumMsrWr_AmdK7HardwareDebugToolCfgMaybe }, + { cpumMsrWr_AmdK7FastFlushCountMaybe }, + { cpumMsrWr_AmdK7NodeId }, + { cpumMsrWr_AmdK7DrXAddrMaskN }, + { cpumMsrWr_AmdK7Dr0DataMatchMaybe }, + { cpumMsrWr_AmdK7Dr0DataMaskMaybe }, + { cpumMsrWr_AmdK7LoadStoreCfg }, + { cpumMsrWr_AmdK7InstrCacheCfg }, + { cpumMsrWr_AmdK7DataCacheCfg }, + { cpumMsrWr_AmdK7BusUnitCfg }, + { cpumMsrWr_AmdK7DebugCtl2Maybe }, + { cpumMsrWr_AmdFam15hFpuCfg }, + { cpumMsrWr_AmdFam15hDecoderCfg }, + { cpumMsrWr_AmdFam10hBusUnitCfg2 }, + { cpumMsrWr_AmdFam15hCombUnitCfg }, + { cpumMsrWr_AmdFam15hCombUnitCfg2 }, + { cpumMsrWr_AmdFam15hCombUnitCfg3 }, + { cpumMsrWr_AmdFam15hExecUnitCfg }, + { cpumMsrWr_AmdFam15hLoadStoreCfg2 }, + { cpumMsrWr_AmdFam10hIbsFetchCtl }, + { cpumMsrWr_AmdFam10hIbsFetchLinAddr }, + { cpumMsrWr_AmdFam10hIbsFetchPhysAddr }, + { cpumMsrWr_AmdFam10hIbsOpExecCtl }, + { cpumMsrWr_AmdFam10hIbsOpRip }, + { cpumMsrWr_AmdFam10hIbsOpData }, + { cpumMsrWr_AmdFam10hIbsOpData2 }, + { cpumMsrWr_AmdFam10hIbsOpData3 }, + { cpumMsrWr_AmdFam10hIbsDcLinAddr }, + { cpumMsrWr_AmdFam10hIbsDcPhysAddr }, + { cpumMsrWr_AmdFam10hIbsCtl }, + { cpumMsrWr_AmdFam14hIbsBrTarget }, + + { cpumMsrWr_Gim }, +}; + + +/** + * Looks up the range for the given MSR. + * + * @returns Pointer to the range if found, NULL if not. + * @param pVM The cross context VM structure. + * @param idMsr The MSR to look up. + */ +# ifndef IN_RING3 +static +# endif +PCPUMMSRRANGE cpumLookupMsrRange(PVM pVM, uint32_t idMsr) +{ + /* + * Binary lookup. + */ + uint32_t cRanges = RT_MIN(pVM->cpum.s.GuestInfo.cMsrRanges, RT_ELEMENTS(pVM->cpum.s.GuestInfo.aMsrRanges)); + if (!cRanges) + return NULL; + PCPUMMSRRANGE paRanges = pVM->cpum.s.GuestInfo.aMsrRanges; + for (;;) + { + uint32_t i = cRanges / 2; + if (idMsr < paRanges[i].uFirst) + { + if (i == 0) + break; + cRanges = i; + } + else if (idMsr > paRanges[i].uLast) + { + i++; + if (i >= cRanges) + break; + cRanges -= i; + paRanges = &paRanges[i]; + } + else + { + if (paRanges[i].enmRdFn == kCpumMsrRdFn_MsrAlias) + return cpumLookupMsrRange(pVM, paRanges[i].uValue); + return &paRanges[i]; + } + } + +# ifdef VBOX_STRICT + /* + * Linear lookup to verify the above binary search. + */ + uint32_t cLeft = RT_MIN(pVM->cpum.s.GuestInfo.cMsrRanges, RT_ELEMENTS(pVM->cpum.s.GuestInfo.aMsrRanges)); + PCPUMMSRRANGE pCur = pVM->cpum.s.GuestInfo.aMsrRanges; + while (cLeft-- > 0) + { + if (idMsr >= pCur->uFirst && idMsr <= pCur->uLast) + { + AssertFailed(); + if (pCur->enmRdFn == kCpumMsrRdFn_MsrAlias) + return cpumLookupMsrRange(pVM, pCur->uValue); + return pCur; + } + pCur++; + } +# endif + return NULL; +} + + +/** + * Query a guest MSR. + * + * The caller is responsible for checking privilege if the call is the result of + * a RDMSR instruction. We'll do the rest. + * + * @retval VINF_SUCCESS on success. + * @retval VINF_CPUM_R3_MSR_READ if the MSR read could not be serviced in the + * current context (raw-mode or ring-0). + * @retval VERR_CPUM_RAISE_GP_0 on failure (invalid MSR), the caller is + * expected to take the appropriate actions. @a *puValue is set to 0. + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR. + * @param puValue Where to return the value. + * + * @remarks This will always return the right values, even when we're in the + * recompiler. + */ +VMMDECL(VBOXSTRICTRC) CPUMQueryGuestMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *puValue) +{ + *puValue = 0; + + VBOXSTRICTRC rcStrict; + PVM pVM = pVCpu->CTX_SUFF(pVM); + PCPUMMSRRANGE pRange = cpumLookupMsrRange(pVM, idMsr); + if (pRange) + { + CPUMMSRRDFN enmRdFn = (CPUMMSRRDFN)pRange->enmRdFn; + AssertReturn(enmRdFn > kCpumMsrRdFn_Invalid && enmRdFn < kCpumMsrRdFn_End, VERR_CPUM_IPE_1); + + PFNCPUMRDMSR pfnRdMsr = g_aCpumRdMsrFns[enmRdFn].pfnRdMsr; + AssertReturn(pfnRdMsr, VERR_CPUM_IPE_2); + + STAM_COUNTER_INC(&pRange->cReads); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReads); + + rcStrict = pfnRdMsr(pVCpu, idMsr, pRange, puValue); + if (rcStrict == VINF_SUCCESS) + Log2(("CPUM: RDMSR %#x (%s) -> %#llx\n", idMsr, pRange->szName, *puValue)); + else if (rcStrict == VERR_CPUM_RAISE_GP_0) + { + Log(("CPUM: RDMSR %#x (%s) -> #GP(0)\n", idMsr, pRange->szName)); + STAM_COUNTER_INC(&pRange->cGps); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReadsRaiseGp); + } +#ifndef IN_RING3 + else if (rcStrict == VINF_CPUM_R3_MSR_READ) + Log(("CPUM: RDMSR %#x (%s) -> ring-3\n", idMsr, pRange->szName)); +#endif + else + { + Log(("CPUM: RDMSR %#x (%s) -> rcStrict=%Rrc\n", idMsr, pRange->szName, VBOXSTRICTRC_VAL(rcStrict))); + AssertMsgStmt(RT_FAILURE_NP(rcStrict), ("%Rrc idMsr=%#x\n", VBOXSTRICTRC_VAL(rcStrict), idMsr), + rcStrict = VERR_IPE_UNEXPECTED_INFO_STATUS); + Assert(rcStrict != VERR_EM_INTERPRETER); + } + } + else + { + Log(("CPUM: Unknown RDMSR %#x -> #GP(0)\n", idMsr)); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReads); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReadsUnknown); + rcStrict = VERR_CPUM_RAISE_GP_0; + } + return rcStrict; +} + + +/** + * Writes to a guest MSR. + * + * The caller is responsible for checking privilege if the call is the result of + * a WRMSR instruction. We'll do the rest. + * + * @retval VINF_SUCCESS on success. + * @retval VINF_CPUM_R3_MSR_WRITE if the MSR write could not be serviced in the + * current context (raw-mode or ring-0). + * @retval VERR_CPUM_RAISE_GP_0 on failure, the caller is expected to take the + * appropriate actions. + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR id. + * @param uValue The value to set. + * + * @remarks Everyone changing MSR values, including the recompiler, shall do it + * by calling this method. This makes sure we have current values and + * that we trigger all the right actions when something changes. + * + * For performance reasons, this actually isn't entirely true for some + * MSRs when in HM mode. The code here and in HM must be aware of + * this. + */ +VMMDECL(VBOXSTRICTRC) CPUMSetGuestMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t uValue) +{ + VBOXSTRICTRC rcStrict; + PVM pVM = pVCpu->CTX_SUFF(pVM); + PCPUMMSRRANGE pRange = cpumLookupMsrRange(pVM, idMsr); + if (pRange) + { + STAM_COUNTER_INC(&pRange->cWrites); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWrites); + + if (!(uValue & pRange->fWrGpMask)) + { + CPUMMSRWRFN enmWrFn = (CPUMMSRWRFN)pRange->enmWrFn; + AssertReturn(enmWrFn > kCpumMsrWrFn_Invalid && enmWrFn < kCpumMsrWrFn_End, VERR_CPUM_IPE_1); + + PFNCPUMWRMSR pfnWrMsr = g_aCpumWrMsrFns[enmWrFn].pfnWrMsr; + AssertReturn(pfnWrMsr, VERR_CPUM_IPE_2); + + uint64_t uValueAdjusted = uValue & ~pRange->fWrIgnMask; + if (uValueAdjusted != uValue) + { + STAM_COUNTER_INC(&pRange->cIgnoredBits); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesToIgnoredBits); + } + + rcStrict = pfnWrMsr(pVCpu, idMsr, pRange, uValueAdjusted, uValue); + if (rcStrict == VINF_SUCCESS) + Log2(("CPUM: WRMSR %#x (%s), %#llx [%#llx]\n", idMsr, pRange->szName, uValueAdjusted, uValue)); + else if (rcStrict == VERR_CPUM_RAISE_GP_0) + { + Log(("CPUM: WRMSR %#x (%s), %#llx [%#llx] -> #GP(0)\n", idMsr, pRange->szName, uValueAdjusted, uValue)); + STAM_COUNTER_INC(&pRange->cGps); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesRaiseGp); + } +#ifndef IN_RING3 + else if (rcStrict == VINF_CPUM_R3_MSR_WRITE) + Log(("CPUM: WRMSR %#x (%s), %#llx [%#llx] -> ring-3\n", idMsr, pRange->szName, uValueAdjusted, uValue)); +#endif + else + { + Log(("CPUM: WRMSR %#x (%s), %#llx [%#llx] -> rcStrict=%Rrc\n", + idMsr, pRange->szName, uValueAdjusted, uValue, VBOXSTRICTRC_VAL(rcStrict))); + AssertMsgStmt(RT_FAILURE_NP(rcStrict), ("%Rrc idMsr=%#x\n", VBOXSTRICTRC_VAL(rcStrict), idMsr), + rcStrict = VERR_IPE_UNEXPECTED_INFO_STATUS); + Assert(rcStrict != VERR_EM_INTERPRETER); + } + } + else + { + Log(("CPUM: WRMSR %#x (%s), %#llx -> #GP(0) - invalid bits %#llx\n", + idMsr, pRange->szName, uValue, uValue & pRange->fWrGpMask)); + STAM_COUNTER_INC(&pRange->cGps); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesRaiseGp); + rcStrict = VERR_CPUM_RAISE_GP_0; + } + } + else + { + Log(("CPUM: Unknown WRMSR %#x, %#llx -> #GP(0)\n", idMsr, uValue)); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWrites); + STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesUnknown); + rcStrict = VERR_CPUM_RAISE_GP_0; + } + return rcStrict; +} + + +#if defined(VBOX_STRICT) && defined(IN_RING3) +/** + * Performs some checks on the static data related to MSRs. + * + * @returns VINF_SUCCESS on success, error on failure. + */ +int cpumR3MsrStrictInitChecks(void) +{ +#define CPUM_ASSERT_RD_MSR_FN(a_Register) \ + AssertReturn(g_aCpumRdMsrFns[kCpumMsrRdFn_##a_Register].pfnRdMsr == cpumMsrRd_##a_Register, VERR_CPUM_IPE_2); +#define CPUM_ASSERT_WR_MSR_FN(a_Register) \ + AssertReturn(g_aCpumWrMsrFns[kCpumMsrWrFn_##a_Register].pfnWrMsr == cpumMsrWr_##a_Register, VERR_CPUM_IPE_2); + + AssertReturn(g_aCpumRdMsrFns[kCpumMsrRdFn_Invalid].pfnRdMsr == NULL, VERR_CPUM_IPE_2); + CPUM_ASSERT_RD_MSR_FN(FixedValue); + CPUM_ASSERT_RD_MSR_FN(WriteOnly); + CPUM_ASSERT_RD_MSR_FN(Ia32P5McAddr); + CPUM_ASSERT_RD_MSR_FN(Ia32P5McType); + CPUM_ASSERT_RD_MSR_FN(Ia32TimestampCounter); + CPUM_ASSERT_RD_MSR_FN(Ia32PlatformId); + CPUM_ASSERT_RD_MSR_FN(Ia32ApicBase); + CPUM_ASSERT_RD_MSR_FN(Ia32FeatureControl); + CPUM_ASSERT_RD_MSR_FN(Ia32BiosSignId); + CPUM_ASSERT_RD_MSR_FN(Ia32SmmMonitorCtl); + CPUM_ASSERT_RD_MSR_FN(Ia32PmcN); + CPUM_ASSERT_RD_MSR_FN(Ia32MonitorFilterLineSize); + CPUM_ASSERT_RD_MSR_FN(Ia32MPerf); + CPUM_ASSERT_RD_MSR_FN(Ia32APerf); + CPUM_ASSERT_RD_MSR_FN(Ia32MtrrCap); + CPUM_ASSERT_RD_MSR_FN(Ia32MtrrPhysBaseN); + CPUM_ASSERT_RD_MSR_FN(Ia32MtrrPhysMaskN); + CPUM_ASSERT_RD_MSR_FN(Ia32MtrrFixed); + CPUM_ASSERT_RD_MSR_FN(Ia32MtrrDefType); + CPUM_ASSERT_RD_MSR_FN(Ia32Pat); + CPUM_ASSERT_RD_MSR_FN(Ia32SysEnterCs); + CPUM_ASSERT_RD_MSR_FN(Ia32SysEnterEsp); + CPUM_ASSERT_RD_MSR_FN(Ia32SysEnterEip); + CPUM_ASSERT_RD_MSR_FN(Ia32McgCap); + CPUM_ASSERT_RD_MSR_FN(Ia32McgStatus); + CPUM_ASSERT_RD_MSR_FN(Ia32McgCtl); + CPUM_ASSERT_RD_MSR_FN(Ia32DebugCtl); + CPUM_ASSERT_RD_MSR_FN(Ia32SmrrPhysBase); + CPUM_ASSERT_RD_MSR_FN(Ia32SmrrPhysMask); + CPUM_ASSERT_RD_MSR_FN(Ia32PlatformDcaCap); + CPUM_ASSERT_RD_MSR_FN(Ia32CpuDcaCap); + CPUM_ASSERT_RD_MSR_FN(Ia32Dca0Cap); + CPUM_ASSERT_RD_MSR_FN(Ia32PerfEvtSelN); + CPUM_ASSERT_RD_MSR_FN(Ia32PerfStatus); + CPUM_ASSERT_RD_MSR_FN(Ia32PerfCtl); + CPUM_ASSERT_RD_MSR_FN(Ia32FixedCtrN); + CPUM_ASSERT_RD_MSR_FN(Ia32PerfCapabilities); + CPUM_ASSERT_RD_MSR_FN(Ia32FixedCtrCtrl); + CPUM_ASSERT_RD_MSR_FN(Ia32PerfGlobalStatus); + CPUM_ASSERT_RD_MSR_FN(Ia32PerfGlobalCtrl); + CPUM_ASSERT_RD_MSR_FN(Ia32PerfGlobalOvfCtrl); + CPUM_ASSERT_RD_MSR_FN(Ia32PebsEnable); + CPUM_ASSERT_RD_MSR_FN(Ia32ClockModulation); + CPUM_ASSERT_RD_MSR_FN(Ia32ThermInterrupt); + CPUM_ASSERT_RD_MSR_FN(Ia32ThermStatus); + CPUM_ASSERT_RD_MSR_FN(Ia32MiscEnable); + CPUM_ASSERT_RD_MSR_FN(Ia32McCtlStatusAddrMiscN); + CPUM_ASSERT_RD_MSR_FN(Ia32McNCtl2); + CPUM_ASSERT_RD_MSR_FN(Ia32DsArea); + CPUM_ASSERT_RD_MSR_FN(Ia32TscDeadline); + CPUM_ASSERT_RD_MSR_FN(Ia32X2ApicN); + CPUM_ASSERT_RD_MSR_FN(Ia32DebugInterface); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxBasic); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxPinbasedCtls); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxProcbasedCtls); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxExitCtls); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxEntryCtls); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxMisc); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr0Fixed0); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr0Fixed1); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr4Fixed0); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr4Fixed1); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxVmcsEnum); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxProcBasedCtls2); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxEptVpidCap); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxTruePinbasedCtls); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxTrueProcbasedCtls); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxTrueExitCtls); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxTrueEntryCtls); + CPUM_ASSERT_RD_MSR_FN(Ia32VmxVmFunc); + CPUM_ASSERT_RD_MSR_FN(Ia32SpecCtrl); + CPUM_ASSERT_RD_MSR_FN(Ia32ArchCapabilities); + + CPUM_ASSERT_RD_MSR_FN(Amd64Efer); + CPUM_ASSERT_RD_MSR_FN(Amd64SyscallTarget); + CPUM_ASSERT_RD_MSR_FN(Amd64LongSyscallTarget); + CPUM_ASSERT_RD_MSR_FN(Amd64CompSyscallTarget); + CPUM_ASSERT_RD_MSR_FN(Amd64SyscallFlagMask); + CPUM_ASSERT_RD_MSR_FN(Amd64FsBase); + CPUM_ASSERT_RD_MSR_FN(Amd64GsBase); + CPUM_ASSERT_RD_MSR_FN(Amd64KernelGsBase); + CPUM_ASSERT_RD_MSR_FN(Amd64TscAux); + + CPUM_ASSERT_RD_MSR_FN(IntelEblCrPowerOn); + CPUM_ASSERT_RD_MSR_FN(IntelI7CoreThreadCount); + CPUM_ASSERT_RD_MSR_FN(IntelP4EbcHardPowerOn); + CPUM_ASSERT_RD_MSR_FN(IntelP4EbcSoftPowerOn); + CPUM_ASSERT_RD_MSR_FN(IntelP4EbcFrequencyId); + CPUM_ASSERT_RD_MSR_FN(IntelP6FsbFrequency); + CPUM_ASSERT_RD_MSR_FN(IntelPlatformInfo); + CPUM_ASSERT_RD_MSR_FN(IntelFlexRatio); + CPUM_ASSERT_RD_MSR_FN(IntelPkgCStConfigControl); + CPUM_ASSERT_RD_MSR_FN(IntelPmgIoCaptureBase); + CPUM_ASSERT_RD_MSR_FN(IntelLastBranchFromToN); + CPUM_ASSERT_RD_MSR_FN(IntelLastBranchFromN); + CPUM_ASSERT_RD_MSR_FN(IntelLastBranchToN); + CPUM_ASSERT_RD_MSR_FN(IntelLastBranchTos); + CPUM_ASSERT_RD_MSR_FN(IntelBblCrCtl); + CPUM_ASSERT_RD_MSR_FN(IntelBblCrCtl3); + CPUM_ASSERT_RD_MSR_FN(IntelI7TemperatureTarget); + CPUM_ASSERT_RD_MSR_FN(IntelI7MsrOffCoreResponseN); + CPUM_ASSERT_RD_MSR_FN(IntelI7MiscPwrMgmt); + CPUM_ASSERT_RD_MSR_FN(IntelP6CrN); + CPUM_ASSERT_RD_MSR_FN(IntelCpuId1FeatureMaskEcdx); + CPUM_ASSERT_RD_MSR_FN(IntelCpuId1FeatureMaskEax); + CPUM_ASSERT_RD_MSR_FN(IntelCpuId80000001FeatureMaskEcdx); + CPUM_ASSERT_RD_MSR_FN(IntelI7SandyAesNiCtl); + CPUM_ASSERT_RD_MSR_FN(IntelI7TurboRatioLimit); + CPUM_ASSERT_RD_MSR_FN(IntelI7LbrSelect); + CPUM_ASSERT_RD_MSR_FN(IntelI7SandyErrorControl); + CPUM_ASSERT_RD_MSR_FN(IntelI7VirtualLegacyWireCap); + CPUM_ASSERT_RD_MSR_FN(IntelI7PowerCtl); + CPUM_ASSERT_RD_MSR_FN(IntelI7SandyPebsNumAlt); + CPUM_ASSERT_RD_MSR_FN(IntelI7PebsLdLat); + CPUM_ASSERT_RD_MSR_FN(IntelI7PkgCnResidencyN); + CPUM_ASSERT_RD_MSR_FN(IntelI7CoreCnResidencyN); + CPUM_ASSERT_RD_MSR_FN(IntelI7SandyVrCurrentConfig); + CPUM_ASSERT_RD_MSR_FN(IntelI7SandyVrMiscConfig); + CPUM_ASSERT_RD_MSR_FN(IntelI7SandyRaplPowerUnit); + CPUM_ASSERT_RD_MSR_FN(IntelI7SandyPkgCnIrtlN); + CPUM_ASSERT_RD_MSR_FN(IntelI7SandyPkgC2Residency); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgPowerLimit); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgEnergyStatus); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgPerfStatus); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgPowerInfo); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramPowerLimit); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramEnergyStatus); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramPerfStatus); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramPowerInfo); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0PowerLimit); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0EnergyStatus); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0Policy); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0PerfStatus); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp1PowerLimit); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp1EnergyStatus); + CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp1Policy); + CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpNominal); + CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpLevel1); + CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpLevel2); + CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpControl); + CPUM_ASSERT_RD_MSR_FN(IntelI7IvyTurboActivationRatio); + CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfGlobalCtrl); + CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfGlobalStatus); + CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfGlobalOvfCtrl); + CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfFixedCtrCtrl); + CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfFixedCtr); + CPUM_ASSERT_RD_MSR_FN(IntelI7UncCBoxConfig); + CPUM_ASSERT_RD_MSR_FN(IntelI7UncArbPerfCtrN); + CPUM_ASSERT_RD_MSR_FN(IntelI7UncArbPerfEvtSelN); + CPUM_ASSERT_RD_MSR_FN(IntelI7SmiCount); + CPUM_ASSERT_RD_MSR_FN(IntelCore2EmttmCrTablesN); + CPUM_ASSERT_RD_MSR_FN(IntelCore2SmmCStMiscInfo); + CPUM_ASSERT_RD_MSR_FN(IntelCore1ExtConfig); + CPUM_ASSERT_RD_MSR_FN(IntelCore1DtsCalControl); + CPUM_ASSERT_RD_MSR_FN(IntelCore2PeciControl); + CPUM_ASSERT_RD_MSR_FN(IntelAtSilvCoreC1Recidency); + + CPUM_ASSERT_RD_MSR_FN(P6LastBranchFromIp); + CPUM_ASSERT_RD_MSR_FN(P6LastBranchToIp); + CPUM_ASSERT_RD_MSR_FN(P6LastIntFromIp); + CPUM_ASSERT_RD_MSR_FN(P6LastIntToIp); + + CPUM_ASSERT_RD_MSR_FN(AmdFam15hTscRate); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hLwpCfg); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hLwpCbAddr); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hMc4MiscN); + CPUM_ASSERT_RD_MSR_FN(AmdK8PerfCtlN); + CPUM_ASSERT_RD_MSR_FN(AmdK8PerfCtrN); + CPUM_ASSERT_RD_MSR_FN(AmdK8SysCfg); + CPUM_ASSERT_RD_MSR_FN(AmdK8HwCr); + CPUM_ASSERT_RD_MSR_FN(AmdK8IorrBaseN); + CPUM_ASSERT_RD_MSR_FN(AmdK8IorrMaskN); + CPUM_ASSERT_RD_MSR_FN(AmdK8TopOfMemN); + CPUM_ASSERT_RD_MSR_FN(AmdK8NbCfg1); + CPUM_ASSERT_RD_MSR_FN(AmdK8McXcptRedir); + CPUM_ASSERT_RD_MSR_FN(AmdK8CpuNameN); + CPUM_ASSERT_RD_MSR_FN(AmdK8HwThermalCtrl); + CPUM_ASSERT_RD_MSR_FN(AmdK8SwThermalCtrl); + CPUM_ASSERT_RD_MSR_FN(AmdK8FidVidControl); + CPUM_ASSERT_RD_MSR_FN(AmdK8FidVidStatus); + CPUM_ASSERT_RD_MSR_FN(AmdK8McCtlMaskN); + CPUM_ASSERT_RD_MSR_FN(AmdK8SmiOnIoTrapN); + CPUM_ASSERT_RD_MSR_FN(AmdK8SmiOnIoTrapCtlSts); + CPUM_ASSERT_RD_MSR_FN(AmdK8IntPendingMessage); + CPUM_ASSERT_RD_MSR_FN(AmdK8SmiTriggerIoCycle); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hMmioCfgBaseAddr); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hTrapCtlMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateCurLimit); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateControl); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateStatus); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateN); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hCofVidControl); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hCofVidStatus); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hCStateIoBaseAddr); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hCpuWatchdogTimer); + CPUM_ASSERT_RD_MSR_FN(AmdK8SmmBase); + CPUM_ASSERT_RD_MSR_FN(AmdK8SmmAddr); + CPUM_ASSERT_RD_MSR_FN(AmdK8SmmMask); + CPUM_ASSERT_RD_MSR_FN(AmdK8VmCr); + CPUM_ASSERT_RD_MSR_FN(AmdK8IgnNe); + CPUM_ASSERT_RD_MSR_FN(AmdK8SmmCtl); + CPUM_ASSERT_RD_MSR_FN(AmdK8VmHSavePa); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hVmLockKey); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hSmmLockKey); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hLocalSmiStatus); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hOsVisWrkIdLength); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hOsVisWrkStatus); + CPUM_ASSERT_RD_MSR_FN(AmdFam16hL2IPerfCtlN); + CPUM_ASSERT_RD_MSR_FN(AmdFam16hL2IPerfCtrN); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hNorthbridgePerfCtlN); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hNorthbridgePerfCtrN); + CPUM_ASSERT_RD_MSR_FN(AmdK7MicrocodeCtl); + CPUM_ASSERT_RD_MSR_FN(AmdK7ClusterIdMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlStd07hEbax); + CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlStd06hEcx); + CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlStd01hEdcx); + CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlExt01hEdcx); + CPUM_ASSERT_RD_MSR_FN(AmdK8PatchLevel); + CPUM_ASSERT_RD_MSR_FN(AmdK7DebugStatusMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdK7BHTraceBaseMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdK7BHTracePtrMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdK7BHTraceLimitMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdK7HardwareDebugToolCfgMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdK7FastFlushCountMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdK7NodeId); + CPUM_ASSERT_RD_MSR_FN(AmdK7DrXAddrMaskN); + CPUM_ASSERT_RD_MSR_FN(AmdK7Dr0DataMatchMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdK7Dr0DataMaskMaybe); + CPUM_ASSERT_RD_MSR_FN(AmdK7LoadStoreCfg); + CPUM_ASSERT_RD_MSR_FN(AmdK7InstrCacheCfg); + CPUM_ASSERT_RD_MSR_FN(AmdK7DataCacheCfg); + CPUM_ASSERT_RD_MSR_FN(AmdK7BusUnitCfg); + CPUM_ASSERT_RD_MSR_FN(AmdK7DebugCtl2Maybe); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hFpuCfg); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hDecoderCfg); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hBusUnitCfg2); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hCombUnitCfg); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hCombUnitCfg2); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hCombUnitCfg3); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hExecUnitCfg); + CPUM_ASSERT_RD_MSR_FN(AmdFam15hLoadStoreCfg2); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsFetchCtl); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsFetchLinAddr); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsFetchPhysAddr); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpExecCtl); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpRip); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpData); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpData2); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpData3); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsDcLinAddr); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsDcPhysAddr); + CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsCtl); + CPUM_ASSERT_RD_MSR_FN(AmdFam14hIbsBrTarget); + + CPUM_ASSERT_RD_MSR_FN(Gim) + + AssertReturn(g_aCpumWrMsrFns[kCpumMsrWrFn_Invalid].pfnWrMsr == NULL, VERR_CPUM_IPE_2); + CPUM_ASSERT_WR_MSR_FN(Ia32P5McAddr); + CPUM_ASSERT_WR_MSR_FN(Ia32P5McType); + CPUM_ASSERT_WR_MSR_FN(Ia32TimestampCounter); + CPUM_ASSERT_WR_MSR_FN(Ia32ApicBase); + CPUM_ASSERT_WR_MSR_FN(Ia32FeatureControl); + CPUM_ASSERT_WR_MSR_FN(Ia32BiosSignId); + CPUM_ASSERT_WR_MSR_FN(Ia32BiosUpdateTrigger); + CPUM_ASSERT_WR_MSR_FN(Ia32SmmMonitorCtl); + CPUM_ASSERT_WR_MSR_FN(Ia32PmcN); + CPUM_ASSERT_WR_MSR_FN(Ia32MonitorFilterLineSize); + CPUM_ASSERT_WR_MSR_FN(Ia32MPerf); + CPUM_ASSERT_WR_MSR_FN(Ia32APerf); + CPUM_ASSERT_WR_MSR_FN(Ia32MtrrPhysBaseN); + CPUM_ASSERT_WR_MSR_FN(Ia32MtrrPhysMaskN); + CPUM_ASSERT_WR_MSR_FN(Ia32MtrrFixed); + CPUM_ASSERT_WR_MSR_FN(Ia32MtrrDefType); + CPUM_ASSERT_WR_MSR_FN(Ia32Pat); + CPUM_ASSERT_WR_MSR_FN(Ia32SysEnterCs); + CPUM_ASSERT_WR_MSR_FN(Ia32SysEnterEsp); + CPUM_ASSERT_WR_MSR_FN(Ia32SysEnterEip); + CPUM_ASSERT_WR_MSR_FN(Ia32McgStatus); + CPUM_ASSERT_WR_MSR_FN(Ia32McgCtl); + CPUM_ASSERT_WR_MSR_FN(Ia32DebugCtl); + CPUM_ASSERT_WR_MSR_FN(Ia32SmrrPhysBase); + CPUM_ASSERT_WR_MSR_FN(Ia32SmrrPhysMask); + CPUM_ASSERT_WR_MSR_FN(Ia32PlatformDcaCap); + CPUM_ASSERT_WR_MSR_FN(Ia32Dca0Cap); + CPUM_ASSERT_WR_MSR_FN(Ia32PerfEvtSelN); + CPUM_ASSERT_WR_MSR_FN(Ia32PerfStatus); + CPUM_ASSERT_WR_MSR_FN(Ia32PerfCtl); + CPUM_ASSERT_WR_MSR_FN(Ia32FixedCtrN); + CPUM_ASSERT_WR_MSR_FN(Ia32PerfCapabilities); + CPUM_ASSERT_WR_MSR_FN(Ia32FixedCtrCtrl); + CPUM_ASSERT_WR_MSR_FN(Ia32PerfGlobalStatus); + CPUM_ASSERT_WR_MSR_FN(Ia32PerfGlobalCtrl); + CPUM_ASSERT_WR_MSR_FN(Ia32PerfGlobalOvfCtrl); + CPUM_ASSERT_WR_MSR_FN(Ia32PebsEnable); + CPUM_ASSERT_WR_MSR_FN(Ia32ClockModulation); + CPUM_ASSERT_WR_MSR_FN(Ia32ThermInterrupt); + CPUM_ASSERT_WR_MSR_FN(Ia32ThermStatus); + CPUM_ASSERT_WR_MSR_FN(Ia32MiscEnable); + CPUM_ASSERT_WR_MSR_FN(Ia32McCtlStatusAddrMiscN); + CPUM_ASSERT_WR_MSR_FN(Ia32McNCtl2); + CPUM_ASSERT_WR_MSR_FN(Ia32DsArea); + CPUM_ASSERT_WR_MSR_FN(Ia32TscDeadline); + CPUM_ASSERT_WR_MSR_FN(Ia32X2ApicN); + CPUM_ASSERT_WR_MSR_FN(Ia32DebugInterface); + CPUM_ASSERT_WR_MSR_FN(Ia32SpecCtrl); + CPUM_ASSERT_WR_MSR_FN(Ia32PredCmd); + CPUM_ASSERT_WR_MSR_FN(Ia32FlushCmd); + + CPUM_ASSERT_WR_MSR_FN(Amd64Efer); + CPUM_ASSERT_WR_MSR_FN(Amd64SyscallTarget); + CPUM_ASSERT_WR_MSR_FN(Amd64LongSyscallTarget); + CPUM_ASSERT_WR_MSR_FN(Amd64CompSyscallTarget); + CPUM_ASSERT_WR_MSR_FN(Amd64SyscallFlagMask); + CPUM_ASSERT_WR_MSR_FN(Amd64FsBase); + CPUM_ASSERT_WR_MSR_FN(Amd64GsBase); + CPUM_ASSERT_WR_MSR_FN(Amd64KernelGsBase); + CPUM_ASSERT_WR_MSR_FN(Amd64TscAux); + + CPUM_ASSERT_WR_MSR_FN(IntelEblCrPowerOn); + CPUM_ASSERT_WR_MSR_FN(IntelP4EbcHardPowerOn); + CPUM_ASSERT_WR_MSR_FN(IntelP4EbcSoftPowerOn); + CPUM_ASSERT_WR_MSR_FN(IntelP4EbcFrequencyId); + CPUM_ASSERT_WR_MSR_FN(IntelFlexRatio); + CPUM_ASSERT_WR_MSR_FN(IntelPkgCStConfigControl); + CPUM_ASSERT_WR_MSR_FN(IntelPmgIoCaptureBase); + CPUM_ASSERT_WR_MSR_FN(IntelLastBranchFromToN); + CPUM_ASSERT_WR_MSR_FN(IntelLastBranchFromN); + CPUM_ASSERT_WR_MSR_FN(IntelLastBranchToN); + CPUM_ASSERT_WR_MSR_FN(IntelLastBranchTos); + CPUM_ASSERT_WR_MSR_FN(IntelBblCrCtl); + CPUM_ASSERT_WR_MSR_FN(IntelBblCrCtl3); + CPUM_ASSERT_WR_MSR_FN(IntelI7TemperatureTarget); + CPUM_ASSERT_WR_MSR_FN(IntelI7MsrOffCoreResponseN); + CPUM_ASSERT_WR_MSR_FN(IntelI7MiscPwrMgmt); + CPUM_ASSERT_WR_MSR_FN(IntelP6CrN); + CPUM_ASSERT_WR_MSR_FN(IntelCpuId1FeatureMaskEcdx); + CPUM_ASSERT_WR_MSR_FN(IntelCpuId1FeatureMaskEax); + CPUM_ASSERT_WR_MSR_FN(IntelCpuId80000001FeatureMaskEcdx); + CPUM_ASSERT_WR_MSR_FN(IntelI7SandyAesNiCtl); + CPUM_ASSERT_WR_MSR_FN(IntelI7TurboRatioLimit); + CPUM_ASSERT_WR_MSR_FN(IntelI7LbrSelect); + CPUM_ASSERT_WR_MSR_FN(IntelI7SandyErrorControl); + CPUM_ASSERT_WR_MSR_FN(IntelI7PowerCtl); + CPUM_ASSERT_WR_MSR_FN(IntelI7SandyPebsNumAlt); + CPUM_ASSERT_WR_MSR_FN(IntelI7PebsLdLat); + CPUM_ASSERT_WR_MSR_FN(IntelI7SandyVrCurrentConfig); + CPUM_ASSERT_WR_MSR_FN(IntelI7SandyVrMiscConfig); + CPUM_ASSERT_WR_MSR_FN(IntelI7SandyPkgCnIrtlN); + CPUM_ASSERT_WR_MSR_FN(IntelI7SandyPkgC2Residency); + CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPkgPowerLimit); + CPUM_ASSERT_WR_MSR_FN(IntelI7RaplDramPowerLimit); + CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp0PowerLimit); + CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp0Policy); + CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp1PowerLimit); + CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp1Policy); + CPUM_ASSERT_WR_MSR_FN(IntelI7IvyConfigTdpControl); + CPUM_ASSERT_WR_MSR_FN(IntelI7IvyTurboActivationRatio); + CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfGlobalCtrl); + CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfGlobalStatus); + CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfGlobalOvfCtrl); + CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfFixedCtrCtrl); + CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfFixedCtr); + CPUM_ASSERT_WR_MSR_FN(IntelI7UncArbPerfCtrN); + CPUM_ASSERT_WR_MSR_FN(IntelI7UncArbPerfEvtSelN); + CPUM_ASSERT_WR_MSR_FN(IntelCore2EmttmCrTablesN); + CPUM_ASSERT_WR_MSR_FN(IntelCore2SmmCStMiscInfo); + CPUM_ASSERT_WR_MSR_FN(IntelCore1ExtConfig); + CPUM_ASSERT_WR_MSR_FN(IntelCore1DtsCalControl); + CPUM_ASSERT_WR_MSR_FN(IntelCore2PeciControl); + + CPUM_ASSERT_WR_MSR_FN(P6LastIntFromIp); + CPUM_ASSERT_WR_MSR_FN(P6LastIntToIp); + + CPUM_ASSERT_WR_MSR_FN(AmdFam15hTscRate); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hLwpCfg); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hLwpCbAddr); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hMc4MiscN); + CPUM_ASSERT_WR_MSR_FN(AmdK8PerfCtlN); + CPUM_ASSERT_WR_MSR_FN(AmdK8PerfCtrN); + CPUM_ASSERT_WR_MSR_FN(AmdK8SysCfg); + CPUM_ASSERT_WR_MSR_FN(AmdK8HwCr); + CPUM_ASSERT_WR_MSR_FN(AmdK8IorrBaseN); + CPUM_ASSERT_WR_MSR_FN(AmdK8IorrMaskN); + CPUM_ASSERT_WR_MSR_FN(AmdK8TopOfMemN); + CPUM_ASSERT_WR_MSR_FN(AmdK8NbCfg1); + CPUM_ASSERT_WR_MSR_FN(AmdK8McXcptRedir); + CPUM_ASSERT_WR_MSR_FN(AmdK8CpuNameN); + CPUM_ASSERT_WR_MSR_FN(AmdK8HwThermalCtrl); + CPUM_ASSERT_WR_MSR_FN(AmdK8SwThermalCtrl); + CPUM_ASSERT_WR_MSR_FN(AmdK8FidVidControl); + CPUM_ASSERT_WR_MSR_FN(AmdK8McCtlMaskN); + CPUM_ASSERT_WR_MSR_FN(AmdK8SmiOnIoTrapN); + CPUM_ASSERT_WR_MSR_FN(AmdK8SmiOnIoTrapCtlSts); + CPUM_ASSERT_WR_MSR_FN(AmdK8IntPendingMessage); + CPUM_ASSERT_WR_MSR_FN(AmdK8SmiTriggerIoCycle); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hMmioCfgBaseAddr); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hTrapCtlMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hPStateControl); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hPStateStatus); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hPStateN); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hCofVidControl); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hCofVidStatus); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hCStateIoBaseAddr); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hCpuWatchdogTimer); + CPUM_ASSERT_WR_MSR_FN(AmdK8SmmBase); + CPUM_ASSERT_WR_MSR_FN(AmdK8SmmAddr); + CPUM_ASSERT_WR_MSR_FN(AmdK8SmmMask); + CPUM_ASSERT_WR_MSR_FN(AmdK8VmCr); + CPUM_ASSERT_WR_MSR_FN(AmdK8IgnNe); + CPUM_ASSERT_WR_MSR_FN(AmdK8SmmCtl); + CPUM_ASSERT_WR_MSR_FN(AmdK8VmHSavePa); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hVmLockKey); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hSmmLockKey); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hLocalSmiStatus); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hOsVisWrkIdLength); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hOsVisWrkStatus); + CPUM_ASSERT_WR_MSR_FN(AmdFam16hL2IPerfCtlN); + CPUM_ASSERT_WR_MSR_FN(AmdFam16hL2IPerfCtrN); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hNorthbridgePerfCtlN); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hNorthbridgePerfCtrN); + CPUM_ASSERT_WR_MSR_FN(AmdK7MicrocodeCtl); + CPUM_ASSERT_WR_MSR_FN(AmdK7ClusterIdMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlStd07hEbax); + CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlStd06hEcx); + CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlStd01hEdcx); + CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlExt01hEdcx); + CPUM_ASSERT_WR_MSR_FN(AmdK8PatchLoader); + CPUM_ASSERT_WR_MSR_FN(AmdK7DebugStatusMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdK7BHTraceBaseMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdK7BHTracePtrMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdK7BHTraceLimitMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdK7HardwareDebugToolCfgMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdK7FastFlushCountMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdK7NodeId); + CPUM_ASSERT_WR_MSR_FN(AmdK7DrXAddrMaskN); + CPUM_ASSERT_WR_MSR_FN(AmdK7Dr0DataMatchMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdK7Dr0DataMaskMaybe); + CPUM_ASSERT_WR_MSR_FN(AmdK7LoadStoreCfg); + CPUM_ASSERT_WR_MSR_FN(AmdK7InstrCacheCfg); + CPUM_ASSERT_WR_MSR_FN(AmdK7DataCacheCfg); + CPUM_ASSERT_WR_MSR_FN(AmdK7BusUnitCfg); + CPUM_ASSERT_WR_MSR_FN(AmdK7DebugCtl2Maybe); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hFpuCfg); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hDecoderCfg); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hBusUnitCfg2); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hCombUnitCfg); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hCombUnitCfg2); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hCombUnitCfg3); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hExecUnitCfg); + CPUM_ASSERT_WR_MSR_FN(AmdFam15hLoadStoreCfg2); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsFetchCtl); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsFetchLinAddr); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsFetchPhysAddr); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpExecCtl); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpRip); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpData); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpData2); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpData3); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsDcLinAddr); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsDcPhysAddr); + CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsCtl); + CPUM_ASSERT_WR_MSR_FN(AmdFam14hIbsBrTarget); + + CPUM_ASSERT_WR_MSR_FN(Gim); + + return VINF_SUCCESS; +} +#endif /* VBOX_STRICT && IN_RING3 */ + + +/** + * Gets the scalable bus frequency. + * + * The bus frequency is used as a base in several MSRs that gives the CPU and + * other frequency ratios. + * + * @returns Scalable bus frequency in Hz. Will not return CPUM_SBUSFREQ_UNKNOWN. + * @param pVM The cross context VM structure. + */ +VMMDECL(uint64_t) CPUMGetGuestScalableBusFrequency(PVM pVM) +{ + uint64_t uFreq = pVM->cpum.s.GuestInfo.uScalableBusFreq; + if (uFreq == CPUM_SBUSFREQ_UNKNOWN) + uFreq = CPUM_SBUSFREQ_100MHZ; + return uFreq; +} + + +/** + * Sets the guest EFER MSR without performing any additional checks. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uOldEfer The previous EFER MSR value. + * @param uValidEfer The new, validated EFER MSR value. + * + * @remarks One would normally call CPUMIsGuestEferMsrWriteValid() before calling + * this function to change the EFER in order to perform an EFER transition. + */ +VMMDECL(void) CPUMSetGuestEferMsrNoChecks(PVMCPUCC pVCpu, uint64_t uOldEfer, uint64_t uValidEfer) +{ + pVCpu->cpum.s.Guest.msrEFER = uValidEfer; + + /* AMD64 Architecture Programmer's Manual: 15.15 TLB Control; flush the TLB + if MSR_K6_EFER_NXE, MSR_K6_EFER_LME or MSR_K6_EFER_LMA are changed. */ + if ( (uOldEfer & (MSR_K6_EFER_NXE | MSR_K6_EFER_LME | MSR_K6_EFER_LMA)) + != (pVCpu->cpum.s.Guest.msrEFER & (MSR_K6_EFER_NXE | MSR_K6_EFER_LME | MSR_K6_EFER_LMA))) + { + /// @todo PGMFlushTLB(pVCpu, cr3, true /*fGlobal*/); + HMFlushTlb(pVCpu); + + /* Notify PGM about NXE changes. */ + if ( (uOldEfer & MSR_K6_EFER_NXE) + != (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_NXE)) + PGMNotifyNxeChanged(pVCpu, !(uOldEfer & MSR_K6_EFER_NXE)); + } +} + + +/** + * Checks if a guest PAT MSR write is valid. + * + * @returns @c true if the PAT bit combination is valid, @c false otherwise. + * @param uValue The PAT MSR value. + */ +VMMDECL(bool) CPUMIsPatMsrValid(uint64_t uValue) +{ + for (uint32_t cShift = 0; cShift < 63; cShift += 8) + { + /* Check all eight bits because the top 5 bits of each byte are reserved. */ + uint8_t uType = (uint8_t)(uValue >> cShift); + if ((uType >= 8) || (uType == 2) || (uType == 3)) + { + Log(("CPUM: Invalid PAT type at %u:%u in IA32_PAT: %#llx (%#llx)\n", cShift + 7, cShift, uValue, uType)); + return false; + } + } + return true; +} + + +/** + * Validates an EFER MSR write and provides the new, validated EFER MSR. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param uCr0 The CR0 of the CPU corresponding to the EFER MSR. + * @param uOldEfer Value of the previous EFER MSR on the CPU if any. + * @param uNewEfer The new EFER MSR value being written. + * @param puValidEfer Where to store the validated EFER (only updated if + * this function returns VINF_SUCCESS). + */ +VMMDECL(int) CPUMIsGuestEferMsrWriteValid(PVM pVM, uint64_t uCr0, uint64_t uOldEfer, uint64_t uNewEfer, uint64_t *puValidEfer) +{ + /* #GP(0) If anything outside the allowed bits is set. */ + uint64_t fMask = CPUMGetGuestEferMsrValidMask(pVM); + if (uNewEfer & ~fMask) + { + Log(("CPUM: Settings disallowed EFER bit. uNewEfer=%#RX64 fAllowed=%#RX64 -> #GP(0)\n", uNewEfer, fMask)); + return VERR_CPUM_RAISE_GP_0; + } + + /* Check for illegal MSR_K6_EFER_LME transitions: not allowed to change LME if + paging is enabled. (AMD Arch. Programmer's Manual Volume 2: Table 14-5) */ + if ( (uOldEfer & MSR_K6_EFER_LME) != (uNewEfer & MSR_K6_EFER_LME) + && (uCr0 & X86_CR0_PG)) + { + Log(("CPUM: Illegal MSR_K6_EFER_LME change: paging is enabled!!\n")); + return VERR_CPUM_RAISE_GP_0; + } + + /* There are a few more: e.g. MSR_K6_EFER_LMSLE. */ + AssertMsg(!(uNewEfer & ~( MSR_K6_EFER_NXE + | MSR_K6_EFER_LME + | MSR_K6_EFER_LMA /* ignored anyway */ + | MSR_K6_EFER_SCE + | MSR_K6_EFER_FFXSR + | MSR_K6_EFER_SVME)), + ("Unexpected value %#RX64\n", uNewEfer)); + + /* Ignore EFER.LMA, it's updated when setting CR0. */ + fMask &= ~MSR_K6_EFER_LMA; + + *puValidEfer = (uOldEfer & ~fMask) | (uNewEfer & fMask); + return VINF_SUCCESS; +} + + +/** + * Gets the mask of valid EFER bits depending on supported guest-CPU features. + * + * @returns Mask of valid EFER bits. + * @param pVM The cross context VM structure. + * + * @remarks EFER.LMA is included as part of the valid mask. It's not invalid but + * rather a read-only bit. + */ +VMMDECL(uint64_t) CPUMGetGuestEferMsrValidMask(PVM pVM) +{ + uint32_t const fExtFeatures = pVM->cpum.s.aGuestCpuIdPatmExt[0].uEax >= 0x80000001 + ? pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx + : 0; + uint64_t fMask = 0; + uint64_t const fIgnoreMask = MSR_K6_EFER_LMA; + + /* Filter out those bits the guest is allowed to change. (e.g. LMA is read-only) */ + if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_NX) + fMask |= MSR_K6_EFER_NXE; + if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE) + fMask |= MSR_K6_EFER_LME; + if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_SYSCALL) + fMask |= MSR_K6_EFER_SCE; + if (fExtFeatures & X86_CPUID_AMD_FEATURE_EDX_FFXSR) + fMask |= MSR_K6_EFER_FFXSR; + if (pVM->cpum.s.GuestFeatures.fSvm) + fMask |= MSR_K6_EFER_SVME; + + return (fIgnoreMask | fMask); +} + + +/** + * Fast way for HM to access the MSR_K8_TSC_AUX register. + * + * @returns The register value. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(uint64_t) CPUMGetGuestTscAux(PVMCPUCC pVCpu) +{ + Assert(!(pVCpu->cpum.s.Guest.fExtrn & CPUMCTX_EXTRN_TSC_AUX)); + return pVCpu->cpum.s.GuestMsrs.msr.TscAux; +} + + +/** + * Fast way for HM to access the MSR_K8_TSC_AUX register. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uValue The new value. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(void) CPUMSetGuestTscAux(PVMCPUCC pVCpu, uint64_t uValue) +{ + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_TSC_AUX; + pVCpu->cpum.s.GuestMsrs.msr.TscAux = uValue; +} + + +/** + * Fast way for HM to access the IA32_SPEC_CTRL register. + * + * @returns The register value. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(uint64_t) CPUMGetGuestSpecCtrl(PVMCPUCC pVCpu) +{ + return pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl; +} + + +/** + * Fast way for HM to access the IA32_SPEC_CTRL register. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uValue The new value. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(void) CPUMSetGuestSpecCtrl(PVMCPUCC pVCpu, uint64_t uValue) +{ + pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl = uValue; +} + diff --git a/src/VBox/VMM/VMMAll/CPUMAllRegs.cpp b/src/VBox/VMM/VMMAll/CPUMAllRegs.cpp new file mode 100644 index 00000000..753aaa71 --- /dev/null +++ b/src/VBox/VMM/VMMAll/CPUMAllRegs.cpp @@ -0,0 +1,3039 @@ +/* $Id: CPUMAllRegs.cpp $ */ +/** @file + * CPUM - CPU Monitor(/Manager) - Getters and Setters. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_CPUM +#include +#include +#include +#include +#include +#include +#include +#include +#include "CPUMInternal.h" +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +# include +#endif +#ifdef IN_RING3 +# include +#endif + +/** Disable stack frame pointer generation here. */ +#if defined(_MSC_VER) && !defined(DEBUG) && defined(RT_ARCH_X86) +# pragma optimize("y", off) +#endif + +AssertCompile2MemberOffsets(VM, cpum.s.GuestFeatures, cpum.ro.GuestFeatures); + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** + * Converts a CPUMCPU::Guest pointer into a VMCPU pointer. + * + * @returns Pointer to the Virtual CPU. + * @param a_pGuestCtx Pointer to the guest context. + */ +#define CPUM_GUEST_CTX_TO_VMCPU(a_pGuestCtx) RT_FROM_MEMBER(a_pGuestCtx, VMCPU, cpum.s.Guest) + +/** + * Lazily loads the hidden parts of a selector register when using raw-mode. + */ +#define CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(a_pVCpu, a_pSReg) \ + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(a_pVCpu, a_pSReg)) + +/** @def CPUM_INT_ASSERT_NOT_EXTRN + * Macro for asserting that @a a_fNotExtrn are present. + * + * @param a_pVCpu The cross context virtual CPU structure of the calling EMT. + * @param a_fNotExtrn Mask of CPUMCTX_EXTRN_XXX bits to check. + */ +#define CPUM_INT_ASSERT_NOT_EXTRN(a_pVCpu, a_fNotExtrn) \ + AssertMsg(!((a_pVCpu)->cpum.s.Guest.fExtrn & (a_fNotExtrn)), \ + ("%#RX64; a_fNotExtrn=%#RX64\n", (a_pVCpu)->cpum.s.Guest.fExtrn, (a_fNotExtrn))) + + +VMMDECL(void) CPUMSetHyperCR3(PVMCPU pVCpu, uint32_t cr3) +{ + pVCpu->cpum.s.Hyper.cr3 = cr3; +} + +VMMDECL(uint32_t) CPUMGetHyperCR3(PVMCPU pVCpu) +{ + return pVCpu->cpum.s.Hyper.cr3; +} + + +/** @def MAYBE_LOAD_DRx + * Macro for updating DRx values in raw-mode and ring-0 contexts. + */ +#ifdef IN_RING0 +# define MAYBE_LOAD_DRx(a_pVCpu, a_fnLoad, a_uValue) do { a_fnLoad(a_uValue); } while (0) +#else +# define MAYBE_LOAD_DRx(a_pVCpu, a_fnLoad, a_uValue) do { } while (0) +#endif + +VMMDECL(void) CPUMSetHyperDR0(PVMCPU pVCpu, RTGCUINTREG uDr0) +{ + pVCpu->cpum.s.Hyper.dr[0] = uDr0; + MAYBE_LOAD_DRx(pVCpu, ASMSetDR0, uDr0); +} + + +VMMDECL(void) CPUMSetHyperDR1(PVMCPU pVCpu, RTGCUINTREG uDr1) +{ + pVCpu->cpum.s.Hyper.dr[1] = uDr1; + MAYBE_LOAD_DRx(pVCpu, ASMSetDR1, uDr1); +} + + +VMMDECL(void) CPUMSetHyperDR2(PVMCPU pVCpu, RTGCUINTREG uDr2) +{ + pVCpu->cpum.s.Hyper.dr[2] = uDr2; + MAYBE_LOAD_DRx(pVCpu, ASMSetDR2, uDr2); +} + + +VMMDECL(void) CPUMSetHyperDR3(PVMCPU pVCpu, RTGCUINTREG uDr3) +{ + pVCpu->cpum.s.Hyper.dr[3] = uDr3; + MAYBE_LOAD_DRx(pVCpu, ASMSetDR3, uDr3); +} + + +VMMDECL(void) CPUMSetHyperDR6(PVMCPU pVCpu, RTGCUINTREG uDr6) +{ + pVCpu->cpum.s.Hyper.dr[6] = uDr6; +} + + +VMMDECL(void) CPUMSetHyperDR7(PVMCPU pVCpu, RTGCUINTREG uDr7) +{ + pVCpu->cpum.s.Hyper.dr[7] = uDr7; +} + + +VMMDECL(RTGCUINTREG) CPUMGetHyperDR0(PVMCPU pVCpu) +{ + return pVCpu->cpum.s.Hyper.dr[0]; +} + + +VMMDECL(RTGCUINTREG) CPUMGetHyperDR1(PVMCPU pVCpu) +{ + return pVCpu->cpum.s.Hyper.dr[1]; +} + + +VMMDECL(RTGCUINTREG) CPUMGetHyperDR2(PVMCPU pVCpu) +{ + return pVCpu->cpum.s.Hyper.dr[2]; +} + + +VMMDECL(RTGCUINTREG) CPUMGetHyperDR3(PVMCPU pVCpu) +{ + return pVCpu->cpum.s.Hyper.dr[3]; +} + + +VMMDECL(RTGCUINTREG) CPUMGetHyperDR6(PVMCPU pVCpu) +{ + return pVCpu->cpum.s.Hyper.dr[6]; +} + + +VMMDECL(RTGCUINTREG) CPUMGetHyperDR7(PVMCPU pVCpu) +{ + return pVCpu->cpum.s.Hyper.dr[7]; +} + + +/** + * Checks that the special cookie stored in unused reserved RFLAGS bits + * + * @retval true if cookie is ok. + * @retval false if cookie is not ok. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) CPUMAssertGuestRFlagsCookie(PVM pVM, PVMCPU pVCpu) +{ + AssertLogRelMsgReturn( ( pVCpu->cpum.s.Guest.rflags.uBoth + & ~(uint64_t)(CPUMX86EFLAGS_HW_MASK_64 | CPUMX86EFLAGS_INT_MASK_64)) + == pVM->cpum.s.fReservedRFlagsCookie + && (pVCpu->cpum.s.Guest.rflags.uBoth & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK + && (pVCpu->cpum.s.Guest.rflags.uBoth & X86_EFL_RAZ_MASK & CPUMX86EFLAGS_HW_MASK_64) == 0, + ("rflags=%#RX64 vs fReservedRFlagsCookie=%#RX64\n", + pVCpu->cpum.s.Guest.rflags.uBoth, pVM->cpum.s.fReservedRFlagsCookie), + false); + return true; +} + + +/** + * Queries the pointer to the internal CPUMCTX structure. + * + * @returns The CPUMCTX pointer. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(PCPUMCTX) CPUMQueryGuestCtxPtr(PVMCPU pVCpu) +{ + return &pVCpu->cpum.s.Guest; +} + + +/** + * Queries the pointer to the internal CPUMCTXMSRS structure. + * + * This is for NEM only. + * + * @returns The CPUMCTX pointer. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(PCPUMCTXMSRS) CPUMQueryGuestCtxMsrsPtr(PVMCPU pVCpu) +{ + return &pVCpu->cpum.s.GuestMsrs; +} + + +VMMDECL(int) CPUMSetGuestGDTR(PVMCPU pVCpu, uint64_t GCPtrBase, uint16_t cbLimit) +{ + pVCpu->cpum.s.Guest.gdtr.cbGdt = cbLimit; + pVCpu->cpum.s.Guest.gdtr.pGdt = GCPtrBase; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_GDTR; + pVCpu->cpum.s.fChanged |= CPUM_CHANGED_GDTR; + return VINF_SUCCESS; /* formality, consider it void. */ +} + + +VMMDECL(int) CPUMSetGuestIDTR(PVMCPU pVCpu, uint64_t GCPtrBase, uint16_t cbLimit) +{ + pVCpu->cpum.s.Guest.idtr.cbIdt = cbLimit; + pVCpu->cpum.s.Guest.idtr.pIdt = GCPtrBase; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_IDTR; + pVCpu->cpum.s.fChanged |= CPUM_CHANGED_IDTR; + return VINF_SUCCESS; /* formality, consider it void. */ +} + + +VMMDECL(int) CPUMSetGuestTR(PVMCPU pVCpu, uint16_t tr) +{ + pVCpu->cpum.s.Guest.tr.Sel = tr; + pVCpu->cpum.s.fChanged |= CPUM_CHANGED_TR; + return VINF_SUCCESS; /* formality, consider it void. */ +} + + +VMMDECL(int) CPUMSetGuestLDTR(PVMCPU pVCpu, uint16_t ldtr) +{ + pVCpu->cpum.s.Guest.ldtr.Sel = ldtr; + /* The caller will set more hidden bits if it has them. */ + pVCpu->cpum.s.Guest.ldtr.ValidSel = 0; + pVCpu->cpum.s.Guest.ldtr.fFlags = 0; + pVCpu->cpum.s.fChanged |= CPUM_CHANGED_LDTR; + return VINF_SUCCESS; /* formality, consider it void. */ +} + + +/** + * Set the guest CR0. + * + * When called in GC, the hyper CR0 may be updated if that is + * required. The caller only has to take special action if AM, + * WP, PG or PE changes. + * + * @returns VINF_SUCCESS (consider it void). + * @param pVCpu The cross context virtual CPU structure. + * @param cr0 The new CR0 value. + */ +VMMDECL(int) CPUMSetGuestCR0(PVMCPUCC pVCpu, uint64_t cr0) +{ + /* + * Check for changes causing TLB flushes (for REM). + * The caller is responsible for calling PGM when appropriate. + */ + if ( (cr0 & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE)) + != (pVCpu->cpum.s.Guest.cr0 & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE))) + pVCpu->cpum.s.fChanged |= CPUM_CHANGED_GLOBAL_TLB_FLUSH; + pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CR0; + + /* + * Let PGM know if the WP goes from 0 to 1 (netware WP0+RO+US hack) + */ + if (((cr0 ^ pVCpu->cpum.s.Guest.cr0) & X86_CR0_WP) && (cr0 & X86_CR0_WP)) + PGMCr0WpEnabled(pVCpu); + + /* The ET flag is settable on a 386 and hardwired on 486+. */ + if ( !(cr0 & X86_CR0_ET) + && pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386) + cr0 |= X86_CR0_ET; + + pVCpu->cpum.s.Guest.cr0 = cr0; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR0; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestCR2(PVMCPU pVCpu, uint64_t cr2) +{ + pVCpu->cpum.s.Guest.cr2 = cr2; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR2; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestCR3(PVMCPU pVCpu, uint64_t cr3) +{ + pVCpu->cpum.s.Guest.cr3 = cr3; + pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CR3; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR3; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestCR4(PVMCPU pVCpu, uint64_t cr4) +{ + /* Note! We don't bother with OSXSAVE and legacy CPUID patches. */ + + if ( (cr4 & (X86_CR4_PGE | X86_CR4_PAE | X86_CR4_PSE)) + != (pVCpu->cpum.s.Guest.cr4 & (X86_CR4_PGE | X86_CR4_PAE | X86_CR4_PSE))) + pVCpu->cpum.s.fChanged |= CPUM_CHANGED_GLOBAL_TLB_FLUSH; + + pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CR4; + pVCpu->cpum.s.Guest.cr4 = cr4; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR4; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestEFlags(PVMCPU pVCpu, uint32_t eflags) +{ + pVCpu->cpum.s.Guest.eflags.u = eflags; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_RFLAGS; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestEIP(PVMCPU pVCpu, uint32_t eip) +{ + pVCpu->cpum.s.Guest.eip = eip; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestEAX(PVMCPU pVCpu, uint32_t eax) +{ + pVCpu->cpum.s.Guest.eax = eax; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestEBX(PVMCPU pVCpu, uint32_t ebx) +{ + pVCpu->cpum.s.Guest.ebx = ebx; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestECX(PVMCPU pVCpu, uint32_t ecx) +{ + pVCpu->cpum.s.Guest.ecx = ecx; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestEDX(PVMCPU pVCpu, uint32_t edx) +{ + pVCpu->cpum.s.Guest.edx = edx; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestESP(PVMCPU pVCpu, uint32_t esp) +{ + pVCpu->cpum.s.Guest.esp = esp; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestEBP(PVMCPU pVCpu, uint32_t ebp) +{ + pVCpu->cpum.s.Guest.ebp = ebp; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestESI(PVMCPU pVCpu, uint32_t esi) +{ + pVCpu->cpum.s.Guest.esi = esi; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestEDI(PVMCPU pVCpu, uint32_t edi) +{ + pVCpu->cpum.s.Guest.edi = edi; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestSS(PVMCPU pVCpu, uint16_t ss) +{ + pVCpu->cpum.s.Guest.ss.Sel = ss; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestCS(PVMCPU pVCpu, uint16_t cs) +{ + pVCpu->cpum.s.Guest.cs.Sel = cs; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestDS(PVMCPU pVCpu, uint16_t ds) +{ + pVCpu->cpum.s.Guest.ds.Sel = ds; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestES(PVMCPU pVCpu, uint16_t es) +{ + pVCpu->cpum.s.Guest.es.Sel = es; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestFS(PVMCPU pVCpu, uint16_t fs) +{ + pVCpu->cpum.s.Guest.fs.Sel = fs; + return VINF_SUCCESS; +} + + +VMMDECL(int) CPUMSetGuestGS(PVMCPU pVCpu, uint16_t gs) +{ + pVCpu->cpum.s.Guest.gs.Sel = gs; + return VINF_SUCCESS; +} + + +VMMDECL(void) CPUMSetGuestEFER(PVMCPU pVCpu, uint64_t val) +{ + pVCpu->cpum.s.Guest.msrEFER = val; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_EFER; +} + + +VMMDECL(RTGCPTR) CPUMGetGuestIDTR(PCVMCPU pVCpu, uint16_t *pcbLimit) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_IDTR); + if (pcbLimit) + *pcbLimit = pVCpu->cpum.s.Guest.idtr.cbIdt; + return pVCpu->cpum.s.Guest.idtr.pIdt; +} + + +VMMDECL(RTSEL) CPUMGetGuestTR(PCVMCPU pVCpu, PCPUMSELREGHID pHidden) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_TR); + if (pHidden) + *pHidden = pVCpu->cpum.s.Guest.tr; + return pVCpu->cpum.s.Guest.tr.Sel; +} + + +VMMDECL(RTSEL) CPUMGetGuestCS(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CS); + return pVCpu->cpum.s.Guest.cs.Sel; +} + + +VMMDECL(RTSEL) CPUMGetGuestDS(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DS); + return pVCpu->cpum.s.Guest.ds.Sel; +} + + +VMMDECL(RTSEL) CPUMGetGuestES(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_ES); + return pVCpu->cpum.s.Guest.es.Sel; +} + + +VMMDECL(RTSEL) CPUMGetGuestFS(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_FS); + return pVCpu->cpum.s.Guest.fs.Sel; +} + + +VMMDECL(RTSEL) CPUMGetGuestGS(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_GS); + return pVCpu->cpum.s.Guest.gs.Sel; +} + + +VMMDECL(RTSEL) CPUMGetGuestSS(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_SS); + return pVCpu->cpum.s.Guest.ss.Sel; +} + + +VMMDECL(uint64_t) CPUMGetGuestFlatPC(PVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER); + CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs); + if ( !CPUMIsGuestInLongMode(pVCpu) + || !pVCpu->cpum.s.Guest.cs.Attr.n.u1Long) + return pVCpu->cpum.s.Guest.eip + (uint32_t)pVCpu->cpum.s.Guest.cs.u64Base; + return pVCpu->cpum.s.Guest.rip + pVCpu->cpum.s.Guest.cs.u64Base; +} + + +VMMDECL(uint64_t) CPUMGetGuestFlatSP(PVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER); + CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.ss); + if ( !CPUMIsGuestInLongMode(pVCpu) + || !pVCpu->cpum.s.Guest.cs.Attr.n.u1Long) + return pVCpu->cpum.s.Guest.eip + (uint32_t)pVCpu->cpum.s.Guest.ss.u64Base; + return pVCpu->cpum.s.Guest.rip + pVCpu->cpum.s.Guest.ss.u64Base; +} + + +VMMDECL(RTSEL) CPUMGetGuestLDTR(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_LDTR); + return pVCpu->cpum.s.Guest.ldtr.Sel; +} + + +VMMDECL(RTSEL) CPUMGetGuestLdtrEx(PCVMCPU pVCpu, uint64_t *pGCPtrBase, uint32_t *pcbLimit) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_LDTR); + *pGCPtrBase = pVCpu->cpum.s.Guest.ldtr.u64Base; + *pcbLimit = pVCpu->cpum.s.Guest.ldtr.u32Limit; + return pVCpu->cpum.s.Guest.ldtr.Sel; +} + + +VMMDECL(uint64_t) CPUMGetGuestCR0(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0); + return pVCpu->cpum.s.Guest.cr0; +} + + +VMMDECL(uint64_t) CPUMGetGuestCR2(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR2); + return pVCpu->cpum.s.Guest.cr2; +} + + +VMMDECL(uint64_t) CPUMGetGuestCR3(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR3); + return pVCpu->cpum.s.Guest.cr3; +} + + +VMMDECL(uint64_t) CPUMGetGuestCR4(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4); + return pVCpu->cpum.s.Guest.cr4; +} + + +VMMDECL(uint64_t) CPUMGetGuestCR8(PCVMCPUCC pVCpu) +{ + uint64_t u64; + int rc = CPUMGetGuestCRx(pVCpu, DISCREG_CR8, &u64); + if (RT_FAILURE(rc)) + u64 = 0; + return u64; +} + + +VMMDECL(void) CPUMGetGuestGDTR(PCVMCPU pVCpu, PVBOXGDTR pGDTR) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_GDTR); + *pGDTR = pVCpu->cpum.s.Guest.gdtr; +} + + +VMMDECL(uint32_t) CPUMGetGuestEIP(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP); + return pVCpu->cpum.s.Guest.eip; +} + + +VMMDECL(uint64_t) CPUMGetGuestRIP(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP); + return pVCpu->cpum.s.Guest.rip; +} + + +VMMDECL(uint32_t) CPUMGetGuestEAX(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RAX); + return pVCpu->cpum.s.Guest.eax; +} + + +VMMDECL(uint32_t) CPUMGetGuestEBX(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RBX); + return pVCpu->cpum.s.Guest.ebx; +} + + +VMMDECL(uint32_t) CPUMGetGuestECX(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RCX); + return pVCpu->cpum.s.Guest.ecx; +} + + +VMMDECL(uint32_t) CPUMGetGuestEDX(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RDX); + return pVCpu->cpum.s.Guest.edx; +} + + +VMMDECL(uint32_t) CPUMGetGuestESI(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RSI); + return pVCpu->cpum.s.Guest.esi; +} + + +VMMDECL(uint32_t) CPUMGetGuestEDI(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RDI); + return pVCpu->cpum.s.Guest.edi; +} + + +VMMDECL(uint32_t) CPUMGetGuestESP(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RSP); + return pVCpu->cpum.s.Guest.esp; +} + + +VMMDECL(uint32_t) CPUMGetGuestEBP(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RBP); + return pVCpu->cpum.s.Guest.ebp; +} + + +VMMDECL(uint32_t) CPUMGetGuestEFlags(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RFLAGS); + return pVCpu->cpum.s.Guest.eflags.u; +} + + +VMMDECL(int) CPUMGetGuestCRx(PCVMCPUCC pVCpu, unsigned iReg, uint64_t *pValue) +{ + switch (iReg) + { + case DISCREG_CR0: + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0); + *pValue = pVCpu->cpum.s.Guest.cr0; + break; + + case DISCREG_CR2: + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR2); + *pValue = pVCpu->cpum.s.Guest.cr2; + break; + + case DISCREG_CR3: + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR3); + *pValue = pVCpu->cpum.s.Guest.cr3; + break; + + case DISCREG_CR4: + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4); + *pValue = pVCpu->cpum.s.Guest.cr4; + break; + + case DISCREG_CR8: + { + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_APIC_TPR); + uint8_t u8Tpr; + int rc = APICGetTpr(pVCpu, &u8Tpr, NULL /* pfPending */, NULL /* pu8PendingIrq */); + if (RT_FAILURE(rc)) + { + AssertMsg(rc == VERR_PDM_NO_APIC_INSTANCE, ("%Rrc\n", rc)); + *pValue = 0; + return rc; + } + *pValue = u8Tpr >> 4; /* bits 7-4 contain the task priority that go in cr8, bits 3-0 */ + break; + } + + default: + return VERR_INVALID_PARAMETER; + } + return VINF_SUCCESS; +} + + +VMMDECL(uint64_t) CPUMGetGuestDR0(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + return pVCpu->cpum.s.Guest.dr[0]; +} + + +VMMDECL(uint64_t) CPUMGetGuestDR1(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + return pVCpu->cpum.s.Guest.dr[1]; +} + + +VMMDECL(uint64_t) CPUMGetGuestDR2(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + return pVCpu->cpum.s.Guest.dr[2]; +} + + +VMMDECL(uint64_t) CPUMGetGuestDR3(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + return pVCpu->cpum.s.Guest.dr[3]; +} + + +VMMDECL(uint64_t) CPUMGetGuestDR6(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR6); + return pVCpu->cpum.s.Guest.dr[6]; +} + + +VMMDECL(uint64_t) CPUMGetGuestDR7(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR7); + return pVCpu->cpum.s.Guest.dr[7]; +} + + +VMMDECL(int) CPUMGetGuestDRx(PCVMCPU pVCpu, uint32_t iReg, uint64_t *pValue) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR_MASK); + AssertReturn(iReg <= DISDREG_DR7, VERR_INVALID_PARAMETER); + /* DR4 is an alias for DR6, and DR5 is an alias for DR7. */ + if (iReg == 4 || iReg == 5) + iReg += 2; + *pValue = pVCpu->cpum.s.Guest.dr[iReg]; + return VINF_SUCCESS; +} + + +VMMDECL(uint64_t) CPUMGetGuestEFER(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_EFER); + return pVCpu->cpum.s.Guest.msrEFER; +} + + +/** + * Looks up a CPUID leaf in the CPUID leaf array, no subleaf. + * + * @returns Pointer to the leaf if found, NULL if not. + * + * @param pVM The cross context VM structure. + * @param uLeaf The leaf to get. + */ +PCPUMCPUIDLEAF cpumCpuIdGetLeaf(PVM pVM, uint32_t uLeaf) +{ + unsigned iEnd = RT_MIN(pVM->cpum.s.GuestInfo.cCpuIdLeaves, RT_ELEMENTS(pVM->cpum.s.GuestInfo.aCpuIdLeaves)); + if (iEnd) + { + unsigned iStart = 0; + PCPUMCPUIDLEAF paLeaves = pVM->cpum.s.GuestInfo.aCpuIdLeaves; + for (;;) + { + unsigned i = iStart + (iEnd - iStart) / 2U; + if (uLeaf < paLeaves[i].uLeaf) + { + if (i <= iStart) + return NULL; + iEnd = i; + } + else if (uLeaf > paLeaves[i].uLeaf) + { + i += 1; + if (i >= iEnd) + return NULL; + iStart = i; + } + else + { + if (RT_LIKELY(paLeaves[i].fSubLeafMask == 0 && paLeaves[i].uSubLeaf == 0)) + return &paLeaves[i]; + + /* This shouldn't normally happen. But in case the it does due + to user configuration overrids or something, just return the + first sub-leaf. */ + AssertMsgFailed(("uLeaf=%#x fSubLeafMask=%#x uSubLeaf=%#x\n", + uLeaf, paLeaves[i].fSubLeafMask, paLeaves[i].uSubLeaf)); + while ( paLeaves[i].uSubLeaf != 0 + && i > 0 + && uLeaf == paLeaves[i - 1].uLeaf) + i--; + return &paLeaves[i]; + } + } + } + + return NULL; +} + + +/** + * Looks up a CPUID leaf in the CPUID leaf array. + * + * @returns Pointer to the leaf if found, NULL if not. + * + * @param pVM The cross context VM structure. + * @param uLeaf The leaf to get. + * @param uSubLeaf The subleaf, if applicable. Just pass 0 if it + * isn't. + * @param pfExactSubLeafHit Whether we've got an exact subleaf hit or not. + */ +PCPUMCPUIDLEAF cpumCpuIdGetLeafEx(PVM pVM, uint32_t uLeaf, uint32_t uSubLeaf, bool *pfExactSubLeafHit) +{ + unsigned iEnd = RT_MIN(pVM->cpum.s.GuestInfo.cCpuIdLeaves, RT_ELEMENTS(pVM->cpum.s.GuestInfo.aCpuIdLeaves)); + if (iEnd) + { + unsigned iStart = 0; + PCPUMCPUIDLEAF paLeaves = pVM->cpum.s.GuestInfo.aCpuIdLeaves; + for (;;) + { + unsigned i = iStart + (iEnd - iStart) / 2U; + if (uLeaf < paLeaves[i].uLeaf) + { + if (i <= iStart) + return NULL; + iEnd = i; + } + else if (uLeaf > paLeaves[i].uLeaf) + { + i += 1; + if (i >= iEnd) + return NULL; + iStart = i; + } + else + { + uSubLeaf &= paLeaves[i].fSubLeafMask; + if (uSubLeaf == paLeaves[i].uSubLeaf) + *pfExactSubLeafHit = true; + else + { + /* Find the right subleaf. We return the last one before + uSubLeaf if we don't find an exact match. */ + if (uSubLeaf < paLeaves[i].uSubLeaf) + while ( i > 0 + && uLeaf == paLeaves[i - 1].uLeaf + && uSubLeaf <= paLeaves[i - 1].uSubLeaf) + i--; + else + while ( i + 1 < pVM->cpum.s.GuestInfo.cCpuIdLeaves + && uLeaf == paLeaves[i + 1].uLeaf + && uSubLeaf >= paLeaves[i + 1].uSubLeaf) + i++; + *pfExactSubLeafHit = uSubLeaf == paLeaves[i].uSubLeaf; + } + return &paLeaves[i]; + } + } + } + + *pfExactSubLeafHit = false; + return NULL; +} + + +/** + * Gets a CPUID leaf. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uLeaf The CPUID leaf to get. + * @param uSubLeaf The CPUID sub-leaf to get, if applicable. + * @param f64BitMode A tristate indicate if the caller is in 64-bit mode or + * not: 1=true, 0=false, 1=whatever. This affect how the + * X86_CPUID_EXT_FEATURE_EDX_SYSCALL flag is returned on + * Intel CPUs, where it's only returned in 64-bit mode. + * @param pEax Where to store the EAX value. + * @param pEbx Where to store the EBX value. + * @param pEcx Where to store the ECX value. + * @param pEdx Where to store the EDX value. + */ +VMMDECL(void) CPUMGetGuestCpuId(PVMCPUCC pVCpu, uint32_t uLeaf, uint32_t uSubLeaf, int f64BitMode, + uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx) +{ + bool fExactSubLeafHit; + PVM pVM = pVCpu->CTX_SUFF(pVM); + PCCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeafEx(pVM, uLeaf, uSubLeaf, &fExactSubLeafHit); + if (pLeaf) + { + AssertMsg(pLeaf->uLeaf == uLeaf, ("%#x %#x\n", pLeaf->uLeaf, uLeaf)); + if (fExactSubLeafHit) + { + *pEax = pLeaf->uEax; + *pEbx = pLeaf->uEbx; + *pEcx = pLeaf->uEcx; + *pEdx = pLeaf->uEdx; + + /* + * Deal with CPU specific information. + */ + if (pLeaf->fFlags & ( CPUMCPUIDLEAF_F_CONTAINS_APIC_ID + | CPUMCPUIDLEAF_F_CONTAINS_OSXSAVE + | CPUMCPUIDLEAF_F_CONTAINS_APIC )) + { + if (uLeaf == 1) + { + /* EBX: Bits 31-24: Initial APIC ID. */ + Assert(pVCpu->idCpu <= 255); + AssertMsg((pLeaf->uEbx >> 24) == 0, ("%#x\n", pLeaf->uEbx)); /* raw-mode assumption */ + *pEbx = (pLeaf->uEbx & UINT32_C(0x00ffffff)) | (pVCpu->idCpu << 24); + + /* EDX: Bit 9: AND with APICBASE.EN. */ + if (!pVCpu->cpum.s.fCpuIdApicFeatureVisible && (pLeaf->fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC)) + *pEdx &= ~X86_CPUID_FEATURE_EDX_APIC; + + /* ECX: Bit 27: CR4.OSXSAVE mirror. */ + *pEcx = (pLeaf->uEcx & ~X86_CPUID_FEATURE_ECX_OSXSAVE) + | (pVCpu->cpum.s.Guest.cr4 & X86_CR4_OSXSAVE ? X86_CPUID_FEATURE_ECX_OSXSAVE : 0); + } + else if (uLeaf == 0xb) + { + /* EDX: Initial extended APIC ID. */ + AssertMsg(pLeaf->uEdx == 0, ("%#x\n", pLeaf->uEdx)); /* raw-mode assumption */ + *pEdx = pVCpu->idCpu; + Assert(!(pLeaf->fFlags & ~(CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES))); + } + else if (uLeaf == UINT32_C(0x8000001e)) + { + /* EAX: Initial extended APIC ID. */ + AssertMsg(pLeaf->uEax == 0, ("%#x\n", pLeaf->uEax)); /* raw-mode assumption */ + *pEax = pVCpu->idCpu; + Assert(!(pLeaf->fFlags & ~CPUMCPUIDLEAF_F_CONTAINS_APIC_ID)); + } + else if (uLeaf == UINT32_C(0x80000001)) + { + /* EDX: Bit 9: AND with APICBASE.EN. */ + if (!pVCpu->cpum.s.fCpuIdApicFeatureVisible) + *pEdx &= ~X86_CPUID_AMD_FEATURE_EDX_APIC; + Assert(!(pLeaf->fFlags & ~CPUMCPUIDLEAF_F_CONTAINS_APIC)); + } + else + AssertMsgFailed(("uLeaf=%#x\n", uLeaf)); + } + + /* Intel CPUs supresses the SYSCALL bit when not executing in 64-bit mode: */ + if ( uLeaf == UINT32_C(0x80000001) + && f64BitMode == false + && (*pEdx & X86_CPUID_EXT_FEATURE_EDX_SYSCALL) + && ( pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_INTEL + || pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_VIA /*?*/ + || pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_SHANGHAI /*?*/ ) ) + *pEdx &= ~X86_CPUID_EXT_FEATURE_EDX_SYSCALL; + + } + /* + * Out of range sub-leaves aren't quite as easy and pretty as we emulate + * them here, but we do the best we can here... + */ + else + { + *pEax = *pEbx = *pEcx = *pEdx = 0; + if (pLeaf->fFlags & CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES) + { + *pEcx = uSubLeaf & 0xff; + *pEdx = pVCpu->idCpu; + } + } + } + else + { + /* + * Different CPUs have different ways of dealing with unknown CPUID leaves. + */ + switch (pVM->cpum.s.GuestInfo.enmUnknownCpuIdMethod) + { + default: + AssertFailed(); + RT_FALL_THRU(); + case CPUMUNKNOWNCPUID_DEFAULTS: + case CPUMUNKNOWNCPUID_LAST_STD_LEAF: /* ASSUME this is executed */ + case CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX: /** @todo Implement CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX */ + *pEax = pVM->cpum.s.GuestInfo.DefCpuId.uEax; + *pEbx = pVM->cpum.s.GuestInfo.DefCpuId.uEbx; + *pEcx = pVM->cpum.s.GuestInfo.DefCpuId.uEcx; + *pEdx = pVM->cpum.s.GuestInfo.DefCpuId.uEdx; + break; + case CPUMUNKNOWNCPUID_PASSTHRU: + *pEax = uLeaf; + *pEbx = 0; + *pEcx = uSubLeaf; + *pEdx = 0; + break; + } + } + Log2(("CPUMGetGuestCpuId: uLeaf=%#010x/%#010x %RX32 %RX32 %RX32 %RX32\n", uLeaf, uSubLeaf, *pEax, *pEbx, *pEcx, *pEdx)); +} + + +/** + * Sets the visibility of the X86_CPUID_FEATURE_EDX_APIC and + * X86_CPUID_AMD_FEATURE_EDX_APIC CPUID bits. + * + * @returns Previous value. + * @param pVCpu The cross context virtual CPU structure to make the + * change on. Usually the calling EMT. + * @param fVisible Whether to make it visible (true) or hide it (false). + * + * @remarks This is "VMMDECL" so that it still links with + * the old APIC code which is in VBoxDD2 and not in + * the VMM module. + */ +VMMDECL(bool) CPUMSetGuestCpuIdPerCpuApicFeature(PVMCPU pVCpu, bool fVisible) +{ + bool fOld = pVCpu->cpum.s.fCpuIdApicFeatureVisible; + pVCpu->cpum.s.fCpuIdApicFeatureVisible = fVisible; + return fOld; +} + + +/** + * Gets the host CPU vendor. + * + * @returns CPU vendor. + * @param pVM The cross context VM structure. + */ +VMMDECL(CPUMCPUVENDOR) CPUMGetHostCpuVendor(PVM pVM) +{ + return (CPUMCPUVENDOR)pVM->cpum.s.HostFeatures.enmCpuVendor; +} + + +/** + * Gets the host CPU microarchitecture. + * + * @returns CPU microarchitecture. + * @param pVM The cross context VM structure. + */ +VMMDECL(CPUMMICROARCH) CPUMGetHostMicroarch(PCVM pVM) +{ + return pVM->cpum.s.HostFeatures.enmMicroarch; +} + + +/** + * Gets the guest CPU vendor. + * + * @returns CPU vendor. + * @param pVM The cross context VM structure. + */ +VMMDECL(CPUMCPUVENDOR) CPUMGetGuestCpuVendor(PVM pVM) +{ + return (CPUMCPUVENDOR)pVM->cpum.s.GuestFeatures.enmCpuVendor; +} + + +/** + * Gets the guest CPU microarchitecture. + * + * @returns CPU microarchitecture. + * @param pVM The cross context VM structure. + */ +VMMDECL(CPUMMICROARCH) CPUMGetGuestMicroarch(PCVM pVM) +{ + return pVM->cpum.s.GuestFeatures.enmMicroarch; +} + + +/** + * Gets the maximum number of physical and linear address bits supported by the + * guest. + * + * @param pVM The cross context VM structure. + * @param pcPhysAddrWidth Where to store the physical address width. + * @param pcLinearAddrWidth Where to store the linear address width. + */ +VMMDECL(void) CPUMGetGuestAddrWidths(PCVM pVM, uint8_t *pcPhysAddrWidth, uint8_t *pcLinearAddrWidth) +{ + AssertPtr(pVM); + AssertReturnVoid(pcPhysAddrWidth); + AssertReturnVoid(pcLinearAddrWidth); + *pcPhysAddrWidth = pVM->cpum.s.GuestFeatures.cMaxPhysAddrWidth; + *pcLinearAddrWidth = pVM->cpum.s.GuestFeatures.cMaxLinearAddrWidth; +} + + +VMMDECL(int) CPUMSetGuestDR0(PVMCPUCC pVCpu, uint64_t uDr0) +{ + pVCpu->cpum.s.Guest.dr[0] = uDr0; + return CPUMRecalcHyperDRx(pVCpu, 0); +} + + +VMMDECL(int) CPUMSetGuestDR1(PVMCPUCC pVCpu, uint64_t uDr1) +{ + pVCpu->cpum.s.Guest.dr[1] = uDr1; + return CPUMRecalcHyperDRx(pVCpu, 1); +} + + +VMMDECL(int) CPUMSetGuestDR2(PVMCPUCC pVCpu, uint64_t uDr2) +{ + pVCpu->cpum.s.Guest.dr[2] = uDr2; + return CPUMRecalcHyperDRx(pVCpu, 2); +} + + +VMMDECL(int) CPUMSetGuestDR3(PVMCPUCC pVCpu, uint64_t uDr3) +{ + pVCpu->cpum.s.Guest.dr[3] = uDr3; + return CPUMRecalcHyperDRx(pVCpu, 3); +} + + +VMMDECL(int) CPUMSetGuestDR6(PVMCPU pVCpu, uint64_t uDr6) +{ + pVCpu->cpum.s.Guest.dr[6] = uDr6; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_DR6; + return VINF_SUCCESS; /* No need to recalc. */ +} + + +VMMDECL(int) CPUMSetGuestDR7(PVMCPUCC pVCpu, uint64_t uDr7) +{ + pVCpu->cpum.s.Guest.dr[7] = uDr7; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_DR7; + return CPUMRecalcHyperDRx(pVCpu, 7); +} + + +VMMDECL(int) CPUMSetGuestDRx(PVMCPUCC pVCpu, uint32_t iReg, uint64_t Value) +{ + AssertReturn(iReg <= DISDREG_DR7, VERR_INVALID_PARAMETER); + /* DR4 is an alias for DR6, and DR5 is an alias for DR7. */ + if (iReg == 4 || iReg == 5) + iReg += 2; + pVCpu->cpum.s.Guest.dr[iReg] = Value; + return CPUMRecalcHyperDRx(pVCpu, iReg); +} + + +/** + * Recalculates the hypervisor DRx register values based on current guest + * registers and DBGF breakpoints, updating changed registers depending on the + * context. + * + * This is called whenever a guest DRx register is modified (any context) and + * when DBGF sets a hardware breakpoint (ring-3 only, rendezvous). + * + * In raw-mode context this function will reload any (hyper) DRx registers which + * comes out with a different value. It may also have to save the host debug + * registers if that haven't been done already. In this context though, we'll + * be intercepting and emulating all DRx accesses, so the hypervisor DRx values + * are only important when breakpoints are actually enabled. + * + * In ring-0 (HM) context DR0-3 will be relocated by us, while DR7 will be + * reloaded by the HM code if it changes. Further more, we will only use the + * combined register set when the VBox debugger is actually using hardware BPs, + * when it isn't we'll keep the guest DR0-3 + (maybe) DR6 loaded (DR6 doesn't + * concern us here). + * + * In ring-3 we won't be loading anything, so well calculate hypervisor values + * all the time. + * + * @returns VINF_SUCCESS. + * @param pVCpu The cross context virtual CPU structure. + * @param iGstReg The guest debug register number that was modified. + * UINT8_MAX if not guest register. + */ +VMMDECL(int) CPUMRecalcHyperDRx(PVMCPUCC pVCpu, uint8_t iGstReg) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); +#ifndef IN_RING0 + RT_NOREF_PV(iGstReg); +#endif + + /* + * Compare the DR7s first. + * + * We only care about the enabled flags. GD is virtualized when we + * dispatch the #DB, we never enable it. The DBGF DR7 value is will + * always have the LE and GE bits set, so no need to check and disable + * stuff if they're cleared like we have to for the guest DR7. + */ + RTGCUINTREG uGstDr7 = CPUMGetGuestDR7(pVCpu); + /** @todo This isn't correct. BPs work without setting LE and GE under AMD-V. They are also documented as unsupported by P6+. */ + if (!(uGstDr7 & (X86_DR7_LE | X86_DR7_GE))) + uGstDr7 = 0; + else if (!(uGstDr7 & X86_DR7_LE)) + uGstDr7 &= ~X86_DR7_LE_ALL; + else if (!(uGstDr7 & X86_DR7_GE)) + uGstDr7 &= ~X86_DR7_GE_ALL; + + const RTGCUINTREG uDbgfDr7 = DBGFBpGetDR7(pVM); + if ((uGstDr7 | uDbgfDr7) & X86_DR7_ENABLED_MASK) + { + Assert(!CPUMIsGuestDebugStateActive(pVCpu)); + + /* + * Ok, something is enabled. Recalc each of the breakpoints, taking + * the VM debugger ones of the guest ones. In raw-mode context we will + * not allow breakpoints with values inside the hypervisor area. + */ + RTGCUINTREG uNewDr7 = X86_DR7_GE | X86_DR7_LE | X86_DR7_RA1_MASK; + + /* bp 0 */ + RTGCUINTREG uNewDr0; + if (uDbgfDr7 & (X86_DR7_L0 | X86_DR7_G0)) + { + uNewDr7 |= uDbgfDr7 & (X86_DR7_L0 | X86_DR7_G0 | X86_DR7_RW0_MASK | X86_DR7_LEN0_MASK); + uNewDr0 = DBGFBpGetDR0(pVM); + } + else if (uGstDr7 & (X86_DR7_L0 | X86_DR7_G0)) + { + uNewDr0 = CPUMGetGuestDR0(pVCpu); + uNewDr7 |= uGstDr7 & (X86_DR7_L0 | X86_DR7_G0 | X86_DR7_RW0_MASK | X86_DR7_LEN0_MASK); + } + else + uNewDr0 = 0; + + /* bp 1 */ + RTGCUINTREG uNewDr1; + if (uDbgfDr7 & (X86_DR7_L1 | X86_DR7_G1)) + { + uNewDr7 |= uDbgfDr7 & (X86_DR7_L1 | X86_DR7_G1 | X86_DR7_RW1_MASK | X86_DR7_LEN1_MASK); + uNewDr1 = DBGFBpGetDR1(pVM); + } + else if (uGstDr7 & (X86_DR7_L1 | X86_DR7_G1)) + { + uNewDr1 = CPUMGetGuestDR1(pVCpu); + uNewDr7 |= uGstDr7 & (X86_DR7_L1 | X86_DR7_G1 | X86_DR7_RW1_MASK | X86_DR7_LEN1_MASK); + } + else + uNewDr1 = 0; + + /* bp 2 */ + RTGCUINTREG uNewDr2; + if (uDbgfDr7 & (X86_DR7_L2 | X86_DR7_G2)) + { + uNewDr7 |= uDbgfDr7 & (X86_DR7_L2 | X86_DR7_G2 | X86_DR7_RW2_MASK | X86_DR7_LEN2_MASK); + uNewDr2 = DBGFBpGetDR2(pVM); + } + else if (uGstDr7 & (X86_DR7_L2 | X86_DR7_G2)) + { + uNewDr2 = CPUMGetGuestDR2(pVCpu); + uNewDr7 |= uGstDr7 & (X86_DR7_L2 | X86_DR7_G2 | X86_DR7_RW2_MASK | X86_DR7_LEN2_MASK); + } + else + uNewDr2 = 0; + + /* bp 3 */ + RTGCUINTREG uNewDr3; + if (uDbgfDr7 & (X86_DR7_L3 | X86_DR7_G3)) + { + uNewDr7 |= uDbgfDr7 & (X86_DR7_L3 | X86_DR7_G3 | X86_DR7_RW3_MASK | X86_DR7_LEN3_MASK); + uNewDr3 = DBGFBpGetDR3(pVM); + } + else if (uGstDr7 & (X86_DR7_L3 | X86_DR7_G3)) + { + uNewDr3 = CPUMGetGuestDR3(pVCpu); + uNewDr7 |= uGstDr7 & (X86_DR7_L3 | X86_DR7_G3 | X86_DR7_RW3_MASK | X86_DR7_LEN3_MASK); + } + else + uNewDr3 = 0; + + /* + * Apply the updates. + */ + pVCpu->cpum.s.fUseFlags |= CPUM_USE_DEBUG_REGS_HYPER; + if (uNewDr3 != pVCpu->cpum.s.Hyper.dr[3]) + CPUMSetHyperDR3(pVCpu, uNewDr3); + if (uNewDr2 != pVCpu->cpum.s.Hyper.dr[2]) + CPUMSetHyperDR2(pVCpu, uNewDr2); + if (uNewDr1 != pVCpu->cpum.s.Hyper.dr[1]) + CPUMSetHyperDR1(pVCpu, uNewDr1); + if (uNewDr0 != pVCpu->cpum.s.Hyper.dr[0]) + CPUMSetHyperDR0(pVCpu, uNewDr0); + if (uNewDr7 != pVCpu->cpum.s.Hyper.dr[7]) + CPUMSetHyperDR7(pVCpu, uNewDr7); + } +#ifdef IN_RING0 + else if (CPUMIsGuestDebugStateActive(pVCpu)) + { + /* + * Reload the register that was modified. Normally this won't happen + * as we won't intercept DRx writes when not having the hyper debug + * state loaded, but in case we do for some reason we'll simply deal + * with it. + */ + switch (iGstReg) + { + case 0: ASMSetDR0(CPUMGetGuestDR0(pVCpu)); break; + case 1: ASMSetDR1(CPUMGetGuestDR1(pVCpu)); break; + case 2: ASMSetDR2(CPUMGetGuestDR2(pVCpu)); break; + case 3: ASMSetDR3(CPUMGetGuestDR3(pVCpu)); break; + default: + AssertReturn(iGstReg != UINT8_MAX, VERR_INTERNAL_ERROR_3); + } + } +#endif + else + { + /* + * No active debug state any more. In raw-mode this means we have to + * make sure DR7 has everything disabled now, if we armed it already. + * In ring-0 we might end up here when just single stepping. + */ +#ifdef IN_RING0 + if (pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_HYPER) + { + if (pVCpu->cpum.s.Hyper.dr[0]) + ASMSetDR0(0); + if (pVCpu->cpum.s.Hyper.dr[1]) + ASMSetDR1(0); + if (pVCpu->cpum.s.Hyper.dr[2]) + ASMSetDR2(0); + if (pVCpu->cpum.s.Hyper.dr[3]) + ASMSetDR3(0); + pVCpu->cpum.s.fUseFlags &= ~CPUM_USED_DEBUG_REGS_HYPER; + } +#endif + pVCpu->cpum.s.fUseFlags &= ~CPUM_USE_DEBUG_REGS_HYPER; + + /* Clear all the registers. */ + pVCpu->cpum.s.Hyper.dr[7] = X86_DR7_RA1_MASK; + pVCpu->cpum.s.Hyper.dr[3] = 0; + pVCpu->cpum.s.Hyper.dr[2] = 0; + pVCpu->cpum.s.Hyper.dr[1] = 0; + pVCpu->cpum.s.Hyper.dr[0] = 0; + + } + Log2(("CPUMRecalcHyperDRx: fUseFlags=%#x %RGr %RGr %RGr %RGr %RGr %RGr\n", + pVCpu->cpum.s.fUseFlags, pVCpu->cpum.s.Hyper.dr[0], pVCpu->cpum.s.Hyper.dr[1], + pVCpu->cpum.s.Hyper.dr[2], pVCpu->cpum.s.Hyper.dr[3], pVCpu->cpum.s.Hyper.dr[6], + pVCpu->cpum.s.Hyper.dr[7])); + + return VINF_SUCCESS; +} + + +/** + * Set the guest XCR0 register. + * + * Will load additional state if the FPU state is already loaded (in ring-0 & + * raw-mode context). + * + * @returns VINF_SUCCESS on success, VERR_CPUM_RAISE_GP_0 on invalid input + * value. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uNewValue The new value. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(int) CPUMSetGuestXcr0(PVMCPUCC pVCpu, uint64_t uNewValue) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_XCRx); + if ( (uNewValue & ~pVCpu->CTX_SUFF(pVM)->cpum.s.fXStateGuestMask) == 0 + /* The X87 bit cannot be cleared. */ + && (uNewValue & XSAVE_C_X87) + /* AVX requires SSE. */ + && (uNewValue & (XSAVE_C_SSE | XSAVE_C_YMM)) != XSAVE_C_YMM + /* AVX-512 requires YMM, SSE and all of its three components to be enabled. */ + && ( (uNewValue & (XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI)) == 0 + || (uNewValue & (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI)) + == (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI) ) + ) + { + pVCpu->cpum.s.Guest.aXcr[0] = uNewValue; + + /* If more state components are enabled, we need to take care to load + them if the FPU/SSE state is already loaded. May otherwise leak + host state to the guest. */ + uint64_t fNewComponents = ~pVCpu->cpum.s.Guest.fXStateMask & uNewValue; + if (fNewComponents) + { +#ifdef IN_RING0 + if (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST) + { + if (pVCpu->cpum.s.Guest.fXStateMask != 0) + /* Adding more components. */ + ASMXRstor(&pVCpu->cpum.s.Guest.XState, fNewComponents); + else + { + /* We're switching from FXSAVE/FXRSTOR to XSAVE/XRSTOR. */ + pVCpu->cpum.s.Guest.fXStateMask |= XSAVE_C_X87 | XSAVE_C_SSE; + if (uNewValue & ~(XSAVE_C_X87 | XSAVE_C_SSE)) + ASMXRstor(&pVCpu->cpum.s.Guest.XState, uNewValue & ~(XSAVE_C_X87 | XSAVE_C_SSE)); + } + } +#endif + pVCpu->cpum.s.Guest.fXStateMask |= uNewValue; + } + return VINF_SUCCESS; + } + return VERR_CPUM_RAISE_GP_0; +} + + +/** + * Tests if the guest has No-Execute Page Protection Enabled (NXE). + * + * @returns true if in real mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestNXEnabled(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_EFER); + return !!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_NXE); +} + + +/** + * Tests if the guest has the Page Size Extension enabled (PSE). + * + * @returns true if in real mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestPageSizeExtEnabled(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4); + /* PAE or AMD64 implies support for big pages regardless of CR4.PSE */ + return !!(pVCpu->cpum.s.Guest.cr4 & (X86_CR4_PSE | X86_CR4_PAE)); +} + + +/** + * Tests if the guest has the paging enabled (PG). + * + * @returns true if in real mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestPagingEnabled(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0); + return !!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PG); +} + + +/** + * Tests if the guest has the paging enabled (PG). + * + * @returns true if in real mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestR0WriteProtEnabled(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0); + return !!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_WP); +} + + +/** + * Tests if the guest is running in real mode or not. + * + * @returns true if in real mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestInRealMode(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0); + return !(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE); +} + + +/** + * Tests if the guest is running in real or virtual 8086 mode. + * + * @returns @c true if it is, @c false if not. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestInRealOrV86Mode(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS); + return !(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE) + || pVCpu->cpum.s.Guest.eflags.Bits.u1VM; /** @todo verify that this cannot be set in long mode. */ +} + + +/** + * Tests if the guest is running in protected or not. + * + * @returns true if in protected mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestInProtectedMode(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0); + return !!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE); +} + + +/** + * Tests if the guest is running in paged protected or not. + * + * @returns true if in paged protected mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestInPagedProtectedMode(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0); + return (pVCpu->cpum.s.Guest.cr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG); +} + + +/** + * Tests if the guest is running in long mode or not. + * + * @returns true if in long mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestInLongMode(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_EFER); + return (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA) == MSR_K6_EFER_LMA; +} + + +/** + * Tests if the guest is running in PAE mode or not. + * + * @returns true if in PAE mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestInPAEMode(PCVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER); + /* Intel mentions EFER.LMA and EFER.LME in different parts of their spec. We shall use EFER.LMA rather + than EFER.LME as it reflects if the CPU has entered paging with EFER.LME set. */ + return (pVCpu->cpum.s.Guest.cr4 & X86_CR4_PAE) + && (pVCpu->cpum.s.Guest.cr0 & X86_CR0_PG) + && !(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA); +} + + +/** + * Tests if the guest is running in 64 bits mode or not. + * + * @returns true if in 64 bits protected mode, otherwise false. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(bool) CPUMIsGuestIn64BitCode(PVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER); + if (!CPUMIsGuestInLongMode(pVCpu)) + return false; + CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs); + return pVCpu->cpum.s.Guest.cs.Attr.n.u1Long; +} + + +/** + * Helper for CPUMIsGuestIn64BitCodeEx that handles lazy resolving of hidden CS + * registers. + * + * @returns true if in 64 bits protected mode, otherwise false. + * @param pCtx Pointer to the current guest CPU context. + */ +VMM_INT_DECL(bool) CPUMIsGuestIn64BitCodeSlow(PCPUMCTX pCtx) +{ + return CPUMIsGuestIn64BitCode(CPUM_GUEST_CTX_TO_VMCPU(pCtx)); +} + + +/** + * Sets the specified changed flags (CPUM_CHANGED_*). + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param fChangedAdd The changed flags to add. + */ +VMMDECL(void) CPUMSetChangedFlags(PVMCPU pVCpu, uint32_t fChangedAdd) +{ + pVCpu->cpum.s.fChanged |= fChangedAdd; +} + + +/** + * Checks if the CPU supports the XSAVE and XRSTOR instruction. + * + * @returns true if supported. + * @returns false if not supported. + * @param pVM The cross context VM structure. + */ +VMMDECL(bool) CPUMSupportsXSave(PVM pVM) +{ + return pVM->cpum.s.HostFeatures.fXSaveRstor != 0; +} + + +/** + * Checks if the host OS uses the SYSENTER / SYSEXIT instructions. + * @returns true if used. + * @returns false if not used. + * @param pVM The cross context VM structure. + */ +VMMDECL(bool) CPUMIsHostUsingSysEnter(PVM pVM) +{ + return RT_BOOL(pVM->cpum.s.fHostUseFlags & CPUM_USE_SYSENTER); +} + + +/** + * Checks if the host OS uses the SYSCALL / SYSRET instructions. + * @returns true if used. + * @returns false if not used. + * @param pVM The cross context VM structure. + */ +VMMDECL(bool) CPUMIsHostUsingSysCall(PVM pVM) +{ + return RT_BOOL(pVM->cpum.s.fHostUseFlags & CPUM_USE_SYSCALL); +} + + +/** + * Checks if we activated the FPU/XMM state of the guest OS. + * + * Obsolete: This differs from CPUMIsGuestFPUStateLoaded() in that it refers to + * the next time we'll be executing guest code, so it may return true for + * 64-on-32 when we still haven't actually loaded the FPU status, just scheduled + * it to be loaded the next time we go thru the world switcher + * (CPUM_SYNC_FPU_STATE). + * + * @returns true / false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestFPUStateActive(PVMCPU pVCpu) +{ + bool fRet = RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST); + AssertMsg(fRet == pVCpu->cpum.s.Guest.fUsedFpuGuest, ("fRet=%d\n", fRet)); + return fRet; +} + + +/** + * Checks if we've really loaded the FPU/XMM state of the guest OS. + * + * @returns true / false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsGuestFPUStateLoaded(PVMCPU pVCpu) +{ + bool fRet = RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST); + AssertMsg(fRet == pVCpu->cpum.s.Guest.fUsedFpuGuest, ("fRet=%d\n", fRet)); + return fRet; +} + + +/** + * Checks if we saved the FPU/XMM state of the host OS. + * + * @returns true / false. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) CPUMIsHostFPUStateSaved(PVMCPU pVCpu) +{ + return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_HOST); +} + + +/** + * Checks if the guest debug state is active. + * + * @returns boolean + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(bool) CPUMIsGuestDebugStateActive(PVMCPU pVCpu) +{ + return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_GUEST); +} + + +/** + * Checks if the hyper debug state is active. + * + * @returns boolean + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(bool) CPUMIsHyperDebugStateActive(PVMCPU pVCpu) +{ + return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_HYPER); +} + + +/** + * Mark the guest's debug state as inactive. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @todo This API doesn't make sense any more. + */ +VMMDECL(void) CPUMDeactivateGuestDebugState(PVMCPU pVCpu) +{ + Assert(!(pVCpu->cpum.s.fUseFlags & (CPUM_USED_DEBUG_REGS_GUEST | CPUM_USED_DEBUG_REGS_HYPER | CPUM_USED_DEBUG_REGS_HOST))); + NOREF(pVCpu); +} + + +/** + * Get the current privilege level of the guest. + * + * @returns CPL + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(uint32_t) CPUMGetGuestCPL(PVMCPU pVCpu) +{ + /* + * CPL can reliably be found in SS.DPL (hidden regs valid) or SS if not. + * + * Note! We used to check CS.DPL here, assuming it was always equal to + * CPL even if a conforming segment was loaded. But this turned out to + * only apply to older AMD-V. With VT-x we had an ACP2 regression + * during install after a far call to ring 2 with VT-x. Then on newer + * AMD-V CPUs we have to move the VMCB.guest.u8CPL into cs.Attr.n.u2Dpl + * as well as ss.Attr.n.u2Dpl to make this (and other) code work right. + * + * So, forget CS.DPL, always use SS.DPL. + * + * Note! The SS RPL is always equal to the CPL, while the CS RPL + * isn't necessarily equal if the segment is conforming. + * See section 4.11.1 in the AMD manual. + * + * Update: Where the heck does it say CS.RPL can differ from CPL other than + * right after real->prot mode switch and when in V8086 mode? That + * section says the RPL specified in a direct transfere (call, jmp, + * ret) is not the one loaded into CS. Besides, if CS.RPL != CPL + * it would be impossible for an exception handle or the iret + * instruction to figure out whether SS:ESP are part of the frame + * or not. VBox or qemu bug must've lead to this misconception. + * + * Update2: On an AMD bulldozer system here, I've no trouble loading a null + * selector into SS with an RPL other than the CPL when CPL != 3 and + * we're in 64-bit mode. The intel dev box doesn't allow this, on + * RPL = CPL. Weird. + */ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS); + uint32_t uCpl; + if (pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE) + { + if (!pVCpu->cpum.s.Guest.eflags.Bits.u1VM) + { + if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.s.Guest.ss)) + uCpl = pVCpu->cpum.s.Guest.ss.Attr.n.u2Dpl; + else + uCpl = (pVCpu->cpum.s.Guest.ss.Sel & X86_SEL_RPL); + } + else + uCpl = 3; /* V86 has CPL=3; REM doesn't set DPL=3 in V8086 mode. See @bugref{5130}. */ + } + else + uCpl = 0; /* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */ + return uCpl; +} + + +/** + * Gets the current guest CPU mode. + * + * If paging mode is what you need, check out PGMGetGuestMode(). + * + * @returns The CPU mode. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(CPUMMODE) CPUMGetGuestMode(PVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER); + CPUMMODE enmMode; + if (!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE)) + enmMode = CPUMMODE_REAL; + else if (!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA)) + enmMode = CPUMMODE_PROTECTED; + else + enmMode = CPUMMODE_LONG; + + return enmMode; +} + + +/** + * Figure whether the CPU is currently executing 16, 32 or 64 bit code. + * + * @returns 16, 32 or 64. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(uint32_t) CPUMGetGuestCodeBits(PVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS); + + if (!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE)) + return 16; + + if (pVCpu->cpum.s.Guest.eflags.Bits.u1VM) + { + Assert(!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA)); + return 16; + } + + CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs); + if ( pVCpu->cpum.s.Guest.cs.Attr.n.u1Long + && (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA)) + return 64; + + if (pVCpu->cpum.s.Guest.cs.Attr.n.u1DefBig) + return 32; + + return 16; +} + + +VMMDECL(DISCPUMODE) CPUMGetGuestDisMode(PVMCPU pVCpu) +{ + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS); + + if (!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE)) + return DISCPUMODE_16BIT; + + if (pVCpu->cpum.s.Guest.eflags.Bits.u1VM) + { + Assert(!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA)); + return DISCPUMODE_16BIT; + } + + CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs); + if ( pVCpu->cpum.s.Guest.cs.Attr.n.u1Long + && (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA)) + return DISCPUMODE_64BIT; + + if (pVCpu->cpum.s.Guest.cs.Attr.n.u1DefBig) + return DISCPUMODE_32BIT; + + return DISCPUMODE_16BIT; +} + + +/** + * Gets the guest MXCSR_MASK value. + * + * This does not access the x87 state, but the value we determined at VM + * initialization. + * + * @returns MXCSR mask. + * @param pVM The cross context VM structure. + */ +VMMDECL(uint32_t) CPUMGetGuestMxCsrMask(PVM pVM) +{ + return pVM->cpum.s.GuestInfo.fMxCsrMask; +} + + +/** + * Returns whether the guest has physical interrupts enabled. + * + * @returns @c true if interrupts are enabled, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks Warning! This function does -not- take into account the global-interrupt + * flag (GIF). + */ +VMM_INT_DECL(bool) CPUMIsGuestPhysIntrEnabled(PVMCPU pVCpu) +{ + switch (CPUMGetGuestInNestedHwvirtMode(&pVCpu->cpum.s.Guest)) + { + case CPUMHWVIRT_NONE: + default: + return pVCpu->cpum.s.Guest.eflags.Bits.u1IF; + case CPUMHWVIRT_VMX: + return CPUMIsGuestVmxPhysIntrEnabled(&pVCpu->cpum.s.Guest); + case CPUMHWVIRT_SVM: + return CPUMIsGuestSvmPhysIntrEnabled(pVCpu, &pVCpu->cpum.s.Guest); + } +} + + +/** + * Returns whether the nested-guest has virtual interrupts enabled. + * + * @returns @c true if interrupts are enabled, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks Warning! This function does -not- take into account the global-interrupt + * flag (GIF). + */ +VMM_INT_DECL(bool) CPUMIsGuestVirtIntrEnabled(PVMCPU pVCpu) +{ + PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest; + Assert(CPUMIsGuestInNestedHwvirtMode(pCtx)); + + if (CPUMIsGuestInVmxNonRootMode(pCtx)) + return CPUMIsGuestVmxVirtIntrEnabled(pCtx); + + Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); + return CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx); +} + + +/** + * Calculates the interruptiblity of the guest. + * + * @returns Interruptibility level. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(CPUMINTERRUPTIBILITY) CPUMGetGuestInterruptibility(PVMCPU pVCpu) +{ +#if 1 + /* Global-interrupt flag blocks pretty much everything we care about here. */ + if (CPUMGetGuestGif(&pVCpu->cpum.s.Guest)) + { + /* + * Physical interrupts are primarily blocked using EFLAGS. However, we cannot access + * it directly here. If and how EFLAGS are used depends on the context (nested-guest + * or raw-mode). Hence we use the function below which handles the details. + */ + if ( !(pVCpu->cpum.s.Guest.eflags.uBoth & CPUMCTX_INHIBIT_ALL_MASK) + || ( !(pVCpu->cpum.s.Guest.eflags.uBoth & CPUMCTX_INHIBIT_NMI) + && pVCpu->cpum.s.Guest.uRipInhibitInt != pVCpu->cpum.s.Guest.rip)) + { + /** @todo OPT: this next call should be inlined! */ + if (CPUMIsGuestPhysIntrEnabled(pVCpu)) + { + /** @todo OPT: type this out as it repeats tests. */ + if ( !CPUMIsGuestInNestedHwvirtMode(&pVCpu->cpum.s.Guest) + || CPUMIsGuestVirtIntrEnabled(pVCpu)) + return CPUMINTERRUPTIBILITY_UNRESTRAINED; + + /* Physical interrupts are enabled, but nested-guest virtual interrupts are disabled. */ + return CPUMINTERRUPTIBILITY_VIRT_INT_DISABLED; + } + return CPUMINTERRUPTIBILITY_INT_DISABLED; + } + + /* + * Blocking the delivery of NMIs during an interrupt shadow is CPU implementation + * specific. Therefore, in practice, we can't deliver an NMI in an interrupt shadow. + * However, there is some uncertainity regarding the converse, i.e. whether + * NMI-blocking until IRET blocks delivery of physical interrupts. + * + * See Intel spec. 25.4.1 "Event Blocking". + */ + /** @todo r=bird: The above comment mixes up VMX root-mode and non-root. Section + * 25.4.1 is only applicable to VMX non-root mode. In root mode / + * non-VMX mode, I have not see any evidence in the intel manuals that + * NMIs are not blocked when in an interrupt shadow. Section "6.7 + * NONMASKABLE INTERRUPT (NMI)" in SDM 3A seems pretty clear to me. + */ + if (!(pVCpu->cpum.s.Guest.eflags.uBoth & CPUMCTX_INHIBIT_NMI)) + return CPUMINTERRUPTIBILITY_INT_INHIBITED; + return CPUMINTERRUPTIBILITY_NMI_INHIBIT; + } + return CPUMINTERRUPTIBILITY_GLOBAL_INHIBIT; +#else + if (pVCpu->cpum.s.Guest.rflags.Bits.u1IF) + { + if (pVCpu->cpum.s.Guest.hwvirt.fGif) + { + if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS | VMCPU_FF_INHIBIT_INTERRUPTS)) + return CPUMINTERRUPTIBILITY_UNRESTRAINED; + + /** @todo does blocking NMIs mean interrupts are also inhibited? */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)) + { + if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)) + return CPUMINTERRUPTIBILITY_INT_INHIBITED; + return CPUMINTERRUPTIBILITY_NMI_INHIBIT; + } + AssertFailed(); + return CPUMINTERRUPTIBILITY_NMI_INHIBIT; + } + return CPUMINTERRUPTIBILITY_GLOBAL_INHIBIT; + } + else + { + if (pVCpu->cpum.s.Guest.hwvirt.fGif) + { + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)) + return CPUMINTERRUPTIBILITY_NMI_INHIBIT; + return CPUMINTERRUPTIBILITY_INT_DISABLED; + } + return CPUMINTERRUPTIBILITY_GLOBAL_INHIBIT; + } +#endif +} + + +/** + * Checks whether the SVM nested-guest has physical interrupts enabled. + * + * @returns true if interrupts are enabled, false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pCtx The guest-CPU context. + * + * @remarks This does -not- take into account the global-interrupt flag. + */ +VMM_INT_DECL(bool) CPUMIsGuestSvmPhysIntrEnabled(PCVMCPU pVCpu, PCCPUMCTX pCtx) +{ + /** @todo Optimization: Avoid this function call and use a pointer to the + * relevant eflags instead (setup during VMRUN instruction emulation). */ + Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); + + X86EFLAGS fEFlags; + if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, pCtx)) + fEFlags.u = pCtx->hwvirt.svm.HostState.rflags.u; + else + fEFlags.u = pCtx->eflags.u; + + return fEFlags.Bits.u1IF; +} + + +/** + * Checks whether the SVM nested-guest is in a state to receive virtual (setup + * for injection by VMRUN instruction) interrupts. + * + * @returns VBox status code. + * @retval true if it's ready, false otherwise. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pCtx The guest-CPU context. + */ +VMM_INT_DECL(bool) CPUMIsGuestSvmVirtIntrEnabled(PCVMCPU pVCpu, PCCPUMCTX pCtx) +{ + RT_NOREF(pVCpu); + Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); + + PCSVMVMCBCTRL pVmcbCtrl = &pCtx->hwvirt.svm.Vmcb.ctrl; + PCSVMINTCTRL pVmcbIntCtrl = &pVmcbCtrl->IntCtrl; + Assert(!pVmcbIntCtrl->n.u1VGifEnable); /* We don't support passing virtual-GIF feature to the guest yet. */ + if ( !pVmcbIntCtrl->n.u1IgnoreTPR + && pVmcbIntCtrl->n.u4VIntrPrio <= pVmcbIntCtrl->n.u8VTPR) + return false; + + return RT_BOOL(pCtx->eflags.u & X86_EFL_IF); +} + + +/** + * Gets the pending SVM nested-guest interruptvector. + * + * @returns The nested-guest interrupt to inject. + * @param pCtx The guest-CPU context. + */ +VMM_INT_DECL(uint8_t) CPUMGetGuestSvmVirtIntrVector(PCCPUMCTX pCtx) +{ + return pCtx->hwvirt.svm.Vmcb.ctrl.IntCtrl.n.u8VIntrVector; +} + + +/** + * Restores the host-state from the host-state save area as part of a \#VMEXIT. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pCtx The guest-CPU context. + */ +VMM_INT_DECL(void) CPUMSvmVmExitRestoreHostState(PVMCPUCC pVCpu, PCPUMCTX pCtx) +{ + /* + * Reload the guest's "host state". + */ + PSVMHOSTSTATE pHostState = &pCtx->hwvirt.svm.HostState; + pCtx->es = pHostState->es; + pCtx->cs = pHostState->cs; + pCtx->ss = pHostState->ss; + pCtx->ds = pHostState->ds; + pCtx->gdtr = pHostState->gdtr; + pCtx->idtr = pHostState->idtr; + CPUMSetGuestEferMsrNoChecks(pVCpu, pCtx->msrEFER, pHostState->uEferMsr); + CPUMSetGuestCR0(pVCpu, pHostState->uCr0 | X86_CR0_PE); + pCtx->cr3 = pHostState->uCr3; + CPUMSetGuestCR4(pVCpu, pHostState->uCr4); + pCtx->rflags.u = pHostState->rflags.u; + pCtx->rflags.Bits.u1VM = 0; + pCtx->rip = pHostState->uRip; + pCtx->rsp = pHostState->uRsp; + pCtx->rax = pHostState->uRax; + pCtx->dr[7] &= ~(X86_DR7_ENABLED_MASK | X86_DR7_RAZ_MASK | X86_DR7_MBZ_MASK); + pCtx->dr[7] |= X86_DR7_RA1_MASK; + Assert(pCtx->ss.Attr.n.u2Dpl == 0); + + /** @todo if RIP is not canonical or outside the CS segment limit, we need to + * raise \#GP(0) in the guest. */ + + /** @todo check the loaded host-state for consistency. Figure out what + * exactly this involves? */ +} + + +/** + * Saves the host-state to the host-state save area as part of a VMRUN. + * + * @param pCtx The guest-CPU context. + * @param cbInstr The length of the VMRUN instruction in bytes. + */ +VMM_INT_DECL(void) CPUMSvmVmRunSaveHostState(PCPUMCTX pCtx, uint8_t cbInstr) +{ + PSVMHOSTSTATE pHostState = &pCtx->hwvirt.svm.HostState; + pHostState->es = pCtx->es; + pHostState->cs = pCtx->cs; + pHostState->ss = pCtx->ss; + pHostState->ds = pCtx->ds; + pHostState->gdtr = pCtx->gdtr; + pHostState->idtr = pCtx->idtr; + pHostState->uEferMsr = pCtx->msrEFER; + pHostState->uCr0 = pCtx->cr0; + pHostState->uCr3 = pCtx->cr3; + pHostState->uCr4 = pCtx->cr4; + pHostState->rflags.u = pCtx->rflags.u; + pHostState->uRip = pCtx->rip + cbInstr; + pHostState->uRsp = pCtx->rsp; + pHostState->uRax = pCtx->rax; +} + + +/** + * Applies the TSC offset of a nested-guest if any and returns the TSC value for the + * nested-guest. + * + * @returns The TSC offset after applying any nested-guest TSC offset. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uTscValue The guest TSC. + * + * @sa CPUMRemoveNestedGuestTscOffset. + */ +VMM_INT_DECL(uint64_t) CPUMApplyNestedGuestTscOffset(PCVMCPU pVCpu, uint64_t uTscValue) +{ + PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest; + if (CPUMIsGuestInVmxNonRootMode(pCtx)) + { + if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_USE_TSC_OFFSETTING)) + return uTscValue + pCtx->hwvirt.vmx.Vmcs.u64TscOffset.u; + return uTscValue; + } + + if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx)) + { + uint64_t offTsc; + if (!HMGetGuestSvmTscOffset(pVCpu, &offTsc)) + offTsc = pCtx->hwvirt.svm.Vmcb.ctrl.u64TSCOffset; + return uTscValue + offTsc; + } + return uTscValue; +} + + +/** + * Removes the TSC offset of a nested-guest if any and returns the TSC value for the + * guest. + * + * @returns The TSC offset after removing any nested-guest TSC offset. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uTscValue The nested-guest TSC. + * + * @sa CPUMApplyNestedGuestTscOffset. + */ +VMM_INT_DECL(uint64_t) CPUMRemoveNestedGuestTscOffset(PCVMCPU pVCpu, uint64_t uTscValue) +{ + PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest; + if (CPUMIsGuestInVmxNonRootMode(pCtx)) + { + if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_USE_TSC_OFFSETTING)) + return uTscValue - pCtx->hwvirt.vmx.Vmcs.u64TscOffset.u; + return uTscValue; + } + + if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx)) + { + uint64_t offTsc; + if (!HMGetGuestSvmTscOffset(pVCpu, &offTsc)) + offTsc = pCtx->hwvirt.svm.Vmcb.ctrl.u64TSCOffset; + return uTscValue - offTsc; + } + return uTscValue; +} + + +/** + * Used to dynamically imports state residing in NEM or HM. + * + * This is a worker for the CPUM_IMPORT_EXTRN_RET() macro and various IEM ones. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param fExtrnImport The fields to import. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(int) CPUMImportGuestStateOnDemand(PVMCPUCC pVCpu, uint64_t fExtrnImport) +{ + VMCPU_ASSERT_EMT(pVCpu); + if (pVCpu->cpum.s.Guest.fExtrn & fExtrnImport) + { + switch (pVCpu->cpum.s.Guest.fExtrn & CPUMCTX_EXTRN_KEEPER_MASK) + { + case CPUMCTX_EXTRN_KEEPER_NEM: + { + int rc = NEMImportStateOnDemand(pVCpu, fExtrnImport); + Assert(rc == VINF_SUCCESS || RT_FAILURE_NP(rc)); + return rc; + } + + case CPUMCTX_EXTRN_KEEPER_HM: + { +#ifdef IN_RING0 + int rc = HMR0ImportStateOnDemand(pVCpu, fExtrnImport); + Assert(rc == VINF_SUCCESS || RT_FAILURE_NP(rc)); + return rc; +#else + AssertLogRelMsgFailed(("TODO Fetch HM state: %#RX64 vs %#RX64\n", pVCpu->cpum.s.Guest.fExtrn, fExtrnImport)); + return VINF_SUCCESS; +#endif + } + default: + AssertLogRelMsgFailedReturn(("%#RX64 vs %#RX64\n", pVCpu->cpum.s.Guest.fExtrn, fExtrnImport), VERR_CPUM_IPE_2); + } + } + return VINF_SUCCESS; +} + + +/** + * Gets valid CR4 bits for the guest. + * + * @returns Valid CR4 bits. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(uint64_t) CPUMGetGuestCR4ValidMask(PVM pVM) +{ + PCCPUMFEATURES pGuestFeatures = &pVM->cpum.s.GuestFeatures; + uint64_t fMask = X86_CR4_VME | X86_CR4_PVI + | X86_CR4_TSD | X86_CR4_DE + | X86_CR4_MCE | X86_CR4_PCE; + if (pGuestFeatures->fPae) + fMask |= X86_CR4_PAE; + if (pGuestFeatures->fPge) + fMask |= X86_CR4_PGE; + if (pGuestFeatures->fPse) + fMask |= X86_CR4_PSE; + if (pGuestFeatures->fFxSaveRstor) + fMask |= X86_CR4_OSFXSR; + if (pGuestFeatures->fVmx) + fMask |= X86_CR4_VMXE; + if (pGuestFeatures->fXSaveRstor) + fMask |= X86_CR4_OSXSAVE; + if (pGuestFeatures->fPcid) + fMask |= X86_CR4_PCIDE; + if (pGuestFeatures->fFsGsBase) + fMask |= X86_CR4_FSGSBASE; + if (pGuestFeatures->fSse) + fMask |= X86_CR4_OSXMMEEXCPT; + return fMask; +} + + +/** + * Sets the PAE PDPEs for the guest. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param paPaePdpes The PAE PDPEs to set. + */ +VMM_INT_DECL(void) CPUMSetGuestPaePdpes(PVMCPU pVCpu, PCX86PDPE paPaePdpes) +{ + Assert(paPaePdpes); + for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->cpum.s.Guest.aPaePdpes); i++) + pVCpu->cpum.s.Guest.aPaePdpes[i].u = paPaePdpes[i].u; + pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR3; +} + + +/** + * Gets the PAE PDPTEs for the guest. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param paPaePdpes Where to store the PAE PDPEs. + */ +VMM_INT_DECL(void) CPUMGetGuestPaePdpes(PVMCPU pVCpu, PX86PDPE paPaePdpes) +{ + Assert(paPaePdpes); + CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR3); + for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->cpum.s.Guest.aPaePdpes); i++) + paPaePdpes[i].u = pVCpu->cpum.s.Guest.aPaePdpes[i].u; +} + + +/** + * Starts a VMX-preemption timer to expire as specified by the nested hypervisor. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param uTimer The VMCS preemption timer value. + * @param cShift The VMX-preemption timer shift (usually based on guest + * VMX MSR rate). + * @param pu64EntryTick Where to store the current tick when the timer is + * programmed. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(int) CPUMStartGuestVmxPremptTimer(PVMCPUCC pVCpu, uint32_t uTimer, uint8_t cShift, uint64_t *pu64EntryTick) +{ + Assert(uTimer); + Assert(cShift <= 31); + Assert(pu64EntryTick); + VMCPU_ASSERT_EMT(pVCpu); + uint64_t const cTicksToNext = uTimer << cShift; + return TMTimerSetRelative(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.s.hNestedVmxPreemptTimer, cTicksToNext, pu64EntryTick); +} + + +/** + * Stops the VMX-preemption timer from firing. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @thread EMT. + * + * @remarks This can be called during VM reset, so we cannot assume it will be on + * the EMT corresponding to @c pVCpu. + */ +VMM_INT_DECL(int) CPUMStopGuestVmxPremptTimer(PVMCPUCC pVCpu) +{ + /* + * CPUM gets initialized before TM, so we defer creation of timers till CPUMR3InitCompleted(). + * However, we still get called during CPUMR3Init() and hence we need to check if we have + * a valid timer object before trying to stop it. + */ + int rc; + TMTIMERHANDLE hTimer = pVCpu->cpum.s.hNestedVmxPreemptTimer; + if (hTimer != NIL_TMTIMERHANDLE) + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + rc = TMTimerLock(pVM, hTimer, VERR_IGNORED); + if (rc == VINF_SUCCESS) + { + if (TMTimerIsActive(pVM, hTimer)) + TMTimerStop(pVM, hTimer); + TMTimerUnlock(pVM, hTimer); + } + } + else + rc = VERR_NOT_FOUND; + return rc; +} + + +/** + * Gets the read and write permission bits for an MSR in an MSR bitmap. + * + * @returns VMXMSRPM_XXX - the MSR permission. + * @param pvMsrBitmap Pointer to the MSR bitmap. + * @param idMsr The MSR to get permissions for. + * + * @sa hmR0VmxSetMsrPermission. + */ +VMM_INT_DECL(uint32_t) CPUMGetVmxMsrPermission(void const *pvMsrBitmap, uint32_t idMsr) +{ + AssertPtrReturn(pvMsrBitmap, VMXMSRPM_EXIT_RD | VMXMSRPM_EXIT_WR); + + uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap; + + /* + * MSR Layout: + * Byte index MSR range Interpreted as + * 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits. + * 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits. + * 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits. + * 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits. + * + * A bit corresponding to an MSR within the above range causes a VM-exit + * if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of + * the MSR range, it always cause a VM-exit. + * + * See Intel spec. 24.6.9 "MSR-Bitmap Address". + */ + uint32_t const offBitmapRead = 0; + uint32_t const offBitmapWrite = 0x800; + uint32_t offMsr; + uint32_t iBit; + if (idMsr <= UINT32_C(0x00001fff)) + { + offMsr = 0; + iBit = idMsr; + } + else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff)) + { + offMsr = 0x400; + iBit = idMsr - UINT32_C(0xc0000000); + } + else + { + LogFunc(("Warning! Out of range MSR %#RX32\n", idMsr)); + return VMXMSRPM_EXIT_RD | VMXMSRPM_EXIT_WR; + } + + /* + * Get the MSR read permissions. + */ + uint32_t fRet; + uint32_t const offMsrRead = offBitmapRead + offMsr; + Assert(offMsrRead + (iBit >> 3) < offBitmapWrite); + if (ASMBitTest(pbMsrBitmap, (offMsrRead << 3) + iBit)) + fRet = VMXMSRPM_EXIT_RD; + else + fRet = VMXMSRPM_ALLOW_RD; + + /* + * Get the MSR write permissions. + */ + uint32_t const offMsrWrite = offBitmapWrite + offMsr; + Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE); + if (ASMBitTest(pbMsrBitmap, (offMsrWrite << 3) + iBit)) + fRet |= VMXMSRPM_EXIT_WR; + else + fRet |= VMXMSRPM_ALLOW_WR; + + Assert(VMXMSRPM_IS_FLAG_VALID(fRet)); + return fRet; +} + + +/** + * Checks the permission bits for the specified I/O port from the given I/O bitmap + * to see if causes a VM-exit. + * + * @returns @c true if the I/O port access must cause a VM-exit, @c false otherwise. + * @param pbIoBitmap Pointer to I/O bitmap. + * @param uPort The I/O port being accessed. + * @param cbAccess e size of the I/O access in bytes (1, 2 or 4 bytes). + */ +static bool cpumGetVmxIoBitmapPermission(uint8_t const *pbIoBitmap, uint16_t uPort, uint8_t cbAccess) +{ + Assert(cbAccess == 1 || cbAccess == 2 || cbAccess == 4); + + /* + * If the I/O port access wraps around the 16-bit port I/O space, we must cause a + * VM-exit. + * + * Reading 1, 2, 4 bytes at ports 0xffff, 0xfffe and 0xfffc are valid and do not + * constitute a wrap around. However, reading 2 bytes at port 0xffff or 4 bytes + * from port 0xffff/0xfffe/0xfffd constitute a wrap around. In other words, any + * access to -both- ports 0xffff and port 0 is a wrap around. + * + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + uint32_t const uPortLast = uPort + cbAccess; + if (uPortLast > 0x10000) + return true; + + /* + * If any bit corresponding to the I/O access is set, we must cause a VM-exit. + */ + uint16_t const offPerm = uPort >> 3; /* Byte offset of the port. */ + uint16_t const idxPermBit = uPort - (offPerm << 3); /* Bit offset within byte. */ + Assert(idxPermBit < 8); + static const uint8_t s_afMask[] = { 0x0, 0x1, 0x3, 0x7, 0xf }; /* Bit-mask for all access sizes. */ + uint16_t const fMask = s_afMask[cbAccess] << idxPermBit; /* Bit-mask of the access. */ + + /* Fetch 8 or 16-bits depending on whether the access spans 8-bit boundary. */ + RTUINT16U uPerm; + uPerm.s.Lo = pbIoBitmap[offPerm]; + if (idxPermBit + cbAccess > 8) + uPerm.s.Hi = pbIoBitmap[offPerm + 1]; + else + uPerm.s.Hi = 0; + + /* If any bit for the access is 1, we must cause a VM-exit. */ + if (uPerm.u & fMask) + return true; + + return false; +} + + +/** + * Returns whether the given VMCS field is valid and supported for the guest. + * + * @param pVM The cross context VM structure. + * @param u64VmcsField The VMCS field. + * + * @remarks This takes into account the CPU features exposed to the guest. + */ +VMM_INT_DECL(bool) CPUMIsGuestVmxVmcsFieldValid(PVMCC pVM, uint64_t u64VmcsField) +{ + uint32_t const uFieldEncHi = RT_HI_U32(u64VmcsField); + uint32_t const uFieldEncLo = RT_LO_U32(u64VmcsField); + if (!uFieldEncHi) + { /* likely */ } + else + return false; + + PCCPUMFEATURES pFeat = &pVM->cpum.s.GuestFeatures; + switch (uFieldEncLo) + { + /* + * 16-bit fields. + */ + /* Control fields. */ + case VMX_VMCS16_VPID: return pFeat->fVmxVpid; + case VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR: return pFeat->fVmxPostedInt; + case VMX_VMCS16_EPTP_INDEX: return pFeat->fVmxEptXcptVe; + + /* Guest-state fields. */ + case VMX_VMCS16_GUEST_ES_SEL: + case VMX_VMCS16_GUEST_CS_SEL: + case VMX_VMCS16_GUEST_SS_SEL: + case VMX_VMCS16_GUEST_DS_SEL: + case VMX_VMCS16_GUEST_FS_SEL: + case VMX_VMCS16_GUEST_GS_SEL: + case VMX_VMCS16_GUEST_LDTR_SEL: + case VMX_VMCS16_GUEST_TR_SEL: return true; + case VMX_VMCS16_GUEST_INTR_STATUS: return pFeat->fVmxVirtIntDelivery; + case VMX_VMCS16_GUEST_PML_INDEX: return pFeat->fVmxPml; + + /* Host-state fields. */ + case VMX_VMCS16_HOST_ES_SEL: + case VMX_VMCS16_HOST_CS_SEL: + case VMX_VMCS16_HOST_SS_SEL: + case VMX_VMCS16_HOST_DS_SEL: + case VMX_VMCS16_HOST_FS_SEL: + case VMX_VMCS16_HOST_GS_SEL: + case VMX_VMCS16_HOST_TR_SEL: return true; + + /* + * 64-bit fields. + */ + /* Control fields. */ + case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL: + case VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH: + case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL: + case VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH: return pFeat->fVmxUseIoBitmaps; + case VMX_VMCS64_CTRL_MSR_BITMAP_FULL: + case VMX_VMCS64_CTRL_MSR_BITMAP_HIGH: return pFeat->fVmxUseMsrBitmaps; + case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL: + case VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH: + case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL: + case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH: + case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL: + case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH: + case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL: + case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH: return true; + case VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL: + case VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH: return pFeat->fVmxPml; + case VMX_VMCS64_CTRL_TSC_OFFSET_FULL: + case VMX_VMCS64_CTRL_TSC_OFFSET_HIGH: return true; + case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL: + case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH: return pFeat->fVmxUseTprShadow; + case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL: + case VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH: return pFeat->fVmxVirtApicAccess; + case VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL: + case VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH: return pFeat->fVmxPostedInt; + case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL: + case VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH: return pFeat->fVmxVmFunc; + case VMX_VMCS64_CTRL_EPTP_FULL: + case VMX_VMCS64_CTRL_EPTP_HIGH: return pFeat->fVmxEpt; + case VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL: + case VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH: + case VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL: + case VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH: + case VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL: + case VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH: + case VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL: + case VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH: return pFeat->fVmxVirtIntDelivery; + case VMX_VMCS64_CTRL_EPTP_LIST_FULL: + case VMX_VMCS64_CTRL_EPTP_LIST_HIGH: + { + PCVMCPU pVCpu = pVM->CTX_SUFF(apCpus)[0]; + uint64_t const uVmFuncMsr = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64VmFunc; + return RT_BOOL(RT_BF_GET(uVmFuncMsr, VMX_BF_VMFUNC_EPTP_SWITCHING)); + } + case VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL: + case VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH: + case VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL: + case VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH: return pFeat->fVmxVmcsShadowing; + case VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL: + case VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH: return pFeat->fVmxEptXcptVe; + case VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL: + case VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH: return pFeat->fVmxXsavesXrstors; + case VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL: + case VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH: return pFeat->fVmxUseTscScaling; + case VMX_VMCS64_CTRL_PROC_EXEC3_FULL: + case VMX_VMCS64_CTRL_PROC_EXEC3_HIGH: return pFeat->fVmxTertiaryExecCtls; + + /* Read-only data fields. */ + case VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL: + case VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH: return pFeat->fVmxEpt; + + /* Guest-state fields. */ + case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL: + case VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH: + case VMX_VMCS64_GUEST_DEBUGCTL_FULL: + case VMX_VMCS64_GUEST_DEBUGCTL_HIGH: return true; + case VMX_VMCS64_GUEST_PAT_FULL: + case VMX_VMCS64_GUEST_PAT_HIGH: return pFeat->fVmxEntryLoadPatMsr || pFeat->fVmxExitSavePatMsr; + case VMX_VMCS64_GUEST_EFER_FULL: + case VMX_VMCS64_GUEST_EFER_HIGH: return pFeat->fVmxEntryLoadEferMsr || pFeat->fVmxExitSaveEferMsr; + case VMX_VMCS64_GUEST_PDPTE0_FULL: + case VMX_VMCS64_GUEST_PDPTE0_HIGH: + case VMX_VMCS64_GUEST_PDPTE1_FULL: + case VMX_VMCS64_GUEST_PDPTE1_HIGH: + case VMX_VMCS64_GUEST_PDPTE2_FULL: + case VMX_VMCS64_GUEST_PDPTE2_HIGH: + case VMX_VMCS64_GUEST_PDPTE3_FULL: + case VMX_VMCS64_GUEST_PDPTE3_HIGH: return pFeat->fVmxEpt; + + /* Host-state fields. */ + case VMX_VMCS64_HOST_PAT_FULL: + case VMX_VMCS64_HOST_PAT_HIGH: return pFeat->fVmxExitLoadPatMsr; + case VMX_VMCS64_HOST_EFER_FULL: + case VMX_VMCS64_HOST_EFER_HIGH: return pFeat->fVmxExitLoadEferMsr; + + /* + * 32-bit fields. + */ + /* Control fields. */ + case VMX_VMCS32_CTRL_PIN_EXEC: + case VMX_VMCS32_CTRL_PROC_EXEC: + case VMX_VMCS32_CTRL_EXCEPTION_BITMAP: + case VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK: + case VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH: + case VMX_VMCS32_CTRL_CR3_TARGET_COUNT: + case VMX_VMCS32_CTRL_EXIT: + case VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT: + case VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT: + case VMX_VMCS32_CTRL_ENTRY: + case VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT: + case VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO: + case VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE: + case VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH: return true; + case VMX_VMCS32_CTRL_TPR_THRESHOLD: return pFeat->fVmxUseTprShadow; + case VMX_VMCS32_CTRL_PROC_EXEC2: return pFeat->fVmxSecondaryExecCtls; + case VMX_VMCS32_CTRL_PLE_GAP: + case VMX_VMCS32_CTRL_PLE_WINDOW: return pFeat->fVmxPauseLoopExit; + + /* Read-only data fields. */ + case VMX_VMCS32_RO_VM_INSTR_ERROR: + case VMX_VMCS32_RO_EXIT_REASON: + case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO: + case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE: + case VMX_VMCS32_RO_IDT_VECTORING_INFO: + case VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE: + case VMX_VMCS32_RO_EXIT_INSTR_LENGTH: + case VMX_VMCS32_RO_EXIT_INSTR_INFO: return true; + + /* Guest-state fields. */ + case VMX_VMCS32_GUEST_ES_LIMIT: + case VMX_VMCS32_GUEST_CS_LIMIT: + case VMX_VMCS32_GUEST_SS_LIMIT: + case VMX_VMCS32_GUEST_DS_LIMIT: + case VMX_VMCS32_GUEST_FS_LIMIT: + case VMX_VMCS32_GUEST_GS_LIMIT: + case VMX_VMCS32_GUEST_LDTR_LIMIT: + case VMX_VMCS32_GUEST_TR_LIMIT: + case VMX_VMCS32_GUEST_GDTR_LIMIT: + case VMX_VMCS32_GUEST_IDTR_LIMIT: + case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS: + case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS: + case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS: + case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS: + case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS: + case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS: + case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS: + case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS: + case VMX_VMCS32_GUEST_INT_STATE: + case VMX_VMCS32_GUEST_ACTIVITY_STATE: + case VMX_VMCS32_GUEST_SMBASE: + case VMX_VMCS32_GUEST_SYSENTER_CS: return true; + case VMX_VMCS32_PREEMPT_TIMER_VALUE: return pFeat->fVmxPreemptTimer; + + /* Host-state fields. */ + case VMX_VMCS32_HOST_SYSENTER_CS: return true; + + /* + * Natural-width fields. + */ + /* Control fields. */ + case VMX_VMCS_CTRL_CR0_MASK: + case VMX_VMCS_CTRL_CR4_MASK: + case VMX_VMCS_CTRL_CR0_READ_SHADOW: + case VMX_VMCS_CTRL_CR4_READ_SHADOW: + case VMX_VMCS_CTRL_CR3_TARGET_VAL0: + case VMX_VMCS_CTRL_CR3_TARGET_VAL1: + case VMX_VMCS_CTRL_CR3_TARGET_VAL2: + case VMX_VMCS_CTRL_CR3_TARGET_VAL3: return true; + + /* Read-only data fields. */ + case VMX_VMCS_RO_EXIT_QUALIFICATION: + case VMX_VMCS_RO_IO_RCX: + case VMX_VMCS_RO_IO_RSI: + case VMX_VMCS_RO_IO_RDI: + case VMX_VMCS_RO_IO_RIP: + case VMX_VMCS_RO_GUEST_LINEAR_ADDR: return true; + + /* Guest-state fields. */ + case VMX_VMCS_GUEST_CR0: + case VMX_VMCS_GUEST_CR3: + case VMX_VMCS_GUEST_CR4: + case VMX_VMCS_GUEST_ES_BASE: + case VMX_VMCS_GUEST_CS_BASE: + case VMX_VMCS_GUEST_SS_BASE: + case VMX_VMCS_GUEST_DS_BASE: + case VMX_VMCS_GUEST_FS_BASE: + case VMX_VMCS_GUEST_GS_BASE: + case VMX_VMCS_GUEST_LDTR_BASE: + case VMX_VMCS_GUEST_TR_BASE: + case VMX_VMCS_GUEST_GDTR_BASE: + case VMX_VMCS_GUEST_IDTR_BASE: + case VMX_VMCS_GUEST_DR7: + case VMX_VMCS_GUEST_RSP: + case VMX_VMCS_GUEST_RIP: + case VMX_VMCS_GUEST_RFLAGS: + case VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS: + case VMX_VMCS_GUEST_SYSENTER_ESP: + case VMX_VMCS_GUEST_SYSENTER_EIP: return true; + + /* Host-state fields. */ + case VMX_VMCS_HOST_CR0: + case VMX_VMCS_HOST_CR3: + case VMX_VMCS_HOST_CR4: + case VMX_VMCS_HOST_FS_BASE: + case VMX_VMCS_HOST_GS_BASE: + case VMX_VMCS_HOST_TR_BASE: + case VMX_VMCS_HOST_GDTR_BASE: + case VMX_VMCS_HOST_IDTR_BASE: + case VMX_VMCS_HOST_SYSENTER_ESP: + case VMX_VMCS_HOST_SYSENTER_EIP: + case VMX_VMCS_HOST_RSP: + case VMX_VMCS_HOST_RIP: return true; + } + + return false; +} + + +/** + * Checks whether the given I/O access should cause a nested-guest VM-exit. + * + * @returns @c true if it causes a VM-exit, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param u16Port The I/O port being accessed. + * @param cbAccess The size of the I/O access in bytes (1, 2 or 4 bytes). + */ +VMM_INT_DECL(bool) CPUMIsGuestVmxIoInterceptSet(PCVMCPU pVCpu, uint16_t u16Port, uint8_t cbAccess) +{ + PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest; + if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_UNCOND_IO_EXIT)) + return true; + + if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_USE_IO_BITMAPS)) + return cpumGetVmxIoBitmapPermission(pCtx->hwvirt.vmx.abIoBitmap, u16Port, cbAccess); + + return false; +} + + +/** + * Checks whether the Mov-to-CR3 instruction causes a nested-guest VM-exit. + * + * @returns @c true if it causes a VM-exit, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uNewCr3 The CR3 value being written. + */ +VMM_INT_DECL(bool) CPUMIsGuestVmxMovToCr3InterceptSet(PVMCPU pVCpu, uint64_t uNewCr3) +{ + /* + * If the CR3-load exiting control is set and the new CR3 value does not + * match any of the CR3-target values in the VMCS, we must cause a VM-exit. + * + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + PCCPUMCTX const pCtx = &pVCpu->cpum.s.Guest; + if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_CR3_LOAD_EXIT)) + { + uint32_t const uCr3TargetCount = pCtx->hwvirt.vmx.Vmcs.u32Cr3TargetCount; + Assert(uCr3TargetCount <= VMX_V_CR3_TARGET_COUNT); + + /* If the CR3-target count is 0, cause a VM-exit. */ + if (uCr3TargetCount == 0) + return true; + + /* If the CR3 being written doesn't match any of the target values, cause a VM-exit. */ + AssertCompile(VMX_V_CR3_TARGET_COUNT == 4); + if ( uNewCr3 != pCtx->hwvirt.vmx.Vmcs.u64Cr3Target0.u + && uNewCr3 != pCtx->hwvirt.vmx.Vmcs.u64Cr3Target1.u + && uNewCr3 != pCtx->hwvirt.vmx.Vmcs.u64Cr3Target2.u + && uNewCr3 != pCtx->hwvirt.vmx.Vmcs.u64Cr3Target3.u) + return true; + } + return false; +} + + +/** + * Checks whether a VMREAD or VMWRITE instruction for the given VMCS field causes a + * VM-exit or not. + * + * @returns @c true if the VMREAD/VMWRITE is intercepted, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason (VMX_EXIT_VMREAD or + * VMX_EXIT_VMREAD). + * @param u64VmcsField The VMCS field. + */ +VMM_INT_DECL(bool) CPUMIsGuestVmxVmreadVmwriteInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint64_t u64VmcsField) +{ + Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest)); + Assert( uExitReason == VMX_EXIT_VMREAD + || uExitReason == VMX_EXIT_VMWRITE); + + /* + * Without VMCS shadowing, all VMREAD and VMWRITE instructions are intercepted. + */ + if (!CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.s.Guest, VMX_PROC_CTLS2_VMCS_SHADOWING)) + return true; + + /* + * If any reserved bit in the 64-bit VMCS field encoding is set, the VMREAD/VMWRITE + * is intercepted. This excludes any reserved bits in the valid parts of the field + * encoding (i.e. bit 12). + */ + if (u64VmcsField & VMX_VMCSFIELD_RSVD_MASK) + return true; + + /* + * Finally, consult the VMREAD/VMWRITE bitmap whether to intercept the instruction or not. + */ + uint32_t const u32VmcsField = RT_LO_U32(u64VmcsField); + uint8_t const * const pbBitmap = uExitReason == VMX_EXIT_VMREAD + ? &pVCpu->cpum.s.Guest.hwvirt.vmx.abVmreadBitmap[0] + : &pVCpu->cpum.s.Guest.hwvirt.vmx.abVmwriteBitmap[0]; + Assert(pbBitmap); + Assert(u32VmcsField >> 3 < VMX_V_VMREAD_VMWRITE_BITMAP_SIZE); + return ASMBitTest(pbBitmap, (u32VmcsField << 3) + (u32VmcsField & 7)); +} + + + +/** + * Determines whether the given I/O access should cause a nested-guest \#VMEXIT. + * + * @param pvIoBitmap Pointer to the nested-guest IO bitmap. + * @param u16Port The IO port being accessed. + * @param enmIoType The type of IO access. + * @param cbReg The IO operand size in bytes. + * @param cAddrSizeBits The address size bits (for 16, 32 or 64). + * @param iEffSeg The effective segment number. + * @param fRep Whether this is a repeating IO instruction (REP prefix). + * @param fStrIo Whether this is a string IO instruction. + * @param pIoExitInfo Pointer to the SVMIOIOEXITINFO struct to be filled. + * Optional, can be NULL. + */ +VMM_INT_DECL(bool) CPUMIsSvmIoInterceptSet(void *pvIoBitmap, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg, + uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo, + PSVMIOIOEXITINFO pIoExitInfo) +{ + Assert(cAddrSizeBits == 16 || cAddrSizeBits == 32 || cAddrSizeBits == 64); + Assert(cbReg == 1 || cbReg == 2 || cbReg == 4 || cbReg == 8); + + /* + * The IOPM layout: + * Each bit represents one 8-bit port. That makes a total of 0..65535 bits or + * two 4K pages. + * + * For IO instructions that access more than a single byte, the permission bits + * for all bytes are checked; if any bit is set to 1, the IO access is intercepted. + * + * Since it's possible to do a 32-bit IO access at port 65534 (accessing 4 bytes), + * we need 3 extra bits beyond the second 4K page. + */ + static const uint16_t s_auSizeMasks[] = { 0, 1, 3, 0, 0xf, 0, 0, 0 }; + + uint16_t const offIopm = u16Port >> 3; + uint16_t const fSizeMask = s_auSizeMasks[(cAddrSizeBits >> SVM_IOIO_OP_SIZE_SHIFT) & 7]; + uint8_t const cShift = u16Port - (offIopm << 3); + uint16_t const fIopmMask = (1 << cShift) | (fSizeMask << cShift); + + uint8_t const *pbIopm = (uint8_t *)pvIoBitmap; + Assert(pbIopm); + pbIopm += offIopm; + uint16_t const u16Iopm = *(uint16_t *)pbIopm; + if (u16Iopm & fIopmMask) + { + if (pIoExitInfo) + { + static const uint32_t s_auIoOpSize[] = + { SVM_IOIO_32_BIT_OP, SVM_IOIO_8_BIT_OP, SVM_IOIO_16_BIT_OP, 0, SVM_IOIO_32_BIT_OP, 0, 0, 0 }; + + static const uint32_t s_auIoAddrSize[] = + { 0, SVM_IOIO_16_BIT_ADDR, SVM_IOIO_32_BIT_ADDR, 0, SVM_IOIO_64_BIT_ADDR, 0, 0, 0 }; + + pIoExitInfo->u = s_auIoOpSize[cbReg & 7]; + pIoExitInfo->u |= s_auIoAddrSize[(cAddrSizeBits >> 4) & 7]; + pIoExitInfo->n.u1Str = fStrIo; + pIoExitInfo->n.u1Rep = fRep; + pIoExitInfo->n.u3Seg = iEffSeg & 7; + pIoExitInfo->n.u1Type = enmIoType; + pIoExitInfo->n.u16Port = u16Port; + } + return true; + } + + /** @todo remove later (for debugging as VirtualBox always traps all IO + * intercepts). */ + AssertMsgFailed(("CPUMSvmIsIOInterceptActive: We expect an IO intercept here!\n")); + return false; +} + + +/** + * Gets the MSR permission bitmap byte and bit offset for the specified MSR. + * + * @returns VBox status code. + * @param idMsr The MSR being requested. + * @param pbOffMsrpm Where to store the byte offset in the MSR permission + * bitmap for @a idMsr. + * @param puMsrpmBit Where to store the bit offset starting at the byte + * returned in @a pbOffMsrpm. + */ +VMM_INT_DECL(int) CPUMGetSvmMsrpmOffsetAndBit(uint32_t idMsr, uint16_t *pbOffMsrpm, uint8_t *puMsrpmBit) +{ + Assert(pbOffMsrpm); + Assert(puMsrpmBit); + + /* + * MSRPM Layout: + * Byte offset MSR range + * 0x000 - 0x7ff 0x00000000 - 0x00001fff + * 0x800 - 0xfff 0xc0000000 - 0xc0001fff + * 0x1000 - 0x17ff 0xc0010000 - 0xc0011fff + * 0x1800 - 0x1fff Reserved + * + * Each MSR is represented by 2 permission bits (read and write). + */ + if (idMsr <= 0x00001fff) + { + /* Pentium-compatible MSRs. */ + uint32_t const bitoffMsr = idMsr << 1; + *pbOffMsrpm = bitoffMsr >> 3; + *puMsrpmBit = bitoffMsr & 7; + return VINF_SUCCESS; + } + + if ( idMsr >= 0xc0000000 + && idMsr <= 0xc0001fff) + { + /* AMD Sixth Generation x86 Processor MSRs. */ + uint32_t const bitoffMsr = (idMsr - 0xc0000000) << 1; + *pbOffMsrpm = 0x800 + (bitoffMsr >> 3); + *puMsrpmBit = bitoffMsr & 7; + return VINF_SUCCESS; + } + + if ( idMsr >= 0xc0010000 + && idMsr <= 0xc0011fff) + { + /* AMD Seventh and Eighth Generation Processor MSRs. */ + uint32_t const bitoffMsr = (idMsr - 0xc0010000) << 1; + *pbOffMsrpm = 0x1000 + (bitoffMsr >> 3); + *puMsrpmBit = bitoffMsr & 7; + return VINF_SUCCESS; + } + + *pbOffMsrpm = 0; + *puMsrpmBit = 0; + return VERR_OUT_OF_RANGE; +} + + +/** + * Checks whether the guest is in VMX non-root mode and using EPT paging. + * + * @returns @c true if in VMX non-root operation with EPT, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) CPUMIsGuestVmxEptPagingEnabled(PCVMCPUCC pVCpu) +{ + return CPUMIsGuestVmxEptPagingEnabledEx(&pVCpu->cpum.s.Guest); +} + + +/** + * Checks whether the guest is in VMX non-root mode and using EPT paging and the + * nested-guest is in PAE mode. + * + * @returns @c true if in VMX non-root operation with EPT, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) CPUMIsGuestVmxEptPaePagingEnabled(PCVMCPUCC pVCpu) +{ + return CPUMIsGuestVmxEptPagingEnabledEx(&pVCpu->cpum.s.Guest) + && CPUMIsGuestInPAEModeEx(&pVCpu->cpum.s.Guest); +} + + +/** + * Returns the guest-physical address of the APIC-access page when executing a + * nested-guest. + * + * @returns The APIC-access page guest-physical address. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(uint64_t) CPUMGetGuestVmxApicAccessPageAddr(PCVMCPUCC pVCpu) +{ + return CPUMGetGuestVmxApicAccessPageAddrEx(&pVCpu->cpum.s.Guest); +} + diff --git a/src/VBox/VMM/VMMAll/DBGFAll.cpp b/src/VBox/VMM/VMMAll/DBGFAll.cpp new file mode 100644 index 00000000..dc1cb1d5 --- /dev/null +++ b/src/VBox/VMM/VMMAll/DBGFAll.cpp @@ -0,0 +1,592 @@ +/* $Id: DBGFAll.cpp $ */ +/** @file + * DBGF - Debugger Facility, All Context Code. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_DBGF +#define VMCPU_INCL_CPUM_GST_CTX +#include +#include "DBGFInternal.h" +#include +#include +#include +#include +#include +#include + + +/* + * Check the read-only VM members. + */ +AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.bmSoftIntBreakpoints, VM, dbgf.ro.bmSoftIntBreakpoints); +AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.bmHardIntBreakpoints, VM, dbgf.ro.bmHardIntBreakpoints); +AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.bmSelectedEvents, VM, dbgf.ro.bmSelectedEvents); +AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.cHardIntBreakpoints, VM, dbgf.ro.cHardIntBreakpoints); +AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.cSoftIntBreakpoints, VM, dbgf.ro.cSoftIntBreakpoints); +AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.cSelectedEvents, VM, dbgf.ro.cSelectedEvents); + + +/** + * Gets the hardware breakpoint configuration as DR7. + * + * @returns DR7 from the DBGF point of view. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR7(PVM pVM) +{ + RTGCUINTREG uDr7 = X86_DR7_GD | X86_DR7_GE | X86_DR7_LE | X86_DR7_RA1_MASK; + for (uint32_t i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); i++) + { + if ( pVM->dbgf.s.aHwBreakpoints[i].fEnabled + && pVM->dbgf.s.aHwBreakpoints[i].hBp != NIL_DBGFBP) + { + static const uint8_t s_au8Sizes[8] = + { + X86_DR7_LEN_BYTE, X86_DR7_LEN_BYTE, X86_DR7_LEN_WORD, X86_DR7_LEN_BYTE, + X86_DR7_LEN_DWORD,X86_DR7_LEN_BYTE, X86_DR7_LEN_BYTE, X86_DR7_LEN_QWORD + }; + uDr7 |= X86_DR7_G(i) + | X86_DR7_RW(i, pVM->dbgf.s.aHwBreakpoints[i].fType) + | X86_DR7_LEN(i, s_au8Sizes[pVM->dbgf.s.aHwBreakpoints[i].cb]); + } + } + return uDr7; +} + + +/** + * Gets the address of the hardware breakpoint number 0. + * + * @returns DR0 from the DBGF point of view. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR0(PVM pVM) +{ + return pVM->dbgf.s.aHwBreakpoints[0].GCPtr; +} + + +/** + * Gets the address of the hardware breakpoint number 1. + * + * @returns DR1 from the DBGF point of view. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR1(PVM pVM) +{ + return pVM->dbgf.s.aHwBreakpoints[1].GCPtr; +} + + +/** + * Gets the address of the hardware breakpoint number 2. + * + * @returns DR2 from the DBGF point of view. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR2(PVM pVM) +{ + return pVM->dbgf.s.aHwBreakpoints[2].GCPtr; +} + + +/** + * Gets the address of the hardware breakpoint number 3. + * + * @returns DR3 from the DBGF point of view. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR3(PVM pVM) +{ + return pVM->dbgf.s.aHwBreakpoints[3].GCPtr; +} + + +/** + * Checks if any of the hardware breakpoints are armed. + * + * @returns true if armed, false if not. + * @param pVM The cross context VM structure. + * @remarks Don't call this from CPUMRecalcHyperDRx! + */ +VMM_INT_DECL(bool) DBGFBpIsHwArmed(PVM pVM) +{ + return pVM->dbgf.s.cEnabledHwBreakpoints > 0; +} + + +/** + * Checks if any of the hardware I/O breakpoints are armed. + * + * @returns true if armed, false if not. + * @param pVM The cross context VM structure. + * @remarks Don't call this from CPUMRecalcHyperDRx! + */ +VMM_INT_DECL(bool) DBGFBpIsHwIoArmed(PVM pVM) +{ + return pVM->dbgf.s.cEnabledHwIoBreakpoints > 0; +} + + +/** + * Checks if any INT3 breakpoints are armed. + * + * @returns true if armed, false if not. + * @param pVM The cross context VM structure. + * @remarks Don't call this from CPUMRecalcHyperDRx! + */ +VMM_INT_DECL(bool) DBGFBpIsInt3Armed(PVM pVM) +{ + /** @todo There was a todo here and returning false when I (bird) removed + * VBOX_WITH_LOTS_OF_DBGF_BPS, so this might not be correct. */ + return pVM->dbgf.s.cEnabledInt3Breakpoints > 0; +} + + +/** + * Checks instruction boundrary for guest or hypervisor hardware breakpoints. + * + * @returns Strict VBox status code. May return DRx register import errors in + * addition to the ones detailed. + * @retval VINF_SUCCESS no breakpoint. + * @retval VINF_EM_DBG_BREAKPOINT hypervisor breakpoint triggered. + * @retval VINF_EM_RAW_GUEST_TRAP caller must trigger \#DB trap, DR6 and DR7 + * have been updated appropriately. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPtrPC The unsegmented PC address. + */ +VMM_INT_DECL(VBOXSTRICTRC) DBGFBpCheckInstruction(PVMCC pVM, PVMCPUCC pVCpu, RTGCPTR GCPtrPC) +{ + CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR7); + + /* + * Check hyper breakpoints first as the VMM debugger has priority over + * the guest. + */ + /** @todo we need some kind of resume flag for these. */ + if (pVM->dbgf.s.cEnabledHwBreakpoints > 0) + for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++) + { + if ( pVM->dbgf.s.aHwBreakpoints[iBp].GCPtr != GCPtrPC + || pVM->dbgf.s.aHwBreakpoints[iBp].fType != X86_DR7_RW_EO + || pVM->dbgf.s.aHwBreakpoints[iBp].cb != 1 + || !pVM->dbgf.s.aHwBreakpoints[iBp].fEnabled + || pVM->dbgf.s.aHwBreakpoints[iBp].hBp == NIL_DBGFBP) + { /*likely*/ } + else + { + /* (See also DBGFRZTrap01Handler.) */ + pVCpu->dbgf.s.hBpActive = pVM->dbgf.s.aHwBreakpoints[iBp].hBp; + pVCpu->dbgf.s.fSingleSteppingRaw = false; + + LogFlow(("DBGFBpCheckInstruction: hit hw breakpoint %u at %04x:%RGv (%RGv)\n", + iBp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPtrPC)); + return VINF_EM_DBG_BREAKPOINT; + } + } + + /* + * Check the guest. + */ + uint32_t const fDr7 = (uint32_t)pVCpu->cpum.GstCtx.dr[7]; + if (X86_DR7_ANY_EO_ENABLED(fDr7) && !pVCpu->cpum.GstCtx.eflags.Bits.u1RF) + { + /* + * The CPU (10980XE & 6700K at least) will set the DR6.BPx bits for any + * DRx that matches the current PC and is configured as an execution + * breakpoint (RWx=EO, LENx=1byte). They don't have to be enabled, + * however one that is enabled must match for the #DB to be raised and + * DR6 to be modified, of course. + */ + CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + uint32_t fMatched = 0; + uint32_t fEnabled = 0; + for (unsigned iBp = 0, uBpMask = 1; iBp < 4; iBp++, uBpMask <<= 1) + if (X86_DR7_IS_EO_CFG(fDr7, iBp)) + { + if (fDr7 & X86_DR7_L_G(iBp)) + fEnabled |= uBpMask; + if (pVCpu->cpum.GstCtx.dr[iBp] == GCPtrPC) + fMatched |= uBpMask; + } + if (!(fEnabled & fMatched)) + { /*likely*/ } + else if (fEnabled & fMatched) + { + /* + * Update DR6 and DR7. + * + * See "AMD64 Architecture Programmer's Manual Volume 2", chapter + * 13.1.1.3 for details on DR6 bits. The basics is that the B0..B3 + * bits are always cleared while the others must be cleared by software. + * + * The following sub chapters says the GD bit is always cleared when + * generating a #DB so the handler can safely access the debug registers. + */ + CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_DR6); + pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK; + if (pVM->cpum.ro.GuestFeatures.enmCpuVendor != CPUMCPUVENDOR_INTEL) + pVCpu->cpum.GstCtx.dr[6] |= fMatched & fEnabled; + else + pVCpu->cpum.GstCtx.dr[6] |= fMatched; /* Intel: All matched, regardless of whether they're enabled or not */ + pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD; + LogFlow(("DBGFBpCheckInstruction: hit hw breakpoints %#x at %04x:%RGv (%RGv)\n", + fMatched, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPtrPC)); + return VINF_EM_RAW_GUEST_TRAP; + } + } + return VINF_SUCCESS; +} + + +/** + * Checks I/O access for guest or hypervisor hardware breakpoints. + * + * @returns Strict VBox status code + * @retval VINF_SUCCESS no breakpoint. + * @retval VINF_EM_DBG_BREAKPOINT hypervisor breakpoint triggered. + * @retval VINF_EM_RAW_GUEST_TRAP guest breakpoint triggered, DR6 and DR7 have + * been updated appropriately. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pCtx The CPU context for the calling EMT. + * @param uIoPort The I/O port being accessed. + * @param cbValue The size/width of the access, in bytes. + */ +VMM_INT_DECL(VBOXSTRICTRC) DBGFBpCheckIo(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTIOPORT uIoPort, uint8_t cbValue) +{ + uint32_t const uIoPortFirst = uIoPort; + uint32_t const uIoPortLast = uIoPortFirst + cbValue - 1; + + /* + * Check hyper breakpoints first as the VMM debugger has priority over + * the guest. + */ + if (pVM->dbgf.s.cEnabledHwIoBreakpoints > 0) + { + for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++) + { + if ( pVM->dbgf.s.aHwBreakpoints[iBp].fType == X86_DR7_RW_IO + && pVM->dbgf.s.aHwBreakpoints[iBp].fEnabled + && pVM->dbgf.s.aHwBreakpoints[iBp].hBp != NIL_DBGFBP) + { + uint8_t cbReg = pVM->dbgf.s.aHwBreakpoints[iBp].cb; Assert(RT_IS_POWER_OF_TWO(cbReg)); + uint64_t uDrXFirst = pVM->dbgf.s.aHwBreakpoints[iBp].GCPtr & ~(uint64_t)(cbReg - 1); + uint64_t uDrXLast = uDrXFirst + cbReg - 1; + if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst) + { + /* (See also DBGFRZTrap01Handler.) */ + pVCpu->dbgf.s.hBpActive = pVM->dbgf.s.aHwBreakpoints[iBp].hBp; + pVCpu->dbgf.s.fSingleSteppingRaw = false; + + LogFlow(("DBGFBpCheckIo: hit hw breakpoint %d at %04x:%RGv (iop %#x)\n", + iBp, pCtx->cs.Sel, pCtx->rip, uIoPort)); + return VINF_EM_DBG_BREAKPOINT; + } + } + } + } + + /* + * Check the guest. + */ + uint32_t const uDr7 = pCtx->dr[7]; + if ( (uDr7 & X86_DR7_ENABLED_MASK) + && X86_DR7_ANY_RW_IO(uDr7) + && (pCtx->cr4 & X86_CR4_DE) ) + { + for (unsigned iBp = 0; iBp < 4; iBp++) + { + if ( (uDr7 & X86_DR7_L_G(iBp)) + && X86_DR7_GET_RW(uDr7, iBp) == X86_DR7_RW_IO) + { + /* ASSUME the breakpoint and the I/O width qualifier uses the same encoding (1 2 x 4). */ + static uint8_t const s_abInvAlign[4] = { 0, 1, 7, 3 }; + uint8_t cbInvAlign = s_abInvAlign[X86_DR7_GET_LEN(uDr7, iBp)]; + uint64_t uDrXFirst = pCtx->dr[iBp] & ~(uint64_t)cbInvAlign; + uint64_t uDrXLast = uDrXFirst + cbInvAlign; + + if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst) + { + /* + * Update DR6 and DR7. + * + * See "AMD64 Architecture Programmer's Manual Volume 2", + * chapter 13.1.1.3 for details on DR6 bits. The basics is + * that the B0..B3 bits are always cleared while the others + * must be cleared by software. + * + * The following sub chapters says the GD bit is always + * cleared when generating a #DB so the handler can safely + * access the debug registers. + */ + pCtx->dr[6] &= ~X86_DR6_B_MASK; + pCtx->dr[6] |= X86_DR6_B(iBp); + pCtx->dr[7] &= ~X86_DR7_GD; + LogFlow(("DBGFBpCheckIo: hit hw breakpoint %d at %04x:%RGv (iop %#x)\n", + iBp, pCtx->cs.Sel, pCtx->rip, uIoPort)); + return VINF_EM_RAW_GUEST_TRAP; + } + } + } + } + return VINF_SUCCESS; +} + + +/** + * Checks I/O access for guest or hypervisor hardware breakpoints. + * + * Caller must make sure DR0-3 and DR7 are present in the CPU context before + * calling this function. + * + * @returns CPUMCTX_DBG_DBGF_BP, CPUMCTX_DBG_HIT_DRX_MASK, or 0 (no match). + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uIoPort The I/O port being accessed. + * @param cbValue The size/width of the access, in bytes. + */ +VMM_INT_DECL(uint32_t) DBGFBpCheckIo2(PVMCC pVM, PVMCPUCC pVCpu, RTIOPORT uIoPort, uint8_t cbValue) +{ + uint32_t const uIoPortFirst = uIoPort; + uint32_t const uIoPortLast = uIoPortFirst + cbValue - 1; + + /* + * Check hyper breakpoints first as the VMM debugger has priority over + * the guest. + */ + if (pVM->dbgf.s.cEnabledHwIoBreakpoints > 0) + for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++) + { + if ( pVM->dbgf.s.aHwBreakpoints[iBp].fType == X86_DR7_RW_IO + && pVM->dbgf.s.aHwBreakpoints[iBp].fEnabled + && pVM->dbgf.s.aHwBreakpoints[iBp].hBp != NIL_DBGFBP) + { + uint8_t cbReg = pVM->dbgf.s.aHwBreakpoints[iBp].cb; Assert(RT_IS_POWER_OF_TWO(cbReg)); + uint64_t uDrXFirst = pVM->dbgf.s.aHwBreakpoints[iBp].GCPtr & ~(uint64_t)(cbReg - 1); + uint64_t uDrXLast = uDrXFirst + cbReg - 1; + if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst) + { + /* (See also DBGFRZTrap01Handler.) */ + pVCpu->dbgf.s.hBpActive = pVM->dbgf.s.aHwBreakpoints[iBp].hBp; + pVCpu->dbgf.s.fSingleSteppingRaw = false; + + LogFlow(("DBGFBpCheckIo2: hit hw breakpoint %d at %04x:%RGv (iop %#x L %u)\n", + iBp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uIoPort, cbValue)); + return CPUMCTX_DBG_DBGF_BP; + } + } + } + + /* + * Check the guest. + */ + uint32_t const fDr7 = pVCpu->cpum.GstCtx.dr[7]; + if ( (fDr7 & X86_DR7_ENABLED_MASK) + && X86_DR7_ANY_RW_IO(fDr7) + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE) ) + { + uint32_t fEnabled = 0; + uint32_t fMatched = 0; + for (unsigned iBp = 0, uBpMask = 1; iBp < 4; iBp++, uBpMask <<= 1) + { + if (fDr7 & X86_DR7_L_G(iBp)) + fEnabled |= uBpMask; + if (X86_DR7_GET_RW(fDr7, iBp) == X86_DR7_RW_IO) + { + /* ASSUME the breakpoint and the I/O width qualifier uses the same encoding (1 2 x 4). */ + static uint8_t const s_abInvAlign[4] = { 0, 1, 7, 3 }; + uint8_t const cbInvAlign = s_abInvAlign[X86_DR7_GET_LEN(fDr7, iBp)]; + uint64_t const uDrXFirst = pVCpu->cpum.GstCtx.dr[iBp] & ~(uint64_t)cbInvAlign; + uint64_t const uDrXLast = uDrXFirst + cbInvAlign; + if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst) + fMatched |= uBpMask; + } + } + if (fEnabled & fMatched) + { + LogFlow(("DBGFBpCheckIo2: hit hw breakpoint %#x at %04x:%RGv (iop %#x L %u)\n", + fMatched, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uIoPort, cbValue)); + return fMatched << CPUMCTX_DBG_HIT_DRX_SHIFT; + } + } + + return 0; +} + + +/** + * Returns the single stepping state for a virtual CPU. + * + * @returns stepping (true) or not (false). + * + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) DBGFIsStepping(PVMCPU pVCpu) +{ + return pVCpu->dbgf.s.fSingleSteppingRaw; +} + + +/** + * Checks if the specified generic event is enabled or not. + * + * @returns true / false. + * @param pVM The cross context VM structure. + * @param enmEvent The generic event being raised. + * @param uEventArg The argument of that event. + */ +DECLINLINE(bool) dbgfEventIsGenericWithArgEnabled(PVM pVM, DBGFEVENTTYPE enmEvent, uint64_t uEventArg) +{ + if (DBGF_IS_EVENT_ENABLED(pVM, enmEvent)) + { + switch (enmEvent) + { + case DBGFEVENT_INTERRUPT_HARDWARE: + AssertReturn(uEventArg < 256, false); + return ASMBitTest(pVM->dbgf.s.bmHardIntBreakpoints, (uint32_t)uEventArg); + + case DBGFEVENT_INTERRUPT_SOFTWARE: + AssertReturn(uEventArg < 256, false); + return ASMBitTest(pVM->dbgf.s.bmSoftIntBreakpoints, (uint32_t)uEventArg); + + default: + return true; + + } + } + return false; +} + + +/** + * Raises a generic debug event if enabled and not being ignored. + * + * @returns Strict VBox status code. + * @retval VINF_EM_DBG_EVENT if the event was raised and the caller should + * return ASAP to the debugger (via EM). We set VMCPU_FF_DBGF so, it + * is okay not to pass this along in some situations. + * @retval VINF_SUCCESS if the event was disabled or ignored. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param enmEvent The generic event being raised. + * @param enmCtx The context in which this event is being raised. + * @param cArgs Number of arguments (0 - 6). + * @param ... Event arguments. + * + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) DBGFEventGenericWithArgs(PVM pVM, PVMCPU pVCpu, DBGFEVENTTYPE enmEvent, DBGFEVENTCTX enmCtx, + unsigned cArgs, ...) +{ + Assert(cArgs < RT_ELEMENTS(pVCpu->dbgf.s.aEvents[0].Event.u.Generic.auArgs)); + + /* + * Is it enabled. + */ + va_list va; + va_start(va, cArgs); + uint64_t uEventArg0 = cArgs ? va_arg(va, uint64_t) : 0; + if (dbgfEventIsGenericWithArgEnabled(pVM, enmEvent, uEventArg0)) + { + /* + * Any events on the stack. Should the incoming event be ignored? + */ + uint64_t const rip = CPUMGetGuestRIP(pVCpu); + uint32_t i = pVCpu->dbgf.s.cEvents; + if (i > 0) + { + while (i-- > 0) + { + if ( pVCpu->dbgf.s.aEvents[i].Event.enmType == enmEvent + && pVCpu->dbgf.s.aEvents[i].enmState == DBGFEVENTSTATE_IGNORE + && pVCpu->dbgf.s.aEvents[i].rip == rip) + { + pVCpu->dbgf.s.aEvents[i].enmState = DBGFEVENTSTATE_RESTORABLE; + va_end(va); + return VINF_SUCCESS; + } + Assert(pVCpu->dbgf.s.aEvents[i].enmState != DBGFEVENTSTATE_CURRENT); + } + + /* + * Trim the event stack. + */ + i = pVCpu->dbgf.s.cEvents; + while (i-- > 0) + { + if ( pVCpu->dbgf.s.aEvents[i].rip == rip + && ( pVCpu->dbgf.s.aEvents[i].enmState == DBGFEVENTSTATE_RESTORABLE + || pVCpu->dbgf.s.aEvents[i].enmState == DBGFEVENTSTATE_IGNORE) ) + pVCpu->dbgf.s.aEvents[i].enmState = DBGFEVENTSTATE_IGNORE; + else + { + if (i + 1 != pVCpu->dbgf.s.cEvents) + memmove(&pVCpu->dbgf.s.aEvents[i], &pVCpu->dbgf.s.aEvents[i + 1], + (pVCpu->dbgf.s.cEvents - i) * sizeof(pVCpu->dbgf.s.aEvents)); + pVCpu->dbgf.s.cEvents--; + } + } + + i = pVCpu->dbgf.s.cEvents; + AssertStmt(i < RT_ELEMENTS(pVCpu->dbgf.s.aEvents), i = RT_ELEMENTS(pVCpu->dbgf.s.aEvents) - 1); + } + + /* + * Push the event. + */ + pVCpu->dbgf.s.aEvents[i].enmState = DBGFEVENTSTATE_CURRENT; + pVCpu->dbgf.s.aEvents[i].rip = rip; + pVCpu->dbgf.s.aEvents[i].Event.enmType = enmEvent; + pVCpu->dbgf.s.aEvents[i].Event.enmCtx = enmCtx; + pVCpu->dbgf.s.aEvents[i].Event.u.Generic.cArgs = cArgs; + pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs[0] = uEventArg0; + if (cArgs > 1) + { + AssertStmt(cArgs < RT_ELEMENTS(pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs), + cArgs = RT_ELEMENTS(pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs)); + for (unsigned iArg = 1; iArg < cArgs; iArg++) + pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs[iArg] = va_arg(va, uint64_t); + } + pVCpu->dbgf.s.cEvents = i + 1; + + VMCPU_FF_SET(pVCpu, VMCPU_FF_DBGF); + va_end(va); + return VINF_EM_DBG_EVENT; + } + + va_end(va); + return VINF_SUCCESS; +} + diff --git a/src/VBox/VMM/VMMAll/DBGFAllBp.cpp b/src/VBox/VMM/VMMAll/DBGFAllBp.cpp new file mode 100644 index 00000000..b6bb2669 --- /dev/null +++ b/src/VBox/VMM/VMMAll/DBGFAllBp.cpp @@ -0,0 +1,604 @@ +/* $Id: DBGFAllBp.cpp $ */ +/** @file + * DBGF - Debugger Facility, All Context breakpoint management part. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_DBGF +#define VMCPU_INCL_CPUM_GST_CTX +#include +#include +#include +#include +#include +#include "DBGFInternal.h" +#include +#include +#include + +#include "DBGFInline.h" + + +#ifdef IN_RC +# error "You lucky person have the pleasure to implement the raw mode part for this!" +#endif + + +/** + * Returns the internal breakpoint state for the given handle. + * + * @returns Pointer to the internal breakpoint state or NULL if the handle is invalid. + * @param pVM The ring-0 VM structure pointer. + * @param hBp The breakpoint handle to resolve. + * @param ppBpR0 Where to store the pointer to the ring-0 only part of the breakpoint + * on success, optional. + */ +#ifdef IN_RING0 +DECLINLINE(PDBGFBPINT) dbgfBpGetByHnd(PVMCC pVM, DBGFBP hBp, PDBGFBPINTR0 *ppBpR0) +#else +DECLINLINE(PDBGFBPINT) dbgfBpGetByHnd(PVMCC pVM, DBGFBP hBp) +#endif +{ + uint32_t idChunk = DBGF_BP_HND_GET_CHUNK_ID(hBp); + uint32_t idxEntry = DBGF_BP_HND_GET_ENTRY(hBp); + + AssertReturn(idChunk < DBGF_BP_CHUNK_COUNT, NULL); + AssertReturn(idxEntry < DBGF_BP_COUNT_PER_CHUNK, NULL); + +#ifdef IN_RING0 + PDBGFBPCHUNKR0 pBpChunk = &pVM->dbgfr0.s.aBpChunks[idChunk]; + AssertPtrReturn(pBpChunk->CTX_SUFF(paBpBaseShared), NULL); + + if (ppBpR0) + *ppBpR0 = &pBpChunk->paBpBaseR0Only[idxEntry]; + return &pBpChunk->CTX_SUFF(paBpBaseShared)[idxEntry]; + +#elif defined(IN_RING3) + PUVM pUVM = pVM->pUVM; + PDBGFBPCHUNKR3 pBpChunk = &pUVM->dbgf.s.aBpChunks[idChunk]; + AssertPtrReturn(pBpChunk->CTX_SUFF(pBpBase), NULL); + + return &pBpChunk->CTX_SUFF(pBpBase)[idxEntry]; + +#else +# error "Unsupported context" +#endif +} + + +/** + * Returns the pointer to the L2 table entry from the given index. + * + * @returns Current context pointer to the L2 table entry or NULL if the provided index value is invalid. + * @param pVM The cross context VM structure. + * @param idxL2 The L2 table index to resolve. + * + * @note The content of the resolved L2 table entry is not validated!. + */ +DECLINLINE(PCDBGFBPL2ENTRY) dbgfBpL2GetByIdx(PVMCC pVM, uint32_t idxL2) +{ + uint32_t idChunk = DBGF_BP_L2_IDX_GET_CHUNK_ID(idxL2); + uint32_t idxEntry = DBGF_BP_L2_IDX_GET_ENTRY(idxL2); + + AssertReturn(idChunk < DBGF_BP_L2_TBL_CHUNK_COUNT, NULL); + AssertReturn(idxEntry < DBGF_BP_L2_TBL_ENTRIES_PER_CHUNK, NULL); + +#ifdef IN_RING0 + PDBGFBPL2TBLCHUNKR0 pL2Chunk = &pVM->dbgfr0.s.aBpL2TblChunks[idChunk]; + AssertPtrReturn(pL2Chunk->CTX_SUFF(paBpL2TblBaseShared), NULL); + + return &pL2Chunk->CTX_SUFF(paBpL2TblBaseShared)[idxEntry]; +#elif defined(IN_RING3) + PUVM pUVM = pVM->pUVM; + PDBGFBPL2TBLCHUNKR3 pL2Chunk = &pUVM->dbgf.s.aBpL2TblChunks[idChunk]; + AssertPtrReturn(pL2Chunk->pbmAlloc, NULL); + AssertReturn(ASMBitTest(pL2Chunk->pbmAlloc, idxEntry), NULL); + + return &pL2Chunk->CTX_SUFF(pL2Base)[idxEntry]; +#endif +} + + +#ifdef IN_RING0 +/** + * Returns the internal breakpoint owner state for the given handle. + * + * @returns Pointer to the internal ring-0 breakpoint owner state or NULL if the handle is invalid. + * @param pVM The cross context VM structure. + * @param hBpOwner The breakpoint owner handle to resolve. + */ +DECLINLINE(PCDBGFBPOWNERINTR0) dbgfR0BpOwnerGetByHnd(PVMCC pVM, DBGFBPOWNER hBpOwner) +{ + if (hBpOwner == NIL_DBGFBPOWNER) + return NULL; + + AssertReturn(hBpOwner < DBGF_BP_OWNER_COUNT_MAX, NULL); + + PCDBGFBPOWNERINTR0 pBpOwnerR0 = &pVM->dbgfr0.s.paBpOwnersR0[hBpOwner]; + AssertReturn(pBpOwnerR0->cRefs > 1, NULL); + + return pBpOwnerR0; +} +#endif + + +/** + * Executes the actions associated with the given breakpoint. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the register context for the CPU. + * @param hBp The breakpoint handle which hit. + * @param pBp The shared breakpoint state. + * @param pBpR0 The ring-0 only breakpoint state. + */ +#ifdef IN_RING0 +DECLINLINE(int) dbgfBpHit(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, DBGFBP hBp, PDBGFBPINT pBp, PDBGFBPINTR0 pBpR0) +#else +DECLINLINE(int) dbgfBpHit(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, DBGFBP hBp, PDBGFBPINT pBp) +#endif +{ + uint64_t cHits = ASMAtomicIncU64(&pBp->Pub.cHits); RT_NOREF(cHits); + + RT_NOREF(pCtx); + LogFlow(("dbgfBpHit: hit breakpoint %u at %04x:%RGv cHits=0x%RX64\n", hBp, pCtx->cs.Sel, pCtx->rip, cHits)); + + int rc = VINF_EM_DBG_BREAKPOINT; +#ifdef IN_RING0 + PCDBGFBPOWNERINTR0 pBpOwnerR0 = dbgfR0BpOwnerGetByHnd(pVM, + pBpR0->fInUse + ? pBpR0->hOwner + : NIL_DBGFBPOWNER); + if (pBpOwnerR0) + { + AssertReturn(pBpOwnerR0->pfnBpIoHitR0, VERR_DBGF_BP_IPE_1); + + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + + if (DBGF_BP_PUB_IS_EXEC_BEFORE(&pBp->Pub)) + rcStrict = pBpOwnerR0->pfnBpHitR0(pVM, pVCpu->idCpu, pBpR0->pvUserR0, hBp, &pBp->Pub, DBGF_BP_F_HIT_EXEC_BEFORE); + if (rcStrict == VINF_SUCCESS) + { + uint8_t abInstr[DBGF_BP_INSN_MAX]; + RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base; + rc = PGMPhysSimpleReadGCPtr(pVCpu, &abInstr[0], GCPtrInstr, sizeof(abInstr)); + AssertRC(rc); + if (RT_SUCCESS(rc)) + { + /* Replace the int3 with the original instruction byte. */ + abInstr[0] = pBp->Pub.u.Int3.bOrg; + rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, GCPtrInstr, &abInstr[0], sizeof(abInstr)); + if ( rcStrict == VINF_SUCCESS + && DBGF_BP_PUB_IS_EXEC_AFTER(&pBp->Pub)) + { + rcStrict = pBpOwnerR0->pfnBpHitR0(pVM, pVCpu->idCpu, pBpR0->pvUserR0, hBp, &pBp->Pub, DBGF_BP_F_HIT_EXEC_AFTER); + if (rcStrict == VINF_SUCCESS) + rc = VINF_SUCCESS; + else if ( rcStrict == VINF_DBGF_BP_HALT + || rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER) + { + pVCpu->dbgf.s.hBpActive = hBp; + if (rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER) + pVCpu->dbgf.s.fBpInvokeOwnerCallback = true; + else + pVCpu->dbgf.s.fBpInvokeOwnerCallback = false; + } + else /* Guru meditation. */ + rc = VERR_DBGF_BP_OWNER_CALLBACK_WRONG_STATUS; + } + else + rc = VBOXSTRICTRC_VAL(rcStrict); + } + } + else if ( rcStrict == VINF_DBGF_BP_HALT + || rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER) + { + pVCpu->dbgf.s.hBpActive = hBp; + if (rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER) + pVCpu->dbgf.s.fBpInvokeOwnerCallback = true; + else + pVCpu->dbgf.s.fBpInvokeOwnerCallback = false; + } + else /* Guru meditation. */ + rc = VERR_DBGF_BP_OWNER_CALLBACK_WRONG_STATUS; + } + else + { + pVCpu->dbgf.s.fBpInvokeOwnerCallback = true; /* Need to check this for ring-3 only owners. */ + pVCpu->dbgf.s.hBpActive = hBp; + } +#else + RT_NOREF(pVM); + pVCpu->dbgf.s.fBpInvokeOwnerCallback = true; + pVCpu->dbgf.s.hBpActive = hBp; +#endif + + return rc; +} + + +/** + * Executes the actions associated with the given port I/O breakpoint. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param fBefore Flag whether the check is done before the access is carried out, + * false if it is done after the access. + * @param fAccess Access flags, see DBGFBPIOACCESS_XXX. + * @param uAddr The address of the access, for port I/O this will hold the port number. + * @param uValue The value read or written (the value for reads is only valid when DBGF_BP_F_HIT_EXEC_AFTER is set). + * @param hBp The breakpoint handle which hit. + * @param pBp The shared breakpoint state. + * @param pBpR0 The ring-0 only breakpoint state. + */ +#ifdef IN_RING0 +DECLINLINE(VBOXSTRICTRC) dbgfBpPortIoHit(PVMCC pVM, PVMCPU pVCpu, bool fBefore, uint32_t fAccess, uint64_t uAddr, uint64_t uValue, + DBGFBP hBp, PDBGFBPINT pBp, PDBGFBPINTR0 pBpR0) +#else +DECLINLINE(VBOXSTRICTRC) dbgfBpPortIoHit(PVMCC pVM, PVMCPU pVCpu, bool fBefore, uint32_t fAccess, uint64_t uAddr, uint64_t uValue, + DBGFBP hBp, PDBGFBPINT pBp) +#endif +{ + ASMAtomicIncU64(&pBp->Pub.cHits); + + VBOXSTRICTRC rcStrict = VINF_EM_DBG_BREAKPOINT; +#ifdef IN_RING0 + PCDBGFBPOWNERINTR0 pBpOwnerR0 = dbgfR0BpOwnerGetByHnd(pVM, + pBpR0->fInUse + ? pBpR0->hOwner + : NIL_DBGFBPOWNER); + if (pBpOwnerR0) + { + AssertReturn(pBpOwnerR0->pfnBpIoHitR0, VERR_DBGF_BP_IPE_1); + rcStrict = pBpOwnerR0->pfnBpIoHitR0(pVM, pVCpu->idCpuUnsafe, pBpR0->pvUserR0, hBp, &pBp->Pub, + fBefore + ? DBGF_BP_F_HIT_EXEC_BEFORE + : DBGF_BP_F_HIT_EXEC_AFTER, + fAccess, uAddr, uValue); + } + else + { + pVCpu->dbgf.s.fBpInvokeOwnerCallback = true; /* Need to check this for ring-3 only owners. */ + pVCpu->dbgf.s.hBpActive = hBp; + pVCpu->dbgf.s.fBpIoActive = true; + pVCpu->dbgf.s.fBpIoBefore = fBefore; + pVCpu->dbgf.s.uBpIoAddress = uAddr; + pVCpu->dbgf.s.fBpIoAccess = fAccess; + pVCpu->dbgf.s.uBpIoValue = uValue; + } +#else + /* Resolve owner (can be NIL_DBGFBPOWNER) and invoke callback if there is one. */ + if (pBp->Pub.hOwner != NIL_DBGFBPOWNER) + { + PCDBGFBPOWNERINT pBpOwner = dbgfR3BpOwnerGetByHnd(pVM->pUVM, pBp->Pub.hOwner); + if (pBpOwner) + { + AssertReturn(pBpOwner->pfnBpIoHitR3, VERR_DBGF_BP_IPE_1); + rcStrict = pBpOwner->pfnBpIoHitR3(pVM, pVCpu->idCpu, pBp->pvUserR3, hBp, &pBp->Pub, + fBefore + ? DBGF_BP_F_HIT_EXEC_BEFORE + : DBGF_BP_F_HIT_EXEC_AFTER, + fAccess, uAddr, uValue); + } + } +#endif + if ( rcStrict == VINF_DBGF_BP_HALT + || rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER) + { + pVCpu->dbgf.s.hBpActive = hBp; + if (rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER) + pVCpu->dbgf.s.fBpInvokeOwnerCallback = true; + else + pVCpu->dbgf.s.fBpInvokeOwnerCallback = false; + rcStrict = VINF_EM_DBG_BREAKPOINT; + } + else if ( rcStrict != VINF_SUCCESS + && rcStrict != VINF_EM_DBG_BREAKPOINT) + rcStrict = VERR_DBGF_BP_OWNER_CALLBACK_WRONG_STATUS; /* Guru meditation. */ + + return rcStrict; +} + + +/** + * Walks the L2 table starting at the given root index searching for the given key. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the register context for the CPU. + * @param idxL2Root L2 table index of the table root. + * @param GCPtrKey The key to search for. + */ +static int dbgfBpL2Walk(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, uint32_t idxL2Root, RTGCUINTPTR GCPtrKey) +{ + /** @todo We don't use the depth right now but abort the walking after a fixed amount of levels. */ + uint8_t iDepth = 32; + PCDBGFBPL2ENTRY pL2Entry = dbgfBpL2GetByIdx(pVM, idxL2Root); + + while (RT_LIKELY( iDepth-- > 0 + && pL2Entry)) + { + /* Make a copy of the entry before verification. */ + DBGFBPL2ENTRY L2Entry; + L2Entry.u64GCPtrKeyAndBpHnd1 = ASMAtomicReadU64((volatile uint64_t *)&pL2Entry->u64GCPtrKeyAndBpHnd1); + L2Entry.u64LeftRightIdxDepthBpHnd2 = ASMAtomicReadU64((volatile uint64_t *)&pL2Entry->u64LeftRightIdxDepthBpHnd2); + + RTGCUINTPTR GCPtrL2Entry = DBGF_BP_L2_ENTRY_GET_GCPTR(L2Entry.u64GCPtrKeyAndBpHnd1); + if (GCPtrKey == GCPtrL2Entry) + { + DBGFBP hBp = DBGF_BP_L2_ENTRY_GET_BP_HND(L2Entry.u64GCPtrKeyAndBpHnd1, L2Entry.u64LeftRightIdxDepthBpHnd2); + + /* Query the internal breakpoint state from the handle. */ +#ifdef IN_RING3 + PDBGFBPINT pBp = dbgfBpGetByHnd(pVM, hBp); +#else + PDBGFBPINTR0 pBpR0 = NULL; + PDBGFBPINT pBp = dbgfBpGetByHnd(pVM, hBp, &pBpR0); +#endif + if ( pBp + && DBGF_BP_PUB_GET_TYPE(&pBp->Pub) == DBGFBPTYPE_INT3) +#ifdef IN_RING3 + return dbgfBpHit(pVM, pVCpu, pCtx, hBp, pBp); +#else + return dbgfBpHit(pVM, pVCpu, pCtx, hBp, pBp, pBpR0); +#endif + + /* The entry got corrupted, just abort. */ + return VERR_DBGF_BP_L2_LOOKUP_FAILED; + } + + /* Not found, get to the next level. */ + uint32_t idxL2Next = (GCPtrKey < GCPtrL2Entry) + ? DBGF_BP_L2_ENTRY_GET_IDX_LEFT(L2Entry.u64LeftRightIdxDepthBpHnd2) + : DBGF_BP_L2_ENTRY_GET_IDX_RIGHT(L2Entry.u64LeftRightIdxDepthBpHnd2); + /* It is genuine guest trap or we hit some assertion if we are at the end. */ + if (idxL2Next == DBGF_BP_L2_ENTRY_IDX_END) + return VINF_EM_RAW_GUEST_TRAP; + + pL2Entry = dbgfBpL2GetByIdx(pVM, idxL2Next); + } + + return VERR_DBGF_BP_L2_LOOKUP_FAILED; +} + + +/** + * Checks whether there is a port I/O breakpoint for the given range and access size. + * + * @returns VBox status code. + * @retval VINF_EM_DBG_BREAKPOINT means there is a breakpoint pending. + * @retval VINF_SUCCESS means everything is fine to continue. + * @retval anything else means a fatal error causing a guru meditation. + * + * @param pVM The current context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param uIoPort The I/O port being accessed. + * @param fAccess Appropriate DBGFBPIOACCESS_XXX. + * @param uValue The value being written to or read from the device + * (The value is only valid for a read when the + * call is made after the access, writes are always valid). + * @param fBefore Flag whether the check is done before the access is carried out, + * false if it is done after the access. + */ +VMM_INT_DECL(VBOXSTRICTRC) DBGFBpCheckPortIo(PVMCC pVM, PVMCPU pVCpu, RTIOPORT uIoPort, + uint32_t fAccess, uint32_t uValue, bool fBefore) +{ + RT_NOREF(uValue); /** @todo Trigger only on specific values. */ + + /* Don't trigger in single stepping mode. */ + if (pVCpu->dbgf.s.fSingleSteppingRaw) + return VINF_SUCCESS; + +#if defined(IN_RING0) + uint32_t volatile *paBpLocPortIo = pVM->dbgfr0.s.CTX_SUFF(paBpLocPortIo); +#elif defined(IN_RING3) + PUVM pUVM = pVM->pUVM; + uint32_t volatile *paBpLocPortIo = pUVM->dbgf.s.CTX_SUFF(paBpLocPortIo); +#else +# error "Unsupported host context" +#endif + if (paBpLocPortIo) + { + const uint32_t u32Entry = ASMAtomicReadU32(&paBpLocPortIo[uIoPort]); + if (u32Entry != DBGF_BP_INT3_L1_ENTRY_TYPE_NULL) + { + uint8_t u8Type = DBGF_BP_INT3_L1_ENTRY_GET_TYPE(u32Entry); + if (RT_LIKELY(u8Type == DBGF_BP_INT3_L1_ENTRY_TYPE_BP_HND)) + { + DBGFBP hBp = DBGF_BP_INT3_L1_ENTRY_GET_BP_HND(u32Entry); + + /* Query the internal breakpoint state from the handle. */ +#ifdef IN_RING3 + PDBGFBPINT pBp = dbgfBpGetByHnd(pVM, hBp); +#else + PDBGFBPINTR0 pBpR0 = NULL; + PDBGFBPINT pBp = dbgfBpGetByHnd(pVM, hBp, &pBpR0); +#endif + if ( pBp + && DBGF_BP_PUB_GET_TYPE(&pBp->Pub) == DBGFBPTYPE_PORT_IO) + { + if ( uIoPort >= pBp->Pub.u.PortIo.uPort + && uIoPort < pBp->Pub.u.PortIo.uPort + pBp->Pub.u.PortIo.cPorts + && pBp->Pub.u.PortIo.fAccess & fAccess + && ( ( fBefore + && DBGF_BP_PUB_IS_EXEC_BEFORE(&pBp->Pub)) + || ( !fBefore + && DBGF_BP_PUB_IS_EXEC_AFTER(&pBp->Pub)))) +#ifdef IN_RING3 + return dbgfBpPortIoHit(pVM, pVCpu, fBefore, fAccess, uIoPort, uValue, hBp, pBp); +#else + return dbgfBpPortIoHit(pVM, pVCpu, fBefore, fAccess, uIoPort, uValue, hBp, pBp, pBpR0); +#endif + /* else: No matching port I/O breakpoint. */ + } + else /* Invalid breakpoint handle or not an port I/O breakpoint. */ + return VERR_DBGF_BP_L1_LOOKUP_FAILED; + } + else /* Some invalid type. */ + return VERR_DBGF_BP_L1_LOOKUP_FAILED; + } + } + + return VINF_SUCCESS; +} + + +/** + * \#DB (Debug event) handler. + * + * @returns VBox status code. + * VINF_SUCCESS means we completely handled this trap, + * other codes are passed execution to host context. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the register context for the CPU. + * @param uDr6 The DR6 hypervisor register value. + * @param fAltStepping Alternative stepping indicator. + */ +VMM_INT_DECL(int) DBGFTrap01Handler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTGCUINTREG uDr6, bool fAltStepping) +{ + /** @todo Intel docs say that X86_DR6_BS has the highest priority... */ + RT_NOREF(pCtx); + + /* + * A breakpoint? + */ + AssertCompile(X86_DR6_B0 == 1 && X86_DR6_B1 == 2 && X86_DR6_B2 == 4 && X86_DR6_B3 == 8); + if ( (uDr6 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3)) + && pVM->dbgf.s.cEnabledHwBreakpoints > 0) + { + for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++) + if ( ((uint32_t)uDr6 & RT_BIT_32(iBp)) + && pVM->dbgf.s.aHwBreakpoints[iBp].hBp != NIL_DBGFBP) + { + pVCpu->dbgf.s.hBpActive = pVM->dbgf.s.aHwBreakpoints[iBp].hBp; + pVCpu->dbgf.s.fSingleSteppingRaw = false; + LogFlow(("DBGFRZTrap03Handler: hit hw breakpoint %x at %04x:%RGv\n", + pVM->dbgf.s.aHwBreakpoints[iBp].hBp, pCtx->cs.Sel, pCtx->rip)); + + return VINF_EM_DBG_BREAKPOINT; + } + } + + /* + * Single step? + * Are we single stepping or is it the guest? + */ + if ( (uDr6 & X86_DR6_BS) + && (pVCpu->dbgf.s.fSingleSteppingRaw || fAltStepping)) + { + pVCpu->dbgf.s.fSingleSteppingRaw = false; + LogFlow(("DBGFRZTrap01Handler: single step at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip)); + return VINF_EM_DBG_STEPPED; + } + + LogFlow(("DBGFRZTrap01Handler: guest debug event %#x at %04x:%RGv!\n", (uint32_t)uDr6, pCtx->cs.Sel, pCtx->rip)); + return VINF_EM_RAW_GUEST_TRAP; +} + + +/** + * \#BP (Breakpoint) handler. + * + * @returns VBox status code. + * VINF_SUCCESS means we completely handled this trap, + * other codes are passed execution to host context. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the register context for the CPU. + */ +VMM_INT_DECL(VBOXSTRICTRC) DBGFTrap03Handler(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx) +{ +#if defined(IN_RING0) + uint32_t volatile *paBpLocL1 = pVM->dbgfr0.s.CTX_SUFF(paBpLocL1); +#elif defined(IN_RING3) + PUVM pUVM = pVM->pUVM; + uint32_t volatile *paBpLocL1 = pUVM->dbgf.s.CTX_SUFF(paBpLocL1); +#else +# error "Unsupported host context" +#endif + if (paBpLocL1) + { + RTGCPTR GCPtrBp; + int rc = SELMValidateAndConvertCSAddr(pVCpu, pCtx->eflags.u, pCtx->ss.Sel, pCtx->cs.Sel, &pCtx->cs, + pCtx->rip /* no -1 outside non-rawmode */, &GCPtrBp); + AssertRCReturn(rc, rc); + + const uint16_t idxL1 = DBGF_BP_INT3_L1_IDX_EXTRACT_FROM_ADDR(GCPtrBp); + const uint32_t u32L1Entry = ASMAtomicReadU32(&paBpLocL1[idxL1]); + + LogFlowFunc(("GCPtrBp=%RGv idxL1=%u u32L1Entry=%#x\n", GCPtrBp, idxL1, u32L1Entry)); + if (u32L1Entry != DBGF_BP_INT3_L1_ENTRY_TYPE_NULL) + { + uint8_t u8Type = DBGF_BP_INT3_L1_ENTRY_GET_TYPE(u32L1Entry); + if (u8Type == DBGF_BP_INT3_L1_ENTRY_TYPE_BP_HND) + { + DBGFBP hBp = DBGF_BP_INT3_L1_ENTRY_GET_BP_HND(u32L1Entry); + + /* Query the internal breakpoint state from the handle. */ +#ifdef IN_RING3 + PDBGFBPINT pBp = dbgfBpGetByHnd(pVM, hBp); +#else + PDBGFBPINTR0 pBpR0 = NULL; + PDBGFBPINT pBp = dbgfBpGetByHnd(pVM, hBp, &pBpR0); +#endif + if ( pBp + && DBGF_BP_PUB_GET_TYPE(&pBp->Pub) == DBGFBPTYPE_INT3) + { + if (pBp->Pub.u.Int3.GCPtr == (RTGCUINTPTR)GCPtrBp) +#ifdef IN_RING3 + rc = dbgfBpHit(pVM, pVCpu, pCtx, hBp, pBp); +#else + rc = dbgfBpHit(pVM, pVCpu, pCtx, hBp, pBp, pBpR0); +#endif + else + rc = VINF_EM_RAW_GUEST_TRAP; /* Genuine guest trap. */ + } + else /* Invalid breakpoint handle or not an int3 breakpoint. */ + rc = VERR_DBGF_BP_L1_LOOKUP_FAILED; + } + else if (u8Type == DBGF_BP_INT3_L1_ENTRY_TYPE_L2_IDX) + rc = dbgfBpL2Walk(pVM, pVCpu, pCtx, DBGF_BP_INT3_L1_ENTRY_GET_L2_IDX(u32L1Entry), + DBGF_BP_INT3_L2_KEY_EXTRACT_FROM_ADDR((RTGCUINTPTR)GCPtrBp)); + else /* Some invalid type. */ + rc = VERR_DBGF_BP_L1_LOOKUP_FAILED; + } + else + rc = VINF_EM_RAW_GUEST_TRAP; /* Genuine guest trap. */ + + return rc; + } + + return VINF_EM_RAW_GUEST_TRAP; +} + diff --git a/src/VBox/VMM/VMMAll/DBGFAllTracer.cpp b/src/VBox/VMM/VMMAll/DBGFAllTracer.cpp new file mode 100644 index 00000000..8e79602b --- /dev/null +++ b/src/VBox/VMM/VMMAll/DBGFAllTracer.cpp @@ -0,0 +1,715 @@ +/* $Id: DBGFAllTracer.cpp $ */ +/** @file + * DBGF - Debugger Facility, All Context Code tracing part. + */ + +/* + * Copyright (C) 2020-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_DBGF +#include "DBGFInternal.h" +#include +#include +#include +#if defined(IN_RING3) +# include +# include +#elif defined(IN_RING0) +# include +#else +# error "Invalid environment" +#endif + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ + +/** + * Returns the current context tracer instance of the given VM instance. + * + * @returns Current context pointer to the DBGF tracer instance. + */ +DECLINLINE(PDBGFTRACERINSCC) dbgfTracerGetInstance(PVMCC pVM) +{ +#if defined(IN_RING0) + return pVM->dbgfr0.s.pTracerR0; +#elif defined(IN_RING3) + PUVM pUVM = pVM->pUVM; + return pUVM->dbgf.s.pTracerR3; +#elif defined(IN_RC) +# error "Not implemented" +#else +# error "No/Invalid context specified" +#endif +} + + +/** + * Returns the size of the tracing ring buffer. + * + * @returns Size of the ring buffer in bytes. + * @param pThisCC The event tracer instance current context data. + */ +DECLINLINE(size_t) dbgfTracerGetRingBufSz(PDBGFTRACERINSCC pThisCC) +{ +#if defined(IN_RING0) /* For R0 we are extra cautious and use the ring buffer size stored in R0 memory so R3 can't corrupt it. */ + return pThisCC->cbRingBuf; +#else + return pThisCC->CTX_SUFF(pShared)->cbRingBuf; +#endif +} + + +/** + * Posts a single event descriptor to the ring buffer of the given tracer instance - extended version. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param pThisCC The event tracer instance current context data. + * @param hEvtSrc The event source for the posted event. + * @param enmTraceEvt The trace event type posted. + * @param idEvtPrev The previous event ID the posted event links to. + * @param pvEvtDesc The event descriptor to copy after the header. + * @param cbEvtDesc Size of the event descriptor. + * @param pidEvt Where to store the assigned event ID, optional. + */ +static int dbgfTracerEvtPostEx(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVTSRC hEvtSrc, + DBGFTRACEREVT enmTraceEvt, uint64_t idEvtPrev, const void *pvEvtDesc, + size_t cbEvtDesc, uint64_t *pidEvt) +{ + LogFlowFunc(("pVM=%p pThisCC=%p hEvtSrc=%llu enmTraceEvt=%u idEvtPrev=%llu pvEvtDesc=%p cbEvtDesc=%zu pidEvt=%p\n", + pVM, pThisCC, hEvtSrc, enmTraceEvt, idEvtPrev, pvEvtDesc, cbEvtDesc, pidEvt)); + + PDBGFTRACERSHARED pSharedCC = pThisCC->CTX_SUFF(pShared); + size_t cRingBufEvts = dbgfTracerGetRingBufSz(pThisCC) / DBGF_TRACER_EVT_SZ; + AssertReturn(cRingBufEvts, VERR_DBGF_TRACER_IPE_1); + AssertReturn(cbEvtDesc <= DBGF_TRACER_EVT_PAYLOAD_SZ, VERR_DBGF_TRACER_IPE_1); + + /* Grab a new event ID first. */ + uint64_t idEvt = ASMAtomicIncU64(&pSharedCC->idEvt) - 1; + uint64_t idxRingBuf = idEvt % cRingBufEvts; /* This gives the index in the ring buffer for the event. */ + PDBGFTRACEREVTHDR pEvtHdr = (PDBGFTRACEREVTHDR)(pThisCC->CTX_SUFF(pbRingBuf) + idxRingBuf * DBGF_TRACER_EVT_SZ); + + if (RT_UNLIKELY(ASMAtomicReadU64(&pEvtHdr->idEvt) != DBGF_TRACER_EVT_HDR_ID_INVALID)) + { + /** @todo The event ring buffer is full and we need to go back (from R0 to R3) and wait for the flusher thread to + * get its act together. + */ + AssertMsgFailed(("Flush thread can't keep up with event amount!\n")); + } + + /* Write the event and kick the flush thread if necessary. */ + if (cbEvtDesc) + memcpy(pEvtHdr + 1, pvEvtDesc, cbEvtDesc); + pEvtHdr->idEvtPrev = idEvtPrev; + pEvtHdr->hEvtSrc = hEvtSrc; + pEvtHdr->enmEvt = enmTraceEvt; + pEvtHdr->fFlags = DBGF_TRACER_EVT_HDR_F_DEFAULT; + ASMAtomicWriteU64(&pEvtHdr->idEvt, idEvt); + + int rc = VINF_SUCCESS; + if (!ASMAtomicXchgBool(&pSharedCC->fEvtsWaiting, true)) + { + if (!ASMAtomicXchgBool(&pSharedCC->fFlushThrdActive, true)) + rc = SUPSemEventSignal(pVM->pSession, pSharedCC->hSupSemEvtFlush); + } + + if (pidEvt) + *pidEvt = idEvt; + return rc; +} + + +/** + * Posts a single event descriptor to the ring buffer of the given tracer instance. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param pThisCC The event tracer instance current context data. + * @param hEvtSrc The event source for the posted event. + * @param enmTraceEvt The trace event type posted. + * @param pvEvtDesc The event descriptor to copy after the header. + * @param pidEvt Where to store the assigned event ID, optional. + */ +DECLINLINE(int) dbgfTracerEvtPostSingle(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVTSRC hEvtSrc, + DBGFTRACEREVT enmTraceEvt, const void *pvEvtDesc, uint64_t *pidEvt) +{ + return dbgfTracerEvtPostEx(pVM, pThisCC, hEvtSrc, enmTraceEvt, DBGF_TRACER_EVT_HDR_ID_INVALID, + pvEvtDesc, DBGF_TRACER_EVT_PAYLOAD_SZ, pidEvt); +} + + +#ifdef IN_RING3 +/** + * Posts a single event descriptor to the ring buffer of the given tracer instance - R3 only variant + * (used for the register/deregister event source events currently). + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param pThisCC The event tracer instance current context data. + * @param hEvtSrc The event source for the posted event. + * @param enmTraceEvt The trace event type posted. + * @param pvEvtDesc The event descriptor to copy after the header. + * @param cbEvtDesc Event descriptor size in bytes. + * @param pidEvt Where to store the assigned event ID, optional. + */ +DECLHIDDEN(int) dbgfTracerR3EvtPostSingle(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVTSRC hEvtSrc, + DBGFTRACEREVT enmTraceEvt, const void *pvEvtDesc, size_t cbEvtDesc, + uint64_t *pidEvt) +{ + return dbgfTracerEvtPostEx(pVM, pThisCC, hEvtSrc, enmTraceEvt, DBGF_TRACER_EVT_HDR_ID_INVALID, + pvEvtDesc, cbEvtDesc, pidEvt); +} +#endif + +/** + * Copies the given MMIO value into the event descriptor based on the given size. + * + * @param pEvtMmio Pointer to the MMIO event descriptor to fill. + * @param pvVal The value to copy. + * @param cbVal Size of the value in bytes. + */ +static void dbgfTracerEvtMmioCopyVal(PDBGFTRACEREVTMMIO pEvtMmio, const void *pvVal, size_t cbVal) +{ + switch (cbVal) + { + case 1: + pEvtMmio->u64Val = *(uint8_t *)pvVal; + break; + case 2: + pEvtMmio->u64Val = *(uint16_t *)pvVal; + break; + case 4: + pEvtMmio->u64Val = *(uint32_t *)pvVal; + break; + case 8: + pEvtMmio->u64Val = *(uint64_t *)pvVal; + break; + default: + AssertMsgFailed(("The value size %zu is not supported!\n", cbVal)); + } +} + + +/** + * Copies the given I/O port value into the event descriptor based on the given size. + * + * @param pEvtIoPort Pointer to the I/O port read/write event descriptor to fill. + * @param pvVal The value to copy. + * @param cbVal Size of the value in bytes. + */ +static void dbgfTracerEvtIoPortCopyVal(PDBGFTRACEREVTIOPORT pEvtIoPort, const void *pvVal, size_t cbVal) +{ + switch (cbVal) + { + case 1: + pEvtIoPort->u32Val = *(uint8_t *)pvVal; + break; + case 2: + pEvtIoPort->u32Val = *(uint16_t *)pvVal; + break; + case 4: + pEvtIoPort->u32Val = *(uint32_t *)pvVal; + break; + default: + AssertMsgFailed(("The value size %zu is not supported!\n", cbVal)); + } +} + + +/** + * Handles a guest memory transfer event. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param pThisCC The event tracer instance current context data. + * @param enmTraceEvt The trace event type posted. + * @param hEvtSrc The event source for the posted event. + * @param GCPhys The guest physical address the transfer starts at. + * @param pvBuf The data being transfered. + * @param cbXfer Number of bytes being transfered. + */ +static int dbgfTracerEvtGCPhys(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVT enmTraceEvt, DBGFTRACEREVTSRC hEvtSrc, + RTGCPHYS GCPhys, const void *pvBuf, size_t cbXfer) +{ + /* Fast path for really small transfers where everything fits into the descriptor. */ + DBGFTRACEREVTGCPHYS EvtGCPhys; + EvtGCPhys.GCPhys = GCPhys; + EvtGCPhys.cbXfer = cbXfer; + if (cbXfer <= sizeof(EvtGCPhys.abData)) + { + memcpy(&EvtGCPhys.abData[0], pvBuf, cbXfer); + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, enmTraceEvt, &EvtGCPhys, NULL /*pidEvt*/); + } + + /* + * Slow path where we have to split the data into multiple entries. + * Each one is linked to the previous one by the previous event ID. + */ + const uint8_t *pbBuf = (const uint8_t *)pvBuf; + size_t cbLeft = cbXfer; + uint64_t idEvtPrev = 0; + memcpy(&EvtGCPhys.abData[0], pbBuf, sizeof(EvtGCPhys.abData)); + pbBuf += sizeof(EvtGCPhys.abData); + cbLeft -= sizeof(EvtGCPhys.abData); + + int rc = dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, enmTraceEvt, &EvtGCPhys, &idEvtPrev); + while ( RT_SUCCESS(rc) + && cbLeft) + { + size_t cbThisXfer = RT_MIN(cbLeft, DBGF_TRACER_EVT_PAYLOAD_SZ); + rc = dbgfTracerEvtPostEx(pVM, pThisCC, hEvtSrc, enmTraceEvt, idEvtPrev, + pbBuf, cbThisXfer, &idEvtPrev); + + pbBuf += cbThisXfer; + cbLeft -= cbThisXfer; + } + + return rc; +} + + +/** + * Handles a I/O port string transfer event. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param pThisCC The event tracer instance current context data. + * @param enmTraceEvt The trace event type posted. + * @param hEvtSrc The event source for the posted event. + * @param hIoPorts The I/O port region handle for the transfer. + * @param offPort The offset into the region where the transfer happened. + * @param pv The data being transfered. + * @param cb Number of bytes of valid data in the buffer. + * @param cbItem Item size in bytes. + * @param cTransfersReq Number of transfers requested. + * @param cTransfersRet Number of transfers done. + */ +static int dbgfTracerEvtIoPortStr(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVT enmTraceEvt, DBGFTRACEREVTSRC hEvtSrc, + uint64_t hIoPorts, RTIOPORT offPort, const void *pv, size_t cb, size_t cbItem, uint32_t cTransfersReq, + uint32_t cTransfersRet) +{ + /* Fast path for really small transfers where everything fits into the descriptor. */ + DBGFTRACEREVTIOPORTSTR EvtIoPortStr; + EvtIoPortStr.hIoPorts = hIoPorts; + EvtIoPortStr.cbItem = (uint32_t)cbItem; + EvtIoPortStr.cTransfersReq = cTransfersReq; + EvtIoPortStr.cTransfersRet = cTransfersRet; + EvtIoPortStr.offPort = offPort; + if (cb <= sizeof(EvtIoPortStr.abData)) + { + memcpy(&EvtIoPortStr.abData[0], pv, cb); + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, enmTraceEvt, &EvtIoPortStr, NULL /*pidEvt*/); + } + + /* + * Slow path where we have to split the data into multiple entries. + * Each one is linked to the previous one by the previous event ID. + */ + const uint8_t *pbBuf = (const uint8_t *)pv; + size_t cbLeft = cb; + uint64_t idEvtPrev = 0; + memcpy(&EvtIoPortStr.abData[0], pbBuf, sizeof(EvtIoPortStr.abData)); + pbBuf += sizeof(EvtIoPortStr.abData); + cbLeft -= sizeof(EvtIoPortStr.abData); + + int rc = dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, enmTraceEvt, &EvtIoPortStr, &idEvtPrev); + while ( RT_SUCCESS(rc) + && cbLeft) + { + size_t cbThisXfer = RT_MIN(cbLeft, DBGF_TRACER_EVT_PAYLOAD_SZ); + rc = dbgfTracerEvtPostEx(pVM, pThisCC, hEvtSrc, enmTraceEvt, idEvtPrev, + pbBuf, cbThisXfer, &idEvtPrev); + + pbBuf += cbThisXfer; + cbLeft -= cbThisXfer; + } + + return rc; +} + + +/** + * Registers an MMIO region mapping event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hRegion The MMIO region handle being mapped. + * @param GCPhysMmio The guest physical address where the region is mapped. + */ +VMM_INT_DECL(int) DBGFTracerEvtMmioMap(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion, RTGCPHYS GCPhysMmio) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTMMIOMAP EvtMmioMap; + EvtMmioMap.hMmioRegion = hRegion; + EvtMmioMap.GCPhysMmioBase = GCPhysMmio; + EvtMmioMap.au64Pad0[0] = 0; + EvtMmioMap.au64Pad0[1] = 0; + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_MAP, &EvtMmioMap, NULL /*pidEvt*/); +} + + +/** + * Registers an MMIO region unmap event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hRegion The MMIO region handle being unmapped. + */ +VMM_INT_DECL(int) DBGFTracerEvtMmioUnmap(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTMMIOUNMAP EvtMmioUnmap; + EvtMmioUnmap.hMmioRegion = hRegion; + EvtMmioUnmap.au64Pad0[0] = 0; + EvtMmioUnmap.au64Pad0[1] = 0; + EvtMmioUnmap.au64Pad0[2] = 0; + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_UNMAP, &EvtMmioUnmap, NULL /*pidEvt*/); +} + + +/** + * Registers an MMIO region read event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hRegion The MMIO region handle being read. + * @param offMmio The MMIO offset into the region where the read happened. + * @param pvVal The value being read. + * @param cbVal Value size in bytes. + */ +VMM_INT_DECL(int) DBGFTracerEvtMmioRead(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion, RTGCPHYS offMmio, const void *pvVal, size_t cbVal) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTMMIO EvtMmio; + EvtMmio.hMmioRegion = hRegion; + EvtMmio.offMmio = offMmio; + EvtMmio.cbXfer = cbVal; + dbgfTracerEvtMmioCopyVal(&EvtMmio, pvVal, cbVal); + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_READ, &EvtMmio, NULL /*pidEvt*/); +} + + +/** + * Registers an MMIO region write event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hRegion The MMIO region handle being written to. + * @param offMmio The MMIO offset into the region where the write happened. + * @param pvVal The value being written. + * @param cbVal Value size in bytes. + */ +VMM_INT_DECL(int) DBGFTracerEvtMmioWrite(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion, RTGCPHYS offMmio, const void *pvVal, size_t cbVal) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTMMIO EvtMmio; + EvtMmio.hMmioRegion = hRegion; + EvtMmio.offMmio = offMmio; + EvtMmio.cbXfer = cbVal; + dbgfTracerEvtMmioCopyVal(&EvtMmio, pvVal, cbVal); + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_WRITE, &EvtMmio, NULL /*pidEvt*/); +} + + +/** + * Registers an MMIO region fill event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hRegion The MMIO region handle being filled. + * @param offMmio The MMIO offset into the region where the fill starts. + * @param u32Item The value being used for filling. + * @param cbItem Item size in bytes. + * @param cItems Number of items being written. + */ +VMM_INT_DECL(int) DBGFTracerEvtMmioFill(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion, RTGCPHYS offMmio, + uint32_t u32Item, uint32_t cbItem, uint32_t cItems) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTMMIOFILL EvtMmioFill; + EvtMmioFill.hMmioRegion = hRegion; + EvtMmioFill.offMmio = offMmio; + EvtMmioFill.cbItem = cbItem; + EvtMmioFill.cItems = cItems; + EvtMmioFill.u32Item = u32Item; + EvtMmioFill.u32Pad0 = 0; + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_FILL, &EvtMmioFill, NULL /*pidEvt*/); +} + + +/** + * Registers an I/O region mapping event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hIoPorts The I/O port region handle being mapped. + * @param IoPortBase The I/O port base where the region is mapped. + */ +VMM_INT_DECL(int) DBGFTracerEvtIoPortMap(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT IoPortBase) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTIOPORTMAP EvtIoPortMap; + RT_ZERO(EvtIoPortMap); + EvtIoPortMap.hIoPorts = hIoPorts; + EvtIoPortMap.IoPortBase = IoPortBase; + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOPORT_MAP, &EvtIoPortMap, NULL /*pidEvt*/); +} + + +/** + * Registers an I/O region unmap event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hIoPorts The I/O port region handle being unmapped. + */ +VMM_INT_DECL(int) DBGFTracerEvtIoPortUnmap(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTIOPORTUNMAP EvtIoPortUnmap; + EvtIoPortUnmap.hIoPorts = hIoPorts; + EvtIoPortUnmap.au64Pad0[0] = 0; + EvtIoPortUnmap.au64Pad0[1] = 0; + EvtIoPortUnmap.au64Pad0[2] = 0; + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOPORT_UNMAP, &EvtIoPortUnmap, NULL /*pidEvt*/); +} + + +/** + * Registers an I/O region read event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hIoPorts The I/O port region handle being read from. + * @param offPort The offset into the region where the read happened. + * @param pvVal The value being read. + * @param cbVal Value size in bytes. + */ +VMM_INT_DECL(int) DBGFTracerEvtIoPortRead(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT offPort, const void *pvVal, size_t cbVal) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTIOPORT EvtIoPort; + RT_ZERO(EvtIoPort); + EvtIoPort.hIoPorts = hIoPorts; + EvtIoPort.offPort = offPort; + EvtIoPort.cbXfer = cbVal; + dbgfTracerEvtIoPortCopyVal(&EvtIoPort, pvVal, cbVal); + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOPORT_READ, &EvtIoPort, NULL /*pidEvt*/); +} + + +/** + * Registers an I/O region string read event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hIoPorts The I/O port region handle being read from. + * @param offPort The offset into the region where the read happened. + * @param pv The data being read. + * @param cb Item size in bytes. + * @param cTransfersReq Number of transfers requested. + * @param cTransfersRet Number of transfers done. + */ +VMM_INT_DECL(int) DBGFTracerEvtIoPortReadStr(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT offPort, const void *pv, size_t cb, + uint32_t cTransfersReq, uint32_t cTransfersRet) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + return dbgfTracerEvtIoPortStr(pVM, pThisCC, DBGFTRACEREVT_IOPORT_READ_STR, hEvtSrc, hIoPorts, offPort, pv, cTransfersRet * cb, + cb, cTransfersReq, cTransfersRet); +} + + +/** + * Registers an I/O region write event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hIoPorts The I/O port region handle being written to. + * @param offPort The offset into the region where the write happened. + * @param pvVal The value being written. + * @param cbVal Value size in bytes. + */ +VMM_INT_DECL(int) DBGFTracerEvtIoPortWrite(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT offPort, const void *pvVal, size_t cbVal) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTIOPORT EvtIoPort; + RT_ZERO(EvtIoPort); + EvtIoPort.hIoPorts = hIoPorts; + EvtIoPort.offPort = offPort; + EvtIoPort.cbXfer = cbVal; + dbgfTracerEvtIoPortCopyVal(&EvtIoPort, pvVal, cbVal); + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOPORT_WRITE, &EvtIoPort, NULL /*pidEvt*/); +} + + +/** + * Registers an I/O region string write event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param hIoPorts The I/O port region handle being written to. + * @param offPort The offset into the region where the write happened. + * @param pv The data being written. + * @param cb Item size in bytes. + * @param cTransfersReq Number of transfers requested. + * @param cTransfersRet Number of transfers done. + */ +VMM_INT_DECL(int) DBGFTracerEvtIoPortWriteStr(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT offPort, const void *pv, size_t cb, + uint32_t cTransfersReq, uint32_t cTransfersRet) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + return dbgfTracerEvtIoPortStr(pVM, pThisCC, DBGFTRACEREVT_IOPORT_WRITE_STR, hEvtSrc, hIoPorts, offPort, pv, cTransfersReq * cb, + cb, cTransfersReq, cTransfersRet); +} + + +/** + * Registers an IRQ change event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param iIrq The IRQ line changed. + * @param fIrqLvl The new IRQ level mask. + */ +VMM_INT_DECL(int) DBGFTracerEvtIrq(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, int32_t iIrq, int32_t fIrqLvl) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTIRQ EvtIrq; + RT_ZERO(EvtIrq); + EvtIrq.iIrq = iIrq; + EvtIrq.fIrqLvl = fIrqLvl; + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IRQ, &EvtIrq, NULL /*pidEvt*/); +} + + +/** + * Registers an I/O APIC MSI event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param GCPhys Guest physical address where the value is written to. + * @param u32Val The MSI event value being written. + */ +VMM_INT_DECL(int) DBGFTracerEvtIoApicMsi(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, RTGCPHYS GCPhys, uint32_t u32Val) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + DBGFTRACEREVTIOAPICMSI EvtMsi; + RT_ZERO(EvtMsi); + EvtMsi.GCPhys = GCPhys; + EvtMsi.u32Val = u32Val; + + return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOAPIC_MSI, &EvtMsi, NULL /*pidEvt*/); +} + + +/** + * Registers an guest physical memory read event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param GCPhys Guest physical address the read started at. + * @param pvBuf The read data. + * @param cbRead Number of bytes read. + */ +VMM_INT_DECL(int) DBGFTracerEvtGCPhysRead(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, RTGCPHYS GCPhys, const void *pvBuf, size_t cbRead) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + return dbgfTracerEvtGCPhys(pVM, pThisCC, DBGFTRACEREVT_GCPHYS_READ, hEvtSrc, GCPhys, pvBuf, cbRead); +} + + +/** + * Registers an guest physical memory write event for the given event source. + * + * @returns VBox status code. + * @param pVM The current context VM instance data. + * @param hEvtSrc The event source for the posted event. + * @param GCPhys Guest physical address the write started at. + * @param pvBuf The written data. + * @param cbWrite Number of bytes written. + */ +VMM_INT_DECL(int) DBGFTracerEvtGCPhysWrite(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite) +{ + PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM); + AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1); + + return dbgfTracerEvtGCPhys(pVM, pThisCC, DBGFTRACEREVT_GCPHYS_WRITE, hEvtSrc, GCPhys, pvBuf, cbWrite); +} + diff --git a/src/VBox/VMM/VMMAll/EMAll.cpp b/src/VBox/VMM/VMMAll/EMAll.cpp new file mode 100644 index 00000000..bc5106d3 --- /dev/null +++ b/src/VBox/VMM/VMMAll/EMAll.cpp @@ -0,0 +1,1018 @@ +/* $Id: EMAll.cpp $ */ +/** @file + * EM - Execution Monitor(/Manager) - All contexts + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_EM +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "EMInternal.h" +#include +#include +#include +#include +#include +#include +#include +#include + + + + +/** + * Get the current execution manager status. + * + * @returns Current status. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(EMSTATE) EMGetState(PVMCPU pVCpu) +{ + return pVCpu->em.s.enmState; +} + + +/** + * Sets the current execution manager status. (use only when you know what you're doing!) + * + * @param pVCpu The cross context virtual CPU structure. + * @param enmNewState The new state, EMSTATE_WAIT_SIPI or EMSTATE_HALTED. + */ +VMM_INT_DECL(void) EMSetState(PVMCPU pVCpu, EMSTATE enmNewState) +{ + /* Only allowed combination: */ + Assert(pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI && enmNewState == EMSTATE_HALTED); + pVCpu->em.s.enmState = enmNewState; +} + + +/** + * Enables / disable hypercall instructions. + * + * This interface is used by GIM to tell the execution monitors whether the + * hypercall instruction (VMMCALL & VMCALL) are allowed or should \#UD. + * + * @param pVCpu The cross context virtual CPU structure this applies to. + * @param fEnabled Whether hypercall instructions are enabled (true) or not. + */ +VMMDECL(void) EMSetHypercallInstructionsEnabled(PVMCPU pVCpu, bool fEnabled) +{ + pVCpu->em.s.fHypercallEnabled = fEnabled; +} + + +/** + * Checks if hypercall instructions (VMMCALL & VMCALL) are enabled or not. + * + * @returns true if enabled, false if not. + * @param pVCpu The cross context virtual CPU structure. + * + * @note If this call becomes a performance factor, we can make the data + * field available thru a read-only view in VMCPU. See VM::cpum.ro. + */ +VMMDECL(bool) EMAreHypercallInstructionsEnabled(PVMCPU pVCpu) +{ + return pVCpu->em.s.fHypercallEnabled; +} + + +/** + * Prepare an MWAIT - essentials of the MONITOR instruction. + * + * @returns VINF_SUCCESS + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param rax The content of RAX. + * @param rcx The content of RCX. + * @param rdx The content of RDX. + * @param GCPhys The physical address corresponding to rax. + */ +VMM_INT_DECL(int) EMMonitorWaitPrepare(PVMCPU pVCpu, uint64_t rax, uint64_t rcx, uint64_t rdx, RTGCPHYS GCPhys) +{ + pVCpu->em.s.MWait.uMonitorRAX = rax; + pVCpu->em.s.MWait.uMonitorRCX = rcx; + pVCpu->em.s.MWait.uMonitorRDX = rdx; + pVCpu->em.s.MWait.fWait |= EMMWAIT_FLAG_MONITOR_ACTIVE; + /** @todo Make use of GCPhys. */ + NOREF(GCPhys); + /** @todo Complete MONITOR implementation. */ + return VINF_SUCCESS; +} + + +/** + * Checks if the monitor hardware is armed / active. + * + * @returns true if armed, false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMM_INT_DECL(bool) EMMonitorIsArmed(PVMCPU pVCpu) +{ + return RT_BOOL(pVCpu->em.s.MWait.fWait & EMMWAIT_FLAG_MONITOR_ACTIVE); +} + + +/** + * Checks if we're in a MWAIT. + * + * @retval 1 if regular, + * @retval > 1 if MWAIT with EMMWAIT_FLAG_BREAKIRQIF0 + * @retval 0 if not armed + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMM_INT_DECL(unsigned) EMMonitorWaitIsActive(PVMCPU pVCpu) +{ + uint32_t fWait = pVCpu->em.s.MWait.fWait; + AssertCompile(EMMWAIT_FLAG_ACTIVE == 1); + AssertCompile(EMMWAIT_FLAG_BREAKIRQIF0 == 2); + AssertCompile((EMMWAIT_FLAG_ACTIVE << 1) == EMMWAIT_FLAG_BREAKIRQIF0); + return fWait & (EMMWAIT_FLAG_ACTIVE | ((fWait & EMMWAIT_FLAG_ACTIVE) << 1)); +} + + +/** + * Performs an MWAIT. + * + * @returns VINF_SUCCESS + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param rax The content of RAX. + * @param rcx The content of RCX. + */ +VMM_INT_DECL(int) EMMonitorWaitPerform(PVMCPU pVCpu, uint64_t rax, uint64_t rcx) +{ + pVCpu->em.s.MWait.uMWaitRAX = rax; + pVCpu->em.s.MWait.uMWaitRCX = rcx; + pVCpu->em.s.MWait.fWait |= EMMWAIT_FLAG_ACTIVE; + if (rcx) + pVCpu->em.s.MWait.fWait |= EMMWAIT_FLAG_BREAKIRQIF0; + else + pVCpu->em.s.MWait.fWait &= ~EMMWAIT_FLAG_BREAKIRQIF0; + /** @todo not completely correct?? */ + return VINF_EM_HALT; +} + + +/** + * Clears any address-range monitoring that is active. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMM_INT_DECL(void) EMMonitorWaitClear(PVMCPU pVCpu) +{ + LogFlowFunc(("Clearing MWAIT\n")); + pVCpu->em.s.MWait.fWait &= ~(EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0); +} + + +/** + * Determine if we should continue execution in HM after encountering an mwait + * instruction. + * + * Clears MWAIT flags if returning @c true. + * + * @returns true if we should continue, false if we should halt. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Current CPU context. + */ +VMM_INT_DECL(bool) EMMonitorWaitShouldContinue(PVMCPU pVCpu, PCPUMCTX pCtx) +{ + if (CPUMGetGuestGif(pCtx)) + { + if ( CPUMIsGuestPhysIntrEnabled(pVCpu) + || ( CPUMIsGuestInNestedHwvirtMode(pCtx) + && CPUMIsGuestVirtIntrEnabled(pVCpu)) + || ( (pVCpu->em.s.MWait.fWait & (EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0)) + == (EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0)) ) + { + if (VMCPU_FF_IS_ANY_SET(pVCpu, ( VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC + | VMCPU_FF_INTERRUPT_NESTED_GUEST))) + { + pVCpu->em.s.MWait.fWait &= ~(EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0); + return true; + } + } + } + + return false; +} + + +/** + * Determine if we should continue execution in HM after encountering a hlt + * instruction. + * + * @returns true if we should continue, false if we should halt. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Current CPU context. + */ +VMM_INT_DECL(bool) EMShouldContinueAfterHalt(PVMCPU pVCpu, PCPUMCTX pCtx) +{ + if (CPUMGetGuestGif(pCtx)) + { + if (CPUMIsGuestPhysIntrEnabled(pVCpu)) + return VMCPU_FF_IS_ANY_SET(pVCpu, (VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)); + + if ( CPUMIsGuestInNestedHwvirtMode(pCtx) + && CPUMIsGuestVirtIntrEnabled(pVCpu)) + return VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST); + } + return false; +} + + +/** + * Unhalts and wakes up the given CPU. + * + * This is an API for assisting the KVM hypercall API in implementing KICK_CPU. + * It sets VMCPU_FF_UNHALT for @a pVCpuDst and makes sure it is woken up. If + * the CPU isn't currently in a halt, the next HLT instruction it executes will + * be affected. + * + * @returns GVMMR0SchedWakeUpEx result or VINF_SUCCESS depending on context. + * @param pVM The cross context VM structure. + * @param pVCpuDst The cross context virtual CPU structure of the + * CPU to unhalt and wake up. This is usually not the + * same as the caller. + * @thread EMT + */ +VMM_INT_DECL(int) EMUnhaltAndWakeUp(PVMCC pVM, PVMCPUCC pVCpuDst) +{ + /* + * Flag the current(/next) HLT to unhalt immediately. + */ + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_UNHALT); + + /* + * Wake up the EMT (technically should be abstracted by VMM/VMEmt, but + * just do it here for now). + */ +#ifdef IN_RING0 + /* We might be here with preemption disabled or enabled (i.e. depending on + thread-context hooks being used), so don't try obtaining the GVMMR0 used + lock here. See @bugref{7270#c148}. */ + int rc = GVMMR0SchedWakeUpNoGVMNoLock(pVM, pVCpuDst->idCpu); + AssertRC(rc); + +#elif defined(IN_RING3) + VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, 0 /*fFlags*/); + int rc = VINF_SUCCESS; + RT_NOREF(pVM); + +#else + /* Nothing to do for raw-mode, shouldn't really be used by raw-mode guests anyway. */ + Assert(pVM->cCpus == 1); NOREF(pVM); + int rc = VINF_SUCCESS; +#endif + return rc; +} + +#ifndef IN_RING3 + +/** + * Makes an I/O port write pending for ring-3 processing. + * + * @returns VINF_EM_PENDING_R3_IOPORT_READ + * @param pVCpu The cross context virtual CPU structure. + * @param uPort The I/O port. + * @param cbInstr The instruction length (for RIP updating). + * @param cbValue The write size. + * @param uValue The value being written. + * @sa emR3ExecutePendingIoPortWrite + * + * @note Must not be used when I/O port breakpoints are pending or when single stepping. + */ +VMMRZ_INT_DECL(VBOXSTRICTRC) +EMRZSetPendingIoPortWrite(PVMCPU pVCpu, RTIOPORT uPort, uint8_t cbInstr, uint8_t cbValue, uint32_t uValue) +{ + Assert(pVCpu->em.s.PendingIoPortAccess.cbValue == 0); + pVCpu->em.s.PendingIoPortAccess.uPort = uPort; + pVCpu->em.s.PendingIoPortAccess.cbValue = cbValue; + pVCpu->em.s.PendingIoPortAccess.cbInstr = cbInstr; + pVCpu->em.s.PendingIoPortAccess.uValue = uValue; + return VINF_EM_PENDING_R3_IOPORT_WRITE; +} + + +/** + * Makes an I/O port read pending for ring-3 processing. + * + * @returns VINF_EM_PENDING_R3_IOPORT_READ + * @param pVCpu The cross context virtual CPU structure. + * @param uPort The I/O port. + * @param cbInstr The instruction length (for RIP updating). + * @param cbValue The read size. + * @sa emR3ExecutePendingIoPortRead + * + * @note Must not be used when I/O port breakpoints are pending or when single stepping. + */ +VMMRZ_INT_DECL(VBOXSTRICTRC) +EMRZSetPendingIoPortRead(PVMCPU pVCpu, RTIOPORT uPort, uint8_t cbInstr, uint8_t cbValue) +{ + Assert(pVCpu->em.s.PendingIoPortAccess.cbValue == 0); + pVCpu->em.s.PendingIoPortAccess.uPort = uPort; + pVCpu->em.s.PendingIoPortAccess.cbValue = cbValue; + pVCpu->em.s.PendingIoPortAccess.cbInstr = cbInstr; + pVCpu->em.s.PendingIoPortAccess.uValue = UINT32_C(0x52454144); /* 'READ' */ + return VINF_EM_PENDING_R3_IOPORT_READ; +} + +#endif /* IN_RING3 */ + + +/** + * Worker for EMHistoryExec that checks for ring-3 returns and flags + * continuation of the EMHistoryExec run there. + */ +DECL_FORCE_INLINE(void) emHistoryExecSetContinueExitRecIdx(PVMCPU pVCpu, VBOXSTRICTRC rcStrict, PCEMEXITREC pExitRec) +{ + pVCpu->em.s.idxContinueExitRec = UINT16_MAX; +#ifdef IN_RING3 + RT_NOREF_PV(rcStrict); RT_NOREF_PV(pExitRec); +#else + switch (VBOXSTRICTRC_VAL(rcStrict)) + { + case VINF_SUCCESS: + default: + break; + + /* + * Only status codes that EMHandleRCTmpl.h will resume EMHistoryExec with. + */ + case VINF_IOM_R3_IOPORT_READ: /* -> emR3ExecuteIOInstruction */ + case VINF_IOM_R3_IOPORT_WRITE: /* -> emR3ExecuteIOInstruction */ + case VINF_IOM_R3_IOPORT_COMMIT_WRITE: /* -> VMCPU_FF_IOM -> VINF_EM_RESUME_R3_HISTORY_EXEC -> emR3ExecuteIOInstruction */ + case VINF_IOM_R3_MMIO_READ: /* -> emR3ExecuteInstruction */ + case VINF_IOM_R3_MMIO_WRITE: /* -> emR3ExecuteInstruction */ + case VINF_IOM_R3_MMIO_READ_WRITE: /* -> emR3ExecuteInstruction */ + case VINF_IOM_R3_MMIO_COMMIT_WRITE: /* -> VMCPU_FF_IOM -> VINF_EM_RESUME_R3_HISTORY_EXEC -> emR3ExecuteIOInstruction */ + case VINF_CPUM_R3_MSR_READ: /* -> emR3ExecuteInstruction */ + case VINF_CPUM_R3_MSR_WRITE: /* -> emR3ExecuteInstruction */ + case VINF_GIM_R3_HYPERCALL: /* -> emR3ExecuteInstruction */ + pVCpu->em.s.idxContinueExitRec = (uint16_t)(pExitRec - &pVCpu->em.s.aExitRecords[0]); + break; + } +#endif /* !IN_RING3 */ +} + + +/** + * Execute using history. + * + * This function will be called when EMHistoryAddExit() and friends returns a + * non-NULL result. This happens in response to probing or when probing has + * uncovered adjacent exits which can more effectively be reached by using IEM + * than restarting execution using the main execution engine and fielding an + * regular exit. + * + * @returns VBox strict status code, see IEMExecForExits. + * @param pVCpu The cross context virtual CPU structure. + * @param pExitRec The exit record return by a previous history add + * or update call. + * @param fWillExit Flags indicating to IEM what will cause exits, TBD. + */ +VMM_INT_DECL(VBOXSTRICTRC) EMHistoryExec(PVMCPUCC pVCpu, PCEMEXITREC pExitRec, uint32_t fWillExit) +{ + Assert(pExitRec); + VMCPU_ASSERT_EMT(pVCpu); + IEMEXECFOREXITSTATS ExecStats; + switch (pExitRec->enmAction) + { + /* + * Executes multiple instruction stopping only when we've gone a given + * number without perceived exits. + */ + case EMEXITACTION_EXEC_WITH_MAX: + { + STAM_REL_PROFILE_START(&pVCpu->em.s.StatHistoryExec, a); + LogFlow(("EMHistoryExec/EXEC_WITH_MAX: %RX64, max %u\n", pExitRec->uFlatPC, pExitRec->cMaxInstructionsWithoutExit)); + VBOXSTRICTRC rcStrict = IEMExecForExits(pVCpu, fWillExit, + pExitRec->cMaxInstructionsWithoutExit /* cMinInstructions*/, + pVCpu->em.s.cHistoryExecMaxInstructions, + pExitRec->cMaxInstructionsWithoutExit, + &ExecStats); + LogFlow(("EMHistoryExec/EXEC_WITH_MAX: %Rrc cExits=%u cMaxExitDistance=%u cInstructions=%u\n", + VBOXSTRICTRC_VAL(rcStrict), ExecStats.cExits, ExecStats.cMaxExitDistance, ExecStats.cInstructions)); + emHistoryExecSetContinueExitRecIdx(pVCpu, rcStrict, pExitRec); + + /* Ignore instructions IEM doesn't know about. */ + if ( ( rcStrict != VERR_IEM_INSTR_NOT_IMPLEMENTED + && rcStrict != VERR_IEM_ASPECT_NOT_IMPLEMENTED) + || ExecStats.cInstructions == 0) + { /* likely */ } + else + rcStrict = VINF_SUCCESS; + + if (ExecStats.cExits > 1) + STAM_REL_COUNTER_ADD(&pVCpu->em.s.StatHistoryExecSavedExits, ExecStats.cExits - 1); + STAM_REL_COUNTER_ADD(&pVCpu->em.s.StatHistoryExecInstructions, ExecStats.cInstructions); + STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHistoryExec, a); + return rcStrict; + } + + /* + * Probe a exit for close by exits. + */ + case EMEXITACTION_EXEC_PROBE: + { + STAM_REL_PROFILE_START(&pVCpu->em.s.StatHistoryProbe, b); + LogFlow(("EMHistoryExec/EXEC_PROBE: %RX64\n", pExitRec->uFlatPC)); + PEMEXITREC pExitRecUnconst = (PEMEXITREC)pExitRec; + VBOXSTRICTRC rcStrict = IEMExecForExits(pVCpu, fWillExit, + pVCpu->em.s.cHistoryProbeMinInstructions, + pVCpu->em.s.cHistoryExecMaxInstructions, + pVCpu->em.s.cHistoryProbeMaxInstructionsWithoutExit, + &ExecStats); + LogFlow(("EMHistoryExec/EXEC_PROBE: %Rrc cExits=%u cMaxExitDistance=%u cInstructions=%u\n", + VBOXSTRICTRC_VAL(rcStrict), ExecStats.cExits, ExecStats.cMaxExitDistance, ExecStats.cInstructions)); + emHistoryExecSetContinueExitRecIdx(pVCpu, rcStrict, pExitRecUnconst); + if ( ExecStats.cExits >= 2 + && RT_SUCCESS(rcStrict)) + { + Assert(ExecStats.cMaxExitDistance > 0 && ExecStats.cMaxExitDistance <= 32); + pExitRecUnconst->cMaxInstructionsWithoutExit = ExecStats.cMaxExitDistance; + pExitRecUnconst->enmAction = EMEXITACTION_EXEC_WITH_MAX; + LogFlow(("EMHistoryExec/EXEC_PROBE: -> EXEC_WITH_MAX %u\n", ExecStats.cMaxExitDistance)); + STAM_REL_COUNTER_INC(&pVCpu->em.s.StatHistoryProbedExecWithMax); + } +#ifndef IN_RING3 + else if ( pVCpu->em.s.idxContinueExitRec != UINT16_MAX + && RT_SUCCESS(rcStrict)) + { + STAM_REL_COUNTER_INC(&pVCpu->em.s.StatHistoryProbedToRing3); + LogFlow(("EMHistoryExec/EXEC_PROBE: -> ring-3\n")); + } +#endif + else + { + pExitRecUnconst->enmAction = EMEXITACTION_NORMAL_PROBED; + pVCpu->em.s.idxContinueExitRec = UINT16_MAX; + LogFlow(("EMHistoryExec/EXEC_PROBE: -> PROBED\n")); + STAM_REL_COUNTER_INC(&pVCpu->em.s.StatHistoryProbedNormal); + if ( rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED + || rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED) + rcStrict = VINF_SUCCESS; + } + STAM_REL_COUNTER_ADD(&pVCpu->em.s.StatHistoryProbeInstructions, ExecStats.cInstructions); + STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHistoryProbe, b); + return rcStrict; + } + + /* We shouldn't ever see these here! */ + case EMEXITACTION_FREE_RECORD: + case EMEXITACTION_NORMAL: + case EMEXITACTION_NORMAL_PROBED: + break; + + /* No default case, want compiler warnings. */ + } + AssertLogRelFailedReturn(VERR_EM_INTERNAL_ERROR); +} + + +/** + * Worker for emHistoryAddOrUpdateRecord. + */ +DECL_FORCE_INLINE(PCEMEXITREC) emHistoryRecordInit(PEMEXITREC pExitRec, uint64_t uFlatPC, uint32_t uFlagsAndType, uint64_t uExitNo) +{ + pExitRec->uFlatPC = uFlatPC; + pExitRec->uFlagsAndType = uFlagsAndType; + pExitRec->enmAction = EMEXITACTION_NORMAL; + pExitRec->bUnused = 0; + pExitRec->cMaxInstructionsWithoutExit = 64; + pExitRec->uLastExitNo = uExitNo; + pExitRec->cHits = 1; + return NULL; +} + + +/** + * Worker for emHistoryAddOrUpdateRecord. + */ +DECL_FORCE_INLINE(PCEMEXITREC) emHistoryRecordInitNew(PVMCPU pVCpu, PEMEXITENTRY pHistEntry, uintptr_t idxSlot, + PEMEXITREC pExitRec, uint64_t uFlatPC, + uint32_t uFlagsAndType, uint64_t uExitNo) +{ + pHistEntry->idxSlot = (uint32_t)idxSlot; + pVCpu->em.s.cExitRecordUsed++; + LogFlow(("emHistoryRecordInitNew: [%#x] = %#07x %016RX64; (%u of %u used)\n", idxSlot, uFlagsAndType, uFlatPC, + pVCpu->em.s.cExitRecordUsed, RT_ELEMENTS(pVCpu->em.s.aExitRecords) )); + return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo); +} + + +/** + * Worker for emHistoryAddOrUpdateRecord. + */ +DECL_FORCE_INLINE(PCEMEXITREC) emHistoryRecordInitReplacement(PEMEXITENTRY pHistEntry, uintptr_t idxSlot, + PEMEXITREC pExitRec, uint64_t uFlatPC, + uint32_t uFlagsAndType, uint64_t uExitNo) +{ + pHistEntry->idxSlot = (uint32_t)idxSlot; + LogFlow(("emHistoryRecordInitReplacement: [%#x] = %#07x %016RX64 replacing %#07x %016RX64 with %u hits, %u exits old\n", + idxSlot, uFlagsAndType, uFlatPC, pExitRec->uFlagsAndType, pExitRec->uFlatPC, pExitRec->cHits, + uExitNo - pExitRec->uLastExitNo)); + return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo); +} + + +/** + * Adds or updates the EMEXITREC for this PC/type and decide on an action. + * + * @returns Pointer to an exit record if special action should be taken using + * EMHistoryExec(). Take normal exit action when NULL. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uFlagsAndType Combined flags and type, EMEXIT_F_KIND_EM set and + * both EMEXIT_F_CS_EIP and EMEXIT_F_UNFLATTENED_PC are clear. + * @param uFlatPC The flattened program counter. + * @param pHistEntry The exit history entry. + * @param uExitNo The current exit number. + */ +static PCEMEXITREC emHistoryAddOrUpdateRecord(PVMCPU pVCpu, uint64_t uFlagsAndType, uint64_t uFlatPC, + PEMEXITENTRY pHistEntry, uint64_t uExitNo) +{ +# ifdef IN_RING0 + /* Disregard the hm flag. */ + uFlagsAndType &= ~EMEXIT_F_HM; +# endif + + /* + * Work the hash table. + */ + AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitRecords) == 1024); +# define EM_EXIT_RECORDS_IDX_MASK 0x3ff + uintptr_t idxSlot = ((uintptr_t)uFlatPC >> 1) & EM_EXIT_RECORDS_IDX_MASK; + PEMEXITREC pExitRec = &pVCpu->em.s.aExitRecords[idxSlot]; + if (pExitRec->uFlatPC == uFlatPC) + { + Assert(pExitRec->enmAction != EMEXITACTION_FREE_RECORD); + pHistEntry->idxSlot = (uint32_t)idxSlot; + if (pExitRec->uFlagsAndType == uFlagsAndType) + { + pExitRec->uLastExitNo = uExitNo; + STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecHits[0]); + } + else + { + STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecTypeChanged[0]); + return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo); + } + } + else if (pExitRec->enmAction == EMEXITACTION_FREE_RECORD) + { + STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecNew[0]); + return emHistoryRecordInitNew(pVCpu, pHistEntry, idxSlot, pExitRec, uFlatPC, uFlagsAndType, uExitNo); + } + else + { + /* + * Collision. We calculate a new hash for stepping away from the first, + * doing up to 8 steps away before replacing the least recently used record. + */ + uintptr_t idxOldest = idxSlot; + uint64_t uOldestExitNo = pExitRec->uLastExitNo; + unsigned iOldestStep = 0; + unsigned iStep = 1; + uintptr_t const idxAdd = (uintptr_t)(uFlatPC >> 11) & (EM_EXIT_RECORDS_IDX_MASK / 4); + for (;;) + { + Assert(iStep < RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits)); + AssertCompile(RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecNew) == RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits)); + AssertCompile(RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecReplaced) == RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits)); + AssertCompile(RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecTypeChanged) == RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits)); + + /* Step to the next slot. */ + idxSlot += idxAdd; + idxSlot &= EM_EXIT_RECORDS_IDX_MASK; + pExitRec = &pVCpu->em.s.aExitRecords[idxSlot]; + + /* Does it match? */ + if (pExitRec->uFlatPC == uFlatPC) + { + Assert(pExitRec->enmAction != EMEXITACTION_FREE_RECORD); + pHistEntry->idxSlot = (uint32_t)idxSlot; + if (pExitRec->uFlagsAndType == uFlagsAndType) + { + pExitRec->uLastExitNo = uExitNo; + STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecHits[iStep]); + break; + } + STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecTypeChanged[iStep]); + return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo); + } + + /* Is it free? */ + if (pExitRec->enmAction == EMEXITACTION_FREE_RECORD) + { + STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecNew[iStep]); + return emHistoryRecordInitNew(pVCpu, pHistEntry, idxSlot, pExitRec, uFlatPC, uFlagsAndType, uExitNo); + } + + /* Is it the least recently used one? */ + if (pExitRec->uLastExitNo < uOldestExitNo) + { + uOldestExitNo = pExitRec->uLastExitNo; + idxOldest = idxSlot; + iOldestStep = iStep; + } + + /* Next iteration? */ + iStep++; + Assert(iStep < RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecReplaced)); + if (RT_LIKELY(iStep < 8 + 1)) + { /* likely */ } + else + { + /* Replace the least recently used slot. */ + STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecReplaced[iOldestStep]); + pExitRec = &pVCpu->em.s.aExitRecords[idxOldest]; + return emHistoryRecordInitReplacement(pHistEntry, idxOldest, pExitRec, uFlatPC, uFlagsAndType, uExitNo); + } + } + } + + /* + * Found an existing record. + */ + switch (pExitRec->enmAction) + { + case EMEXITACTION_NORMAL: + { + uint64_t const cHits = ++pExitRec->cHits; + if (cHits < 256) + return NULL; + LogFlow(("emHistoryAddOrUpdateRecord: [%#x] %#07x %16RX64: -> EXEC_PROBE\n", idxSlot, uFlagsAndType, uFlatPC)); + pExitRec->enmAction = EMEXITACTION_EXEC_PROBE; + return pExitRec; + } + + case EMEXITACTION_NORMAL_PROBED: + pExitRec->cHits += 1; + return NULL; + + default: + pExitRec->cHits += 1; + return pExitRec; + + /* This will happen if the caller ignores or cannot serve the probe + request (forced to ring-3, whatever). We retry this 256 times. */ + case EMEXITACTION_EXEC_PROBE: + { + uint64_t const cHits = ++pExitRec->cHits; + if (cHits < 512) + return pExitRec; + pExitRec->enmAction = EMEXITACTION_NORMAL_PROBED; + LogFlow(("emHistoryAddOrUpdateRecord: [%#x] %#07x %16RX64: -> PROBED\n", idxSlot, uFlagsAndType, uFlatPC)); + return NULL; + } + } +} + + +/** + * Adds an exit to the history for this CPU. + * + * @returns Pointer to an exit record if special action should be taken using + * EMHistoryExec(). Take normal exit action when NULL. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uFlagsAndType Combined flags and type (see EMEXIT_MAKE_FT). + * @param uFlatPC The flattened program counter (RIP). UINT64_MAX if not available. + * @param uTimestamp The TSC value for the exit, 0 if not available. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(PCEMEXITREC) EMHistoryAddExit(PVMCPUCC pVCpu, uint32_t uFlagsAndType, uint64_t uFlatPC, uint64_t uTimestamp) +{ + VMCPU_ASSERT_EMT(pVCpu); + + /* + * Add the exit history entry. + */ + AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256); + uint64_t uExitNo = pVCpu->em.s.iNextExit++; + PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff]; + pHistEntry->uFlatPC = uFlatPC; + pHistEntry->uTimestamp = uTimestamp; + pHistEntry->uFlagsAndType = uFlagsAndType; + pHistEntry->idxSlot = UINT32_MAX; + + /* + * If common exit type, we will insert/update the exit into the exit record hash table. + */ + if ( (uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)) == EMEXIT_F_KIND_EM +#ifdef IN_RING0 + && pVCpu->em.s.fExitOptimizationEnabledR0 + && ( !(uFlagsAndType & EMEXIT_F_HM) || pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled) +#else + && pVCpu->em.s.fExitOptimizationEnabled +#endif + && uFlatPC != UINT64_MAX + ) + return emHistoryAddOrUpdateRecord(pVCpu, uFlagsAndType, uFlatPC, pHistEntry, uExitNo); + return NULL; +} + + +/** + * Interface that VT-x uses to supply the PC of an exit when CS:RIP is being read. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uFlatPC The flattened program counter (RIP). + * @param fFlattened Set if RIP was subjected to CS.BASE, clear if not. + */ +VMM_INT_DECL(void) EMHistoryUpdatePC(PVMCPUCC pVCpu, uint64_t uFlatPC, bool fFlattened) +{ + VMCPU_ASSERT_EMT(pVCpu); + + AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256); + uint64_t uExitNo = pVCpu->em.s.iNextExit - 1; + PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff]; + pHistEntry->uFlatPC = uFlatPC; + if (fFlattened) + pHistEntry->uFlagsAndType &= ~EMEXIT_F_UNFLATTENED_PC; + else + pHistEntry->uFlagsAndType |= EMEXIT_F_UNFLATTENED_PC; +} + + +/** + * Interface for convering a engine specific exit to a generic one and get guidance. + * + * @returns Pointer to an exit record if special action should be taken using + * EMHistoryExec(). Take normal exit action when NULL. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uFlagsAndType Combined flags and type (see EMEXIT_MAKE_FLAGS_AND_TYPE). + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(PCEMEXITREC) EMHistoryUpdateFlagsAndType(PVMCPUCC pVCpu, uint32_t uFlagsAndType) +{ + VMCPU_ASSERT_EMT(pVCpu); + + /* + * Do the updating. + */ + AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256); + uint64_t uExitNo = pVCpu->em.s.iNextExit - 1; + PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff]; + pHistEntry->uFlagsAndType = uFlagsAndType | (pHistEntry->uFlagsAndType & (EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)); + + /* + * If common exit type, we will insert/update the exit into the exit record hash table. + */ + if ( (uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)) == EMEXIT_F_KIND_EM +#ifdef IN_RING0 + && pVCpu->em.s.fExitOptimizationEnabledR0 + && ( !(uFlagsAndType & EMEXIT_F_HM) || pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled) +#else + && pVCpu->em.s.fExitOptimizationEnabled +#endif + && pHistEntry->uFlatPC != UINT64_MAX + ) + return emHistoryAddOrUpdateRecord(pVCpu, uFlagsAndType, pHistEntry->uFlatPC, pHistEntry, uExitNo); + return NULL; +} + + +/** + * Interface for convering a engine specific exit to a generic one and get + * guidance, supplying flattened PC too. + * + * @returns Pointer to an exit record if special action should be taken using + * EMHistoryExec(). Take normal exit action when NULL. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uFlagsAndType Combined flags and type (see EMEXIT_MAKE_FLAGS_AND_TYPE). + * @param uFlatPC The flattened program counter (RIP). + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(PCEMEXITREC) EMHistoryUpdateFlagsAndTypeAndPC(PVMCPUCC pVCpu, uint32_t uFlagsAndType, uint64_t uFlatPC) +{ + VMCPU_ASSERT_EMT(pVCpu); + //Assert(uFlatPC != UINT64_MAX); - disable to make the pc wrapping tests in bs3-cpu-weird-1 work. + + /* + * Do the updating. + */ + AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256); + uint64_t uExitNo = pVCpu->em.s.iNextExit - 1; + PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff]; + pHistEntry->uFlagsAndType = uFlagsAndType; + pHistEntry->uFlatPC = uFlatPC; + + /* + * If common exit type, we will insert/update the exit into the exit record hash table. + */ + if ( (uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)) == EMEXIT_F_KIND_EM +#ifdef IN_RING0 + && pVCpu->em.s.fExitOptimizationEnabledR0 + && ( !(uFlagsAndType & EMEXIT_F_HM) || pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled) +#else + && pVCpu->em.s.fExitOptimizationEnabled +#endif + ) + return emHistoryAddOrUpdateRecord(pVCpu, uFlagsAndType, uFlatPC, pHistEntry, uExitNo); + return NULL; +} + + +/** + * @callback_method_impl{FNDISREADBYTES} + */ +static DECLCALLBACK(int) emReadBytes(PDISCPUSTATE pDis, uint8_t offInstr, uint8_t cbMinRead, uint8_t cbMaxRead) +{ + PVMCPUCC pVCpu = (PVMCPUCC)pDis->pvUser; + RTUINTPTR uSrcAddr = pDis->uInstrAddr + offInstr; + + /* + * Figure how much we can or must read. + */ + size_t cbToRead = GUEST_PAGE_SIZE - (uSrcAddr & (GUEST_PAGE_SIZE - 1)); + if (cbToRead > cbMaxRead) + cbToRead = cbMaxRead; + else if (cbToRead < cbMinRead) + cbToRead = cbMinRead; + + int rc = PGMPhysSimpleReadGCPtr(pVCpu, &pDis->abInstr[offInstr], uSrcAddr, cbToRead); + if (RT_FAILURE(rc)) + { + if (cbToRead > cbMinRead) + { + cbToRead = cbMinRead; + rc = PGMPhysSimpleReadGCPtr(pVCpu, &pDis->abInstr[offInstr], uSrcAddr, cbToRead); + } + if (RT_FAILURE(rc)) + { + /* + * If we fail to find the page via the guest's page tables + * we invalidate the page in the host TLB (pertaining to + * the guest in the NestedPaging case). See @bugref{6043}. + */ + if (rc == VERR_PAGE_TABLE_NOT_PRESENT || rc == VERR_PAGE_NOT_PRESENT) + { + HMInvalidatePage(pVCpu, uSrcAddr); + if (((uSrcAddr + cbToRead - 1) >> GUEST_PAGE_SHIFT) != (uSrcAddr >> GUEST_PAGE_SHIFT)) + HMInvalidatePage(pVCpu, uSrcAddr + cbToRead - 1); + } + } + } + + pDis->cbCachedInstr = offInstr + (uint8_t)cbToRead; + return rc; +} + + +/** + * Disassembles the current instruction. + * + * @returns VBox status code, see SELMToFlatEx and EMInterpretDisasOneEx for + * details. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pDis Where to return the parsed instruction info. + * @param pcbInstr Where to return the instruction size. (optional) + */ +VMM_INT_DECL(int) EMInterpretDisasCurrent(PVMCPUCC pVCpu, PDISCPUSTATE pDis, unsigned *pcbInstr) +{ + PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu); + RTGCPTR GCPtrInstr; +#if 0 + int rc = SELMToFlatEx(pVCpu, DISSELREG_CS, pCtx, pCtx->rip, 0, &GCPtrInstr); +#else +/** @todo Get the CPU mode as well while we're at it! */ + int rc = SELMValidateAndConvertCSAddr(pVCpu, pCtx->eflags.u, pCtx->ss.Sel, pCtx->cs.Sel, &pCtx->cs, pCtx->rip, &GCPtrInstr); +#endif + if (RT_SUCCESS(rc)) + return EMInterpretDisasOneEx(pVCpu, (RTGCUINTPTR)GCPtrInstr, pDis, pcbInstr); + + Log(("EMInterpretDisasOne: Failed to convert %RTsel:%RGv (cpl=%d) - rc=%Rrc !!\n", + pCtx->cs.Sel, (RTGCPTR)pCtx->rip, pCtx->ss.Sel & X86_SEL_RPL, rc)); + return rc; +} + + +/** + * Disassembles one instruction. + * + * This is used by internally by the interpreter and by trap/access handlers. + * + * @returns VBox status code. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtrInstr The flat address of the instruction. + * @param pDis Where to return the parsed instruction info. + * @param pcbInstr Where to return the instruction size. (optional) + */ +VMM_INT_DECL(int) EMInterpretDisasOneEx(PVMCPUCC pVCpu, RTGCUINTPTR GCPtrInstr, PDISCPUSTATE pDis, unsigned *pcbInstr) +{ + DISCPUMODE enmCpuMode = CPUMGetGuestDisMode(pVCpu); + /** @todo Deal with too long instruction (=> \#GP), opcode read errors (=> + * \#PF, \#GP, \#??), undefined opcodes (=> \#UD), and such. */ + int rc = DISInstrWithReader(GCPtrInstr, enmCpuMode, emReadBytes, pVCpu, pDis, pcbInstr); + if (RT_SUCCESS(rc)) + return VINF_SUCCESS; + AssertMsg(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT, ("DISCoreOne failed to GCPtrInstr=%RGv rc=%Rrc\n", GCPtrInstr, rc)); + return rc; +} + + +/** + * Interprets the current instruction. + * + * @returns VBox status code. + * @retval VINF_* Scheduling instructions. + * @retval VERR_EM_INTERPRETER Something we can't cope with. + * @retval VERR_* Fatal errors. + * + * @param pVCpu The cross context virtual CPU structure. + * + * @remark Invalid opcode exceptions have a higher priority than \#GP (see + * Intel Architecture System Developers Manual, Vol 3, 5.5) so we don't + * need to worry about e.g. invalid modrm combinations (!) + */ +VMM_INT_DECL(VBOXSTRICTRC) EMInterpretInstruction(PVMCPUCC pVCpu) +{ + LogFlow(("EMInterpretInstruction %RGv\n", (RTGCPTR)CPUMGetGuestRIP(pVCpu))); + + VBOXSTRICTRC rc = IEMExecOneBypassEx(pVCpu, NULL /*pcbWritten*/); + if (RT_UNLIKELY( rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED + || rc == VERR_IEM_INSTR_NOT_IMPLEMENTED)) + rc = VERR_EM_INTERPRETER; + if (rc != VINF_SUCCESS) + Log(("EMInterpretInstruction: returns %Rrc\n", VBOXSTRICTRC_VAL(rc))); + + return rc; +} + + +/** + * Interprets the current instruction using the supplied DISCPUSTATE structure. + * + * IP/EIP/RIP *IS* updated! + * + * @returns VBox strict status code. + * @retval VINF_* Scheduling instructions. When these are returned, it + * starts to get a bit tricky to know whether code was + * executed or not... We'll address this when it becomes a problem. + * @retval VERR_EM_INTERPRETER Something we can't cope with. + * @retval VERR_* Fatal errors. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pDis The disassembler cpu state for the instruction to be + * interpreted. + * @param rip The instruction pointer value. + * + * @remark Invalid opcode exceptions have a higher priority than GP (see Intel + * Architecture System Developers Manual, Vol 3, 5.5) so we don't need + * to worry about e.g. invalid modrm combinations (!) + * + * @todo At this time we do NOT check if the instruction overwrites vital information. + * Make sure this can't happen!! (will add some assertions/checks later) + */ +VMM_INT_DECL(VBOXSTRICTRC) EMInterpretInstructionDisasState(PVMCPUCC pVCpu, PDISCPUSTATE pDis, uint64_t rip) +{ + LogFlow(("EMInterpretInstructionDisasState %RGv\n", (RTGCPTR)rip)); + + VBOXSTRICTRC rc = IEMExecOneBypassWithPrefetchedByPC(pVCpu, rip, pDis->abInstr, pDis->cbCachedInstr); + if (RT_UNLIKELY( rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED + || rc == VERR_IEM_INSTR_NOT_IMPLEMENTED)) + rc = VERR_EM_INTERPRETER; + + if (rc != VINF_SUCCESS) + Log(("EMInterpretInstructionDisasState: returns %Rrc\n", VBOXSTRICTRC_VAL(rc))); + + return rc; +} + diff --git a/src/VBox/VMM/VMMAll/GCMAll.cpp b/src/VBox/VMM/VMMAll/GCMAll.cpp new file mode 100644 index 00000000..6773e8c3 --- /dev/null +++ b/src/VBox/VMM/VMMAll/GCMAll.cpp @@ -0,0 +1,245 @@ +/** @file + * GCM - Guest Compatibility Manager - All Contexts. + */ + +/* + * Copyright (C) 2022-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_GIM +#include +#include /* For EMInterpretDisasCurrent */ +#include "GCMInternal.h" +#include + +#include /* For DISCPUSTATE */ +#include +#include + + +/** + * Checks whether GCM is enabled for this VM. + * + * @retval true if GCM is on. + * @retval false if no GCM fixer is enabled. + * + * @param pVM The cross context VM structure. + */ +VMMDECL(bool) GCMIsEnabled(PVM pVM) +{ + return pVM->gcm.s.enmFixerIds != GCMFIXER_NONE; +} + + +/** + * Gets the GCM fixers configured for this VM. + * + * @returns The GCM provider Id. + * @param pVM The cross context VM structure. + */ +VMMDECL(int32_t) GCMGetFixers(PVM pVM) +{ + return pVM->gcm.s.enmFixerIds; +} + + +/** + * Whether \#DE exceptions in the guest should be intercepted by GCM and + * possibly fixed up. + * + * @returns true if needed, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) GCMShouldTrapXcptDE(PVMCPUCC pVCpu) +{ + LogFlowFunc(("entered\n")); + PVM pVM = pVCpu->CTX_SUFF(pVM); + if (!GCMIsEnabled(pVM)) + return false; + + LogFunc(("GCM checking if #DE needs trapping\n")); + + /* See if the enabled fixers need to intercept #DE. */ + if ( pVM->gcm.s.enmFixerIds + & (GCMFIXER_DBZ_DOS | GCMFIXER_DBZ_OS2 | GCMFIXER_DBZ_WIN9X)) + { + LogRel(("GCM: #DE should be trapped\n")); + return true; + } + + return false; +} + + +/** + * Exception handler for \#DE when registered by GCM. + * + * @returns Strict VBox status code. + * @retval VINF_SUCCESS retry division and continue. + * @retval VERR_NOT_FOUND deliver exception to guest. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * @param pDis Pointer to the disassembled instruction state at RIP. + * If NULL is passed, it implies the disassembly of the + * the instruction at RIP is the + * responsibility of GCM. + * @param pcbInstr Where to store the instruction length of + * the divide instruction. Optional, can be + * NULL. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(VBOXSTRICTRC) GCMXcptDE(PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + Assert(GCMIsEnabled(pVM)); + Assert(pDis || pcbInstr); + RT_NOREF(pDis); + RT_NOREF(pcbInstr); + + LogRel(("GCM: Intercepted #DE at CS:RIP=%04x:%RX64 (%RX64 linear) RDX:RAX=%RX64:%RX64 RCX=%RX64 RBX=%RX64\n", + pCtx->cs.Sel, pCtx->rip, pCtx->cs.u64Base + pCtx->rip, pCtx->rdx, pCtx->rax, pCtx->rcx, pCtx->rbx)); + + if (pVM->gcm.s.enmFixerIds & GCMFIXER_DBZ_OS2) + { + if (pCtx->rcx == 0 && pCtx->rdx == 1 && pCtx->rax == 0x86a0) + { + /* OS/2 1.x drivers loaded during boot: DX:AX = 100,000, CX < 2 causes overflow. */ + /* Example: OS/2 1.0 KBD01.SYS, 16,945 bytes, dated 10/21/1987, div cx at offset 2:2ffeh */ + /* Code later merged into BASEDD01.SYS, crash fixed in OS/2 1.30.1; this should + * fix all affected versions of OS/2 1.x. + */ + pCtx->rcx = 2; + return VINF_SUCCESS; + } + if ((uint16_t)pCtx->rbx == 0 && (uint16_t)pCtx->rdx == 0 && (uint16_t)pCtx->rax == 0x1000) + { + /* OS/2 2.1 and later boot loader: DX:AX = 0x1000, zero BX. May have junk in high words of all registers. */ + /* Example: OS/2 MCP2 OS2LDR, 44,544 bytes, dated 03/08/2002, idiv bx at offset 847ah */ + pCtx->rbx = (pCtx->rbx & ~0xffff) | 2; + return VINF_SUCCESS; + } + if (pCtx->rbx == 0 && pCtx->rdx == 0 && pCtx->rax == 0x100) + { + /* OS/2 2.0 boot loader: DX:AX = 0x100, zero BX. May have junk in high words of registers. */ + /* Example: OS/2 2.0 OS2LDR, 32,256 bytes, dated 03/30/1992, idiv bx at offset 2298h */ + pCtx->rbx = 2; + return VINF_SUCCESS; + } + } + + if (pVM->gcm.s.enmFixerIds & GCMFIXER_DBZ_DOS) + { + /* NB: For 16-bit DOS software, we must generally only compare 16-bit registers. + * The contents of the high words may be unpredictable depending on the environment. + * For 32-bit Windows 3.x code that is not the case. + */ + if (pCtx->rcx == 0 && pCtx->rdx == 0 && pCtx->rax == 0x100000) + { + /* NDIS.386 in WfW 3.11: CalibrateStall, EDX:EAX = 0x100000, zero ECX. + * Occurs when NDIS.386 loads. + */ + pCtx->rcx = 0x20000; /* Want a large divisor to shorten stalls. */ + return VINF_SUCCESS; + } + if (pCtx->rcx == 0 && pCtx->rdx == 0 && pCtx->rax > 0x100000) + { + /* NDIS.386 in WfW 3.11: NdisStallExecution, EDX:EAX = 0xYY00000, zero ECX. + * EDX:EAX is variable, but low 20 bits of EAX must be zero and EDX is likely + * to be zero as well. + * Only occurs if NdisStallExecution is called to do a longish stall. + */ + pCtx->rcx = 22; + return VINF_SUCCESS; + } + if ((uint16_t)pCtx->rbx == 0 && (uint16_t)pCtx->rdx == 0 && (uint16_t)pCtx->rax == 0x64) + { + /* Norton Sysinfo or Diagnostics 8.0 DX:AX = 0x64 (100 decimal), zero BX. */ + pCtx->rbx = (pCtx->rbx & 0xffff0000) | 1; /* BX = 1 */ + return VINF_SUCCESS; + } + if ((uint16_t)pCtx->rbx == 0 && (uint16_t)pCtx->rdx == 0 && (uint16_t)pCtx->rax == 0xff) + { + /* IBM PC LAN Program 1.3: DX:AX=0xff (255 decimal), zero BX. */ + /* NETWORK1.CMD, 64,324 bytes, dated 06/06/1988, div bx at offset 0xa400 in file. */ + pCtx->rbx = (pCtx->rbx & 0xffff0000) | 1; /* BX = 1 */ + return VINF_SUCCESS; + } + if ((uint16_t)pCtx->rdx == 0xffff && (uint16_t)pCtx->rax == 0xffff && (uint16_t)pCtx->rcx == 0xa8c0) + { + /* QNX 2.15C: DX:AX=0xffffffff (-1), constant CX = 0xa8c0 (43200). */ + /* div cx at e.g. 2220:fa5 and 2220:10a0 in memory. */ + pCtx->rdx = (pCtx->rdx & 0xffff0000) | 8; /* DX = 8 */ + return VINF_SUCCESS; + } + if ((uint16_t)pCtx->rax > 0x1800 && ((uint16_t)pCtx->rax & 0x3f) == 0 && (uint16_t)pCtx->rbx == 0x19) + { + /* 3C501.COM ODI driver v1.21: AX > ~0x1900 (-1), BX = 0x19 (25). */ + /* AX was shifted left by 6 bits so low bits must be zero. */ + /* div bl at e.g. 06b3:2f80 and offset 0x2E80 in file. */ + pCtx->rax = (pCtx->rax & 0xffff0000) | 0x8c0; /* AX = 0x8c0 */ + return VINF_SUCCESS; + } + if ((uint16_t)pCtx->rcx == 0x37 && ((uint16_t)pCtx->rdx > 0x34)) + { + /* Turbo Pascal, classic Runtime Error 200: CX = 55, DX > ~54, AX/BX variable. */ + /* div cx at variable offset in file. */ + pCtx->rdx = (pCtx->rdx & 0xffff0000) | 0x30; /* DX = 48 */ + return VINF_SUCCESS; + } + } + + if (pVM->gcm.s.enmFixerIds & GCMFIXER_DBZ_WIN9X) + { + if (pCtx->rcx == 0 && pCtx->rdx == 0 && pCtx->rax == 0x100000) + { + /* NDIS.VXD in Win9x: EDX:EAX = 0x100000, zero ECX. */ + /* Example: Windows 95 NDIS.VXD, 99,084 bytes, dated 07/11/1994, div ecx at 28:Cxxxx80B */ + /* Crash fixed in Windows 98 SE. */ + pCtx->rcx = 0x20000; /* Want a large divisor to shorten stalls. */ + return VINF_SUCCESS; + } + if (pCtx->rcx < 3 && pCtx->rdx == 2 && pCtx->rax == 0x540be400) + { + /* SCSI.PDR, ESDI506.PDR in Win95: EDX:EAX = 0x2540be400 (10,000,000,000 decimal), ECX < 3. */ + /* Example: Windows 95 SCSIPORT.PDR, 23,133 bytes, dated 07/11/1995, div ecx at 28:Cxxxx876 */ + /* Example: Win95 OSR2 ESDI506.PDR, 24,390 bytes, dated 04/24/1996, div ecx at 28:Cxxxx8E3 */ + /* Crash fixed in Windows 98. */ + pCtx->rcx = 1000; + return VINF_SUCCESS; + } + if (pCtx->rcx == 0 && pCtx->rdx == 0x3d && pCtx->rax == 0x9000000) + { + /* Unknown source, Win9x shutdown, div ecx. */ + /* GCM: Intercepted #DE at CS:RIP=0028:c0050f8e RDX:RAX=3d:9000000 (250000*1024*1024) RCX=0 RBX=c19200e8 [RBX variable] */ + pCtx->rcx = 4096; + return VINF_SUCCESS; + } + } + + /* If we got this far, deliver exception to guest. */ + return VERR_NOT_FOUND; +} diff --git a/src/VBox/VMM/VMMAll/GIMAll.cpp b/src/VBox/VMM/VMMAll/GIMAll.cpp new file mode 100644 index 00000000..470e653e --- /dev/null +++ b/src/VBox/VMM/VMMAll/GIMAll.cpp @@ -0,0 +1,506 @@ +/* $Id: GIMAll.cpp $ */ +/** @file + * GIM - Guest Interface Manager - All Contexts. + */ + +/* + * Copyright (C) 2014-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_GIM +#include +#include /* For EMInterpretDisasCurrent */ +#include "GIMInternal.h" +#include + +#include /* For DISCPUSTATE */ +#include +#include + +/* Include all the providers. */ +#include "GIMHvInternal.h" +#include "GIMMinimalInternal.h" + + +/** + * Checks whether GIM is being used by this VM. + * + * @retval true if used. + * @retval false if no GIM provider ("none") is used. + * + * @param pVM The cross context VM structure. + */ +VMMDECL(bool) GIMIsEnabled(PVM pVM) +{ + return pVM->gim.s.enmProviderId != GIMPROVIDERID_NONE; +} + + +/** + * Gets the GIM provider configured for this VM. + * + * @returns The GIM provider Id. + * @param pVM The cross context VM structure. + */ +VMMDECL(GIMPROVIDERID) GIMGetProvider(PVM pVM) +{ + return pVM->gim.s.enmProviderId; +} + + +/** + * Returns the array of MMIO2 regions that are expected to be registered and + * later mapped into the guest-physical address space for the GIM provider + * configured for the VM. + * + * @returns Pointer to an array of GIM MMIO2 regions, may return NULL. + * @param pVM The cross context VM structure. + * @param pcRegions Where to store the number of items in the array. + * + * @remarks The caller does not own and therefore must -NOT- try to free the + * returned pointer. + */ +VMMDECL(PGIMMMIO2REGION) GIMGetMmio2Regions(PVMCC pVM, uint32_t *pcRegions) +{ + Assert(pVM); + Assert(pcRegions); + + *pcRegions = 0; + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_HYPERV: + return gimHvGetMmio2Regions(pVM, pcRegions); + + default: + break; + } + + return NULL; +} + + +/** + * Returns whether the guest has configured and enabled calls to the hypervisor. + * + * @returns true if hypercalls are enabled and usable, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) GIMAreHypercallsEnabled(PVMCPUCC pVCpu) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + if (!GIMIsEnabled(pVM)) + return false; + + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_HYPERV: + return gimHvAreHypercallsEnabled(pVM); + + case GIMPROVIDERID_KVM: + return gimKvmAreHypercallsEnabled(pVCpu); + + default: + return false; + } +} + + +/** + * Implements a GIM hypercall with the provider configured for the VM. + * + * @returns Strict VBox status code. + * @retval VINF_SUCCESS if the hypercall succeeded (even if its operation + * failed). + * @retval VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating + * RIP. + * @retval VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3. + * @retval VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient. + * @retval VERR_GIM_HYPERCALLS_NOT_AVAILABLE hypercalls unavailable. + * @retval VERR_GIM_NOT_ENABLED GIM is not enabled (shouldn't really happen) + * @retval VERR_GIM_HYPERCALL_MEMORY_READ_FAILED hypercall failed while reading + * memory. + * @retval VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED hypercall failed while + * writing memory. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * + * @remarks The caller of this function needs to advance RIP as required. + * @thread EMT. + */ +VMM_INT_DECL(VBOXSTRICTRC) GIMHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPU_ASSERT_EMT(pVCpu); + + if (RT_UNLIKELY(!GIMIsEnabled(pVM))) + return VERR_GIM_NOT_ENABLED; + + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_HYPERV: + return gimHvHypercall(pVCpu, pCtx); + + case GIMPROVIDERID_KVM: + return gimKvmHypercall(pVCpu, pCtx); + + default: + AssertMsgFailed(("GIMHypercall: for provider %u not available/implemented\n", pVM->gim.s.enmProviderId)); + return VERR_GIM_HYPERCALLS_NOT_AVAILABLE; + } +} + + +/** + * Same as GIMHypercall, except with disassembler opcode and instruction length. + * + * This is the interface used by IEM. + * + * @returns Strict VBox status code. + * @retval VINF_SUCCESS if the hypercall succeeded (even if its operation + * failed). + * @retval VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating + * RIP. + * @retval VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3. + * @retval VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient. + * @retval VERR_GIM_HYPERCALLS_NOT_AVAILABLE hypercalls unavailable. + * @retval VERR_GIM_NOT_ENABLED GIM is not enabled (shouldn't really happen) + * @retval VERR_GIM_HYPERCALL_MEMORY_READ_FAILED hypercall failed while reading + * memory. + * @retval VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED hypercall failed while + * writing memory. + * @retval VERR_GIM_INVALID_HYPERCALL_INSTR if uDisOpcode is the wrong one; raise \#UD. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * @param uDisOpcode The disassembler opcode. + * @param cbInstr The instruction length. + * + * @remarks The caller of this function needs to advance RIP as required. + * @thread EMT. + */ +VMM_INT_DECL(VBOXSTRICTRC) GIMHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPU_ASSERT_EMT(pVCpu); + + if (RT_UNLIKELY(!GIMIsEnabled(pVM))) + return VERR_GIM_NOT_ENABLED; + + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_HYPERV: + return gimHvHypercallEx(pVCpu, pCtx, uDisOpcode, cbInstr); + + case GIMPROVIDERID_KVM: + return gimKvmHypercallEx(pVCpu, pCtx, uDisOpcode, cbInstr); + + default: + AssertMsgFailedReturn(("enmProviderId=%u\n", pVM->gim.s.enmProviderId), VERR_GIM_HYPERCALLS_NOT_AVAILABLE); + } +} + + +/** + * Disassembles the instruction at RIP and if it's a hypercall + * instruction, performs the hypercall. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * @param pcbInstr Where to store the disassembled instruction length. + * Optional, can be NULL. + * + * @todo This interface should disappear when IEM/REM execution engines + * handle VMCALL/VMMCALL instructions to call into GIM when + * required. See @bugref{7270#c168}. + */ +VMM_INT_DECL(VBOXSTRICTRC) GIMExecHypercallInstr(PVMCPUCC pVCpu, PCPUMCTX pCtx, uint8_t *pcbInstr) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPU_ASSERT_EMT(pVCpu); + + if (RT_UNLIKELY(!GIMIsEnabled(pVM))) + return VERR_GIM_NOT_ENABLED; + + unsigned cbInstr; + DISCPUSTATE Dis; + int rc = EMInterpretDisasCurrent(pVCpu, &Dis, &cbInstr); + if (RT_SUCCESS(rc)) + { + if (pcbInstr) + *pcbInstr = (uint8_t)cbInstr; + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_HYPERV: + return gimHvHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr); + + case GIMPROVIDERID_KVM: + return gimKvmHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr); + + default: + AssertMsgFailed(("GIMExecHypercallInstr: for provider %u not available/implemented\n", pVM->gim.s.enmProviderId)); + return VERR_GIM_HYPERCALLS_NOT_AVAILABLE; + } + } + + Log(("GIM: GIMExecHypercallInstr: Failed to disassemble CS:RIP=%04x:%08RX64. rc=%Rrc\n", pCtx->cs.Sel, pCtx->rip, rc)); + return rc; +} + + +/** + * Returns whether the guest has configured and setup the use of paravirtualized + * TSC. + * + * Paravirtualized TSCs are per-VM and the rest of the execution engine logic + * relies on that. + * + * @returns true if enabled and usable, false otherwise. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) GIMIsParavirtTscEnabled(PVMCC pVM) +{ + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_HYPERV: + return gimHvIsParavirtTscEnabled(pVM); + + case GIMPROVIDERID_KVM: + return gimKvmIsParavirtTscEnabled(pVM); + + default: + break; + } + return false; +} + + +/** + * Whether \#UD exceptions in the guest needs to be intercepted by the GIM + * provider. + * + * At the moment, the reason why this isn't a more generic interface wrt to + * exceptions is because of performance (each VM-exit would have to manually + * check whether or not GIM needs to be notified). Left as a todo for later if + * really required. + * + * @returns true if needed, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) GIMShouldTrapXcptUD(PVMCPUCC pVCpu) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + if (!GIMIsEnabled(pVM)) + return false; + + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_KVM: + return gimKvmShouldTrapXcptUD(pVM); + + case GIMPROVIDERID_HYPERV: + return gimHvShouldTrapXcptUD(pVCpu); + + default: + return false; + } +} + + +/** + * Exception handler for \#UD when requested by the GIM provider. + * + * @returns Strict VBox status code. + * @retval VINF_SUCCESS if the hypercall succeeded (even if its operation + * failed). + * @retval VINF_GIM_R3_HYPERCALL restart the hypercall from ring-3. + * @retval VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating + * RIP. + * @retval VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient. + * @retval VERR_GIM_INVALID_HYPERCALL_INSTR instruction at RIP is not a valid + * hypercall instruction. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * @param pDis Pointer to the disassembled instruction state at RIP. + * If NULL is passed, it implies the disassembly of the + * the instruction at RIP is the responsibility of the + * GIM provider. + * @param pcbInstr Where to store the instruction length of the hypercall + * instruction. Optional, can be NULL. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(VBOXSTRICTRC) GIMXcptUD(PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + Assert(GIMIsEnabled(pVM)); + Assert(pDis || pcbInstr); + + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_KVM: + return gimKvmXcptUD(pVM, pVCpu, pCtx, pDis, pcbInstr); + + case GIMPROVIDERID_HYPERV: + return gimHvXcptUD(pVCpu, pCtx, pDis, pcbInstr); + + default: + return VERR_GIM_OPERATION_FAILED; + } +} + + +/** + * Invokes the read-MSR handler for the GIM provider configured for the VM. + * + * @returns Strict VBox status code like CPUMQueryGuestMsr. + * @retval VINF_CPUM_R3_MSR_READ + * @retval VERR_CPUM_RAISE_GP_0 + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR to read. + * @param pRange The range this MSR belongs to. + * @param puValue Where to store the MSR value read. + */ +VMM_INT_DECL(VBOXSTRICTRC) GIMReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + Assert(pVCpu); + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + Assert(GIMIsEnabled(pVM)); + VMCPU_ASSERT_EMT(pVCpu); + + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_HYPERV: + return gimHvReadMsr(pVCpu, idMsr, pRange, puValue); + + case GIMPROVIDERID_KVM: + return gimKvmReadMsr(pVCpu, idMsr, pRange, puValue); + + default: + AssertMsgFailed(("GIMReadMsr: for unknown provider %u idMsr=%#RX32 -> #GP(0)", pVM->gim.s.enmProviderId, idMsr)); + return VERR_CPUM_RAISE_GP_0; + } +} + + +/** + * Invokes the write-MSR handler for the GIM provider configured for the VM. + * + * @returns Strict VBox status code like CPUMSetGuestMsr. + * @retval VINF_CPUM_R3_MSR_WRITE + * @retval VERR_CPUM_RAISE_GP_0 + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR to write. + * @param pRange The range this MSR belongs to. + * @param uValue The value to set, ignored bits masked. + * @param uRawValue The raw value with the ignored bits not masked. + */ +VMM_INT_DECL(VBOXSTRICTRC) GIMWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue) +{ + AssertPtr(pVCpu); + NOREF(uValue); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + Assert(GIMIsEnabled(pVM)); + VMCPU_ASSERT_EMT(pVCpu); + + switch (pVM->gim.s.enmProviderId) + { + case GIMPROVIDERID_HYPERV: + return gimHvWriteMsr(pVCpu, idMsr, pRange, uRawValue); + + case GIMPROVIDERID_KVM: + return gimKvmWriteMsr(pVCpu, idMsr, pRange, uRawValue); + + default: + AssertMsgFailed(("GIMWriteMsr: for unknown provider %u idMsr=%#RX32 -> #GP(0)", pVM->gim.s.enmProviderId, idMsr)); + return VERR_CPUM_RAISE_GP_0; + } +} + + +/** + * Queries the opcode bytes for a native hypercall. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pvBuf The destination buffer. + * @param cbBuf The size of the buffer. + * @param pcbWritten Where to return the number of bytes written. This is + * reliably updated only on successful return. Optional. + * @param puDisOpcode Where to return the disassembler opcode. Optional. + */ +VMM_INT_DECL(int) GIMQueryHypercallOpcodeBytes(PVM pVM, void *pvBuf, size_t cbBuf, size_t *pcbWritten, uint16_t *puDisOpcode) +{ + AssertPtrReturn(pvBuf, VERR_INVALID_POINTER); + +#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64) + CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM); +#else + CPUMCPUVENDOR enmCpuVendor = CPUMGetGuestCpuVendor(pVM); /* Use what is presented to the guest. */ +#endif + uint8_t const *pbSrc; + size_t cbSrc; + switch (enmCpuVendor) + { + case CPUMCPUVENDOR_AMD: + case CPUMCPUVENDOR_HYGON: + { + if (puDisOpcode) + *puDisOpcode = OP_VMMCALL; + static uint8_t const s_abHypercall[] = { 0x0F, 0x01, 0xD9 }; /* VMMCALL */ + pbSrc = s_abHypercall; + cbSrc = sizeof(s_abHypercall); + break; + } + + case CPUMCPUVENDOR_INTEL: + case CPUMCPUVENDOR_VIA: + case CPUMCPUVENDOR_SHANGHAI: + { + if (puDisOpcode) + *puDisOpcode = OP_VMCALL; + static uint8_t const s_abHypercall[] = { 0x0F, 0x01, 0xC1 }; /* VMCALL */ + pbSrc = s_abHypercall; + cbSrc = sizeof(s_abHypercall); + break; + } + + default: + AssertMsgFailedReturn(("%d\n", enmCpuVendor), VERR_UNSUPPORTED_CPU); + } + if (RT_LIKELY(cbBuf >= cbSrc)) + { + memcpy(pvBuf, pbSrc, cbSrc); + if (pcbWritten) + *pcbWritten = cbSrc; + return VINF_SUCCESS; + } + return VERR_BUFFER_OVERFLOW; +} + diff --git a/src/VBox/VMM/VMMAll/GIMAllHv.cpp b/src/VBox/VMM/VMMAll/GIMAllHv.cpp new file mode 100644 index 00000000..2f71989a --- /dev/null +++ b/src/VBox/VMM/VMMAll/GIMAllHv.cpp @@ -0,0 +1,1495 @@ +/* $Id: GIMAllHv.cpp $ */ +/** @file + * GIM - Guest Interface Manager, Microsoft Hyper-V, All Contexts. + */ + +/* + * Copyright (C) 2014-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_GIM +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "GIMHvInternal.h" +#include "GIMInternal.h" +#include + +#include + +#ifdef IN_RING3 +# include +#endif + + +#ifdef IN_RING3 +/** + * Read and validate slow hypercall parameters. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pCtx Pointer to the guest-CPU context. + * @param fIs64BitMode Whether the guest is currently in 64-bit mode or not. + * @param enmParam The hypercall parameter type. + * @param prcHv Where to store the Hyper-V status code. Only valid + * to the caller when this function returns + * VINF_SUCCESS. + */ +static int gimHvReadSlowHypercallParam(PVM pVM, PCPUMCTX pCtx, bool fIs64BitMode, GIMHVHYPERCALLPARAM enmParam, int *prcHv) +{ + int rc = VINF_SUCCESS; + PGIMHV pHv = &pVM->gim.s.u.Hv; + RTGCPHYS GCPhysParam; + void *pvDst; + if (enmParam == GIMHVHYPERCALLPARAM_IN) + { + GCPhysParam = fIs64BitMode ? pCtx->rdx : (pCtx->rbx << 32) | pCtx->ecx; + pvDst = pHv->pbHypercallIn; + pHv->GCPhysHypercallIn = GCPhysParam; + } + else + { + GCPhysParam = fIs64BitMode ? pCtx->r8 : (pCtx->rdi << 32) | pCtx->esi; + pvDst = pHv->pbHypercallOut; + pHv->GCPhysHypercallOut = GCPhysParam; + Assert(enmParam == GIMHVHYPERCALLPARAM_OUT); + } + + const char *pcszParam = enmParam == GIMHVHYPERCALLPARAM_IN ? "input" : "output"; NOREF(pcszParam); + if (RT_ALIGN_64(GCPhysParam, 8) == GCPhysParam) + { + if (PGMPhysIsGCPhysNormal(pVM, GCPhysParam)) + { + rc = PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysParam, GIM_HV_PAGE_SIZE); + if (RT_SUCCESS(rc)) + { + *prcHv = GIM_HV_STATUS_SUCCESS; + return VINF_SUCCESS; + } + LogRel(("GIM: HyperV: Failed reading %s param at %#RGp. rc=%Rrc\n", pcszParam, GCPhysParam, rc)); + rc = VERR_GIM_HYPERCALL_MEMORY_READ_FAILED; + } + else + { + Log(("GIM: HyperV: Invalid %s param address %#RGp\n", pcszParam, GCPhysParam)); + *prcHv = GIM_HV_STATUS_INVALID_PARAMETER; + } + } + else + { + Log(("GIM: HyperV: Misaligned %s param address %#RGp\n", pcszParam, GCPhysParam)); + *prcHv = GIM_HV_STATUS_INVALID_ALIGNMENT; + } + return rc; +} + + +/** + * Helper for reading and validating slow hypercall input and output parameters. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pCtx Pointer to the guest-CPU context. + * @param fIs64BitMode Whether the guest is currently in 64-bit mode or not. + * @param prcHv Where to store the Hyper-V status code. Only valid + * to the caller when this function returns + * VINF_SUCCESS. + */ +static int gimHvReadSlowHypercallParamsInOut(PVM pVM, PCPUMCTX pCtx, bool fIs64BitMode, int *prcHv) +{ + int rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_IN, prcHv); + if ( RT_SUCCESS(rc) + && *prcHv == GIM_HV_STATUS_SUCCESS) + rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_OUT, prcHv); + return rc; +} +#endif + + +/** + * Handles all Hyper-V hypercalls. + * + * @returns Strict VBox status code. + * @retval VINF_SUCCESS if the hypercall succeeded (even if its operation + * failed). + * @retval VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3. + * @retval VERR_GIM_HYPERCALLS_NOT_ENABLED hypercalls are disabled by the + * guest. + * @retval VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient. + * @retval VERR_GIM_HYPERCALL_MEMORY_READ_FAILED hypercall failed while reading + * memory. + * @retval VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED hypercall failed while + * writing memory. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(VBOXSTRICTRC) gimHvHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx) +{ + VMCPU_ASSERT_EMT(pVCpu); + +#ifndef IN_RING3 + RT_NOREF_PV(pVCpu); + RT_NOREF_PV(pCtx); + return VINF_GIM_R3_HYPERCALL; +#else + PVM pVM = pVCpu->CTX_SUFF(pVM); + STAM_REL_COUNTER_INC(&pVM->gim.s.StatHypercalls); + + /* + * Verify that hypercalls are enabled by the guest. + */ + if (!gimHvAreHypercallsEnabled(pVM)) + return VERR_GIM_HYPERCALLS_NOT_ENABLED; + + /* + * Verify guest is in ring-0 protected mode. + */ + uint32_t uCpl = CPUMGetGuestCPL(pVCpu); + if ( uCpl + || CPUMIsGuestInRealModeEx(pCtx)) + { + return VERR_GIM_HYPERCALL_ACCESS_DENIED; + } + + /* + * Get the hypercall operation code and modes. + * Fast hypercalls have only two or fewer inputs but no output parameters. + */ + const bool fIs64BitMode = CPUMIsGuestIn64BitCodeEx(pCtx); + const uint64_t uHyperIn = fIs64BitMode ? pCtx->rcx : (pCtx->rdx << 32) | pCtx->eax; + const uint16_t uHyperOp = GIM_HV_HYPERCALL_IN_CALL_CODE(uHyperIn); + const bool fHyperFast = GIM_HV_HYPERCALL_IN_IS_FAST(uHyperIn); + const uint16_t cHyperReps = GIM_HV_HYPERCALL_IN_REP_COUNT(uHyperIn); + const uint16_t idxHyperRepStart = GIM_HV_HYPERCALL_IN_REP_START_IDX(uHyperIn); + uint64_t cHyperRepsDone = 0; + + /* Currently no repeating hypercalls are supported. */ + RT_NOREF2(cHyperReps, idxHyperRepStart); + + int rc = VINF_SUCCESS; + int rcHv = GIM_HV_STATUS_OPERATION_DENIED; + PGIMHV pHv = &pVM->gim.s.u.Hv; + + /* + * Validate common hypercall input parameters. + */ + if ( !GIM_HV_HYPERCALL_IN_RSVD_1(uHyperIn) + && !GIM_HV_HYPERCALL_IN_RSVD_2(uHyperIn) + && !GIM_HV_HYPERCALL_IN_RSVD_3(uHyperIn)) + { + /* + * Perform the hypercall. + */ + switch (uHyperOp) + { + case GIM_HV_HYPERCALL_OP_RETREIVE_DEBUG_DATA: /* Non-rep, memory IO. */ + { + if (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING) + { + rc = gimHvReadSlowHypercallParamsInOut(pVM, pCtx, fIs64BitMode, &rcHv); + if ( RT_SUCCESS(rc) + && rcHv == GIM_HV_STATUS_SUCCESS) + { + LogRelMax(1, ("GIM: HyperV: Initiated debug data reception via hypercall\n")); + rc = gimR3HvHypercallRetrieveDebugData(pVM, &rcHv); + if (RT_FAILURE(rc)) + LogRelMax(10, ("GIM: HyperV: gimR3HvHypercallRetrieveDebugData failed. rc=%Rrc\n", rc)); + } + } + else + rcHv = GIM_HV_STATUS_ACCESS_DENIED; + break; + } + + case GIM_HV_HYPERCALL_OP_POST_DEBUG_DATA: /* Non-rep, memory IO. */ + { + if (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING) + { + rc = gimHvReadSlowHypercallParamsInOut(pVM, pCtx, fIs64BitMode, &rcHv); + if ( RT_SUCCESS(rc) + && rcHv == GIM_HV_STATUS_SUCCESS) + { + LogRelMax(1, ("GIM: HyperV: Initiated debug data transmission via hypercall\n")); + rc = gimR3HvHypercallPostDebugData(pVM, &rcHv); + if (RT_FAILURE(rc)) + LogRelMax(10, ("GIM: HyperV: gimR3HvHypercallPostDebugData failed. rc=%Rrc\n", rc)); + } + } + else + rcHv = GIM_HV_STATUS_ACCESS_DENIED; + break; + } + + case GIM_HV_HYPERCALL_OP_RESET_DEBUG_SESSION: /* Non-rep, fast (register IO). */ + { + if (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING) + { + uint32_t fFlags = 0; + if (!fHyperFast) + { + rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_IN, &rcHv); + if ( RT_SUCCESS(rc) + && rcHv == GIM_HV_STATUS_SUCCESS) + { + PGIMHVDEBUGRESETIN pIn = (PGIMHVDEBUGRESETIN)pHv->pbHypercallIn; + fFlags = pIn->fFlags; + } + } + else + { + rcHv = GIM_HV_STATUS_SUCCESS; + fFlags = fIs64BitMode ? pCtx->rdx : pCtx->ebx; + } + + /* + * Nothing to flush on the sending side as we don't maintain our own buffers. + */ + /** @todo We should probably ask the debug receive thread to flush it's buffer. */ + if (rcHv == GIM_HV_STATUS_SUCCESS) + { + if (fFlags) + LogRel(("GIM: HyperV: Resetting debug session via hypercall\n")); + else + rcHv = GIM_HV_STATUS_INVALID_PARAMETER; + } + } + else + rcHv = GIM_HV_STATUS_ACCESS_DENIED; + break; + } + + case GIM_HV_HYPERCALL_OP_POST_MESSAGE: /* Non-rep, memory IO. */ + { + if (pHv->fIsInterfaceVs) + { + rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_IN, &rcHv); + if ( RT_SUCCESS(rc) + && rcHv == GIM_HV_STATUS_SUCCESS) + { + PGIMHVPOSTMESSAGEIN pMsgIn = (PGIMHVPOSTMESSAGEIN)pHv->pbHypercallIn; + PCGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + if ( pMsgIn->uConnectionId == GIM_HV_VMBUS_MSG_CONNECTION_ID + && pMsgIn->enmMessageType == GIMHVMSGTYPE_VMBUS + && !MSR_GIM_HV_SINT_IS_MASKED(pHvCpu->auSintMsrs[GIM_HV_VMBUS_MSG_SINT]) + && MSR_GIM_HV_SIMP_IS_ENABLED(pHvCpu->uSimpMsr)) + { + RTGCPHYS GCPhysSimp = MSR_GIM_HV_SIMP_GPA(pHvCpu->uSimpMsr); + if (PGMPhysIsGCPhysNormal(pVM, GCPhysSimp)) + { + /* + * The VMBus client (guest) expects to see 0xf at offsets 4 and 16 and 1 at offset 0. + */ + GIMHVMSG HvMsg; + RT_ZERO(HvMsg); + HvMsg.MsgHdr.enmMessageType = GIMHVMSGTYPE_VMBUS; + HvMsg.MsgHdr.cbPayload = 0xf; + HvMsg.aPayload[0] = 0xf; + uint16_t const offMsg = GIM_HV_VMBUS_MSG_SINT * sizeof(GIMHVMSG); + int rc2 = PGMPhysSimpleWriteGCPhys(pVM, GCPhysSimp + offMsg, &HvMsg, sizeof(HvMsg)); + if (RT_SUCCESS(rc2)) + LogRel(("GIM: HyperV: SIMP hypercall faking message at %#RGp:%u\n", GCPhysSimp, offMsg)); + else + { + LogRel(("GIM: HyperV: Failed to write SIMP message at %#RGp:%u, rc=%Rrc\n", GCPhysSimp, + offMsg, rc)); + } + } + } + + /* + * Make the call fail after updating the SIMP, so the guest can go back to using + * the Hyper-V debug MSR interface. Any error code below GIM_HV_STATUS_NOT_ACKNOWLEDGED + * and the guest tries to proceed with initializing VMBus which is totally unnecessary + * for what we're trying to accomplish, i.e. convince guest to use Hyper-V debugging. Also, + * we don't implement other VMBus/SynIC functionality so the guest would #GP and die. + */ + rcHv = GIM_HV_STATUS_NOT_ACKNOWLEDGED; + } + else + rcHv = GIM_HV_STATUS_INVALID_PARAMETER; + } + else + rcHv = GIM_HV_STATUS_ACCESS_DENIED; + break; + } + + case GIM_HV_EXT_HYPERCALL_OP_QUERY_CAP: /* Non-rep, extended hypercall. */ + { + if (pHv->uPartFlags & GIM_HV_PART_FLAGS_EXTENDED_HYPERCALLS) + { + rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_OUT, &rcHv); + if ( RT_SUCCESS(rc) + && rcHv == GIM_HV_STATUS_SUCCESS) + { + rc = gimR3HvHypercallExtQueryCap(pVM, &rcHv); + } + } + else + { + LogRel(("GIM: HyperV: Denied HvExtCallQueryCapabilities when the feature is not exposed\n")); + rcHv = GIM_HV_STATUS_ACCESS_DENIED; + } + break; + } + + case GIM_HV_EXT_HYPERCALL_OP_GET_BOOT_ZEROED_MEM: /* Non-rep, extended hypercall. */ + { + if (pHv->uPartFlags & GIM_HV_PART_FLAGS_EXTENDED_HYPERCALLS) + { + rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_OUT, &rcHv); + if ( RT_SUCCESS(rc) + && rcHv == GIM_HV_STATUS_SUCCESS) + { + rc = gimR3HvHypercallExtGetBootZeroedMem(pVM, &rcHv); + } + } + else + { + LogRel(("GIM: HyperV: Denied HvExtCallGetBootZeroedMemory when the feature is not exposed\n")); + rcHv = GIM_HV_STATUS_ACCESS_DENIED; + } + break; + } + + default: + { + LogRel(("GIM: HyperV: Unknown/invalid hypercall opcode %#x (%u)\n", uHyperOp, uHyperOp)); + rcHv = GIM_HV_STATUS_INVALID_HYPERCALL_CODE; + break; + } + } + } + else + rcHv = GIM_HV_STATUS_INVALID_HYPERCALL_INPUT; + + /* + * Update the guest with results of the hypercall. + */ + if (RT_SUCCESS(rc)) + { + if (fIs64BitMode) + pCtx->rax = (cHyperRepsDone << 32) | rcHv; + else + { + pCtx->edx = cHyperRepsDone; + pCtx->eax = rcHv; + } + } + + return rc; +#endif +} + + +/** + * Returns a pointer to the MMIO2 regions supported by Hyper-V. + * + * @returns Pointer to an array of MMIO2 regions. + * @param pVM The cross context VM structure. + * @param pcRegions Where to store the number of regions in the array. + */ +VMM_INT_DECL(PGIMMMIO2REGION) gimHvGetMmio2Regions(PVM pVM, uint32_t *pcRegions) +{ + Assert(GIMIsEnabled(pVM)); + PGIMHV pHv = &pVM->gim.s.u.Hv; + + AssertCompile(RT_ELEMENTS(pHv->aMmio2Regions) <= 8); + *pcRegions = RT_ELEMENTS(pHv->aMmio2Regions); + return pHv->aMmio2Regions; +} + + +/** + * Returns whether the guest has configured and enabled the use of Hyper-V's + * hypercall interface. + * + * @returns true if hypercalls are enabled, false otherwise. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) gimHvAreHypercallsEnabled(PCVM pVM) +{ + return RT_BOOL(pVM->gim.s.u.Hv.u64GuestOsIdMsr != 0); +} + + +/** + * Returns whether the guest has configured and enabled the use of Hyper-V's + * paravirtualized TSC. + * + * @returns true if paravirt. TSC is enabled, false otherwise. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) gimHvIsParavirtTscEnabled(PVM pVM) +{ + return MSR_GIM_HV_REF_TSC_IS_ENABLED(pVM->gim.s.u.Hv.u64TscPageMsr); +} + + +#ifdef IN_RING3 +/** + * Gets the descriptive OS ID variant as identified via the + * MSR_GIM_HV_GUEST_OS_ID MSR. + * + * @returns The name. + * @param uGuestOsIdMsr The MSR_GIM_HV_GUEST_OS_ID MSR. + */ +static const char *gimHvGetGuestOsIdVariantName(uint64_t uGuestOsIdMsr) +{ + /* Refer the Hyper-V spec, section 3.6 "Reporting the Guest OS Identity". */ + uint32_t uVendor = MSR_GIM_HV_GUEST_OS_ID_VENDOR(uGuestOsIdMsr); + if (uVendor == 1 /* Microsoft */) + { + uint32_t uOsVariant = MSR_GIM_HV_GUEST_OS_ID_OS_VARIANT(uGuestOsIdMsr); + switch (uOsVariant) + { + case 0: return "Undefined"; + case 1: return "MS-DOS"; + case 2: return "Windows 3.x"; + case 3: return "Windows 9x"; + case 4: return "Windows NT or derivative"; + case 5: return "Windows CE"; + default: return "Unknown"; + } + } + return "Unknown"; +} +#endif + +/** + * Gets the time reference count for the current VM. + * + * @returns The time reference count. + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(uint64_t) gimHvGetTimeRefCount(PVMCPUCC pVCpu) +{ + /* Hyper-V reports the time in 100 ns units (10 MHz). */ + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + PCGIMHV pHv = &pVCpu->CTX_SUFF(pVM)->gim.s.u.Hv; + uint64_t const u64Tsc = TMCpuTickGet(pVCpu); /** @todo should we be passing VCPU0 always? */ + uint64_t const u64TscHz = pHv->cTscTicksPerSecond; + uint64_t const u64Tsc100NS = u64TscHz / UINT64_C(10000000); /* 100 ns */ + uint64_t const uTimeRefCount = (u64Tsc / u64Tsc100NS); + return uTimeRefCount; +} + + +/** + * Starts the synthetic timer. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pHvStimer Pointer to the Hyper-V synthetic timer. + * + * @remarks Caller needs to hold the timer critical section. + * @thread Any. + */ +VMM_INT_DECL(void) gimHvStartStimer(PVMCPUCC pVCpu, PCGIMHVSTIMER pHvStimer) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + TMTIMERHANDLE hTimer = pHvStimer->hTimer; + Assert(TMTimerIsLockOwner(pVM, hTimer)); + + uint64_t const uTimerCount = pHvStimer->uStimerCountMsr; + if (uTimerCount) + { + uint64_t const uTimerCountNS = uTimerCount * 100; + + /* For periodic timers, 'uTimerCountNS' represents the relative interval. */ + if (MSR_GIM_HV_STIMER_IS_PERIODIC(pHvStimer->uStimerConfigMsr)) + { + TMTimerSetNano(pVM, hTimer, uTimerCountNS); + LogFlow(("GIM%u: HyperV: Started relative periodic STIMER%u with uTimerCountNS=%RU64\n", pVCpu->idCpu, + pHvStimer->idxStimer, uTimerCountNS)); + } + else + { + /* For one-shot timers, 'uTimerCountNS' represents an absolute expiration wrt to Hyper-V reference time, + we convert it to a relative time and program the timer. */ + uint64_t const uCurRefTimeNS = gimHvGetTimeRefCount(pVCpu) * 100; + if (uTimerCountNS > uCurRefTimeNS) + { + uint64_t const uRelativeNS = uTimerCountNS - uCurRefTimeNS; + TMTimerSetNano(pVM, hTimer, uRelativeNS); + LogFlow(("GIM%u: HyperV: Started one-shot relative STIMER%u with uRelativeNS=%RU64\n", pVCpu->idCpu, + pHvStimer->idxStimer, uRelativeNS)); + } + } + /** @todo frequency hinting? */ + } +} + + +/** + * Stops the synthetic timer for the given VCPU. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pHvStimer Pointer to the Hyper-V synthetic timer. + * + * @remarks Caller needs to the hold the timer critical section. + * @thread EMT(pVCpu). + */ +static void gimHvStopStimer(PVMCPUCC pVCpu, PGIMHVSTIMER pHvStimer) +{ + VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + TMTIMERHANDLE hTimer = pHvStimer->hTimer; + Assert(TMTimerIsLockOwner(pVM, hTimer)); + + if (TMTimerIsActive(pVM, hTimer)) + TMTimerStop(pVM, hTimer); +} + + +/** + * MSR read handler for Hyper-V. + * + * @returns Strict VBox status code like CPUMQueryGuestMsr(). + * @retval VINF_CPUM_R3_MSR_READ + * @retval VERR_CPUM_RAISE_GP_0 + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR being read. + * @param pRange The range this MSR belongs to. + * @param puValue Where to store the MSR value read. + * + * @thread EMT. + */ +VMM_INT_DECL(VBOXSTRICTRC) gimHvReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + NOREF(pRange); + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PCGIMHV pHv = &pVM->gim.s.u.Hv; + + switch (idMsr) + { + case MSR_GIM_HV_TIME_REF_COUNT: + *puValue = gimHvGetTimeRefCount(pVCpu); + return VINF_SUCCESS; + + case MSR_GIM_HV_VP_INDEX: + *puValue = pVCpu->idCpu; + return VINF_SUCCESS; + + case MSR_GIM_HV_TPR: + *puValue = APICHvGetTpr(pVCpu); + return VINF_SUCCESS; + + case MSR_GIM_HV_ICR: + *puValue = APICHvGetIcr(pVCpu); + return VINF_SUCCESS; + + case MSR_GIM_HV_GUEST_OS_ID: + *puValue = pHv->u64GuestOsIdMsr; + return VINF_SUCCESS; + + case MSR_GIM_HV_HYPERCALL: + *puValue = pHv->u64HypercallMsr; + return VINF_SUCCESS; + + case MSR_GIM_HV_REF_TSC: + *puValue = pHv->u64TscPageMsr; + return VINF_SUCCESS; + + case MSR_GIM_HV_TSC_FREQ: + *puValue = TMCpuTicksPerSecond(pVM); + return VINF_SUCCESS; + + case MSR_GIM_HV_APIC_FREQ: + { + int rc = APICGetTimerFreq(pVM, puValue); + if (RT_FAILURE(rc)) + return VERR_CPUM_RAISE_GP_0; + return VINF_SUCCESS; + } + + case MSR_GIM_HV_SYNTH_DEBUG_STATUS: + *puValue = pHv->uDbgStatusMsr; + return VINF_SUCCESS; + + case MSR_GIM_HV_SINT0: case MSR_GIM_HV_SINT1: case MSR_GIM_HV_SINT2: case MSR_GIM_HV_SINT3: + case MSR_GIM_HV_SINT4: case MSR_GIM_HV_SINT5: case MSR_GIM_HV_SINT6: case MSR_GIM_HV_SINT7: + case MSR_GIM_HV_SINT8: case MSR_GIM_HV_SINT9: case MSR_GIM_HV_SINT10: case MSR_GIM_HV_SINT11: + case MSR_GIM_HV_SINT12: case MSR_GIM_HV_SINT13: case MSR_GIM_HV_SINT14: case MSR_GIM_HV_SINT15: + { + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + *puValue = pHvCpu->auSintMsrs[idMsr - MSR_GIM_HV_SINT0]; + return VINF_SUCCESS; + } + + case MSR_GIM_HV_STIMER0_CONFIG: + case MSR_GIM_HV_STIMER1_CONFIG: + case MSR_GIM_HV_STIMER2_CONFIG: + case MSR_GIM_HV_STIMER3_CONFIG: + { + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + uint8_t const idxStimer = (idMsr - MSR_GIM_HV_STIMER0_CONFIG) >> 1; + PCGIMHVSTIMER pcHvStimer = &pHvCpu->aStimers[idxStimer]; + *puValue = pcHvStimer->uStimerConfigMsr; + return VINF_SUCCESS; + } + + case MSR_GIM_HV_STIMER0_COUNT: + case MSR_GIM_HV_STIMER1_COUNT: + case MSR_GIM_HV_STIMER2_COUNT: + case MSR_GIM_HV_STIMER3_COUNT: + { + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + uint8_t const idxStimer = (idMsr - MSR_GIM_HV_STIMER0_COUNT) >> 1; + PCGIMHVSTIMER pcHvStimer = &pHvCpu->aStimers[idxStimer]; + *puValue = pcHvStimer->uStimerCountMsr; + return VINF_SUCCESS; + } + + case MSR_GIM_HV_EOM: + { + *puValue = 0; + return VINF_SUCCESS; + } + + case MSR_GIM_HV_SCONTROL: + { + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + *puValue = pHvCpu->uSControlMsr; + return VINF_SUCCESS; + } + + case MSR_GIM_HV_SIMP: + { + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + *puValue = pHvCpu->uSimpMsr; + return VINF_SUCCESS; + } + + case MSR_GIM_HV_SVERSION: + *puValue = GIM_HV_SVERSION; + return VINF_SUCCESS; + + case MSR_GIM_HV_RESET: + *puValue = 0; + return VINF_SUCCESS; + + case MSR_GIM_HV_CRASH_CTL: + *puValue = pHv->uCrashCtlMsr; + return VINF_SUCCESS; + + case MSR_GIM_HV_CRASH_P0: *puValue = pHv->uCrashP0Msr; return VINF_SUCCESS; + case MSR_GIM_HV_CRASH_P1: *puValue = pHv->uCrashP1Msr; return VINF_SUCCESS; + case MSR_GIM_HV_CRASH_P2: *puValue = pHv->uCrashP2Msr; return VINF_SUCCESS; + case MSR_GIM_HV_CRASH_P3: *puValue = pHv->uCrashP3Msr; return VINF_SUCCESS; + case MSR_GIM_HV_CRASH_P4: *puValue = pHv->uCrashP4Msr; return VINF_SUCCESS; + + case MSR_GIM_HV_DEBUG_OPTIONS_MSR: + { + if (pHv->fIsVendorMsHv) + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_READ; +#else + LogRelMax(1, ("GIM: HyperV: Guest querying debug options, suggesting %s interface\n", + pHv->fDbgHypercallInterface ? "hypercall" : "MSR")); + *puValue = pHv->fDbgHypercallInterface ? GIM_HV_DEBUG_OPTIONS_USE_HYPERCALLS : 0; + return VINF_SUCCESS; +#endif + } + break; + } + + /* Write-only MSRs: */ + case MSR_GIM_HV_EOI: + /* Reserved/unknown MSRs: */ + default: + { +#ifdef IN_RING3 + static uint32_t s_cTimes = 0; + if (s_cTimes++ < 20) + LogRel(("GIM: HyperV: Unknown/invalid RdMsr (%#x) -> #GP(0)\n", idMsr)); + LogFunc(("Unknown/invalid RdMsr (%#RX32) -> #GP(0)\n", idMsr)); + break; +#else + return VINF_CPUM_R3_MSR_READ; +#endif + } + } + + return VERR_CPUM_RAISE_GP_0; +} + + +/** + * MSR write handler for Hyper-V. + * + * @returns Strict VBox status code like CPUMSetGuestMsr(). + * @retval VINF_CPUM_R3_MSR_WRITE + * @retval VERR_CPUM_RAISE_GP_0 + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR being written. + * @param pRange The range this MSR belongs to. + * @param uRawValue The raw value with the ignored bits not masked. + * + * @thread EMT. + */ +VMM_INT_DECL(VBOXSTRICTRC) gimHvWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uRawValue) +{ + NOREF(pRange); + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGIMHV pHv = &pVM->gim.s.u.Hv; + + switch (idMsr) + { + case MSR_GIM_HV_TPR: + return APICHvSetTpr(pVCpu, uRawValue); + + case MSR_GIM_HV_EOI: + return APICHvSetEoi(pVCpu, uRawValue); + + case MSR_GIM_HV_ICR: + return APICHvSetIcr(pVCpu, uRawValue); + + case MSR_GIM_HV_GUEST_OS_ID: + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + /* Disable the hypercall-page and hypercalls if 0 is written to this MSR. */ + if (!uRawValue) + { + if (MSR_GIM_HV_HYPERCALL_PAGE_IS_ENABLED(pHv->u64HypercallMsr)) + { + gimR3HvDisableHypercallPage(pVM); + pHv->u64HypercallMsr &= ~MSR_GIM_HV_HYPERCALL_PAGE_ENABLE; + LogRel(("GIM: HyperV: Hypercall page disabled via Guest OS ID MSR\n")); + } + } + else + { + LogRel(("GIM: HyperV: Guest OS reported ID %#RX64\n", uRawValue)); + LogRel(("GIM: HyperV: Open-source=%RTbool Vendor=%#x OS=%#x (%s) Major=%u Minor=%u ServicePack=%u Build=%u\n", + MSR_GIM_HV_GUEST_OS_ID_IS_OPENSOURCE(uRawValue), MSR_GIM_HV_GUEST_OS_ID_VENDOR(uRawValue), + MSR_GIM_HV_GUEST_OS_ID_OS_VARIANT(uRawValue), gimHvGetGuestOsIdVariantName(uRawValue), + MSR_GIM_HV_GUEST_OS_ID_MAJOR_VERSION(uRawValue), MSR_GIM_HV_GUEST_OS_ID_MINOR_VERSION(uRawValue), + MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(uRawValue), MSR_GIM_HV_GUEST_OS_ID_BUILD(uRawValue))); + + /* Update the CPUID leaf, see Hyper-V spec. "Microsoft Hypervisor CPUID Leaves". */ + CPUMCPUIDLEAF HyperLeaf; + RT_ZERO(HyperLeaf); + HyperLeaf.uLeaf = UINT32_C(0x40000002); + HyperLeaf.uEax = MSR_GIM_HV_GUEST_OS_ID_BUILD(uRawValue); + HyperLeaf.uEbx = MSR_GIM_HV_GUEST_OS_ID_MINOR_VERSION(uRawValue) + | (MSR_GIM_HV_GUEST_OS_ID_MAJOR_VERSION(uRawValue) << 16); + HyperLeaf.uEcx = MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(uRawValue); + HyperLeaf.uEdx = MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(uRawValue) + | (MSR_GIM_HV_GUEST_OS_ID_BUILD(uRawValue) << 24); + int rc2 = CPUMR3CpuIdInsert(pVM, &HyperLeaf); + AssertRC(rc2); + } + + pHv->u64GuestOsIdMsr = uRawValue; + + /* + * Update EM on hypercall instruction enabled state. + */ + if (uRawValue) + for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++) + EMSetHypercallInstructionsEnabled(pVM->CTX_SUFF(apCpus)[idCpu], true); + else + for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++) + EMSetHypercallInstructionsEnabled(pVM->CTX_SUFF(apCpus)[idCpu], false); + + return VINF_SUCCESS; +#endif /* IN_RING3 */ + } + + case MSR_GIM_HV_HYPERCALL: + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + /** @todo There is/was a problem with hypercalls for FreeBSD 10.1 guests, + * see @bugref{7270#c116}. */ + /* First, update all but the hypercall page enable bit. */ + pHv->u64HypercallMsr = (uRawValue & ~MSR_GIM_HV_HYPERCALL_PAGE_ENABLE); + + /* Hypercall page can only be enabled when the guest has enabled hypercalls. */ + bool fEnable = MSR_GIM_HV_HYPERCALL_PAGE_IS_ENABLED(uRawValue); + if ( fEnable + && !gimHvAreHypercallsEnabled(pVM)) + { + return VINF_SUCCESS; + } + + /* Is the guest disabling the hypercall-page? Allow it regardless of the Guest-OS Id Msr. */ + if (!fEnable) + { + gimR3HvDisableHypercallPage(pVM); + pHv->u64HypercallMsr = uRawValue; + return VINF_SUCCESS; + } + + /* Enable the hypercall-page. */ + RTGCPHYS GCPhysHypercallPage = MSR_GIM_HV_HYPERCALL_GUEST_PFN(uRawValue) << GUEST_PAGE_SHIFT; + int rc = gimR3HvEnableHypercallPage(pVM, GCPhysHypercallPage); + if (RT_SUCCESS(rc)) + { + pHv->u64HypercallMsr = uRawValue; + return VINF_SUCCESS; + } + + return VERR_CPUM_RAISE_GP_0; +#endif + } + + case MSR_GIM_HV_REF_TSC: + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else /* IN_RING3 */ + /* First, update all but the TSC page enable bit. */ + pHv->u64TscPageMsr = (uRawValue & ~MSR_GIM_HV_REF_TSC_ENABLE); + + /* Is the guest disabling the TSC page? */ + bool fEnable = MSR_GIM_HV_REF_TSC_IS_ENABLED(uRawValue); + if (!fEnable) + { + gimR3HvDisableTscPage(pVM); + pHv->u64TscPageMsr = uRawValue; + return VINF_SUCCESS; + } + + /* Enable the TSC page. */ + RTGCPHYS GCPhysTscPage = MSR_GIM_HV_REF_TSC_GUEST_PFN(uRawValue) << GUEST_PAGE_SHIFT; + int rc = gimR3HvEnableTscPage(pVM, GCPhysTscPage, false /* fUseThisTscSequence */, 0 /* uTscSequence */); + if (RT_SUCCESS(rc)) + { + pHv->u64TscPageMsr = uRawValue; + return VINF_SUCCESS; + } + + return VERR_CPUM_RAISE_GP_0; +#endif /* IN_RING3 */ + } + + case MSR_GIM_HV_APIC_ASSIST_PAGE: + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else /* IN_RING3 */ + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + pHvCpu->uApicAssistPageMsr = uRawValue; + + if (MSR_GIM_HV_APICASSIST_PAGE_IS_ENABLED(uRawValue)) + { + RTGCPHYS GCPhysApicAssistPage = MSR_GIM_HV_APICASSIST_GUEST_PFN(uRawValue) << GUEST_PAGE_SHIFT; + if (PGMPhysIsGCPhysNormal(pVM, GCPhysApicAssistPage)) + { + int rc = gimR3HvEnableApicAssistPage(pVCpu, GCPhysApicAssistPage); + if (RT_SUCCESS(rc)) + { + pHvCpu->uApicAssistPageMsr = uRawValue; + return VINF_SUCCESS; + } + } + else + { + LogRelMax(5, ("GIM%u: HyperV: APIC-assist page address %#RGp invalid!\n", pVCpu->idCpu, + GCPhysApicAssistPage)); + } + } + else + gimR3HvDisableApicAssistPage(pVCpu); + + return VERR_CPUM_RAISE_GP_0; +#endif /* IN_RING3 */ + } + + case MSR_GIM_HV_RESET: + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + if (MSR_GIM_HV_RESET_IS_ENABLED(uRawValue)) + { + LogRel(("GIM: HyperV: Reset initiated through MSR\n")); + int rc = PDMDevHlpVMReset(pVM->gim.s.pDevInsR3, PDMVMRESET_F_GIM); + AssertRC(rc); /* Note! Not allowed to return VINF_EM_RESET / VINF_EM_HALT here, so ignore them. */ + } + /* else: Ignore writes to other bits. */ + return VINF_SUCCESS; +#endif /* IN_RING3 */ + } + + case MSR_GIM_HV_CRASH_CTL: + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + if (uRawValue & MSR_GIM_HV_CRASH_CTL_NOTIFY) + { + LogRel(("GIM: HyperV: Guest indicates a fatal condition! P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64\n", + pHv->uCrashP0Msr, pHv->uCrashP1Msr, pHv->uCrashP2Msr, pHv->uCrashP3Msr, pHv->uCrashP4Msr)); + DBGFR3ReportBugCheck(pVM, pVCpu, DBGFEVENT_BSOD_MSR, pHv->uCrashP0Msr, pHv->uCrashP1Msr, + pHv->uCrashP2Msr, pHv->uCrashP3Msr, pHv->uCrashP4Msr); + /* (Do not try pass VINF_EM_DBG_EVENT, doesn't work from here!) */ + } + return VINF_SUCCESS; +#endif + } + + case MSR_GIM_HV_SYNTH_DEBUG_SEND_BUFFER: + { + if (!pHv->fDbgEnabled) + return VERR_CPUM_RAISE_GP_0; +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + RTGCPHYS GCPhysBuffer = (RTGCPHYS)uRawValue; + pHv->uDbgSendBufferMsr = GCPhysBuffer; + if (PGMPhysIsGCPhysNormal(pVM, GCPhysBuffer)) + LogRel(("GIM: HyperV: Set up debug send buffer at %#RGp\n", GCPhysBuffer)); + else + LogRel(("GIM: HyperV: Destroyed debug send buffer\n")); + pHv->uDbgSendBufferMsr = uRawValue; + return VINF_SUCCESS; +#endif + } + + case MSR_GIM_HV_SYNTH_DEBUG_RECEIVE_BUFFER: + { + if (!pHv->fDbgEnabled) + return VERR_CPUM_RAISE_GP_0; +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + RTGCPHYS GCPhysBuffer = (RTGCPHYS)uRawValue; + pHv->uDbgRecvBufferMsr = GCPhysBuffer; + if (PGMPhysIsGCPhysNormal(pVM, GCPhysBuffer)) + LogRel(("GIM: HyperV: Set up debug receive buffer at %#RGp\n", GCPhysBuffer)); + else + LogRel(("GIM: HyperV: Destroyed debug receive buffer\n")); + return VINF_SUCCESS; +#endif + } + + case MSR_GIM_HV_SYNTH_DEBUG_PENDING_BUFFER: + { + if (!pHv->fDbgEnabled) + return VERR_CPUM_RAISE_GP_0; +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + RTGCPHYS GCPhysBuffer = (RTGCPHYS)uRawValue; + pHv->uDbgPendingBufferMsr = GCPhysBuffer; + if (PGMPhysIsGCPhysNormal(pVM, GCPhysBuffer)) + LogRel(("GIM: HyperV: Set up debug pending buffer at %#RGp\n", uRawValue)); + else + LogRel(("GIM: HyperV: Destroyed debug pending buffer\n")); + return VINF_SUCCESS; +#endif + } + + case MSR_GIM_HV_SYNTH_DEBUG_CONTROL: + { + if (!pHv->fDbgEnabled) + return VERR_CPUM_RAISE_GP_0; +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + if ( MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_WRITE(uRawValue) + && MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_READ(uRawValue)) + { + LogRel(("GIM: HyperV: Requesting both read and write through debug control MSR -> #GP(0)\n")); + return VERR_CPUM_RAISE_GP_0; + } + + if (MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_WRITE(uRawValue)) + { + uint32_t cbWrite = MSR_GIM_HV_SYNTH_DEBUG_CONTROL_W_LEN(uRawValue); + if ( cbWrite > 0 + && cbWrite < GIM_HV_PAGE_SIZE) + { + if (PGMPhysIsGCPhysNormal(pVM, (RTGCPHYS)pHv->uDbgSendBufferMsr)) + { + Assert(pHv->pvDbgBuffer); + int rc = PGMPhysSimpleReadGCPhys(pVM, pHv->pvDbgBuffer, (RTGCPHYS)pHv->uDbgSendBufferMsr, cbWrite); + if (RT_SUCCESS(rc)) + { + LogRelMax(1, ("GIM: HyperV: Initiated debug data transmission via MSR\n")); + uint32_t cbWritten = 0; + rc = gimR3HvDebugWrite(pVM, pHv->pvDbgBuffer, cbWrite, &cbWritten, false /*fUdpPkt*/); + if ( RT_SUCCESS(rc) + && cbWrite == cbWritten) + pHv->uDbgStatusMsr = MSR_GIM_HV_SYNTH_DEBUG_STATUS_W_SUCCESS; + else + pHv->uDbgStatusMsr = 0; + } + else + LogRelMax(5, ("GIM: HyperV: Failed to read debug send buffer at %#RGp, rc=%Rrc\n", + (RTGCPHYS)pHv->uDbgSendBufferMsr, rc)); + } + else + LogRelMax(5, ("GIM: HyperV: Debug send buffer address %#RGp invalid! Ignoring debug write!\n", + (RTGCPHYS)pHv->uDbgSendBufferMsr)); + } + else + LogRelMax(5, ("GIM: HyperV: Invalid write size %u specified in MSR, ignoring debug write!\n", + MSR_GIM_HV_SYNTH_DEBUG_CONTROL_W_LEN(uRawValue))); + } + else if (MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_READ(uRawValue)) + { + if (PGMPhysIsGCPhysNormal(pVM, (RTGCPHYS)pHv->uDbgRecvBufferMsr)) + { + LogRelMax(1, ("GIM: HyperV: Initiated debug data reception via MSR\n")); + uint32_t cbReallyRead; + Assert(pHv->pvDbgBuffer); + int rc = gimR3HvDebugRead(pVM, pHv->pvDbgBuffer, GIM_HV_PAGE_SIZE, GIM_HV_PAGE_SIZE, + &cbReallyRead, 0, false /*fUdpPkt*/); + if ( RT_SUCCESS(rc) + && cbReallyRead > 0) + { + rc = PGMPhysSimpleWriteGCPhys(pVM, (RTGCPHYS)pHv->uDbgRecvBufferMsr, pHv->pvDbgBuffer, cbReallyRead); + if (RT_SUCCESS(rc)) + { + pHv->uDbgStatusMsr = ((uint16_t)cbReallyRead) << 16; + pHv->uDbgStatusMsr |= MSR_GIM_HV_SYNTH_DEBUG_STATUS_R_SUCCESS; + } + else + { + pHv->uDbgStatusMsr = 0; + LogRelMax(5, ("GIM: HyperV: PGMPhysSimpleWriteGCPhys failed. rc=%Rrc\n", rc)); + } + } + else + pHv->uDbgStatusMsr = 0; + } + else + { + LogRelMax(5, ("GIM: HyperV: Debug receive buffer address %#RGp invalid! Ignoring debug read!\n", + (RTGCPHYS)pHv->uDbgRecvBufferMsr)); + } + } + return VINF_SUCCESS; +#endif + } + + case MSR_GIM_HV_SINT0: case MSR_GIM_HV_SINT1: case MSR_GIM_HV_SINT2: case MSR_GIM_HV_SINT3: + case MSR_GIM_HV_SINT4: case MSR_GIM_HV_SINT5: case MSR_GIM_HV_SINT6: case MSR_GIM_HV_SINT7: + case MSR_GIM_HV_SINT8: case MSR_GIM_HV_SINT9: case MSR_GIM_HV_SINT10: case MSR_GIM_HV_SINT11: + case MSR_GIM_HV_SINT12: case MSR_GIM_HV_SINT13: case MSR_GIM_HV_SINT14: case MSR_GIM_HV_SINT15: + { + uint8_t uVector = MSR_GIM_HV_SINT_GET_VECTOR(uRawValue); + bool const fVMBusMsg = RT_BOOL(idMsr == GIM_HV_VMBUS_MSG_SINT); + size_t const idxSintMsr = idMsr - MSR_GIM_HV_SINT0; + const char *pszDesc = fVMBusMsg ? "VMBus Message" : "Generic"; + if (uVector < GIM_HV_SINT_VECTOR_VALID_MIN) + { + LogRel(("GIM%u: HyperV: Programmed an invalid vector in SINT%u (%s), uVector=%u -> #GP(0)\n", pVCpu->idCpu, + idxSintMsr, pszDesc, uVector)); + return VERR_CPUM_RAISE_GP_0; + } + + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + pHvCpu->auSintMsrs[idxSintMsr] = uRawValue; + if (fVMBusMsg) + { + if (MSR_GIM_HV_SINT_IS_MASKED(uRawValue)) + Log(("GIM%u: HyperV: Masked SINT%u (%s)\n", pVCpu->idCpu, idxSintMsr, pszDesc)); + else + Log(("GIM%u: HyperV: Unmasked SINT%u (%s), uVector=%u\n", pVCpu->idCpu, idxSintMsr, pszDesc, uVector)); + } + Log(("GIM%u: HyperV: Written SINT%u=%#RX64\n", pVCpu->idCpu, idxSintMsr, uRawValue)); + return VINF_SUCCESS; + } + + case MSR_GIM_HV_SCONTROL: + { +#ifndef IN_RING3 + /** @todo make this RZ later? */ + return VINF_CPUM_R3_MSR_WRITE; +#else + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + pHvCpu->uSControlMsr = uRawValue; + if (MSR_GIM_HV_SCONTROL_IS_ENABLED(uRawValue)) + LogRel(("GIM%u: HyperV: Synthetic interrupt control enabled\n", pVCpu->idCpu)); + else + LogRel(("GIM%u: HyperV: Synthetic interrupt control disabled\n", pVCpu->idCpu)); + return VINF_SUCCESS; +#endif + } + + case MSR_GIM_HV_STIMER0_CONFIG: + case MSR_GIM_HV_STIMER1_CONFIG: + case MSR_GIM_HV_STIMER2_CONFIG: + case MSR_GIM_HV_STIMER3_CONFIG: + { + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + uint8_t const idxStimer = (idMsr - MSR_GIM_HV_STIMER0_CONFIG) >> 1; + + /* Validate the writable bits. */ + if (RT_LIKELY(!(uRawValue & ~MSR_GIM_HV_STIMER_RW_VALID))) + { + Assert(idxStimer < RT_ELEMENTS(pHvCpu->aStimers)); + PGIMHVSTIMER pHvStimer = &pHvCpu->aStimers[idxStimer]; + + /* Lock to prevent concurrent access from the timer callback. */ + int rc = TMTimerLock(pVM, pHvStimer->hTimer, VERR_IGNORED); + if (rc == VINF_SUCCESS) + { + /* Update the MSR value. */ + pHvStimer->uStimerConfigMsr = uRawValue; + Log(("GIM%u: HyperV: Set STIMER_CONFIG%u=%#RX64\n", pVCpu->idCpu, idxStimer, uRawValue)); + + /* Process the MSR bits. */ + if ( !MSR_GIM_HV_STIMER_GET_SINTX(uRawValue) /* Writing SINTx as 0 causes the timer to be disabled. */ + || !MSR_GIM_HV_STIMER_IS_ENABLED(uRawValue)) + { + pHvStimer->uStimerConfigMsr &= ~MSR_GIM_HV_STIMER_ENABLE; + gimHvStopStimer(pVCpu, pHvStimer); + Log(("GIM%u: HyperV: Disabled STIMER_CONFIG%u\n", pVCpu->idCpu, idxStimer)); + } + else if (MSR_GIM_HV_STIMER_IS_ENABLED(uRawValue)) + { + /* Auto-enable implies writing to the STIMERx_COUNT MSR is what starts the timer. */ + if (!MSR_GIM_HV_STIMER_IS_AUTO_ENABLED(uRawValue)) + { + if (!TMTimerIsActive(pVM, pHvStimer->hTimer)) + { + gimHvStartStimer(pVCpu, pHvStimer); + Log(("GIM%u: HyperV: Started STIMER%u\n", pVCpu->idCpu, idxStimer)); + } + else + { + /* + * Enabling a timer that's already enabled is undefined behaviour, + * see Hyper-V spec. 15.3.1 "Synthetic Timer Configuration Register". + * + * Our implementation just re-starts the timer. Guests that comform to + * the Hyper-V specs. should not be doing this anyway. + */ + AssertFailed(); + gimHvStopStimer(pVCpu, pHvStimer); + gimHvStartStimer(pVCpu, pHvStimer); + } + } + } + + TMTimerUnlock(pVM, pHvStimer->hTimer); + } + return rc; + } +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + LogRel(("GIM%u: HyperV: Setting reserved bits of STIMER%u MSR (uRawValue=%#RX64) -> #GP(0)\n", pVCpu->idCpu, + idxStimer, uRawValue)); + return VERR_CPUM_RAISE_GP_0; +#endif + } + + case MSR_GIM_HV_STIMER0_COUNT: + case MSR_GIM_HV_STIMER1_COUNT: + case MSR_GIM_HV_STIMER2_COUNT: + case MSR_GIM_HV_STIMER3_COUNT: + { + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + uint8_t const idxStimer = (idMsr - MSR_GIM_HV_STIMER0_CONFIG) >> 1; + Assert(idxStimer < RT_ELEMENTS(pHvCpu->aStimers)); + PGIMHVSTIMER pHvStimer = &pHvCpu->aStimers[idxStimer]; + int const rcBusy = VINF_CPUM_R3_MSR_WRITE; + + /* + * Writing zero to this MSR disables the timer regardless of whether the auto-enable + * flag is set in the config MSR corresponding to the timer. + */ + if (!uRawValue) + { + gimHvStopStimer(pVCpu, pHvStimer); + pHvStimer->uStimerCountMsr = 0; + Log(("GIM%u: HyperV: Set STIMER_COUNT%u=%RU64, stopped timer\n", pVCpu->idCpu, idxStimer, uRawValue)); + return VINF_SUCCESS; + } + + /* + * Concurrent writes to the config. MSR can't happen as it's serialized by way + * of being done on the same EMT as this. + */ + if (MSR_GIM_HV_STIMER_IS_AUTO_ENABLED(pHvStimer->uStimerConfigMsr)) + { + int rc = TMTimerLock(pVM, pHvStimer->hTimer, rcBusy); + if (rc == VINF_SUCCESS) + { + pHvStimer->uStimerCountMsr = uRawValue; + gimHvStartStimer(pVCpu, pHvStimer); + TMTimerUnlock(pVM, pHvStimer->hTimer); + Log(("GIM%u: HyperV: Set STIMER_COUNT%u=%RU64 %RU64 msec, auto-started timer\n", pVCpu->idCpu, idxStimer, + uRawValue, (uRawValue * 100) / RT_NS_1MS_64)); + } + return rc; + } + + /* Simple update of the counter without any timer start/stop side-effects. */ + pHvStimer->uStimerCountMsr = uRawValue; + Log(("GIM%u: HyperV: Set STIMER_COUNT%u=%RU64\n", pVCpu->idCpu, idxStimer, uRawValue)); + return VINF_SUCCESS; + } + + case MSR_GIM_HV_EOM: + { + /** @todo implement EOM. */ + Log(("GIM%u: HyperV: EOM\n", pVCpu->idCpu)); + return VINF_SUCCESS; + } + + case MSR_GIM_HV_SIEFP: + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + pHvCpu->uSiefpMsr = uRawValue; + if (MSR_GIM_HV_SIEF_PAGE_IS_ENABLED(uRawValue)) + { + RTGCPHYS GCPhysSiefPage = MSR_GIM_HV_SIEF_GUEST_PFN(uRawValue) << GUEST_PAGE_SHIFT; + if (PGMPhysIsGCPhysNormal(pVM, GCPhysSiefPage)) + { + int rc = gimR3HvEnableSiefPage(pVCpu, GCPhysSiefPage); + if (RT_SUCCESS(rc)) + { + LogRel(("GIM%u: HyperV: Enabled synthetic interrupt event flags page at %#RGp\n", pVCpu->idCpu, + GCPhysSiefPage)); + /** @todo SIEF setup. */ + return VINF_SUCCESS; + } + } + else + LogRelMax(5, ("GIM%u: HyperV: SIEF page address %#RGp invalid!\n", pVCpu->idCpu, GCPhysSiefPage)); + } + else + gimR3HvDisableSiefPage(pVCpu); + + return VERR_CPUM_RAISE_GP_0; +#endif + break; + } + + case MSR_GIM_HV_SIMP: + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu; + pHvCpu->uSimpMsr = uRawValue; + if (MSR_GIM_HV_SIMP_IS_ENABLED(uRawValue)) + { + RTGCPHYS GCPhysSimp = MSR_GIM_HV_SIMP_GPA(uRawValue); + if (PGMPhysIsGCPhysNormal(pVM, GCPhysSimp)) + { + uint8_t abSimp[GIM_HV_PAGE_SIZE]; + RT_ZERO(abSimp); + int rc2 = PGMPhysSimpleWriteGCPhys(pVM, GCPhysSimp, &abSimp[0], sizeof(abSimp)); + if (RT_SUCCESS(rc2)) + LogRel(("GIM%u: HyperV: Enabled synthetic interrupt message page at %#RGp\n", pVCpu->idCpu, GCPhysSimp)); + else + { + LogRel(("GIM%u: HyperV: Failed to update synthetic interrupt message page at %#RGp. uSimpMsr=%#RX64 rc=%Rrc\n", + pVCpu->idCpu, pHvCpu->uSimpMsr, GCPhysSimp, rc2)); + return VERR_CPUM_RAISE_GP_0; + } + } + else + { + LogRel(("GIM%u: HyperV: Enabled synthetic interrupt message page at invalid address %#RGp\n", pVCpu->idCpu, + GCPhysSimp)); + } + } + else + LogRel(("GIM%u: HyperV: Disabled synthetic interrupt message page\n", pVCpu->idCpu)); + return VINF_SUCCESS; +#endif + } + + case MSR_GIM_HV_CRASH_P0: pHv->uCrashP0Msr = uRawValue; return VINF_SUCCESS; + case MSR_GIM_HV_CRASH_P1: pHv->uCrashP1Msr = uRawValue; return VINF_SUCCESS; + case MSR_GIM_HV_CRASH_P2: pHv->uCrashP2Msr = uRawValue; return VINF_SUCCESS; + case MSR_GIM_HV_CRASH_P3: pHv->uCrashP3Msr = uRawValue; return VINF_SUCCESS; + case MSR_GIM_HV_CRASH_P4: pHv->uCrashP4Msr = uRawValue; return VINF_SUCCESS; + + case MSR_GIM_HV_TIME_REF_COUNT: /* Read-only MSRs. */ + case MSR_GIM_HV_VP_INDEX: + case MSR_GIM_HV_TSC_FREQ: + case MSR_GIM_HV_APIC_FREQ: + LogFunc(("WrMsr on read-only MSR %#RX32 -> #GP(0)\n", idMsr)); + break; + + case MSR_GIM_HV_DEBUG_OPTIONS_MSR: + { + if (pHv->fIsVendorMsHv) + { +#ifndef IN_RING3 + return VINF_CPUM_R3_MSR_WRITE; +#else + LogRelMax(5, ("GIM: HyperV: Write debug options MSR with %#RX64 ignored\n", uRawValue)); + return VINF_SUCCESS; +#endif + } + return VERR_CPUM_RAISE_GP_0; + } + + default: + { +#ifdef IN_RING3 + static uint32_t s_cTimes = 0; + if (s_cTimes++ < 20) + LogRel(("GIM: HyperV: Unknown/invalid WrMsr (%#x,%#x`%08x) -> #GP(0)\n", idMsr, + uRawValue & UINT64_C(0xffffffff00000000), uRawValue & UINT64_C(0xffffffff))); + LogFunc(("Unknown/invalid WrMsr (%#RX32,%#RX64) -> #GP(0)\n", idMsr, uRawValue)); + break; +#else + return VINF_CPUM_R3_MSR_WRITE; +#endif + } + } + + return VERR_CPUM_RAISE_GP_0; +} + + +/** + * Whether we need to trap \#UD exceptions in the guest. + * + * We only needed to trap \#UD exceptions for the old raw-mode guests when + * hypercalls are enabled. For HM VMs, the hypercall would be handled via the + * VMCALL/VMMCALL VM-exit. + * + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) gimHvShouldTrapXcptUD(PVMCPU pVCpu) +{ + RT_NOREF(pVCpu); + return false; +} + + +/** + * Checks the instruction and executes the hypercall if it's a valid hypercall + * instruction. + * + * This interface is used by \#UD handlers and IEM. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * @param uDisOpcode The disassembler opcode. + * @param cbInstr The instruction length. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(VBOXSTRICTRC) gimHvHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr) +{ + Assert(pVCpu); + Assert(pCtx); + VMCPU_ASSERT_EMT(pVCpu); + + PVM pVM = pVCpu->CTX_SUFF(pVM); + CPUMCPUVENDOR const enmGuestCpuVendor = (CPUMCPUVENDOR)pVM->cpum.ro.GuestFeatures.enmCpuVendor; + if ( ( uDisOpcode == OP_VMCALL + && ( enmGuestCpuVendor == CPUMCPUVENDOR_INTEL + || enmGuestCpuVendor == CPUMCPUVENDOR_VIA + || enmGuestCpuVendor == CPUMCPUVENDOR_SHANGHAI)) + || ( uDisOpcode == OP_VMMCALL + && ( enmGuestCpuVendor == CPUMCPUVENDOR_AMD + || enmGuestCpuVendor == CPUMCPUVENDOR_HYGON)) ) + return gimHvHypercall(pVCpu, pCtx); + + RT_NOREF_PV(cbInstr); + return VERR_GIM_INVALID_HYPERCALL_INSTR; +} + + +/** + * Exception handler for \#UD. + * + * @returns Strict VBox status code. + * @retval VINF_SUCCESS if the hypercall succeeded (even if its operation + * failed). + * @retval VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3. + * @retval VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating + * RIP. + * @retval VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient. + * @retval VERR_GIM_INVALID_HYPERCALL_INSTR instruction at RIP is not a valid + * hypercall instruction. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * @param pDis Pointer to the disassembled instruction state at RIP. + * Optional, can be NULL. + * @param pcbInstr Where to store the instruction length of the hypercall + * instruction. Optional, can be NULL. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(VBOXSTRICTRC) gimHvXcptUD(PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr) +{ + VMCPU_ASSERT_EMT(pVCpu); + + /* + * If we didn't ask for #UD to be trapped, bail. + */ + if (!gimHvShouldTrapXcptUD(pVCpu)) + return VERR_GIM_IPE_1; + + if (!pDis) + { + /* + * Disassemble the instruction at RIP to figure out if it's the Intel VMCALL instruction + * or the AMD VMMCALL instruction and if so, handle it as a hypercall. + */ + unsigned cbInstr; + DISCPUSTATE Dis; + int rc = EMInterpretDisasCurrent(pVCpu, &Dis, &cbInstr); + if (RT_SUCCESS(rc)) + { + if (pcbInstr) + *pcbInstr = (uint8_t)cbInstr; + return gimHvHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr); + } + + Log(("GIM: HyperV: Failed to disassemble instruction at CS:RIP=%04x:%08RX64. rc=%Rrc\n", pCtx->cs.Sel, pCtx->rip, rc)); + return rc; + } + + return gimHvHypercallEx(pVCpu, pCtx, pDis->pCurInstr->uOpcode, pDis->cbInstr); +} + diff --git a/src/VBox/VMM/VMMAll/GIMAllKvm.cpp b/src/VBox/VMM/VMMAll/GIMAllKvm.cpp new file mode 100644 index 00000000..91d858ee --- /dev/null +++ b/src/VBox/VMM/VMMAll/GIMAllKvm.cpp @@ -0,0 +1,450 @@ +/* $Id: GIMAllKvm.cpp $ */ +/** @file + * GIM - Guest Interface Manager, KVM, All Contexts. + */ + +/* + * Copyright (C) 2015-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_GIM +#include +#include +#include +#include +#include +#include +#include +#include "GIMKvmInternal.h" +#include "GIMInternal.h" +#include + +#include +#include +#include + +#include + + +/** + * Handles the KVM hypercall. + * + * @returns Strict VBox status code. + * @retval VINF_SUCCESS if the hypercall succeeded (even if its operation + * failed). + * @retval VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3. + * @retval VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(VBOXSTRICTRC) gimKvmHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx) +{ + VMCPU_ASSERT_EMT(pVCpu); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + STAM_REL_COUNTER_INC(&pVM->gim.s.StatHypercalls); + + /* + * Get the hypercall operation and arguments. + */ + bool const fIs64BitMode = CPUMIsGuestIn64BitCodeEx(pCtx); + uint64_t uHyperOp = pCtx->rax; + uint64_t uHyperArg0 = pCtx->rbx; + uint64_t uHyperArg1 = pCtx->rcx; + uint64_t uHyperArg2 = pCtx->rdi; + uint64_t uHyperArg3 = pCtx->rsi; + uint64_t uHyperRet = KVM_HYPERCALL_RET_ENOSYS; + uint64_t uAndMask = UINT64_C(0xffffffffffffffff); + if (!fIs64BitMode) + { + uAndMask = UINT64_C(0xffffffff); + uHyperOp &= UINT64_C(0xffffffff); + uHyperArg0 &= UINT64_C(0xffffffff); + uHyperArg1 &= UINT64_C(0xffffffff); + uHyperArg2 &= UINT64_C(0xffffffff); + uHyperArg3 &= UINT64_C(0xffffffff); + uHyperRet &= UINT64_C(0xffffffff); + } + + /* + * Verify that guest ring-0 is the one making the hypercall. + */ + uint32_t uCpl = CPUMGetGuestCPL(pVCpu); + if (RT_UNLIKELY(uCpl)) + { + pCtx->rax = KVM_HYPERCALL_RET_EPERM & uAndMask; + return VERR_GIM_HYPERCALL_ACCESS_DENIED; + } + + /* + * Do the work. + */ + int rc = VINF_SUCCESS; + switch (uHyperOp) + { + case KVM_HYPERCALL_OP_KICK_CPU: + { + if (uHyperArg1 < pVM->cCpus) + { + PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, uHyperArg1); /* ASSUMES pVCpu index == ApicId of the VCPU. */ + EMUnhaltAndWakeUp(pVM, pVCpuDst); + uHyperRet = KVM_HYPERCALL_RET_SUCCESS; + } + else + { + /* Shouldn't ever happen! If it does, throw a guru, as otherwise it'll lead to deadlocks in the guest anyway! */ + rc = VERR_GIM_HYPERCALL_FAILED; + } + break; + } + + case KVM_HYPERCALL_OP_VAPIC_POLL_IRQ: + uHyperRet = KVM_HYPERCALL_RET_SUCCESS; + break; + + default: + break; + } + + /* + * Place the result in rax/eax. + */ + pCtx->rax = uHyperRet & uAndMask; + return rc; +} + + +/** + * Returns whether the guest has configured and enabled the use of KVM's + * hypercall interface. + * + * @returns true if hypercalls are enabled, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) gimKvmAreHypercallsEnabled(PVMCPU pVCpu) +{ + NOREF(pVCpu); + /* KVM paravirt interface doesn't have hypercall control bits (like Hyper-V does) + that guests can control, i.e. hypercalls are always enabled. */ + return true; +} + + +/** + * Returns whether the guest has configured and enabled the use of KVM's + * paravirtualized TSC. + * + * @returns true if paravirt. TSC is enabled, false otherwise. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) gimKvmIsParavirtTscEnabled(PVMCC pVM) +{ + uint32_t const cCpus = pVM->cCpus; + for (uint32_t idCpu = 0; idCpu < cCpus; idCpu++) + { + PVMCPUCC pVCpu = pVM->CTX_SUFF(apCpus)[idCpu]; + PGIMKVMCPU pGimKvmCpu = &pVCpu->gim.s.u.KvmCpu; + if (MSR_GIM_KVM_SYSTEM_TIME_IS_ENABLED(pGimKvmCpu->u64SystemTimeMsr)) + return true; + } + return false; +} + + +/** + * MSR read handler for KVM. + * + * @returns Strict VBox status code like CPUMQueryGuestMsr(). + * @retval VINF_CPUM_R3_MSR_READ + * @retval VERR_CPUM_RAISE_GP_0 + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR being read. + * @param pRange The range this MSR belongs to. + * @param puValue Where to store the MSR value read. + */ +VMM_INT_DECL(VBOXSTRICTRC) gimKvmReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue) +{ + NOREF(pRange); + PVM pVM = pVCpu->CTX_SUFF(pVM); + PGIMKVM pKvm = &pVM->gim.s.u.Kvm; + PGIMKVMCPU pKvmCpu = &pVCpu->gim.s.u.KvmCpu; + + switch (idMsr) + { + case MSR_GIM_KVM_SYSTEM_TIME: + case MSR_GIM_KVM_SYSTEM_TIME_OLD: + *puValue = pKvmCpu->u64SystemTimeMsr; + return VINF_SUCCESS; + + case MSR_GIM_KVM_WALL_CLOCK: + case MSR_GIM_KVM_WALL_CLOCK_OLD: + *puValue = pKvm->u64WallClockMsr; + return VINF_SUCCESS; + + default: + { +#ifdef IN_RING3 + static uint32_t s_cTimes = 0; + if (s_cTimes++ < 20) + LogRel(("GIM: KVM: Unknown/invalid RdMsr (%#x) -> #GP(0)\n", idMsr)); +#endif + LogFunc(("Unknown/invalid RdMsr (%#RX32) -> #GP(0)\n", idMsr)); + break; + } + } + + return VERR_CPUM_RAISE_GP_0; +} + + +/** + * MSR write handler for KVM. + * + * @returns Strict VBox status code like CPUMSetGuestMsr(). + * @retval VINF_CPUM_R3_MSR_WRITE + * @retval VERR_CPUM_RAISE_GP_0 + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR being written. + * @param pRange The range this MSR belongs to. + * @param uRawValue The raw value with the ignored bits not masked. + */ +VMM_INT_DECL(VBOXSTRICTRC) gimKvmWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uRawValue) +{ + NOREF(pRange); + switch (idMsr) + { + case MSR_GIM_KVM_SYSTEM_TIME: + case MSR_GIM_KVM_SYSTEM_TIME_OLD: + { +#ifndef IN_RING3 + RT_NOREF2(pVCpu, uRawValue); + return VINF_CPUM_R3_MSR_WRITE; +#else + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGIMKVMCPU pKvmCpu = &pVCpu->gim.s.u.KvmCpu; + if (uRawValue & MSR_GIM_KVM_SYSTEM_TIME_ENABLE_BIT) + gimR3KvmEnableSystemTime(pVM, pVCpu, uRawValue); + else + gimR3KvmDisableSystemTime(pVM); + + pKvmCpu->u64SystemTimeMsr = uRawValue; + return VINF_SUCCESS; +#endif /* IN_RING3 */ + } + + case MSR_GIM_KVM_WALL_CLOCK: + case MSR_GIM_KVM_WALL_CLOCK_OLD: + { +#ifndef IN_RING3 + RT_NOREF2(pVCpu, uRawValue); + return VINF_CPUM_R3_MSR_WRITE; +#else + /* Enable the wall-clock struct. */ + RTGCPHYS GCPhysWallClock = MSR_GIM_KVM_WALL_CLOCK_GUEST_GPA(uRawValue); + if (RT_LIKELY(RT_ALIGN_64(GCPhysWallClock, 4) == GCPhysWallClock)) + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + int rc = gimR3KvmEnableWallClock(pVM, GCPhysWallClock); + if (RT_SUCCESS(rc)) + { + PGIMKVM pKvm = &pVM->gim.s.u.Kvm; + pKvm->u64WallClockMsr = uRawValue; + return VINF_SUCCESS; + } + } + return VERR_CPUM_RAISE_GP_0; +#endif /* IN_RING3 */ + } + + default: + { +#ifdef IN_RING3 + static uint32_t s_cTimes = 0; + if (s_cTimes++ < 20) + LogRel(("GIM: KVM: Unknown/invalid WrMsr (%#x,%#x`%08x) -> #GP(0)\n", idMsr, + uRawValue & UINT64_C(0xffffffff00000000), uRawValue & UINT64_C(0xffffffff))); +#endif + LogFunc(("Unknown/invalid WrMsr (%#RX32,%#RX64) -> #GP(0)\n", idMsr, uRawValue)); + break; + } + } + + return VERR_CPUM_RAISE_GP_0; +} + + +/** + * Whether we need to trap \#UD exceptions in the guest. + * + * On AMD-V we need to trap them because paravirtualized Linux/KVM guests use + * the Intel VMCALL instruction to make hypercalls and we need to trap and + * optionally patch them to the AMD-V VMMCALL instruction and handle the + * hypercall. + * + * I guess this was done so that guest teleporation between an AMD and an Intel + * machine would working without any changes at the time of teleporation. + * However, this also means we -always- need to intercept \#UD exceptions on one + * of the two CPU models (Intel or AMD). Hyper-V solves this problem more + * elegantly by letting the hypervisor supply an opaque hypercall page. + * + * For raw-mode VMs, this function will always return true. See gimR3KvmInit(). + * + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) gimKvmShouldTrapXcptUD(PVM pVM) +{ + return pVM->gim.s.u.Kvm.fTrapXcptUD; +} + + +/** + * Checks the instruction and executes the hypercall if it's a valid hypercall + * instruction. + * + * This interface is used by \#UD handlers and IEM. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * @param uDisOpcode The disassembler opcode. + * @param cbInstr The instruction length. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(VBOXSTRICTRC) gimKvmHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr) +{ + Assert(pVCpu); + Assert(pCtx); + VMCPU_ASSERT_EMT(pVCpu); + + /* + * If the instruction at RIP is the Intel VMCALL instruction or + * the AMD VMMCALL instruction handle it as a hypercall. + * + * Linux/KVM guests always uses the Intel VMCALL instruction but we patch + * it to the host-native one whenever we encounter it so subsequent calls + * will not require disassembly (when coming from HM). + */ + if ( uDisOpcode == OP_VMCALL + || uDisOpcode == OP_VMMCALL) + { + /* + * Perform the hypercall. + * + * For HM, we can simply resume guest execution without performing the hypercall now and + * do it on the next VMCALL/VMMCALL exit handler on the patched instruction. + * + * For raw-mode we need to do this now anyway. So we do it here regardless with an added + * advantage is that it saves one world-switch for the HM case. + */ + VBOXSTRICTRC rcStrict = gimKvmHypercall(pVCpu, pCtx); + if (rcStrict == VINF_SUCCESS) + { + /* + * Patch the instruction to so we don't have to spend time disassembling it each time. + * Makes sense only for HM as with raw-mode we will be getting a #UD regardless. + */ + PVM pVM = pVCpu->CTX_SUFF(pVM); + PCGIMKVM pKvm = &pVM->gim.s.u.Kvm; + if ( uDisOpcode != pKvm->uOpcodeNative + && cbInstr == sizeof(pKvm->abOpcodeNative) ) + { + /** @todo r=ramshankar: we probably should be doing this in an + * EMT rendezvous. */ + /** @todo Add stats for patching. */ + int rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, pKvm->abOpcodeNative, sizeof(pKvm->abOpcodeNative)); + AssertRC(rc); + } + } + else + { + /* The KVM provider doesn't have any concept of continuing hypercalls. */ + Assert(rcStrict != VINF_GIM_HYPERCALL_CONTINUING); +#ifdef IN_RING3 + Assert(rcStrict != VINF_GIM_R3_HYPERCALL); +#endif + } + return rcStrict; + } + + return VERR_GIM_INVALID_HYPERCALL_INSTR; +} + + +/** + * Exception handler for \#UD. + * + * @returns Strict VBox status code. + * @retval VINF_SUCCESS if the hypercall succeeded (even if its operation + * failed). + * @retval VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3. + * @retval VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient. + * @retval VERR_GIM_INVALID_HYPERCALL_INSTR instruction at RIP is not a valid + * hypercall instruction. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * @param pDis Pointer to the disassembled instruction state at RIP. + * Optional, can be NULL. + * @param pcbInstr Where to store the instruction length of the hypercall + * instruction. Optional, can be NULL. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(VBOXSTRICTRC) gimKvmXcptUD(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr) +{ + VMCPU_ASSERT_EMT(pVCpu); + + /* + * If we didn't ask for #UD to be trapped, bail. + */ + if (RT_UNLIKELY(!pVM->gim.s.u.Kvm.fTrapXcptUD)) + return VERR_GIM_IPE_3; + + if (!pDis) + { + unsigned cbInstr; + DISCPUSTATE Dis; + int rc = EMInterpretDisasCurrent(pVCpu, &Dis, &cbInstr); + if (RT_SUCCESS(rc)) + { + if (pcbInstr) + *pcbInstr = (uint8_t)cbInstr; + return gimKvmHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr); + } + + Log(("GIM: KVM: Failed to disassemble instruction at CS:RIP=%04x:%08RX64. rc=%Rrc\n", pCtx->cs.Sel, pCtx->rip, rc)); + return rc; + } + + return gimKvmHypercallEx(pVCpu, pCtx, pDis->pCurInstr->uOpcode, pDis->cbInstr); +} + diff --git a/src/VBox/VMM/VMMAll/HMAll.cpp b/src/VBox/VMM/VMMAll/HMAll.cpp new file mode 100644 index 00000000..1d2f4170 --- /dev/null +++ b/src/VBox/VMM/VMMAll/HMAll.cpp @@ -0,0 +1,904 @@ +/* $Id: HMAll.cpp $ */ +/** @file + * HM - All contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_HM +#define VMCPU_INCL_CPUM_GST_CTX +#include +#include +#include "HMInternal.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +#define EXIT_REASON(a_Def, a_Val, a_Str) #a_Def " - " #a_Val " - " a_Str +#define EXIT_REASON_NIL() NULL + +/** Exit reason descriptions for VT-x, used to describe statistics and exit + * history. */ +static const char * const g_apszVmxExitReasons[MAX_EXITREASON_STAT] = +{ + EXIT_REASON(VMX_EXIT_XCPT_OR_NMI , 0, "Exception or non-maskable interrupt (NMI)."), + EXIT_REASON(VMX_EXIT_EXT_INT , 1, "External interrupt."), + EXIT_REASON(VMX_EXIT_TRIPLE_FAULT , 2, "Triple fault."), + EXIT_REASON(VMX_EXIT_INIT_SIGNAL , 3, "INIT signal."), + EXIT_REASON(VMX_EXIT_SIPI , 4, "Start-up IPI (SIPI)."), + EXIT_REASON(VMX_EXIT_IO_SMI_IRQ , 5, "I/O system-management interrupt (SMI)."), + EXIT_REASON(VMX_EXIT_SMI_IRQ , 6, "Other SMI."), + EXIT_REASON(VMX_EXIT_INT_WINDOW , 7, "Interrupt window."), + EXIT_REASON(VMX_EXIT_NMI_WINDOW , 8, "NMI window."), + EXIT_REASON(VMX_EXIT_TASK_SWITCH , 9, "Task switch."), + EXIT_REASON(VMX_EXIT_CPUID , 10, "CPUID instruction."), + EXIT_REASON(VMX_EXIT_GETSEC , 11, "GETSEC instruction."), + EXIT_REASON(VMX_EXIT_HLT , 12, "HLT instruction."), + EXIT_REASON(VMX_EXIT_INVD , 13, "INVD instruction."), + EXIT_REASON(VMX_EXIT_INVLPG , 14, "INVLPG instruction."), + EXIT_REASON(VMX_EXIT_RDPMC , 15, "RDPMC instruction."), + EXIT_REASON(VMX_EXIT_RDTSC , 16, "RDTSC instruction."), + EXIT_REASON(VMX_EXIT_RSM , 17, "RSM instruction in SMM."), + EXIT_REASON(VMX_EXIT_VMCALL , 18, "VMCALL instruction."), + EXIT_REASON(VMX_EXIT_VMCLEAR , 19, "VMCLEAR instruction."), + EXIT_REASON(VMX_EXIT_VMLAUNCH , 20, "VMLAUNCH instruction."), + EXIT_REASON(VMX_EXIT_VMPTRLD , 21, "VMPTRLD instruction."), + EXIT_REASON(VMX_EXIT_VMPTRST , 22, "VMPTRST instruction."), + EXIT_REASON(VMX_EXIT_VMREAD , 23, "VMREAD instruction."), + EXIT_REASON(VMX_EXIT_VMRESUME , 24, "VMRESUME instruction."), + EXIT_REASON(VMX_EXIT_VMWRITE , 25, "VMWRITE instruction."), + EXIT_REASON(VMX_EXIT_VMXOFF , 26, "VMXOFF instruction."), + EXIT_REASON(VMX_EXIT_VMXON , 27, "VMXON instruction."), + EXIT_REASON(VMX_EXIT_MOV_CRX , 28, "Control-register accesses."), + EXIT_REASON(VMX_EXIT_MOV_DRX , 29, "Debug-register accesses."), + EXIT_REASON(VMX_EXIT_PORT_IO , 30, "I/O instruction."), + EXIT_REASON(VMX_EXIT_RDMSR , 31, "RDMSR instruction."), + EXIT_REASON(VMX_EXIT_WRMSR , 32, "WRMSR instruction."), + EXIT_REASON(VMX_EXIT_ERR_INVALID_GUEST_STATE, 33, "VM-entry failure due to invalid guest state."), + EXIT_REASON(VMX_EXIT_ERR_MSR_LOAD , 34, "VM-entry failure due to MSR loading."), + EXIT_REASON_NIL(), + EXIT_REASON(VMX_EXIT_MWAIT , 36, "MWAIT instruction."), + EXIT_REASON(VMX_EXIT_MTF , 37, "Monitor Trap Flag."), + EXIT_REASON_NIL(), + EXIT_REASON(VMX_EXIT_MONITOR , 39, "MONITOR instruction."), + EXIT_REASON(VMX_EXIT_PAUSE , 40, "PAUSE instruction."), + EXIT_REASON(VMX_EXIT_ERR_MACHINE_CHECK , 41, "VM-entry failure due to machine-check."), + EXIT_REASON_NIL(), + EXIT_REASON(VMX_EXIT_TPR_BELOW_THRESHOLD , 43, "TPR below threshold (MOV to CR8)."), + EXIT_REASON(VMX_EXIT_APIC_ACCESS , 44, "APIC access."), + EXIT_REASON(VMX_EXIT_VIRTUALIZED_EOI , 45, "Virtualized EOI."), + EXIT_REASON(VMX_EXIT_GDTR_IDTR_ACCESS , 46, "GDTR/IDTR access using LGDT/SGDT/LIDT/SIDT."), + EXIT_REASON(VMX_EXIT_LDTR_TR_ACCESS , 47, "LDTR/TR access using LLDT/SLDT/LTR/STR."), + EXIT_REASON(VMX_EXIT_EPT_VIOLATION , 48, "EPT violation."), + EXIT_REASON(VMX_EXIT_EPT_MISCONFIG , 49, "EPT misconfiguration."), + EXIT_REASON(VMX_EXIT_INVEPT , 50, "INVEPT instruction."), + EXIT_REASON(VMX_EXIT_RDTSCP , 51, "RDTSCP instruction."), + EXIT_REASON(VMX_EXIT_PREEMPT_TIMER , 52, "VMX-preemption timer expired."), + EXIT_REASON(VMX_EXIT_INVVPID , 53, "INVVPID instruction."), + EXIT_REASON(VMX_EXIT_WBINVD , 54, "WBINVD instruction."), + EXIT_REASON(VMX_EXIT_XSETBV , 55, "XSETBV instruction."), + EXIT_REASON(VMX_EXIT_APIC_WRITE , 56, "APIC write completed to virtual-APIC page."), + EXIT_REASON(VMX_EXIT_RDRAND , 57, "RDRAND instruction."), + EXIT_REASON(VMX_EXIT_INVPCID , 58, "INVPCID instruction."), + EXIT_REASON(VMX_EXIT_VMFUNC , 59, "VMFUNC instruction."), + EXIT_REASON(VMX_EXIT_ENCLS , 60, "ENCLS instruction."), + EXIT_REASON(VMX_EXIT_RDSEED , 61, "RDSEED instruction."), + EXIT_REASON(VMX_EXIT_PML_FULL , 62, "Page-modification log full."), + EXIT_REASON(VMX_EXIT_XSAVES , 63, "XSAVES instruction."), + EXIT_REASON(VMX_EXIT_XRSTORS , 64, "XRSTORS instruction."), + EXIT_REASON_NIL(), + EXIT_REASON(VMX_EXIT_SPP_EVENT , 66, "SPP-related event."), + EXIT_REASON(VMX_EXIT_UMWAIT , 67, "UMWAIT instruction."), + EXIT_REASON(VMX_EXIT_TPAUSE , 68, "TPAUSE instruction.") +}; +/** Array index of the last valid VT-x exit reason. */ +#define MAX_EXITREASON_VTX 68 + +/** A partial list of \#EXIT reason descriptions for AMD-V, used to describe + * statistics and exit history. + * + * @note AMD-V have annoyingly large gaps (e.g. \#NPF VMEXIT comes at 1024), + * this array doesn't contain the entire set of exit reasons, we + * handle them via hmSvmGetSpecialExitReasonDesc(). */ +static const char * const g_apszSvmExitReasons[MAX_EXITREASON_STAT] = +{ + EXIT_REASON(SVM_EXIT_READ_CR0 , 0, "Read CR0."), + EXIT_REASON(SVM_EXIT_READ_CR1 , 1, "Read CR1."), + EXIT_REASON(SVM_EXIT_READ_CR2 , 2, "Read CR2."), + EXIT_REASON(SVM_EXIT_READ_CR3 , 3, "Read CR3."), + EXIT_REASON(SVM_EXIT_READ_CR4 , 4, "Read CR4."), + EXIT_REASON(SVM_EXIT_READ_CR5 , 5, "Read CR5."), + EXIT_REASON(SVM_EXIT_READ_CR6 , 6, "Read CR6."), + EXIT_REASON(SVM_EXIT_READ_CR7 , 7, "Read CR7."), + EXIT_REASON(SVM_EXIT_READ_CR8 , 8, "Read CR8."), + EXIT_REASON(SVM_EXIT_READ_CR9 , 9, "Read CR9."), + EXIT_REASON(SVM_EXIT_READ_CR10 , 10, "Read CR10."), + EXIT_REASON(SVM_EXIT_READ_CR11 , 11, "Read CR11."), + EXIT_REASON(SVM_EXIT_READ_CR12 , 12, "Read CR12."), + EXIT_REASON(SVM_EXIT_READ_CR13 , 13, "Read CR13."), + EXIT_REASON(SVM_EXIT_READ_CR14 , 14, "Read CR14."), + EXIT_REASON(SVM_EXIT_READ_CR15 , 15, "Read CR15."), + EXIT_REASON(SVM_EXIT_WRITE_CR0 , 16, "Write CR0."), + EXIT_REASON(SVM_EXIT_WRITE_CR1 , 17, "Write CR1."), + EXIT_REASON(SVM_EXIT_WRITE_CR2 , 18, "Write CR2."), + EXIT_REASON(SVM_EXIT_WRITE_CR3 , 19, "Write CR3."), + EXIT_REASON(SVM_EXIT_WRITE_CR4 , 20, "Write CR4."), + EXIT_REASON(SVM_EXIT_WRITE_CR5 , 21, "Write CR5."), + EXIT_REASON(SVM_EXIT_WRITE_CR6 , 22, "Write CR6."), + EXIT_REASON(SVM_EXIT_WRITE_CR7 , 23, "Write CR7."), + EXIT_REASON(SVM_EXIT_WRITE_CR8 , 24, "Write CR8."), + EXIT_REASON(SVM_EXIT_WRITE_CR9 , 25, "Write CR9."), + EXIT_REASON(SVM_EXIT_WRITE_CR10 , 26, "Write CR10."), + EXIT_REASON(SVM_EXIT_WRITE_CR11 , 27, "Write CR11."), + EXIT_REASON(SVM_EXIT_WRITE_CR12 , 28, "Write CR12."), + EXIT_REASON(SVM_EXIT_WRITE_CR13 , 29, "Write CR13."), + EXIT_REASON(SVM_EXIT_WRITE_CR14 , 30, "Write CR14."), + EXIT_REASON(SVM_EXIT_WRITE_CR15 , 31, "Write CR15."), + EXIT_REASON(SVM_EXIT_READ_DR0 , 32, "Read DR0."), + EXIT_REASON(SVM_EXIT_READ_DR1 , 33, "Read DR1."), + EXIT_REASON(SVM_EXIT_READ_DR2 , 34, "Read DR2."), + EXIT_REASON(SVM_EXIT_READ_DR3 , 35, "Read DR3."), + EXIT_REASON(SVM_EXIT_READ_DR4 , 36, "Read DR4."), + EXIT_REASON(SVM_EXIT_READ_DR5 , 37, "Read DR5."), + EXIT_REASON(SVM_EXIT_READ_DR6 , 38, "Read DR6."), + EXIT_REASON(SVM_EXIT_READ_DR7 , 39, "Read DR7."), + EXIT_REASON(SVM_EXIT_READ_DR8 , 40, "Read DR8."), + EXIT_REASON(SVM_EXIT_READ_DR9 , 41, "Read DR9."), + EXIT_REASON(SVM_EXIT_READ_DR10 , 42, "Read DR10."), + EXIT_REASON(SVM_EXIT_READ_DR11 , 43, "Read DR11"), + EXIT_REASON(SVM_EXIT_READ_DR12 , 44, "Read DR12."), + EXIT_REASON(SVM_EXIT_READ_DR13 , 45, "Read DR13."), + EXIT_REASON(SVM_EXIT_READ_DR14 , 46, "Read DR14."), + EXIT_REASON(SVM_EXIT_READ_DR15 , 47, "Read DR15."), + EXIT_REASON(SVM_EXIT_WRITE_DR0 , 48, "Write DR0."), + EXIT_REASON(SVM_EXIT_WRITE_DR1 , 49, "Write DR1."), + EXIT_REASON(SVM_EXIT_WRITE_DR2 , 50, "Write DR2."), + EXIT_REASON(SVM_EXIT_WRITE_DR3 , 51, "Write DR3."), + EXIT_REASON(SVM_EXIT_WRITE_DR4 , 52, "Write DR4."), + EXIT_REASON(SVM_EXIT_WRITE_DR5 , 53, "Write DR5."), + EXIT_REASON(SVM_EXIT_WRITE_DR6 , 54, "Write DR6."), + EXIT_REASON(SVM_EXIT_WRITE_DR7 , 55, "Write DR7."), + EXIT_REASON(SVM_EXIT_WRITE_DR8 , 56, "Write DR8."), + EXIT_REASON(SVM_EXIT_WRITE_DR9 , 57, "Write DR9."), + EXIT_REASON(SVM_EXIT_WRITE_DR10 , 58, "Write DR10."), + EXIT_REASON(SVM_EXIT_WRITE_DR11 , 59, "Write DR11."), + EXIT_REASON(SVM_EXIT_WRITE_DR12 , 60, "Write DR12."), + EXIT_REASON(SVM_EXIT_WRITE_DR13 , 61, "Write DR13."), + EXIT_REASON(SVM_EXIT_WRITE_DR14 , 62, "Write DR14."), + EXIT_REASON(SVM_EXIT_WRITE_DR15 , 63, "Write DR15."), + EXIT_REASON(SVM_EXIT_XCPT_0 , 64, "Exception 0 (#DE)."), + EXIT_REASON(SVM_EXIT_XCPT_1 , 65, "Exception 1 (#DB)."), + EXIT_REASON(SVM_EXIT_XCPT_2 , 66, "Exception 2 (#NMI)."), + EXIT_REASON(SVM_EXIT_XCPT_3 , 67, "Exception 3 (#BP)."), + EXIT_REASON(SVM_EXIT_XCPT_4 , 68, "Exception 4 (#OF)."), + EXIT_REASON(SVM_EXIT_XCPT_5 , 69, "Exception 5 (#BR)."), + EXIT_REASON(SVM_EXIT_XCPT_6 , 70, "Exception 6 (#UD)."), + EXIT_REASON(SVM_EXIT_XCPT_7 , 71, "Exception 7 (#NM)."), + EXIT_REASON(SVM_EXIT_XCPT_8 , 72, "Exception 8 (#DF)."), + EXIT_REASON(SVM_EXIT_XCPT_9 , 73, "Exception 9 (#CO_SEG_OVERRUN)."), + EXIT_REASON(SVM_EXIT_XCPT_10 , 74, "Exception 10 (#TS)."), + EXIT_REASON(SVM_EXIT_XCPT_11 , 75, "Exception 11 (#NP)."), + EXIT_REASON(SVM_EXIT_XCPT_12 , 76, "Exception 12 (#SS)."), + EXIT_REASON(SVM_EXIT_XCPT_13 , 77, "Exception 13 (#GP)."), + EXIT_REASON(SVM_EXIT_XCPT_14 , 78, "Exception 14 (#PF)."), + EXIT_REASON(SVM_EXIT_XCPT_15 , 79, "Exception 15 (0x0f)."), + EXIT_REASON(SVM_EXIT_XCPT_16 , 80, "Exception 16 (#MF)."), + EXIT_REASON(SVM_EXIT_XCPT_17 , 81, "Exception 17 (#AC)."), + EXIT_REASON(SVM_EXIT_XCPT_18 , 82, "Exception 18 (#MC)."), + EXIT_REASON(SVM_EXIT_XCPT_19 , 83, "Exception 19 (#XF)."), + EXIT_REASON(SVM_EXIT_XCPT_20 , 84, "Exception 20 (#VE)."), + EXIT_REASON(SVM_EXIT_XCPT_21 , 85, "Exception 22 (0x15)."), + EXIT_REASON(SVM_EXIT_XCPT_22 , 86, "Exception 22 (0x16)."), + EXIT_REASON(SVM_EXIT_XCPT_23 , 87, "Exception 23 (0x17)."), + EXIT_REASON(SVM_EXIT_XCPT_24 , 88, "Exception 24 (0x18)."), + EXIT_REASON(SVM_EXIT_XCPT_25 , 89, "Exception 25 (0x19)."), + EXIT_REASON(SVM_EXIT_XCPT_26 , 90, "Exception 26 (0x1a)."), + EXIT_REASON(SVM_EXIT_XCPT_27 , 91, "Exception 27 (0x1b)."), + EXIT_REASON(SVM_EXIT_XCPT_28 , 92, "Exception 28 (0x1c)."), + EXIT_REASON(SVM_EXIT_XCPT_29 , 93, "Exception 29 (0x1d)."), + EXIT_REASON(SVM_EXIT_XCPT_30 , 94, "Exception 30 (#SX)."), + EXIT_REASON(SVM_EXIT_XCPT_31 , 95, "Exception 31 (0x1F)."), + EXIT_REASON(SVM_EXIT_INTR , 96, "Physical maskable interrupt (host)."), + EXIT_REASON(SVM_EXIT_NMI , 97, "Physical non-maskable interrupt (host)."), + EXIT_REASON(SVM_EXIT_SMI , 98, "System management interrupt (host)."), + EXIT_REASON(SVM_EXIT_INIT , 99, "Physical INIT signal (host)."), + EXIT_REASON(SVM_EXIT_VINTR , 100, "Virtual interrupt-window exit."), + EXIT_REASON(SVM_EXIT_CR0_SEL_WRITE , 101, "Selective CR0 Write (to bits other than CR0.TS and CR0.MP)."), + EXIT_REASON(SVM_EXIT_IDTR_READ , 102, "Read IDTR."), + EXIT_REASON(SVM_EXIT_GDTR_READ , 103, "Read GDTR."), + EXIT_REASON(SVM_EXIT_LDTR_READ , 104, "Read LDTR."), + EXIT_REASON(SVM_EXIT_TR_READ , 105, "Read TR."), + EXIT_REASON(SVM_EXIT_IDTR_WRITE , 106, "Write IDTR."), + EXIT_REASON(SVM_EXIT_GDTR_WRITE , 107, "Write GDTR."), + EXIT_REASON(SVM_EXIT_LDTR_WRITE , 108, "Write LDTR."), + EXIT_REASON(SVM_EXIT_TR_WRITE , 109, "Write TR."), + EXIT_REASON(SVM_EXIT_RDTSC , 110, "RDTSC instruction."), + EXIT_REASON(SVM_EXIT_RDPMC , 111, "RDPMC instruction."), + EXIT_REASON(SVM_EXIT_PUSHF , 112, "PUSHF instruction."), + EXIT_REASON(SVM_EXIT_POPF , 113, "POPF instruction."), + EXIT_REASON(SVM_EXIT_CPUID , 114, "CPUID instruction."), + EXIT_REASON(SVM_EXIT_RSM , 115, "RSM instruction."), + EXIT_REASON(SVM_EXIT_IRET , 116, "IRET instruction."), + EXIT_REASON(SVM_EXIT_SWINT , 117, "Software interrupt (INTn instructions)."), + EXIT_REASON(SVM_EXIT_INVD , 118, "INVD instruction."), + EXIT_REASON(SVM_EXIT_PAUSE , 119, "PAUSE instruction."), + EXIT_REASON(SVM_EXIT_HLT , 120, "HLT instruction."), + EXIT_REASON(SVM_EXIT_INVLPG , 121, "INVLPG instruction."), + EXIT_REASON(SVM_EXIT_INVLPGA , 122, "INVLPGA instruction."), + EXIT_REASON(SVM_EXIT_IOIO , 123, "IN/OUT/INS/OUTS instruction."), + EXIT_REASON(SVM_EXIT_MSR , 124, "RDMSR or WRMSR access to protected MSR."), + EXIT_REASON(SVM_EXIT_TASK_SWITCH , 125, "Task switch."), + EXIT_REASON(SVM_EXIT_FERR_FREEZE , 126, "FERR Freeze; CPU frozen in an x87/mmx instruction waiting for interrupt."), + EXIT_REASON(SVM_EXIT_SHUTDOWN , 127, "Shutdown."), + EXIT_REASON(SVM_EXIT_VMRUN , 128, "VMRUN instruction."), + EXIT_REASON(SVM_EXIT_VMMCALL , 129, "VMCALL instruction."), + EXIT_REASON(SVM_EXIT_VMLOAD , 130, "VMLOAD instruction."), + EXIT_REASON(SVM_EXIT_VMSAVE , 131, "VMSAVE instruction."), + EXIT_REASON(SVM_EXIT_STGI , 132, "STGI instruction."), + EXIT_REASON(SVM_EXIT_CLGI , 133, "CLGI instruction."), + EXIT_REASON(SVM_EXIT_SKINIT , 134, "SKINIT instruction."), + EXIT_REASON(SVM_EXIT_RDTSCP , 135, "RDTSCP instruction."), + EXIT_REASON(SVM_EXIT_ICEBP , 136, "ICEBP instruction."), + EXIT_REASON(SVM_EXIT_WBINVD , 137, "WBINVD instruction."), + EXIT_REASON(SVM_EXIT_MONITOR , 138, "MONITOR instruction."), + EXIT_REASON(SVM_EXIT_MWAIT , 139, "MWAIT instruction."), + EXIT_REASON(SVM_EXIT_MWAIT_ARMED , 140, "MWAIT instruction when armed."), + EXIT_REASON(SVM_EXIT_XSETBV , 141, "XSETBV instruction."), + EXIT_REASON(SVM_EXIT_RDPRU , 142, "RDPRU instruction."), + EXIT_REASON(SVM_EXIT_WRITE_EFER_TRAP, 143, "Write EFER (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR0_TRAP , 144, "Write CR0 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR1_TRAP , 145, "Write CR1 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR2_TRAP , 146, "Write CR2 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR3_TRAP , 147, "Write CR3 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR4_TRAP , 148, "Write CR4 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR5_TRAP , 149, "Write CR5 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR6_TRAP , 150, "Write CR6 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR7_TRAP , 151, "Write CR7 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR8_TRAP , 152, "Write CR8 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR9_TRAP , 153, "Write CR9 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR10_TRAP, 154, "Write CR10 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR11_TRAP, 155, "Write CR11 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR12_TRAP, 156, "Write CR12 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR13_TRAP, 157, "Write CR13 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR14_TRAP, 158, "Write CR14 (trap-like)."), + EXIT_REASON(SVM_EXIT_WRITE_CR15_TRAP, 159, "Write CR15 (trap-like)."), + EXIT_REASON_NIL() , + EXIT_REASON_NIL() , + EXIT_REASON_NIL() , + EXIT_REASON(SVM_EXIT_MCOMMIT , 163, "MCOMMIT instruction."), +}; +/** Array index of the last valid AMD-V exit reason. */ +#define MAX_EXITREASON_AMDV 163 + +/** Special exit reasons not covered in the array above. */ +#define SVM_EXIT_REASON_NPF EXIT_REASON(SVM_EXIT_NPF , 1024, "Nested Page Fault.") +#define SVM_EXIT_REASON_AVIC_INCOMPLETE_IPI EXIT_REASON(SVM_EXIT_AVIC_INCOMPLETE_IPI, 1025, "AVIC - Incomplete IPI delivery.") +#define SVM_EXIT_REASON_AVIC_NOACCEL EXIT_REASON(SVM_EXIT_AVIC_NOACCEL , 1026, "AVIC - Unhandled register.") + +/** + * Gets the SVM exit reason if it's one of the reasons not present in the @c + * g_apszSvmExitReasons array. + * + * @returns The exit reason or NULL if unknown. + * @param uExit The exit. + */ +DECLINLINE(const char *) hmSvmGetSpecialExitReasonDesc(uint16_t uExit) +{ + switch (uExit) + { + case SVM_EXIT_NPF: return SVM_EXIT_REASON_NPF; + case SVM_EXIT_AVIC_INCOMPLETE_IPI: return SVM_EXIT_REASON_AVIC_INCOMPLETE_IPI; + case SVM_EXIT_AVIC_NOACCEL: return SVM_EXIT_REASON_AVIC_NOACCEL; + } + return EXIT_REASON_NIL(); +} +#undef EXIT_REASON_NIL +#undef EXIT_REASON + + +/** + * Checks whether HM (VT-x/AMD-V) is being used by this VM. + * + * @retval true if used. + * @retval false if software virtualization (raw-mode) is used. + * @param pVM The cross context VM structure. + * @sa HMIsEnabled, HMR3IsEnabled + * @internal + */ +VMMDECL(bool) HMIsEnabledNotMacro(PVM pVM) +{ + Assert(pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NOT_SET); + return pVM->fHMEnabled; +} + + +/** + * Checks if the guest is in a suitable state for hardware-assisted execution. + * + * @returns @c true if it is suitable, @c false otherwise. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest CPU context. + * + * @remarks @a pCtx can be a partial context created and not necessarily the same as + * pVCpu->cpum.GstCtx. + */ +VMMDECL(bool) HMCanExecuteGuest(PVMCC pVM, PVMCPUCC pVCpu, PCCPUMCTX pCtx) +{ + Assert(HMIsEnabled(pVM)); + +#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM + if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx) + || CPUMIsGuestInVmxNonRootMode(pCtx)) + { + LogFunc(("In nested-guest mode - returning false")); + return false; + } +#endif + + /* AMD-V supports real & protected mode with or without paging. */ + if (pVM->hm.s.svm.fEnabled) + { + pVCpu->hm.s.fActive = true; + return true; + } + + bool rc = HMCanExecuteVmxGuest(pVM, pVCpu, pCtx); + LogFlowFunc(("returning %RTbool\n", rc)); + return rc; +} + + +/** + * Queues a guest page for invalidation. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCVirt Page to invalidate. + */ +static void hmQueueInvlPage(PVMCPU pVCpu, RTGCPTR GCVirt) +{ + /* Nothing to do if a TLB flush is already pending */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH)) + return; + VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH); + NOREF(GCVirt); +} + + +/** + * Invalidates a guest page. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCVirt Page to invalidate. + */ +VMM_INT_DECL(int) HMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt) +{ + STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushPageManual); +#ifdef IN_RING0 + return HMR0InvalidatePage(pVCpu, GCVirt); +#else + hmQueueInvlPage(pVCpu, GCVirt); + return VINF_SUCCESS; +#endif +} + + +#ifdef IN_RING0 + +/** + * Dummy RTMpOnSpecific handler since RTMpPokeCpu couldn't be used. + * + */ +static DECLCALLBACK(void) hmFlushHandler(RTCPUID idCpu, void *pvUser1, void *pvUser2) +{ + NOREF(idCpu); NOREF(pvUser1); NOREF(pvUser2); + return; +} + + +/** + * Wrapper for RTMpPokeCpu to deal with VERR_NOT_SUPPORTED. + */ +static void hmR0PokeCpu(PVMCPUCC pVCpu, RTCPUID idHostCpu) +{ + uint32_t cWorldSwitchExits = ASMAtomicUoReadU32(&pVCpu->hmr0.s.cWorldSwitchExits); + + STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatPoke, x); + int rc = RTMpPokeCpu(idHostCpu); + STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPoke, x); + + /* Not implemented on some platforms (Darwin, Linux kernel < 2.6.19); fall + back to a less efficient implementation (broadcast). */ + if (rc == VERR_NOT_SUPPORTED) + { + STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatSpinPoke, z); + /* synchronous. */ + RTMpOnSpecific(idHostCpu, hmFlushHandler, 0, 0); + STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatSpinPoke, z); + } + else + { + if (rc == VINF_SUCCESS) + STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatSpinPoke, z); + else + STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatSpinPokeFailed, z); + +/** @todo If more than one CPU is going to be poked, we could optimize this + * operation by poking them first and wait afterwards. Would require + * recording who to poke and their current cWorldSwitchExits values, + * that's something not suitable for stack... So, pVCpu->hm.s.something + * then. */ + /* Spin until the VCPU has switched back (poking is async). */ + while ( ASMAtomicUoReadBool(&pVCpu->hm.s.fCheckedTLBFlush) + && cWorldSwitchExits == ASMAtomicUoReadU32(&pVCpu->hmr0.s.cWorldSwitchExits)) + ASMNopPause(); + + if (rc == VINF_SUCCESS) + STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatSpinPoke, z); + else + STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatSpinPokeFailed, z); + } +} + +#endif /* IN_RING0 */ + +/** + * Flushes the guest TLB. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(int) HMFlushTlb(PVMCPU pVCpu) +{ + VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH); + STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbManual); + return VINF_SUCCESS; +} + + +/** + * Poke an EMT so it can perform the appropriate TLB shootdowns. + * + * @param pVCpu The cross context virtual CPU structure of the + * EMT poke. + * @param fAccountFlushStat Whether to account the call to + * StatTlbShootdownFlush or StatTlbShootdown. + */ +static void hmPokeCpuForTlbFlush(PVMCPUCC pVCpu, bool fAccountFlushStat) +{ + if (ASMAtomicUoReadBool(&pVCpu->hm.s.fCheckedTLBFlush)) + { + if (fAccountFlushStat) + STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdownFlush); + else + STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown); +#ifdef IN_RING0 + RTCPUID idHostCpu = pVCpu->hmr0.s.idEnteredCpu; + if (idHostCpu != NIL_RTCPUID) + hmR0PokeCpu(pVCpu, idHostCpu); +#else + VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_POKE); +#endif + } + else + STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushPageManual); +} + + +/** + * Invalidates a guest page on all VCPUs. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCVirt Page to invalidate. + */ +VMM_INT_DECL(int) HMInvalidatePageOnAllVCpus(PVMCC pVM, RTGCPTR GCVirt) +{ + /* + * The VT-x/AMD-V code will be flushing TLB each time a VCPU migrates to a different + * host CPU, see hmR0VmxFlushTaggedTlbBoth() and hmR0SvmFlushTaggedTlb(). + * + * This is the reason why we do not care about thread preemption here and just + * execute HMInvalidatePage() assuming it might be the 'right' CPU. + */ + VMCPUID const idCurCpu = VMMGetCpuId(pVM); + STAM_COUNTER_INC(&VMCC_GET_CPU(pVM, idCurCpu)->hm.s.StatFlushPage); + + for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++) + { + PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu); + + /* Nothing to do if a TLB flush is already pending; the VCPU should + have already been poked if it were active. */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH)) + continue; + + if (pVCpu->idCpu == idCurCpu) + HMInvalidatePage(pVCpu, GCVirt); + else + { + hmQueueInvlPage(pVCpu, GCVirt); + hmPokeCpuForTlbFlush(pVCpu, false /* fAccountFlushStat */); + } + } + + return VINF_SUCCESS; +} + + +/** + * Flush the TLBs of all VCPUs. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(int) HMFlushTlbOnAllVCpus(PVMCC pVM) +{ + if (pVM->cCpus == 1) + return HMFlushTlb(VMCC_GET_CPU_0(pVM)); + + VMCPUID const idThisCpu = VMMGetCpuId(pVM); + + STAM_COUNTER_INC(&VMCC_GET_CPU(pVM, idThisCpu)->hm.s.StatFlushTlb); + + for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++) + { + PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu); + + /* Nothing to do if a TLB flush is already pending; the VCPU should + have already been poked if it were active. */ + if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH)) + { + VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH); + if (idThisCpu != idCpu) + hmPokeCpuForTlbFlush(pVCpu, true /* fAccountFlushStat */); + } + } + + return VINF_SUCCESS; +} + + +/** + * Invalidates a guest page by physical address. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys Page to invalidate. + * + * @remarks Assumes the current instruction references this physical page + * though a virtual address! + */ +VMM_INT_DECL(int) HMInvalidatePhysPage(PVMCC pVM, RTGCPHYS GCPhys) +{ + if (!HMIsNestedPagingActive(pVM)) + return VINF_SUCCESS; + + /* + * AMD-V: Doesn't support invalidation with guest physical addresses. + * + * VT-x: Doesn't support invalidation with guest physical addresses. + * INVVPID instruction takes only a linear address while invept only flushes by EPT + * not individual addresses. + * + * We update the force flag and flush before the next VM-entry, see @bugref{6568}. + */ + RT_NOREF(GCPhys); + /** @todo Remove or figure out to way to update the Phys STAT counter. */ + /* STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys); */ + return HMFlushTlbOnAllVCpus(pVM); +} + + +/** + * Checks if nested paging is enabled. + * + * @returns true if nested paging is active, false otherwise. + * @param pVM The cross context VM structure. + * + * @remarks Works before hmR3InitFinalizeR0. + */ +VMM_INT_DECL(bool) HMIsNestedPagingActive(PVMCC pVM) +{ + return HMIsEnabled(pVM) && CTX_EXPR(pVM->hm.s.fNestedPagingCfg, pVM->hmr0.s.fNestedPaging, RT_NOTHING); +} + + +/** + * Checks if both nested paging and unhampered guest execution are enabled. + * + * The almost complete guest execution in hardware is only applicable to VT-x. + * + * @returns true if we have both enabled, otherwise false. + * @param pVM The cross context VM structure. + * + * @remarks Works before hmR3InitFinalizeR0. + */ +VMM_INT_DECL(bool) HMAreNestedPagingAndFullGuestExecEnabled(PVMCC pVM) +{ + return HMIsEnabled(pVM) + && CTX_EXPR(pVM->hm.s.fNestedPagingCfg, pVM->hmr0.s.fNestedPaging, RT_NOTHING) + && ( CTX_EXPR(pVM->hm.s.vmx.fUnrestrictedGuestCfg, pVM->hmr0.s.vmx.fUnrestrictedGuest, RT_NOTHING) + || pVM->hm.s.svm.fSupported); +} + + +/** + * Checks if this VM is using HM and is long-mode capable. + * + * Use VMR3IsLongModeAllowed() instead of this, when possible. + * + * @returns true if long mode is allowed, false otherwise. + * @param pVM The cross context VM structure. + * @sa VMR3IsLongModeAllowed, NEMHCIsLongModeAllowed + */ +VMM_INT_DECL(bool) HMIsLongModeAllowed(PVMCC pVM) +{ + return HMIsEnabled(pVM) && CTX_EXPR(pVM->hm.s.fAllow64BitGuestsCfg, pVM->hmr0.s.fAllow64BitGuests, RT_NOTHING); +} + + +/** + * Checks if MSR bitmaps are active. It is assumed that when it's available + * it will be used as well. + * + * @returns true if MSR bitmaps are available, false otherwise. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) HMIsMsrBitmapActive(PVM pVM) +{ + if (HMIsEnabled(pVM)) + { + if (pVM->hm.s.svm.fSupported) + return true; + + if ( pVM->hm.s.vmx.fSupported + && ( CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.ProcCtls.n.allowed1, g_HmMsrs.u.vmx.ProcCtls.n.allowed1, RT_NOTHING) + & VMX_PROC_CTLS_USE_MSR_BITMAPS)) + return true; + } + return false; +} + + +/** + * Checks if AMD-V is active. + * + * @returns true if AMD-V is active. + * @param pVM The cross context VM structure. + * + * @remarks Works before hmR3InitFinalizeR0. + */ +VMM_INT_DECL(bool) HMIsSvmActive(PVM pVM) +{ + return pVM->hm.s.svm.fSupported && HMIsEnabled(pVM); +} + + +/** + * Checks if VT-x is active. + * + * @returns true if VT-x is active. + * @param pVM The cross context VM structure. + * + * @remarks Works before hmR3InitFinalizeR0. + */ +VMM_INT_DECL(bool) HMIsVmxActive(PVM pVM) +{ + return pVM->hm.s.vmx.fSupported && HMIsEnabled(pVM); +} + + +/** + * Checks if an interrupt event is currently pending. + * + * @returns Interrupt event pending state. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) HMHasPendingIrq(PVMCC pVM) +{ + PVMCPUCC pVCpu = VMMGetCpu(pVM); + return !!pVCpu->hm.s.Event.fPending; +} + + +/** + * Sets or clears the single instruction flag. + * + * When set, HM will try its best to return to ring-3 after executing a single + * instruction. This can be used for debugging. See also + * EMR3HmSingleInstruction. + * + * @returns The old flag state. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param fEnable The new flag state. + */ +VMM_INT_DECL(bool) HMSetSingleInstruction(PVMCC pVM, PVMCPUCC pVCpu, bool fEnable) +{ + VMCPU_ASSERT_EMT(pVCpu); + bool fOld = pVCpu->hm.s.fSingleInstruction; + pVCpu->hm.s.fSingleInstruction = fEnable; + pVCpu->hm.s.fUseDebugLoop = fEnable || pVM->hm.s.fUseDebugLoop; + return fOld; +} + + +/** + * Notification callback which is called whenever there is a chance that a CR3 + * value might have changed. + * + * This is called by PGM. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param enmShadowMode New shadow paging mode. + * @param enmGuestMode New guest paging mode. + */ +VMM_INT_DECL(void) HMHCChangedPagingMode(PVM pVM, PVMCPUCC pVCpu, PGMMODE enmShadowMode, PGMMODE enmGuestMode) +{ +#ifdef IN_RING3 + /* Ignore page mode changes during state loading. */ + if (VMR3GetState(pVM) == VMSTATE_LOADING) + return; +#endif + + pVCpu->hm.s.enmShadowMode = enmShadowMode; + + /* + * If the guest left protected mode VMX execution, we'll have to be + * extra careful if/when the guest switches back to protected mode. + */ + if (enmGuestMode == PGMMODE_REAL) + { + PVMXVMCSINFOSHARED pVmcsInfoShared = hmGetVmxActiveVmcsInfoShared(pVCpu); + pVmcsInfoShared->fWasInRealMode = true; + } + +#ifdef IN_RING0 + /* + * We need to tickle SVM and VT-x state updates. + * + * Note! We could probably reduce this depending on what exactly changed. + */ + if (VM_IS_HM_ENABLED(pVM)) + { + CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER); /* No recursion! */ + uint64_t fChanged = HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_CR3 | HM_CHANGED_GUEST_CR4 | HM_CHANGED_GUEST_EFER_MSR; + if (pVM->hm.s.svm.fSupported) + fChanged |= HM_CHANGED_SVM_XCPT_INTERCEPTS; + else + fChanged |= HM_CHANGED_VMX_XCPT_INTERCEPTS | HM_CHANGED_VMX_ENTRY_EXIT_CTLS; + ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fChanged); + } +#endif + + Log4(("HMHCChangedPagingMode: Guest paging mode '%s', shadow paging mode '%s'\n", PGMGetModeName(enmGuestMode), + PGMGetModeName(enmShadowMode))); +} + + +/** + * Gets VMX MSRs from the provided hardware-virtualization MSRs struct. + * + * This abstraction exists to insulate the support driver from including VMX + * structures from HM headers. + * + * @param pHwvirtMsrs The hardware-virtualization MSRs. + * @param pVmxMsrs Where to store the VMX MSRs. + */ +VMM_INT_DECL(void) HMGetVmxMsrsFromHwvirtMsrs(PCSUPHWVIRTMSRS pHwvirtMsrs, PVMXMSRS pVmxMsrs) +{ + AssertReturnVoid(pHwvirtMsrs); + AssertReturnVoid(pVmxMsrs); + pVmxMsrs->u64Basic = pHwvirtMsrs->u.vmx.u64Basic; + pVmxMsrs->PinCtls.u = pHwvirtMsrs->u.vmx.PinCtls.u; + pVmxMsrs->ProcCtls.u = pHwvirtMsrs->u.vmx.ProcCtls.u; + pVmxMsrs->ProcCtls2.u = pHwvirtMsrs->u.vmx.ProcCtls2.u; + pVmxMsrs->ExitCtls.u = pHwvirtMsrs->u.vmx.ExitCtls.u; + pVmxMsrs->EntryCtls.u = pHwvirtMsrs->u.vmx.EntryCtls.u; + pVmxMsrs->TruePinCtls.u = pHwvirtMsrs->u.vmx.TruePinCtls.u; + pVmxMsrs->TrueProcCtls.u = pHwvirtMsrs->u.vmx.TrueProcCtls.u; + pVmxMsrs->TrueEntryCtls.u = pHwvirtMsrs->u.vmx.TrueEntryCtls.u; + pVmxMsrs->TrueExitCtls.u = pHwvirtMsrs->u.vmx.TrueExitCtls.u; + pVmxMsrs->u64Misc = pHwvirtMsrs->u.vmx.u64Misc; + pVmxMsrs->u64Cr0Fixed0 = pHwvirtMsrs->u.vmx.u64Cr0Fixed0; + pVmxMsrs->u64Cr0Fixed1 = pHwvirtMsrs->u.vmx.u64Cr0Fixed1; + pVmxMsrs->u64Cr4Fixed0 = pHwvirtMsrs->u.vmx.u64Cr4Fixed0; + pVmxMsrs->u64Cr4Fixed1 = pHwvirtMsrs->u.vmx.u64Cr4Fixed1; + pVmxMsrs->u64VmcsEnum = pHwvirtMsrs->u.vmx.u64VmcsEnum; + pVmxMsrs->u64VmFunc = pHwvirtMsrs->u.vmx.u64VmFunc; + pVmxMsrs->u64EptVpidCaps = pHwvirtMsrs->u.vmx.u64EptVpidCaps; + pVmxMsrs->u64ProcCtls3 = pHwvirtMsrs->u.vmx.u64ProcCtls3; + pVmxMsrs->u64ExitCtls2 = pHwvirtMsrs->u.vmx.u64ExitCtls2; +} + + +/** + * Gets SVM MSRs from the provided hardware-virtualization MSRs struct. + * + * This abstraction exists to insulate the support driver from including SVM + * structures from HM headers. + * + * @param pHwvirtMsrs The hardware-virtualization MSRs. + * @param pSvmMsrs Where to store the SVM MSRs. + */ +VMM_INT_DECL(void) HMGetSvmMsrsFromHwvirtMsrs(PCSUPHWVIRTMSRS pHwvirtMsrs, PSVMMSRS pSvmMsrs) +{ + AssertReturnVoid(pHwvirtMsrs); + AssertReturnVoid(pSvmMsrs); + pSvmMsrs->u64MsrHwcr = pHwvirtMsrs->u.svm.u64MsrHwcr; +} + + +/** + * Gets the name of a VT-x exit code. + * + * @returns Pointer to read only string if @a uExit is known, otherwise NULL. + * @param uExit The VT-x exit to name. + */ +VMM_INT_DECL(const char *) HMGetVmxExitName(uint32_t uExit) +{ + uint16_t const uReason = VMX_EXIT_REASON_BASIC(uExit); + if (uReason <= MAX_EXITREASON_VTX) + { + Assert(uReason < RT_ELEMENTS(g_apszVmxExitReasons)); + return g_apszVmxExitReasons[uReason]; + } + return NULL; +} + + +/** + * Gets the name of an AMD-V exit code. + * + * @returns Pointer to read only string if @a uExit is known, otherwise NULL. + * @param uExit The AMD-V exit to name. + */ +VMM_INT_DECL(const char *) HMGetSvmExitName(uint32_t uExit) +{ + if (uExit <= MAX_EXITREASON_AMDV) + { + Assert(uExit < RT_ELEMENTS(g_apszSvmExitReasons)); + return g_apszSvmExitReasons[uExit]; + } + return hmSvmGetSpecialExitReasonDesc(uExit); +} + diff --git a/src/VBox/VMM/VMMAll/HMSVMAll.cpp b/src/VBox/VMM/VMMAll/HMSVMAll.cpp new file mode 100644 index 00000000..02e66339 --- /dev/null +++ b/src/VBox/VMM/VMMAll/HMSVMAll.cpp @@ -0,0 +1,553 @@ +/* $Id: HMSVMAll.cpp $ */ +/** @file + * HM SVM (AMD-V) - All contexts. + */ + +/* + * Copyright (C) 2017-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_HM +#define VMCPU_INCL_CPUM_GST_CTX +#include "HMInternal.h" +#include +#include +#include +#include + +#include +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +# include /* ASMCpuId */ +#endif + + + +/** + * Emulates a simple MOV TPR (CR8) instruction. + * + * Used for TPR patching on 32-bit guests. This simply looks up the patch record + * at EIP and does the required. + * + * This VMMCALL is used a fallback mechanism when mov to/from cr8 isn't exactly + * like how we want it to be (e.g. not followed by shr 4 as is usually done for + * TPR). See hmR3ReplaceTprInstr() for the details. + * + * @returns VBox status code. + * @retval VINF_SUCCESS if the access was handled successfully, RIP + RFLAGS updated. + * @retval VERR_NOT_FOUND if no patch record for this RIP could be found. + * @retval VERR_SVM_UNEXPECTED_PATCH_TYPE if the found patch type is invalid. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(int) hmEmulateSvmMovTpr(PVMCC pVM, PVMCPUCC pVCpu) +{ + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + Log4(("Emulated VMMCall TPR access replacement at RIP=%RGv\n", pCtx->rip)); + + AssertCompile(DISGREG_EAX == X86_GREG_xAX); + AssertCompile(DISGREG_ECX == X86_GREG_xCX); + AssertCompile(DISGREG_EDX == X86_GREG_xDX); + AssertCompile(DISGREG_EBX == X86_GREG_xBX); + AssertCompile(DISGREG_ESP == X86_GREG_xSP); + AssertCompile(DISGREG_EBP == X86_GREG_xBP); + AssertCompile(DISGREG_ESI == X86_GREG_xSI); + AssertCompile(DISGREG_EDI == X86_GREG_xDI); + AssertCompile(DISGREG_R8D == X86_GREG_x8); + AssertCompile(DISGREG_R9D == X86_GREG_x9); + AssertCompile(DISGREG_R10D == X86_GREG_x10); + AssertCompile(DISGREG_R11D == X86_GREG_x11); + AssertCompile(DISGREG_R12D == X86_GREG_x12); + AssertCompile(DISGREG_R13D == X86_GREG_x13); + AssertCompile(DISGREG_R14D == X86_GREG_x14); + AssertCompile(DISGREG_R15D == X86_GREG_x15); + + /* + * We do this in a loop as we increment the RIP after a successful emulation + * and the new RIP may be a patched instruction which needs emulation as well. + */ + bool fPatchFound = false; + for (;;) + { + PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip); + if (!pPatch) + break; + fPatchFound = true; + + uint8_t u8Tpr; + switch (pPatch->enmType) + { + case HMTPRINSTR_READ: + { + bool fPending; + int rc = APICGetTpr(pVCpu, &u8Tpr, &fPending, NULL /* pu8PendingIrq */); + AssertRC(rc); + + uint8_t idxReg = pPatch->uDstOperand; + AssertStmt(idxReg < RT_ELEMENTS(pCtx->aGRegs), idxReg = RT_ELEMENTS(pCtx->aGRegs) - 1); + pCtx->aGRegs[idxReg].u64 = u8Tpr; + pCtx->rip += pPatch->cbOp; + pCtx->eflags.Bits.u1RF = 0; + break; + } + + case HMTPRINSTR_WRITE_REG: + case HMTPRINSTR_WRITE_IMM: + { + if (pPatch->enmType == HMTPRINSTR_WRITE_REG) + { + uint8_t idxReg = pPatch->uSrcOperand; + AssertStmt(idxReg < RT_ELEMENTS(pCtx->aGRegs), idxReg = RT_ELEMENTS(pCtx->aGRegs) - 1); + u8Tpr = pCtx->aGRegs[idxReg].u8; + } + else + u8Tpr = (uint8_t)pPatch->uSrcOperand; + + int rc2 = APICSetTpr(pVCpu, u8Tpr); + AssertRC(rc2); + pCtx->rip += pPatch->cbOp; + pCtx->eflags.Bits.u1RF = 0; + ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR + | HM_CHANGED_GUEST_RIP + | HM_CHANGED_GUEST_RFLAGS); + break; + } + + default: + { + AssertMsgFailed(("Unexpected patch type %d\n", pPatch->enmType)); + pVCpu->hm.s.u32HMError = pPatch->enmType; + return VERR_SVM_UNEXPECTED_PATCH_TYPE; + } + } + } + + return fPatchFound ? VINF_SUCCESS : VERR_NOT_FOUND; +} + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +/** + * Notification callback for when a \#VMEXIT happens outside SVM R0 code (e.g. + * in IEM). + * + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest-CPU context. + * + * @sa hmR0SvmVmRunCacheVmcb. + */ +VMM_INT_DECL(void) HMNotifySvmNstGstVmexit(PVMCPUCC pVCpu, PCPUMCTX pCtx) +{ + PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + /* + * Restore fields as our own code might look at the VMCB controls as part + * of the #VMEXIT handling in IEM. Otherwise, strictly speaking we don't need to + * restore these fields because currently none of them are written back to memory + * by a physical CPU on #VMEXIT. + */ + PSVMVMCBCTRL pVmcbNstGstCtrl = &pCtx->hwvirt.svm.Vmcb.ctrl; + pVmcbNstGstCtrl->u16InterceptRdCRx = pVmcbNstGstCache->u16InterceptRdCRx; + pVmcbNstGstCtrl->u16InterceptWrCRx = pVmcbNstGstCache->u16InterceptWrCRx; + pVmcbNstGstCtrl->u16InterceptRdDRx = pVmcbNstGstCache->u16InterceptRdDRx; + pVmcbNstGstCtrl->u16InterceptWrDRx = pVmcbNstGstCache->u16InterceptWrDRx; + pVmcbNstGstCtrl->u16PauseFilterThreshold = pVmcbNstGstCache->u16PauseFilterThreshold; + pVmcbNstGstCtrl->u16PauseFilterCount = pVmcbNstGstCache->u16PauseFilterCount; + pVmcbNstGstCtrl->u32InterceptXcpt = pVmcbNstGstCache->u32InterceptXcpt; + pVmcbNstGstCtrl->u64InterceptCtrl = pVmcbNstGstCache->u64InterceptCtrl; + pVmcbNstGstCtrl->u64TSCOffset = pVmcbNstGstCache->u64TSCOffset; + pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = pVmcbNstGstCache->fVIntrMasking; + pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVmcbNstGstCache->fNestedPaging; + pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcbNstGstCache->fLbrVirt; + pVmcbNstGstCache->fCacheValid = false; + } + + /* + * Transitions to ring-3 flag a full CPU-state change except if we transition to ring-3 + * in response to a physical CPU interrupt as no changes to the guest-CPU state are + * expected (see VINF_EM_RAW_INTERRUPT handling in hmR0SvmExitToRing3). + * + * However, with nested-guests, the state -can- change on trips to ring-3 for we might + * try to inject a nested-guest physical interrupt and cause a SVM_EXIT_INTR #VMEXIT for + * the nested-guest from ring-3. Import the complete state here as we will be swapping + * to the guest VMCB after the #VMEXIT. + */ + CPUMImportGuestStateOnDemand(pVCpu, CPUMCTX_EXTRN_ALL); + CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_ALL); + ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); +} +#endif + +/** + * Checks if the Virtual GIF (Global Interrupt Flag) feature is supported and + * enabled for the VM. + * + * @returns @c true if VGIF is enabled, @c false otherwise. + * @param pVM The cross context VM structure. + * + * @remarks This value returned by this functions is expected by the callers not + * to change throughout the lifetime of the VM. + */ +VMM_INT_DECL(bool) HMIsSvmVGifActive(PCVMCC pVM) +{ +#ifdef IN_RING0 + bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); +#else + bool const fVGif = RT_BOOL(pVM->hm.s.ForR3.svm.fFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); +#endif + return fVGif && pVM->hm.s.svm.fVGif; +} + + +/** + * Interface used by IEM to handle patched TPR accesses. + * + * @returns VBox status code + * @retval VINF_SUCCESS if hypercall was handled, RIP + RFLAGS all dealt with. + * @retval VERR_NOT_FOUND if hypercall was _not_ handled. + * @retval VERR_SVM_UNEXPECTED_PATCH_TYPE on IPE. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(int) HMHCMaybeMovTprSvmHypercall(PVMCC pVM, PVMCPUCC pVCpu) +{ + if (pVM->hm.s.fTprPatchingAllowed) + { + int rc = hmEmulateSvmMovTpr(pVM, pVCpu); + if (RT_SUCCESS(rc)) + return VINF_SUCCESS; + return rc; + } + return VERR_NOT_FOUND; +} + + +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +/** + * Checks if the current AMD CPU is subject to erratum 170 "In SVM mode, + * incorrect code bytes may be fetched after a world-switch". + * + * @param pu32Family Where to store the CPU family (can be NULL). + * @param pu32Model Where to store the CPU model (can be NULL). + * @param pu32Stepping Where to store the CPU stepping (can be NULL). + * @returns true if the erratum applies, false otherwise. + */ +VMM_INT_DECL(int) HMIsSubjectToSvmErratum170(uint32_t *pu32Family, uint32_t *pu32Model, uint32_t *pu32Stepping) +{ + /* + * Erratum 170 which requires a forced TLB flush for each world switch: + * See AMD spec. "Revision Guide for AMD NPT Family 0Fh Processors". + * + * All BH-G1/2 and DH-G1/2 models include a fix: + * Athlon X2: 0x6b 1/2 + * 0x68 1/2 + * Athlon 64: 0x7f 1 + * 0x6f 2 + * Sempron: 0x7f 1/2 + * 0x6f 2 + * 0x6c 2 + * 0x7c 2 + * Turion 64: 0x68 2 + */ + uint32_t u32Dummy; + uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily; + ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy); + u32BaseFamily = (u32Version >> 8) & 0xf; + u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0); + u32Model = ((u32Version >> 4) & 0xf); + u32Model = u32Model | ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4); + u32Stepping = u32Version & 0xf; + + bool fErratumApplies = false; + if ( u32Family == 0xf + && !((u32Model == 0x68 || u32Model == 0x6b || u32Model == 0x7f) && u32Stepping >= 1) + && !((u32Model == 0x6f || u32Model == 0x6c || u32Model == 0x7c) && u32Stepping >= 2)) + fErratumApplies = true; + + if (pu32Family) + *pu32Family = u32Family; + if (pu32Model) + *pu32Model = u32Model; + if (pu32Stepping) + *pu32Stepping = u32Stepping; + + return fErratumApplies; +} +#endif + + +/** + * Converts an SVM event type to a TRPM event type. + * + * @returns The TRPM event type. + * @retval TRPM_32BIT_HACK if the specified type of event isn't among the set + * of recognized trap types. + * + * @param pEvent Pointer to the SVM event. + * @param uVector The vector associated with the event. + */ +VMM_INT_DECL(TRPMEVENT) HMSvmEventToTrpmEventType(PCSVMEVENT pEvent, uint8_t uVector) +{ + uint8_t const uType = pEvent->n.u3Type; + switch (uType) + { + case SVM_EVENT_EXTERNAL_IRQ: return TRPM_HARDWARE_INT; + case SVM_EVENT_SOFTWARE_INT: return TRPM_SOFTWARE_INT; + case SVM_EVENT_NMI: return TRPM_TRAP; + case SVM_EVENT_EXCEPTION: + { + if ( uVector == X86_XCPT_BP + || uVector == X86_XCPT_OF) + return TRPM_SOFTWARE_INT; + return TRPM_TRAP; + } + default: + break; + } + AssertMsgFailed(("HMSvmEventToTrpmEvent: Invalid pending-event type %#x\n", uType)); + return TRPM_32BIT_HACK; +} + + +/** + * Gets the SVM nested-guest control intercepts if cached by HM. + * + * @returns @c true on success, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling + * EMT. + * @param pu64Intercepts Where to store the control intercepts. Only updated when + * @c true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmCtrlIntercepts(PCVMCPU pVCpu, uint64_t *pu64Intercepts) +{ + Assert(pu64Intercepts); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pu64Intercepts = pVmcbNstGstCache->u64InterceptCtrl; + return true; + } + return false; +} + + +/** + * Gets the SVM nested-guest CRx-read intercepts if cached by HM. + * + * @returns @c true on success, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling + * EMT. + * @param pu16Intercepts Where to store the CRx-read intercepts. Only updated + * when @c true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmReadCRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts) +{ + Assert(pu16Intercepts); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pu16Intercepts = pVmcbNstGstCache->u16InterceptRdCRx; + return true; + } + return false; +} + + +/** + * Gets the SVM nested-guest CRx-write intercepts if cached by HM. + * + * @returns @c true on success, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling + * EMT. + * @param pu16Intercepts Where to store the CRx-write intercepts. Only updated + * when @c true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmWriteCRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts) +{ + Assert(pu16Intercepts); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pu16Intercepts = pVmcbNstGstCache->u16InterceptWrCRx; + return true; + } + return false; +} + + +/** + * Gets the SVM nested-guest DRx-read intercepts if cached by HM. + * + * @returns @c true on success, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling + * EMT. + * @param pu16Intercepts Where to store the DRx-read intercepts. Only updated + * when @c true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmReadDRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts) +{ + Assert(pu16Intercepts); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pu16Intercepts = pVmcbNstGstCache->u16InterceptRdDRx; + return true; + } + return false; +} + + +/** + * Gets the SVM nested-guest DRx-write intercepts if cached by HM. + * + * @returns @c true on success, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling + * EMT. + * @param pu16Intercepts Where to store the DRx-write intercepts. Only updated + * when @c true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmWriteDRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts) +{ + Assert(pu16Intercepts); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pu16Intercepts = pVmcbNstGstCache->u16InterceptWrDRx; + return true; + } + return false; +} + + +/** + * Gets the SVM nested-guest exception intercepts if cached by HM. + * + * @returns @c true on success, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling + * EMT. + * @param pu32Intercepts Where to store the exception intercepts. Only updated + * when @c true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmXcptIntercepts(PCVMCPU pVCpu, uint32_t *pu32Intercepts) +{ + Assert(pu32Intercepts); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pu32Intercepts = pVmcbNstGstCache->u32InterceptXcpt; + return true; + } + return false; +} + + +/** + * Checks if the nested-guest VMCB has virtual-interrupts masking enabled. + * + * @returns @c true on success, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling + * EMT. + * @param pfVIntrMasking Where to store the virtual-interrupt masking bit. + * Updated only when @c true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmVirtIntrMasking(PCVMCPU pVCpu, bool *pfVIntrMasking) +{ + Assert(pfVIntrMasking); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pfVIntrMasking = pVmcbNstGstCache->fVIntrMasking; + return true; + } + return false; +} + + +/** + * Gets the SVM nested-guest nested-paging bit if cached by HM. + * + * @returns @c true on success, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the + * calling EMT. + * @param pfNestedPaging Where to store the nested-paging bit. Updated only + * when @c true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmNestedPaging(PCVMCPU pVCpu, bool *pfNestedPaging) +{ + Assert(pfNestedPaging); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pfNestedPaging = pVmcbNstGstCache->fNestedPaging; + return true; + } + return false; +} + + +/** + * Returns the nested-guest VMCB pause-filter count. + * + * @returns @c true on success, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the + * calling EMT. + * @param pu16PauseFilterCount Where to store the pause-filter count. Only + * updated @c true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmPauseFilterCount(PCVMCPU pVCpu, uint16_t *pu16PauseFilterCount) +{ + Assert(pu16PauseFilterCount); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pu16PauseFilterCount = pVmcbNstGstCache->u16PauseFilterCount; + return true; + } + return false; +} + + +/** + * Returns the SVM nested-guest TSC offset if cached by HM. + * + * @returns The TSC offset after applying any nested-guest TSC offset. + * @param pVCpu The cross context virtual CPU structure of the calling + * EMT. + * @param pu64TscOffset Where to store the TSC offset. Only updated when @c + * true is returned. + */ +VMM_INT_DECL(bool) HMGetGuestSvmTscOffset(PCVMCPU pVCpu, uint64_t *pu64TscOffset) +{ + Assert(pu64TscOffset); + PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache; + if (pVmcbNstGstCache->fCacheValid) + { + *pu64TscOffset = pVmcbNstGstCache->u64TSCOffset; + return true; + } + return false; +} + diff --git a/src/VBox/VMM/VMMAll/HMVMXAll.cpp b/src/VBox/VMM/VMMAll/HMVMXAll.cpp new file mode 100644 index 00000000..e574fe27 --- /dev/null +++ b/src/VBox/VMM/VMMAll/HMVMXAll.cpp @@ -0,0 +1,1346 @@ +/* $Id: HMVMXAll.cpp $ */ +/** @file + * HM VMX (VT-x) - All contexts. + */ + +/* + * Copyright (C) 2018-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_HM +#define VMCPU_INCL_CPUM_GST_CTX +#include "HMInternal.h" +#include +#include +#include +#include + +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +# include /* ASMCpuId_EAX */ +#endif + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +#define VMXV_DIAG_DESC(a_Def, a_Desc) #a_Def " - " #a_Desc +/** VMX virtual-instructions and VM-exit diagnostics. */ +static const char * const g_apszVmxVDiagDesc[] = +{ + /* Internal processing errors. */ + VMXV_DIAG_DESC(kVmxVDiag_None , "None" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_1 , "Ipe_1" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_2 , "Ipe_2" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_3 , "Ipe_3" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_4 , "Ipe_4" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_5 , "Ipe_5" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_6 , "Ipe_6" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_7 , "Ipe_7" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_8 , "Ipe_8" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_9 , "Ipe_9" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_10 , "Ipe_10" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_11 , "Ipe_11" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_12 , "Ipe_12" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_13 , "Ipe_13" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_14 , "Ipe_14" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_15 , "Ipe_15" ), + VMXV_DIAG_DESC(kVmxVDiag_Ipe_16 , "Ipe_16" ), + /* VMXON. */ + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_A20M , "A20M" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr0Fixed0 , "Cr0Fixed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr0Fixed1 , "Cr0Fixed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr4Fixed0 , "Cr4Fixed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr4Fixed1 , "Cr4Fixed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Intercept , "Intercept" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_MsrFeatCtl , "MsrFeatCtl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrAbnormal , "PtrAbnormal" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrAlign , "PtrAlign" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrMap , "PtrMap" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrReadPhys , "PtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrWidth , "PtrWidth" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_ShadowVmcs , "ShadowVmcs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_VmxAlreadyRoot , "VmxAlreadyRoot" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Vmxe , "Vmxe" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_VmcsRevId , "VmcsRevId" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxon_VmxRootCpl , "VmxRootCpl" ), + /* VMXOFF. */ + VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_Intercept , "Intercept" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_Vmxe , "Vmxe" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_VmxRoot , "VmxRoot" ), + /* VMPTRLD. */ + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrAbnormal , "PtrAbnormal" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrAlign , "PtrAlign" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrMap , "PtrMap" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrReadPhys , "PtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrVmxon , "PtrVmxon" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrWidth , "PtrWidth" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_RevPtrReadPhys , "RevPtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_ShadowVmcs , "ShadowVmcs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_VmcsRevId , "VmcsRevId" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_VmxRoot , "VmxRoot" ), + /* VMPTRST. */ + VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_PtrMap , "PtrMap" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_VmxRoot , "VmxRoot" ), + /* VMCLEAR. */ + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrAbnormal , "PtrAbnormal" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrAlign , "PtrAlign" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrMap , "PtrMap" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrReadPhys , "PtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrVmxon , "PtrVmxon" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrWidth , "PtrWidth" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmclear_VmxRoot , "VmxRoot" ), + /* VMWRITE. */ + VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_FieldInvalid , "FieldInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_FieldRo , "FieldRo" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_LinkPtrInvalid , "LinkPtrInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_PtrInvalid , "PtrInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_PtrMap , "PtrMap" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_VmxRoot , "VmxRoot" ), + /* VMREAD. */ + VMXV_DIAG_DESC(kVmxVDiag_Vmread_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmread_FieldInvalid , "FieldInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmread_LinkPtrInvalid , "LinkPtrInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmread_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmread_PtrInvalid , "PtrInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmread_PtrMap , "PtrMap" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmread_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmread_VmxRoot , "VmxRoot" ), + /* INVVPID. */ + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_DescRsvd , "DescRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_TypeInvalid , "TypeInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type0InvalidAddr , "Type0InvalidAddr" ), + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type0InvalidVpid , "Type0InvalidVpid" ), + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type1InvalidVpid , "Type1InvalidVpid" ), + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type3InvalidVpid , "Type3InvalidVpid" ), + VMXV_DIAG_DESC(kVmxVDiag_Invvpid_VmxRoot , "VmxRoot" ), + /* INVEPT. */ + VMXV_DIAG_DESC(kVmxVDiag_Invept_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Invept_DescRsvd , "DescRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Invept_EptpInvalid , "EptpInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Invept_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Invept_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Invept_TypeInvalid , "TypeInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Invept_VmxRoot , "VmxRoot" ), + /* VMLAUNCH/VMRESUME. */ + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrApicAccess , "AddrApicAccess" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrApicAccessEqVirtApic , "AddrApicAccessEqVirtApic" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrApicAccessHandlerReg , "AddrApicAccessHandlerReg" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrEntryMsrLoad , "AddrEntryMsrLoad" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrExitMsrLoad , "AddrExitMsrLoad" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrExitMsrStore , "AddrExitMsrStore" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrIoBitmapA , "AddrIoBitmapA" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrIoBitmapB , "AddrIoBitmapB" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrMsrBitmap , "AddrMsrBitmap" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVirtApicPage , "AddrVirtApicPage" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVmcsLinkPtr , "AddrVmcsLinkPtr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVmreadBitmap , "AddrVmreadBitmap" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVmwriteBitmap , "AddrVmwriteBitmap" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ApicRegVirt , "ApicRegVirt" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_BlocKMovSS , "BlockMovSS" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_Cpl , "Cpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_Cr3TargetCount , "Cr3TargetCount" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryCtlsAllowed1 , "EntryCtlsAllowed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryCtlsDisallowed0 , "EntryCtlsDisallowed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryInstrLen , "EntryInstrLen" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryInstrLenZero , "EntryInstrLenZero" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryIntInfoErrCodePe , "EntryIntInfoErrCodePe" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryIntInfoErrCodeVec , "EntryIntInfoErrCodeVec" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryIntInfoTypeVecRsvd , "EntryIntInfoTypeVecRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryXcptErrCodeRsvd , "EntryXcptErrCodeRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EptpAccessDirty , "EptpAccessDirty" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EptpPageWalkLength , "EptpPageWalkLength" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EptpMemType , "EptpMemType" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EptpRsvd , "EptpRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ExitCtlsAllowed1 , "ExitCtlsAllowed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ExitCtlsDisallowed0 , "ExitCtlsDisallowed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateHlt , "GuestActStateHlt" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateRsvd , "GuestActStateRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateShutdown , "GuestActStateShutdown" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateSsDpl , "GuestActStateSsDpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateStiMovSs , "GuestActStateStiMovSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr0Fixed0 , "GuestCr0Fixed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr0Fixed1 , "GuestCr0Fixed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr0PgPe , "GuestCr0PgPe" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr3 , "GuestCr3" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr4Fixed0 , "GuestCr4Fixed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr4Fixed1 , "GuestCr4Fixed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestDebugCtl , "GuestDebugCtl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestDr7 , "GuestDr7" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestEferMsr , "GuestEferMsr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestEferMsrRsvd , "GuestEferMsrRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestGdtrBase , "GuestGdtrBase" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestGdtrLimit , "GuestGdtrLimit" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIdtrBase , "GuestIdtrBase" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIdtrLimit , "GuestIdtrLimit" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateEnclave , "GuestIntStateEnclave" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateExtInt , "GuestIntStateExtInt" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateNmi , "GuestIntStateNmi" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateRFlagsSti , "GuestIntStateRFlagsSti" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateRsvd , "GuestIntStateRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateSmi , "GuestIntStateSmi" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateStiMovSs , "GuestIntStateStiMovSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateVirtNmi , "GuestIntStateVirtNmi" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPae , "GuestPae" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPatMsr , "GuestPatMsr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPcide , "GuestPcide" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPdpte , "GuestPdpteRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptBsNoTf , "GuestPndDbgXcptBsNoTf" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptBsTf , "GuestPndDbgXcptBsTf" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptRsvd , "GuestPndDbgXcptRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptRtm , "GuestPndDbgXcptRtm" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRip , "GuestRip" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRipRsvd , "GuestRipRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRFlagsIf , "GuestRFlagsIf" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRFlagsRsvd , "GuestRFlagsRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRFlagsVm , "GuestRFlagsVm" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDefBig , "GuestSegAttrCsDefBig" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDplEqSs , "GuestSegAttrCsDplEqSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDplLtSs , "GuestSegAttrCsDplLtSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDplZero , "GuestSegAttrCsDplZero" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsType , "GuestSegAttrCsType" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsTypeRead , "GuestSegAttrCsTypeRead" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeCs , "GuestSegAttrDescTypeCs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeDs , "GuestSegAttrDescTypeDs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeEs , "GuestSegAttrDescTypeEs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeFs , "GuestSegAttrDescTypeFs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeGs , "GuestSegAttrDescTypeGs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeSs , "GuestSegAttrDescTypeSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplCs , "GuestSegAttrDplRplCs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplDs , "GuestSegAttrDplRplDs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplEs , "GuestSegAttrDplRplEs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplFs , "GuestSegAttrDplRplFs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplGs , "GuestSegAttrDplRplGs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplSs , "GuestSegAttrDplRplSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranCs , "GuestSegAttrGranCs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranDs , "GuestSegAttrGranDs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranEs , "GuestSegAttrGranEs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranFs , "GuestSegAttrGranFs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranGs , "GuestSegAttrGranGs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranSs , "GuestSegAttrGranSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrDescType , "GuestSegAttrLdtrDescType" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrGran , "GuestSegAttrLdtrGran" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrPresent , "GuestSegAttrLdtrPresent" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrRsvd , "GuestSegAttrLdtrRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrType , "GuestSegAttrLdtrType" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentCs , "GuestSegAttrPresentCs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentDs , "GuestSegAttrPresentDs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentEs , "GuestSegAttrPresentEs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentFs , "GuestSegAttrPresentFs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentGs , "GuestSegAttrPresentGs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentSs , "GuestSegAttrPresentSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdCs , "GuestSegAttrRsvdCs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdDs , "GuestSegAttrRsvdDs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdEs , "GuestSegAttrRsvdEs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdFs , "GuestSegAttrRsvdFs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdGs , "GuestSegAttrRsvdGs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdSs , "GuestSegAttrRsvdSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrSsDplEqRpl , "GuestSegAttrSsDplEqRpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrSsDplZero , "GuestSegAttrSsDplZero " ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrSsType , "GuestSegAttrSsType" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrDescType , "GuestSegAttrTrDescType" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrGran , "GuestSegAttrTrGran" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrPresent , "GuestSegAttrTrPresent" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrRsvd , "GuestSegAttrTrRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrType , "GuestSegAttrTrType" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrUnusable , "GuestSegAttrTrUnusable" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccCs , "GuestSegAttrTypeAccCs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccDs , "GuestSegAttrTypeAccDs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccEs , "GuestSegAttrTypeAccEs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccFs , "GuestSegAttrTypeAccFs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccGs , "GuestSegAttrTypeAccGs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccSs , "GuestSegAttrTypeAccSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Cs , "GuestSegAttrV86Cs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Ds , "GuestSegAttrV86Ds" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Es , "GuestSegAttrV86Es" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Fs , "GuestSegAttrV86Fs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Gs , "GuestSegAttrV86Gs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Ss , "GuestSegAttrV86Ss" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseCs , "GuestSegBaseCs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseDs , "GuestSegBaseDs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseEs , "GuestSegBaseEs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseFs , "GuestSegBaseFs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseGs , "GuestSegBaseGs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseLdtr , "GuestSegBaseLdtr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseSs , "GuestSegBaseSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseTr , "GuestSegBaseTr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Cs , "GuestSegBaseV86Cs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Ds , "GuestSegBaseV86Ds" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Es , "GuestSegBaseV86Es" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Fs , "GuestSegBaseV86Fs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Gs , "GuestSegBaseV86Gs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Ss , "GuestSegBaseV86Ss" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Cs , "GuestSegLimitV86Cs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Ds , "GuestSegLimitV86Ds" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Es , "GuestSegLimitV86Es" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Fs , "GuestSegLimitV86Fs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Gs , "GuestSegLimitV86Gs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Ss , "GuestSegLimitV86Ss" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegSelCsSsRpl , "GuestSegSelCsSsRpl" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegSelLdtr , "GuestSegSelLdtr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegSelTr , "GuestSegSelTr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSysenterEspEip , "GuestSysenterEspEip" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrCurVmcs , "VmcsLinkPtrCurVmcs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrReadPhys , "VmcsLinkPtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrRevId , "VmcsLinkPtrRevId" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrShadow , "VmcsLinkPtrShadow" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr0Fixed0 , "HostCr0Fixed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr0Fixed1 , "HostCr0Fixed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr3 , "HostCr3" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Fixed0 , "HostCr4Fixed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Fixed1 , "HostCr4Fixed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Pae , "HostCr4Pae" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Pcide , "HostCr4Pcide" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCsTr , "HostCsTr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostEferMsr , "HostEferMsr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostEferMsrRsvd , "HostEferMsrRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostGuestLongMode , "HostGuestLongMode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostGuestLongModeNoCpu , "HostGuestLongModeNoCpu" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostLongMode , "HostLongMode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostPatMsr , "HostPatMsr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostRip , "HostRip" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostRipRsvd , "HostRipRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSel , "HostSel" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSegBase , "HostSegBase" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSs , "HostSs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSysenterEspEip , "HostSysenterEspEip" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_IoBitmapAPtrReadPhys , "IoBitmapAPtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_IoBitmapBPtrReadPhys , "IoBitmapBPtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_LongModeCS , "LongModeCS" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrBitmapPtrReadPhys , "MsrBitmapPtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoad , "MsrLoad" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadCount , "MsrLoadCount" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadPtrReadPhys , "MsrLoadPtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadRing3 , "MsrLoadRing3" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadRsvd , "MsrLoadRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_NmiWindowExit , "NmiWindowExit" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PinCtlsAllowed1 , "PinCtlsAllowed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PinCtlsDisallowed0 , "PinCtlsDisallowed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtlsAllowed1 , "ProcCtlsAllowed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtlsDisallowed0 , "ProcCtlsDisallowed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtls2Allowed1 , "ProcCtls2Allowed1" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtls2Disallowed0 , "ProcCtls2Disallowed0" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PtrInvalid , "PtrInvalid" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PtrShadowVmcs , "PtrShadowVmcs" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_RealOrV86Mode , "RealOrV86Mode" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_SavePreemptTimer , "SavePreemptTimer" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_TprThresholdRsvd , "TprThresholdRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_TprThresholdVTpr , "TprThresholdVTpr" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtApicPagePtrReadPhys , "VirtApicPageReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtIntDelivery , "VirtIntDelivery" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtNmi , "VirtNmi" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtX2ApicTprShadow , "VirtX2ApicTprShadow" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtX2ApicVirtApic , "VirtX2ApicVirtApic" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsClear , "VmcsClear" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLaunch , "VmcsLaunch" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmreadBitmapPtrReadPhys , "VmreadBitmapPtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmwriteBitmapPtrReadPhys , "VmwriteBitmapPtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmxRoot , "VmxRoot" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmentry_Vpid , "Vpid" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_HostPdpte , "HostPdpte" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoad , "MsrLoad" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadCount , "MsrLoadCount" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadPtrReadPhys , "MsrLoadPtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadRing3 , "MsrLoadRing3" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadRsvd , "MsrLoadRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStore , "MsrStore" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStoreCount , "MsrStoreCount" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStorePtrReadPhys , "MsrStorePtrReadPhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStorePtrWritePhys , "MsrStorePtrWritePhys" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStoreRing3 , "MsrStoreRing3" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStoreRsvd , "MsrStoreRsvd" ), + VMXV_DIAG_DESC(kVmxVDiag_Vmexit_VirtApicPagePtrWritePhys , "VirtApicPagePtrWritePhys" ) + /* kVmxVDiag_End */ +}; +AssertCompile(RT_ELEMENTS(g_apszVmxVDiagDesc) == kVmxVDiag_End); +#undef VMXV_DIAG_DESC + + +/** + * Gets the descriptive name of a VMX instruction/VM-exit diagnostic code. + * + * @returns The descriptive string. + * @param enmDiag The VMX diagnostic. + */ +VMM_INT_DECL(const char *) HMGetVmxDiagDesc(VMXVDIAG enmDiag) +{ + if (RT_LIKELY((unsigned)enmDiag < RT_ELEMENTS(g_apszVmxVDiagDesc))) + return g_apszVmxVDiagDesc[enmDiag]; + return "Unknown/invalid"; +} + + +/** + * Checks if a code selector (CS) is suitable for execution using hardware-assisted + * VMX when unrestricted execution isn't available. + * + * @returns true if selector is suitable for VMX, otherwise + * false. + * @param pSel Pointer to the selector to check (CS). + * @param uStackDpl The CPL, aka the DPL of the stack segment. + */ +static bool hmVmxIsCodeSelectorOk(PCCPUMSELREG pSel, unsigned uStackDpl) +{ + /* + * Segment must be an accessed code segment, it must be present and it must + * be usable. + * Note! These are all standard requirements and if CS holds anything else + * we've got buggy code somewhere! + */ + AssertCompile(X86DESCATTR_TYPE == 0xf); + AssertMsgReturn( (pSel->Attr.u & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_CODE | X86DESCATTR_DT | X86DESCATTR_P | X86DESCATTR_UNUSABLE)) + == (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_CODE | X86DESCATTR_DT | X86DESCATTR_P), + ("%#x\n", pSel->Attr.u), + false); + + /* + * For conforming segments, CS.DPL must be <= SS.DPL, while CS.DPL must equal + * SS.DPL for non-confroming segments. + * Note! This is also a hard requirement like above. + */ + AssertMsgReturn( pSel->Attr.n.u4Type & X86_SEL_TYPE_CONF + ? pSel->Attr.n.u2Dpl <= uStackDpl + : pSel->Attr.n.u2Dpl == uStackDpl, + ("u4Type=%#x u2Dpl=%u uStackDpl=%u\n", pSel->Attr.n.u4Type, pSel->Attr.n.u2Dpl, uStackDpl), + false); + + /* + * The following two requirements are VT-x specific: + * - G bit must be set if any high limit bits are set. + * - G bit must be clear if any low limit bits are clear. + */ + if ( ((pSel->u32Limit & 0xfff00000) == 0x00000000 || pSel->Attr.n.u1Granularity) + && ((pSel->u32Limit & 0x00000fff) == 0x00000fff || !pSel->Attr.n.u1Granularity)) + return true; + return false; +} + + +/** + * Checks if a data selector (DS/ES/FS/GS) is suitable for execution using + * hardware-assisted VMX when unrestricted execution isn't available. + * + * @returns true if selector is suitable for VMX, otherwise + * false. + * @param pSel Pointer to the selector to check + * (DS/ES/FS/GS). + */ +static bool hmVmxIsDataSelectorOk(PCCPUMSELREG pSel) +{ + /* + * Unusable segments are OK. These days they should be marked as such, as + * but as an alternative we for old saved states and AMD<->VT-x migration + * we also treat segments with all the attributes cleared as unusable. + */ + if (pSel->Attr.n.u1Unusable || !pSel->Attr.u) + return true; + + /** @todo tighten these checks. Will require CPUM load adjusting. */ + + /* Segment must be accessed. */ + if (pSel->Attr.u & X86_SEL_TYPE_ACCESSED) + { + /* Code segments must also be readable. */ + if ( !(pSel->Attr.u & X86_SEL_TYPE_CODE) + || (pSel->Attr.u & X86_SEL_TYPE_READ)) + { + /* The S bit must be set. */ + if (pSel->Attr.n.u1DescType) + { + /* Except for conforming segments, DPL >= RPL. */ + if ( pSel->Attr.n.u2Dpl >= (pSel->Sel & X86_SEL_RPL) + || pSel->Attr.n.u4Type >= X86_SEL_TYPE_ER_ACC) + { + /* Segment must be present. */ + if (pSel->Attr.n.u1Present) + { + /* + * The following two requirements are VT-x specific: + * - G bit must be set if any high limit bits are set. + * - G bit must be clear if any low limit bits are clear. + */ + if ( ((pSel->u32Limit & 0xfff00000) == 0x00000000 || pSel->Attr.n.u1Granularity) + && ((pSel->u32Limit & 0x00000fff) == 0x00000fff || !pSel->Attr.n.u1Granularity)) + return true; + } + } + } + } + } + + return false; +} + + +/** + * Checks if the stack selector (SS) is suitable for execution using + * hardware-assisted VMX when unrestricted execution isn't available. + * + * @returns true if selector is suitable for VMX, otherwise + * false. + * @param pSel Pointer to the selector to check (SS). + */ +static bool hmVmxIsStackSelectorOk(PCCPUMSELREG pSel) +{ + /* + * Unusable segments are OK. These days they should be marked as such, as + * but as an alternative we for old saved states and AMD<->VT-x migration + * we also treat segments with all the attributes cleared as unusable. + */ + /** @todo r=bird: actually all zeroes isn't gonna cut it... SS.DPL == CPL. */ + if (pSel->Attr.n.u1Unusable || !pSel->Attr.u) + return true; + + /* + * Segment must be an accessed writable segment, it must be present. + * Note! These are all standard requirements and if SS holds anything else + * we've got buggy code somewhere! + */ + AssertCompile(X86DESCATTR_TYPE == 0xf); + AssertMsgReturn( (pSel->Attr.u & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_WRITE | X86DESCATTR_DT | X86DESCATTR_P | X86_SEL_TYPE_CODE)) + == (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_WRITE | X86DESCATTR_DT | X86DESCATTR_P), + ("%#x\n", pSel->Attr.u), false); + + /* + * DPL must equal RPL. But in real mode or soon after enabling protected + * mode, it might not be. + */ + if (pSel->Attr.n.u2Dpl == (pSel->Sel & X86_SEL_RPL)) + { + /* + * The following two requirements are VT-x specific: + * - G bit must be set if any high limit bits are set. + * - G bit must be clear if any low limit bits are clear. + */ + if ( ((pSel->u32Limit & 0xfff00000) == 0x00000000 || pSel->Attr.n.u1Granularity) + && ((pSel->u32Limit & 0x00000fff) == 0x00000fff || !pSel->Attr.n.u1Granularity)) + return true; + } + return false; +} + + +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +/** + * Checks if the CPU is subject to the "VMX-Preemption Timer Does Not Count Down at + * the Rate Specified" erratum. + * + * Errata names and related steppings: + * - BA86 - D0. + * - AAX65 - C2. + * - AAU65 - C2, K0. + * - AAO95 - B1. + * - AAT59 - C2. + * - AAK139 - D0. + * - AAM126 - C0, C1, D0. + * - AAN92 - B1. + * - AAJ124 - C0, D0. + * - AAP86 - B1. + * + * Steppings: B1, C0, C1, C2, D0, K0. + * + * @returns @c true if subject to it, @c false if not. + */ +VMM_INT_DECL(bool) HMIsSubjectToVmxPreemptTimerErratum(void) +{ + uint32_t u = ASMCpuId_EAX(1); + u &= ~(RT_BIT_32(14) | RT_BIT_32(15) | RT_BIT_32(28) | RT_BIT_32(29) | RT_BIT_32(30) | RT_BIT_32(31)); + if ( u == 0x000206E6 /* 323344.pdf - BA86 - D0 - Xeon Processor 7500 Series */ + || u == 0x00020652 /* 323056.pdf - AAX65 - C2 - Xeon Processor L3406 */ + /* 322814.pdf - AAT59 - C2 - CoreTM i7-600, i5-500, i5-400 and i3-300 Mobile Processor Series */ + /* 322911.pdf - AAU65 - C2 - CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */ + || u == 0x00020655 /* 322911.pdf - AAU65 - K0 - CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */ + || u == 0x000106E5 /* 322373.pdf - AAO95 - B1 - Xeon Processor 3400 Series */ + /* 322166.pdf - AAN92 - B1 - CoreTM i7-800 and i5-700 Desktop Processor Series */ + /* 320767.pdf - AAP86 - B1 - Core i7-900 Mobile Processor Extreme Edition Series, Intel Core i7-800 and i7-700 Mobile Processor Series */ + || u == 0x000106A0 /* 321333.pdf - AAM126 - C0 - Xeon Processor 3500 Series Specification */ + || u == 0x000106A1 /* 321333.pdf - AAM126 - C1 - Xeon Processor 3500 Series Specification */ + || u == 0x000106A4 /* 320836.pdf - AAJ124 - C0 - Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */ + || u == 0x000106A5 /* 321333.pdf - AAM126 - D0 - Xeon Processor 3500 Series Specification */ + /* 321324.pdf - AAK139 - D0 - Xeon Processor 5500 Series Specification */ + /* 320836.pdf - AAJ124 - D0 - Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */ + || u == 0x000306A8 /* ?????????? - ?????? - ?? - Xeon E3-1220 v2 */ + ) + return true; + return false; +} +#endif + + +/** + * Checks if the guest is in a suitable state for hardware-assisted VMX execution. + * + * @returns @c true if it is suitable, @c false otherwise. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the guest CPU context. + * + * @remarks @a pCtx can be a partial context and thus may not be necessarily the + * same as pVCpu->cpum.GstCtx! Thus don't eliminate the @a pCtx parameter. + * Secondly, if additional checks are added that require more of the CPU + * state, make sure REM (which supplies a partial state) is updated. + */ +VMM_INT_DECL(bool) HMCanExecuteVmxGuest(PVMCC pVM, PVMCPUCC pVCpu, PCCPUMCTX pCtx) +{ + Assert(HMIsEnabled(pVM)); + bool const fUnrestrictedGuest = CTX_EXPR(pVM->hm.s.vmx.fUnrestrictedGuestCfg, pVM->hmr0.s.vmx.fUnrestrictedGuest, RT_NOTHING); + Assert( ( fUnrestrictedGuest && !pVM->hm.s.vmx.pRealModeTSS) + || (!fUnrestrictedGuest && pVM->hm.s.vmx.pRealModeTSS)); + + pVCpu->hm.s.fActive = false; + + bool const fSupportsRealMode = fUnrestrictedGuest || PDMVmmDevHeapIsEnabled(pVM); + if (!fUnrestrictedGuest) + { + /* + * The VMM device heap is a requirement for emulating real mode or protected mode without paging with the unrestricted + * guest execution feature is missing (VT-x only). + */ + if (fSupportsRealMode) + { + if (CPUMIsGuestInRealModeEx(pCtx)) + { + /* + * In V86 mode (VT-x or not), the CPU enforces real-mode compatible selector + * bases, limits, and attributes, i.e. limit must be 64K, base must be selector * 16, + * and attributes must be 0x9b for code and 0x93 for code segments. + * If this is not true, we cannot execute real mode as V86 and have to fall + * back to emulation. + */ + if ( pCtx->cs.Sel != (pCtx->cs.u64Base >> 4) + || pCtx->ds.Sel != (pCtx->ds.u64Base >> 4) + || pCtx->es.Sel != (pCtx->es.u64Base >> 4) + || pCtx->ss.Sel != (pCtx->ss.u64Base >> 4) + || pCtx->fs.Sel != (pCtx->fs.u64Base >> 4) + || pCtx->gs.Sel != (pCtx->gs.u64Base >> 4)) + { + STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRmSelBase); + return false; + } + if ( (pCtx->cs.u32Limit != 0xffff) + || (pCtx->ds.u32Limit != 0xffff) + || (pCtx->es.u32Limit != 0xffff) + || (pCtx->ss.u32Limit != 0xffff) + || (pCtx->fs.u32Limit != 0xffff) + || (pCtx->gs.u32Limit != 0xffff)) + { + STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRmSelLimit); + return false; + } + if ( (pCtx->cs.Attr.u != 0x9b) + || (pCtx->ds.Attr.u != 0x93) + || (pCtx->es.Attr.u != 0x93) + || (pCtx->ss.Attr.u != 0x93) + || (pCtx->fs.Attr.u != 0x93) + || (pCtx->gs.Attr.u != 0x93)) + { + STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRmSelAttr); + return false; + } + STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckRmOk); + } + else + { + /* + * Verify the requirements for executing code in protected mode. VT-x can't + * handle the CPU state right after a switch from real to protected mode + * (all sorts of RPL & DPL assumptions). + */ + PCVMXVMCSINFOSHARED pVmcsInfo = hmGetVmxActiveVmcsInfoShared(pVCpu); + if (pVmcsInfo->fWasInRealMode) + { + if (!CPUMIsGuestInV86ModeEx(pCtx)) + { + /* The guest switched to protected mode, check if the state is suitable for VT-x. */ + if ((pCtx->cs.Sel & X86_SEL_RPL) != (pCtx->ss.Sel & X86_SEL_RPL)) + { + STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRpl); + return false; + } + if ( !hmVmxIsCodeSelectorOk(&pCtx->cs, pCtx->ss.Attr.n.u2Dpl) + || !hmVmxIsDataSelectorOk(&pCtx->ds) + || !hmVmxIsDataSelectorOk(&pCtx->es) + || !hmVmxIsDataSelectorOk(&pCtx->fs) + || !hmVmxIsDataSelectorOk(&pCtx->gs) + || !hmVmxIsStackSelectorOk(&pCtx->ss)) + { + STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadSel); + return false; + } + } + else + { + /* The guest switched to V86 mode, check if the state is suitable for VT-x. */ + if ( pCtx->cs.Sel != (pCtx->cs.u64Base >> 4) + || pCtx->ds.Sel != (pCtx->ds.u64Base >> 4) + || pCtx->es.Sel != (pCtx->es.u64Base >> 4) + || pCtx->ss.Sel != (pCtx->ss.u64Base >> 4) + || pCtx->fs.Sel != (pCtx->fs.u64Base >> 4) + || pCtx->gs.Sel != (pCtx->gs.u64Base >> 4)) + { + STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadV86SelBase); + return false; + } + if ( pCtx->cs.u32Limit != 0xffff + || pCtx->ds.u32Limit != 0xffff + || pCtx->es.u32Limit != 0xffff + || pCtx->ss.u32Limit != 0xffff + || pCtx->fs.u32Limit != 0xffff + || pCtx->gs.u32Limit != 0xffff) + { + STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadV86SelLimit); + return false; + } + if ( pCtx->cs.Attr.u != 0xf3 + || pCtx->ds.Attr.u != 0xf3 + || pCtx->es.Attr.u != 0xf3 + || pCtx->ss.Attr.u != 0xf3 + || pCtx->fs.Attr.u != 0xf3 + || pCtx->gs.Attr.u != 0xf3) + { + STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadV86SelAttr); + return false; + } + } + } + } + } + else + { + if (!CPUMIsGuestInLongModeEx(pCtx)) + { + if (/* Requires a fake PD for real *and* protected mode without paging - stored in the VMM device heap: */ + !CTX_EXPR(pVM->hm.s.fNestedPagingCfg, pVM->hmr0.s.fNestedPaging, RT_NOTHING) + /* Requires a fake TSS for real mode - stored in the VMM device heap: */ + || CPUMIsGuestInRealModeEx(pCtx)) + return false; + + /* Too early for VT-x; Solaris guests will fail with a guru meditation otherwise; same for XP. */ + if (pCtx->idtr.pIdt == 0 || pCtx->idtr.cbIdt == 0 || pCtx->tr.Sel == 0) + return false; + + /* + * The guest is about to complete the switch to protected mode. Wait a bit longer. + * Windows XP; switch to protected mode; all selectors are marked not present + * in the hidden registers (possible recompiler bug; see load_seg_vm). + */ + /** @todo Is this supposed recompiler bug still relevant with IEM? */ + if (pCtx->cs.Attr.n.u1Present == 0) + return false; + if (pCtx->ss.Attr.n.u1Present == 0) + return false; + + /* + * Windows XP: possible same as above, but new recompiler requires new + * heuristics? VT-x doesn't seem to like something about the guest state and + * this stuff avoids it. + */ + /** @todo This check is actually wrong, it doesn't take the direction of the + * stack segment into account. But, it does the job for now. */ + if (pCtx->rsp >= pCtx->ss.u32Limit) + return false; + } + } + } + + if (pVM->hm.s.vmx.fEnabled) + { + /* If bit N is set in cr0_fixed0, then it must be set in the guest's cr0. */ + uint32_t uCr0Mask = (uint32_t)CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.u64Cr0Fixed0, g_HmMsrs.u.vmx.u64Cr0Fixed0, RT_NOTHING); + + /* We ignore the NE bit here on purpose; see HMR0.cpp for details. */ + uCr0Mask &= ~X86_CR0_NE; + + if (fSupportsRealMode) + { + /* We ignore the PE & PG bits here on purpose; we emulate real and protected mode without paging. */ + uCr0Mask &= ~(X86_CR0_PG | X86_CR0_PE); + } + else + { + /* We support protected mode without paging using identity mapping. */ + uCr0Mask &= ~X86_CR0_PG; + } + if ((pCtx->cr0 & uCr0Mask) != uCr0Mask) + return false; + + /* If bit N is cleared in cr0_fixed1, then it must be zero in the guest's cr0. */ + uCr0Mask = (uint32_t)~CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.u64Cr0Fixed1, g_HmMsrs.u.vmx.u64Cr0Fixed1, RT_NOTHING); + if ((pCtx->cr0 & uCr0Mask) != 0) + return false; + + /* If bit N is set in cr4_fixed0, then it must be set in the guest's cr4. */ + uCr0Mask = (uint32_t)CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.u64Cr4Fixed0, g_HmMsrs.u.vmx.u64Cr4Fixed0, RT_NOTHING); + uCr0Mask &= ~X86_CR4_VMXE; + if ((pCtx->cr4 & uCr0Mask) != uCr0Mask) + return false; + + /* If bit N is cleared in cr4_fixed1, then it must be zero in the guest's cr4. */ + uCr0Mask = (uint32_t)~CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.u64Cr4Fixed1, g_HmMsrs.u.vmx.u64Cr4Fixed1, RT_NOTHING); + if ((pCtx->cr4 & uCr0Mask) != 0) + return false; + + pVCpu->hm.s.fActive = true; + return true; + } + + return false; +} + + +/** + * Dumps the virtual VMCS state to the release log. + * + * This is a purely a convenience function to output to the release log because + * cpumR3InfoVmxVmcs dumps only to the debug console and isn't always easy to use in + * case of a crash. + * + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(void) HMDumpHwvirtVmxState(PVMCPU pVCpu) +{ + /* The string width of -4 used in the macros below to cover 'LDTR', 'GDTR', 'IDTR. */ +#define HMVMX_DUMP_HOST_XDTR(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \ + do { \ + LogRel((" %s%-4s = {base=%016RX64}\n", \ + (a_pszPrefix), (a_SegName), (a_pVmcs)->u64Host##a_Seg##Base.u)); \ + } while (0) +#define HMVMX_DUMP_HOST_FS_GS_TR(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \ + do { \ + LogRel((" %s%-4s = {%04x base=%016RX64}\n", \ + (a_pszPrefix), (a_SegName), (a_pVmcs)->Host##a_Seg, (a_pVmcs)->u64Host##a_Seg##Base.u)); \ + } while (0) +#define HMVMX_DUMP_GUEST_SEGREG(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \ + do { \ + LogRel((" %s%-4s = {%04x base=%016RX64 limit=%08x flags=%04x}\n", \ + (a_pszPrefix), (a_SegName), (a_pVmcs)->Guest##a_Seg, (a_pVmcs)->u64Guest##a_Seg##Base.u, \ + (a_pVmcs)->u32Guest##a_Seg##Limit, (a_pVmcs)->u32Guest##a_Seg##Attr)); \ + } while (0) +#define HMVMX_DUMP_GUEST_XDTR(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \ + do { \ + LogRel((" %s%-4s = {base=%016RX64 limit=%08x}\n", \ + (a_pszPrefix), (a_SegName), (a_pVmcs)->u64Guest##a_Seg##Base.u, (a_pVmcs)->u32Guest##a_Seg##Limit)); \ + } while (0) + + PCCPUMCTX const pCtx = &pVCpu->cpum.GstCtx; + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + if (!pVmcs) + { + LogRel(("Virtual VMCS not allocated\n")); + return; + } + LogRel(("GCPhysVmxon = %#RGp\n", pCtx->hwvirt.vmx.GCPhysVmxon)); + LogRel(("GCPhysVmcs = %#RGp\n", pCtx->hwvirt.vmx.GCPhysVmcs)); + LogRel(("GCPhysShadowVmcs = %#RGp\n", pCtx->hwvirt.vmx.GCPhysShadowVmcs)); + LogRel(("enmDiag = %u (%s)\n", pCtx->hwvirt.vmx.enmDiag, HMGetVmxDiagDesc(pCtx->hwvirt.vmx.enmDiag))); + LogRel(("uDiagAux = %#RX64\n", pCtx->hwvirt.vmx.uDiagAux)); + LogRel(("enmAbort = %u (%s)\n", pCtx->hwvirt.vmx.enmAbort, VMXGetAbortDesc(pCtx->hwvirt.vmx.enmAbort))); + LogRel(("uAbortAux = %u (%#x)\n", pCtx->hwvirt.vmx.uAbortAux, pCtx->hwvirt.vmx.uAbortAux)); + LogRel(("fInVmxRootMode = %RTbool\n", pCtx->hwvirt.vmx.fInVmxRootMode)); + LogRel(("fInVmxNonRootMode = %RTbool\n", pCtx->hwvirt.vmx.fInVmxNonRootMode)); + LogRel(("fInterceptEvents = %RTbool\n", pCtx->hwvirt.vmx.fInterceptEvents)); + LogRel(("fNmiUnblockingIret = %RTbool\n", pCtx->hwvirt.vmx.fNmiUnblockingIret)); + LogRel(("uFirstPauseLoopTick = %RX64\n", pCtx->hwvirt.vmx.uFirstPauseLoopTick)); + LogRel(("uPrevPauseTick = %RX64\n", pCtx->hwvirt.vmx.uPrevPauseTick)); + LogRel(("uEntryTick = %RX64\n", pCtx->hwvirt.vmx.uEntryTick)); + LogRel(("offVirtApicWrite = %#RX16\n", pCtx->hwvirt.vmx.offVirtApicWrite)); + LogRel(("fVirtNmiBlocking = %RTbool\n", pCtx->hwvirt.vmx.fVirtNmiBlocking)); + LogRel(("VMCS cache:\n")); + + const char *pszPrefix = " "; + /* Header. */ + { + LogRel(("%sHeader:\n", pszPrefix)); + LogRel((" %sVMCS revision id = %#RX32\n", pszPrefix, pVmcs->u32VmcsRevId)); + LogRel((" %sVMX-abort id = %#RX32 (%s)\n", pszPrefix, pVmcs->enmVmxAbort, VMXGetAbortDesc(pVmcs->enmVmxAbort))); + LogRel((" %sVMCS state = %#x (%s)\n", pszPrefix, pVmcs->fVmcsState, VMXGetVmcsStateDesc(pVmcs->fVmcsState))); + } + + /* Control fields. */ + { + /* 16-bit. */ + LogRel(("%sControl:\n", pszPrefix)); + LogRel((" %sVPID = %#RX16\n", pszPrefix, pVmcs->u16Vpid)); + LogRel((" %sPosted intr notify vector = %#RX16\n", pszPrefix, pVmcs->u16PostIntNotifyVector)); + LogRel((" %sEPTP index = %#RX16\n", pszPrefix, pVmcs->u16EptpIndex)); + + /* 32-bit. */ + LogRel((" %sPin ctls = %#RX32\n", pszPrefix, pVmcs->u32PinCtls)); + LogRel((" %sProcessor ctls = %#RX32\n", pszPrefix, pVmcs->u32ProcCtls)); + LogRel((" %sSecondary processor ctls = %#RX32\n", pszPrefix, pVmcs->u32ProcCtls2)); + LogRel((" %sVM-exit ctls = %#RX32\n", pszPrefix, pVmcs->u32ExitCtls)); + LogRel((" %sVM-entry ctls = %#RX32\n", pszPrefix, pVmcs->u32EntryCtls)); + LogRel((" %sException bitmap = %#RX32\n", pszPrefix, pVmcs->u32XcptBitmap)); + LogRel((" %sPage-fault mask = %#RX32\n", pszPrefix, pVmcs->u32XcptPFMask)); + LogRel((" %sPage-fault match = %#RX32\n", pszPrefix, pVmcs->u32XcptPFMatch)); + LogRel((" %sCR3-target count = %RU32\n", pszPrefix, pVmcs->u32Cr3TargetCount)); + LogRel((" %sVM-exit MSR store count = %RU32\n", pszPrefix, pVmcs->u32ExitMsrStoreCount)); + LogRel((" %sVM-exit MSR load count = %RU32\n", pszPrefix, pVmcs->u32ExitMsrLoadCount)); + LogRel((" %sVM-entry MSR load count = %RU32\n", pszPrefix, pVmcs->u32EntryMsrLoadCount)); + LogRel((" %sVM-entry interruption info = %#RX32\n", pszPrefix, pVmcs->u32EntryIntInfo)); + { + uint32_t const fInfo = pVmcs->u32EntryIntInfo; + uint8_t const uType = VMX_ENTRY_INT_INFO_TYPE(fInfo); + LogRel((" %sValid = %RTbool\n", pszPrefix, VMX_ENTRY_INT_INFO_IS_VALID(fInfo))); + LogRel((" %sType = %#x (%s)\n", pszPrefix, uType, VMXGetEntryIntInfoTypeDesc(uType))); + LogRel((" %sVector = %#x\n", pszPrefix, VMX_ENTRY_INT_INFO_VECTOR(fInfo))); + LogRel((" %sNMI-unblocking-IRET = %RTbool\n", pszPrefix, VMX_ENTRY_INT_INFO_IS_NMI_UNBLOCK_IRET(fInfo))); + LogRel((" %sError-code valid = %RTbool\n", pszPrefix, VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(fInfo))); + } + LogRel((" %sVM-entry xcpt error-code = %#RX32\n", pszPrefix, pVmcs->u32EntryXcptErrCode)); + LogRel((" %sVM-entry instr length = %u byte(s)\n", pszPrefix, pVmcs->u32EntryInstrLen)); + LogRel((" %sTPR threshold = %#RX32\n", pszPrefix, pVmcs->u32TprThreshold)); + LogRel((" %sPLE gap = %#RX32\n", pszPrefix, pVmcs->u32PleGap)); + LogRel((" %sPLE window = %#RX32\n", pszPrefix, pVmcs->u32PleWindow)); + + /* 64-bit. */ + LogRel((" %sIO-bitmap A addr = %#RX64\n", pszPrefix, pVmcs->u64AddrIoBitmapA.u)); + LogRel((" %sIO-bitmap B addr = %#RX64\n", pszPrefix, pVmcs->u64AddrIoBitmapB.u)); + LogRel((" %sMSR-bitmap addr = %#RX64\n", pszPrefix, pVmcs->u64AddrMsrBitmap.u)); + LogRel((" %sVM-exit MSR store addr = %#RX64\n", pszPrefix, pVmcs->u64AddrExitMsrStore.u)); + LogRel((" %sVM-exit MSR load addr = %#RX64\n", pszPrefix, pVmcs->u64AddrExitMsrLoad.u)); + LogRel((" %sVM-entry MSR load addr = %#RX64\n", pszPrefix, pVmcs->u64AddrEntryMsrLoad.u)); + LogRel((" %sExecutive VMCS ptr = %#RX64\n", pszPrefix, pVmcs->u64ExecVmcsPtr.u)); + LogRel((" %sPML addr = %#RX64\n", pszPrefix, pVmcs->u64AddrPml.u)); + LogRel((" %sTSC offset = %#RX64\n", pszPrefix, pVmcs->u64TscOffset.u)); + LogRel((" %sVirtual-APIC addr = %#RX64\n", pszPrefix, pVmcs->u64AddrVirtApic.u)); + LogRel((" %sAPIC-access addr = %#RX64\n", pszPrefix, pVmcs->u64AddrApicAccess.u)); + LogRel((" %sPosted-intr desc addr = %#RX64\n", pszPrefix, pVmcs->u64AddrPostedIntDesc.u)); + LogRel((" %sVM-functions control = %#RX64\n", pszPrefix, pVmcs->u64VmFuncCtls.u)); + LogRel((" %sEPTP ptr = %#RX64\n", pszPrefix, pVmcs->u64EptPtr.u)); + LogRel((" %sEOI-exit bitmap 0 = %#RX64\n", pszPrefix, pVmcs->u64EoiExitBitmap0.u)); + LogRel((" %sEOI-exit bitmap 1 = %#RX64\n", pszPrefix, pVmcs->u64EoiExitBitmap1.u)); + LogRel((" %sEOI-exit bitmap 2 = %#RX64\n", pszPrefix, pVmcs->u64EoiExitBitmap2.u)); + LogRel((" %sEOI-exit bitmap 3 = %#RX64\n", pszPrefix, pVmcs->u64EoiExitBitmap3.u)); + LogRel((" %sEPTP-list addr = %#RX64\n", pszPrefix, pVmcs->u64AddrEptpList.u)); + LogRel((" %sVMREAD-bitmap addr = %#RX64\n", pszPrefix, pVmcs->u64AddrVmreadBitmap.u)); + LogRel((" %sVMWRITE-bitmap addr = %#RX64\n", pszPrefix, pVmcs->u64AddrVmwriteBitmap.u)); + LogRel((" %sVirt-Xcpt info addr = %#RX64\n", pszPrefix, pVmcs->u64AddrXcptVeInfo.u)); + LogRel((" %sXSS-exiting bitmap = %#RX64\n", pszPrefix, pVmcs->u64XssExitBitmap.u)); + LogRel((" %sENCLS-exiting bitmap = %#RX64\n", pszPrefix, pVmcs->u64EnclsExitBitmap.u)); + LogRel((" %sSPP table pointer = %#RX64\n", pszPrefix, pVmcs->u64SppTablePtr.u)); + LogRel((" %sTSC multiplier = %#RX64\n", pszPrefix, pVmcs->u64TscMultiplier.u)); + LogRel((" %sENCLV-exiting bitmap = %#RX64\n", pszPrefix, pVmcs->u64EnclvExitBitmap.u)); + + /* Natural width. */ + LogRel((" %sCR0 guest/host mask = %#RX64\n", pszPrefix, pVmcs->u64Cr0Mask.u)); + LogRel((" %sCR4 guest/host mask = %#RX64\n", pszPrefix, pVmcs->u64Cr4Mask.u)); + LogRel((" %sCR0 read shadow = %#RX64\n", pszPrefix, pVmcs->u64Cr0ReadShadow.u)); + LogRel((" %sCR4 read shadow = %#RX64\n", pszPrefix, pVmcs->u64Cr4ReadShadow.u)); + LogRel((" %sCR3-target 0 = %#RX64\n", pszPrefix, pVmcs->u64Cr3Target0.u)); + LogRel((" %sCR3-target 1 = %#RX64\n", pszPrefix, pVmcs->u64Cr3Target1.u)); + LogRel((" %sCR3-target 2 = %#RX64\n", pszPrefix, pVmcs->u64Cr3Target2.u)); + LogRel((" %sCR3-target 3 = %#RX64\n", pszPrefix, pVmcs->u64Cr3Target3.u)); + } + + /* Guest state. */ + { + LogRel(("%sGuest state:\n", pszPrefix)); + + /* 16-bit. */ + HMVMX_DUMP_GUEST_SEGREG(pVmcs, Cs, "cs", pszPrefix); + HMVMX_DUMP_GUEST_SEGREG(pVmcs, Ss, "ss", pszPrefix); + HMVMX_DUMP_GUEST_SEGREG(pVmcs, Es, "es", pszPrefix); + HMVMX_DUMP_GUEST_SEGREG(pVmcs, Ds, "ds", pszPrefix); + HMVMX_DUMP_GUEST_SEGREG(pVmcs, Fs, "fs", pszPrefix); + HMVMX_DUMP_GUEST_SEGREG(pVmcs, Gs, "gs", pszPrefix); + HMVMX_DUMP_GUEST_SEGREG(pVmcs, Ldtr, "ldtr", pszPrefix); + HMVMX_DUMP_GUEST_SEGREG(pVmcs, Tr, "tr", pszPrefix); + HMVMX_DUMP_GUEST_XDTR( pVmcs, Gdtr, "gdtr", pszPrefix); + HMVMX_DUMP_GUEST_XDTR( pVmcs, Idtr, "idtr", pszPrefix); + LogRel((" %sInterrupt status = %#RX16\n", pszPrefix, pVmcs->u16GuestIntStatus)); + LogRel((" %sPML index = %#RX16\n", pszPrefix, pVmcs->u16PmlIndex)); + + /* 32-bit. */ + LogRel((" %sInterruptibility state = %#RX32\n", pszPrefix, pVmcs->u32GuestIntrState)); + LogRel((" %sActivity state = %#RX32\n", pszPrefix, pVmcs->u32GuestActivityState)); + LogRel((" %sSMBASE = %#RX32\n", pszPrefix, pVmcs->u32GuestSmBase)); + LogRel((" %sSysEnter CS = %#RX32\n", pszPrefix, pVmcs->u32GuestSysenterCS)); + LogRel((" %sVMX-preemption timer value = %#RX32\n", pszPrefix, pVmcs->u32PreemptTimer)); + + /* 64-bit. */ + LogRel((" %sVMCS link ptr = %#RX64\n", pszPrefix, pVmcs->u64VmcsLinkPtr.u)); + LogRel((" %sDBGCTL = %#RX64\n", pszPrefix, pVmcs->u64GuestDebugCtlMsr.u)); + LogRel((" %sPAT = %#RX64\n", pszPrefix, pVmcs->u64GuestPatMsr.u)); + LogRel((" %sEFER = %#RX64\n", pszPrefix, pVmcs->u64GuestEferMsr.u)); + LogRel((" %sPERFGLOBALCTRL = %#RX64\n", pszPrefix, pVmcs->u64GuestPerfGlobalCtlMsr.u)); + LogRel((" %sPDPTE 0 = %#RX64\n", pszPrefix, pVmcs->u64GuestPdpte0.u)); + LogRel((" %sPDPTE 1 = %#RX64\n", pszPrefix, pVmcs->u64GuestPdpte1.u)); + LogRel((" %sPDPTE 2 = %#RX64\n", pszPrefix, pVmcs->u64GuestPdpte2.u)); + LogRel((" %sPDPTE 3 = %#RX64\n", pszPrefix, pVmcs->u64GuestPdpte3.u)); + LogRel((" %sBNDCFGS = %#RX64\n", pszPrefix, pVmcs->u64GuestBndcfgsMsr.u)); + LogRel((" %sRTIT_CTL = %#RX64\n", pszPrefix, pVmcs->u64GuestRtitCtlMsr.u)); + + /* Natural width. */ + LogRel((" %scr0 = %#RX64\n", pszPrefix, pVmcs->u64GuestCr0.u)); + LogRel((" %scr3 = %#RX64\n", pszPrefix, pVmcs->u64GuestCr3.u)); + LogRel((" %scr4 = %#RX64\n", pszPrefix, pVmcs->u64GuestCr4.u)); + LogRel((" %sdr7 = %#RX64\n", pszPrefix, pVmcs->u64GuestDr7.u)); + LogRel((" %srsp = %#RX64\n", pszPrefix, pVmcs->u64GuestRsp.u)); + LogRel((" %srip = %#RX64\n", pszPrefix, pVmcs->u64GuestRip.u)); + LogRel((" %srflags = %#RX64\n", pszPrefix, pVmcs->u64GuestRFlags.u)); + LogRel((" %sPending debug xcpts = %#RX64\n", pszPrefix, pVmcs->u64GuestPendingDbgXcpts.u)); + LogRel((" %sSysEnter ESP = %#RX64\n", pszPrefix, pVmcs->u64GuestSysenterEsp.u)); + LogRel((" %sSysEnter EIP = %#RX64\n", pszPrefix, pVmcs->u64GuestSysenterEip.u)); + } + + /* Host state. */ + { + LogRel(("%sHost state:\n", pszPrefix)); + + /* 16-bit. */ + LogRel((" %scs = %#RX16\n", pszPrefix, pVmcs->HostCs)); + LogRel((" %sss = %#RX16\n", pszPrefix, pVmcs->HostSs)); + LogRel((" %sds = %#RX16\n", pszPrefix, pVmcs->HostDs)); + LogRel((" %ses = %#RX16\n", pszPrefix, pVmcs->HostEs)); + HMVMX_DUMP_HOST_FS_GS_TR(pVmcs, Fs, "fs", pszPrefix); + HMVMX_DUMP_HOST_FS_GS_TR(pVmcs, Gs, "gs", pszPrefix); + HMVMX_DUMP_HOST_FS_GS_TR(pVmcs, Tr, "tr", pszPrefix); + HMVMX_DUMP_HOST_XDTR(pVmcs, Gdtr, "gdtr", pszPrefix); + HMVMX_DUMP_HOST_XDTR(pVmcs, Idtr, "idtr", pszPrefix); + + /* 32-bit. */ + LogRel((" %sSysEnter CS = %#RX32\n", pszPrefix, pVmcs->u32HostSysenterCs)); + + /* 64-bit. */ + LogRel((" %sEFER = %#RX64\n", pszPrefix, pVmcs->u64HostEferMsr.u)); + LogRel((" %sPAT = %#RX64\n", pszPrefix, pVmcs->u64HostPatMsr.u)); + LogRel((" %sPERFGLOBALCTRL = %#RX64\n", pszPrefix, pVmcs->u64HostPerfGlobalCtlMsr.u)); + + /* Natural width. */ + LogRel((" %scr0 = %#RX64\n", pszPrefix, pVmcs->u64HostCr0.u)); + LogRel((" %scr3 = %#RX64\n", pszPrefix, pVmcs->u64HostCr3.u)); + LogRel((" %scr4 = %#RX64\n", pszPrefix, pVmcs->u64HostCr4.u)); + LogRel((" %sSysEnter ESP = %#RX64\n", pszPrefix, pVmcs->u64HostSysenterEsp.u)); + LogRel((" %sSysEnter EIP = %#RX64\n", pszPrefix, pVmcs->u64HostSysenterEip.u)); + LogRel((" %srsp = %#RX64\n", pszPrefix, pVmcs->u64HostRsp.u)); + LogRel((" %srip = %#RX64\n", pszPrefix, pVmcs->u64HostRip.u)); + } + + /* Read-only fields. */ + { + LogRel(("%sRead-only data fields:\n", pszPrefix)); + + /* 16-bit (none currently). */ + + /* 32-bit. */ + uint32_t const uExitReason = pVmcs->u32RoExitReason; + LogRel((" %sExit reason = %u (%s)\n", pszPrefix, uExitReason, HMGetVmxExitName(uExitReason))); + LogRel((" %sExit qualification = %#RX64\n", pszPrefix, pVmcs->u64RoExitQual.u)); + LogRel((" %sVM-instruction error = %#RX32\n", pszPrefix, pVmcs->u32RoVmInstrError)); + LogRel((" %sVM-exit intr info = %#RX32\n", pszPrefix, pVmcs->u32RoExitIntInfo)); + { + uint32_t const fInfo = pVmcs->u32RoExitIntInfo; + uint8_t const uType = VMX_EXIT_INT_INFO_TYPE(fInfo); + LogRel((" %sValid = %RTbool\n", pszPrefix, VMX_EXIT_INT_INFO_IS_VALID(fInfo))); + LogRel((" %sType = %#x (%s)\n", pszPrefix, uType, VMXGetExitIntInfoTypeDesc(uType))); + LogRel((" %sVector = %#x\n", pszPrefix, VMX_EXIT_INT_INFO_VECTOR(fInfo))); + LogRel((" %sNMI-unblocking-IRET = %RTbool\n", pszPrefix, VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(fInfo))); + LogRel((" %sError-code valid = %RTbool\n", pszPrefix, VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(fInfo))); + } + LogRel((" %sVM-exit intr error-code = %#RX32\n", pszPrefix, pVmcs->u32RoExitIntErrCode)); + LogRel((" %sIDT-vectoring info = %#RX32\n", pszPrefix, pVmcs->u32RoIdtVectoringInfo)); + { + uint32_t const fInfo = pVmcs->u32RoIdtVectoringInfo; + uint8_t const uType = VMX_IDT_VECTORING_INFO_TYPE(fInfo); + LogRel((" %sValid = %RTbool\n", pszPrefix, VMX_IDT_VECTORING_INFO_IS_VALID(fInfo))); + LogRel((" %sType = %#x (%s)\n", pszPrefix, uType, VMXGetIdtVectoringInfoTypeDesc(uType))); + LogRel((" %sVector = %#x\n", pszPrefix, VMX_IDT_VECTORING_INFO_VECTOR(fInfo))); + LogRel((" %sError-code valid = %RTbool\n", pszPrefix, VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(fInfo))); + } + LogRel((" %sIDT-vectoring error-code = %#RX32\n", pszPrefix, pVmcs->u32RoIdtVectoringErrCode)); + LogRel((" %sVM-exit instruction length = %u bytes\n", pszPrefix, pVmcs->u32RoExitInstrLen)); + LogRel((" %sVM-exit instruction info = %#RX64\n", pszPrefix, pVmcs->u32RoExitInstrInfo)); + + /* 64-bit. */ + LogRel((" %sGuest-physical addr = %#RX64\n", pszPrefix, pVmcs->u64RoGuestPhysAddr.u)); + + /* Natural width. */ + LogRel((" %sI/O RCX = %#RX64\n", pszPrefix, pVmcs->u64RoIoRcx.u)); + LogRel((" %sI/O RSI = %#RX64\n", pszPrefix, pVmcs->u64RoIoRsi.u)); + LogRel((" %sI/O RDI = %#RX64\n", pszPrefix, pVmcs->u64RoIoRdi.u)); + LogRel((" %sI/O RIP = %#RX64\n", pszPrefix, pVmcs->u64RoIoRip.u)); + LogRel((" %sGuest-linear addr = %#RX64\n", pszPrefix, pVmcs->u64RoGuestLinearAddr.u)); + } + +#undef HMVMX_DUMP_HOST_XDTR +#undef HMVMX_DUMP_HOST_FS_GS_TR +#undef HMVMX_DUMP_GUEST_SEGREG +#undef HMVMX_DUMP_GUEST_XDTR +} + + +/** + * Gets the active (in use) VMCS info. object for the specified VCPU. + * + * This is either the guest or nested-guest VMCS info. and need not necessarily + * pertain to the "current" VMCS (in the VMX definition of the term). For instance, + * if the VM-entry failed due to an invalid-guest state, we may have "cleared" the + * current VMCS while returning to ring-3. However, the VMCS info. object for that + * VMCS would still be active and returned here so that we could dump the VMCS + * fields to ring-3 for diagnostics. This function is thus only used to + * distinguish between the nested-guest or guest VMCS. + * + * @returns The active VMCS information. + * @param pVCpu The cross context virtual CPU structure. + * + * @thread EMT. + * @remarks This function may be called with preemption or interrupts disabled! + */ +VMM_INT_DECL(PVMXVMCSINFOSHARED) hmGetVmxActiveVmcsInfoShared(PVMCPUCC pVCpu) +{ +#ifdef IN_RING0 + if (!pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs) +#else + if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3) +#endif + return &pVCpu->hm.s.vmx.VmcsInfo; + return &pVCpu->hm.s.vmx.VmcsInfoNstGst; +} + + +/** + * Converts a VMX event type into an appropriate TRPM event type. + * + * @returns TRPM event. + * @param uIntInfo The VMX event. + */ +VMM_INT_DECL(TRPMEVENT) HMVmxEventTypeToTrpmEventType(uint32_t uIntInfo) +{ + Assert(VMX_IDT_VECTORING_INFO_IS_VALID(uIntInfo)); + + TRPMEVENT enmTrapType; + uint8_t const uType = VMX_IDT_VECTORING_INFO_TYPE(uIntInfo); + uint8_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(uIntInfo); + + switch (uType) + { + case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT: + enmTrapType = TRPM_HARDWARE_INT; + break; + + case VMX_IDT_VECTORING_INFO_TYPE_NMI: + case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT: + enmTrapType = TRPM_TRAP; + break; + + case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* INT1 (ICEBP). */ + Assert(uVector == X86_XCPT_DB); NOREF(uVector); + enmTrapType = TRPM_SOFTWARE_INT; + break; + + case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* INT3 (#BP) and INTO (#OF) */ + Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF); NOREF(uVector); + enmTrapType = TRPM_SOFTWARE_INT; + break; + + case VMX_IDT_VECTORING_INFO_TYPE_SW_INT: + enmTrapType = TRPM_SOFTWARE_INT; + break; + + default: + AssertMsgFailed(("Invalid trap type %#x\n", uType)); + enmTrapType = TRPM_32BIT_HACK; + break; + } + + return enmTrapType; +} + + +/** + * Converts a TRPM event type into an appropriate VMX event type. + * + * @returns VMX event type mask. + * @param uVector The event vector. + * @param enmTrpmEvent The TRPM event. + * @param fIcebp Whether the \#DB vector is caused by an INT1/ICEBP + * instruction. + */ +VMM_INT_DECL(uint32_t) HMTrpmEventTypeToVmxEventType(uint8_t uVector, TRPMEVENT enmTrpmEvent, bool fIcebp) +{ + uint32_t uIntInfoType = 0; + if (enmTrpmEvent == TRPM_TRAP) + { + Assert(!fIcebp); + switch (uVector) + { + case X86_XCPT_NMI: + uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_NMI << VMX_IDT_VECTORING_INFO_TYPE_SHIFT); + break; + + case X86_XCPT_BP: + case X86_XCPT_OF: + uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT); + break; + + case X86_XCPT_PF: + case X86_XCPT_DF: + case X86_XCPT_TS: + case X86_XCPT_NP: + case X86_XCPT_SS: + case X86_XCPT_GP: + case X86_XCPT_AC: + uIntInfoType |= VMX_IDT_VECTORING_INFO_ERROR_CODE_VALID; + RT_FALL_THRU(); + default: + uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT); + break; + } + } + else if (enmTrpmEvent == TRPM_HARDWARE_INT) + { + Assert(!fIcebp); + uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_EXT_INT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT); + } + else if (enmTrpmEvent == TRPM_SOFTWARE_INT) + { + switch (uVector) + { + case X86_XCPT_BP: + case X86_XCPT_OF: + uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT); + break; + + case X86_XCPT_DB: + { + if (fIcebp) + { + uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT); + break; + } + RT_FALL_THRU(); + } + default: + uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_SW_INT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT); + break; + } + } + else + AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent)); + return uIntInfoType; +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** + * Notification callback for when a VM-exit happens outside VMX R0 code (e.g. in + * IEM). + * + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks Can be called from ring-0 as well as ring-3. + */ +VMM_INT_DECL(void) HMNotifyVmxNstGstVmexit(PVMCPU pVCpu) +{ + LogFlowFunc(("\n")); + + /* + * Transitions to ring-3 flag a full CPU-state change except if we transition to ring-3 + * in response to a physical CPU interrupt as no changes to the guest-CPU state are + * expected (see VINF_EM_RAW_INTERRUPT handling in hmR0VmxExitToRing3). + * + * However, with nested-guests, the state -can- change on trips to ring-3 for we might + * try to inject a nested-guest physical interrupt and cause a VMX_EXIT_EXT_INT VM-exit + * for the nested-guest from ring-3. + * + * Signalling reload of just the guest-CPU state that changed with the VM-exit is -not- + * sufficient since HM also needs to reload state related to VM-entry/VM-exit controls + * etc. So signal reloading of the entire state. It does not seem worth making this any + * more fine grained at the moment. + */ + CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_ALL); + ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); + + /* + * Make sure we need to merge the guest VMCS controls with the nested-guest + * VMCS controls on the next nested-guest VM-entry. + */ + pVCpu->hm.s.vmx.fMergedNstGstCtls = false; + + /* + * Flush the TLB before entering the outer guest execution (mainly required since the + * APIC-access guest-physical address would have changed and probably more things in + * the future). + */ + pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true; + + /** @todo Handle releasing of the page-mapping lock later. */ +#if 0 + if (pVCpu->hm.s.vmx.fVirtApicPageLocked) + { + PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->hm.s.vmx.PgMapLockVirtApic); + pVCpu->hm.s.vmx.fVirtApicPageLocked = false; + } +#endif +} + + +/** + * Notification callback for when the nested hypervisor's current VMCS is loaded or + * changed outside VMX R0 code (e.g. in IEM). + * + * This need -not- be called for modifications to the nested hypervisor's current + * VMCS when the guest is in VMX non-root mode as VMCS shadowing is not applicable + * there. + * + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks Can be called from ring-0 as well as ring-3. + */ +VMM_INT_DECL(void) HMNotifyVmxNstGstCurrentVmcsChanged(PVMCPU pVCpu) +{ + CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_HWVIRT); + ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, CPUMCTX_EXTRN_HWVIRT); + + /* + * Make sure we need to copy the nested hypervisor's current VMCS into the shadow VMCS + * on the next guest VM-entry. + */ + pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = false; +} + +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + diff --git a/src/VBox/VMM/VMMAll/IEMAll.cpp b/src/VBox/VMM/VMMAll/IEMAll.cpp new file mode 100644 index 00000000..106c51c5 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAll.cpp @@ -0,0 +1,11599 @@ +/* $Id: IEMAll.cpp $ */ +/** @file + * IEM - Interpreted Execution Manager - All Contexts. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/** @page pg_iem IEM - Interpreted Execution Manager + * + * The interpreted exeuction manager (IEM) is for executing short guest code + * sequences that are causing too many exits / virtualization traps. It will + * also be used to interpret single instructions, thus replacing the selective + * interpreters in EM and IOM. + * + * Design goals: + * - Relatively small footprint, although we favour speed and correctness + * over size. + * - Reasonably fast. + * - Correctly handle lock prefixed instructions. + * - Complete instruction set - eventually. + * - Refactorable into a recompiler, maybe. + * - Replace EMInterpret*. + * + * Using the existing disassembler has been considered, however this is thought + * to conflict with speed as the disassembler chews things a bit too much while + * leaving us with a somewhat complicated state to interpret afterwards. + * + * + * The current code is very much work in progress. You've been warned! + * + * + * @section sec_iem_fpu_instr FPU Instructions + * + * On x86 and AMD64 hosts, the FPU instructions are implemented by executing the + * same or equivalent instructions on the host FPU. To make life easy, we also + * let the FPU prioritize the unmasked exceptions for us. This however, only + * works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13 + * for FPU exception delivery, because with CR0.NE=0 there is a window where we + * can trigger spurious FPU exceptions. + * + * The guest FPU state is not loaded into the host CPU and kept there till we + * leave IEM because the calling conventions have declared an all year open + * season on much of the FPU state. For instance an innocent looking call to + * memcpy might end up using a whole bunch of XMM or MM registers if the + * particular implementation finds it worthwhile. + * + * + * @section sec_iem_logging Logging + * + * The IEM code uses the \"IEM\" log group for the main logging. The different + * logging levels/flags are generally used for the following purposes: + * - Level 1 (Log) : Errors, exceptions, interrupts and such major events. + * - Flow (LogFlow) : Basic enter/exit IEM state info. + * - Level 2 (Log2) : ? + * - Level 3 (Log3) : More detailed enter/exit IEM state info. + * - Level 4 (Log4) : Decoding mnemonics w/ EIP. + * - Level 5 (Log5) : Decoding details. + * - Level 6 (Log6) : Enables/disables the lockstep comparison with REM. + * - Level 7 (Log7) : iret++ execution logging. + * - Level 8 (Log8) : Memory writes. + * - Level 9 (Log9) : Memory reads. + * - Level 10 (Log10): TLBs. + * - Level 11 (Log11): Unmasked FPU exceptions. + * + * The SVM (AMD-V) and VMX (VT-x) code has the following assignments: + * - Level 1 (Log) : Errors and other major events. + * - Flow (LogFlow) : Misc flow stuff (cleanup?) + * - Level 2 (Log2) : VM exits. + */ + +/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */ +#ifdef _MSC_VER +# pragma warning(disable:4505) +#endif + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_IEM +#define VMCPU_INCL_CPUM_GST_CTX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +# include +# include +#endif +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +# include +#endif +#include +#include +#include +#include "IEMInternal.h" +#include +#include +#include +#include +#include +#include +#include +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +# include +#elif defined(RT_ARCH_ARM64) || defined(RT_ARCH_ARM32) +# include +#endif +#include +#include +#include + +#include "IEMInline.h" + + +/********************************************************************************************************************************* +* Structures and Typedefs * +*********************************************************************************************************************************/ +/** + * CPU exception classes. + */ +typedef enum IEMXCPTCLASS +{ + IEMXCPTCLASS_BENIGN, + IEMXCPTCLASS_CONTRIBUTORY, + IEMXCPTCLASS_PAGE_FAULT, + IEMXCPTCLASS_DOUBLE_FAULT +} IEMXCPTCLASS; + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +#if defined(IEM_LOG_MEMORY_WRITES) +/** What IEM just wrote. */ +uint8_t g_abIemWrote[256]; +/** How much IEM just wrote. */ +size_t g_cbIemWrote; +#endif + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, + uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT; + + +/** + * Slow path of iemInitDecoder() and iemInitExec() that checks what kind of + * breakpoints are enabled. + * + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + */ +void iemInitPendingBreakpointsSlow(PVMCPUCC pVCpu) +{ + /* + * Process guest breakpoints. + */ +#define PROCESS_ONE_BP(a_fDr7, a_iBp) do { \ + if (a_fDr7 & X86_DR7_L_G(a_iBp)) \ + { \ + switch (X86_DR7_GET_RW(a_fDr7, a_iBp)) \ + { \ + case X86_DR7_RW_EO: \ + pVCpu->iem.s.fPendingInstructionBreakpoints = true; \ + break; \ + case X86_DR7_RW_WO: \ + case X86_DR7_RW_RW: \ + pVCpu->iem.s.fPendingDataBreakpoints = true; \ + break; \ + case X86_DR7_RW_IO: \ + pVCpu->iem.s.fPendingIoBreakpoints = true; \ + break; \ + } \ + } \ + } while (0) + uint32_t const fGstDr7 = (uint32_t)pVCpu->cpum.GstCtx.dr[7]; + if (fGstDr7 & X86_DR7_ENABLED_MASK) + { + PROCESS_ONE_BP(fGstDr7, 0); + PROCESS_ONE_BP(fGstDr7, 1); + PROCESS_ONE_BP(fGstDr7, 2); + PROCESS_ONE_BP(fGstDr7, 3); + } + + /* + * Process hypervisor breakpoints. + */ + uint32_t const fHyperDr7 = DBGFBpGetDR7(pVCpu->CTX_SUFF(pVM)); + if (fHyperDr7 & X86_DR7_ENABLED_MASK) + { + PROCESS_ONE_BP(fHyperDr7, 0); + PROCESS_ONE_BP(fHyperDr7, 1); + PROCESS_ONE_BP(fHyperDr7, 2); + PROCESS_ONE_BP(fHyperDr7, 3); + } +} + + +/** + * Initializes the decoder state. + * + * iemReInitDecoder is mostly a copy of this function. + * + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param fBypassHandlers Whether to bypass access handlers. + * @param fDisregardLock Whether to disregard the LOCK prefix. + */ +DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock) +{ + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK); + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr)); + + pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); + IEMMODE enmMode = iemCalcCpuMode(pVCpu); + pVCpu->iem.s.enmCpuMode = enmMode; + pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */ + pVCpu->iem.s.enmEffAddrMode = enmMode; + if (enmMode != IEMMODE_64BIT) + { + pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */ + pVCpu->iem.s.enmEffOpSize = enmMode; + } + else + { + pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT; + pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT; + } + pVCpu->iem.s.fPrefixes = 0; + pVCpu->iem.s.uRexReg = 0; + pVCpu->iem.s.uRexB = 0; + pVCpu->iem.s.uRexIndex = 0; + pVCpu->iem.s.idxPrefix = 0; + pVCpu->iem.s.uVex3rdReg = 0; + pVCpu->iem.s.uVexLength = 0; + pVCpu->iem.s.fEvexStuff = 0; + pVCpu->iem.s.iEffSeg = X86_SREG_DS; +#ifdef IEM_WITH_CODE_TLB + pVCpu->iem.s.pbInstrBuf = NULL; + pVCpu->iem.s.offInstrNextByte = 0; + pVCpu->iem.s.offCurInstrStart = 0; +# ifdef VBOX_STRICT + pVCpu->iem.s.cbInstrBuf = UINT16_MAX; + pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX; + pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff); +# endif +#else + pVCpu->iem.s.offOpcode = 0; + pVCpu->iem.s.cbOpcode = 0; +#endif + pVCpu->iem.s.offModRm = 0; + pVCpu->iem.s.cActiveMappings = 0; + pVCpu->iem.s.iNextMapping = 0; + pVCpu->iem.s.rcPassUp = VINF_SUCCESS; + pVCpu->iem.s.fBypassHandlers = fBypassHandlers; + pVCpu->iem.s.fDisregardLock = fDisregardLock; + pVCpu->iem.s.fPendingInstructionBreakpoints = false; + pVCpu->iem.s.fPendingDataBreakpoints = false; + pVCpu->iem.s.fPendingIoBreakpoints = false; + if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK) + && pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0)) + { /* likely */ } + else + iemInitPendingBreakpointsSlow(pVCpu); + +#ifdef DBGFTRACE_ENABLED + switch (enmMode) + { + case IEMMODE_64BIT: + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip); + break; + case IEMMODE_32BIT: + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip); + break; + case IEMMODE_16BIT: + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip); + break; + } +#endif +} + + +/** + * Reinitializes the decoder state 2nd+ loop of IEMExecLots. + * + * This is mostly a copy of iemInitDecoder. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu) +{ + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr)); + + pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); /** @todo this should be updated during execution! */ + IEMMODE enmMode = iemCalcCpuMode(pVCpu); + pVCpu->iem.s.enmCpuMode = enmMode; /** @todo this should be updated during execution! */ + pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */ + pVCpu->iem.s.enmEffAddrMode = enmMode; + if (enmMode != IEMMODE_64BIT) + { + pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */ + pVCpu->iem.s.enmEffOpSize = enmMode; + } + else + { + pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT; + pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT; + } + pVCpu->iem.s.fPrefixes = 0; + pVCpu->iem.s.uRexReg = 0; + pVCpu->iem.s.uRexB = 0; + pVCpu->iem.s.uRexIndex = 0; + pVCpu->iem.s.idxPrefix = 0; + pVCpu->iem.s.uVex3rdReg = 0; + pVCpu->iem.s.uVexLength = 0; + pVCpu->iem.s.fEvexStuff = 0; + pVCpu->iem.s.iEffSeg = X86_SREG_DS; +#ifdef IEM_WITH_CODE_TLB + if (pVCpu->iem.s.pbInstrBuf) + { + uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT + ? pVCpu->cpum.GstCtx.rip + : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base) + - pVCpu->iem.s.uInstrBufPc; + if (off < pVCpu->iem.s.cbInstrBufTotal) + { + pVCpu->iem.s.offInstrNextByte = (uint32_t)off; + pVCpu->iem.s.offCurInstrStart = (uint16_t)off; + if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal) + pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15; + else + pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal; + } + else + { + pVCpu->iem.s.pbInstrBuf = NULL; + pVCpu->iem.s.offInstrNextByte = 0; + pVCpu->iem.s.offCurInstrStart = 0; + pVCpu->iem.s.cbInstrBuf = 0; + pVCpu->iem.s.cbInstrBufTotal = 0; + } + } + else + { + pVCpu->iem.s.offInstrNextByte = 0; + pVCpu->iem.s.offCurInstrStart = 0; + pVCpu->iem.s.cbInstrBuf = 0; + pVCpu->iem.s.cbInstrBufTotal = 0; + } +#else + pVCpu->iem.s.cbOpcode = 0; + pVCpu->iem.s.offOpcode = 0; +#endif + pVCpu->iem.s.offModRm = 0; + Assert(pVCpu->iem.s.cActiveMappings == 0); + pVCpu->iem.s.iNextMapping = 0; + Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS); + Assert(pVCpu->iem.s.fBypassHandlers == false); + +#ifdef DBGFTRACE_ENABLED + switch (enmMode) + { + case IEMMODE_64BIT: + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip); + break; + case IEMMODE_32BIT: + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip); + break; + case IEMMODE_16BIT: + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip); + break; + } +#endif +} + + + +/** + * Prefetch opcodes the first time when starting executing. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param fBypassHandlers Whether to bypass access handlers. + * @param fDisregardLock Whether to disregard LOCK prefixes. + * + * @todo Combine fDisregardLock and fBypassHandlers into a flag parameter and + * store them as such. + */ +static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock) RT_NOEXCEPT +{ + iemInitDecoder(pVCpu, fBypassHandlers, fDisregardLock); + +#ifndef IEM_WITH_CODE_TLB + /* + * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap. + * + * First translate CS:rIP to a physical address. + * + * Note! The iemOpcodeFetchMoreBytes code depends on this here code to fetch + * all relevant bytes from the first page, as it ASSUMES it's only ever + * called for dealing with CS.LIM, page crossing and instructions that + * are too long. + */ + uint32_t cbToTryRead; + RTGCPTR GCPtrPC; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + cbToTryRead = GUEST_PAGE_SIZE; + GCPtrPC = pVCpu->cpum.GstCtx.rip; + if (IEM_IS_CANONICAL(GCPtrPC)) + cbToTryRead = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK); + else + return iemRaiseGeneralProtectionFault0(pVCpu); + } + else + { + uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip; + AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit) + cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1; + else + return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION); + if (cbToTryRead) { /* likely */ } + else /* overflowed */ + { + Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX); + cbToTryRead = UINT32_MAX; + } + GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32; + Assert(GCPtrPC <= UINT32_MAX); + } + + PGMPTWALK Walk; + int rc = PGMGstGetPage(pVCpu, GCPtrPC, &Walk); + if (RT_SUCCESS(rc)) + Assert(Walk.fSucceeded); /* probable. */ + else + { + Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc)); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */); +# endif + return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, rc); + } + if ((Walk.fEffective & X86_PTE_US) || pVCpu->iem.s.uCpl != 3) { /* likely */ } + else + { + Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC)); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +# endif + return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED); + } + if (!(Walk.fEffective & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ } + else + { + Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC)); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +# endif + return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED); + } + RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrPC & GUEST_PAGE_OFFSET_MASK); + /** @todo Check reserved bits and such stuff. PGM is better at doing + * that, so do it when implementing the guest virtual address + * TLB... */ + + /* + * Read the bytes at this address. + */ + uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK); + if (cbToTryRead > cbLeftOnPage) + cbToTryRead = cbLeftOnPage; + if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode)) + cbToTryRead = sizeof(pVCpu->iem.s.abOpcode); + + if (!pVCpu->iem.s.fBypassHandlers) + { + VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + { + Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n", + GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead)); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } + else + { + Log((RT_SUCCESS(rcStrict) + ? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n" + : "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n", + GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead)); + return rcStrict; + } + } + else + { + rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n", + GCPtrPC, GCPhys, rc, cbToTryRead)); + return rc; + } + } + pVCpu->iem.s.cbOpcode = cbToTryRead; +#endif /* !IEM_WITH_CODE_TLB */ + return VINF_SUCCESS; +} + + +/** + * Invalidates the IEM TLBs. + * + * This is called internally as well as by PGM when moving GC mappings. + * + * @param pVCpu The cross context virtual CPU structure of the calling + * thread. + */ +VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu) +{ +#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB) + Log10(("IEMTlbInvalidateAll\n")); +# ifdef IEM_WITH_CODE_TLB + pVCpu->iem.s.cbInstrBufTotal = 0; + pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR; + if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0) + { /* very likely */ } + else + { + pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR; + unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries); + while (i-- > 0) + pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0; + } +# endif + +# ifdef IEM_WITH_DATA_TLB + pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR; + if (pVCpu->iem.s.DataTlb.uTlbRevision != 0) + { /* very likely */ } + else + { + pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR; + unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries); + while (i-- > 0) + pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0; + } +# endif +#else + RT_NOREF(pVCpu); +#endif +} + + +/** + * Invalidates a page in the TLBs. + * + * @param pVCpu The cross context virtual CPU structure of the calling + * thread. + * @param GCPtr The address of the page to invalidate + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr) +{ +#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB) + Log10(("IEMTlbInvalidatePage: GCPtr=%RGv\n", GCPtr)); + GCPtr = IEMTLB_CALC_TAG_NO_REV(GCPtr); + Assert(!(GCPtr >> (48 - X86_PAGE_SHIFT))); + uintptr_t const idx = IEMTLB_TAG_TO_INDEX(GCPtr); + +# ifdef IEM_WITH_CODE_TLB + if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.CodeTlb.uTlbRevision)) + { + pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0; + if (GCPtr == IEMTLB_CALC_TAG_NO_REV(pVCpu->iem.s.uInstrBufPc)) + pVCpu->iem.s.cbInstrBufTotal = 0; + } +# endif + +# ifdef IEM_WITH_DATA_TLB + if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.DataTlb.uTlbRevision)) + pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0; +# endif +#else + NOREF(pVCpu); NOREF(GCPtr); +#endif +} + + +#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB) +/** + * Invalid both TLBs slow fashion following a rollover. + * + * Worker for IEMTlbInvalidateAllPhysical, + * IEMTlbInvalidateAllPhysicalAllCpus, iemOpcodeFetchBytesJmp, iemMemMap, + * iemMemMapJmp and others. + * + * @thread EMT(pVCpu) + */ +static void IEMTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu) +{ + Log10(("IEMTlbInvalidateAllPhysicalSlow\n")); + ASMAtomicWriteU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2); + ASMAtomicWriteU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2); + + unsigned i; +# ifdef IEM_WITH_CODE_TLB + i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries); + while (i-- > 0) + { + pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL; + pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ + | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PHYS_REV); + } +# endif +# ifdef IEM_WITH_DATA_TLB + i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries); + while (i-- > 0) + { + pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL; + pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ + | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PHYS_REV); + } +# endif + +} +#endif + + +/** + * Invalidates the host physical aspects of the IEM TLBs. + * + * This is called internally as well as by PGM when moving GC mappings. + * + * @param pVCpu The cross context virtual CPU structure of the calling + * thread. + * @note Currently not used. + */ +VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu) +{ +#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB) + /* Note! This probably won't end up looking exactly like this, but it give an idea... */ + Log10(("IEMTlbInvalidateAllPhysical\n")); + +# ifdef IEM_WITH_CODE_TLB + pVCpu->iem.s.cbInstrBufTotal = 0; +# endif + uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR; + if (RT_LIKELY(uTlbPhysRev > IEMTLB_PHYS_REV_INCR * 2)) + { + pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev; + pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev; + } + else + IEMTlbInvalidateAllPhysicalSlow(pVCpu); +#else + NOREF(pVCpu); +#endif +} + + +/** + * Invalidates the host physical aspects of the IEM TLBs. + * + * This is called internally as well as by PGM when moving GC mappings. + * + * @param pVM The cross context VM structure. + * @param idCpuCaller The ID of the calling EMT if available to the caller, + * otherwise NIL_VMCPUID. + * + * @remarks Caller holds the PGM lock. + */ +VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVMCC pVM, VMCPUID idCpuCaller) +{ +#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB) + PVMCPUCC const pVCpuCaller = idCpuCaller >= pVM->cCpus ? VMMGetCpu(pVM) : VMMGetCpuById(pVM, idCpuCaller); + if (pVCpuCaller) + VMCPU_ASSERT_EMT(pVCpuCaller); + Log10(("IEMTlbInvalidateAllPhysicalAllCpus\n")); + + VMCC_FOR_EACH_VMCPU(pVM) + { +# ifdef IEM_WITH_CODE_TLB + if (pVCpuCaller == pVCpu) + pVCpu->iem.s.cbInstrBufTotal = 0; +# endif + + uint64_t const uTlbPhysRevPrev = ASMAtomicUoReadU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev); + uint64_t uTlbPhysRevNew = uTlbPhysRevPrev + IEMTLB_PHYS_REV_INCR; + if (RT_LIKELY(uTlbPhysRevNew > IEMTLB_PHYS_REV_INCR * 2)) + { /* likely */} + else if (pVCpuCaller == pVCpu) + uTlbPhysRevNew = IEMTLB_PHYS_REV_INCR; + else + { + IEMTlbInvalidateAllPhysicalSlow(pVCpu); + continue; + } + ASMAtomicCmpXchgU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev); + ASMAtomicCmpXchgU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev); + } + VMCC_FOR_EACH_VMCPU_END(pVM); + +#else + RT_NOREF(pVM, idCpuCaller); +#endif +} + +#ifdef IEM_WITH_CODE_TLB + +/** + * Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on + * failure and jumps. + * + * We end up here for a number of reasons: + * - pbInstrBuf isn't yet initialized. + * - Advancing beyond the buffer boundrary (e.g. cross page). + * - Advancing beyond the CS segment limit. + * - Fetching from non-mappable page (e.g. MMIO). + * + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param pvDst Where to return the bytes. + * @param cbDst Number of bytes to read. + * + * @todo Make cbDst = 0 a way of initializing pbInstrBuf? + */ +void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IN_RING3 + for (;;) + { + Assert(cbDst <= 8); + uint32_t offBuf = pVCpu->iem.s.offInstrNextByte; + + /* + * We might have a partial buffer match, deal with that first to make the + * rest simpler. This is the first part of the cross page/buffer case. + */ + if (pVCpu->iem.s.pbInstrBuf != NULL) + { + if (offBuf < pVCpu->iem.s.cbInstrBuf) + { + Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf); + uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte; + memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy); + + cbDst -= cbCopy; + pvDst = (uint8_t *)pvDst + cbCopy; + offBuf += cbCopy; + pVCpu->iem.s.offInstrNextByte += offBuf; + } + } + + /* + * Check segment limit, figuring how much we're allowed to access at this point. + * + * We will fault immediately if RIP is past the segment limit / in non-canonical + * territory. If we do continue, there are one or more bytes to read before we + * end up in trouble and we need to do that first before faulting. + */ + RTGCPTR GCPtrFirst; + uint32_t cbMaxRead; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart); + if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst))) + { /* likely */ } + else + iemRaiseGeneralProtectionFault0Jmp(pVCpu); + cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK); + } + else + { + GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart); + /* Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT); - this is allowed */ + if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit)) + { /* likely */ } + else /** @todo For CPUs older than the 386, we should not necessarily generate \#GP here but wrap around! */ + iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION); + cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1; + if (cbMaxRead != 0) + { /* likely */ } + else + { + /* Overflowed because address is 0 and limit is max. */ + Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX); + cbMaxRead = X86_PAGE_SIZE; + } + GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base; + uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK); + if (cbMaxRead2 < cbMaxRead) + cbMaxRead = cbMaxRead2; + /** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */ + } + + /* + * Get the TLB entry for this piece of code. + */ + uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.CodeTlb, GCPtrFirst); + PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.CodeTlb, uTag); + if (pTlbe->uTag == uTag) + { + /* likely when executing lots of code, otherwise unlikely */ +# ifdef VBOX_WITH_STATISTICS + pVCpu->iem.s.CodeTlb.cTlbHits++; +# endif + } + else + { + pVCpu->iem.s.CodeTlb.cTlbMisses++; + PGMPTWALK Walk; + int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &Walk); + if (RT_FAILURE(rc)) + { +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + /** @todo Nested VMX: Need to handle EPT violation/misconfig here? */ + Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT)); +#endif + Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc)); + iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, rc); + } + + AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1); + Assert(Walk.fSucceeded); + pTlbe->uTag = uTag; + pTlbe->fFlagsAndPhysRev = (~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A)) + | (Walk.fEffective >> X86_PTE_PAE_BIT_NX); + pTlbe->GCPhys = Walk.GCPhys; + pTlbe->pbMappingR3 = NULL; + } + + /* + * Check TLB page table level access flags. + */ + if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_EXEC)) + { + if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3) + { + Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst)); + iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED); + } + if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) + { + Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst)); + iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED); + } + } + + /* + * Look up the physical page info if necessary. + */ + if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev) + { /* not necessary */ } + else + { + AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE); + AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ); + AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3); + AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED); + if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR)) + { /* likely */ } + else + IEMTlbInvalidateAllPhysicalSlow(pVCpu); + pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV + | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_UNASSIGNED); + int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev, + &pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev); + AssertRCStmt(rc, IEM_DO_LONGJMP(pVCpu, rc)); + } + +# if defined(IN_RING3) || defined(IN_RING0) /** @todo fixme */ + /* + * Try do a direct read using the pbMappingR3 pointer. + */ + if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ)) + == pVCpu->iem.s.CodeTlb.uTlbPhysRev) + { + uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK); + pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead; + if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart) + { + pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead); + pVCpu->iem.s.offCurInstrStart = (int16_t)offPg; + } + else + { + uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart; + if (cbInstr + (uint32_t)cbDst <= 15) + { + pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr; + pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr); + } + else + { + Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0)\n", + pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, cbInstr, cbDst)); + iemRaiseGeneralProtectionFault0Jmp(pVCpu); + } + } + if (cbDst <= cbMaxRead) + { + pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst; + pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK; + pVCpu->iem.s.pbInstrBuf = pTlbe->pbMappingR3; + memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst); + return; + } + pVCpu->iem.s.pbInstrBuf = NULL; + + memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead); + pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead; + } +# else +# error "refactor as needed" + /* + * If there is no special read handling, so we can read a bit more and + * put it in the prefetch buffer. + */ + if ( cbDst < cbMaxRead + && (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev) + { + VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys, + &pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + { + Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n", + GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead)); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + AssertStmt(rcStrict == VINF_SUCCESS, IEM_DO_LONGJMP(pVCpu, VBOXSTRICRC_VAL(rcStrict))); + } + else + { + Log((RT_SUCCESS(rcStrict) + ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n" + : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n", + GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead)); + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + } + } +# endif + /* + * Special read handling, so only read exactly what's needed. + * This is a highly unlikely scenario. + */ + else + { + pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++; + + /* Check instruction length. */ + uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart; + if (RT_LIKELY(cbInstr + cbDst <= 15)) + { /* likely */ } + else + { + Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0) [slow]\n", + pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, cbInstr, cbDst)); + iemRaiseGeneralProtectionFault0Jmp(pVCpu); + } + + /* Do the reading. */ + uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead); + VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), + pvDst, cbToRead, PGMACCESSORIGIN_IEM); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + { + Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n", + GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead)); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + AssertStmt(rcStrict == VINF_SUCCESS, IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict))); + } + else + { + Log((RT_SUCCESS(rcStrict) + ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n" + : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n", + GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead)); + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + } + pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead; + if (cbToRead == cbDst) + return; + } + + /* + * More to read, loop. + */ + cbDst -= cbMaxRead; + pvDst = (uint8_t *)pvDst + cbMaxRead; + } +# else /* !IN_RING3 */ + RT_NOREF(pvDst, cbDst); + if (pvDst || cbDst) + IEM_DO_LONGJMP(pVCpu, VERR_INTERNAL_ERROR); +# endif /* !IN_RING3 */ +} + +#else + +/** + * Try fetch at least @a cbMin bytes more opcodes, raise the appropriate + * exception if it fails. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param cbMin The minimum number of bytes relative offOpcode + * that must be read. + */ +VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT +{ + /* + * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap. + * + * First translate CS:rIP to a physical address. + */ + uint8_t const cbOpcode = pVCpu->iem.s.cbOpcode; + uint8_t const offOpcode = pVCpu->iem.s.offOpcode; + uint8_t const cbLeft = cbOpcode - offOpcode; + Assert(cbLeft < cbMin); + Assert(cbOpcode <= sizeof(pVCpu->iem.s.abOpcode)); + + uint32_t cbToTryRead; + RTGCPTR GCPtrNext; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + GCPtrNext = pVCpu->cpum.GstCtx.rip + cbOpcode; + if (!IEM_IS_CANONICAL(GCPtrNext)) + return iemRaiseGeneralProtectionFault0(pVCpu); + cbToTryRead = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK); + } + else + { + uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip; + /* Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT); - this is allowed */ + GCPtrNext32 += cbOpcode; + if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit) + /** @todo For CPUs older than the 386, we should not generate \#GP here but wrap around! */ + return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION); + cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1; + if (!cbToTryRead) /* overflowed */ + { + Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX); + cbToTryRead = UINT32_MAX; + /** @todo check out wrapping around the code segment. */ + } + if (cbToTryRead < cbMin - cbLeft) + return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION); + GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32; + + uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK); + if (cbToTryRead > cbLeftOnPage) + cbToTryRead = cbLeftOnPage; + } + + /* Restrict to opcode buffer space. + + We're making ASSUMPTIONS here based on work done previously in + iemInitDecoderAndPrefetchOpcodes, where bytes from the first page will + be fetched in case of an instruction crossing two pages. */ + if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - cbOpcode) + cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - cbOpcode; + if (RT_LIKELY(cbToTryRead + cbLeft >= cbMin)) + { /* likely */ } + else + { + Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0)\n", + pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, offOpcode, cbMin)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + PGMPTWALK Walk; + int rc = PGMGstGetPage(pVCpu, GCPtrNext, &Walk); + if (RT_FAILURE(rc)) + { + Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc)); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */); +#endif + return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, rc); + } + if (!(Walk.fEffective & X86_PTE_US) && pVCpu->iem.s.uCpl == 3) + { + Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext)); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +#endif + return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED); + } + if ((Walk.fEffective & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) + { + Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext)); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +#endif + return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED); + } + RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrNext & GUEST_PAGE_OFFSET_MASK); + Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, cbOpcode)); + /** @todo Check reserved bits and such stuff. PGM is better at doing + * that, so do it when implementing the guest virtual address + * TLB... */ + + /* + * Read the bytes at this address. + * + * We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already, + * and since PATM should only patch the start of an instruction there + * should be no need to check again here. + */ + if (!pVCpu->iem.s.fBypassHandlers) + { + VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[cbOpcode], + cbToTryRead, PGMACCESSORIGIN_IEM); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + { + Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n", + GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead)); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } + else + { + Log((RT_SUCCESS(rcStrict) + ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n" + : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n", + GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead)); + return rcStrict; + } + } + else + { + rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[cbOpcode], GCPhys, cbToTryRead); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc)); + return rc; + } + } + pVCpu->iem.s.cbOpcode = cbOpcode + cbToTryRead; + Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode)); + + return VINF_SUCCESS; +} + +#endif /* !IEM_WITH_CODE_TLB */ +#ifndef IEM_WITH_SETJMP + +/** + * Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param pb Where to return the opcode byte. + */ +VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; + *pb = pVCpu->iem.s.abOpcode[offOpcode]; + pVCpu->iem.s.offOpcode = offOpcode + 1; + } + else + *pb = 0; + return rcStrict; +} + +#else /* IEM_WITH_SETJMP */ + +/** + * Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error. + * + * @returns The opcode byte. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IEM_WITH_CODE_TLB + uint8_t u8; + iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8); + return u8; +# else + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1); + if (rcStrict == VINF_SUCCESS) + return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++]; + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); +# endif +} + +#endif /* IEM_WITH_SETJMP */ + +#ifndef IEM_WITH_SETJMP + +/** + * Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu16 Where to return the opcode dword. + */ +VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT +{ + uint8_t u8; + VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8); + if (rcStrict == VINF_SUCCESS) + *pu16 = (int8_t)u8; + return rcStrict; +} + + +/** + * Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32 Where to return the opcode dword. + */ +VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT +{ + uint8_t u8; + VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8); + if (rcStrict == VINF_SUCCESS) + *pu32 = (int8_t)u8; + return rcStrict; +} + + +/** + * Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the opcode qword. + */ +VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + uint8_t u8; + VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8); + if (rcStrict == VINF_SUCCESS) + *pu64 = (int8_t)u8; + return rcStrict; +} + +#endif /* !IEM_WITH_SETJMP */ + + +#ifndef IEM_WITH_SETJMP + +/** + * Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu16 Where to return the opcode word. + */ +VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); +# endif + pVCpu->iem.s.offOpcode = offOpcode + 2; + } + else + *pu16 = 0; + return rcStrict; +} + +#else /* IEM_WITH_SETJMP */ + +/** + * Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error + * + * @returns The opcode word. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IEM_WITH_CODE_TLB + uint16_t u16; + iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16); + return u16; +# else + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; + pVCpu->iem.s.offOpcode += 2; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); +# endif + } + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); +# endif +} + +#endif /* IEM_WITH_SETJMP */ + +#ifndef IEM_WITH_SETJMP + +/** + * Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32 Where to return the opcode double word. + */ +VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; + *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); + pVCpu->iem.s.offOpcode = offOpcode + 2; + } + else + *pu32 = 0; + return rcStrict; +} + + +/** + * Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the opcode quad word. + */ +VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; + *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); + pVCpu->iem.s.offOpcode = offOpcode + 2; + } + else + *pu64 = 0; + return rcStrict; +} + +#endif /* !IEM_WITH_SETJMP */ + +#ifndef IEM_WITH_SETJMP + +/** + * Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32 Where to return the opcode dword. + */ +VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3]); +# endif + pVCpu->iem.s.offOpcode = offOpcode + 4; + } + else + *pu32 = 0; + return rcStrict; +} + +#else /* IEM_WITH_SETJMP */ + +/** + * Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error. + * + * @returns The opcode dword. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IEM_WITH_CODE_TLB + uint32_t u32; + iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32); + return u32; +# else + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; + pVCpu->iem.s.offOpcode = offOpcode + 4; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3]); +# endif + } + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); +# endif +} + +#endif /* IEM_WITH_SETJMP */ + +#ifndef IEM_WITH_SETJMP + +/** + * Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the opcode dword. + */ +VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; + *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3]); + pVCpu->iem.s.offOpcode = offOpcode + 4; + } + else + *pu64 = 0; + return rcStrict; +} + + +/** + * Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the opcode qword. + */ +VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; + *pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3]); + pVCpu->iem.s.offOpcode = offOpcode + 4; + } + else + *pu64 = 0; + return rcStrict; +} + +#endif /* !IEM_WITH_SETJMP */ + +#ifndef IEM_WITH_SETJMP + +/** + * Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the opcode qword. + */ +VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3], + pVCpu->iem.s.abOpcode[offOpcode + 4], + pVCpu->iem.s.abOpcode[offOpcode + 5], + pVCpu->iem.s.abOpcode[offOpcode + 6], + pVCpu->iem.s.abOpcode[offOpcode + 7]); +# endif + pVCpu->iem.s.offOpcode = offOpcode + 8; + } + else + *pu64 = 0; + return rcStrict; +} + +#else /* IEM_WITH_SETJMP */ + +/** + * Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error. + * + * @returns The opcode qword. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IEM_WITH_CODE_TLB + uint64_t u64; + iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64); + return u64; +# else + VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8); + if (rcStrict == VINF_SUCCESS) + { + uint8_t offOpcode = pVCpu->iem.s.offOpcode; + pVCpu->iem.s.offOpcode = offOpcode + 8; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3], + pVCpu->iem.s.abOpcode[offOpcode + 4], + pVCpu->iem.s.abOpcode[offOpcode + 5], + pVCpu->iem.s.abOpcode[offOpcode + 6], + pVCpu->iem.s.abOpcode[offOpcode + 7]); +# endif + } + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); +# endif +} + +#endif /* IEM_WITH_SETJMP */ + + + +/** @name Misc Worker Functions. + * @{ + */ + +/** + * Gets the exception class for the specified exception vector. + * + * @returns The class of the specified exception. + * @param uVector The exception vector. + */ +static IEMXCPTCLASS iemGetXcptClass(uint8_t uVector) RT_NOEXCEPT +{ + Assert(uVector <= X86_XCPT_LAST); + switch (uVector) + { + case X86_XCPT_DE: + case X86_XCPT_TS: + case X86_XCPT_NP: + case X86_XCPT_SS: + case X86_XCPT_GP: + case X86_XCPT_SX: /* AMD only */ + return IEMXCPTCLASS_CONTRIBUTORY; + + case X86_XCPT_PF: + case X86_XCPT_VE: /* Intel only */ + return IEMXCPTCLASS_PAGE_FAULT; + + case X86_XCPT_DF: + return IEMXCPTCLASS_DOUBLE_FAULT; + } + return IEMXCPTCLASS_BENIGN; +} + + +/** + * Evaluates how to handle an exception caused during delivery of another event + * (exception / interrupt). + * + * @returns How to handle the recursive exception. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param fPrevFlags The flags of the previous event. + * @param uPrevVector The vector of the previous event. + * @param fCurFlags The flags of the current exception. + * @param uCurVector The vector of the current exception. + * @param pfXcptRaiseInfo Where to store additional information about the + * exception condition. Optional. + */ +VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags, + uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo) +{ + /* + * Only CPU exceptions can be raised while delivering other events, software interrupt + * (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception. + */ + AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID); + Assert(pVCpu); RT_NOREF(pVCpu); + Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector)); + + IEMXCPTRAISE enmRaise = IEMXCPTRAISE_CURRENT_XCPT; + IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE; + if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT) + { + IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector); + if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN) + { + IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector); + if ( enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT + && ( enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT + || enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY)) + { + enmRaise = IEMXCPTRAISE_DOUBLE_FAULT; + fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF + : IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT; + Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector, + uCurVector, pVCpu->cpum.GstCtx.cr2)); + } + else if ( enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY + && enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY) + { + enmRaise = IEMXCPTRAISE_DOUBLE_FAULT; + Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector)); + } + else if ( enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT + && ( enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY + || enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT)) + { + enmRaise = IEMXCPTRAISE_TRIPLE_FAULT; + Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector)); + } + } + else + { + if (uPrevVector == X86_XCPT_NMI) + { + fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT; + if (uCurVector == X86_XCPT_PF) + { + fRaiseInfo |= IEMXCPTRAISEINFO_NMI_PF; + Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n")); + } + } + else if ( uPrevVector == X86_XCPT_AC + && uCurVector == X86_XCPT_AC) + { + enmRaise = IEMXCPTRAISE_CPU_HANG; + fRaiseInfo = IEMXCPTRAISEINFO_AC_AC; + Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n")); + } + } + } + else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT) + { + fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT; + if (uCurVector == X86_XCPT_PF) + fRaiseInfo |= IEMXCPTRAISEINFO_EXT_INT_PF; + } + else + { + Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT); + fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT; + } + + if (pfXcptRaiseInfo) + *pfXcptRaiseInfo = fRaiseInfo; + return enmRaise; +} + + +/** + * Enters the CPU shutdown state initiated by a triple fault or other + * unrecoverable conditions. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + */ +static VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */); + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN)) + { + Log2(("shutdown: Guest intercept -> #VMEXIT\n")); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + RT_NOREF(pVCpu); + return VINF_EM_TRIPLE_FAULT; +} + + +/** + * Validates a new SS segment. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param NewSS The new SS selctor. + * @param uCpl The CPL to load the stack for. + * @param pDesc Where to return the descriptor. + */ +static VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc) RT_NOEXCEPT +{ + /* Null selectors are not allowed (we're not called for dispatching + interrupts with SS=0 in long mode). */ + if (!(NewSS & X86_SEL_MASK_OFF_RPL)) + { + Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS)); + return iemRaiseTaskSwitchFault0(pVCpu); + } + + /** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */ + if ((NewSS & X86_SEL_RPL) != uCpl) + { + Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS); + } + + /* + * Read the descriptor. + */ + VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Perform the descriptor validation documented for LSS, POP SS and MOV SS. + */ + if (!pDesc->Legacy.Gen.u1DescType) + { + Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS); + } + + if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) + || !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) ) + { + Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS); + } + if (pDesc->Legacy.Gen.u2Dpl != uCpl) + { + Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS); + } + + /* Is it there? */ + /** @todo testcase: Is this checked before the canonical / limit check below? */ + if (!pDesc->Legacy.Gen.u1Present) + { + Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS); + } + + return VINF_SUCCESS; +} + +/** @} */ + + +/** @name Raising Exceptions. + * + * @{ + */ + + +/** + * Loads the specified stack far pointer from the TSS. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param uCpl The CPL to load the stack for. + * @param pSelSS Where to return the new stack segment. + * @param puEsp Where to return the new stack pointer. + */ +static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict; + Assert(uCpl < 4); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR); + switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type) + { + /* + * 16-bit TSS (X86TSS16). + */ + case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU(); + case X86_SEL_TYPE_SYS_286_TSS_BUSY: + { + uint32_t off = uCpl * 4 + 2; + if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit) + { + /** @todo check actual access pattern here. */ + uint32_t u32Tmp = 0; /* gcc maybe... */ + rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off); + if (rcStrict == VINF_SUCCESS) + { + *puEsp = RT_LOWORD(u32Tmp); + *pSelSS = RT_HIWORD(u32Tmp); + return VINF_SUCCESS; + } + } + else + { + Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit)); + rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu); + } + break; + } + + /* + * 32-bit TSS (X86TSS32). + */ + case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU(); + case X86_SEL_TYPE_SYS_386_TSS_BUSY: + { + uint32_t off = uCpl * 8 + 4; + if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit) + { +/** @todo check actual access pattern here. */ + uint64_t u64Tmp; + rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off); + if (rcStrict == VINF_SUCCESS) + { + *puEsp = u64Tmp & UINT32_MAX; + *pSelSS = (RTSEL)(u64Tmp >> 32); + return VINF_SUCCESS; + } + } + else + { + Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit)); + rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu); + } + break; + } + + default: + AssertFailed(); + rcStrict = VERR_IEM_IPE_4; + break; + } + + *puEsp = 0; /* make gcc happy */ + *pSelSS = 0; /* make gcc happy */ + return rcStrict; +} + + +/** + * Loads the specified stack pointer from the 64-bit TSS. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param uCpl The CPL to load the stack for. + * @param uIst The interrupt stack table index, 0 if to use uCpl. + * @param puRsp Where to return the new stack pointer. + */ +static VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp) RT_NOEXCEPT +{ + Assert(uCpl < 4); + Assert(uIst < 8); + *puRsp = 0; /* make gcc happy */ + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR); + AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5); + + uint32_t off; + if (uIst) + off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1); + else + off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0); + if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit) + { + Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit)); + return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu); + } + + return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off); +} + + +/** + * Adjust the CPU state according to the exception being raised. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u8Vector The exception that has been raised. + */ +DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector) +{ + switch (u8Vector) + { + case X86_XCPT_DB: + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7); + pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD; + break; + /** @todo Read the AMD and Intel exception reference... */ + } +} + + +/** + * Implements exceptions and interrupts for real mode. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr The number of bytes to offset rIP by in the return + * address. + * @param u8Vector The interrupt / exception vector number. + * @param fFlags The flags. + * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set. + * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set. + */ +static VBOXSTRICTRC +iemRaiseXcptOrIntInRealMode(PVMCPUCC pVCpu, + uint8_t cbInstr, + uint8_t u8Vector, + uint32_t fFlags, + uint16_t uErr, + uint64_t uCr2) RT_NOEXCEPT +{ + NOREF(uErr); NOREF(uCr2); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + + /* + * Read the IDT entry. + */ + if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3) + { + Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt)); + return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + RTFAR16 Idte; + VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t *)&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) * u8Vector); + if (RT_UNLIKELY(rcStrict != VINF_SUCCESS)) + { + Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* + * Push the stack frame. + */ + uint16_t *pu16Frame; + uint64_t uNewRsp; + rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, 3, (void **)&pu16Frame, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + uint32_t fEfl = IEMMISC_GET_EFL(pVCpu); +#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC + AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186); + if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186) + fEfl |= UINT16_C(0xf000); +#endif + pu16Frame[2] = (uint16_t)fEfl; + pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel; + pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip; + rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp); + if (RT_UNLIKELY(rcStrict != VINF_SUCCESS)) + return rcStrict; + + /* + * Load the vector address into cs:ip and make exception specific state + * adjustments. + */ + pVCpu->cpum.GstCtx.cs.Sel = Idte.sel; + pVCpu->cpum.GstCtx.cs.ValidSel = Idte.sel; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)Idte.sel << 4; + /** @todo do we load attribs and limit as well? Should we check against limit like far jump? */ + pVCpu->cpum.GstCtx.rip = Idte.off; + fEfl &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_AC); + IEMMISC_SET_EFL(pVCpu, fEfl); + + /** @todo do we actually do this in real mode? */ + if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT) + iemRaiseXcptAdjustState(pVCpu, u8Vector); + + return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS; +} + + +/** + * Loads a NULL data selector into when coming from V8086 mode. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pSReg Pointer to the segment register. + */ +DECLINLINE(void) iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg) +{ + pSReg->Sel = 0; + pSReg->ValidSel = 0; + if (IEM_IS_GUEST_CPU_INTEL(pVCpu)) + { + /* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */ + pSReg->Attr.u &= X86DESCATTR_DT | X86DESCATTR_TYPE | X86DESCATTR_DPL | X86DESCATTR_G | X86DESCATTR_D; + pSReg->Attr.u |= X86DESCATTR_UNUSABLE; + } + else + { + pSReg->fFlags = CPUMSELREG_FLAGS_VALID; + /** @todo check this on AMD-V */ + pSReg->u64Base = 0; + pSReg->u32Limit = 0; + } +} + + +/** + * Loads a segment selector during a task switch in V8086 mode. + * + * @param pSReg Pointer to the segment register. + * @param uSel The selector value to load. + */ +DECLINLINE(void) iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel) +{ + /* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */ + pSReg->Sel = uSel; + pSReg->ValidSel = uSel; + pSReg->fFlags = CPUMSELREG_FLAGS_VALID; + pSReg->u64Base = uSel << 4; + pSReg->u32Limit = 0xffff; + pSReg->Attr.u = 0xf3; +} + + +/** + * Loads a segment selector during a task switch in protected mode. + * + * In this task switch scenario, we would throw \#TS exceptions rather than + * \#GPs. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pSReg Pointer to the segment register. + * @param uSel The new selector value. + * + * @remarks This does _not_ handle CS or SS. + * @remarks This expects pVCpu->iem.s.uCpl to be up to date. + */ +static VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel) RT_NOEXCEPT +{ + Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT); + + /* Null data selector. */ + if (!(uSel & X86_SEL_MASK_OFF_RPL)) + { + iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)); + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS); + return VINF_SUCCESS; + } + + /* Fetch the descriptor. */ + IEMSELDESC Desc; + VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Must be a data segment or readable code segment. */ + if ( !Desc.Legacy.Gen.u1DescType + || (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE) + { + Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel, + Desc.Legacy.Gen.u4Type)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL); + } + + /* Check privileges for data segments and non-conforming code segments. */ + if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) + != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) + { + /* The RPL and the new CPL must be less than or equal to the DPL. */ + if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl + || (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)) + { + Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n", + uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL); + } + } + + /* Is it there? */ + if (!Desc.Legacy.Gen.u1Present) + { + Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel)); + return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL); + } + + /* The base and limit. */ + uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy); + uint64_t u64Base = X86DESC_BASE(&Desc.Legacy); + + /* + * Ok, everything checked out fine. Now set the accessed bit before + * committing the result into the registers. + */ + if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* Commit */ + pSReg->Sel = uSel; + pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy); + pSReg->u32Limit = cbLimit; + pSReg->u64Base = u64Base; /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */ + pSReg->ValidSel = uSel; + pSReg->fFlags = CPUMSELREG_FLAGS_VALID; + if (IEM_IS_GUEST_CPU_INTEL(pVCpu)) + pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE; + + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)); + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS); + return VINF_SUCCESS; +} + + +/** + * Performs a task switch. + * + * If the task switch is the result of a JMP, CALL or IRET instruction, the + * caller is responsible for performing the necessary checks (like DPL, TSS + * present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction + * reference for JMP, CALL, IRET. + * + * If the task switch is the due to a software interrupt or hardware exception, + * the caller is responsible for validating the TSS selector and descriptor. See + * Intel Instruction reference for INT n. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param enmTaskSwitch The cause of the task switch. + * @param uNextEip The EIP effective after the task switch. + * @param fFlags The flags, see IEM_XCPT_FLAGS_XXX. + * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set. + * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set. + * @param SelTSS The TSS selector of the new task. + * @param pNewDescTSS Pointer to the new TSS descriptor. + */ +VBOXSTRICTRC +iemTaskSwitch(PVMCPUCC pVCpu, + IEMTASKSWITCH enmTaskSwitch, + uint32_t uNextEip, + uint32_t fFlags, + uint16_t uErr, + uint64_t uCr2, + RTSEL SelTSS, + PIEMSELDESC pNewDescTSS) RT_NOEXCEPT +{ + Assert(!IEM_IS_REAL_MODE(pVCpu)); + Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + + uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type; + Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL + || uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY + || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL + || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY); + + bool const fIsNewTSS386 = ( uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL + || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY); + + Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS, + fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip)); + + /* Update CR2 in case it's a page-fault. */ + /** @todo This should probably be done much earlier in IEM/PGM. See + * @bugref{5653#c49}. */ + if (fFlags & IEM_XCPT_FLAGS_CR2) + pVCpu->cpum.GstCtx.cr2 = uCr2; + + /* + * Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference" + * subsection "Interrupt 10 - Invalid TSS Exception (#TS)". + */ + uint32_t const uNewTSSLimit = pNewDescTSS->Legacy.Gen.u16LimitLow | (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16); + uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN; + if (uNewTSSLimit < uNewTSSLimitMin) + { + Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n", + enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL); + } + + /* + * Task switches in VMX non-root mode always cause task switches. + * The new TSS must have been read and validated (DPL, limits etc.) before a + * task-switch VM-exit commences. + * + * See Intel spec. 25.4.2 "Treatment of Task Switches". + */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTSS)); + IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTSS, uNextEip - pVCpu->cpum.GstCtx.eip); + } + + /* + * The SVM nested-guest intercept for task-switch takes priority over all exceptions + * after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept". + */ + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH)) + { + uint32_t const uExitInfo1 = SelTSS; + uint32_t uExitInfo2 = uErr; + switch (enmTaskSwitch) + { + case IEMTASKSWITCH_JUMP: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_JUMP; break; + case IEMTASKSWITCH_IRET: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_IRET; break; + default: break; + } + if (fFlags & IEM_XCPT_FLAGS_ERR) + uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE; + if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF) + uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF; + + Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2)); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2); + RT_NOREF2(uExitInfo1, uExitInfo2); + } + + /* + * Check the current TSS limit. The last written byte to the current TSS during the + * task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit). + * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields. + * + * The AMD docs doesn't mention anything about limit checks with LTR which suggests you can + * end up with smaller than "legal" TSS limits. + */ + uint32_t const uCurTSSLimit = pVCpu->cpum.GstCtx.tr.u32Limit; + uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29; + if (uCurTSSLimit < uCurTSSLimitMin) + { + Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n", + enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL); + } + + /* + * Verify that the new TSS can be accessed and map it. Map only the required contents + * and not the entire TSS. + */ + void *pvNewTSS; + uint32_t const cbNewTSS = uNewTSSLimitMin + 1; + RTGCPTR const GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy); + AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1); + /** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may + * not perform correct translation if this happens. See Intel spec. 7.2.1 + * "Task-State Segment". */ + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW, 0); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch, + cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* + * Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET. + */ + uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u; + if ( enmTaskSwitch == IEMTASKSWITCH_JUMP + || enmTaskSwitch == IEMTASKSWITCH_IRET) + { + PX86DESC pDescCurTSS; + rcStrict = iemMemMap(pVCpu, (void **)&pDescCurTSS, sizeof(*pDescCurTSS), UINT8_MAX, + pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n", + enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK; + rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n", + enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */ + if (enmTaskSwitch == IEMTASKSWITCH_IRET) + { + Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY + || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY); + fEFlags &= ~X86_EFL_NT; + } + } + + /* + * Save the CPU state into the current TSS. + */ + RTGCPTR const GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base; + if (GCPtrNewTSS == GCPtrCurTSS) + { + Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS)); + Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n", + pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, pVCpu->cpum.GstCtx.eax, + pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, + pVCpu->cpum.GstCtx.ldtr.Sel)); + } + if (fIsNewTSS386) + { + /* + * Verify that the current TSS (32-bit) can be accessed, only the minimum required size. + * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields. + */ + void *pvCurTSS32; + uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS32, eip); + uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip); + AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64); + rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n", + enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */ + PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS); + pCurTSS32->eip = uNextEip; + pCurTSS32->eflags = fEFlags; + pCurTSS32->eax = pVCpu->cpum.GstCtx.eax; + pCurTSS32->ecx = pVCpu->cpum.GstCtx.ecx; + pCurTSS32->edx = pVCpu->cpum.GstCtx.edx; + pCurTSS32->ebx = pVCpu->cpum.GstCtx.ebx; + pCurTSS32->esp = pVCpu->cpum.GstCtx.esp; + pCurTSS32->ebp = pVCpu->cpum.GstCtx.ebp; + pCurTSS32->esi = pVCpu->cpum.GstCtx.esi; + pCurTSS32->edi = pVCpu->cpum.GstCtx.edi; + pCurTSS32->es = pVCpu->cpum.GstCtx.es.Sel; + pCurTSS32->cs = pVCpu->cpum.GstCtx.cs.Sel; + pCurTSS32->ss = pVCpu->cpum.GstCtx.ss.Sel; + pCurTSS32->ds = pVCpu->cpum.GstCtx.ds.Sel; + pCurTSS32->fs = pVCpu->cpum.GstCtx.fs.Sel; + pCurTSS32->gs = pVCpu->cpum.GstCtx.gs.Sel; + + rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } + else + { + /* + * Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size. + */ + void *pvCurTSS16; + uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS16, ip); + uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip); + AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28); + rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n", + enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */ + PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS); + pCurTSS16->ip = uNextEip; + pCurTSS16->flags = (uint16_t)fEFlags; + pCurTSS16->ax = pVCpu->cpum.GstCtx.ax; + pCurTSS16->cx = pVCpu->cpum.GstCtx.cx; + pCurTSS16->dx = pVCpu->cpum.GstCtx.dx; + pCurTSS16->bx = pVCpu->cpum.GstCtx.bx; + pCurTSS16->sp = pVCpu->cpum.GstCtx.sp; + pCurTSS16->bp = pVCpu->cpum.GstCtx.bp; + pCurTSS16->si = pVCpu->cpum.GstCtx.si; + pCurTSS16->di = pVCpu->cpum.GstCtx.di; + pCurTSS16->es = pVCpu->cpum.GstCtx.es.Sel; + pCurTSS16->cs = pVCpu->cpum.GstCtx.cs.Sel; + pCurTSS16->ss = pVCpu->cpum.GstCtx.ss.Sel; + pCurTSS16->ds = pVCpu->cpum.GstCtx.ds.Sel; + + rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } + + /* + * Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT. + */ + if ( enmTaskSwitch == IEMTASKSWITCH_CALL + || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT) + { + /* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */ + PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS; + pNewTSS->selPrev = pVCpu->cpum.GstCtx.tr.Sel; + } + + /* + * Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky, + * it's done further below with error handling (e.g. CR3 changes will go through PGM). + */ + uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi; + uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt; + bool fNewDebugTrap; + if (fIsNewTSS386) + { + PCX86TSS32 pNewTSS32 = (PCX86TSS32)pvNewTSS; + uNewCr3 = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0; + uNewEip = pNewTSS32->eip; + uNewEflags = pNewTSS32->eflags; + uNewEax = pNewTSS32->eax; + uNewEcx = pNewTSS32->ecx; + uNewEdx = pNewTSS32->edx; + uNewEbx = pNewTSS32->ebx; + uNewEsp = pNewTSS32->esp; + uNewEbp = pNewTSS32->ebp; + uNewEsi = pNewTSS32->esi; + uNewEdi = pNewTSS32->edi; + uNewES = pNewTSS32->es; + uNewCS = pNewTSS32->cs; + uNewSS = pNewTSS32->ss; + uNewDS = pNewTSS32->ds; + uNewFS = pNewTSS32->fs; + uNewGS = pNewTSS32->gs; + uNewLdt = pNewTSS32->selLdt; + fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap); + } + else + { + PCX86TSS16 pNewTSS16 = (PCX86TSS16)pvNewTSS; + uNewCr3 = 0; + uNewEip = pNewTSS16->ip; + uNewEflags = pNewTSS16->flags; + uNewEax = UINT32_C(0xffff0000) | pNewTSS16->ax; + uNewEcx = UINT32_C(0xffff0000) | pNewTSS16->cx; + uNewEdx = UINT32_C(0xffff0000) | pNewTSS16->dx; + uNewEbx = UINT32_C(0xffff0000) | pNewTSS16->bx; + uNewEsp = UINT32_C(0xffff0000) | pNewTSS16->sp; + uNewEbp = UINT32_C(0xffff0000) | pNewTSS16->bp; + uNewEsi = UINT32_C(0xffff0000) | pNewTSS16->si; + uNewEdi = UINT32_C(0xffff0000) | pNewTSS16->di; + uNewES = pNewTSS16->es; + uNewCS = pNewTSS16->cs; + uNewSS = pNewTSS16->ss; + uNewDS = pNewTSS16->ds; + uNewFS = 0; + uNewGS = 0; + uNewLdt = pNewTSS16->selLdt; + fNewDebugTrap = false; + } + + if (GCPtrNewTSS == GCPtrCurTSS) + Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n", + uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt)); + + /* + * We're done accessing the new TSS. + */ + rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* + * Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT. + */ + if (enmTaskSwitch != IEMTASKSWITCH_IRET) + { + rcStrict = iemMemMap(pVCpu, (void **)&pNewDescTSS, sizeof(*pNewDescTSS), UINT8_MAX, + pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n", + enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Check that the descriptor indicates the new TSS is available (not busy). */ + AssertMsg( pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL + || pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL, + ("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type)); + + pNewDescTSS->Legacy.Gate.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK; + rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n", + enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } + + /* + * From this point on, we're technically in the new task. We will defer exceptions + * until the completion of the task switch but before executing any instructions in the new task. + */ + pVCpu->cpum.GstCtx.tr.Sel = SelTSS; + pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS; + pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy); + pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy); + pVCpu->cpum.GstCtx.tr.u64Base = X86DESC_BASE(&pNewDescTSS->Legacy); + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR); + + /* Set the busy bit in TR. */ + pVCpu->cpum.GstCtx.tr.Attr.n.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK; + + /* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */ + if ( enmTaskSwitch == IEMTASKSWITCH_CALL + || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT) + { + uNewEflags |= X86_EFL_NT; + } + + pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */ + pVCpu->cpum.GstCtx.cr0 |= X86_CR0_TS; + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0); + + pVCpu->cpum.GstCtx.eip = uNewEip; + pVCpu->cpum.GstCtx.eax = uNewEax; + pVCpu->cpum.GstCtx.ecx = uNewEcx; + pVCpu->cpum.GstCtx.edx = uNewEdx; + pVCpu->cpum.GstCtx.ebx = uNewEbx; + pVCpu->cpum.GstCtx.esp = uNewEsp; + pVCpu->cpum.GstCtx.ebp = uNewEbp; + pVCpu->cpum.GstCtx.esi = uNewEsi; + pVCpu->cpum.GstCtx.edi = uNewEdi; + + uNewEflags &= X86_EFL_LIVE_MASK; + uNewEflags |= X86_EFL_RA1_MASK; + IEMMISC_SET_EFL(pVCpu, uNewEflags); + + /* + * Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors + * will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3 + * due to the hidden part data originating from the guest LDT/GDT which is accessed through paging. + */ + pVCpu->cpum.GstCtx.es.Sel = uNewES; + pVCpu->cpum.GstCtx.es.Attr.u &= ~X86DESCATTR_P; + + pVCpu->cpum.GstCtx.cs.Sel = uNewCS; + pVCpu->cpum.GstCtx.cs.Attr.u &= ~X86DESCATTR_P; + + pVCpu->cpum.GstCtx.ss.Sel = uNewSS; + pVCpu->cpum.GstCtx.ss.Attr.u &= ~X86DESCATTR_P; + + pVCpu->cpum.GstCtx.ds.Sel = uNewDS; + pVCpu->cpum.GstCtx.ds.Attr.u &= ~X86DESCATTR_P; + + pVCpu->cpum.GstCtx.fs.Sel = uNewFS; + pVCpu->cpum.GstCtx.fs.Attr.u &= ~X86DESCATTR_P; + + pVCpu->cpum.GstCtx.gs.Sel = uNewGS; + pVCpu->cpum.GstCtx.gs.Attr.u &= ~X86DESCATTR_P; + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS); + + pVCpu->cpum.GstCtx.ldtr.Sel = uNewLdt; + pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_STALE; + pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P; + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR); + + if (IEM_IS_GUEST_CPU_INTEL(pVCpu)) + { + pVCpu->cpum.GstCtx.es.Attr.u |= X86DESCATTR_UNUSABLE; + pVCpu->cpum.GstCtx.cs.Attr.u |= X86DESCATTR_UNUSABLE; + pVCpu->cpum.GstCtx.ss.Attr.u |= X86DESCATTR_UNUSABLE; + pVCpu->cpum.GstCtx.ds.Attr.u |= X86DESCATTR_UNUSABLE; + pVCpu->cpum.GstCtx.fs.Attr.u |= X86DESCATTR_UNUSABLE; + pVCpu->cpum.GstCtx.gs.Attr.u |= X86DESCATTR_UNUSABLE; + pVCpu->cpum.GstCtx.ldtr.Attr.u |= X86DESCATTR_UNUSABLE; + } + + /* + * Switch CR3 for the new task. + */ + if ( fIsNewTSS386 + && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG)) + { + /** @todo Should we update and flush TLBs only if CR3 value actually changes? */ + int rc = CPUMSetGuestCR3(pVCpu, uNewCr3); + AssertRCSuccessReturn(rc, rc); + + /* Inform PGM. */ + /** @todo Should we raise \#GP(0) here when PAE PDPEs are invalid? */ + rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE)); + AssertRCReturn(rc, rc); + /* ignore informational status codes */ + + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3); + } + + /* + * Switch LDTR for the new task. + */ + if (!(uNewLdt & X86_SEL_MASK_OFF_RPL)) + iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt); + else + { + Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */ + + IEMSELDESC DescNewLdt; + rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch, + uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + if ( !DescNewLdt.Legacy.Gen.u1Present + || DescNewLdt.Legacy.Gen.u1DescType + || DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT) + { + Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch, + uNewLdt, DescNewLdt.Legacy.u)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL); + } + + pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt; + pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy); + pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy); + pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy); + if (IEM_IS_GUEST_CPU_INTEL(pVCpu)) + pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE; + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr)); + } + + IEMSELDESC DescSS; + if (IEM_IS_V86_MODE(pVCpu)) + { + pVCpu->iem.s.uCpl = 3; + iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES); + iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS); + iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS); + iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS); + iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS); + iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS); + + /* Quick fix: fake DescSS. */ /** @todo fix the code further down? */ + DescSS.Legacy.u = 0; + DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit; + DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16; + DescSS.Legacy.Gen.u16BaseLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base; + DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16); + DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24); + DescSS.Legacy.Gen.u4Type = X86_SEL_TYPE_RW_ACC; + DescSS.Legacy.Gen.u2Dpl = 3; + } + else + { + uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL); + + /* + * Load the stack segment for the new task. + */ + if (!(uNewSS & X86_SEL_MASK_OFF_RPL)) + { + Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL); + } + + /* Fetch the descriptor. */ + rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* SS must be a data segment and writable. */ + if ( !DescSS.Legacy.Gen.u1DescType + || (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) + || !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE)) + { + Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n", + uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL); + } + + /* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */ + if ( (uNewSS & X86_SEL_RPL) != uNewCpl + || DescSS.Legacy.Gen.u2Dpl != uNewCpl) + { + Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl, + uNewCpl)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL); + } + + /* Is it there? */ + if (!DescSS.Legacy.Gen.u1Present) + { + Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS)); + return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL); + } + + uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy); + uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy); + + /* Set the accessed bit before committing the result into SS. */ + if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* Commit SS. */ + pVCpu->cpum.GstCtx.ss.Sel = uNewSS; + pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy); + pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit; + pVCpu->cpum.GstCtx.ss.u64Base = u64Base; + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + + /* CPL has changed, update IEM before loading rest of segments. */ + pVCpu->iem.s.uCpl = uNewCpl; + + /* + * Load the data segments for the new task. + */ + rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Load the code segment for the new task. + */ + if (!(uNewCS & X86_SEL_MASK_OFF_RPL)) + { + Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL); + } + + /* Fetch the descriptor. */ + IEMSELDESC DescCS; + rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* CS must be a code segment. */ + if ( !DescCS.Legacy.Gen.u1DescType + || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)) + { + Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS, + DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL); + } + + /* For conforming CS, DPL must be less than or equal to the RPL. */ + if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) + && DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL)) + { + Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type, + DescCS.Legacy.Gen.u2Dpl)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL); + } + + /* For non-conforming CS, DPL must match RPL. */ + if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) + && DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL)) + { + Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, + DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl)); + return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL); + } + + /* Is it there? */ + if (!DescCS.Legacy.Gen.u1Present) + { + Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS)); + return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL); + } + + cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy); + u64Base = X86DESC_BASE(&DescCS.Legacy); + + /* Set the accessed bit before committing the result into CS. */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* Commit CS. */ + pVCpu->cpum.GstCtx.cs.Sel = uNewCS; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit; + pVCpu->cpum.GstCtx.cs.u64Base = u64Base; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); + } + + /** @todo Debug trap. */ + if (fIsNewTSS386 && fNewDebugTrap) + Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n")); + + /* + * Construct the error code masks based on what caused this task switch. + * See Intel Instruction reference for INT. + */ + uint16_t uExt; + if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT + && ( !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) + || (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR))) + { + uExt = 1; + } + else + uExt = 0; + + /* + * Push any error code on to the new stack. + */ + if (fFlags & IEM_XCPT_FLAGS_ERR) + { + Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT); + uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy); + uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2; + + /* Check that there is sufficient space on the stack. */ + /** @todo Factor out segment limit checking for normal/expand down segments + * into a separate function. */ + if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN)) + { + if ( pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS + || pVCpu->cpum.GstCtx.esp < cbStackFrame) + { + /** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */ + Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n", + pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame)); + return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt); + } + } + else + { + if ( pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)) + || pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1)) + { + Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n", + pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame)); + return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt); + } + } + + + if (fIsNewTSS386) + rcStrict = iemMemStackPushU32(pVCpu, uErr); + else + rcStrict = iemMemStackPushU16(pVCpu, uErr); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n", + fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } + + /* Check the new EIP against the new CS limit. */ + if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit) + { + Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n", + pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit)); + /** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */ + return iemRaiseGeneralProtectionFault(pVCpu, uExt); + } + + Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, + pVCpu->cpum.GstCtx.ss.Sel)); + return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS; +} + + +/** + * Implements exceptions and interrupts for protected mode. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr The number of bytes to offset rIP by in the return + * address. + * @param u8Vector The interrupt / exception vector number. + * @param fFlags The flags. + * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set. + * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set. + */ +static VBOXSTRICTRC +iemRaiseXcptOrIntInProtMode(PVMCPUCC pVCpu, + uint8_t cbInstr, + uint8_t u8Vector, + uint32_t fFlags, + uint16_t uErr, + uint64_t uCr2) RT_NOEXCEPT +{ + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + + /* + * Read the IDT entry. + */ + if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7) + { + Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt)); + return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + X86DESC Idte; + VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX, + pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector); + if (RT_UNLIKELY(rcStrict != VINF_SUCCESS)) + { + Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n", + u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type, + Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow)); + + /* + * Check the descriptor type, DPL and such. + * ASSUMES this is done in the same order as described for call-gate calls. + */ + if (Idte.Gate.u1DescType) + { + Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + bool fTaskGate = false; + uint8_t f32BitGate = true; + uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM; + switch (Idte.Gate.u4Type) + { + case X86_SEL_TYPE_SYS_UNDEFINED: + case X86_SEL_TYPE_SYS_286_TSS_AVAIL: + case X86_SEL_TYPE_SYS_LDT: + case X86_SEL_TYPE_SYS_286_TSS_BUSY: + case X86_SEL_TYPE_SYS_286_CALL_GATE: + case X86_SEL_TYPE_SYS_UNDEFINED2: + case X86_SEL_TYPE_SYS_386_TSS_AVAIL: + case X86_SEL_TYPE_SYS_UNDEFINED3: + case X86_SEL_TYPE_SYS_386_TSS_BUSY: + case X86_SEL_TYPE_SYS_386_CALL_GATE: + case X86_SEL_TYPE_SYS_UNDEFINED4: + { + /** @todo check what actually happens when the type is wrong... + * esp. call gates. */ + Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + + case X86_SEL_TYPE_SYS_286_INT_GATE: + f32BitGate = false; + RT_FALL_THRU(); + case X86_SEL_TYPE_SYS_386_INT_GATE: + fEflToClear |= X86_EFL_IF; + break; + + case X86_SEL_TYPE_SYS_TASK_GATE: + fTaskGate = true; +#ifndef IEM_IMPLEMENTS_TASKSWITCH + IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n")); +#endif + break; + + case X86_SEL_TYPE_SYS_286_TRAP_GATE: + f32BitGate = false; + case X86_SEL_TYPE_SYS_386_TRAP_GATE: + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + /* Check DPL against CPL if applicable. */ + if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT) + { + if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl) + { + Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl)); + return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + } + + /* Is it there? */ + if (!Idte.Gate.u1Present) + { + Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector)); + return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + + /* Is it a task-gate? */ + if (fTaskGate) + { + /* + * Construct the error code masks based on what caused this task switch. + * See Intel Instruction reference for INT. + */ + uint16_t const uExt = ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) + && !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1; + uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL; + RTSEL SelTSS = Idte.Gate.u16Sel; + + /* + * Fetch the TSS descriptor in the GDT. + */ + IEMSELDESC DescTSS; + rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) | uExt); + if (rcStrict != VINF_SUCCESS) + { + Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* The TSS descriptor must be a system segment and be available (not busy). */ + if ( DescTSS.Legacy.Gen.u1DescType + || ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL + && DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL)) + { + Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n", + u8Vector, SelTSS, DescTSS.Legacy.au64)); + return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) | uExt); + } + + /* The TSS must be present. */ + if (!DescTSS.Legacy.Gen.u1Present) + { + Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64)); + return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) | uExt); + } + + /* Do the actual task switch. */ + return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT, + (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip, + fFlags, uErr, uCr2, SelTSS, &DescTSS); + } + + /* A null CS is bad. */ + RTSEL NewCS = Idte.Gate.u16Sel; + if (!(NewCS & X86_SEL_MASK_OFF_RPL)) + { + Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Fetch the descriptor for the new CS. */ + IEMSELDESC DescCS; + rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */ + if (rcStrict != VINF_SUCCESS) + { + Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Must be a code segment. */ + if (!DescCS.Legacy.Gen.u1DescType) + { + Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL); + } + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)) + { + Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL); + } + + /* Don't allow lowering the privilege level. */ + /** @todo Does the lowering of privileges apply to software interrupts + * only? This has bearings on the more-privileged or + * same-privilege stack behavior further down. A testcase would + * be nice. */ + if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl) + { + Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n", + u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL); + } + + /* Make sure the selector is present. */ + if (!DescCS.Legacy.Gen.u1Present) + { + Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS); + } + + /* Check the new EIP against the new CS limit. */ + uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE + || Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE + ? Idte.Gate.u16OffsetLow + : Idte.Gate.u16OffsetLow | ((uint32_t)Idte.Gate.u16OffsetHigh << 16); + uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy); + if (uNewEip > cbLimitCS) + { + Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n", + u8Vector, uNewEip, cbLimitCS, NewCS)); + return iemRaiseGeneralProtectionFault(pVCpu, 0); + } + Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS)); + + /* Calc the flag image to push. */ + uint32_t fEfl = IEMMISC_GET_EFL(pVCpu); + if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT)) + fEfl &= ~X86_EFL_RF; + else + fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */ + + /* From V8086 mode only go to CPL 0. */ + uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF + ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl; + if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */ + { + Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl)); + return iemRaiseGeneralProtectionFault(pVCpu, 0); + } + + /* + * If the privilege level changes, we need to get a new stack from the TSS. + * This in turns means validating the new SS and ESP... + */ + if (uNewCpl != pVCpu->iem.s.uCpl) + { + RTSEL NewSS; + uint32_t uNewEsp; + rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + IEMSELDESC DescSS; + rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + /* If the new SS is 16-bit, we are only going to use SP, not ESP. */ + if (!DescSS.Legacy.Gen.u1DefBig) + { + Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp)); + uNewEsp = (uint16_t)uNewEsp; + } + + Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp)); + + /* Check that there is sufficient space for the stack frame. */ + uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy); + uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM) + ? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate + : (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate; + + if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN)) + { + if ( uNewEsp - 1 > cbLimitSS + || uNewEsp < cbStackFrame) + { + Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n", + u8Vector, NewSS, uNewEsp, cbStackFrame)); + return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS); + } + } + else + { + if ( uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX) + || uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1)) + { + Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n", + u8Vector, NewSS, uNewEsp, cbStackFrame)); + return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS); + } + } + + /* + * Start making changes. + */ + + /* Set the new CPL so that stack accesses use it. */ + uint8_t const uOldCpl = pVCpu->iem.s.uCpl; + pVCpu->iem.s.uCpl = uNewCpl; + + /* Create the stack frame. */ + RTPTRUNION uStackFrame; + rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX, + uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy), + IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */ + if (rcStrict != VINF_SUCCESS) + return rcStrict; + void * const pvStackFrame = uStackFrame.pv; + if (f32BitGate) + { + if (fFlags & IEM_XCPT_FLAGS_ERR) + *uStackFrame.pu32++ = uErr; + uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip; + uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl; + uStackFrame.pu32[2] = fEfl; + uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp; + uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel; + Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp)); + if (fEfl & X86_EFL_VM) + { + uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; + uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel; + uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel; + uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel; + uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel; + } + } + else + { + if (fFlags & IEM_XCPT_FLAGS_ERR) + *uStackFrame.pu16++ = uErr; + uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip; + uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl; + uStackFrame.pu16[2] = fEfl; + uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp; + uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel; + Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp)); + if (fEfl & X86_EFL_VM) + { + uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel; + uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel; + uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel; + uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel; + uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel; + } + } + rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Mark the selectors 'accessed' (hope this is the correct time). */ + /** @todo testcase: excatly _when_ are the accessed bits set - before or + * after pushing the stack frame? (Write protect the gdt + stack to + * find out.) */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* + * Start comitting the register changes (joins with the DPL=CPL branch). + */ + pVCpu->cpum.GstCtx.ss.Sel = NewSS; + pVCpu->cpum.GstCtx.ss.ValidSel = NewSS; + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSS; + pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy); + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy); + /** @todo When coming from 32-bit code and operating with a 16-bit TSS and + * 16-bit handler, the high word of ESP remains unchanged (i.e. only + * SP is loaded). + * Need to check the other combinations too: + * - 16-bit TSS, 32-bit handler + * - 32-bit TSS, 16-bit handler */ + if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + pVCpu->cpum.GstCtx.sp = (uint16_t)(uNewEsp - cbStackFrame); + else + pVCpu->cpum.GstCtx.rsp = uNewEsp - cbStackFrame; + + if (fEfl & X86_EFL_VM) + { + iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs); + iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs); + iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es); + iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds); + } + } + /* + * Same privilege, no stack change and smaller stack frame. + */ + else + { + uint64_t uNewRsp; + RTPTRUNION uStackFrame; + uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate; + rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, f32BitGate ? 3 : 1, &uStackFrame.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + void * const pvStackFrame = uStackFrame.pv; + + if (f32BitGate) + { + if (fFlags & IEM_XCPT_FLAGS_ERR) + *uStackFrame.pu32++ = uErr; + uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip; + uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl; + uStackFrame.pu32[2] = fEfl; + } + else + { + if (fFlags & IEM_XCPT_FLAGS_ERR) + *uStackFrame.pu16++ = uErr; + uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip; + uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl; + uStackFrame.pu16[2] = fEfl; + } + rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */ + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Mark the CS selector as 'accessed'. */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* + * Start committing the register changes (joins with the other branch). + */ + pVCpu->cpum.GstCtx.rsp = uNewRsp; + } + + /* ... register committing continues. */ + pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl; + pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS; + pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy); + + pVCpu->cpum.GstCtx.rip = uNewEip; /* (The entire register is modified, see pe16_32 bs3kit tests.) */ + fEfl &= ~fEflToClear; + IEMMISC_SET_EFL(pVCpu, fEfl); + + if (fFlags & IEM_XCPT_FLAGS_CR2) + pVCpu->cpum.GstCtx.cr2 = uCr2; + + if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT) + iemRaiseXcptAdjustState(pVCpu, u8Vector); + + return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS; +} + + +/** + * Implements exceptions and interrupts for long mode. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr The number of bytes to offset rIP by in the return + * address. + * @param u8Vector The interrupt / exception vector number. + * @param fFlags The flags. + * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set. + * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set. + */ +static VBOXSTRICTRC +iemRaiseXcptOrIntInLongMode(PVMCPUCC pVCpu, + uint8_t cbInstr, + uint8_t u8Vector, + uint32_t fFlags, + uint16_t uErr, + uint64_t uCr2) RT_NOEXCEPT +{ + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + + /* + * Read the IDT entry. + */ + uint16_t offIdt = (uint16_t)u8Vector << 4; + if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7) + { + Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt)); + return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + X86DESC64 Idte; +#ifdef _MSC_VER /* Shut up silly compiler warning. */ + Idte.au64[0] = 0; + Idte.au64[1] = 0; +#endif + VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8); + if (RT_UNLIKELY(rcStrict != VINF_SUCCESS)) + { + Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n", + u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type, + Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow)); + + /* + * Check the descriptor type, DPL and such. + * ASSUMES this is done in the same order as described for call-gate calls. + */ + if (Idte.Gate.u1DescType) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM; + switch (Idte.Gate.u4Type) + { + case AMD64_SEL_TYPE_SYS_INT_GATE: + fEflToClear |= X86_EFL_IF; + break; + case AMD64_SEL_TYPE_SYS_TRAP_GATE: + break; + + default: + Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + + /* Check DPL against CPL if applicable. */ + if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT) + { + if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl)); + return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + } + + /* Is it there? */ + if (!Idte.Gate.u1Present) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector)); + return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT)); + } + + /* A null CS is bad. */ + RTSEL NewCS = Idte.Gate.u16Sel; + if (!(NewCS & X86_SEL_MASK_OFF_RPL)) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Fetch the descriptor for the new CS. */ + IEMSELDESC DescCS; + rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Must be a 64-bit code segment. */ + if (!DescCS.Long.Gen.u1DescType) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL); + } + if ( !DescCS.Long.Gen.u1Long + || DescCS.Long.Gen.u1DefBig + || !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) ) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n", + u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig)); + return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL); + } + + /* Don't allow lowering the privilege level. For non-conforming CS + selectors, the CS.DPL sets the privilege level the trap/interrupt + handler runs at. For conforming CS selectors, the CPL remains + unchanged, but the CS.DPL must be <= CPL. */ + /** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched + * when CPU in Ring-0. Result \#GP? */ + if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n", + u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL); + } + + + /* Make sure the selector is present. */ + if (!DescCS.Legacy.Gen.u1Present) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS); + } + + /* Check that the new RIP is canonical. */ + uint64_t const uNewRip = Idte.Gate.u16OffsetLow + | ((uint32_t)Idte.Gate.u16OffsetHigh << 16) + | ((uint64_t)Idte.Gate.u32OffsetTop << 32); + if (!IEM_IS_CANONICAL(uNewRip)) + { + Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * If the privilege level changes or if the IST isn't zero, we need to get + * a new stack from the TSS. + */ + uint64_t uNewRsp; + uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF + ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl; + if ( uNewCpl != pVCpu->iem.s.uCpl + || Idte.Gate.u3IST != 0) + { + rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + else + uNewRsp = pVCpu->cpum.GstCtx.rsp; + uNewRsp &= ~(uint64_t)0xf; + + /* + * Calc the flag image to push. + */ + uint32_t fEfl = IEMMISC_GET_EFL(pVCpu); + if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT)) + fEfl &= ~X86_EFL_RF; + else + fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */ + + /* + * Start making changes. + */ + /* Set the new CPL so that stack accesses use it. */ + uint8_t const uOldCpl = pVCpu->iem.s.uCpl; + pVCpu->iem.s.uCpl = uNewCpl; + + /* Create the stack frame. */ + uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR)); + RTPTRUNION uStackFrame; + rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX, + uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */ + if (rcStrict != VINF_SUCCESS) + return rcStrict; + void * const pvStackFrame = uStackFrame.pv; + + if (fFlags & IEM_XCPT_FLAGS_ERR) + *uStackFrame.pu64++ = uErr; + uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip; + uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl; /* CPL paranoia */ + uStackFrame.pu64[2] = fEfl; + uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp; + uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel; + rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Mark the CS selectors 'accessed' (hope this is the correct time). */ + /** @todo testcase: excatly _when_ are the accessed bits set - before or + * after pushing the stack frame? (Write protect the gdt + stack to + * find out.) */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* + * Start comitting the register changes. + */ + /** @todo research/testcase: Figure out what VT-x and AMD-V loads into the + * hidden registers when interrupting 32-bit or 16-bit code! */ + if (uNewCpl != uOldCpl) + { + pVCpu->cpum.GstCtx.ss.Sel = 0 | uNewCpl; + pVCpu->cpum.GstCtx.ss.ValidSel = 0 | uNewCpl; + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX; + pVCpu->cpum.GstCtx.ss.u64Base = 0; + pVCpu->cpum.GstCtx.ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) | X86DESCATTR_UNUSABLE; + } + pVCpu->cpum.GstCtx.rsp = uNewRsp - cbStackFrame; + pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl; + pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy); + pVCpu->cpum.GstCtx.rip = uNewRip; + + fEfl &= ~fEflToClear; + IEMMISC_SET_EFL(pVCpu, fEfl); + + if (fFlags & IEM_XCPT_FLAGS_CR2) + pVCpu->cpum.GstCtx.cr2 = uCr2; + + if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT) + iemRaiseXcptAdjustState(pVCpu, u8Vector); + + return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS; +} + + +/** + * Implements exceptions and interrupts. + * + * All exceptions and interrupts goes thru this function! + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr The number of bytes to offset rIP by in the return + * address. + * @param u8Vector The interrupt / exception vector number. + * @param fFlags The flags. + * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set. + * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set. + */ +VBOXSTRICTRC +iemRaiseXcptOrInt(PVMCPUCC pVCpu, + uint8_t cbInstr, + uint8_t u8Vector, + uint32_t fFlags, + uint16_t uErr, + uint64_t uCr2) RT_NOEXCEPT +{ + /* + * Get all the state that we might need here. + */ + IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + +#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */ + /* + * Flush prefetch buffer + */ + pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode; +#endif + + /* + * Perform the V8086 IOPL check and upgrade the fault without nesting. + */ + if ( pVCpu->cpum.GstCtx.eflags.Bits.u1VM + && pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3 + && (fFlags & ( IEM_XCPT_FLAGS_T_SOFT_INT + | IEM_XCPT_FLAGS_BP_INSTR + | IEM_XCPT_FLAGS_ICEBP_INSTR + | IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT + && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) ) + { + Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector)); + fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR; + u8Vector = X86_XCPT_GP; + uErr = 0; + } +#ifdef DBGFTRACE_ENABLED + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx", + pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2, + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp); +#endif + + /* + * Evaluate whether NMI blocking should be in effect. + * Normally, NMI blocking is in effect whenever we inject an NMI. + */ + bool fBlockNmi = u8Vector == X86_XCPT_NMI + && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT); + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr); + if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict0; + + /* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */ + if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking) + { + Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI)); + fBlockNmi = false; + } + } +#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))) + { + /* + * If the event is being injected as part of VMRUN, it isn't subject to event + * intercepts in the nested-guest. However, secondary exceptions that occur + * during injection of any event -are- subject to exception intercepts. + * + * See AMD spec. 15.20 "Event Injection". + */ + if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents) + pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true; + else + { + /* + * Check and handle if the event being raised is intercepted. + */ + VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2); + if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE) + return rcStrict0; + } + } +#endif + + /* + * Set NMI blocking if necessary. + */ + if (fBlockNmi) + CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx); + + /* + * Do recursion accounting. + */ + uint8_t const uPrevXcpt = pVCpu->iem.s.uCurXcpt; + uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt; + if (pVCpu->iem.s.cXcptRecursions == 0) + Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n", + u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2)); + else + { + Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n", + u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt, + pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt)); + + if (pVCpu->iem.s.cXcptRecursions >= 4) + { +#ifdef DEBUG_bird + AssertFailed(); +#endif + IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n")); + } + + /* + * Evaluate the sequence of recurring events. + */ + IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector, + NULL /* pXcptRaiseInfo */); + if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT) + { /* likely */ } + else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT) + { + Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt)); + fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR; + u8Vector = X86_XCPT_DF; + uErr = 0; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* VMX nested-guest #DF intercept needs to be checked here. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu); + if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict0; + } +#endif + /* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */ + if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF)) + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT) + { + Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt)); + return iemInitiateCpuShutdown(pVCpu); + } + else if (enmRaise == IEMXCPTRAISE_CPU_HANG) + { + /* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */ + Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n")); + if ( !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)) + && !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu))) + return VERR_EM_GUEST_CPU_HANG; + } + else + { + AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n", + enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags)); + return VERR_IEM_IPE_9; + } + + /* + * The 'EXT' bit is set when an exception occurs during deliver of an external + * event (such as an interrupt or earlier exception)[1]. Privileged software + * exception (INT1) also sets the EXT bit[2]. Exceptions generated by software + * interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set. + * + * [1] - Intel spec. 6.13 "Error Code" + * [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection". + * [3] - Intel Instruction reference for INT n. + */ + if ( (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_EXT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) + && (fFlags & IEM_XCPT_FLAGS_ERR) + && u8Vector != X86_XCPT_PF + && u8Vector != X86_XCPT_DF) + { + uErr |= X86_TRAP_ERR_EXTERNAL; + } + } + + pVCpu->iem.s.cXcptRecursions++; + pVCpu->iem.s.uCurXcpt = u8Vector; + pVCpu->iem.s.fCurXcpt = fFlags; + pVCpu->iem.s.uCurXcptErr = uErr; + pVCpu->iem.s.uCurXcptCr2 = uCr2; + + /* + * Extensive logging. + */ +#if defined(LOG_ENABLED) && defined(IN_RING3) + if (LogIs3Enabled()) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK); + PVM pVM = pVCpu->CTX_SUFF(pVM); + char szRegs[4096]; + DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs), + "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n" + "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n" + "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n" + "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n" + "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n" + "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n" + "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n" + "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n" + "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n" + "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n" + "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n" + "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n" + "dr6=%016VR{dr6} dr7=%016VR{dr7}\n" + "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n" + "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n" + "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n" + " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n" + " efer=%016VR{efer}\n" + " pat=%016VR{pat}\n" + " sf_mask=%016VR{sf_mask}\n" + "krnl_gs_base=%016VR{krnl_gs_base}\n" + " lstar=%016VR{lstar}\n" + " star=%016VR{star} cstar=%016VR{cstar}\n" + "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n" + ); + + char szInstr[256]; + DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0, + DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE, + szInstr, sizeof(szInstr), NULL); + Log3(("%s%s\n", szRegs, szInstr)); + } +#endif /* LOG_ENABLED */ + + /* + * Stats. + */ + if (!(fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)) + STAM_REL_STATS({ pVCpu->iem.s.aStatInts[u8Vector] += 1; }); + else if (u8Vector <= X86_XCPT_LAST) + { + STAM_REL_COUNTER_INC(&pVCpu->iem.s.aStatXcpts[u8Vector]); + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_XCPT, u8Vector), + pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, ASMReadTSC()); + } + + /* + * #PF's implies a INVLPG for the CR2 value (see 4.10.1.1 in Intel SDM Vol 3) + * to ensure that a stale TLB or paging cache entry will only cause one + * spurious #PF. + */ + if ( u8Vector == X86_XCPT_PF + && (fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_CR2)) == (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_CR2)) + IEMTlbInvalidatePage(pVCpu, uCr2); + + /* + * Call the mode specific worker function. + */ + VBOXSTRICTRC rcStrict; + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)) + rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2); + else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA) + rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2); + else + rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2); + + /* Flush the prefetch buffer. */ +#ifdef IEM_WITH_CODE_TLB + pVCpu->iem.s.pbInstrBuf = NULL; +#else + pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu); +#endif + + /* + * Unwind. + */ + pVCpu->iem.s.cXcptRecursions--; + pVCpu->iem.s.uCurXcpt = uPrevXcpt; + pVCpu->iem.s.fCurXcpt = fPrevXcpt; + Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n", + VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl, + pVCpu->iem.s.cXcptRecursions + 1)); + return rcStrict; +} + +#ifdef IEM_WITH_SETJMP +/** + * See iemRaiseXcptOrInt. Will not return. + */ +DECL_NO_RETURN(void) +iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu, + uint8_t cbInstr, + uint8_t u8Vector, + uint32_t fFlags, + uint16_t uErr, + uint64_t uCr2) IEM_NOEXCEPT_MAY_LONGJMP +{ + VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2); + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); +} +#endif + + +/** \#DE - 00. */ +VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0); +} + + +/** \#DB - 01. + * @note This automatically clear DR7.GD. */ +VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* This always clears RF (via IEM_XCPT_FLAGS_DRx_INSTR_BP). */ + pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD; + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_DRx_INSTR_BP, 0, 0); +} + + +/** \#BR - 05. */ +VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0); +} + + +/** \#UD - 06. */ +VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0); +} + + +/** \#NM - 07. */ +VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0); +} + + +/** \#TS(err) - 0a. */ +VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0); +} + + +/** \#TS(tr) - 0a. */ +VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, + pVCpu->cpum.GstCtx.tr.Sel, 0); +} + + +/** \#TS(0) - 0a. */ +VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, + 0, 0); +} + + +/** \#TS(err) - 0a. */ +VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, + uSel & X86_SEL_MASK_OFF_RPL, 0); +} + + +/** \#NP(err) - 0b. */ +VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0); +} + + +/** \#NP(sel) - 0b. */ +VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, + uSel & ~X86_SEL_RPL, 0); +} + + +/** \#SS(seg) - 0c. */ +VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, + uSel & ~X86_SEL_RPL, 0); +} + + +/** \#SS(err) - 0c. */ +VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0); +} + + +/** \#GP(n) - 0d. */ +VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0); +} + + +/** \#GP(0) - 0d. */ +VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} + +#ifdef IEM_WITH_SETJMP +/** \#GP(0) - 0d. */ +DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ + iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} +#endif + + +/** \#GP(sel) - 0d. */ +VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, + Sel & ~X86_SEL_RPL, 0); +} + + +/** \#GP(0) - 0d. */ +VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} + + +/** \#GP(sel) - 0d. */ +VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT +{ + NOREF(iSegReg); NOREF(fAccess); + return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP, + IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} + +#ifdef IEM_WITH_SETJMP +/** \#GP(sel) - 0d, longjmp. */ +DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) IEM_NOEXCEPT_MAY_LONGJMP +{ + NOREF(iSegReg); NOREF(fAccess); + iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP, + IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} +#endif + +/** \#GP(sel) - 0d. */ +VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT +{ + NOREF(Sel); + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} + +#ifdef IEM_WITH_SETJMP +/** \#GP(sel) - 0d, longjmp. */ +DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) IEM_NOEXCEPT_MAY_LONGJMP +{ + NOREF(Sel); + iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} +#endif + + +/** \#GP(sel) - 0d. */ +VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT +{ + NOREF(iSegReg); NOREF(fAccess); + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} + +#ifdef IEM_WITH_SETJMP +/** \#GP(sel) - 0d, longjmp. */ +DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) IEM_NOEXCEPT_MAY_LONGJMP +{ + NOREF(iSegReg); NOREF(fAccess); + iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} +#endif + + +/** \#PF(n) - 0e. */ +VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t cbAccess, uint32_t fAccess, int rc) RT_NOEXCEPT +{ + uint16_t uErr; + switch (rc) + { + case VERR_PAGE_NOT_PRESENT: + case VERR_PAGE_TABLE_NOT_PRESENT: + case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT: + case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT: + uErr = 0; + break; + + default: + AssertMsgFailed(("%Rrc\n", rc)); + RT_FALL_THRU(); + case VERR_ACCESS_DENIED: + uErr = X86_TRAP_PF_P; + break; + + /** @todo reserved */ + } + + if (pVCpu->iem.s.uCpl == 3) + uErr |= X86_TRAP_PF_US; + + if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE + && ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE) + && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) ) + uErr |= X86_TRAP_PF_ID; + +#if 0 /* This is so much non-sense, really. Why was it done like that? */ + /* Note! RW access callers reporting a WRITE protection fault, will clear + the READ flag before calling. So, read-modify-write accesses (RW) + can safely be reported as READ faults. */ + if ((fAccess & (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE) + uErr |= X86_TRAP_PF_RW; +#else + if (fAccess & IEM_ACCESS_TYPE_WRITE) + { + /// @todo r=bird: bs3-cpu-basic-2 wants X86_TRAP_PF_RW for xchg and cmpxchg + /// (regardless of outcome of the comparison in the latter case). + //if (!(fAccess & IEM_ACCESS_TYPE_READ)) + uErr |= X86_TRAP_PF_RW; + } +#endif + + /* For FXSAVE and FRSTOR the #PF is typically reported at the max address + of the memory operand rather than at the start of it. (Not sure what + happens if it crosses a page boundrary.) The current heuristics for + this is to report the #PF for the last byte if the access is more than + 64 bytes. This is probably not correct, but we can work that out later, + main objective now is to get FXSAVE to work like for real hardware and + make bs3-cpu-basic2 work. */ + if (cbAccess <= 64) + { /* likely*/ } + else + GCPtrWhere += cbAccess - 1; + + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR | IEM_XCPT_FLAGS_CR2, + uErr, GCPtrWhere); +} + +#ifdef IEM_WITH_SETJMP +/** \#PF(n) - 0e, longjmp. */ +DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t cbAccess, + uint32_t fAccess, int rc) IEM_NOEXCEPT_MAY_LONGJMP +{ + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, cbAccess, fAccess, rc))); +} +#endif + + +/** \#MF(0) - 10. */ +VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE) + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0); + + /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */ + PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq */, 1 /* u8Level */, 0 /* uTagSrc */); + return iemRegUpdateRipAndFinishClearingRF(pVCpu); +} + + +/** \#AC(0) - 11. */ +VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0); +} + +#ifdef IEM_WITH_SETJMP +/** \#AC(0) - 11, longjmp. */ +DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(iemRaiseAlignmentCheckException(pVCpu))); +} +#endif + + +/** \#XF(0)/\#XM(0) - 19. */ +VBOXSTRICTRC iemRaiseSimdFpException(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_XF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0); +} + + +/** Accessed via IEMOP_RAISE_DIVIDE_ERROR. */ +IEM_CIMPL_DEF_0(iemCImplRaiseDivideError) +{ + NOREF(cbInstr); + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0); +} + + +/** Accessed via IEMOP_RAISE_INVALID_LOCK_PREFIX. */ +IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix) +{ + NOREF(cbInstr); + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0); +} + + +/** Accessed via IEMOP_RAISE_INVALID_OPCODE. */ +IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode) +{ + NOREF(cbInstr); + return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0); +} + + +/** @} */ + +/** @name Common opcode decoders. + * @{ + */ +//#include + +/** + * Used to add extra details about a stub case. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +void iemOpStubMsg2(PVMCPUCC pVCpu) RT_NOEXCEPT +{ +#if defined(LOG_ENABLED) && defined(IN_RING3) + PVM pVM = pVCpu->CTX_SUFF(pVM); + char szRegs[4096]; + DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs), + "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n" + "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n" + "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n" + "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n" + "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n" + "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n" + "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n" + "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n" + "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n" + "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n" + "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n" + "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n" + "dr6=%016VR{dr6} dr7=%016VR{dr7}\n" + "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n" + "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n" + "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n" + " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n" + " efer=%016VR{efer}\n" + " pat=%016VR{pat}\n" + " sf_mask=%016VR{sf_mask}\n" + "krnl_gs_base=%016VR{krnl_gs_base}\n" + " lstar=%016VR{lstar}\n" + " star=%016VR{star} cstar=%016VR{cstar}\n" + "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n" + ); + + char szInstr[256]; + DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0, + DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE, + szInstr, sizeof(szInstr), NULL); + + RTAssertMsg2Weak("%s%s\n", szRegs, szInstr); +#else + RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip); +#endif +} + +/** @} */ + + + +/** @name Register Access. + * @{ + */ + +/** + * Adds a 8-bit signed jump offset to RIP/EIP/IP. + * + * May raise a \#GP(0) if the new RIP is non-canonical or outside the code + * segment limit. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr Instruction size. + * @param offNextInstr The offset of the next instruction. + * @param enmEffOpSize Effective operand size. + */ +VBOXSTRICTRC iemRegRipRelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr, + IEMMODE enmEffOpSize) RT_NOEXCEPT +{ + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr; + if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit + || pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT /* no CS limit checks in 64-bit mode */)) + pVCpu->cpum.GstCtx.rip = uNewIp; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + break; + } + + case IEMMODE_32BIT: + { + Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT); + Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); + + uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr; + if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit)) + pVCpu->cpum.GstCtx.rip = uNewEip; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + break; + } + + case IEMMODE_64BIT: + { + Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT); + + uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; + if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip))) + pVCpu->cpum.GstCtx.rip = uNewRip; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + +#ifndef IEM_WITH_CODE_TLB + /* Flush the prefetch buffer. */ + pVCpu->iem.s.cbOpcode = cbInstr; +#endif + + /* + * Clear RF and finish the instruction (maybe raise #DB). + */ + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Adds a 16-bit signed jump offset to RIP/EIP/IP. + * + * May raise a \#GP(0) if the new RIP is non-canonical or outside the code + * segment limit. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr Instruction size. + * @param offNextInstr The offset of the next instruction. + */ +VBOXSTRICTRC iemRegRipRelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offNextInstr) RT_NOEXCEPT +{ + Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT); + + uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr; + if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit + || pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT /* no limit checking in 64-bit mode */)) + pVCpu->cpum.GstCtx.rip = uNewIp; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + +#ifndef IEM_WITH_CODE_TLB + /* Flush the prefetch buffer. */ + pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu); +#endif + + /* + * Clear RF and finish the instruction (maybe raise #DB). + */ + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Adds a 32-bit signed jump offset to RIP/EIP/IP. + * + * May raise a \#GP(0) if the new RIP is non-canonical or outside the code + * segment limit. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr Instruction size. + * @param offNextInstr The offset of the next instruction. + * @param enmEffOpSize Effective operand size. + */ +VBOXSTRICTRC iemRegRipRelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offNextInstr, + IEMMODE enmEffOpSize) RT_NOEXCEPT +{ + if (enmEffOpSize == IEMMODE_32BIT) + { + Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT); + + uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr; + if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit)) + pVCpu->cpum.GstCtx.rip = uNewEip; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + } + else + { + Assert(enmEffOpSize == IEMMODE_64BIT); + + uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr; + if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip))) + pVCpu->cpum.GstCtx.rip = uNewRip; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + } + +#ifndef IEM_WITH_CODE_TLB + /* Flush the prefetch buffer. */ + pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu); +#endif + + /* + * Clear RF and finish the instruction (maybe raise #DB). + */ + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Performs a near jump to the specified address. + * + * May raise a \#GP(0) if the new IP outside the code segment limit. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param uNewIp The new IP value. + */ +VBOXSTRICTRC iemRegRipJumpU16AndFinishClearningRF(PVMCPUCC pVCpu, uint16_t uNewIp) RT_NOEXCEPT +{ + if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit + || pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT /* no limit checks in 64-bit mode */)) + pVCpu->cpum.GstCtx.rip = uNewIp; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + /** @todo Test 16-bit jump in 64-bit mode. */ + +#ifndef IEM_WITH_CODE_TLB + /* Flush the prefetch buffer. */ + pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu); +#endif + + /* + * Clear RF and finish the instruction (maybe raise #DB). + */ + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Performs a near jump to the specified address. + * + * May raise a \#GP(0) if the new RIP is outside the code segment limit. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param uNewEip The new EIP value. + */ +VBOXSTRICTRC iemRegRipJumpU32AndFinishClearningRF(PVMCPUCC pVCpu, uint32_t uNewEip) RT_NOEXCEPT +{ + Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); + Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT); + + if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit)) + pVCpu->cpum.GstCtx.rip = uNewEip; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + +#ifndef IEM_WITH_CODE_TLB + /* Flush the prefetch buffer. */ + pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu); +#endif + + /* + * Clear RF and finish the instruction (maybe raise #DB). + */ + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Performs a near jump to the specified address. + * + * May raise a \#GP(0) if the new RIP is non-canonical or outside the code + * segment limit. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param uNewRip The new RIP value. + */ +VBOXSTRICTRC iemRegRipJumpU64AndFinishClearningRF(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT +{ + Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT); + + if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip))) + pVCpu->cpum.GstCtx.rip = uNewRip; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + +#ifndef IEM_WITH_CODE_TLB + /* Flush the prefetch buffer. */ + pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu); +#endif + + /* + * Clear RF and finish the instruction (maybe raise #DB). + */ + return iemRegFinishClearingRF(pVCpu); +} + +/** @} */ + + +/** @name FPU access and helpers. + * + * @{ + */ + +/** + * Updates the x87.DS and FPUDP registers. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pFpuCtx The FPU context. + * @param iEffSeg The effective segment register. + * @param GCPtrEff The effective address relative to @a iEffSeg. + */ +DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff) +{ + RTSEL sel; + switch (iEffSeg) + { + case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break; + case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break; + case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break; + case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break; + case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break; + case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break; + default: + AssertMsgFailed(("%d\n", iEffSeg)); + sel = pVCpu->cpum.GstCtx.ds.Sel; + } + /** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */ + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + pFpuCtx->DS = 0; + pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4); + } + else if (!IEM_IS_LONG_MODE(pVCpu)) + { + pFpuCtx->DS = sel; + pFpuCtx->FPUDP = GCPtrEff; + } + else + *(uint64_t *)&pFpuCtx->FPUDP = GCPtrEff; +} + + +/** + * Rotates the stack registers in the push direction. + * + * @param pFpuCtx The FPU context. + * @remarks This is a complete waste of time, but fxsave stores the registers in + * stack order. + */ +DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx) +{ + RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80; + pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80; + pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80; + pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80; + pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80; + pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80; + pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80; + pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80; + pFpuCtx->aRegs[0].r80 = r80Tmp; +} + + +/** + * Rotates the stack registers in the pop direction. + * + * @param pFpuCtx The FPU context. + * @remarks This is a complete waste of time, but fxsave stores the registers in + * stack order. + */ +DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx) +{ + RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80; + pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80; + pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80; + pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80; + pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80; + pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80; + pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80; + pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80; + pFpuCtx->aRegs[7].r80 = r80Tmp; +} + + +/** + * Updates FSW and pushes a FPU result onto the FPU stack if no pending + * exception prevents it. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pResult The FPU operation result to push. + * @param pFpuCtx The FPU context. + */ +static void iemFpuMaybePushResult(PVMCPU pVCpu, PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx) RT_NOEXCEPT +{ + /* Update FSW and bail if there are pending exceptions afterwards. */ + uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK; + fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK; + if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE)) + & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM))) + { + if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FCW & X86_FSW_ES)) + Log11(("iemFpuMaybePushResult: %04x:%08RX64: FSW %#x -> %#x\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw)); + pFpuCtx->FSW = fFsw; + return; + } + + uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK; + if (!(pFpuCtx->FTW & RT_BIT(iNewTop))) + { + /* All is fine, push the actual value. */ + pFpuCtx->FTW |= RT_BIT(iNewTop); + pFpuCtx->aRegs[7].r80 = pResult->r80Result; + } + else if (pFpuCtx->FCW & X86_FCW_IM) + { + /* Masked stack overflow, push QNaN. */ + fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1; + iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80); + } + else + { + /* Raise stack overflow, don't push anything. */ + pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK; + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES; + Log11(("iemFpuMaybePushResult: %04x:%08RX64: stack overflow (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW)); + return; + } + + fFsw &= ~X86_FSW_TOP_MASK; + fFsw |= iNewTop << X86_FSW_TOP_SHIFT; + pFpuCtx->FSW = fFsw; + + iemFpuRotateStackPush(pFpuCtx); + RT_NOREF(pVCpu); +} + + +/** + * Stores a result in a FPU register and updates the FSW and FTW. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pFpuCtx The FPU context. + * @param pResult The result to store. + * @param iStReg Which FPU register to store it in. + */ +static void iemFpuStoreResultOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT +{ + Assert(iStReg < 8); + uint16_t fNewFsw = pFpuCtx->FSW; + uint16_t const iReg = (X86_FSW_TOP_GET(fNewFsw) + iStReg) & X86_FSW_TOP_SMASK; + fNewFsw &= ~X86_FSW_C_MASK; + fNewFsw |= pResult->FSW & ~X86_FSW_TOP_MASK; + if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES)) + Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw)); + pFpuCtx->FSW = fNewFsw; + pFpuCtx->FTW |= RT_BIT(iReg); + pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result; + RT_NOREF(pVCpu); +} + + +/** + * Only updates the FPU status word (FSW) with the result of the current + * instruction. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pFpuCtx The FPU context. + * @param u16FSW The FSW output of the current instruction. + */ +static void iemFpuUpdateFSWOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint16_t u16FSW) RT_NOEXCEPT +{ + uint16_t fNewFsw = pFpuCtx->FSW; + fNewFsw &= ~X86_FSW_C_MASK; + fNewFsw |= u16FSW & ~X86_FSW_TOP_MASK; + if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES)) + Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw)); + pFpuCtx->FSW = fNewFsw; + RT_NOREF(pVCpu); +} + + +/** + * Pops one item off the FPU stack if no pending exception prevents it. + * + * @param pFpuCtx The FPU context. + */ +static void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx) RT_NOEXCEPT +{ + /* Check pending exceptions. */ + uint16_t uFSW = pFpuCtx->FSW; + if ( (pFpuCtx->FSW & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE)) + & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM))) + return; + + /* TOP--. */ + uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK; + uFSW &= ~X86_FSW_TOP_MASK; + uFSW |= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK; + pFpuCtx->FSW = uFSW; + + /* Mark the previous ST0 as empty. */ + iOldTop >>= X86_FSW_TOP_SHIFT; + pFpuCtx->FTW &= ~RT_BIT(iOldTop); + + /* Rotate the registers. */ + iemFpuRotateStackPop(pFpuCtx); +} + + +/** + * Pushes a FPU result onto the FPU stack if no pending exception prevents it. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pResult The FPU operation result to push. + */ +void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx); +} + + +/** + * Pushes a FPU result onto the FPU stack if no pending exception prevents it, + * and sets FPUDP and FPUDS. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pResult The FPU operation result to push. + * @param iEffSeg The effective segment register. + * @param GCPtrEff The effective address relative to @a iEffSeg. + */ +void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff); + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx); +} + + +/** + * Replace ST0 with the first value and push the second onto the FPU stack, + * unless a pending exception prevents it. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pResult The FPU operation result to store and push. + */ +void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + + /* Update FSW and bail if there are pending exceptions afterwards. */ + uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK; + fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK; + if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE)) + & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM))) + { + if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES)) + Log11(("iemFpuPushResultTwo: %04x:%08RX64: FSW %#x -> %#x\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw)); + pFpuCtx->FSW = fFsw; + return; + } + + uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK; + if (!(pFpuCtx->FTW & RT_BIT(iNewTop))) + { + /* All is fine, push the actual value. */ + pFpuCtx->FTW |= RT_BIT(iNewTop); + pFpuCtx->aRegs[0].r80 = pResult->r80Result1; + pFpuCtx->aRegs[7].r80 = pResult->r80Result2; + } + else if (pFpuCtx->FCW & X86_FCW_IM) + { + /* Masked stack overflow, push QNaN. */ + fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1; + iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80); + iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80); + } + else + { + /* Raise stack overflow, don't push anything. */ + pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK; + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES; + Log11(("iemFpuPushResultTwo: %04x:%08RX64: stack overflow (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW)); + return; + } + + fFsw &= ~X86_FSW_TOP_MASK; + fFsw |= iNewTop << X86_FSW_TOP_SHIFT; + pFpuCtx->FSW = fFsw; + + iemFpuRotateStackPush(pFpuCtx); +} + + +/** + * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and + * FOP. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pResult The result to store. + * @param iStReg Which FPU register to store it in. + */ +void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg); +} + + +/** + * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and + * FOP, and then pops the stack. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pResult The result to store. + * @param iStReg Which FPU register to store it in. + */ +void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg); + iemFpuMaybePopOne(pFpuCtx); +} + + +/** + * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP, + * FPUDP, and FPUDS. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pResult The result to store. + * @param iStReg Which FPU register to store it in. + * @param iEffSeg The effective memory operand selector register. + * @param GCPtrEff The effective memory operand offset. + */ +void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg, + uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff); + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg); +} + + +/** + * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP, + * FPUDP, and FPUDS, and then pops the stack. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pResult The result to store. + * @param iStReg Which FPU register to store it in. + * @param iEffSeg The effective memory operand selector register. + * @param GCPtrEff The effective memory operand offset. + */ +void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, + uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff); + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg); + iemFpuMaybePopOne(pFpuCtx); +} + + +/** + * Updates the FOP, FPUIP, and FPUCS. For FNOP. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); +} + + +/** + * Updates the FSW, FOP, FPUIP, and FPUCS. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16FSW The FSW from the current instruction. + */ +void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW); +} + + +/** + * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16FSW The FSW from the current instruction. + */ +void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW); + iemFpuMaybePopOne(pFpuCtx); +} + + +/** + * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16FSW The FSW from the current instruction. + * @param iEffSeg The effective memory operand selector register. + * @param GCPtrEff The effective memory operand offset. + */ +void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff); + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW); +} + + +/** + * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16FSW The FSW from the current instruction. + */ +void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW); + iemFpuMaybePopOne(pFpuCtx); + iemFpuMaybePopOne(pFpuCtx); +} + + +/** + * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16FSW The FSW from the current instruction. + * @param iEffSeg The effective memory operand selector register. + * @param GCPtrEff The effective memory operand offset. + */ +void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff); + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW); + iemFpuMaybePopOne(pFpuCtx); +} + + +/** + * Worker routine for raising an FPU stack underflow exception. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pFpuCtx The FPU context. + * @param iStReg The stack register being accessed. + */ +static void iemFpuStackUnderflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint8_t iStReg) +{ + Assert(iStReg < 8 || iStReg == UINT8_MAX); + if (pFpuCtx->FCW & X86_FCW_IM) + { + /* Masked underflow. */ + pFpuCtx->FSW &= ~X86_FSW_C_MASK; + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF; + uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK; + if (iStReg != UINT8_MAX) + { + pFpuCtx->FTW |= RT_BIT(iReg); + iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80); + } + } + else + { + pFpuCtx->FSW &= ~X86_FSW_C_MASK; + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B; + Log11(("iemFpuStackUnderflowOnly: %04x:%08RX64: underflow (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW)); + } + RT_NOREF(pVCpu); +} + + +/** + * Raises a FPU stack underflow exception. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iStReg The destination register that should be loaded + * with QNaN if \#IS is not masked. Specify + * UINT8_MAX if none (like for fcom). + */ +void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg); +} + + +void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff); + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg); +} + + +void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg); + iemFpuMaybePopOne(pFpuCtx); +} + + +void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff); + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg); + iemFpuMaybePopOne(pFpuCtx); +} + + +void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, UINT8_MAX); + iemFpuMaybePopOne(pFpuCtx); + iemFpuMaybePopOne(pFpuCtx); +} + + +void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + + if (pFpuCtx->FCW & X86_FCW_IM) + { + /* Masked overflow - Push QNaN. */ + uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK; + pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK); + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF; + pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT; + pFpuCtx->FTW |= RT_BIT(iNewTop); + iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80); + iemFpuRotateStackPush(pFpuCtx); + } + else + { + /* Exception pending - don't change TOP or the register stack. */ + pFpuCtx->FSW &= ~X86_FSW_C_MASK; + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B; + Log11(("iemFpuStackPushUnderflow: %04x:%08RX64: underflow (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW)); + } +} + + +void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + + if (pFpuCtx->FCW & X86_FCW_IM) + { + /* Masked overflow - Push QNaN. */ + uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK; + pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK); + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF; + pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT; + pFpuCtx->FTW |= RT_BIT(iNewTop); + iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80); + iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80); + iemFpuRotateStackPush(pFpuCtx); + } + else + { + /* Exception pending - don't change TOP or the register stack. */ + pFpuCtx->FSW &= ~X86_FSW_C_MASK; + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B; + Log11(("iemFpuStackPushUnderflowTwo: %04x:%08RX64: underflow (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW)); + } +} + + +/** + * Worker routine for raising an FPU stack overflow exception on a push. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pFpuCtx The FPU context. + */ +static void iemFpuStackPushOverflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT +{ + if (pFpuCtx->FCW & X86_FCW_IM) + { + /* Masked overflow. */ + uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK; + pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK); + pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF; + pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT; + pFpuCtx->FTW |= RT_BIT(iNewTop); + iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80); + iemFpuRotateStackPush(pFpuCtx); + } + else + { + /* Exception pending - don't change TOP or the register stack. */ + pFpuCtx->FSW &= ~X86_FSW_C_MASK; + pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B; + Log11(("iemFpuStackPushOverflowOnly: %04x:%08RX64: overflow (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW)); + } + RT_NOREF(pVCpu); +} + + +/** + * Raises a FPU stack overflow exception on a push. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx); +} + + +/** + * Raises a FPU stack overflow exception on a push with a memory operand. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iEffSeg The effective memory operand selector register. + * @param GCPtrEff The effective memory operand offset. + */ +void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff); + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx); +} + +/** @} */ + + +/** @name SSE+AVX SIMD access and helpers. + * + * @{ + */ +/** + * Stores a result in a SIMD XMM register, updates the MXCSR. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pResult The result to store. + * @param iXmmReg Which SIMD XMM register to store the result in. + */ +void iemSseStoreResult(PVMCPUCC pVCpu, PCIEMSSERESULT pResult, uint8_t iXmmReg) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + pFpuCtx->MXCSR |= pResult->MXCSR & X86_MXCSR_XCPT_FLAGS; + + /* The result is only updated if there is no unmasked exception pending. */ + if (( ~((pFpuCtx->MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) + & (pFpuCtx->MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0) + pVCpu->cpum.GstCtx.XState.x87.aXMM[iXmmReg] = pResult->uResult; +} + + +/** + * Updates the MXCSR. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param fMxcsr The new MXCSR value. + */ +void iemSseUpdateMxcsr(PVMCPUCC pVCpu, uint32_t fMxcsr) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + pFpuCtx->MXCSR |= fMxcsr & X86_MXCSR_XCPT_FLAGS; +} +/** @} */ + + +/** @name Memory access. + * + * @{ + */ + + +/** + * Updates the IEMCPU::cbWritten counter if applicable. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param fAccess The access being accounted for. + * @param cbMem The access size. + */ +DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem) +{ + if ( (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK | IEM_ACCESS_TYPE_WRITE) + || (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA | IEM_ACCESS_TYPE_WRITE) ) + pVCpu->iem.s.cbWritten += (uint32_t)cbMem; +} + + +/** + * Applies the segment limit, base and attributes. + * + * This may raise a \#GP or \#SS. + * + * @returns VBox strict status code. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param fAccess The kind of access which is being performed. + * @param iSegReg The index of the segment register to apply. + * This is UINT8_MAX if none (for IDT, GDT, LDT, + * TSS, ++). + * @param cbMem The access size. + * @param pGCPtrMem Pointer to the guest memory address to apply + * segmentation to. Input and output parameter. + */ +VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT +{ + if (iSegReg == UINT8_MAX) + return VINF_SUCCESS; + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg); + switch (pVCpu->iem.s.enmCpuMode) + { + case IEMMODE_16BIT: + case IEMMODE_32BIT: + { + RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem; + RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1; + + if ( pSel->Attr.n.u1Present + && !pSel->Attr.n.u1Unusable) + { + Assert(pSel->Attr.n.u1DescType); + if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE)) + { + if ( (fAccess & IEM_ACCESS_TYPE_WRITE) + && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) ) + return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess); + + if (!IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + /** @todo CPL check. */ + } + + /* + * There are two kinds of data selectors, normal and expand down. + */ + if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN)) + { + if ( GCPtrFirst32 > pSel->u32Limit + || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */ + return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess); + } + else + { + /* + * The upper boundary is defined by the B bit, not the G bit! + */ + if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1) + || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))) + return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess); + } + *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base; + } + else + { + /* + * Code selector and usually be used to read thru, writing is + * only permitted in real and V8086 mode. + */ + if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE) + || ( (fAccess & IEM_ACCESS_TYPE_READ) + && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) ) + && !IEM_IS_REAL_OR_V86_MODE(pVCpu) ) + return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess); + + if ( GCPtrFirst32 > pSel->u32Limit + || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */ + return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess); + + if (!IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + /** @todo CPL check. */ + } + + *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base; + } + } + else + return iemRaiseGeneralProtectionFault0(pVCpu); + return VINF_SUCCESS; + } + + case IEMMODE_64BIT: + { + RTGCPTR GCPtrMem = *pGCPtrMem; + if (iSegReg == X86_SREG_GS || iSegReg == X86_SREG_FS) + *pGCPtrMem = GCPtrMem + pSel->u64Base; + + Assert(cbMem >= 1); + if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1))) + return VINF_SUCCESS; + /** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec. + * 4.12.2 "Data Limit Checks in 64-bit Mode". */ + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + default: + AssertFailedReturn(VERR_IEM_IPE_7); + } +} + + +/** + * Translates a virtual address to a physical physical address and checks if we + * can access the page as specified. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param GCPtrMem The virtual address. + * @param cbAccess The access size, for raising \#PF correctly for + * FXSAVE and such. + * @param fAccess The intended access. + * @param pGCPhysMem Where to return the physical address. + */ +VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t cbAccess, + uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT +{ + /** @todo Need a different PGM interface here. We're currently using + * generic / REM interfaces. this won't cut it for R0. */ + /** @todo If/when PGM handles paged real-mode, we can remove the hack in + * iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault + * here. */ + PGMPTWALK Walk; + int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk); + if (RT_FAILURE(rc)) + { + Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem)); + /** @todo Check unassigned memory in unpaged mode. */ + /** @todo Reserved bits in page tables. Requires new PGM interface. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */); +#endif + *pGCPhysMem = NIL_RTGCPHYS; + return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess, rc); + } + + /* If the page is writable and does not have the no-exec bit set, all + access is allowed. Otherwise we'll have to check more carefully... */ + if ((Walk.fEffective & (X86_PTE_RW | X86_PTE_US | X86_PTE_PAE_NX)) != (X86_PTE_RW | X86_PTE_US)) + { + /* Write to read only memory? */ + if ( (fAccess & IEM_ACCESS_TYPE_WRITE) + && !(Walk.fEffective & X86_PTE_RW) + && ( ( pVCpu->iem.s.uCpl == 3 + && !(fAccess & IEM_ACCESS_WHAT_SYS)) + || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP))) + { + Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem)); + *pGCPhysMem = NIL_RTGCPHYS; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +#endif + return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED); + } + + /* Kernel memory accessed by userland? */ + if ( !(Walk.fEffective & X86_PTE_US) + && pVCpu->iem.s.uCpl == 3 + && !(fAccess & IEM_ACCESS_WHAT_SYS)) + { + Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem)); + *pGCPhysMem = NIL_RTGCPHYS; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +#endif + return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess, VERR_ACCESS_DENIED); + } + + /* Executing non-executable memory? */ + if ( (fAccess & IEM_ACCESS_TYPE_EXEC) + && (Walk.fEffective & X86_PTE_PAE_NX) + && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) + { + Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem)); + *pGCPhysMem = NIL_RTGCPHYS; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +#endif + return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess & ~(IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE), + VERR_ACCESS_DENIED); + } + } + + /* + * Set the dirty / access flags. + * ASSUMES this is set when the address is translated rather than on committ... + */ + /** @todo testcase: check when A and D bits are actually set by the CPU. */ + uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A; + if ((Walk.fEffective & fAccessedDirty) != fAccessedDirty) + { + int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty); + AssertRC(rc2); + /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */ + Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK)); + } + + RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK); + *pGCPhysMem = GCPhys; + return VINF_SUCCESS; +} + + +/** + * Looks up a memory mapping entry. + * + * @returns The mapping index (positive) or VERR_NOT_FOUND (negative). + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pvMem The memory address. + * @param fAccess The access to. + */ +DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) +{ + Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)); + fAccess &= IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK; + if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem + && (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess) + return 0; + if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem + && (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess) + return 1; + if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem + && (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess) + return 2; + return VERR_NOT_FOUND; +} + + +/** + * Finds a free memmap entry when using iNextMapping doesn't work. + * + * @returns Memory mapping index, 1024 on failure. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +static unsigned iemMemMapFindFree(PVMCPUCC pVCpu) +{ + /* + * The easy case. + */ + if (pVCpu->iem.s.cActiveMappings == 0) + { + pVCpu->iem.s.iNextMapping = 1; + return 0; + } + + /* There should be enough mappings for all instructions. */ + AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024); + + for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++) + if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID) + return i; + + AssertFailedReturn(1024); +} + + +/** + * Commits a bounce buffer that needs writing back and unmaps it. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iMemMap The index of the buffer to commit. + * @param fPostponeFail Whether we can postpone writer failures to ring-3. + * Always false in ring-3, obviously. + */ +static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail) +{ + Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED); + Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE); +#ifdef IN_RING3 + Assert(!fPostponeFail); + RT_NOREF_PV(fPostponeFail); +#endif + + /* + * Do the writing. + */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned) + { + uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst; + uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond; + uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]; + if (!pVCpu->iem.s.fBypassHandlers) + { + /* + * Carefully and efficiently dealing with access handler return + * codes make this a little bloated. + */ + VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, + pbBuf, + cbFirst, + PGMACCESSORIGIN_IEM); + if (rcStrict == VINF_SUCCESS) + { + if (cbSecond) + { + rcStrict = PGMPhysWrite(pVM, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, + pbBuf + cbFirst, + cbSecond, + PGMACCESSORIGIN_IEM); + if (rcStrict == VINF_SUCCESS) + { /* nothing */ } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) )); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } +#ifndef IN_RING3 + else if (fPostponeFail) + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) )); + pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND; + VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM); + return iemSetPassUpStatus(pVCpu, rcStrict); + } +#endif + else + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) )); + return rcStrict; + } + } + } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + { + if (!cbSecond) + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) )); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } + else + { + VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, + pbBuf + cbFirst, + cbSecond, + PGMACCESSORIGIN_IEM); + if (rcStrict2 == VINF_SUCCESS) + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict), + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond)); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2)) + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict), + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) )); + PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } +#ifndef IN_RING3 + else if (fPostponeFail) + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) )); + pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND; + VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM); + return iemSetPassUpStatus(pVCpu, rcStrict); + } +#endif + else + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict), + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) )); + return rcStrict2; + } + } + } +#ifndef IN_RING3 + else if (fPostponeFail) + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) )); + if (!cbSecond) + pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST; + else + pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND; + VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM); + return iemSetPassUpStatus(pVCpu, rcStrict); + } +#endif + else + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict), + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond)); + return rcStrict; + } + } + else + { + /* + * No access handlers, much simpler. + */ + int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst); + if (RT_SUCCESS(rc)) + { + if (cbSecond) + { + rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc)); + return rc; + } + } + } + else + { + Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n", + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond)); + return rc; + } + } + } + +#if defined(IEM_LOG_MEMORY_WRITES) + Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, + RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0])); + if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond) + Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, + RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64), + &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst])); + + size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond; + g_cbIemWrote = cbWrote; + memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote))); +#endif + + /* + * Free the mapping entry. + */ + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID; + Assert(pVCpu->iem.s.cActiveMappings != 0); + pVCpu->iem.s.cActiveMappings--; + return VINF_SUCCESS; +} + + +/** + * iemMemMap worker that deals with a request crossing pages. + */ +static VBOXSTRICTRC +iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess) +{ + Assert(cbMem <= GUEST_PAGE_SIZE); + + /* + * Do the address translations. + */ + uint32_t const cbFirstPage = GUEST_PAGE_SIZE - (uint32_t)(GCPtrFirst & GUEST_PAGE_OFFSET_MASK); + RTGCPHYS GCPhysFirst; + VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, cbFirstPage, fAccess, &GCPhysFirst); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + Assert((GCPhysFirst & GUEST_PAGE_OFFSET_MASK) == (GCPtrFirst & GUEST_PAGE_OFFSET_MASK)); + + uint32_t const cbSecondPage = (uint32_t)cbMem - cbFirstPage; + RTGCPHYS GCPhysSecond; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK, + cbSecondPage, fAccess, &GCPhysSecond); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + Assert((GCPhysSecond & GUEST_PAGE_OFFSET_MASK) == 0); + GCPhysSecond &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; /** @todo why? */ + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Read in the current memory content if it's a read, execute or partial + * write access. + */ + uint8_t * const pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]; + + if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE)) + { + if (!pVCpu->iem.s.fBypassHandlers) + { + /* + * Must carefully deal with access handler status codes here, + * makes the code a bit bloated. + */ + rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM); + if (rcStrict == VINF_SUCCESS) + { + rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM); + if (rcStrict == VINF_SUCCESS) + { /*likely */ } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + else + { + Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n", + GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) )); + return rcStrict; + } + } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + { + VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM); + if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2)) + { + PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } + else + { + Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n", + GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) )); + return rcStrict2; + } + } + else + { + Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n", + GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) )); + return rcStrict; + } + } + else + { + /* + * No informational status codes here, much more straight forward. + */ + int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage); + if (RT_SUCCESS(rc)) + { + Assert(rc == VINF_SUCCESS); + rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage); + if (RT_SUCCESS(rc)) + Assert(rc == VINF_SUCCESS); + else + { + Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc)); + return rc; + } + } + else + { + Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc)); + return rc; + } + } + } +#ifdef VBOX_STRICT + else + memset(pbBuf, 0xcc, cbMem); + if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab)) + memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem); +#endif + AssertCompileMemberAlignment(VMCPU, iem.s.aBounceBuffers, 64); + + /* + * Commit the bounce buffer entry. + */ + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst; + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond; + pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage; + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage; + pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false; + pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf; + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED; + pVCpu->iem.s.iNextMapping = iMemMap + 1; + pVCpu->iem.s.cActiveMappings++; + + iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem); + *ppvMem = pbBuf; + return VINF_SUCCESS; +} + + +/** + * iemMemMap woker that deals with iemMemPageMap failures. + */ +static VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem, + RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap) +{ + /* + * Filter out conditions we can handle and the ones which shouldn't happen. + */ + if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE + && rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL + && rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED) + { + AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8); + return rcMap; + } + pVCpu->iem.s.cPotentialExits++; + + /* + * Read in the current memory content if it's a read, execute or partial + * write access. + */ + uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]; + if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE)) + { + if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED) + memset(pbBuf, 0xff, cbMem); + else + { + int rc; + if (!pVCpu->iem.s.fBypassHandlers) + { + VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM); + if (rcStrict == VINF_SUCCESS) + { /* nothing */ } + else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + else + { + Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n", + GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) )); + return rcStrict; + } + } + else + { + rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n", + GCPhysFirst, rc)); + return rc; + } + } + } + } +#ifdef VBOX_STRICT + else + memset(pbBuf, 0xcc, cbMem); +#endif +#ifdef VBOX_STRICT + if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab)) + memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem); +#endif + + /* + * Commit the bounce buffer entry. + */ + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst; + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS; + pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem; + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0; + pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED; + pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf; + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED; + pVCpu->iem.s.iNextMapping = iMemMap + 1; + pVCpu->iem.s.cActiveMappings++; + + iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem); + *ppvMem = pbBuf; + return VINF_SUCCESS; +} + + + +/** + * Maps the specified guest memory for the given kind of access. + * + * This may be using bounce buffering of the memory if it's crossing a page + * boundary or if there is an access handler installed for any of it. Because + * of lock prefix guarantees, we're in for some extra clutter when this + * happens. + * + * This may raise a \#GP, \#SS, \#PF or \#AC. + * + * @returns VBox strict status code. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param ppvMem Where to return the pointer to the mapped memory. + * @param cbMem The number of bytes to map. This is usually 1, 2, 4, 6, + * 8, 12, 16, 32 or 512. When used by string operations + * it can be up to a page. + * @param iSegReg The index of the segment register to use for this + * access. The base and limits are checked. Use UINT8_MAX + * to indicate that no segmentation is required (for IDT, + * GDT and LDT accesses). + * @param GCPtrMem The address of the guest memory. + * @param fAccess How the memory is being accessed. The + * IEM_ACCESS_TYPE_XXX bit is used to figure out how to map + * the memory, while the IEM_ACCESS_WHAT_XXX bit is used + * when raising exceptions. + * @param uAlignCtl Alignment control: + * - Bits 15:0 is the alignment mask. + * - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP, + * IEM_MEMMAP_F_ALIGN_SSE, and + * IEM_MEMMAP_F_ALIGN_GP_OR_AC. + * Pass zero to skip alignment. + */ +VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, + uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT +{ + /* + * Check the input and figure out which mapping entry to use. + */ + Assert(cbMem <= sizeof(pVCpu->iem.s.aBounceBuffers[0])); + Assert( cbMem <= 64 || cbMem == 512 || cbMem == 256 || cbMem == 108 || cbMem == 104 || cbMem == 102 || cbMem == 94 + || (iSegReg == UINT8_MAX && uAlignCtl == 0 && fAccess == IEM_ACCESS_DATA_R /* for the CPUID logging interface */) ); + Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK))); + Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)); + + unsigned iMemMap = pVCpu->iem.s.iNextMapping; + if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings) + || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID) + { + iMemMap = iemMemMapFindFree(pVCpu); + AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), + ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings, + pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, + pVCpu->iem.s.aMemMappings[2].fAccess), + VERR_IEM_IPE_9); + } + + /* + * Map the memory, checking that we can actually access it. If something + * slightly complicated happens, fall back on bounce buffering. + */ + VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem); + if (rcStrict == VINF_SUCCESS) + { /* likely */ } + else + return rcStrict; + + if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE) /* Crossing a page boundary? */ + { /* likely */ } + else + return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess); + + /* + * Alignment check. + */ + if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 ) + { /* likelyish */ } + else + { + /* Misaligned access. */ + if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS) + { + if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP) + || ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE) + && (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) ) + { + AssertCompile(X86_CR0_AM == X86_EFL_AC); + + if (iemMemAreAlignmentChecksEnabled(pVCpu)) + return iemRaiseAlignmentCheckException(pVCpu); + } + else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC) + && (GCPtrMem & 3) /* The value 4 matches 10980xe's FXSAVE and helps make bs3-cpu-basic2 work. */ + /** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU + * implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. Using 4 for now as + * that's what FXSAVE does on a 10980xe. */ + && iemMemAreAlignmentChecksEnabled(pVCpu)) + return iemRaiseAlignmentCheckException(pVCpu); + else + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + +#ifdef IEM_WITH_DATA_TLB + Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC)); + + /* + * Get the TLB entry for this page. + */ + uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem); + PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag); + if (pTlbe->uTag == uTag) + { +# ifdef VBOX_WITH_STATISTICS + pVCpu->iem.s.DataTlb.cTlbHits++; +# endif + } + else + { + pVCpu->iem.s.DataTlb.cTlbMisses++; + PGMPTWALK Walk; + int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk); + if (RT_FAILURE(rc)) + { + Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem)); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */); +# endif + return iemRaisePageFault(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, rc); + } + + Assert(Walk.fSucceeded); + pTlbe->uTag = uTag; + pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A); /* skipping NX */ + pTlbe->GCPhys = Walk.GCPhys; + pTlbe->pbMappingR3 = NULL; + } + + /* + * Check TLB page table level access flags. + */ + /* If the page is either supervisor only or non-writable, we need to do + more careful access checks. */ + if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_WRITE)) + { + /* Write to read only memory? */ + if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_WRITE) + && (fAccess & IEM_ACCESS_TYPE_WRITE) + && ( ( pVCpu->iem.s.uCpl == 3 + && !(fAccess & IEM_ACCESS_WHAT_SYS)) + || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP))) + { + Log(("iemMemMap: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem)); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +# endif + return iemRaisePageFault(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED); + } + + /* Kernel memory accessed by userland? */ + if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) + && pVCpu->iem.s.uCpl == 3 + && !(fAccess & IEM_ACCESS_WHAT_SYS)) + { + Log(("iemMemMap: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem)); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +# endif + return iemRaisePageFault(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, VERR_ACCESS_DENIED); + } + } + + /* + * Set the dirty / access flags. + * ASSUMES this is set when the address is translated rather than on commit... + */ + /** @todo testcase: check when A and D bits are actually set by the CPU. */ + uint64_t const fTlbAccessedDirty = (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PT_NO_DIRTY : 0) | IEMTLBE_F_PT_NO_ACCESSED; + if (pTlbe->fFlagsAndPhysRev & fTlbAccessedDirty) + { + uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A; + int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty); + AssertRC(rc2); + /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */ + Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK)); + pTlbe->fFlagsAndPhysRev &= ~fTlbAccessedDirty; + } + + /* + * Look up the physical page info if necessary. + */ + uint8_t *pbMem = NULL; + if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev) +# ifdef IN_RING3 + pbMem = pTlbe->pbMappingR3; +# else + pbMem = NULL; +# endif + else + { + AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE); + AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ); + AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3); + AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED); + if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR)) + { /* likely */ } + else + IEMTlbInvalidateAllPhysicalSlow(pVCpu); + pTlbe->pbMappingR3 = NULL; + pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV + | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_UNASSIGNED); + int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev, + &pbMem, &pTlbe->fFlagsAndPhysRev); + AssertRCReturn(rc, rc); +# ifdef IN_RING3 + pTlbe->pbMappingR3 = pbMem; +# endif + } + + /* + * Check the physical page level access and mapping. + */ + if ( !(pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ)) + || !(pTlbe->fFlagsAndPhysRev & ( (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PG_NO_WRITE : 0) + | (fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0))) ) + { /* probably likely */ } + else + return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, + pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess, + pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED + : pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL + : VERR_PGM_PHYS_TLB_CATCH_WRITE); + Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */ + + if (pbMem) + { + Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK)); + pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK); + fAccess |= IEM_ACCESS_NOT_LOCKED; + } + else + { + Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED)); + RTGCPHYS const GCPhysFirst = pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK); + rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock); + if (rcStrict != VINF_SUCCESS) + return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict); + } + + void * const pvMem = pbMem; + + if (fAccess & IEM_ACCESS_TYPE_WRITE) + Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem)); + if (fAccess & IEM_ACCESS_TYPE_READ) + Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem)); + +#else /* !IEM_WITH_DATA_TLB */ + + RTGCPHYS GCPhysFirst; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, &GCPhysFirst); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + if (fAccess & IEM_ACCESS_TYPE_WRITE) + Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem)); + if (fAccess & IEM_ACCESS_TYPE_READ) + Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem)); + + void *pvMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock); + if (rcStrict != VINF_SUCCESS) + return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict); + +#endif /* !IEM_WITH_DATA_TLB */ + + /* + * Fill in the mapping table entry. + */ + pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem; + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess; + pVCpu->iem.s.iNextMapping = iMemMap + 1; + pVCpu->iem.s.cActiveMappings += 1; + + iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem); + *ppvMem = pvMem; + + return VINF_SUCCESS; +} + + +/** + * Commits the guest memory if bounce buffered and unmaps it. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pvMem The mapping. + * @param fAccess The kind of access. + */ +VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT +{ + int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess); + AssertReturn(iMemMap >= 0, iMemMap); + + /* If it's bounce buffered, we may need to write back the buffer. */ + if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED) + { + if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE) + return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/); + } + /* Otherwise unlock it. */ + else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED)) + PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock); + + /* Free the entry. */ + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID; + Assert(pVCpu->iem.s.cActiveMappings != 0); + pVCpu->iem.s.cActiveMappings--; + return VINF_SUCCESS; +} + +#ifdef IEM_WITH_SETJMP + +/** + * Maps the specified guest memory for the given kind of access, longjmp on + * error. + * + * This may be using bounce buffering of the memory if it's crossing a page + * boundary or if there is an access handler installed for any of it. Because + * of lock prefix guarantees, we're in for some extra clutter when this + * happens. + * + * This may raise a \#GP, \#SS, \#PF or \#AC. + * + * @returns Pointer to the mapped memory. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbMem The number of bytes to map. This is usually 1, + * 2, 4, 6, 8, 12, 16, 32 or 512. When used by + * string operations it can be up to a page. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * Use UINT8_MAX to indicate that no segmentation + * is required (for IDT, GDT and LDT accesses). + * @param GCPtrMem The address of the guest memory. + * @param fAccess How the memory is being accessed. The + * IEM_ACCESS_TYPE_XXX bit is used to figure out + * how to map the memory, while the + * IEM_ACCESS_WHAT_XXX bit is used when raising + * exceptions. + * @param uAlignCtl Alignment control: + * - Bits 15:0 is the alignment mask. + * - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP, + * IEM_MEMMAP_F_ALIGN_SSE, and + * IEM_MEMMAP_F_ALIGN_GP_OR_AC. + * Pass zero to skip alignment. + */ +void *iemMemMapJmp(PVMCPUCC pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess, + uint32_t uAlignCtl) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* + * Check the input, check segment access and adjust address + * with segment base. + */ + Assert(cbMem <= 64 || cbMem == 512 || cbMem == 108 || cbMem == 104 || cbMem == 94); /* 512 is the max! */ + Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK))); + Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)); + + VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem); + if (rcStrict == VINF_SUCCESS) { /*likely*/ } + else IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + + /* + * Alignment check. + */ + if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 ) + { /* likelyish */ } + else + { + /* Misaligned access. */ + if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS) + { + if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP) + || ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE) + && (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) ) + { + AssertCompile(X86_CR0_AM == X86_EFL_AC); + + if (iemMemAreAlignmentChecksEnabled(pVCpu)) + iemRaiseAlignmentCheckExceptionJmp(pVCpu); + } + else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC) + && (GCPtrMem & 3) /* The value 4 matches 10980xe's FXSAVE and helps make bs3-cpu-basic2 work. */ + /** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU + * implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. Using 4 for now as + * that's what FXSAVE does on a 10980xe. */ + && iemMemAreAlignmentChecksEnabled(pVCpu)) + iemRaiseAlignmentCheckExceptionJmp(pVCpu); + else + iemRaiseGeneralProtectionFault0Jmp(pVCpu); + } + } + + /* + * Figure out which mapping entry to use. + */ + unsigned iMemMap = pVCpu->iem.s.iNextMapping; + if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings) + || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID) + { + iMemMap = iemMemMapFindFree(pVCpu); + AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), + ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings, + pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, + pVCpu->iem.s.aMemMappings[2].fAccess), + IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_9)); + } + + /* + * Crossing a page boundary? + */ + if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE) + { /* No (likely). */ } + else + { + void *pvMem; + rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess); + if (rcStrict == VINF_SUCCESS) + return pvMem; + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + } + +#ifdef IEM_WITH_DATA_TLB + Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC)); + + /* + * Get the TLB entry for this page. + */ + uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem); + PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag); + if (pTlbe->uTag == uTag) + STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;}); + else + { + pVCpu->iem.s.DataTlb.cTlbMisses++; + PGMPTWALK Walk; + int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk); + if (RT_FAILURE(rc)) + { + Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem)); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */); +# endif + iemRaisePageFaultJmp(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, rc); + } + + Assert(Walk.fSucceeded); + pTlbe->uTag = uTag; + pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A); /* skipping NX */ + pTlbe->GCPhys = Walk.GCPhys; + pTlbe->pbMappingR3 = NULL; + } + + /* + * Check the flags and physical revision. + */ + /** @todo make the caller pass these in with fAccess. */ + uint64_t const fNoUser = (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS && pVCpu->iem.s.uCpl == 3 + ? IEMTLBE_F_PT_NO_USER : 0; + uint64_t const fNoWriteNoDirty = fAccess & IEM_ACCESS_TYPE_WRITE + ? IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PT_NO_DIRTY + | ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP) + || (pVCpu->iem.s.uCpl == 3 && (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS) + ? IEMTLBE_F_PT_NO_WRITE : 0) + : 0; + uint64_t const fNoRead = fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0; + uint8_t *pbMem = NULL; + if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PT_NO_ACCESSED | fNoRead | fNoWriteNoDirty | fNoUser)) + == pVCpu->iem.s.DataTlb.uTlbPhysRev) +# ifdef IN_RING3 + pbMem = pTlbe->pbMappingR3; +# else + pbMem = NULL; +# endif + else + { + /* + * Okay, something isn't quite right or needs refreshing. + */ + /* Write to read only memory? */ + if (pTlbe->fFlagsAndPhysRev & fNoWriteNoDirty & IEMTLBE_F_PT_NO_WRITE) + { + Log(("iemMemMapJmp: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem)); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +# endif + iemRaisePageFaultJmp(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED); + } + + /* Kernel memory accessed by userland? */ + if (pTlbe->fFlagsAndPhysRev & fNoUser & IEMTLBE_F_PT_NO_USER) + { + Log(("iemMemMapJmp: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem)); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (Walk.fFailed & PGM_WALKFAIL_EPT) + IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */); +# endif + iemRaisePageFaultJmp(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, VERR_ACCESS_DENIED); + } + + /* Set the dirty / access flags. + ASSUMES this is set when the address is translated rather than on commit... */ + /** @todo testcase: check when A and D bits are actually set by the CPU. */ + if (pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) | IEMTLBE_F_PT_NO_ACCESSED)) + { + uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A; + int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty); + AssertRC(rc2); + /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */ + Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK)); + pTlbe->fFlagsAndPhysRev &= ~((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) | IEMTLBE_F_PT_NO_ACCESSED); + } + + /* + * Check if the physical page info needs updating. + */ + if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev) +# ifdef IN_RING3 + pbMem = pTlbe->pbMappingR3; +# else + pbMem = NULL; +# endif + else + { + AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE); + AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ); + AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3); + AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED); + pTlbe->pbMappingR3 = NULL; + pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV + | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_UNASSIGNED); + int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev, + &pbMem, &pTlbe->fFlagsAndPhysRev); + AssertRCStmt(rc, IEM_DO_LONGJMP(pVCpu, rc)); +# ifdef IN_RING3 + pTlbe->pbMappingR3 = pbMem; +# endif + } + + /* + * Check the physical page level access and mapping. + */ + if (!(pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty | fNoRead) & (IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ)))) + { /* probably likely */ } + else + { + rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, (void **)&pbMem, cbMem, + pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess, + pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED + : pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL + : VERR_PGM_PHYS_TLB_CATCH_WRITE); + if (rcStrict == VINF_SUCCESS) + return pbMem; + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + } + } + Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */ + + if (pbMem) + { + Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK)); + pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK); + fAccess |= IEM_ACCESS_NOT_LOCKED; + } + else + { + Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED)); + RTGCPHYS const GCPhysFirst = pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK); + rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock); + if (rcStrict == VINF_SUCCESS) + return pbMem; + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + } + + void * const pvMem = pbMem; + + if (fAccess & IEM_ACCESS_TYPE_WRITE) + Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem)); + if (fAccess & IEM_ACCESS_TYPE_READ) + Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem)); + +#else /* !IEM_WITH_DATA_TLB */ + + + RTGCPHYS GCPhysFirst; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, &GCPhysFirst); + if (rcStrict == VINF_SUCCESS) { /*likely*/ } + else IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + + if (fAccess & IEM_ACCESS_TYPE_WRITE) + Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem)); + if (fAccess & IEM_ACCESS_TYPE_READ) + Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem)); + + void *pvMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock); + if (rcStrict == VINF_SUCCESS) + { /* likely */ } + else + { + rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict); + if (rcStrict == VINF_SUCCESS) + return pvMem; + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + } + +#endif /* !IEM_WITH_DATA_TLB */ + + /* + * Fill in the mapping table entry. + */ + pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem; + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess; + pVCpu->iem.s.iNextMapping = iMemMap + 1; + pVCpu->iem.s.cActiveMappings++; + + iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem); + return pvMem; +} + + +/** + * Commits the guest memory if bounce buffered and unmaps it, longjmp on error. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pvMem The mapping. + * @param fAccess The kind of access. + */ +void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) IEM_NOEXCEPT_MAY_LONGJMP +{ + int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess); + AssertStmt(iMemMap >= 0, IEM_DO_LONGJMP(pVCpu, iMemMap)); + + /* If it's bounce buffered, we may need to write back the buffer. */ + if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED) + { + if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE) + { + VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/); + if (rcStrict == VINF_SUCCESS) + return; + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + } + } + /* Otherwise unlock it. */ + else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED)) + PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock); + + /* Free the entry. */ + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID; + Assert(pVCpu->iem.s.cActiveMappings != 0); + pVCpu->iem.s.cActiveMappings--; +} + +#endif /* IEM_WITH_SETJMP */ + +#ifndef IN_RING3 +/** + * Commits the guest memory if bounce buffered and unmaps it, if any bounce + * buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff). + * + * Allows the instruction to be completed and retired, while the IEM user will + * return to ring-3 immediately afterwards and do the postponed writes there. + * + * @returns VBox status code (no strict statuses). Caller must check + * VMCPU_FF_IEM before repeating string instructions and similar stuff. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pvMem The mapping. + * @param fAccess The kind of access. + */ +VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT +{ + int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess); + AssertReturn(iMemMap >= 0, iMemMap); + + /* If it's bounce buffered, we may need to write back the buffer. */ + if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED) + { + if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE) + return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /*fPostponeFail*/); + } + /* Otherwise unlock it. */ + else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED)) + PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock); + + /* Free the entry. */ + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID; + Assert(pVCpu->iem.s.cActiveMappings != 0); + pVCpu->iem.s.cActiveMappings--; + return VINF_SUCCESS; +} +#endif + + +/** + * Rollbacks mappings, releasing page locks and such. + * + * The caller shall only call this after checking cActiveMappings. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + Assert(pVCpu->iem.s.cActiveMappings > 0); + + uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings); + while (iMemMap-- > 0) + { + uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess; + if (fAccess != IEM_ACCESS_INVALID) + { + AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess)); + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID; + if (!(fAccess & (IEM_ACCESS_BOUNCE_BUFFERED | IEM_ACCESS_NOT_LOCKED))) + PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock); + AssertMsg(pVCpu->iem.s.cActiveMappings > 0, + ("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n", + iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond)); + pVCpu->iem.s.cActiveMappings--; + } + } +} + + +/** + * Fetches a data byte. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu8Dst Where to return the byte. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint8_t const *pu8Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Src, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0); + if (rc == VINF_SUCCESS) + { + *pu8Dst = *pu8Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data byte, longjmp on error. + * + * @returns The byte. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + uint8_t const *pu8Src = (uint8_t const *)iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0); + uint8_t const bRet = *pu8Src; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R); + return bRet; +} +#endif /* IEM_WITH_SETJMP */ + + +/** + * Fetches a data word. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu16Dst Where to return the word. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint16_t const *pu16Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, sizeof(*pu16Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu16Dst = *pu16Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data word, longjmp on error. + * + * @returns The word + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + uint16_t const *pu16Src = (uint16_t const *)iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, + sizeof(*pu16Src) - 1); + uint16_t const u16Ret = *pu16Src; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R); + return u16Ret; +} +#endif + + +/** + * Fetches a data dword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32Dst Where to return the dword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint32_t const *pu32Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu32Dst = *pu32Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +/** + * Fetches a data dword and zero extends it to a qword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint32_t const *pu32Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu64Dst = *pu32Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP + +/** + * Fetches a data dword, longjmp on error, fallback/safe version. + * + * @returns The dword + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +uint32_t iemMemFetchDataU32SafeJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, + sizeof(*pu32Src) - 1); + uint32_t const u32Ret = *pu32Src; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R); + return u32Ret; +} + + +/** + * Fetches a data dword, longjmp on error. + * + * @returns The dword + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ +# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3) + /* + * Convert from segmented to flat address and check that it doesn't cross a page boundrary. + */ + RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem); + if (RT_LIKELY((GCPtrEff & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(uint32_t))) + { + /* + * TLB lookup. + */ + uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrEff); + PIEMTLBENTRY pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag); + if (pTlbe->uTag == uTag) + { + /* + * Check TLB page table level access flags. + */ + uint64_t const fNoUser = pVCpu->iem.s.uCpl == 3 ? IEMTLBE_F_PT_NO_USER : 0; + if ( (pTlbe->fFlagsAndPhysRev & ( IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PG_NO_READ + | IEMTLBE_F_PT_NO_ACCESSED | IEMTLBE_F_NO_MAPPINGR3 | fNoUser)) + == pVCpu->iem.s.DataTlb.uTlbPhysRev) + { + STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;}); + + /* + * Alignment check: + */ + /** @todo check priority \#AC vs \#PF */ + if ( !(GCPtrEff & (sizeof(uint32_t) - 1)) + || !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM) + || !pVCpu->cpum.GstCtx.eflags.Bits.u1AC + || pVCpu->iem.s.uCpl != 3) + { + /* + * Fetch and return the dword + */ + Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */ + Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK)); + return *(uint32_t const *)&pTlbe->pbMappingR3[GCPtrEff & GUEST_PAGE_OFFSET_MASK]; + } + Log10(("iemMemFetchDataU32Jmp: Raising #AC for %RGv\n", GCPtrEff)); + iemRaiseAlignmentCheckExceptionJmp(pVCpu); + } + } + } + + /* Fall back on the slow careful approach in case of TLB miss, MMIO, exception + outdated page pointer, or other troubles. */ + Log10(("iemMemFetchDataU32Jmp: %u:%RGv fallback\n", iSegReg, GCPtrMem)); + return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem); + +# else + uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1); + uint32_t const u32Ret = *pu32Src; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R); + return u32Ret; +# endif +} +#endif + + +#ifdef SOME_UNUSED_FUNCTION +/** + * Fetches a data dword and sign extends it to a qword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64Dst Where to return the sign extended value. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + int32_t const *pi32Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pi32Src, sizeof(*pi32Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, sizeof(*pi32Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu64Dst = *pi32Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R); + } +#ifdef __GNUC__ /* warning: GCC may be a royal pain */ + else + *pu64Dst = 0; +#endif + return rc; +} +#endif + + +/** + * Fetches a data qword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint64_t const *pu64Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu64Dst = *pu64Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data qword, longjmp on error. + * + * @returns The qword. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1); + uint64_t const u64Ret = *pu64Src; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R); + return u64Ret; +} +#endif + + +/** + * Fetches a data qword, aligned at a 16 byte boundrary (for SSE). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint64_t const *pu64Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, 15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE); + if (rc == VINF_SUCCESS) + { + *pu64Dst = *pu64Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data qword, longjmp on error. + * + * @returns The qword. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, + 15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE); + uint64_t const u64Ret = *pu64Src; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R); + return u64Ret; +} +#endif + + +/** + * Fetches a data tword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pr80Dst Where to return the tword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PCRTFLOAT80U pr80Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pr80Src, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7); + if (rc == VINF_SUCCESS) + { + *pr80Dst = *pr80Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data tword, longjmp on error. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pr80Dst Where to return the tword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7); + *pr80Dst = *pr80Src; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R); +} +#endif + + +/** + * Fetches a data decimal tword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pd80Dst Where to return the tword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PCRTPBCD80U pd80Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pd80Src, sizeof(*pd80Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, 7 /** @todo FBLD alignment check */); + if (rc == VINF_SUCCESS) + { + *pd80Dst = *pd80Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data decimal tword, longjmp on error. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pd80Dst Where to return the tword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PCRTPBCD80U pd80Src = (PCRTPBCD80U)iemMemMapJmp(pVCpu, sizeof(*pd80Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, 7 /** @todo FBSTP alignment check */); + *pd80Dst = *pd80Src; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R); +} +#endif + + +/** + * Fetches a data dqword (double qword), generally SSE related. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu128Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PCRTUINT128U pu128Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, 0 /* NO_AC variant */); + if (rc == VINF_SUCCESS) + { + pu128Dst->au64[0] = pu128Src->au64[0]; + pu128Dst->au64[1] = pu128Src->au64[1]; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data dqword (double qword), generally SSE related. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu128Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, 0 /* NO_AC variant */); + pu128Dst->au64[0] = pu128Src->au64[0]; + pu128Dst->au64[1] = pu128Src->au64[1]; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R); +} +#endif + + +/** + * Fetches a data dqword (double qword) at an aligned address, generally SSE + * related. + * + * Raises \#GP(0) if not aligned. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu128Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PCRTUINT128U pu128Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, (sizeof(*pu128Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE); + if (rc == VINF_SUCCESS) + { + pu128Dst->au64[0] = pu128Src->au64[0]; + pu128Dst->au64[1] = pu128Src->au64[1]; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data dqword (double qword) at an aligned address, generally SSE + * related, longjmp on error. + * + * Raises \#GP(0) if not aligned. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu128Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, + RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, + (sizeof(*pu128Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE); + pu128Dst->au64[0] = pu128Src->au64[0]; + pu128Dst->au64[1] = pu128Src->au64[1]; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R); +} +#endif + + +/** + * Fetches a data oword (octo word), generally AVX related. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu256Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PCRTUINT256U pu256Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, 0 /* NO_AC variant */); + if (rc == VINF_SUCCESS) + { + pu256Dst->au64[0] = pu256Src->au64[0]; + pu256Dst->au64[1] = pu256Src->au64[1]; + pu256Dst->au64[2] = pu256Src->au64[2]; + pu256Dst->au64[3] = pu256Src->au64[3]; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data oword (octo word), generally AVX related. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu256Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, 0 /* NO_AC variant */); + pu256Dst->au64[0] = pu256Src->au64[0]; + pu256Dst->au64[1] = pu256Src->au64[1]; + pu256Dst->au64[2] = pu256Src->au64[2]; + pu256Dst->au64[3] = pu256Src->au64[3]; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R); +} +#endif + + +/** + * Fetches a data oword (octo word) at an aligned address, generally AVX + * related. + * + * Raises \#GP(0) if not aligned. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu256Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PCRTUINT256U pu256Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_R, (sizeof(*pu256Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE); + if (rc == VINF_SUCCESS) + { + pu256Dst->au64[0] = pu256Src->au64[0]; + pu256Dst->au64[1] = pu256Src->au64[1]; + pu256Dst->au64[2] = pu256Src->au64[2]; + pu256Dst->au64[3] = pu256Src->au64[3]; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Fetches a data oword (octo word) at an aligned address, generally AVX + * related, longjmp on error. + * + * Raises \#GP(0) if not aligned. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu256Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, + RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, + (sizeof(*pu256Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE); + pu256Dst->au64[0] = pu256Src->au64[0]; + pu256Dst->au64[1] = pu256Src->au64[1]; + pu256Dst->au64[2] = pu256Src->au64[2]; + pu256Dst->au64[3] = pu256Src->au64[3]; + iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R); +} +#endif + + + +/** + * Fetches a descriptor register (lgdt, lidt). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pcbLimit Where to return the limit. + * @param pGCPtrBase Where to return the base. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param enmOpSize The effective operand size. + */ +VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg, + RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT +{ + /* + * Just like SIDT and SGDT, the LIDT and LGDT instructions are a + * little special: + * - The two reads are done separately. + * - Operand size override works in 16-bit and 32-bit code, but 64-bit. + * - We suspect the 386 to actually commit the limit before the base in + * some cases (search for 386 in bs3CpuBasic2_lidt_lgdt_One). We + * don't try emulate this eccentric behavior, because it's not well + * enough understood and rather hard to trigger. + * - The 486 seems to do a dword limit read when the operand size is 32-bit. + */ + VBOXSTRICTRC rcStrict; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2); + } + else + { + uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */ + if (enmOpSize == IEMMODE_32BIT) + { + if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486) + { + rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2); + } + else + { + rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem); + if (rcStrict == VINF_SUCCESS) + { + *pcbLimit = (uint16_t)uTmp; + rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2); + } + } + if (rcStrict == VINF_SUCCESS) + *pGCPtrBase = uTmp; + } + else + { + rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem); + if (rcStrict == VINF_SUCCESS) + { + rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2); + if (rcStrict == VINF_SUCCESS) + *pGCPtrBase = uTmp & UINT32_C(0x00ffffff); + } + } + } + return rcStrict; +} + + + +/** + * Stores a data byte. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u8Value The value to store. + */ +VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint8_t *pu8Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Dst, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0); + if (rc == VINF_SUCCESS) + { + *pu8Dst = u8Value; + rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Stores a data byte, longjmp on error. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u8Value The value to store. + */ +void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + uint8_t *pu8Dst = (uint8_t *)iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0); + *pu8Dst = u8Value; + iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W); +} +#endif + + +/** + * Stores a data word. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u16Value The value to store. + */ +VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint16_t *pu16Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1); + if (rc == VINF_SUCCESS) + { + *pu16Dst = u16Value; + rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Stores a data word, longjmp on error. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u16Value The value to store. + */ +void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + uint16_t *pu16Dst = (uint16_t *)iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1); + *pu16Dst = u16Value; + iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W); +} +#endif + + +/** + * Stores a data dword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u32Value The value to store. + */ +VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint32_t *pu32Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1); + if (rc == VINF_SUCCESS) + { + *pu32Dst = u32Value; + rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Stores a data dword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u32Value The value to store. + */ +void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + uint32_t *pu32Dst = (uint32_t *)iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1); + *pu32Dst = u32Value; + iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W); +} +#endif + + +/** + * Stores a data qword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u64Value The value to store. + */ +VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint64_t *pu64Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1); + if (rc == VINF_SUCCESS) + { + *pu64Dst = u64Value; + rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Stores a data qword, longjmp on error. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u64Value The value to store. + */ +void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + uint64_t *pu64Dst = (uint64_t *)iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1); + *pu64Dst = u64Value; + iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W); +} +#endif + + +/** + * Stores a data dqword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u128Value The value to store. + */ +VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PRTUINT128U pu128Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, 0 /* NO_AC variant */); + if (rc == VINF_SUCCESS) + { + pu128Dst->au64[0] = u128Value.au64[0]; + pu128Dst->au64[1] = u128Value.au64[1]; + rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Stores a data dqword, longjmp on error. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u128Value The value to store. + */ +void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, 0 /* NO_AC variant */); + pu128Dst->au64[0] = u128Value.au64[0]; + pu128Dst->au64[1] = u128Value.au64[1]; + iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W); +} +#endif + + +/** + * Stores a data dqword, SSE aligned. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u128Value The value to store. + */ +VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PRTUINT128U pu128Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, + (sizeof(*pu128Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE); + if (rc == VINF_SUCCESS) + { + pu128Dst->au64[0] = u128Value.au64[0]; + pu128Dst->au64[1] = u128Value.au64[1]; + rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Stores a data dqword, SSE aligned. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param u128Value The value to store. + */ +void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, + RTUINT128U u128Value) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, + (sizeof(*pu128Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE); + pu128Dst->au64[0] = u128Value.au64[0]; + pu128Dst->au64[1] = u128Value.au64[1]; + iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W); +} +#endif + + +/** + * Stores a data dqword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param pu256Value Pointer to the value to store. + */ +VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PRTUINT256U pu256Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, 0 /* NO_AC variant */); + if (rc == VINF_SUCCESS) + { + pu256Dst->au64[0] = pu256Value->au64[0]; + pu256Dst->au64[1] = pu256Value->au64[1]; + pu256Dst->au64[2] = pu256Value->au64[2]; + pu256Dst->au64[3] = pu256Value->au64[3]; + rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Stores a data dqword, longjmp on error. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param pu256Value Pointer to the value to store. + */ +void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, 0 /* NO_AC variant */); + pu256Dst->au64[0] = pu256Value->au64[0]; + pu256Dst->au64[1] = pu256Value->au64[1]; + pu256Dst->au64[2] = pu256Value->au64[2]; + pu256Dst->au64[3] = pu256Value->au64[3]; + iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W); +} +#endif + + +/** + * Stores a data dqword, AVX \#GP(0) aligned. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param pu256Value Pointer to the value to store. + */ +VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + PRTUINT256U pu256Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP); + if (rc == VINF_SUCCESS) + { + pu256Dst->au64[0] = pu256Value->au64[0]; + pu256Dst->au64[1] = pu256Value->au64[1]; + pu256Dst->au64[2] = pu256Value->au64[2]; + pu256Dst->au64[3] = pu256Value->au64[3]; + rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W); + } + return rc; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Stores a data dqword, AVX aligned. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + * @param pu256Value Pointer to the value to store. + */ +void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, + PCRTUINT256U pu256Value) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* The lazy approach for now... */ + PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, + IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP); + pu256Dst->au64[0] = pu256Value->au64[0]; + pu256Dst->au64[1] = pu256Value->au64[1]; + pu256Dst->au64[2] = pu256Value->au64[2]; + pu256Dst->au64[3] = pu256Value->au64[3]; + iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W); +} +#endif + + +/** + * Stores a descriptor register (sgdt, sidt). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbLimit The limit. + * @param GCPtrBase The base address. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* + * The SIDT and SGDT instructions actually stores the data using two + * independent writes (see bs3CpuBasic2_sidt_sgdt_One). The instructions + * does not respond to opsize prefixes. + */ + VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit); + if (rcStrict == VINF_SUCCESS) + { + if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT) + rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, + IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286 + ? (uint32_t)GCPtrBase | UINT32_C(0xff000000) : (uint32_t)GCPtrBase); + else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT) + rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase); + else + rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase); + } + return rcStrict; +} + + +/** + * Pushes a word onto the stack. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16Value The value to push. + */ +VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + uint64_t uNewRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp); + + /* Write the word the lazy way. */ + uint16_t *pu16Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1); + if (rc == VINF_SUCCESS) + { + *pu16Dst = u16Value; + rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W); + } + + /* Commit the new RSP value unless we an access handler made trouble. */ + if (rc == VINF_SUCCESS) + pVCpu->cpum.GstCtx.rsp = uNewRsp; + + return rc; +} + + +/** + * Pushes a dword onto the stack. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u32Value The value to push. + */ +VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + uint64_t uNewRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp); + + /* Write the dword the lazy way. */ + uint32_t *pu32Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1); + if (rc == VINF_SUCCESS) + { + *pu32Dst = u32Value; + rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W); + } + + /* Commit the new RSP value unless we an access handler made trouble. */ + if (rc == VINF_SUCCESS) + pVCpu->cpum.GstCtx.rsp = uNewRsp; + + return rc; +} + + +/** + * Pushes a dword segment register value onto the stack. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u32Value The value to push. + */ +VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + uint64_t uNewRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp); + + /* The intel docs talks about zero extending the selector register + value. My actual intel CPU here might be zero extending the value + but it still only writes the lower word... */ + /** @todo Test this on new HW and on AMD and in 64-bit mode. Also test what + * happens when crossing an electric page boundrary, is the high word checked + * for write accessibility or not? Probably it is. What about segment limits? + * It appears this behavior is also shared with trap error codes. + * + * Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check + * ancient hardware when it actually did change. */ + uint16_t *pu16Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_RW, sizeof(*pu16Dst) - 1); /** @todo 2 or 4 alignment check for PUSH SS? */ + if (rc == VINF_SUCCESS) + { + *pu16Dst = (uint16_t)u32Value; + rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW); + } + + /* Commit the new RSP value unless we an access handler made trouble. */ + if (rc == VINF_SUCCESS) + pVCpu->cpum.GstCtx.rsp = uNewRsp; + + return rc; +} + + +/** + * Pushes a qword onto the stack. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u64Value The value to push. + */ +VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + uint64_t uNewRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp); + + /* Write the word the lazy way. */ + uint64_t *pu64Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1); + if (rc == VINF_SUCCESS) + { + *pu64Dst = u64Value; + rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W); + } + + /* Commit the new RSP value unless we an access handler made trouble. */ + if (rc == VINF_SUCCESS) + pVCpu->cpum.GstCtx.rsp = uNewRsp; + + return rc; +} + + +/** + * Pops a word from the stack. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu16Value Where to store the popped value. + */ +VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + uint64_t uNewRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp); + + /* Write the word the lazy way. */ + uint16_t const *pu16Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu16Value = *pu16Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R); + + /* Commit the new RSP value. */ + if (rc == VINF_SUCCESS) + pVCpu->cpum.GstCtx.rsp = uNewRsp; + } + + return rc; +} + + +/** + * Pops a dword from the stack. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32Value Where to store the popped value. + */ +VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + uint64_t uNewRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp); + + /* Write the word the lazy way. */ + uint32_t const *pu32Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu32Value = *pu32Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R); + + /* Commit the new RSP value. */ + if (rc == VINF_SUCCESS) + pVCpu->cpum.GstCtx.rsp = uNewRsp; + } + + return rc; +} + + +/** + * Pops a qword from the stack. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64Value Where to store the popped value. + */ +VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + uint64_t uNewRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp); + + /* Write the word the lazy way. */ + uint64_t const *pu64Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu64Value = *pu64Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R); + + /* Commit the new RSP value. */ + if (rc == VINF_SUCCESS) + pVCpu->cpum.GstCtx.rsp = uNewRsp; + } + + return rc; +} + + +/** + * Pushes a word onto the stack, using a temporary stack pointer. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16Value The value to push. + * @param pTmpRsp Pointer to the temporary stack pointer. + */ +VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + RTUINT64U NewRsp = *pTmpRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2); + + /* Write the word the lazy way. */ + uint16_t *pu16Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1); + if (rc == VINF_SUCCESS) + { + *pu16Dst = u16Value; + rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W); + } + + /* Commit the new RSP value unless we an access handler made trouble. */ + if (rc == VINF_SUCCESS) + *pTmpRsp = NewRsp; + + return rc; +} + + +/** + * Pushes a dword onto the stack, using a temporary stack pointer. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u32Value The value to push. + * @param pTmpRsp Pointer to the temporary stack pointer. + */ +VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + RTUINT64U NewRsp = *pTmpRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4); + + /* Write the word the lazy way. */ + uint32_t *pu32Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1); + if (rc == VINF_SUCCESS) + { + *pu32Dst = u32Value; + rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W); + } + + /* Commit the new RSP value unless we an access handler made trouble. */ + if (rc == VINF_SUCCESS) + *pTmpRsp = NewRsp; + + return rc; +} + + +/** + * Pushes a dword onto the stack, using a temporary stack pointer. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u64Value The value to push. + * @param pTmpRsp Pointer to the temporary stack pointer. + */ +VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + RTUINT64U NewRsp = *pTmpRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8); + + /* Write the word the lazy way. */ + uint64_t *pu64Dst; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1); + if (rc == VINF_SUCCESS) + { + *pu64Dst = u64Value; + rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W); + } + + /* Commit the new RSP value unless we an access handler made trouble. */ + if (rc == VINF_SUCCESS) + *pTmpRsp = NewRsp; + + return rc; +} + + +/** + * Pops a word from the stack, using a temporary stack pointer. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu16Value Where to store the popped value. + * @param pTmpRsp Pointer to the temporary stack pointer. + */ +VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + RTUINT64U NewRsp = *pTmpRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2); + + /* Write the word the lazy way. */ + uint16_t const *pu16Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu16Value = *pu16Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R); + + /* Commit the new RSP value. */ + if (rc == VINF_SUCCESS) + *pTmpRsp = NewRsp; + } + + return rc; +} + + +/** + * Pops a dword from the stack, using a temporary stack pointer. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32Value Where to store the popped value. + * @param pTmpRsp Pointer to the temporary stack pointer. + */ +VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + RTUINT64U NewRsp = *pTmpRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4); + + /* Write the word the lazy way. */ + uint32_t const *pu32Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1); + if (rc == VINF_SUCCESS) + { + *pu32Value = *pu32Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R); + + /* Commit the new RSP value. */ + if (rc == VINF_SUCCESS) + *pTmpRsp = NewRsp; + } + + return rc; +} + + +/** + * Pops a qword from the stack, using a temporary stack pointer. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64Value Where to store the popped value. + * @param pTmpRsp Pointer to the temporary stack pointer. + */ +VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT +{ + /* Increment the stack pointer. */ + RTUINT64U NewRsp = *pTmpRsp; + RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8); + + /* Write the word the lazy way. */ + uint64_t const *pu64Src; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1); + if (rcStrict == VINF_SUCCESS) + { + *pu64Value = *pu64Src; + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R); + + /* Commit the new RSP value. */ + if (rcStrict == VINF_SUCCESS) + *pTmpRsp = NewRsp; + } + + return rcStrict; +} + + +/** + * Begin a special stack push (used by interrupt, exceptions and such). + * + * This will raise \#SS or \#PF if appropriate. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbMem The number of bytes to push onto the stack. + * @param cbAlign The alignment mask (7, 3, 1). + * @param ppvMem Where to return the pointer to the stack memory. + * As with the other memory functions this could be + * direct access or bounce buffered access, so + * don't commit register until the commit call + * succeeds. + * @param puNewRsp Where to return the new RSP value. This must be + * passed unchanged to + * iemMemStackPushCommitSpecial(). + */ +VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign, + void **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT +{ + Assert(cbMem < UINT8_MAX); + RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp); + return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop, + IEM_ACCESS_STACK_W, cbAlign); +} + + +/** + * Commits a special stack push (started by iemMemStackPushBeginSpecial). + * + * This will update the rSP. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pvMem The pointer returned by + * iemMemStackPushBeginSpecial(). + * @param uNewRsp The new RSP value returned by + * iemMemStackPushBeginSpecial(). + */ +VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W); + if (rcStrict == VINF_SUCCESS) + pVCpu->cpum.GstCtx.rsp = uNewRsp; + return rcStrict; +} + + +/** + * Begin a special stack pop (used by iret, retf and such). + * + * This will raise \#SS or \#PF if appropriate. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbMem The number of bytes to pop from the stack. + * @param cbAlign The alignment mask (7, 3, 1). + * @param ppvMem Where to return the pointer to the stack memory. + * @param puNewRsp Where to return the new RSP value. This must be + * assigned to CPUMCTX::rsp manually some time + * after iemMemStackPopDoneSpecial() has been + * called. + */ +VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign, + void const **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT +{ + Assert(cbMem < UINT8_MAX); + RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp); + return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R, cbAlign); +} + + +/** + * Continue a special stack pop (used by iret and retf), for the purpose of + * retrieving a new stack pointer. + * + * This will raise \#SS or \#PF if appropriate. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param off Offset from the top of the stack. This is zero + * except in the retf case. + * @param cbMem The number of bytes to pop from the stack. + * @param ppvMem Where to return the pointer to the stack memory. + * @param uCurNewRsp The current uncommitted RSP value. (No need to + * return this because all use of this function is + * to retrieve a new value and anything we return + * here would be discarded.) + */ +VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t off, size_t cbMem, + void const **ppvMem, uint64_t uCurNewRsp) RT_NOEXCEPT +{ + Assert(cbMem < UINT8_MAX); + + /* The essense of iemRegGetRspForPopEx and friends: */ /** @todo put this into a inlined function? */ + RTGCPTR GCPtrTop; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + GCPtrTop = uCurNewRsp; + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + GCPtrTop = (uint32_t)uCurNewRsp; + else + GCPtrTop = (uint16_t)uCurNewRsp; + + return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop + off, IEM_ACCESS_STACK_R, + 0 /* checked in iemMemStackPopBeginSpecial */); +} + + +/** + * Done with a special stack pop (started by iemMemStackPopBeginSpecial or + * iemMemStackPopContinueSpecial). + * + * The caller will manually commit the rSP. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pvMem The pointer returned by + * iemMemStackPopBeginSpecial() or + * iemMemStackPopContinueSpecial(). + */ +VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT +{ + return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R); +} + + +/** + * Fetches a system table byte. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pbDst Where to return the byte. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint8_t const *pbSrc; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pbSrc, sizeof(*pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0); + if (rc == VINF_SUCCESS) + { + *pbDst = *pbSrc; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R); + } + return rc; +} + + +/** + * Fetches a system table word. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu16Dst Where to return the word. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint16_t const *pu16Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0); + if (rc == VINF_SUCCESS) + { + *pu16Dst = *pu16Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R); + } + return rc; +} + + +/** + * Fetches a system table dword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32Dst Where to return the dword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint32_t const *pu32Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0); + if (rc == VINF_SUCCESS) + { + *pu32Dst = *pu32Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R); + } + return rc; +} + + +/** + * Fetches a system table qword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64Dst Where to return the qword. + * @param iSegReg The index of the segment register to use for + * this access. The base and limits are checked. + * @param GCPtrMem The address of the guest memory. + */ +VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT +{ + /* The lazy approach for now... */ + uint64_t const *pu64Src; + VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0); + if (rc == VINF_SUCCESS) + { + *pu64Dst = *pu64Src; + rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R); + } + return rc; +} + + +/** + * Fetches a descriptor table entry with caller specified error code. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pDesc Where to return the descriptor table entry. + * @param uSel The selector which table entry to fetch. + * @param uXcpt The exception to raise on table lookup error. + * @param uErrorCode The error code associated with the exception. + */ +static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, + uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT +{ + AssertPtr(pDesc); + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR); + + /** @todo did the 286 require all 8 bytes to be accessible? */ + /* + * Get the selector table base and check bounds. + */ + RTGCPTR GCPtrBase; + if (uSel & X86_SEL_LDT) + { + if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present + || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit ) + { + Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n", + uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel)); + return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, + uErrorCode, 0); + } + + Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); + GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base; + } + else + { + if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt) + { + Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt)); + return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, + uErrorCode, 0); + } + GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt; + } + + /* + * Read the legacy descriptor and maybe the long mode extensions if + * required. + */ + VBOXSTRICTRC rcStrict; + if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286) + rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK)); + else + { + rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4); + if (rcStrict == VINF_SUCCESS) + pDesc->Legacy.au16[3] = 0; + else + return rcStrict; + } + + if (rcStrict == VINF_SUCCESS) + { + if ( !IEM_IS_LONG_MODE(pVCpu) + || pDesc->Legacy.Gen.u1DescType) + pDesc->Long.au64[1] = 0; + else if ( (uint32_t)(uSel | X86_SEL_RPL_LDT) + 8 + <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt)) + rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel | X86_SEL_RPL_LDT) + 1); + else + { + Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel)); + /** @todo is this the right exception? */ + return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErrorCode, 0); + } + } + return rcStrict; +} + + +/** + * Fetches a descriptor table entry. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pDesc Where to return the descriptor table entry. + * @param uSel The selector which table entry to fetch. + * @param uXcpt The exception to raise on table lookup error. + */ +VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT +{ + return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL); +} + + +/** + * Marks the selector descriptor as accessed (only non-system descriptors). + * + * This function ASSUMES that iemMemFetchSelDesc has be called previously and + * will therefore skip the limit checks. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param uSel The selector. + */ +VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT +{ + /* + * Get the selector table base and calculate the entry address. + */ + RTGCPTR GCPtr = uSel & X86_SEL_LDT + ? pVCpu->cpum.GstCtx.ldtr.u64Base + : pVCpu->cpum.GstCtx.gdtr.pGdt; + GCPtr += uSel & X86_SEL_MASK; + + /* + * ASMAtomicBitSet will assert if the address is misaligned, so do some + * ugly stuff to avoid this. This will make sure it's an atomic access + * as well more or less remove any question about 8-bit or 32-bit accesss. + */ + VBOXSTRICTRC rcStrict; + uint32_t volatile *pu32; + if ((GCPtr & 3) == 0) + { + /* The normal case, map the 32-bit bits around the accessed bit (40). */ + GCPtr += 2 + 2; + rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */ + } + else + { + /* The misaligned GDT/LDT case, map the whole thing. */ + rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + switch ((uintptr_t)pu32 & 3) + { + case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break; + case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break; + case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break; + case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break; + } + } + + return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW); +} + +/** @} */ + +/** @name Opcode Helpers. + * @{ + */ + +/** + * Calculates the effective address of a ModR/M memory operand. + * + * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR. + * + * @return Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param bRm The ModRM byte. + * @param cbImm The size of any immediate following the + * effective address opcode bytes. Important for + * RIP relative addressing. + * @param pGCPtrEff Where to return the effective address. + */ +VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff) RT_NOEXCEPT +{ + Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm)); +# define SET_SS_DEF() \ + do \ + { \ + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \ + pVCpu->iem.s.iEffSeg = X86_SREG_SS; \ + } while (0) + + if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) + { +/** @todo Check the effective address size crap! */ + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT) + { + uint16_t u16EffAddr; + + /* Handle the disp16 form with no registers first. */ + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6) + IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); + else + { + /* Get the displacment. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: u16EffAddr = 0; break; + case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break; + case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break; + default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */ + } + + /* Add the base and index registers to the disp. */ + switch (bRm & X86_MODRM_RM_MASK) + { + case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break; + case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break; + case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break; + case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break; + case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break; + case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break; + case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break; + case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break; + } + } + + *pGCPtrEff = u16EffAddr; + } + else + { + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT); + uint32_t u32EffAddr; + + /* Handle the disp32 form with no registers first. */ + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5) + IEM_OPCODE_GET_NEXT_U32(&u32EffAddr); + else + { + /* Get the register (or SIB) value. */ + switch ((bRm & X86_MODRM_RM_MASK)) + { + case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break; + case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break; + case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break; + case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break; + case 4: /* SIB */ + { + uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib); + + /* Get the index and scale it. */ + switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) + { + case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break; + case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break; + case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break; + case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break; + case 4: u32EffAddr = 0; /*none */ break; + case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break; + case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break; + case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK; + + /* add base */ + switch (bSib & X86_SIB_BASE_MASK) + { + case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break; + case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break; + case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break; + case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break; + case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break; + case 5: + if ((bRm & X86_MODRM_MOD_MASK) != 0) + { + u32EffAddr += pVCpu->cpum.GstCtx.ebp; + SET_SS_DEF(); + } + else + { + uint32_t u32Disp; + IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u32EffAddr += u32Disp; + } + break; + case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break; + case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + } + case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break; + case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break; + case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + /* Get and add the displacement. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: + break; + case 1: + { + int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp); + u32EffAddr += i8Disp; + break; + } + case 2: + { + uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u32EffAddr += u32Disp; + break; + } + default: + AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */ + } + + } + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT) + *pGCPtrEff = u32EffAddr; + else + { + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT); + *pGCPtrEff = u32EffAddr & UINT16_MAX; + } + } + } + else + { + uint64_t u64EffAddr; + + /* Handle the rip+disp32 form with no registers first. */ + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5) + { + IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr); + u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm; + } + else + { + /* Get the register (or SIB) value. */ + switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB) + { + case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break; + case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break; + case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break; + case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break; + case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break; + case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break; + case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break; + case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break; + case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break; + case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break; + case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break; + case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break; + case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break; + case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break; + /* SIB */ + case 4: + case 12: + { + uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib); + + /* Get the index and scale it. */ + switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex) + { + case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break; + case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break; + case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break; + case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break; + case 4: u64EffAddr = 0; /*none */ break; + case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break; + case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break; + case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break; + case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break; + case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break; + case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break; + case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break; + case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break; + case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break; + case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break; + case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK; + + /* add base */ + switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB) + { + case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break; + case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break; + case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break; + case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break; + case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break; + case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break; + case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break; + case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break; + case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break; + case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break; + case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break; + case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break; + case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break; + case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break; + /* complicated encodings */ + case 5: + case 13: + if ((bRm & X86_MODRM_MOD_MASK) != 0) + { + if (!pVCpu->iem.s.uRexB) + { + u64EffAddr += pVCpu->cpum.GstCtx.rbp; + SET_SS_DEF(); + } + else + u64EffAddr += pVCpu->cpum.GstCtx.r13; + } + else + { + uint32_t u32Disp; + IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u64EffAddr += (int32_t)u32Disp; + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + /* Get and add the displacement. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: + break; + case 1: + { + int8_t i8Disp; + IEM_OPCODE_GET_NEXT_S8(&i8Disp); + u64EffAddr += i8Disp; + break; + } + case 2: + { + uint32_t u32Disp; + IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u64EffAddr += (int32_t)u32Disp; + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */ + } + + } + + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT) + *pGCPtrEff = u64EffAddr; + else + { + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT); + *pGCPtrEff = u64EffAddr & UINT32_MAX; + } + } + + Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff)); + return VINF_SUCCESS; +} + + +/** + * Calculates the effective address of a ModR/M memory operand. + * + * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR. + * + * @return Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param bRm The ModRM byte. + * @param cbImm The size of any immediate following the + * effective address opcode bytes. Important for + * RIP relative addressing. + * @param pGCPtrEff Where to return the effective address. + * @param offRsp RSP displacement. + */ +VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp) RT_NOEXCEPT +{ + Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm)); +# define SET_SS_DEF() \ + do \ + { \ + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \ + pVCpu->iem.s.iEffSeg = X86_SREG_SS; \ + } while (0) + + if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) + { +/** @todo Check the effective address size crap! */ + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT) + { + uint16_t u16EffAddr; + + /* Handle the disp16 form with no registers first. */ + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6) + IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); + else + { + /* Get the displacment. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: u16EffAddr = 0; break; + case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break; + case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break; + default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */ + } + + /* Add the base and index registers to the disp. */ + switch (bRm & X86_MODRM_RM_MASK) + { + case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break; + case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break; + case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break; + case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break; + case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break; + case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break; + case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break; + case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break; + } + } + + *pGCPtrEff = u16EffAddr; + } + else + { + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT); + uint32_t u32EffAddr; + + /* Handle the disp32 form with no registers first. */ + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5) + IEM_OPCODE_GET_NEXT_U32(&u32EffAddr); + else + { + /* Get the register (or SIB) value. */ + switch ((bRm & X86_MODRM_RM_MASK)) + { + case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break; + case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break; + case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break; + case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break; + case 4: /* SIB */ + { + uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib); + + /* Get the index and scale it. */ + switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) + { + case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break; + case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break; + case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break; + case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break; + case 4: u32EffAddr = 0; /*none */ break; + case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break; + case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break; + case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK; + + /* add base */ + switch (bSib & X86_SIB_BASE_MASK) + { + case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break; + case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break; + case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break; + case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break; + case 4: + u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp; + SET_SS_DEF(); + break; + case 5: + if ((bRm & X86_MODRM_MOD_MASK) != 0) + { + u32EffAddr += pVCpu->cpum.GstCtx.ebp; + SET_SS_DEF(); + } + else + { + uint32_t u32Disp; + IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u32EffAddr += u32Disp; + } + break; + case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break; + case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + } + case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break; + case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break; + case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + /* Get and add the displacement. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: + break; + case 1: + { + int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp); + u32EffAddr += i8Disp; + break; + } + case 2: + { + uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u32EffAddr += u32Disp; + break; + } + default: + AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */ + } + + } + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT) + *pGCPtrEff = u32EffAddr; + else + { + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT); + *pGCPtrEff = u32EffAddr & UINT16_MAX; + } + } + } + else + { + uint64_t u64EffAddr; + + /* Handle the rip+disp32 form with no registers first. */ + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5) + { + IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr); + u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm; + } + else + { + /* Get the register (or SIB) value. */ + switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB) + { + case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break; + case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break; + case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break; + case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break; + case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break; + case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break; + case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break; + case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break; + case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break; + case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break; + case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break; + case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break; + case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break; + case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break; + /* SIB */ + case 4: + case 12: + { + uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib); + + /* Get the index and scale it. */ + switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex) + { + case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break; + case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break; + case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break; + case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break; + case 4: u64EffAddr = 0; /*none */ break; + case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break; + case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break; + case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break; + case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break; + case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break; + case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break; + case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break; + case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break; + case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break; + case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break; + case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK; + + /* add base */ + switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB) + { + case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break; + case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break; + case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break; + case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break; + case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break; + case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break; + case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break; + case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break; + case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break; + case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break; + case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break; + case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break; + case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break; + case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break; + /* complicated encodings */ + case 5: + case 13: + if ((bRm & X86_MODRM_MOD_MASK) != 0) + { + if (!pVCpu->iem.s.uRexB) + { + u64EffAddr += pVCpu->cpum.GstCtx.rbp; + SET_SS_DEF(); + } + else + u64EffAddr += pVCpu->cpum.GstCtx.r13; + } + else + { + uint32_t u32Disp; + IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u64EffAddr += (int32_t)u32Disp; + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + /* Get and add the displacement. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: + break; + case 1: + { + int8_t i8Disp; + IEM_OPCODE_GET_NEXT_S8(&i8Disp); + u64EffAddr += i8Disp; + break; + } + case 2: + { + uint32_t u32Disp; + IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u64EffAddr += (int32_t)u32Disp; + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */ + } + + } + + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT) + *pGCPtrEff = u64EffAddr; + else + { + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT); + *pGCPtrEff = u64EffAddr & UINT32_MAX; + } + } + + Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff)); + return VINF_SUCCESS; +} + + +#ifdef IEM_WITH_SETJMP +/** + * Calculates the effective address of a ModR/M memory operand. + * + * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR. + * + * May longjmp on internal error. + * + * @return The effective address. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param bRm The ModRM byte. + * @param cbImm The size of any immediate following the + * effective address opcode bytes. Important for + * RIP relative addressing. + */ +RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm) IEM_NOEXCEPT_MAY_LONGJMP +{ + Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm)); +# define SET_SS_DEF() \ + do \ + { \ + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \ + pVCpu->iem.s.iEffSeg = X86_SREG_SS; \ + } while (0) + + if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) + { +/** @todo Check the effective address size crap! */ + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT) + { + uint16_t u16EffAddr; + + /* Handle the disp16 form with no registers first. */ + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6) + IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); + else + { + /* Get the displacment. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: u16EffAddr = 0; break; + case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break; + case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break; + default: AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_1)); /* (caller checked for these) */ + } + + /* Add the base and index registers to the disp. */ + switch (bRm & X86_MODRM_RM_MASK) + { + case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break; + case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break; + case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break; + case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break; + case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break; + case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break; + case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break; + case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break; + } + } + + Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr)); + return u16EffAddr; + } + + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT); + uint32_t u32EffAddr; + + /* Handle the disp32 form with no registers first. */ + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5) + IEM_OPCODE_GET_NEXT_U32(&u32EffAddr); + else + { + /* Get the register (or SIB) value. */ + switch ((bRm & X86_MODRM_RM_MASK)) + { + case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break; + case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break; + case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break; + case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break; + case 4: /* SIB */ + { + uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib); + + /* Get the index and scale it. */ + switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) + { + case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break; + case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break; + case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break; + case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break; + case 4: u32EffAddr = 0; /*none */ break; + case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break; + case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break; + case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); + } + u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK; + + /* add base */ + switch (bSib & X86_SIB_BASE_MASK) + { + case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break; + case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break; + case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break; + case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break; + case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break; + case 5: + if ((bRm & X86_MODRM_MOD_MASK) != 0) + { + u32EffAddr += pVCpu->cpum.GstCtx.ebp; + SET_SS_DEF(); + } + else + { + uint32_t u32Disp; + IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u32EffAddr += u32Disp; + } + break; + case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break; + case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); + } + break; + } + case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break; + case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break; + case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); + } + + /* Get and add the displacement. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: + break; + case 1: + { + int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp); + u32EffAddr += i8Disp; + break; + } + case 2: + { + uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u32EffAddr += u32Disp; + break; + } + default: + AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_2)); /* (caller checked for these) */ + } + } + + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT) + { + Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr)); + return u32EffAddr; + } + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT); + Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX)); + return u32EffAddr & UINT16_MAX; + } + + uint64_t u64EffAddr; + + /* Handle the rip+disp32 form with no registers first. */ + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5) + { + IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr); + u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm; + } + else + { + /* Get the register (or SIB) value. */ + switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB) + { + case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break; + case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break; + case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break; + case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break; + case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break; + case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break; + case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break; + case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break; + case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break; + case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break; + case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break; + case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break; + case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break; + case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break; + /* SIB */ + case 4: + case 12: + { + uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib); + + /* Get the index and scale it. */ + switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex) + { + case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break; + case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break; + case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break; + case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break; + case 4: u64EffAddr = 0; /*none */ break; + case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break; + case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break; + case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break; + case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break; + case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break; + case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break; + case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break; + case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break; + case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break; + case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break; + case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); + } + u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK; + + /* add base */ + switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB) + { + case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break; + case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break; + case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break; + case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break; + case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break; + case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break; + case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break; + case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break; + case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break; + case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break; + case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break; + case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break; + case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break; + case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break; + /* complicated encodings */ + case 5: + case 13: + if ((bRm & X86_MODRM_MOD_MASK) != 0) + { + if (!pVCpu->iem.s.uRexB) + { + u64EffAddr += pVCpu->cpum.GstCtx.rbp; + SET_SS_DEF(); + } + else + u64EffAddr += pVCpu->cpum.GstCtx.r13; + } + else + { + uint32_t u32Disp; + IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u64EffAddr += (int32_t)u32Disp; + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); + } + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); + } + + /* Get and add the displacement. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: + break; + case 1: + { + int8_t i8Disp; + IEM_OPCODE_GET_NEXT_S8(&i8Disp); + u64EffAddr += i8Disp; + break; + } + case 2: + { + uint32_t u32Disp; + IEM_OPCODE_GET_NEXT_U32(&u32Disp); + u64EffAddr += (int32_t)u32Disp; + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */ + } + + } + + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT) + { + Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr)); + return u64EffAddr; + } + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT); + Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX)); + return u64EffAddr & UINT32_MAX; +} +#endif /* IEM_WITH_SETJMP */ + +/** @} */ + + +#ifdef LOG_ENABLED +/** + * Logs the current instruction. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param fSameCtx Set if we have the same context information as the VMM, + * clear if we may have already executed an instruction in + * our debug context. When clear, we assume IEMCPU holds + * valid CPU mode info. + * + * The @a fSameCtx parameter is now misleading and obsolete. + * @param pszFunction The IEM function doing the execution. + */ +static void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction) RT_NOEXCEPT +{ +# ifdef IN_RING3 + if (LogIs2Enabled()) + { + char szInstr[256]; + uint32_t cbInstr = 0; + if (fSameCtx) + DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0, + DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE, + szInstr, sizeof(szInstr), &cbInstr); + else + { + uint32_t fFlags = 0; + switch (pVCpu->iem.s.enmCpuMode) + { + case IEMMODE_64BIT: fFlags |= DBGF_DISAS_FLAGS_64BIT_MODE; break; + case IEMMODE_32BIT: fFlags |= DBGF_DISAS_FLAGS_32BIT_MODE; break; + case IEMMODE_16BIT: + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) || pVCpu->cpum.GstCtx.eflags.Bits.u1VM) + fFlags |= DBGF_DISAS_FLAGS_16BIT_REAL_MODE; + else + fFlags |= DBGF_DISAS_FLAGS_16BIT_MODE; + break; + } + DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags, + szInstr, sizeof(szInstr), &cbInstr); + } + + PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + Log2(("**** %s\n" + " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n" + " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n" + " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n" + " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n" + " %s\n" + , pszFunction, + pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi, + pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel, + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel, + pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u, + pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK, + szInstr)); + + if (LogIs3Enabled()) + DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL); + } + else +# endif + LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel, + pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u)); + RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx); +} +#endif /* LOG_ENABLED */ + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** + * Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW, + * VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW. + * + * @returns Modified rcStrict. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param rcStrict The instruction execution status. + */ +static VBOXSTRICTRC iemHandleNestedInstructionBoundaryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT +{ + Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu))); + if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF)) + { + /* VMX preemption timer takes priority over NMI-window exits. */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER)) + { + rcStrict = iemVmxVmexitPreemptTimer(pVCpu); + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER)); + } + /* + * Check remaining intercepts. + * + * NMI-window and Interrupt-window VM-exits. + * Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs. + * Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits. + * + * See Intel spec. 26.7.6 "NMI-Window Exiting". + * See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery". + */ + else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW | VMCPU_FF_VMX_INT_WINDOW) + && !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx) + && !TRPMHasTrap(pVCpu)) + { + Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx)); + if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW) + && CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx)) + { + rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */); + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)); + } + else if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW) + && CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx)) + { + rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */); + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)); + } + } + } + /* TPR-below threshold/APIC write has the highest priority. */ + else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE)) + { + rcStrict = iemVmxApicWriteEmulation(pVCpu); + Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)); + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE)); + } + /* MTF takes priority over VMX-preemption timer. */ + else + { + rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */); + Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)); + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF)); + } + return rcStrict; +} +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + + +/** @def IEM_TRY_SETJMP + * Wrapper around setjmp / try, hiding all the ugly differences. + * + * @note Use with extreme care as this is a fragile macro. + * @param a_pVCpu The cross context virtual CPU structure of the calling EMT. + * @param a_rcTarget The variable that should receive the status code in case + * of a longjmp/throw. + */ +/** @def IEM_TRY_SETJMP_AGAIN + * For when setjmp / try is used again in the same variable scope as a previous + * IEM_TRY_SETJMP invocation. + */ +/** @def IEM_CATCH_LONGJMP_BEGIN + * Start wrapper for catch / setjmp-else. + * + * This will set up a scope. + * + * @note Use with extreme care as this is a fragile macro. + * @param a_pVCpu The cross context virtual CPU structure of the calling EMT. + * @param a_rcTarget The variable that should receive the status code in case + * of a longjmp/throw. + */ +/** @def IEM_CATCH_LONGJMP_END + * End wrapper for catch / setjmp-else. + * + * This will close the scope set up by IEM_CATCH_LONGJMP_BEGIN and clean up the + * state. + * + * @note Use with extreme care as this is a fragile macro. + * @param a_pVCpu The cross context virtual CPU structure of the calling EMT. + */ +#if defined(IEM_WITH_SETJMP) || defined(DOXYGEN_RUNNING) +# ifdef IEM_WITH_THROW_CATCH +# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \ + a_rcTarget = VINF_SUCCESS; \ + try +# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \ + IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) +# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \ + catch (int rcThrown) \ + { \ + a_rcTarget = rcThrown +# define IEM_CATCH_LONGJMP_END(a_pVCpu) \ + } \ + ((void)0) +# else /* !IEM_WITH_THROW_CATCH */ +# define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \ + jmp_buf JmpBuf; \ + jmp_buf * volatile pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf); \ + pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \ + if ((rcStrict = setjmp(JmpBuf)) == 0) +# define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \ + pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf); \ + pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \ + if ((rcStrict = setjmp(JmpBuf)) == 0) +# define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \ + else \ + { \ + ((void)0) +# define IEM_CATCH_LONGJMP_END(a_pVCpu) \ + } \ + (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf +# endif /* !IEM_WITH_THROW_CATCH */ +#endif /* IEM_WITH_SETJMP */ + + +/** + * The actual code execution bits of IEMExecOne, IEMExecOneEx, and + * IEMExecOneWithPrefetchedByPC. + * + * Similar code is found in IEMExecLots. + * + * @return Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param fExecuteInhibit If set, execute the instruction following CLI, + * POP SS and MOV SS,GR. + * @param pszFunction The calling function name. + */ +DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction) +{ + AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst)); + AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst)); + AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst)); + RT_NOREF_PV(pszFunction); + +#ifdef IEM_WITH_SETJMP + VBOXSTRICTRC rcStrict; + IEM_TRY_SETJMP(pVCpu, rcStrict) + { + uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b); + rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict); + { + pVCpu->iem.s.cLongJumps++; + } + IEM_CATCH_LONGJMP_END(pVCpu); +#else + uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b); + VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]); +#endif + if (rcStrict == VINF_SUCCESS) + pVCpu->iem.s.cInstructions++; + if (pVCpu->iem.s.cActiveMappings > 0) + { + Assert(rcStrict != VINF_SUCCESS); + iemMemRollback(pVCpu); + } + AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst)); + AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst)); + AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst)); + +//#ifdef DEBUG +// AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr || rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr)); +//#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* + * Perform any VMX nested-guest instruction boundary actions. + * + * If any of these causes a VM-exit, we must skip executing the next + * instruction (would run into stale page tables). A VM-exit makes sure + * there is no interrupt-inhibition, so that should ensure we don't go + * to try execute the next instruction. Clearing fExecuteInhibit is + * problematic because of the setjmp/longjmp clobbering above. + */ + if ( !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER + | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW) + || rcStrict != VINF_SUCCESS) + { /* likely */ } + else + rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict); +#endif + + /* Execute the next instruction as well if a cli, pop ss or + mov ss, Gr has just completed successfully. */ + if ( fExecuteInhibit + && rcStrict == VINF_SUCCESS + && CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)) + { + rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers, pVCpu->iem.s.fDisregardLock); + if (rcStrict == VINF_SUCCESS) + { +#ifdef LOG_ENABLED + iemLogCurInstr(pVCpu, false, pszFunction); +#endif +#ifdef IEM_WITH_SETJMP + IEM_TRY_SETJMP_AGAIN(pVCpu, rcStrict) + { + uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b); + rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict); + { + pVCpu->iem.s.cLongJumps++; + } + IEM_CATCH_LONGJMP_END(pVCpu); +#else + IEM_OPCODE_GET_FIRST_U8(&b); + rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]); +#endif + if (rcStrict == VINF_SUCCESS) + { + pVCpu->iem.s.cInstructions++; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER + | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)) + { /* likely */ } + else + rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict); +#endif + } + if (pVCpu->iem.s.cActiveMappings > 0) + { + Assert(rcStrict != VINF_SUCCESS); + iemMemRollback(pVCpu); + } + AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst)); + AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst)); + AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst)); + } + else if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + /** @todo drop this after we bake this change into RIP advancing. */ + CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); /* hope this is correct for all exceptional cases... */ + } + + /* + * Return value fiddling, statistics and sanity assertions. + */ + rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict); + + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + return rcStrict; +} + + +/** + * Execute one instruction. + * + * @return Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu) +{ + AssertCompile(sizeof(pVCpu->iem.s) <= sizeof(pVCpu->iem.padding)); /* (tstVMStruct can't do it's job w/o instruction stats) */ +#ifdef LOG_ENABLED + iemLogCurInstr(pVCpu, true, "IEMExecOne"); +#endif + + /* + * Do the decoding and emulation. + */ + VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne"); + else if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + + if (rcStrict != VINF_SUCCESS) + LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +} + + +VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, uint32_t *pcbWritten) +{ + uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten; + VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false); + if (rcStrict == VINF_SUCCESS) + { + rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx"); + if (pcbWritten) + *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten; + } + else if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + + return rcStrict; +} + + +VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, uint64_t OpcodeBytesPC, + const void *pvOpcodeBytes, size_t cbOpcodeBytes) +{ + VBOXSTRICTRC rcStrict; + if ( cbOpcodeBytes + && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC) + { + iemInitDecoder(pVCpu, false, false); +#ifdef IEM_WITH_CODE_TLB + pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC; + pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes; + pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes); + pVCpu->iem.s.offCurInstrStart = 0; + pVCpu->iem.s.offInstrNextByte = 0; +#else + pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode)); + memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode); +#endif + rcStrict = VINF_SUCCESS; + } + else + rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC"); + else if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + + return rcStrict; +} + + +VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, uint32_t *pcbWritten) +{ + uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten; + VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false); + if (rcStrict == VINF_SUCCESS) + { + rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx"); + if (pcbWritten) + *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten; + } + else if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + + return rcStrict; +} + + +VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, uint64_t OpcodeBytesPC, + const void *pvOpcodeBytes, size_t cbOpcodeBytes) +{ + VBOXSTRICTRC rcStrict; + if ( cbOpcodeBytes + && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC) + { + iemInitDecoder(pVCpu, true, false); +#ifdef IEM_WITH_CODE_TLB + pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC; + pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes; + pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes); + pVCpu->iem.s.offCurInstrStart = 0; + pVCpu->iem.s.offInstrNextByte = 0; +#else + pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode)); + memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode); +#endif + rcStrict = VINF_SUCCESS; + } + else + rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC"); + else if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + + return rcStrict; +} + + +/** + * For handling split cacheline lock operations when the host has split-lock + * detection enabled. + * + * This will cause the interpreter to disregard the lock prefix and implicit + * locking (xchg). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMMDECL(VBOXSTRICTRC) IEMExecOneIgnoreLock(PVMCPUCC pVCpu) +{ + /* + * Do the decoding and emulation. + */ + VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, true /*fDisregardLock*/); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneIgnoreLock"); + else if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + + if (rcStrict != VINF_SUCCESS) + LogFlow(("IEMExecOneIgnoreLock: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +} + + +VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions) +{ + uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions; + AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate)); + + /* + * See if there is an interrupt pending in TRPM, inject it if we can. + */ + /** @todo What if we are injecting an exception and not an interrupt? Is that + * possible here? For now we assert it is indeed only an interrupt. */ + if (!TRPMHasTrap(pVCpu)) + { /* likely */ } + else + { + if ( !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx) + && !CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx)) + { + /** @todo Can we centralize this under CPUMCanInjectInterrupt()? */ +#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) + bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx); + if (fIntrEnabled) + { + if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu))) + fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF; + else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu))) + fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu)); + else + { + Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))); + fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu)); + } + } +#else + bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF; +#endif + if (fIntrEnabled) + { + uint8_t u8TrapNo; + TRPMEVENT enmType; + uint32_t uErrCode; + RTGCPTR uCr2; + int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /*pu8InstLen*/, NULL /*fIcebp*/); + AssertRC(rc2); + Assert(enmType == TRPM_HARDWARE_INT); + VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /*cbInstr*/); + + TRPMResetTrap(pVCpu); + +#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) + /* Injecting an event may cause a VM-exit. */ + if ( rcStrict != VINF_SUCCESS + && rcStrict != VINF_IEM_RAISED_XCPT) + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +#else + NOREF(rcStrict); +#endif + } + } + } + + /* + * Initial decoder init w/ prefetch, then setup setjmp. + */ + VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false); + if (rcStrict == VINF_SUCCESS) + { +#ifdef IEM_WITH_SETJMP + pVCpu->iem.s.cActiveMappings = 0; /** @todo wtf? */ + IEM_TRY_SETJMP(pVCpu, rcStrict) +#endif + { + /* + * The run loop. We limit ourselves to 4096 instructions right now. + */ + uint32_t cMaxInstructionsGccStupidity = cMaxInstructions; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + for (;;) + { + /* + * Log the state. + */ +#ifdef LOG_ENABLED + iemLogCurInstr(pVCpu, true, "IEMExecLots"); +#endif + + /* + * Do the decoding and emulation. + */ + uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b); + rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]); +#ifdef VBOX_STRICT + CPUMAssertGuestRFlagsCookie(pVM, pVCpu); +#endif + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + Assert(pVCpu->iem.s.cActiveMappings == 0); + pVCpu->iem.s.cInstructions++; + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* Perform any VMX nested-guest instruction boundary actions. */ + uint64_t fCpu = pVCpu->fLocalForcedActions; + if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER + | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW))) + { /* likely */ } + else + { + rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + fCpu = pVCpu->fLocalForcedActions; + else + { + rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict); + break; + } + } +#endif + if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS)) + { +#ifndef VBOX_WITH_NESTED_HWVIRT_VMX + uint64_t fCpu = pVCpu->fLocalForcedActions; +#endif + fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3 + | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL + | VMCPU_FF_TLB_FLUSH + | VMCPU_FF_UNHALT ); + + if (RT_LIKELY( ( !fCpu + || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)) + && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) ) + && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )) + { + if (cMaxInstructionsGccStupidity-- > 0) + { + /* Poll timers every now an then according to the caller's specs. */ + if ( (cMaxInstructionsGccStupidity & cPollRate) != 0 + || !TMTimerPollBool(pVM, pVCpu)) + { + Assert(pVCpu->iem.s.cActiveMappings == 0); + iemReInitDecoder(pVCpu); + continue; + } + } + } + } + Assert(pVCpu->iem.s.cActiveMappings == 0); + } + else if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict); + break; + } + } +#ifdef IEM_WITH_SETJMP + IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict); + { + if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); +# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) + rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict); +# endif + pVCpu->iem.s.cLongJumps++; + } + IEM_CATCH_LONGJMP_END(pVCpu); +#endif + + /* + * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder). + */ + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + } + else + { + if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + +#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) + /* + * When a nested-guest causes an exception intercept (e.g. #PF) when fetching + * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT. + */ + rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict); +#endif + } + + /* + * Maybe re-enter raw-mode and log. + */ + if (rcStrict != VINF_SUCCESS) + LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict))); + if (pcInstructions) + *pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart; + return rcStrict; +} + + +/** + * Interface used by EMExecuteExec, does exit statistics and limits. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param fWillExit To be defined. + * @param cMinInstructions Minimum number of instructions to execute before checking for FFs. + * @param cMaxInstructions Maximum number of instructions to execute. + * @param cMaxInstructionsWithoutExits + * The max number of instructions without exits. + * @param pStats Where to return statistics. + */ +VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions, + uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats) +{ + NOREF(fWillExit); /** @todo define flexible exit crits */ + + /* + * Initialize return stats. + */ + pStats->cInstructions = 0; + pStats->cExits = 0; + pStats->cMaxExitDistance = 0; + pStats->cReserved = 0; + + /* + * Initial decoder init w/ prefetch, then setup setjmp. + */ + VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false); + if (rcStrict == VINF_SUCCESS) + { +#ifdef IEM_WITH_SETJMP + pVCpu->iem.s.cActiveMappings = 0; /** @todo wtf?!? */ + IEM_TRY_SETJMP(pVCpu, rcStrict) +#endif + { +#ifdef IN_RING0 + bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() || !RTThreadPreemptIsEnabled(NIL_RTTHREAD); +#endif + uint32_t cInstructionSinceLastExit = 0; + + /* + * The run loop. We limit ourselves to 4096 instructions right now. + */ + PVM pVM = pVCpu->CTX_SUFF(pVM); + for (;;) + { + /* + * Log the state. + */ +#ifdef LOG_ENABLED + iemLogCurInstr(pVCpu, true, "IEMExecForExits"); +#endif + + /* + * Do the decoding and emulation. + */ + uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits; + + uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b); + rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]); + + if ( cPotentialExits != pVCpu->iem.s.cPotentialExits + && cInstructionSinceLastExit > 0 /* don't count the first */ ) + { + pStats->cExits += 1; + if (cInstructionSinceLastExit > pStats->cMaxExitDistance) + pStats->cMaxExitDistance = cInstructionSinceLastExit; + cInstructionSinceLastExit = 0; + } + + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + Assert(pVCpu->iem.s.cActiveMappings == 0); + pVCpu->iem.s.cInstructions++; + pStats->cInstructions++; + cInstructionSinceLastExit++; + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* Perform any VMX nested-guest instruction boundary actions. */ + uint64_t fCpu = pVCpu->fLocalForcedActions; + if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER + | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW))) + { /* likely */ } + else + { + rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + fCpu = pVCpu->fLocalForcedActions; + else + { + rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict); + break; + } + } +#endif + if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS)) + { +#ifndef VBOX_WITH_NESTED_HWVIRT_VMX + uint64_t fCpu = pVCpu->fLocalForcedActions; +#endif + fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3 + | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL + | VMCPU_FF_TLB_FLUSH + | VMCPU_FF_UNHALT ); + if (RT_LIKELY( ( ( !fCpu + || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)) + && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF)) + && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ) + || pStats->cInstructions < cMinInstructions)) + { + if (pStats->cInstructions < cMaxInstructions) + { + if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits) + { +#ifdef IN_RING0 + if ( !fCheckPreemptionPending + || !RTThreadPreemptIsPending(NIL_RTTHREAD)) +#endif + { + Assert(pVCpu->iem.s.cActiveMappings == 0); + iemReInitDecoder(pVCpu); + continue; + } +#ifdef IN_RING0 + rcStrict = VINF_EM_RAW_INTERRUPT; + break; +#endif + } + } + } + Assert(!(fCpu & VMCPU_FF_IEM)); + } + Assert(pVCpu->iem.s.cActiveMappings == 0); + } + else if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict); + break; + } + } +#ifdef IEM_WITH_SETJMP + IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict); + { + if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + pVCpu->iem.s.cLongJumps++; + } + IEM_CATCH_LONGJMP_END(pVCpu); +#endif + + /* + * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder). + */ + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + } + else + { + if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + +#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) + /* + * When a nested-guest causes an exception intercept (e.g. #PF) when fetching + * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT. + */ + rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict); +#endif + } + + /* + * Maybe re-enter raw-mode and log. + */ + if (rcStrict != VINF_SUCCESS) + LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, + pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance)); + return rcStrict; +} + + +/** + * Injects a trap, fault, abort, software interrupt or external interrupt. + * + * The parameter list matches TRPMQueryTrapAll pretty closely. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param u8TrapNo The trap number. + * @param enmType What type is it (trap/fault/abort), software + * interrupt or hardware interrupt. + * @param uErrCode The error code if applicable. + * @param uCr2 The CR2 value if applicable. + * @param cbInstr The instruction length (only relevant for + * software interrupts). + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2, + uint8_t cbInstr) +{ + iemInitDecoder(pVCpu, false, false); +#ifdef DBGFTRACE_ENABLED + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx", + u8TrapNo, enmType, uErrCode, uCr2); +#endif + + uint32_t fFlags; + switch (enmType) + { + case TRPM_HARDWARE_INT: + Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo)); + fFlags = IEM_XCPT_FLAGS_T_EXT_INT; + uErrCode = uCr2 = 0; + break; + + case TRPM_SOFTWARE_INT: + Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo)); + fFlags = IEM_XCPT_FLAGS_T_SOFT_INT; + uErrCode = uCr2 = 0; + break; + + case TRPM_TRAP: + Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2)); + fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT; + if (u8TrapNo == X86_XCPT_PF) + fFlags |= IEM_XCPT_FLAGS_CR2; + switch (u8TrapNo) + { + case X86_XCPT_DF: + case X86_XCPT_TS: + case X86_XCPT_NP: + case X86_XCPT_SS: + case X86_XCPT_PF: + case X86_XCPT_AC: + case X86_XCPT_GP: + fFlags |= IEM_XCPT_FLAGS_ERR; + break; + } + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2); + + if (pVCpu->iem.s.cActiveMappings > 0) + iemMemRollback(pVCpu); + + return rcStrict; +} + + +/** + * Injects the active TRPM event. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu) +{ +#ifndef IEM_IMPLEMENTS_TASKSWITCH + IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n")); +#else + uint8_t u8TrapNo; + TRPMEVENT enmType; + uint32_t uErrCode; + RTGCUINTPTR uCr2; + uint8_t cbInstr; + int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */); + if (RT_FAILURE(rc)) + return rc; + + /** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle + * ICEBP \#DB injection as a special case. */ + VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr); +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (rcStrict == VINF_SVM_VMEXIT) + rcStrict = VINF_SUCCESS; +#endif +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (rcStrict == VINF_VMX_VMEXIT) + rcStrict = VINF_SUCCESS; +#endif + /** @todo Are there any other codes that imply the event was successfully + * delivered to the guest? See @bugref{6607}. */ + if ( rcStrict == VINF_SUCCESS + || rcStrict == VINF_IEM_RAISED_XCPT) + TRPMResetTrap(pVCpu); + + return rcStrict; +#endif +} + + +VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp) +{ + RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp); + return VERR_NOT_IMPLEMENTED; +} + + +VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp) +{ + RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp); + return VERR_NOT_IMPLEMENTED; +} + + +/** + * Interface for HM and EM for executing string I/O OUT (write) instructions. + * + * This API ASSUMES that the caller has already verified that the guest code is + * allowed to access the I/O port. (The I/O port is in the DX register in the + * guest state.) + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbValue The size of the I/O port access (1, 2, or 4). + * @param enmAddrMode The addressing mode. + * @param fRepPrefix Indicates whether a repeat prefix is used + * (doesn't matter which for this instruction). + * @param cbInstr The instruction length in bytes. + * @param iEffSeg The effective segment address. + * @param fIoChecked Whether the access to the I/O port has been + * checked or not. It's typically checked in the + * HM scenario. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode, + bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked) +{ + AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG); + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1); + + /* + * State init. + */ + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + /* + * Switch orgy for getting to the right handler. + */ + VBOXSTRICTRC rcStrict; + if (fRepPrefix) + { + switch (enmAddrMode) + { + case IEMMODE_16BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + case IEMMODE_32BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + case IEMMODE_64BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + default: + AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE); + } + } + else + { + switch (enmAddrMode) + { + case IEMMODE_16BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + case IEMMODE_32BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + case IEMMODE_64BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + default: + AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE); + } + } + + if (pVCpu->iem.s.cActiveMappings) + iemMemRollback(pVCpu); + + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM for executing string I/O IN (read) instructions. + * + * This API ASSUMES that the caller has already verified that the guest code is + * allowed to access the I/O port. (The I/O port is in the DX register in the + * guest state.) + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbValue The size of the I/O port access (1, 2, or 4). + * @param enmAddrMode The addressing mode. + * @param fRepPrefix Indicates whether a repeat prefix is used + * (doesn't matter which for this instruction). + * @param cbInstr The instruction length in bytes. + * @param fIoChecked Whether the access to the I/O port has been + * checked or not. It's typically checked in the + * HM scenario. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode, + bool fRepPrefix, uint8_t cbInstr, bool fIoChecked) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1); + + /* + * State init. + */ + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + /* + * Switch orgy for getting to the right handler. + */ + VBOXSTRICTRC rcStrict; + if (fRepPrefix) + { + switch (enmAddrMode) + { + case IEMMODE_16BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break; + case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break; + case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + case IEMMODE_32BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break; + case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break; + case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + case IEMMODE_64BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break; + case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break; + case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + default: + AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE); + } + } + else + { + switch (enmAddrMode) + { + case IEMMODE_16BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break; + case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break; + case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + case IEMMODE_32BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break; + case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break; + case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + case IEMMODE_64BIT: + switch (cbValue) + { + case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break; + case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break; + case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break; + default: + AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE); + } + break; + + default: + AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE); + } + } + + if ( pVCpu->iem.s.cActiveMappings == 0 + || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM)) + { /* likely */ } + else + { + AssertMsg(!IOM_SUCCESS(rcStrict), ("%#x\n", VBOXSTRICTRC_VAL(rcStrict))); + iemMemRollback(pVCpu); + } + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for rawmode to write execute an OUT instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param u16Port The port to read. + * @param fImm Whether the port is specified using an immediate operand or + * using the implicit DX register. + * @param cbReg The register size. + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1); + Assert(cbReg <= 4 && cbReg != 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, fImm, cbReg); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for rawmode to write execute an IN instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param u16Port The port to read. + * @param fImm Whether the port is specified using an immediate operand or + * using the implicit DX. + * @param cbReg The register size. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1); + Assert(cbReg <= 4 && cbReg != 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, fImm, cbReg); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to write to a CRx register. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iCrReg The control register number (destination). + * @param iGReg The general purpose register number (source). + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + Assert(iCrReg < 16); + Assert(iGReg < 16); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to read from a CRx register. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iGReg The general purpose register number (destination). + * @param iCrReg The control register number (source). + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 + | CPUMCTX_EXTRN_APIC_TPR); + Assert(iCrReg < 16); + Assert(iGReg < 16); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to write to a DRx register. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iDrReg The debug register number (destination). + * @param iGReg The general purpose register number (source). + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovDRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iDrReg, uint8_t iGReg) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_DR7); + Assert(iDrReg < 8); + Assert(iGReg < 16); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Dd_Rd, iDrReg, iGReg); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to read from a DRx register. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iGReg The general purpose register number (destination). + * @param iDrReg The debug register number (source). + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovDRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iDrReg) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_DR7); + Assert(iDrReg < 8); + Assert(iGReg < 16); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Dd, iGReg, iDrReg); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to clear the CR0[TS] bit. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the LMSW instruction (loads CR0). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param uValue The value to load into CR0. + * @param GCPtrEffDst The guest-linear address if the LMSW instruction has a + * memory operand. Otherwise pass NIL_RTGCPTR. + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the XSETBV instruction (loads XCRx). + * + * Takes input values in ecx and edx:eax of the CPU context of the calling EMT. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cbInstr The instruction length in bytes. + * @remarks In ring-0 not all of the state needs to be synced in. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the WBINVD instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the INVD instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the INVLPG instruction. + * + * @returns Strict VBox status code. + * @retval VINF_PGM_SYNC_CR3 + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param GCPtrPage The effective address of the page to invalidate. + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the INVPCID instruction. + * + * @returns Strict VBox status code. + * @retval VINF_PGM_SYNC_CR3 + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iEffSeg The effective segment register. + * @param GCPtrDesc The effective address of the INVPCID descriptor. + * @param uType The invalidation type. + * + * @remarks In ring-0 not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc, + uint64_t uType) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the CPUID instruction. + * + * @returns Strict VBox status code. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the RDPMC instruction. + * + * @returns Strict VBox status code. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks Not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the RDTSC instruction. + * + * @returns Strict VBox status code. + * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks Not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the RDTSCP instruction. + * + * @returns Strict VBox status code. + * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks Not all of the state needs to be synced in. Recommended + * to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_TSC_AUX); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the RDMSR instruction. + * + * @returns Strict VBox status code. + * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks Not all of the state needs to be synced in. Requires RCX and + * (currently) all MSRs. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_ALL_MSRS); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the WRMSR instruction. + * + * @returns Strict VBox status code. + * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks Not all of the state needs to be synced in. Requires RCX, RAX, RDX, + * and (currently) all MSRs. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK + | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_ALL_MSRS); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the MONITOR instruction. + * + * @returns Strict VBox status code. + * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks Not all of the state needs to be synced in. + * @remarks ASSUMES the default segment of DS and no segment override prefixes + * are used. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the MWAIT instruction. + * + * @returns Strict VBox status code. + * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks Not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate the HLT instruction. + * + * @returns Strict VBox status code. + * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * + * @remarks Not all of the state needs to be synced in. + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Checks if IEM is in the process of delivering an event (interrupt or + * exception). + * + * @returns true if we're in the process of raising an interrupt or exception, + * false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param puVector Where to store the vector associated with the + * currently delivered event, optional. + * @param pfFlags Where to store th event delivery flags (see + * IEM_XCPT_FLAGS_XXX), optional. + * @param puErr Where to store the error code associated with the + * event, optional. + * @param puCr2 Where to store the CR2 associated with the event, + * optional. + * @remarks The caller should check the flags to determine if the error code and + * CR2 are valid for the event. + */ +VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t *puVector, uint32_t *pfFlags, uint32_t *puErr, uint64_t *puCr2) +{ + bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0; + if (fRaisingXcpt) + { + if (puVector) + *puVector = pVCpu->iem.s.uCurXcpt; + if (pfFlags) + *pfFlags = pVCpu->iem.s.fCurXcpt; + if (puErr) + *puErr = pVCpu->iem.s.uCurXcptErr; + if (puCr2) + *puCr2 = pVCpu->iem.s.uCurXcptCr2; + } + return fRaisingXcpt; +} + +#ifdef IN_RING3 + +/** + * Handles the unlikely and probably fatal merge cases. + * + * @returns Merged status code. + * @param rcStrict Current EM status code. + * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge + * with @a rcStrict. + * @param iMemMap The memory mapping index. For error reporting only. + * @param pVCpu The cross context virtual CPU structure of the calling + * thread, for error reporting only. + */ +DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, + unsigned iMemMap, PVMCPUCC pVCpu) +{ + if (RT_FAILURE_NP(rcStrict)) + return rcStrict; + + if (RT_FAILURE_NP(rcStrictCommit)) + return rcStrictCommit; + + if (rcStrict == rcStrictCommit) + return rcStrictCommit; + + AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n", + VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap, + pVCpu->iem.s.aMemMappings[iMemMap].fAccess, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)); + return VERR_IOM_FF_STATUS_IPE; +} + + +/** + * Helper for IOMR3ProcessForceFlag. + * + * @returns Merged status code. + * @param rcStrict Current EM status code. + * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge + * with @a rcStrict. + * @param iMemMap The memory mapping index. For error reporting only. + * @param pVCpu The cross context virtual CPU structure of the calling + * thread, for error reporting only. + */ +DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu) +{ + /* Simple. */ + if (RT_LIKELY(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RAW_TO_R3)) + return rcStrictCommit; + + if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS)) + return rcStrict; + + /* EM scheduling status codes. */ + if (RT_LIKELY( rcStrict >= VINF_EM_FIRST + && rcStrict <= VINF_EM_LAST)) + { + if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST + && rcStrictCommit <= VINF_EM_LAST)) + return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit; + } + + /* Unlikely */ + return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu); +} + + +/** + * Called by force-flag handling code when VMCPU_FF_IEM is set. + * + * @returns Merge between @a rcStrict and what the commit operation returned. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param rcStrict The status code returned by ring-0 or raw-mode. + */ +VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) +{ + /* + * Reset the pending commit. + */ + AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess | pVCpu->iem.s.aMemMappings[1].fAccess | pVCpu->iem.s.aMemMappings[2].fAccess) + & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND), + ("%#x %#x %#x\n", + pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess)); + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM); + + /* + * Commit the pending bounce buffers (usually just one). + */ + unsigned cBufs = 0; + unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings); + while (iMemMap-- > 0) + if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND)) + { + Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE); + Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED); + Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned); + + uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst; + uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond; + uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]; + + if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST) + { + VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, + pbBuf, + cbFirst, + PGMACCESSORIGIN_IEM); + rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu); + Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n", + iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, + VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict))); + } + + if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND) + { + VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM, + pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, + pbBuf + cbFirst, + cbSecond, + PGMACCESSORIGIN_IEM); + rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu); + Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n", + iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, + VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict))); + } + cBufs++; + pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID; + } + + AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings, + ("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings, + pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess)); + pVCpu->iem.s.cActiveMappings = 0; + return rcStrict; +} + +#endif /* IN_RING3 */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllAImpl.asm b/src/VBox/VMM/VMMAll/IEMAllAImpl.asm new file mode 100644 index 00000000..0d6e8517 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllAImpl.asm @@ -0,0 +1,6458 @@ +; $Id: IEMAllAImpl.asm $ +;; @file +; IEM - Instruction Implementation in Assembly. +; + +; +; Copyright (C) 2011-2023 Oracle and/or its affiliates. +; +; This file is part of VirtualBox base platform packages, as +; available from https://www.virtualbox.org. +; +; This program is free software; you can redistribute it and/or +; modify it under the terms of the GNU General Public License +; as published by the Free Software Foundation, in version 3 of the +; License. +; +; This program is distributed in the hope that it will be useful, but +; WITHOUT ANY WARRANTY; without even the implied warranty of +; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +; General Public License for more details. +; +; You should have received a copy of the GNU General Public License +; along with this program; if not, see . +; +; SPDX-License-Identifier: GPL-3.0-only +; + + +;********************************************************************************************************************************* +;* Header Files * +;********************************************************************************************************************************* +%include "VBox/asmdefs.mac" +%include "VBox/err.mac" +%include "iprt/x86.mac" + + +;********************************************************************************************************************************* +;* Defined Constants And Macros * +;********************************************************************************************************************************* + +;; +; RET XX / RET wrapper for fastcall. +; +%macro RET_FASTCALL 1 +%ifdef RT_ARCH_X86 + %ifdef RT_OS_WINDOWS + ret %1 + %else + ret + %endif +%else + ret +%endif +%endmacro + +;; +; NAME for fastcall functions. +; +;; @todo 'global @fastcall@12' is still broken in yasm and requires dollar +; escaping (or whatever the dollar is good for here). Thus the ugly +; prefix argument. +; +%define NAME_FASTCALL(a_Name, a_cbArgs, a_Prefix) NAME(a_Name) +%ifdef RT_ARCH_X86 + %ifdef RT_OS_WINDOWS + %undef NAME_FASTCALL + %define NAME_FASTCALL(a_Name, a_cbArgs, a_Prefix) a_Prefix %+ a_Name %+ @ %+ a_cbArgs + %endif +%endif + +;; +; BEGINPROC for fastcall functions. +; +; @param 1 The function name (C). +; @param 2 The argument size on x86. +; +%macro BEGINPROC_FASTCALL 2 + %ifdef ASM_FORMAT_PE + export %1=NAME_FASTCALL(%1,%2,$@) + %endif + %ifdef __NASM__ + %ifdef ASM_FORMAT_OMF + export NAME(%1) NAME_FASTCALL(%1,%2,$@) + %endif + %endif + %ifndef ASM_FORMAT_BIN + global NAME_FASTCALL(%1,%2,$@) + %endif +NAME_FASTCALL(%1,%2,@): + IBT_ENDBRxx +%endmacro + + +; +; We employ some macro assembly here to hid the calling convention differences. +; +%ifdef RT_ARCH_AMD64 + %macro PROLOGUE_1_ARGS 0 + %endmacro + %macro EPILOGUE_1_ARGS 0 + ret + %endmacro + %macro EPILOGUE_1_ARGS_EX 0 + ret + %endmacro + + %macro PROLOGUE_2_ARGS 0 + %endmacro + %macro EPILOGUE_2_ARGS 0 + ret + %endmacro + %macro EPILOGUE_2_ARGS_EX 1 + ret + %endmacro + + %macro PROLOGUE_3_ARGS 0 + %endmacro + %macro EPILOGUE_3_ARGS 0 + ret + %endmacro + %macro EPILOGUE_3_ARGS_EX 1 + ret + %endmacro + + %macro PROLOGUE_4_ARGS 0 + %endmacro + %macro EPILOGUE_4_ARGS 0 + ret + %endmacro + %macro EPILOGUE_4_ARGS_EX 1 + ret + %endmacro + + %ifdef ASM_CALL64_GCC + %define A0 rdi + %define A0_32 edi + %define A0_16 di + %define A0_8 dil + + %define A1 rsi + %define A1_32 esi + %define A1_16 si + %define A1_8 sil + + %define A2 rdx + %define A2_32 edx + %define A2_16 dx + %define A2_8 dl + + %define A3 rcx + %define A3_32 ecx + %define A3_16 cx + %endif + + %ifdef ASM_CALL64_MSC + %define A0 rcx + %define A0_32 ecx + %define A0_16 cx + %define A0_8 cl + + %define A1 rdx + %define A1_32 edx + %define A1_16 dx + %define A1_8 dl + + %define A2 r8 + %define A2_32 r8d + %define A2_16 r8w + %define A2_8 r8b + + %define A3 r9 + %define A3_32 r9d + %define A3_16 r9w + %endif + + %define T0 rax + %define T0_32 eax + %define T0_16 ax + %define T0_8 al + + %define T1 r11 + %define T1_32 r11d + %define T1_16 r11w + %define T1_8 r11b + + %define T2 r10 ; only AMD64 + %define T2_32 r10d + %define T2_16 r10w + %define T2_8 r10b + +%else + ; x86 + %macro PROLOGUE_1_ARGS 0 + push edi + %endmacro + %macro EPILOGUE_1_ARGS 0 + pop edi + ret 0 + %endmacro + %macro EPILOGUE_1_ARGS_EX 1 + pop edi + ret %1 + %endmacro + + %macro PROLOGUE_2_ARGS 0 + push edi + %endmacro + %macro EPILOGUE_2_ARGS 0 + pop edi + ret 0 + %endmacro + %macro EPILOGUE_2_ARGS_EX 1 + pop edi + ret %1 + %endmacro + + %macro PROLOGUE_3_ARGS 0 + push ebx + mov ebx, [esp + 4 + 4] + push edi + %endmacro + %macro EPILOGUE_3_ARGS_EX 1 + %if (%1) < 4 + %error "With three args, at least 4 bytes must be remove from the stack upon return (32-bit)." + %endif + pop edi + pop ebx + ret %1 + %endmacro + %macro EPILOGUE_3_ARGS 0 + EPILOGUE_3_ARGS_EX 4 + %endmacro + + %macro PROLOGUE_4_ARGS 0 + push ebx + push edi + push esi + mov ebx, [esp + 12 + 4 + 0] + mov esi, [esp + 12 + 4 + 4] + %endmacro + %macro EPILOGUE_4_ARGS_EX 1 + %if (%1) < 8 + %error "With four args, at least 8 bytes must be remove from the stack upon return (32-bit)." + %endif + pop esi + pop edi + pop ebx + ret %1 + %endmacro + %macro EPILOGUE_4_ARGS 0 + EPILOGUE_4_ARGS_EX 8 + %endmacro + + %define A0 ecx + %define A0_32 ecx + %define A0_16 cx + %define A0_8 cl + + %define A1 edx + %define A1_32 edx + %define A1_16 dx + %define A1_8 dl + + %define A2 ebx + %define A2_32 ebx + %define A2_16 bx + %define A2_8 bl + + %define A3 esi + %define A3_32 esi + %define A3_16 si + + %define T0 eax + %define T0_32 eax + %define T0_16 ax + %define T0_8 al + + %define T1 edi + %define T1_32 edi + %define T1_16 di +%endif + + +;; +; Load the relevant flags from [%1] if there are undefined flags (%3). +; +; @remarks Clobbers T0, stack. Changes EFLAGS. +; @param A2 The register pointing to the flags. +; @param 1 The parameter (A0..A3) pointing to the eflags. +; @param 2 The set of modified flags. +; @param 3 The set of undefined flags. +; +%macro IEM_MAYBE_LOAD_FLAGS 3 + ;%if (%3) != 0 + pushf ; store current flags + mov T0_32, [%1] ; load the guest flags + and dword [xSP], ~(%2 | %3) ; mask out the modified and undefined flags + and T0_32, (%2 | %3) ; select the modified and undefined flags. + or [xSP], T0 ; merge guest flags with host flags. + popf ; load the mixed flags. + ;%endif +%endmacro + +;; +; Update the flag. +; +; @remarks Clobbers T0, T1, stack. +; @param 1 The register pointing to the EFLAGS. +; @param 2 The mask of modified flags to save. +; @param 3 The mask of undefined flags to (maybe) save. +; +%macro IEM_SAVE_FLAGS 3 + %if (%2 | %3) != 0 + pushf + pop T1 + mov T0_32, [%1] ; flags + and T0_32, ~(%2 | %3) ; clear the modified & undefined flags. + and T1_32, (%2 | %3) ; select the modified and undefined flags. + or T0_32, T1_32 ; combine the flags. + mov [%1], T0_32 ; save the flags. + %endif +%endmacro + +;; +; Calculates the new EFLAGS based on the CPU EFLAGS and fixed clear and set bit masks. +; +; @remarks Clobbers T0, T1, stack. +; @param 1 The register pointing to the EFLAGS. +; @param 2 The mask of modified flags to save. +; @param 3 Mask of additional flags to always clear +; @param 4 Mask of additional flags to always set. +; +%macro IEM_SAVE_AND_ADJUST_FLAGS 4 + %if (%2 | %3 | %4) != 0 + pushf + pop T1 + mov T0_32, [%1] ; load flags. + and T0_32, ~(%2 | %3) ; clear the modified and always cleared flags. + and T1_32, (%2) ; select the modified flags. + or T0_32, T1_32 ; combine the flags. + %if (%4) != 0 + or T0_32, %4 ; add the always set flags. + %endif + mov [%1], T0_32 ; save the result. + %endif +%endmacro + +;; +; Calculates the new EFLAGS based on the CPU EFLAGS (%2), a clear mask (%3), +; signed input (%4[%5]) and parity index (%6). +; +; This is used by MUL and IMUL, where we got result (%4 & %6) in xAX which is +; also T0. So, we have to use T1 for the EFLAGS calculation and save T0/xAX +; while we extract the %2 flags from the CPU EFLAGS or use T2 (only AMD64). +; +; @remarks Clobbers T0, T1, stack, %6, EFLAGS. +; @param 1 The register pointing to the EFLAGS. +; @param 2 The mask of modified flags to save. +; @param 3 Mask of additional flags to always clear +; @param 4 The result register to set SF by. +; @param 5 The width of the %4 register in bits (8, 16, 32, or 64). +; @param 6 The (full) register containing the parity table index. Will be modified! + +%macro IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF 6 + %ifdef RT_ARCH_AMD64 + pushf + pop T2 + %else + push T0 + pushf + pop T0 + %endif + mov T1_32, [%1] ; load flags. + and T1_32, ~(%2 | %3 | X86_EFL_PF | X86_EFL_SF) ; clear the modified, always cleared flags and the two flags we calc. + %ifdef RT_ARCH_AMD64 + and T2_32, (%2) ; select the modified flags. + or T1_32, T2_32 ; combine the flags. + %else + and T0_32, (%2) ; select the modified flags. + or T1_32, T0_32 ; combine the flags. + pop T0 + %endif + + ; First calculate SF as it's likely to be refereing to the same register as %6 does. + bt %4, %5 - 1 + jnc %%sf_clear + or T1_32, X86_EFL_SF + %%sf_clear: + + ; Parity last. + and %6, 0xff + %ifdef RT_ARCH_AMD64 + lea T2, [NAME(g_afParity) xWrtRIP] + or T1_8, [T2 + %6] + %else + or T1_8, [NAME(g_afParity) + %6] + %endif + + mov [%1], T1_32 ; save the result. +%endmacro + +;; +; Calculates the new EFLAGS using fixed clear and set bit masks. +; +; @remarks Clobbers T0. +; @param 1 The register pointing to the EFLAGS. +; @param 2 Mask of additional flags to always clear +; @param 3 Mask of additional flags to always set. +; +%macro IEM_ADJUST_FLAGS 3 + %if (%2 | %3) != 0 + mov T0_32, [%1] ; Load flags. + %if (%2) != 0 + and T0_32, ~(%2) ; Remove the always cleared flags. + %endif + %if (%3) != 0 + or T0_32, %3 ; Add the always set flags. + %endif + mov [%1], T0_32 ; Save the result. + %endif +%endmacro + +;; +; Calculates the new EFLAGS using fixed clear and set bit masks. +; +; @remarks Clobbers T0, %4, EFLAGS. +; @param 1 The register pointing to the EFLAGS. +; @param 2 Mask of additional flags to always clear +; @param 3 Mask of additional flags to always set. +; @param 4 The (full) register containing the parity table index. Will be modified! +; +%macro IEM_ADJUST_FLAGS_WITH_PARITY 4 + mov T0_32, [%1] ; Load flags. + and T0_32, ~(%2 | X86_EFL_PF) ; Remove PF and the always cleared flags. + %if (%3) != 0 + or T0_32, %3 ; Add the always set flags. + %endif + and %4, 0xff + %ifdef RT_ARCH_AMD64 + lea T2, [NAME(g_afParity) xWrtRIP] + or T0_8, [T2 + %4] + %else + or T0_8, [NAME(g_afParity) + %4] + %endif + mov [%1], T0_32 ; Save the result. +%endmacro + + +;; +; Checks that the size expression %1 matches %2 adjusted according to +; RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK and for 256 entries. +; @param 1 The jump array size assembly expression. +; @param 2 The size without accounting for the IBT_ENDBRxx_WITHOUT_NOTRACK instruction. +; +%macro IEMCHECK_256_JUMP_ARRAY_SIZE 2 + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + dw (0xffff - %2 - 256*4) + %1 ; will cause warning if entries are too big. + dw (0xffff + %2 + 256*4) - %1 ; will cause warning if entries are too small. + %else + dw (0xffff - %2) + %1 ; will cause warning if entries are too big. + dw (0xffff + %2) - %1 ; will cause warning if entries are too small. + %endif +%endmacro + + +;********************************************************************************************************************************* +;* External Symbols * +;********************************************************************************************************************************* +extern NAME(g_afParity) + + +;; +; Macro for implementing a binary operator. +; +; This will generate code for the 8, 16, 32 and 64 bit accesses with locked +; variants, except on 32-bit system where the 64-bit accesses requires hand +; coding. +; +; All the functions takes a pointer to the destination memory operand in A0, +; the source register operand in A1 and a pointer to eflags in A2. +; +; @param 1 The instruction mnemonic. +; @param 2 Non-zero if there should be a locked version. +; @param 3 The modified flags. +; @param 4 The undefined flags. +; +%macro IEMIMPL_BIN_OP 4 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + %1 byte [A0], A1_8 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u8 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + %1 word [A0], A1_16 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + %1 dword [A0], A1_32 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + %1 qword [A0], A1 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS_EX 8 +ENDPROC iemAImpl_ %+ %1 %+ _u64 + %endif ; RT_ARCH_AMD64 + + %if %2 != 0 ; locked versions requested? + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8_locked, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + lock %1 byte [A0], A1_8 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u8_locked + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + lock %1 word [A0], A1_16 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16_locked + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + lock %1 dword [A0], A1_32 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32_locked + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 16 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + lock %1 qword [A0], A1 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS_EX 8 +ENDPROC iemAImpl_ %+ %1 %+ _u64_locked + %endif ; RT_ARCH_AMD64 + %endif ; locked +%endmacro + +; instr,lock, modified-flags, undefined flags +IEMIMPL_BIN_OP add, 1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 +IEMIMPL_BIN_OP adc, 1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 +IEMIMPL_BIN_OP sub, 1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 +IEMIMPL_BIN_OP sbb, 1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 +IEMIMPL_BIN_OP or, 1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), X86_EFL_AF +IEMIMPL_BIN_OP xor, 1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), X86_EFL_AF +IEMIMPL_BIN_OP and, 1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), X86_EFL_AF +IEMIMPL_BIN_OP cmp, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 +IEMIMPL_BIN_OP test, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), X86_EFL_AF + + +;; +; Macro for implementing a binary operator, VEX variant with separate input/output. +; +; This will generate code for the 32 and 64 bit accesses, except on 32-bit system +; where the 64-bit accesses requires hand coding. +; +; All the functions takes a pointer to the destination memory operand in A0, +; the first source register operand in A1, the second source register operand +; in A2 and a pointer to eflags in A3. +; +; @param 1 The instruction mnemonic. +; @param 2 The modified flags. +; @param 3 The undefined flags. +; +%macro IEMIMPL_VEX_BIN_OP 3 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + %1 T0_32, A1_32, A2_32 + mov [A0], T0_32 + IEM_SAVE_FLAGS A3, %2, %3 + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + %1 T0, A1, A2 + mov [A0], T0 + IEM_SAVE_FLAGS A3, %2, %3 + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64 + %endif ; RT_ARCH_AMD64 +%endmacro + +; instr, modified-flags, undefined-flags +IEMIMPL_VEX_BIN_OP andn, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF) +IEMIMPL_VEX_BIN_OP bextr, (X86_EFL_OF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_AF | X86_EFL_PF) +IEMIMPL_VEX_BIN_OP bzhi, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF) + +;; +; Macro for implementing BLSR, BLCMSK and BLSI (fallbacks implemented in C). +; +; This will generate code for the 32 and 64 bit accesses, except on 32-bit system +; where the 64-bit accesses requires hand coding. +; +; All the functions takes a pointer to the destination memory operand in A0, +; the source register operand in A1 and a pointer to eflags in A2. +; +; @param 1 The instruction mnemonic. +; @param 2 The modified flags. +; @param 3 The undefined flags. +; +%macro IEMIMPL_VEX_BIN_OP_2 3 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + mov T0_32, [A0] + %1 T0_32, A1_32 + mov [A0], T0_32 + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + mov T0, [A0] + %1 T0, A1 + mov [A0], T0 + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64 + %endif ; RT_ARCH_AMD64 +%endmacro + +; instr, modified-flags, undefined-flags +IEMIMPL_VEX_BIN_OP_2 blsr, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF) +IEMIMPL_VEX_BIN_OP_2 blsmsk, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF) +IEMIMPL_VEX_BIN_OP_2 blsi, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF) + + +;; +; Macro for implementing a binary operator w/o flags, VEX variant with separate input/output. +; +; This will generate code for the 32 and 64 bit accesses, except on 32-bit system +; where the 64-bit accesses requires hand coding. +; +; All the functions takes a pointer to the destination memory operand in A0, +; the first source register operand in A1, the second source register operand +; in A2 and a pointer to eflags in A3. +; +; @param 1 The instruction mnemonic. +; @param 2 Fallback instruction if applicable. +; @param 3 Whether to emit fallback or not. +; +%macro IEMIMPL_VEX_BIN_OP_NOEFL 3 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12 + PROLOGUE_3_ARGS + %1 T0_32, A1_32, A2_32 + mov [A0], T0_32 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + + %if %3 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_fallback, 12 + PROLOGUE_3_ARGS + %ifdef ASM_CALL64_GCC + mov cl, A2_8 + %2 A1_32, cl + mov [A0], A1_32 + %else + xchg A2, A0 + %2 A1_32, cl + mov [A2], A1_32 + %endif + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32_fallback + %endif + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12 + PROLOGUE_3_ARGS + %1 T0, A1, A2 + mov [A0], T0 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64 + + %if %3 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_fallback, 12 + PROLOGUE_3_ARGS + %ifdef ASM_CALL64_GCC + mov cl, A2_8 + %2 A1, cl + mov [A0], A1_32 + %else + xchg A2, A0 + %2 A1, cl + mov [A2], A1_32 + %endif + mov [A0], A1 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64_fallback + %endif + %endif ; RT_ARCH_AMD64 +%endmacro + +; instr, fallback instr, emit fallback +IEMIMPL_VEX_BIN_OP_NOEFL sarx, sar, 1 +IEMIMPL_VEX_BIN_OP_NOEFL shlx, shl, 1 +IEMIMPL_VEX_BIN_OP_NOEFL shrx, shr, 1 +IEMIMPL_VEX_BIN_OP_NOEFL pdep, nop, 0 +IEMIMPL_VEX_BIN_OP_NOEFL pext, nop, 0 + + +; +; RORX uses a immediate byte for the shift count, so we only do +; fallback implementation of that one. +; +BEGINPROC_FASTCALL iemAImpl_rorx_u32, 12 + PROLOGUE_3_ARGS + %ifdef ASM_CALL64_GCC + mov cl, A2_8 + ror A1_32, cl + mov [A0], A1_32 + %else + xchg A2, A0 + ror A1_32, cl + mov [A2], A1_32 + %endif + EPILOGUE_3_ARGS +ENDPROC iemAImpl_rorx_u32 + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_rorx_u64, 12 + PROLOGUE_3_ARGS + %ifdef ASM_CALL64_GCC + mov cl, A2_8 + ror A1, cl + mov [A0], A1_32 + %else + xchg A2, A0 + ror A1, cl + mov [A2], A1_32 + %endif + mov [A0], A1 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_rorx_u64 + %endif ; RT_ARCH_AMD64 + + +; +; MULX +; +BEGINPROC_FASTCALL iemAImpl_mulx_u32, 16 + PROLOGUE_4_ARGS +%ifdef ASM_CALL64_GCC + ; A2_32 is EDX - prefect + mulx T0_32, T1_32, A3_32 + mov [A1], T1_32 ; Low value first, as we should return the high part if same destination registers. + mov [A0], T0_32 +%else + ; A1 is xDX - must switch A1 and A2, so EDX=uSrc1 + xchg A1, A2 + mulx T0_32, T1_32, A3_32 + mov [A2], T1_32 ; Low value first, as we should return the high part if same destination registers. + mov [A0], T0_32 +%endif + EPILOGUE_4_ARGS +ENDPROC iemAImpl_mulx_u32 + + +BEGINPROC_FASTCALL iemAImpl_mulx_u32_fallback, 16 + PROLOGUE_4_ARGS +%ifdef ASM_CALL64_GCC + ; A2_32 is EDX, T0_32 is EAX + mov eax, A3_32 + mul A2_32 + mov [A1], eax ; Low value first, as we should return the high part if same destination registers. + mov [A0], edx +%else + ; A1 is xDX, T0_32 is EAX - must switch A1 and A2, so EDX=uSrc1 + xchg A1, A2 + mov eax, A3_32 + mul A2_32 + mov [A2], eax ; Low value first, as we should return the high part if same destination registers. + mov [A0], edx +%endif + EPILOGUE_4_ARGS +ENDPROC iemAImpl_mulx_u32_fallback + +%ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_mulx_u64, 16 + PROLOGUE_4_ARGS +%ifdef ASM_CALL64_GCC + ; A2 is RDX - prefect + mulx T0, T1, A3 + mov [A1], T1 ; Low value first, as we should return the high part if same destination registers. + mov [A0], T0 +%else + ; A1 is xDX - must switch A1 and A2, so RDX=uSrc1 + xchg A1, A2 + mulx T0, T1, A3 + mov [A2], T1 ; Low value first, as we should return the high part if same destination registers. + mov [A0], T0 +%endif + EPILOGUE_4_ARGS +ENDPROC iemAImpl_mulx_u64 + + +BEGINPROC_FASTCALL iemAImpl_mulx_u64_fallback, 16 + PROLOGUE_4_ARGS +%ifdef ASM_CALL64_GCC + ; A2 is RDX, T0 is RAX + mov rax, A3 + mul A2 + mov [A1], rax ; Low value first, as we should return the high part if same destination registers. + mov [A0], rdx +%else + ; A1 is xDX, T0 is RAX - must switch A1 and A2, so RDX=uSrc1 + xchg A1, A2 + mov rax, A3 + mul A2 + mov [A2], rax ; Low value first, as we should return the high part if same destination registers. + mov [A0], rdx +%endif + EPILOGUE_4_ARGS +ENDPROC iemAImpl_mulx_u64_fallback + +%endif + + +;; +; Macro for implementing a bit operator. +; +; This will generate code for the 16, 32 and 64 bit accesses with locked +; variants, except on 32-bit system where the 64-bit accesses requires hand +; coding. +; +; All the functions takes a pointer to the destination memory operand in A0, +; the source register operand in A1 and a pointer to eflags in A2. +; +; @param 1 The instruction mnemonic. +; @param 2 Non-zero if there should be a locked version. +; @param 3 The modified flags. +; @param 4 The undefined flags. +; +%macro IEMIMPL_BIT_OP 4 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + %1 word [A0], A1_16 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + %1 dword [A0], A1_32 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + %1 qword [A0], A1 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS_EX 8 +ENDPROC iemAImpl_ %+ %1 %+ _u64 + %endif ; RT_ARCH_AMD64 + + %if %2 != 0 ; locked versions requested? + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + lock %1 word [A0], A1_16 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16_locked + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + lock %1 dword [A0], A1_32 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32_locked + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 16 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %3, %4 + lock %1 qword [A0], A1 + IEM_SAVE_FLAGS A2, %3, %4 + EPILOGUE_3_ARGS_EX 8 +ENDPROC iemAImpl_ %+ %1 %+ _u64_locked + %endif ; RT_ARCH_AMD64 + %endif ; locked +%endmacro +IEMIMPL_BIT_OP bt, 0, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF) +IEMIMPL_BIT_OP btc, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF) +IEMIMPL_BIT_OP bts, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF) +IEMIMPL_BIT_OP btr, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF) + +;; +; Macro for implementing a bit search operator. +; +; This will generate code for the 16, 32 and 64 bit accesses, except on 32-bit +; system where the 64-bit accesses requires hand coding. +; +; All the functions takes a pointer to the destination memory operand in A0, +; the source register operand in A1 and a pointer to eflags in A2. +; +; In the ZF case the destination register is 'undefined', however it seems that +; both AMD and Intel just leaves it as is. The undefined EFLAGS differs between +; AMD and Intel and accoridng to https://www.sandpile.org/x86/flags.htm between +; Intel microarchitectures. We only implement 'intel' and 'amd' variation with +; the behaviour of more recent CPUs (Intel 10980X and AMD 3990X). +; +; @param 1 The instruction mnemonic. +; @param 2 The modified flags. +; @param 3 The undefined flags. +; @param 4 Non-zero if destination isn't written when ZF=1. Zero if always written. +; +%macro IEMIMPL_BIT_OP2 4 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %1 T0_16, A1_16 +%if %4 != 0 + jz .unchanged_dst +%endif + mov [A0], T0_16 +.unchanged_dst: + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _intel, 12 + PROLOGUE_3_ARGS + %1 T1_16, A1_16 +%if %4 != 0 + jz .unchanged_dst +%endif + mov [A0], T1_16 + IEM_ADJUST_FLAGS_WITH_PARITY A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1 + EPILOGUE_3_ARGS +.unchanged_dst: + IEM_ADJUST_FLAGS A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16_intel + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _amd, 12 + PROLOGUE_3_ARGS + %1 T0_16, A1_16 +%if %4 != 0 + jz .unchanged_dst +%endif + mov [A0], T0_16 +.unchanged_dst: + IEM_SAVE_AND_ADJUST_FLAGS A2, %2, 0, 0 ; Only the ZF flag is modified on AMD Zen 2. + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16_amd + + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %1 T0_32, A1_32 +%if %4 != 0 + jz .unchanged_dst +%endif + mov [A0], T0_32 +.unchanged_dst: + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _intel, 12 + PROLOGUE_3_ARGS + %1 T1_32, A1_32 +%if %4 != 0 + jz .unchanged_dst +%endif + mov [A0], T1_32 + IEM_ADJUST_FLAGS_WITH_PARITY A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1 + EPILOGUE_3_ARGS +.unchanged_dst: + IEM_ADJUST_FLAGS A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32_intel + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _amd, 12 + PROLOGUE_3_ARGS + %1 T0_32, A1_32 +%if %4 != 0 + jz .unchanged_dst +%endif + mov [A0], T0_32 +.unchanged_dst: + IEM_SAVE_AND_ADJUST_FLAGS A2, %2, 0, 0 ; Only the ZF flag is modified on AMD Zen 2. + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32_amd + + + %ifdef RT_ARCH_AMD64 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %1 T0, A1 +%if %4 != 0 + jz .unchanged_dst +%endif + mov [A0], T0 +.unchanged_dst: + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS_EX 8 +ENDPROC iemAImpl_ %+ %1 %+ _u64 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _intel, 16 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %1 T1, A1 +%if %4 != 0 + jz .unchanged_dst +%endif + mov [A0], T1 + IEM_ADJUST_FLAGS_WITH_PARITY A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1 + EPILOGUE_3_ARGS +.unchanged_dst: + IEM_ADJUST_FLAGS A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64_intel + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _amd, 16 + PROLOGUE_3_ARGS + %1 T0, A1 +%if %4 != 0 + jz .unchanged_dst +%endif + mov [A0], T0 +.unchanged_dst: + IEM_SAVE_AND_ADJUST_FLAGS A2, %2, 0, 0 ; Only the ZF flag is modified on AMD Zen 2. + EPILOGUE_3_ARGS_EX 8 +ENDPROC iemAImpl_ %+ %1 %+ _u64_amd + + %endif ; RT_ARCH_AMD64 +%endmacro + +IEMIMPL_BIT_OP2 bsf, (X86_EFL_ZF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1 +IEMIMPL_BIT_OP2 bsr, (X86_EFL_ZF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1 +IEMIMPL_BIT_OP2 tzcnt, (X86_EFL_ZF | X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0 +IEMIMPL_BIT_OP2 lzcnt, (X86_EFL_ZF | X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0 + + +;; +; Macro for implementing POPCNT. +; +; This will generate code for the 16, 32 and 64 bit accesses, except on 32-bit +; system where the 64-bit accesses requires hand coding. +; +; All the functions takes a pointer to the destination memory operand in A0, +; the source register operand in A1 and a pointer to eflags in A2. +; +; ASSUMES Intel and AMD set EFLAGS the same way. +; +; ASSUMES the instruction does not support memory destination. +; +; @param 1 The instruction mnemonic. +; @param 2 The modified flags. +; @param 3 The undefined flags. +; +%macro IEMIMPL_BIT_OP3 3 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %1 T0_16, A1_16 + mov [A0], T0_16 + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %1 T0_32, A1_32 + mov [A0], T0_32 + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %1 T0, A1 + mov [A0], T0 + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS_EX 8 +ENDPROC iemAImpl_ %+ %1 %+ _u64 + %endif ; RT_ARCH_AMD64 +%endmacro +IEMIMPL_BIT_OP3 popcnt, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0 + + +; +; IMUL is also a similar but yet different case (no lock, no mem dst). +; The rDX:rAX variant of imul is handled together with mul further down. +; +BEGINCODE +; @param 1 EFLAGS that are modified. +; @param 2 Undefined EFLAGS. +; @param 3 Function suffix. +; @param 4 EFLAGS variation: 0 for native, 1 for intel (ignored), +; 2 for AMD (set AF, clear PF, ZF and SF). +%macro IEMIMPL_IMUL_TWO 4 +BEGINPROC_FASTCALL iemAImpl_imul_two_u16 %+ %3, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %1, %2 + imul A1_16, word [A0] + mov [A0], A1_16 + %if %4 != 1 + IEM_SAVE_FLAGS A2, %1, %2 + %else + IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_16, 16, A1 + %endif + EPILOGUE_3_ARGS +ENDPROC iemAImpl_imul_two_u16 %+ %3 + +BEGINPROC_FASTCALL iemAImpl_imul_two_u32 %+ %3, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %1, %2 + imul A1_32, dword [A0] + mov [A0], A1_32 + %if %4 != 1 + IEM_SAVE_FLAGS A2, %1, %2 + %else + IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_32, 32, A1 + %endif + EPILOGUE_3_ARGS +ENDPROC iemAImpl_imul_two_u32 %+ %3 + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_imul_two_u64 %+ %3, 16 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %1, %2 + imul A1, qword [A0] + mov [A0], A1 + %if %4 != 1 + IEM_SAVE_FLAGS A2, %1, %2 + %else + IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF A2, %1, X86_EFL_AF | X86_EFL_ZF, A1, 64, A1 + %endif + EPILOGUE_3_ARGS_EX 8 +ENDPROC iemAImpl_imul_two_u64 %+ %3 + %endif ; RT_ARCH_AMD64 +%endmacro +IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF, , 0 +IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, 0, _intel, 1 +IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, 0, _amd, 2 + + +; +; XCHG for memory operands. This implies locking. No flag changes. +; +; Each function takes two arguments, first the pointer to the memory, +; then the pointer to the register. They all return void. +; +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_xchg_u8_locked, 8 + PROLOGUE_2_ARGS + mov T0_8, [A1] + xchg [A0], T0_8 + mov [A1], T0_8 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_xchg_u8_locked + +BEGINPROC_FASTCALL iemAImpl_xchg_u16_locked, 8 + PROLOGUE_2_ARGS + mov T0_16, [A1] + xchg [A0], T0_16 + mov [A1], T0_16 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_xchg_u16_locked + +BEGINPROC_FASTCALL iemAImpl_xchg_u32_locked, 8 + PROLOGUE_2_ARGS + mov T0_32, [A1] + xchg [A0], T0_32 + mov [A1], T0_32 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_xchg_u32_locked + +%ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_xchg_u64_locked, 8 + PROLOGUE_2_ARGS + mov T0, [A1] + xchg [A0], T0 + mov [A1], T0 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_xchg_u64_locked +%endif + +; Unlocked variants for fDisregardLock mode. + +BEGINPROC_FASTCALL iemAImpl_xchg_u8_unlocked, 8 + PROLOGUE_2_ARGS + mov T0_8, [A1] + mov T1_8, [A0] + mov [A0], T0_8 + mov [A1], T1_8 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_xchg_u8_unlocked + +BEGINPROC_FASTCALL iemAImpl_xchg_u16_unlocked, 8 + PROLOGUE_2_ARGS + mov T0_16, [A1] + mov T1_16, [A0] + mov [A0], T0_16 + mov [A1], T1_16 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_xchg_u16_unlocked + +BEGINPROC_FASTCALL iemAImpl_xchg_u32_unlocked, 8 + PROLOGUE_2_ARGS + mov T0_32, [A1] + mov T1_32, [A0] + mov [A0], T0_32 + mov [A1], T1_32 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_xchg_u32_unlocked + +%ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_xchg_u64_unlocked, 8 + PROLOGUE_2_ARGS + mov T0, [A1] + mov T1, [A0] + mov [A0], T0 + mov [A1], T1 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_xchg_u64_unlocked +%endif + + +; +; XADD for memory operands. +; +; Each function takes three arguments, first the pointer to the +; memory/register, then the pointer to the register, and finally a pointer to +; eflags. They all return void. +; +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_xadd_u8, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + mov T0_8, [A1] + xadd [A0], T0_8 + mov [A1], T0_8 + IEM_SAVE_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_xadd_u8 + +BEGINPROC_FASTCALL iemAImpl_xadd_u16, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + mov T0_16, [A1] + xadd [A0], T0_16 + mov [A1], T0_16 + IEM_SAVE_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_xadd_u16 + +BEGINPROC_FASTCALL iemAImpl_xadd_u32, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + mov T0_32, [A1] + xadd [A0], T0_32 + mov [A1], T0_32 + IEM_SAVE_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_xadd_u32 + +%ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_xadd_u64, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + mov T0, [A1] + xadd [A0], T0 + mov [A1], T0 + IEM_SAVE_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_xadd_u64 +%endif ; RT_ARCH_AMD64 + +BEGINPROC_FASTCALL iemAImpl_xadd_u8_locked, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + mov T0_8, [A1] + lock xadd [A0], T0_8 + mov [A1], T0_8 + IEM_SAVE_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_xadd_u8_locked + +BEGINPROC_FASTCALL iemAImpl_xadd_u16_locked, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + mov T0_16, [A1] + lock xadd [A0], T0_16 + mov [A1], T0_16 + IEM_SAVE_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_xadd_u16_locked + +BEGINPROC_FASTCALL iemAImpl_xadd_u32_locked, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + mov T0_32, [A1] + lock xadd [A0], T0_32 + mov [A1], T0_32 + IEM_SAVE_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_xadd_u32_locked + +%ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_xadd_u64_locked, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + mov T0, [A1] + lock xadd [A0], T0 + mov [A1], T0 + IEM_SAVE_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_xadd_u64_locked +%endif ; RT_ARCH_AMD64 + + +; +; CMPXCHG8B. +; +; These are tricky register wise, so the code is duplicated for each calling +; convention. +; +; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to! +; +; C-proto: +; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx, +; uint32_t *pEFlags)); +; +; Note! Identical to iemAImpl_cmpxchg16b. +; +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_cmpxchg8b, 16 +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_MSC + push rbx + + mov r11, rdx ; pu64EaxEdx (is also T1) + mov r10, rcx ; pu64Dst + + mov ebx, [r8] + mov ecx, [r8 + 4] + IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov eax, [r11] + mov edx, [r11 + 4] + + cmpxchg8b [r10] + + mov [r11], eax + mov [r11 + 4], edx + IEM_SAVE_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11) + + pop rbx + ret + %else + push rbx + + mov r10, rcx ; pEFlags + mov r11, rdx ; pu64EbxEcx (is also T1) + + mov ebx, [r11] + mov ecx, [r11 + 4] + IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov eax, [rsi] + mov edx, [rsi + 4] + + cmpxchg8b [rdi] + + mov [rsi], eax + mov [rsi + 4], edx + IEM_SAVE_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11) + + pop rbx + ret + + %endif +%else + push esi + push edi + push ebx + push ebp + + mov edi, ecx ; pu64Dst + mov esi, edx ; pu64EaxEdx + mov ecx, [esp + 16 + 4 + 0] ; pu64EbxEcx + mov ebp, [esp + 16 + 4 + 4] ; pEFlags + + mov ebx, [ecx] + mov ecx, [ecx + 4] + IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov eax, [esi] + mov edx, [esi + 4] + + cmpxchg8b [edi] + + mov [esi], eax + mov [esi + 4], edx + IEM_SAVE_FLAGS ebp, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, edi) + + pop ebp + pop ebx + pop edi + pop esi + ret 8 +%endif +ENDPROC iemAImpl_cmpxchg8b + +BEGINPROC_FASTCALL iemAImpl_cmpxchg8b_locked, 16 +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_MSC + push rbx + + mov r11, rdx ; pu64EaxEdx (is also T1) + mov r10, rcx ; pu64Dst + + mov ebx, [r8] + mov ecx, [r8 + 4] + IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov eax, [r11] + mov edx, [r11 + 4] + + lock cmpxchg8b [r10] + + mov [r11], eax + mov [r11 + 4], edx + IEM_SAVE_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11) + + pop rbx + ret + %else + push rbx + + mov r10, rcx ; pEFlags + mov r11, rdx ; pu64EbxEcx (is also T1) + + mov ebx, [r11] + mov ecx, [r11 + 4] + IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov eax, [rsi] + mov edx, [rsi + 4] + + lock cmpxchg8b [rdi] + + mov [rsi], eax + mov [rsi + 4], edx + IEM_SAVE_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11) + + pop rbx + ret + + %endif +%else + push esi + push edi + push ebx + push ebp + + mov edi, ecx ; pu64Dst + mov esi, edx ; pu64EaxEdx + mov ecx, [esp + 16 + 4 + 0] ; pu64EbxEcx + mov ebp, [esp + 16 + 4 + 4] ; pEFlags + + mov ebx, [ecx] + mov ecx, [ecx + 4] + IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov eax, [esi] + mov edx, [esi + 4] + + lock cmpxchg8b [edi] + + mov [esi], eax + mov [esi + 4], edx + IEM_SAVE_FLAGS ebp, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, edi) + + pop ebp + pop ebx + pop edi + pop esi + ret 8 +%endif +ENDPROC iemAImpl_cmpxchg8b_locked + +%ifdef RT_ARCH_AMD64 + +; +; CMPXCHG16B. +; +; These are tricky register wise, so the code is duplicated for each calling +; convention. +; +; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to! +; +; C-proto: +; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu1284RaxRdx, PRTUINT128U pu128RbxRcx, +; uint32_t *pEFlags)); +; +; Note! Identical to iemAImpl_cmpxchg8b. +; +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_cmpxchg16b, 16 + %ifdef ASM_CALL64_MSC + push rbx + + mov r11, rdx ; pu64RaxRdx (is also T1) + mov r10, rcx ; pu64Dst + + mov rbx, [r8] + mov rcx, [r8 + 8] + IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov rax, [r11] + mov rdx, [r11 + 8] + + cmpxchg16b [r10] + + mov [r11], rax + mov [r11 + 8], rdx + IEM_SAVE_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11) + + pop rbx + ret + %else + push rbx + + mov r10, rcx ; pEFlags + mov r11, rdx ; pu64RbxRcx (is also T1) + + mov rbx, [r11] + mov rcx, [r11 + 8] + IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov rax, [rsi] + mov rdx, [rsi + 8] + + cmpxchg16b [rdi] + + mov [rsi], rax + mov [rsi + 8], rdx + IEM_SAVE_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11) + + pop rbx + ret + + %endif +ENDPROC iemAImpl_cmpxchg16b + +BEGINPROC_FASTCALL iemAImpl_cmpxchg16b_locked, 16 + %ifdef ASM_CALL64_MSC + push rbx + + mov r11, rdx ; pu64RaxRdx (is also T1) + mov r10, rcx ; pu64Dst + + mov rbx, [r8] + mov rcx, [r8 + 8] + IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov rax, [r11] + mov rdx, [r11 + 8] + + lock cmpxchg16b [r10] + + mov [r11], rax + mov [r11 + 8], rdx + IEM_SAVE_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11) + + pop rbx + ret + %else + push rbx + + mov r10, rcx ; pEFlags + mov r11, rdx ; pu64RbxRcx (is also T1) + + mov rbx, [r11] + mov rcx, [r11 + 8] + IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax) + mov rax, [rsi] + mov rdx, [rsi + 8] + + lock cmpxchg16b [rdi] + + mov [rsi], rax + mov [rsi + 8], rdx + IEM_SAVE_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11) + + pop rbx + ret + + %endif +ENDPROC iemAImpl_cmpxchg16b_locked + +%endif ; RT_ARCH_AMD64 + + +; +; CMPXCHG. +; +; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to! +; +; C-proto: +; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg,(uintX_t *puXDst, uintX_t puEax, uintX_t uReg, uint32_t *pEFlags)); +; +BEGINCODE +%macro IEMIMPL_CMPXCHG 2 +BEGINPROC_FASTCALL iemAImpl_cmpxchg_u8 %+ %2, 16 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax) + mov al, [A1] + %1 cmpxchg [A0], A2_8 + mov [A1], al + IEM_SAVE_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi) + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cmpxchg_u8 %+ %2 + +BEGINPROC_FASTCALL iemAImpl_cmpxchg_u16 %+ %2, 16 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax) + mov ax, [A1] + %1 cmpxchg [A0], A2_16 + mov [A1], ax + IEM_SAVE_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi) + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cmpxchg_u16 %+ %2 + +BEGINPROC_FASTCALL iemAImpl_cmpxchg_u32 %+ %2, 16 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax) + mov eax, [A1] + %1 cmpxchg [A0], A2_32 + mov [A1], eax + IEM_SAVE_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi) + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cmpxchg_u32 %+ %2 + +BEGINPROC_FASTCALL iemAImpl_cmpxchg_u64 %+ %2, 16 +%ifdef RT_ARCH_AMD64 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax) + mov rax, [A1] + %1 cmpxchg [A0], A2 + mov [A1], rax + IEM_SAVE_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi) + EPILOGUE_4_ARGS +%else + ; + ; Must use cmpxchg8b here. See also iemAImpl_cmpxchg8b. + ; + push esi + push edi + push ebx + push ebp + + mov edi, ecx ; pu64Dst + mov esi, edx ; pu64Rax + mov ecx, [esp + 16 + 4 + 0] ; pu64Reg - Note! Pointer on 32-bit hosts! + mov ebp, [esp + 16 + 4 + 4] ; pEFlags + + mov ebx, [ecx] + mov ecx, [ecx + 4] + IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax) + mov eax, [esi] + mov edx, [esi + 4] + + lock cmpxchg8b [edi] + + ; cmpxchg8b doesn't set CF, PF, AF, SF and OF, so we have to do that. + jz .cmpxchg8b_not_equal + cmp eax, eax ; just set the other flags. +.store: + mov [esi], eax + mov [esi + 4], edx + IEM_SAVE_FLAGS ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, edi) + + pop ebp + pop ebx + pop edi + pop esi + ret 8 + +.cmpxchg8b_not_equal: + cmp [esi + 4], edx ;; @todo FIXME - verify 64-bit compare implementation + jne .store + cmp [esi], eax + jmp .store + +%endif +ENDPROC iemAImpl_cmpxchg_u64 %+ %2 +%endmacro ; IEMIMPL_CMPXCHG + +IEMIMPL_CMPXCHG , , +IEMIMPL_CMPXCHG lock, _locked + +;; +; Macro for implementing a unary operator. +; +; This will generate code for the 8, 16, 32 and 64 bit accesses with locked +; variants, except on 32-bit system where the 64-bit accesses requires hand +; coding. +; +; All the functions takes a pointer to the destination memory operand in A0, +; the source register operand in A1 and a pointer to eflags in A2. +; +; @param 1 The instruction mnemonic. +; @param 2 The modified flags. +; @param 3 The undefined flags. +; +%macro IEMIMPL_UNARY_OP 3 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 8 + PROLOGUE_2_ARGS + IEM_MAYBE_LOAD_FLAGS A1, %2, %3 + %1 byte [A0] + IEM_SAVE_FLAGS A1, %2, %3 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u8 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8_locked, 8 + PROLOGUE_2_ARGS + IEM_MAYBE_LOAD_FLAGS A1, %2, %3 + lock %1 byte [A0] + IEM_SAVE_FLAGS A1, %2, %3 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u8_locked + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 8 + PROLOGUE_2_ARGS + IEM_MAYBE_LOAD_FLAGS A1, %2, %3 + %1 word [A0] + IEM_SAVE_FLAGS A1, %2, %3 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 8 + PROLOGUE_2_ARGS + IEM_MAYBE_LOAD_FLAGS A1, %2, %3 + lock %1 word [A0] + IEM_SAVE_FLAGS A1, %2, %3 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16_locked + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 8 + PROLOGUE_2_ARGS + IEM_MAYBE_LOAD_FLAGS A1, %2, %3 + %1 dword [A0] + IEM_SAVE_FLAGS A1, %2, %3 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 8 + PROLOGUE_2_ARGS + IEM_MAYBE_LOAD_FLAGS A1, %2, %3 + lock %1 dword [A0] + IEM_SAVE_FLAGS A1, %2, %3 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32_locked + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8 + PROLOGUE_2_ARGS + IEM_MAYBE_LOAD_FLAGS A1, %2, %3 + %1 qword [A0] + IEM_SAVE_FLAGS A1, %2, %3 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 8 + PROLOGUE_2_ARGS + IEM_MAYBE_LOAD_FLAGS A1, %2, %3 + lock %1 qword [A0] + IEM_SAVE_FLAGS A1, %2, %3 + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64_locked + %endif ; RT_ARCH_AMD64 + +%endmacro + +IEMIMPL_UNARY_OP inc, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), 0 +IEMIMPL_UNARY_OP dec, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), 0 +IEMIMPL_UNARY_OP neg, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0 +IEMIMPL_UNARY_OP not, 0, 0 + + +; +; BSWAP. No flag changes. +; +; Each function takes one argument, pointer to the value to bswap +; (input/output). They all return void. +; +BEGINPROC_FASTCALL iemAImpl_bswap_u16, 4 + PROLOGUE_1_ARGS + mov T0_32, [A0] ; just in case any of the upper bits are used. + db 66h + bswap T0_32 + mov [A0], T0_32 + EPILOGUE_1_ARGS +ENDPROC iemAImpl_bswap_u16 + +BEGINPROC_FASTCALL iemAImpl_bswap_u32, 4 + PROLOGUE_1_ARGS + mov T0_32, [A0] + bswap T0_32 + mov [A0], T0_32 + EPILOGUE_1_ARGS +ENDPROC iemAImpl_bswap_u32 + +BEGINPROC_FASTCALL iemAImpl_bswap_u64, 4 +%ifdef RT_ARCH_AMD64 + PROLOGUE_1_ARGS + mov T0, [A0] + bswap T0 + mov [A0], T0 + EPILOGUE_1_ARGS +%else + PROLOGUE_1_ARGS + mov T0, [A0] + mov T1, [A0 + 4] + bswap T0 + bswap T1 + mov [A0 + 4], T0 + mov [A0], T1 + EPILOGUE_1_ARGS +%endif +ENDPROC iemAImpl_bswap_u64 + + +;; +; Macro for implementing a shift operation. +; +; This will generate code for the 8, 16, 32 and 64 bit accesses, except on +; 32-bit system where the 64-bit accesses requires hand coding. +; +; All the functions takes a pointer to the destination memory operand in A0, +; the shift count in A1 and a pointer to eflags in A2. +; +; @param 1 The instruction mnemonic. +; @param 2 The modified flags. +; @param 3 The undefined flags. +; +; Makes ASSUMPTIONS about A0, A1 and A2 assignments. +; +; @note the _intel and _amd variants are implemented in C. +; +%macro IEMIMPL_SHIFT_OP 3 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %ifdef ASM_CALL64_GCC + mov cl, A1_8 + %1 byte [A0], cl + %else + xchg A1, A0 + %1 byte [A1], cl + %endif + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u8 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %ifdef ASM_CALL64_GCC + mov cl, A1_8 + %1 word [A0], cl + %else + xchg A1, A0 + %1 word [A1], cl + %endif + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %ifdef ASM_CALL64_GCC + mov cl, A1_8 + %1 dword [A0], cl + %else + xchg A1, A0 + %1 dword [A1], cl + %endif + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + %ifdef ASM_CALL64_GCC + mov cl, A1_8 + %1 qword [A0], cl + %else + xchg A1, A0 + %1 qword [A1], cl + %endif + IEM_SAVE_FLAGS A2, %2, %3 + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64 + %endif ; RT_ARCH_AMD64 + +%endmacro + +IEMIMPL_SHIFT_OP rol, (X86_EFL_OF | X86_EFL_CF), 0 +IEMIMPL_SHIFT_OP ror, (X86_EFL_OF | X86_EFL_CF), 0 +IEMIMPL_SHIFT_OP rcl, (X86_EFL_OF | X86_EFL_CF), 0 +IEMIMPL_SHIFT_OP rcr, (X86_EFL_OF | X86_EFL_CF), 0 +IEMIMPL_SHIFT_OP shl, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF) +IEMIMPL_SHIFT_OP shr, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF) +IEMIMPL_SHIFT_OP sar, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF) + + +;; +; Macro for implementing a double precision shift operation. +; +; This will generate code for the 16, 32 and 64 bit accesses, except on +; 32-bit system where the 64-bit accesses requires hand coding. +; +; The functions takes the destination operand (r/m) in A0, the source (reg) in +; A1, the shift count in A2 and a pointer to the eflags variable/register in A3. +; +; @param 1 The instruction mnemonic. +; @param 2 The modified flags. +; @param 3 The undefined flags. +; +; Makes ASSUMPTIONS about A0, A1, A2 and A3 assignments. +; +; @note the _intel and _amd variants are implemented in C. +; +%macro IEMIMPL_SHIFT_DBL_OP 3 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 16 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + %ifdef ASM_CALL64_GCC + xchg A3, A2 + %1 [A0], A1_16, cl + xchg A3, A2 + %else + xchg A0, A2 + %1 [A2], A1_16, cl + %endif + IEM_SAVE_FLAGS A3, %2, %3 + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + %ifdef ASM_CALL64_GCC + xchg A3, A2 + %1 [A0], A1_32, cl + xchg A3, A2 + %else + xchg A0, A2 + %1 [A2], A1_32, cl + %endif + IEM_SAVE_FLAGS A3, %2, %3 + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 + + %ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 20 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + %ifdef ASM_CALL64_GCC + xchg A3, A2 + %1 [A0], A1, cl + xchg A3, A2 + %else + xchg A0, A2 + %1 [A2], A1, cl + %endif + IEM_SAVE_FLAGS A3, %2, %3 + EPILOGUE_4_ARGS_EX 12 +ENDPROC iemAImpl_ %+ %1 %+ _u64 + %endif ; RT_ARCH_AMD64 + +%endmacro + +IEMIMPL_SHIFT_DBL_OP shld, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF) +IEMIMPL_SHIFT_DBL_OP shrd, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF) + + +;; +; Macro for implementing a multiplication operations. +; +; This will generate code for the 8, 16, 32 and 64 bit accesses, except on +; 32-bit system where the 64-bit accesses requires hand coding. +; +; The 8-bit function only operates on AX, so it takes no DX pointer. The other +; functions takes a pointer to rAX in A0, rDX in A1, the operand in A2 and a +; pointer to eflags in A3. +; +; The functions all return 0 so the caller can be used for div/idiv as well as +; for the mul/imul implementation. +; +; @param 1 The instruction mnemonic. +; @param 2 The modified flags. +; @param 3 The undefined flags. +; @param 4 Name suffix. +; @param 5 EFLAGS behaviour: 0 for native, 1 for intel and 2 for AMD. +; +; Makes ASSUMPTIONS about A0, A1, A2, A3, T0 and T1 assignments. +; +%macro IEMIMPL_MUL_OP 5 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8 %+ %4, 12 + PROLOGUE_3_ARGS + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + mov al, [A0] + %1 A1_8 + mov [A0], ax + %if %5 != 1 + IEM_SAVE_FLAGS A2, %2, %3 + %else + IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF A2, %2, X86_EFL_AF | X86_EFL_ZF, ax, 8, xAX + %endif + xor eax, eax + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u8 %+ %4 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ %4, 16 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + mov ax, [A0] + %ifdef ASM_CALL64_GCC + %1 A2_16 + mov [A0], ax + mov [A1], dx + %else + mov T1, A1 + %1 A2_16 + mov [A0], ax + mov [T1], dx + %endif + %if %5 != 1 + IEM_SAVE_FLAGS A3, %2, %3 + %else + IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF A3, %2, X86_EFL_AF | X86_EFL_ZF, ax, 16, xAX + %endif + xor eax, eax + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u16 %+ %4 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ %4, 16 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + mov eax, [A0] + %ifdef ASM_CALL64_GCC + %1 A2_32 + mov [A0], eax + mov [A1], edx + %else + mov T1, A1 + %1 A2_32 + mov [A0], eax + mov [T1], edx + %endif + %if %5 != 1 + IEM_SAVE_FLAGS A3, %2, %3 + %else + IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF A3, %2, X86_EFL_AF | X86_EFL_ZF, eax, 32, xAX + %endif + xor eax, eax + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u32 %+ %4 + + %ifdef RT_ARCH_AMD64 ; The 32-bit host version lives in IEMAllAImplC.cpp. +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ %4, 20 + PROLOGUE_4_ARGS + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + mov rax, [A0] + %ifdef ASM_CALL64_GCC + %1 A2 + mov [A0], rax + mov [A1], rdx + %else + mov T1, A1 + %1 A2 + mov [A0], rax + mov [T1], rdx + %endif + %if %5 != 1 + IEM_SAVE_FLAGS A3, %2, %3 + %else + IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF A3, %2, X86_EFL_AF | X86_EFL_ZF, rax, 64, xAX + %endif + xor eax, eax + EPILOGUE_4_ARGS_EX 12 +ENDPROC iemAImpl_ %+ %1 %+ _u64 %+ %4 + %endif ; !RT_ARCH_AMD64 + +%endmacro + +IEMIMPL_MUL_OP mul, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), , 0 +IEMIMPL_MUL_OP mul, (X86_EFL_OF | X86_EFL_CF), 0, _intel, 1 +IEMIMPL_MUL_OP mul, (X86_EFL_OF | X86_EFL_CF), 0, _amd, 2 +IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), , 0 +IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), 0, _intel, 1 +IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), 0, _amd, 2 + + +BEGINCODE +;; +; Worker function for negating a 32-bit number in T1:T0 +; @uses None (T0,T1) +BEGINPROC iemAImpl_negate_T0_T1_u32 + push 0 + push 0 + xchg T0_32, [xSP] + xchg T1_32, [xSP + xCB] + sub T0_32, [xSP] + sbb T1_32, [xSP + xCB] + add xSP, xCB*2 + ret +ENDPROC iemAImpl_negate_T0_T1_u32 + +%ifdef RT_ARCH_AMD64 +;; +; Worker function for negating a 64-bit number in T1:T0 +; @uses None (T0,T1) +BEGINPROC iemAImpl_negate_T0_T1_u64 + push 0 + push 0 + xchg T0, [xSP] + xchg T1, [xSP + xCB] + sub T0, [xSP] + sbb T1, [xSP + xCB] + add xSP, xCB*2 + ret +ENDPROC iemAImpl_negate_T0_T1_u64 +%endif + + +;; +; Macro for implementing a division operations. +; +; This will generate code for the 8, 16, 32 and 64 bit accesses, except on +; 32-bit system where the 64-bit accesses requires hand coding. +; +; The 8-bit function only operates on AX, so it takes no DX pointer. The other +; functions takes a pointer to rAX in A0, rDX in A1, the operand in A2 and a +; pointer to eflags in A3. +; +; The functions all return 0 on success and -1 if a divide error should be +; raised by the caller. +; +; @param 1 The instruction mnemonic. +; @param 2 The modified flags. +; @param 3 The undefined flags. +; @param 4 1 if signed, 0 if unsigned. +; @param 5 Function suffix. +; @param 6 EFLAGS variation: 0 for native, 1 for intel (ignored), +; 2 for AMD (set AF, clear PF, ZF and SF). +; +; Makes ASSUMPTIONS about A0, A1, A2, A3, T0 and T1 assignments. +; +%macro IEMIMPL_DIV_OP 6 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8 %+ %5, 12 + PROLOGUE_3_ARGS + + ; div by chainsaw check. + test A1_8, A1_8 + jz .div_zero + + ; Overflow check - unsigned division is simple to verify, haven't + ; found a simple way to check signed division yet unfortunately. + %if %4 == 0 + cmp [A0 + 1], A1_8 + jae .div_overflow + %else + mov T0_16, [A0] ; T0 = dividend + mov T1, A1 ; T1 = saved divisor (because of missing T1_8 in 32-bit) + test A1_8, A1_8 + js .divisor_negative + test T0_16, T0_16 + jns .both_positive + neg T0_16 +.one_of_each: ; OK range is 2^(result-with - 1) + (divisor - 1). + push T0 ; Start off like unsigned below. + shr T0_16, 7 + cmp T0_8, A1_8 + pop T0 + jb .div_no_overflow + ja .div_overflow + and T0_8, 0x7f ; Special case for covering (divisor - 1). + cmp T0_8, A1_8 + jae .div_overflow + jmp .div_no_overflow + +.divisor_negative: + neg A1_8 + test T0_16, T0_16 + jns .one_of_each + neg T0_16 +.both_positive: ; Same as unsigned shifted by sign indicator bit. + shr T0_16, 7 + cmp T0_8, A1_8 + jae .div_overflow +.div_no_overflow: + mov A1, T1 ; restore divisor + %endif + + IEM_MAYBE_LOAD_FLAGS A2, %2, %3 + mov ax, [A0] + %1 A1_8 + mov [A0], ax + %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF. + IEM_ADJUST_FLAGS A2, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF + %else + IEM_SAVE_FLAGS A2, %2, %3 + %endif + xor eax, eax + +.return: + EPILOGUE_3_ARGS + +.div_zero: +.div_overflow: + mov eax, -1 + jmp .return +ENDPROC iemAImpl_ %+ %1 %+ _u8 %+ %5 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ %5, 16 + PROLOGUE_4_ARGS + + ; div by chainsaw check. + test A2_16, A2_16 + jz .div_zero + + ; Overflow check - unsigned division is simple to verify, haven't + ; found a simple way to check signed division yet unfortunately. + %if %4 == 0 + cmp [A1], A2_16 + jae .div_overflow + %else + mov T0_16, [A1] + shl T0_32, 16 + mov T0_16, [A0] ; T0 = dividend + mov T1, A2 ; T1 = divisor + test T1_16, T1_16 + js .divisor_negative + test T0_32, T0_32 + jns .both_positive + neg T0_32 +.one_of_each: ; OK range is 2^(result-with - 1) + (divisor - 1). + push T0 ; Start off like unsigned below. + shr T0_32, 15 + cmp T0_16, T1_16 + pop T0 + jb .div_no_overflow + ja .div_overflow + and T0_16, 0x7fff ; Special case for covering (divisor - 1). + cmp T0_16, T1_16 + jae .div_overflow + jmp .div_no_overflow + +.divisor_negative: + neg T1_16 + test T0_32, T0_32 + jns .one_of_each + neg T0_32 +.both_positive: ; Same as unsigned shifted by sign indicator bit. + shr T0_32, 15 + cmp T0_16, T1_16 + jae .div_overflow +.div_no_overflow: + %endif + + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + %ifdef ASM_CALL64_GCC + mov T1, A2 + mov ax, [A0] + mov dx, [A1] + %1 T1_16 + mov [A0], ax + mov [A1], dx + %else + mov T1, A1 + mov ax, [A0] + mov dx, [T1] + %1 A2_16 + mov [A0], ax + mov [T1], dx + %endif + %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF. + IEM_ADJUST_FLAGS A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF + %else + IEM_SAVE_FLAGS A3, %2, %3 + %endif + xor eax, eax + +.return: + EPILOGUE_4_ARGS + +.div_zero: +.div_overflow: + mov eax, -1 + jmp .return +ENDPROC iemAImpl_ %+ %1 %+ _u16 %+ %5 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ %5, 16 + PROLOGUE_4_ARGS + + ; div by chainsaw check. + test A2_32, A2_32 + jz .div_zero + + ; Overflow check - unsigned division is simple to verify, haven't + ; found a simple way to check signed division yet unfortunately. + %if %4 == 0 + cmp [A1], A2_32 + jae .div_overflow + %else + push A2 ; save A2 so we modify it (we out of regs on x86). + mov T0_32, [A0] ; T0 = dividend low + mov T1_32, [A1] ; T1 = dividend high + test A2_32, A2_32 + js .divisor_negative + test T1_32, T1_32 + jns .both_positive + call NAME(iemAImpl_negate_T0_T1_u32) +.one_of_each: ; OK range is 2^(result-with - 1) + (divisor - 1). + push T0 ; Start off like unsigned below. + shl T1_32, 1 + shr T0_32, 31 + or T1_32, T0_32 + cmp T1_32, A2_32 + pop T0 + jb .div_no_overflow + ja .div_overflow + and T0_32, 0x7fffffff ; Special case for covering (divisor - 1). + cmp T0_32, A2_32 + jae .div_overflow + jmp .div_no_overflow + +.divisor_negative: + neg A2_32 + test T1_32, T1_32 + jns .one_of_each + call NAME(iemAImpl_negate_T0_T1_u32) +.both_positive: ; Same as unsigned shifted by sign indicator bit. + shl T1_32, 1 + shr T0_32, 31 + or T1_32, T0_32 + cmp T1_32, A2_32 + jae .div_overflow +.div_no_overflow: + pop A2 + %endif + + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + mov eax, [A0] + %ifdef ASM_CALL64_GCC + mov T1, A2 + mov eax, [A0] + mov edx, [A1] + %1 T1_32 + mov [A0], eax + mov [A1], edx + %else + mov T1, A1 + mov eax, [A0] + mov edx, [T1] + %1 A2_32 + mov [A0], eax + mov [T1], edx + %endif + %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF. + IEM_ADJUST_FLAGS A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF + %else + IEM_SAVE_FLAGS A3, %2, %3 + %endif + xor eax, eax + +.return: + EPILOGUE_4_ARGS + +.div_overflow: + %if %4 != 0 + pop A2 + %endif +.div_zero: + mov eax, -1 + jmp .return +ENDPROC iemAImpl_ %+ %1 %+ _u32 %+ %5 + + %ifdef RT_ARCH_AMD64 ; The 32-bit host version lives in IEMAllAImplC.cpp. +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ %5, 20 + PROLOGUE_4_ARGS + + test A2, A2 + jz .div_zero + %if %4 == 0 + cmp [A1], A2 + jae .div_overflow + %else + push A2 ; save A2 so we modify it (we out of regs on x86). + mov T0, [A0] ; T0 = dividend low + mov T1, [A1] ; T1 = dividend high + test A2, A2 + js .divisor_negative + test T1, T1 + jns .both_positive + call NAME(iemAImpl_negate_T0_T1_u64) +.one_of_each: ; OK range is 2^(result-with - 1) + (divisor - 1). + push T0 ; Start off like unsigned below. + shl T1, 1 + shr T0, 63 + or T1, T0 + cmp T1, A2 + pop T0 + jb .div_no_overflow + ja .div_overflow + mov T1, 0x7fffffffffffffff + and T0, T1 ; Special case for covering (divisor - 1). + cmp T0, A2 + jae .div_overflow + jmp .div_no_overflow + +.divisor_negative: + neg A2 + test T1, T1 + jns .one_of_each + call NAME(iemAImpl_negate_T0_T1_u64) +.both_positive: ; Same as unsigned shifted by sign indicator bit. + shl T1, 1 + shr T0, 63 + or T1, T0 + cmp T1, A2 + jae .div_overflow +.div_no_overflow: + pop A2 + %endif + + IEM_MAYBE_LOAD_FLAGS A3, %2, %3 + mov rax, [A0] + %ifdef ASM_CALL64_GCC + mov T1, A2 + mov rax, [A0] + mov rdx, [A1] + %1 T1 + mov [A0], rax + mov [A1], rdx + %else + mov T1, A1 + mov rax, [A0] + mov rdx, [T1] + %1 A2 + mov [A0], rax + mov [T1], rdx + %endif + %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF. + IEM_ADJUST_FLAGS A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF + %else + IEM_SAVE_FLAGS A3, %2, %3 + %endif + xor eax, eax + +.return: + EPILOGUE_4_ARGS_EX 12 + +.div_overflow: + %if %4 != 0 + pop A2 + %endif +.div_zero: + mov eax, -1 + jmp .return +ENDPROC iemAImpl_ %+ %1 %+ _u64 %+ %5 + %endif ; !RT_ARCH_AMD64 + +%endmacro + +IEMIMPL_DIV_OP div, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0, , 0 +IEMIMPL_DIV_OP div, 0, 0, 0, _intel, 1 +IEMIMPL_DIV_OP div, 0, 0, 0, _amd, 2 +IEMIMPL_DIV_OP idiv, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1, , 0 +IEMIMPL_DIV_OP idiv, 0, 0, 1, _intel, 1 +IEMIMPL_DIV_OP idiv, 0, 0, 1, _amd, 2 + + +;; +; Macro for implementing memory fence operation. +; +; No return value, no operands or anything. +; +; @param 1 The instruction. +; +%macro IEMIMPL_MEM_FENCE 1 +BEGINCODE +BEGINPROC_FASTCALL iemAImpl_ %+ %1, 0 + %1 + ret +ENDPROC iemAImpl_ %+ %1 +%endmacro + +IEMIMPL_MEM_FENCE lfence +IEMIMPL_MEM_FENCE sfence +IEMIMPL_MEM_FENCE mfence + +;; +; Alternative for non-SSE2 host. +; +BEGINPROC_FASTCALL iemAImpl_alt_mem_fence, 0 + push xAX + xchg xAX, [xSP] + add xSP, xCB + ret +ENDPROC iemAImpl_alt_mem_fence + + +;; +; Initialize the FPU for the actual instruction being emulated, this means +; loading parts of the guest's control word and status word. +; +; @uses 24 bytes of stack. T0, T1 +; @param 1 Expression giving the address of the FXSTATE of the guest. +; +%macro FPU_LD_FXSTATE_FCW_AND_SAFE_FSW 1 + fnstenv [xSP] + + ; FCW - for exception, precision and rounding control. + movzx T0, word [%1 + X86FXSTATE.FCW] + and T0, X86_FCW_MASK_ALL | X86_FCW_PC_MASK | X86_FCW_RC_MASK + mov [xSP + X86FSTENV32P.FCW], T0_16 + + ; FSW - for undefined C0, C1, C2, and C3. + movzx T1, word [%1 + X86FXSTATE.FSW] + and T1, X86_FSW_C_MASK + movzx T0, word [xSP + X86FSTENV32P.FSW] + and T0, X86_FSW_TOP_MASK + or T0, T1 + mov [xSP + X86FSTENV32P.FSW], T0_16 + + fldenv [xSP] +%endmacro + + +;; +; Initialize the FPU for the actual instruction being emulated, this means +; loading parts of the guest's control word, status word, and update the +; tag word for the top register if it's empty. +; +; ASSUMES actual TOP=7 +; +; @uses 24 bytes of stack. T0, T1 +; @param 1 Expression giving the address of the FXSTATE of the guest. +; +%macro FPU_LD_FXSTATE_FCW_AND_SAFE_FSW_AND_FTW_0 1 + fnstenv [xSP] + + ; FCW - for exception, precision and rounding control. + movzx T0_32, word [%1 + X86FXSTATE.FCW] + and T0_32, X86_FCW_MASK_ALL | X86_FCW_PC_MASK | X86_FCW_RC_MASK + mov [xSP + X86FSTENV32P.FCW], T0_16 + + ; FSW - for undefined C0, C1, C2, and C3. + movzx T1_32, word [%1 + X86FXSTATE.FSW] + and T1_32, X86_FSW_C_MASK + movzx T0_32, word [xSP + X86FSTENV32P.FSW] + and T0_32, X86_FSW_TOP_MASK + or T0_32, T1_32 + mov [xSP + X86FSTENV32P.FSW], T0_16 + + ; FTW - Only for ST0 (in/out). + movzx T1_32, word [%1 + X86FXSTATE.FSW] + shr T1_32, X86_FSW_TOP_SHIFT + and T1_32, X86_FSW_TOP_SMASK + bt [%1 + X86FXSTATE.FTW], T1_16 ; Empty if FTW bit is clear. Fixed register order. + jc %%st0_not_empty + or word [xSP + X86FSTENV32P.FTW], 0c000h ; TOP=7, so set TAG(7)=3 +%%st0_not_empty: + + fldenv [xSP] +%endmacro + + +;; +; Need to move this as well somewhere better? +; +struc IEMFPURESULT + .r80Result resw 5 + .FSW resw 1 +endstruc + + +;; +; Need to move this as well somewhere better? +; +struc IEMFPURESULTTWO + .r80Result1 resw 5 + .FSW resw 1 + .r80Result2 resw 5 +endstruc + + +; +;---------------------- 16-bit signed integer operations ---------------------- +; + + +;; +; Converts a 16-bit floating point value to a 80-bit one (fpu register). +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 16-bit floating point value to convert. +; +BEGINPROC_FASTCALL iemAImpl_fild_r80_from_i16, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fild word [A2] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_fild_r80_from_i16 + + +;; +; Store a 80-bit floating point value (register) as a 16-bit signed integer (memory). +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 16-bit signed integer value. +; @param A3 Pointer to the 80-bit value. +; +BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i16, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fistp word [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fist_r80_to_i16 + + +;; +; Store a 80-bit floating point value (register) as a 16-bit signed integer +; (memory) with truncation. +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 16-bit signed integer value. +; @param A3 Pointer to the 80-bit value. +; +BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i16, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fisttp word [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fistt_r80_to_i16 + + +;; +; FPU instruction working on one 80-bit and one 16-bit signed integer value. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 80-bit value. +; @param A3 Pointer to the 16-bit value. +; +%macro IEMIMPL_FPU_R80_BY_I16 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i16, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 word [A3] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i16 +%endmacro + +IEMIMPL_FPU_R80_BY_I16 fiadd +IEMIMPL_FPU_R80_BY_I16 fimul +IEMIMPL_FPU_R80_BY_I16 fisub +IEMIMPL_FPU_R80_BY_I16 fisubr +IEMIMPL_FPU_R80_BY_I16 fidiv +IEMIMPL_FPU_R80_BY_I16 fidivr + + +;; +; FPU instruction working on one 80-bit and one 16-bit signed integer value, +; only returning FSW. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Where to store the output FSW. +; @param A2 Pointer to the 80-bit value. +; @param A3 Pointer to the 64-bit value. +; +%macro IEMIMPL_FPU_R80_BY_I16_FSW 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i16, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 word [A3] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i16 +%endmacro + +IEMIMPL_FPU_R80_BY_I16_FSW ficom + + + +; +;---------------------- 32-bit signed integer operations ---------------------- +; + + +;; +; Converts a 32-bit floating point value to a 80-bit one (fpu register). +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 32-bit floating point value to convert. +; +BEGINPROC_FASTCALL iemAImpl_fild_r80_from_i32, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fild dword [A2] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_fild_r80_from_i32 + + +;; +; Store a 80-bit floating point value (register) as a 32-bit signed integer (memory). +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 32-bit signed integer value. +; @param A3 Pointer to the 80-bit value. +; +BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i32, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fistp dword [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fist_r80_to_i32 + + +;; +; Store a 80-bit floating point value (register) as a 32-bit signed integer +; (memory) with truncation. +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 32-bit signed integer value. +; @param A3 Pointer to the 80-bit value. +; +BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i32, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fisttp dword [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fistt_r80_to_i32 + + +;; +; FPU instruction working on one 80-bit and one 32-bit signed integer value. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 80-bit value. +; @param A3 Pointer to the 32-bit value. +; +%macro IEMIMPL_FPU_R80_BY_I32 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i32, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 dword [A3] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i32 +%endmacro + +IEMIMPL_FPU_R80_BY_I32 fiadd +IEMIMPL_FPU_R80_BY_I32 fimul +IEMIMPL_FPU_R80_BY_I32 fisub +IEMIMPL_FPU_R80_BY_I32 fisubr +IEMIMPL_FPU_R80_BY_I32 fidiv +IEMIMPL_FPU_R80_BY_I32 fidivr + + +;; +; FPU instruction working on one 80-bit and one 32-bit signed integer value, +; only returning FSW. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Where to store the output FSW. +; @param A2 Pointer to the 80-bit value. +; @param A3 Pointer to the 64-bit value. +; +%macro IEMIMPL_FPU_R80_BY_I32_FSW 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i32, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 dword [A3] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i32 +%endmacro + +IEMIMPL_FPU_R80_BY_I32_FSW ficom + + + +; +;---------------------- 64-bit signed integer operations ---------------------- +; + + +;; +; Converts a 64-bit floating point value to a 80-bit one (fpu register). +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 64-bit floating point value to convert. +; +BEGINPROC_FASTCALL iemAImpl_fild_r80_from_i64, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fild qword [A2] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_fild_r80_from_i64 + + +;; +; Store a 80-bit floating point value (register) as a 64-bit signed integer (memory). +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 64-bit signed integer value. +; @param A3 Pointer to the 80-bit value. +; +BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i64, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fistp qword [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fist_r80_to_i64 + + +;; +; Store a 80-bit floating point value (register) as a 64-bit signed integer +; (memory) with truncation. +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 64-bit signed integer value. +; @param A3 Pointer to the 80-bit value. +; +BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i64, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fisttp qword [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fistt_r80_to_i64 + + + +; +;---------------------- 32-bit floating point operations ---------------------- +; + +;; +; Converts a 32-bit floating point value to a 80-bit one (fpu register). +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 32-bit floating point value to convert. +; +BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r32, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fld dword [A2] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_fld_r80_from_r32 + + +;; +; Store a 80-bit floating point value (register) as a 32-bit one (memory). +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 32-bit value. +; @param A3 Pointer to the 80-bit value. +; +BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r32, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fst dword [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fst_r80_to_r32 + + +;; +; FPU instruction working on one 80-bit and one 32-bit floating point value. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 80-bit value. +; @param A3 Pointer to the 32-bit value. +; +%macro IEMIMPL_FPU_R80_BY_R32 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r32, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 dword [A3] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r32 +%endmacro + +IEMIMPL_FPU_R80_BY_R32 fadd +IEMIMPL_FPU_R80_BY_R32 fmul +IEMIMPL_FPU_R80_BY_R32 fsub +IEMIMPL_FPU_R80_BY_R32 fsubr +IEMIMPL_FPU_R80_BY_R32 fdiv +IEMIMPL_FPU_R80_BY_R32 fdivr + + +;; +; FPU instruction working on one 80-bit and one 32-bit floating point value, +; only returning FSW. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Where to store the output FSW. +; @param A2 Pointer to the 80-bit value. +; @param A3 Pointer to the 64-bit value. +; +%macro IEMIMPL_FPU_R80_BY_R32_FSW 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r32, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 dword [A3] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r32 +%endmacro + +IEMIMPL_FPU_R80_BY_R32_FSW fcom + + + +; +;---------------------- 64-bit floating point operations ---------------------- +; + +;; +; Converts a 64-bit floating point value to a 80-bit one (fpu register). +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 64-bit floating point value to convert. +; +BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r64, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fld qword [A2] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_fld_r80_from_r64 + + +;; +; Store a 80-bit floating point value (register) as a 64-bit one (memory). +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 64-bit value. +; @param A3 Pointer to the 80-bit value. +; +BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r64, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fst qword [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fst_r80_to_r64 + + +;; +; FPU instruction working on one 80-bit and one 64-bit floating point value. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 80-bit value. +; @param A3 Pointer to the 64-bit value. +; +%macro IEMIMPL_FPU_R80_BY_R64 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r64, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 qword [A3] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r64 +%endmacro + +IEMIMPL_FPU_R80_BY_R64 fadd +IEMIMPL_FPU_R80_BY_R64 fmul +IEMIMPL_FPU_R80_BY_R64 fsub +IEMIMPL_FPU_R80_BY_R64 fsubr +IEMIMPL_FPU_R80_BY_R64 fdiv +IEMIMPL_FPU_R80_BY_R64 fdivr + +;; +; FPU instruction working on one 80-bit and one 64-bit floating point value, +; only returning FSW. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Where to store the output FSW. +; @param A2 Pointer to the 80-bit value. +; @param A3 Pointer to the 64-bit value. +; +%macro IEMIMPL_FPU_R80_BY_R64_FSW 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r64, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 qword [A3] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r64 +%endmacro + +IEMIMPL_FPU_R80_BY_R64_FSW fcom + + + +; +;---------------------- 80-bit floating point operations ---------------------- +; + +;; +; Loads a 80-bit floating point register value from memory. +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 80-bit floating point value to load. +; +BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r80, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fld tword [A2] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_fld_r80_from_r80 + + +;; +; Store a 80-bit floating point register to memory +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 80-bit value. +; @param A3 Pointer to the 80-bit register value. +; +BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r80, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fstp tword [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fst_r80_to_r80 + + +;; +; Loads an 80-bit floating point register value in BCD format from memory. +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 80-bit BCD value to load. +; +BEGINPROC_FASTCALL iemAImpl_fld_r80_from_d80, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fbld tword [A2] + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_fld_r80_from_d80 + + +;; +; Store a 80-bit floating point register to memory as BCD +; +; @param A0 FPU context (fxsave). +; @param A1 Where to return the output FSW. +; @param A2 Where to store the 80-bit BCD value. +; @param A3 Pointer to the 80-bit register value. +; +BEGINPROC_FASTCALL iemAImpl_fst_r80_to_d80, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + fbstp tword [A2] + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_fst_r80_to_d80 + + +;; +; FPU instruction working on two 80-bit floating point values. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the first 80-bit value (ST0) +; @param A3 Pointer to the second 80-bit value (STn). +; +%macro IEMIMPL_FPU_R80_BY_R80 2 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 %2 + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80 +%endmacro + +IEMIMPL_FPU_R80_BY_R80 fadd, {st0, st1} +IEMIMPL_FPU_R80_BY_R80 fmul, {st0, st1} +IEMIMPL_FPU_R80_BY_R80 fsub, {st0, st1} +IEMIMPL_FPU_R80_BY_R80 fsubr, {st0, st1} +IEMIMPL_FPU_R80_BY_R80 fdiv, {st0, st1} +IEMIMPL_FPU_R80_BY_R80 fdivr, {st0, st1} +IEMIMPL_FPU_R80_BY_R80 fprem, {} +IEMIMPL_FPU_R80_BY_R80 fprem1, {} +IEMIMPL_FPU_R80_BY_R80 fscale, {} + + +;; +; FPU instruction working on two 80-bit floating point values, ST1 and ST0, +; storing the result in ST1 and popping the stack. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the first 80-bit value (ST1). +; @param A3 Pointer to the second 80-bit value (ST0). +; +%macro IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + fld tword [A3] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80 +%endmacro + +IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fpatan +IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fyl2x +IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fyl2xp1 + + +;; +; FPU instruction working on two 80-bit floating point values, only +; returning FSW. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a uint16_t for the resulting FSW. +; @param A2 Pointer to the first 80-bit value. +; @param A3 Pointer to the second 80-bit value. +; +%macro IEMIMPL_FPU_R80_BY_R80_FSW 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 st0, st1 + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80 +%endmacro + +IEMIMPL_FPU_R80_BY_R80_FSW fcom +IEMIMPL_FPU_R80_BY_R80_FSW fucom + + +;; +; FPU instruction working on two 80-bit floating point values, +; returning FSW and EFLAGS (eax). +; +; @param 1 The instruction +; +; @returns EFLAGS in EAX. +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a uint16_t for the resulting FSW. +; @param A2 Pointer to the first 80-bit value. +; @param A3 Pointer to the second 80-bit value. +; +%macro IEMIMPL_FPU_R80_BY_R80_EFL 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16 + PROLOGUE_4_ARGS + sub xSP, 20h + + fninit + fld tword [A3] + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 st1 + + fnstsw word [A1] + pushf + pop xAX + + fninit + add xSP, 20h + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80 +%endmacro + +IEMIMPL_FPU_R80_BY_R80_EFL fcomi +IEMIMPL_FPU_R80_BY_R80_EFL fucomi + + +;; +; FPU instruction working on one 80-bit floating point value. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; @param A2 Pointer to the 80-bit value. +; +%macro IEMIMPL_FPU_R80 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80 +%endmacro + +IEMIMPL_FPU_R80 fchs +IEMIMPL_FPU_R80 fabs +IEMIMPL_FPU_R80 f2xm1 +IEMIMPL_FPU_R80 fsqrt +IEMIMPL_FPU_R80 frndint +IEMIMPL_FPU_R80 fsin +IEMIMPL_FPU_R80 fcos + + +;; +; FPU instruction working on one 80-bit floating point value, only +; returning FSW. +; +; @param 1 The instruction +; @param 2 Non-zero to also restore FTW. +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a uint16_t for the resulting FSW. +; @param A2 Pointer to the 80-bit value. +; +%macro IEMIMPL_FPU_R80_FSW 2 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + fld tword [A2] +%if %2 != 0 + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW_AND_FTW_0 A0 +%else + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 +%endif + %1 + + fnstsw word [A1] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80 +%endmacro + +IEMIMPL_FPU_R80_FSW ftst, 0 +IEMIMPL_FPU_R80_FSW fxam, 1 ; No #IS or any other FP exceptions. + + + +;; +; FPU instruction loading a 80-bit floating point constant. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULT for the output. +; +%macro IEMIMPL_FPU_R80_CONST 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1, 8 + PROLOGUE_2_ARGS + sub xSP, 20h + + fninit + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 + + fnstsw word [A1 + IEMFPURESULT.FSW] + fnclex + fstp tword [A1 + IEMFPURESULT.r80Result] + + fninit + add xSP, 20h + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ +%endmacro + +IEMIMPL_FPU_R80_CONST fld1 +IEMIMPL_FPU_R80_CONST fldl2t +IEMIMPL_FPU_R80_CONST fldl2e +IEMIMPL_FPU_R80_CONST fldpi +IEMIMPL_FPU_R80_CONST fldlg2 +IEMIMPL_FPU_R80_CONST fldln2 +IEMIMPL_FPU_R80_CONST fldz + + +;; +; FPU instruction working on one 80-bit floating point value, outputing two. +; +; @param 1 The instruction +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to a IEMFPURESULTTWO for the output. +; @param A2 Pointer to the 80-bit value. +; +%macro IEMIMPL_FPU_R80_R80 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_r80, 12 + PROLOGUE_3_ARGS + sub xSP, 20h + + fninit + fld tword [A2] + FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0 + %1 + + fnstsw word [A1 + IEMFPURESULTTWO.FSW] + fnclex + fstp tword [A1 + IEMFPURESULTTWO.r80Result2] + fnclex + fstp tword [A1 + IEMFPURESULTTWO.r80Result1] + + fninit + add xSP, 20h + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _r80_r80 +%endmacro + +IEMIMPL_FPU_R80_R80 fptan +IEMIMPL_FPU_R80_R80 fxtract +IEMIMPL_FPU_R80_R80 fsincos + + + + +;---------------------- SSE and MMX Operations ---------------------- + +;; @todo what do we need to do for MMX? +%macro IEMIMPL_MMX_PROLOGUE 0 +%endmacro +%macro IEMIMPL_MMX_EPILOGUE 0 +%endmacro + +;; @todo what do we need to do for SSE? +%macro IEMIMPL_SSE_PROLOGUE 0 +%endmacro +%macro IEMIMPL_SSE_EPILOGUE 0 +%endmacro + +;; @todo what do we need to do for AVX? +%macro IEMIMPL_AVX_PROLOGUE 0 +%endmacro +%macro IEMIMPL_AVX_EPILOGUE 0 +%endmacro + + +;; +; Media instruction working on two full sized registers. +; +; @param 1 The instruction +; @param 2 Whether there is an MMX variant (1) or not (0). +; +; @param A0 FPU context (fxsave). +; @param A1 Pointer to the first media register size operand (input/output). +; @param A2 Pointer to the second media register size operand (input). +; +%macro IEMIMPL_MEDIA_F2 2 +%if %2 != 0 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12 + PROLOGUE_3_ARGS + IEMIMPL_MMX_PROLOGUE + + movq mm0, [A1] + movq mm1, [A2] + %1 mm0, mm1 + movq [A1], mm0 + + IEMIMPL_MMX_EPILOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64 +%endif + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A1] + movdqu xmm1, [A2] + %1 xmm0, xmm1 + movdqu [A1], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_F2 pshufb, 1 +IEMIMPL_MEDIA_F2 pand, 1 +IEMIMPL_MEDIA_F2 pandn, 1 +IEMIMPL_MEDIA_F2 por, 1 +IEMIMPL_MEDIA_F2 pxor, 1 +IEMIMPL_MEDIA_F2 pcmpeqb, 1 +IEMIMPL_MEDIA_F2 pcmpeqw, 1 +IEMIMPL_MEDIA_F2 pcmpeqd, 1 +IEMIMPL_MEDIA_F2 pcmpeqq, 0 +IEMIMPL_MEDIA_F2 pcmpgtb, 1 +IEMIMPL_MEDIA_F2 pcmpgtw, 1 +IEMIMPL_MEDIA_F2 pcmpgtd, 1 +IEMIMPL_MEDIA_F2 pcmpgtq, 0 +IEMIMPL_MEDIA_F2 paddb, 1 +IEMIMPL_MEDIA_F2 paddw, 1 +IEMIMPL_MEDIA_F2 paddd, 1 +IEMIMPL_MEDIA_F2 paddq, 1 +IEMIMPL_MEDIA_F2 paddsb, 1 +IEMIMPL_MEDIA_F2 paddsw, 1 +IEMIMPL_MEDIA_F2 paddusb, 1 +IEMIMPL_MEDIA_F2 paddusw, 1 +IEMIMPL_MEDIA_F2 psubb, 1 +IEMIMPL_MEDIA_F2 psubw, 1 +IEMIMPL_MEDIA_F2 psubd, 1 +IEMIMPL_MEDIA_F2 psubq, 1 +IEMIMPL_MEDIA_F2 psubsb, 1 +IEMIMPL_MEDIA_F2 psubsw, 1 +IEMIMPL_MEDIA_F2 psubusb, 1 +IEMIMPL_MEDIA_F2 psubusw, 1 +IEMIMPL_MEDIA_F2 pmullw, 1 +IEMIMPL_MEDIA_F2 pmulld, 0 +IEMIMPL_MEDIA_F2 pmulhw, 1 +IEMIMPL_MEDIA_F2 pmaddwd, 1 +IEMIMPL_MEDIA_F2 pminub, 1 +IEMIMPL_MEDIA_F2 pminuw, 0 +IEMIMPL_MEDIA_F2 pminud, 0 +IEMIMPL_MEDIA_F2 pminsb, 0 +IEMIMPL_MEDIA_F2 pminsw, 1 +IEMIMPL_MEDIA_F2 pminsd, 0 +IEMIMPL_MEDIA_F2 pmaxub, 1 +IEMIMPL_MEDIA_F2 pmaxuw, 0 +IEMIMPL_MEDIA_F2 pmaxud, 0 +IEMIMPL_MEDIA_F2 pmaxsb, 0 +IEMIMPL_MEDIA_F2 pmaxsw, 1 +IEMIMPL_MEDIA_F2 pmaxsd, 0 +IEMIMPL_MEDIA_F2 pabsb, 1 +IEMIMPL_MEDIA_F2 pabsw, 1 +IEMIMPL_MEDIA_F2 pabsd, 1 +IEMIMPL_MEDIA_F2 psignb, 1 +IEMIMPL_MEDIA_F2 psignw, 1 +IEMIMPL_MEDIA_F2 psignd, 1 +IEMIMPL_MEDIA_F2 phaddw, 1 +IEMIMPL_MEDIA_F2 phaddd, 1 +IEMIMPL_MEDIA_F2 phsubw, 1 +IEMIMPL_MEDIA_F2 phsubd, 1 +IEMIMPL_MEDIA_F2 phaddsw, 1 +IEMIMPL_MEDIA_F2 phsubsw, 1 +IEMIMPL_MEDIA_F2 pmaddubsw, 1 +IEMIMPL_MEDIA_F2 pmulhrsw, 1 +IEMIMPL_MEDIA_F2 pmuludq, 1 + + +;; +; Media instruction working on two full sized registers, but no FXSAVE state argument. +; +; @param 1 The instruction +; @param 2 Whether there is an MMX variant (1) or not (0). +; +; @param A0 Pointer to the first media register size operand (input/output). +; @param A1 Pointer to the second media register size operand (input). +; +%macro IEMIMPL_MEDIA_OPT_F2 2 +%if %2 != 0 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8 + PROLOGUE_2_ARGS + IEMIMPL_MMX_PROLOGUE + + movq mm0, [A0] + movq mm1, [A1] + %1 mm0, mm1 + movq [A0], mm0 + + IEMIMPL_MMX_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64 +%endif + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 8 + PROLOGUE_2_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A0] + movdqu xmm1, [A1] + %1 xmm0, xmm1 + movdqu [A0], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_OPT_F2 packsswb, 1 +IEMIMPL_MEDIA_OPT_F2 packssdw, 1 +IEMIMPL_MEDIA_OPT_F2 packuswb, 1 +IEMIMPL_MEDIA_OPT_F2 packusdw, 0 +IEMIMPL_MEDIA_OPT_F2 psllw, 1 +IEMIMPL_MEDIA_OPT_F2 pslld, 1 +IEMIMPL_MEDIA_OPT_F2 psllq, 1 +IEMIMPL_MEDIA_OPT_F2 psrlw, 1 +IEMIMPL_MEDIA_OPT_F2 psrld, 1 +IEMIMPL_MEDIA_OPT_F2 psrlq, 1 +IEMIMPL_MEDIA_OPT_F2 psraw, 1 +IEMIMPL_MEDIA_OPT_F2 psrad, 1 +IEMIMPL_MEDIA_OPT_F2 pmulhuw, 1 +IEMIMPL_MEDIA_OPT_F2 pavgb, 1 +IEMIMPL_MEDIA_OPT_F2 pavgw, 1 +IEMIMPL_MEDIA_OPT_F2 psadbw, 1 +IEMIMPL_MEDIA_OPT_F2 pmuldq, 0 +IEMIMPL_MEDIA_OPT_F2 unpcklps, 0 +IEMIMPL_MEDIA_OPT_F2 unpcklpd, 0 +IEMIMPL_MEDIA_OPT_F2 unpckhps, 0 +IEMIMPL_MEDIA_OPT_F2 unpckhpd, 0 +IEMIMPL_MEDIA_OPT_F2 phminposuw, 0 +IEMIMPL_MEDIA_OPT_F2 aesimc, 0 +IEMIMPL_MEDIA_OPT_F2 aesenc, 0 +IEMIMPL_MEDIA_OPT_F2 aesdec, 0 +IEMIMPL_MEDIA_OPT_F2 aesenclast, 0 +IEMIMPL_MEDIA_OPT_F2 aesdeclast, 0 + +;; +; Media instruction working on one full sized and one half sized register (lower half). +; +; @param 1 The instruction +; @param 2 1 if MMX is included, 0 if not. +; +; @param A0 Pointer to the first full sized media register operand (input/output). +; @param A1 Pointer to the second half sized media register operand (input). +; +%macro IEMIMPL_MEDIA_F1L1 2 + %if %2 != 0 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8 + PROLOGUE_2_ARGS + IEMIMPL_MMX_PROLOGUE + + movq mm0, [A0] + movq mm1, [A1] + %1 mm0, mm1 + movq [A0], mm0 + + IEMIMPL_MMX_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u64 + %endif + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 8 + PROLOGUE_2_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A0] + movdqu xmm1, [A1] + %1 xmm0, xmm1 + movdqu [A0], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_F1L1 punpcklbw, 1 +IEMIMPL_MEDIA_F1L1 punpcklwd, 1 +IEMIMPL_MEDIA_F1L1 punpckldq, 1 +IEMIMPL_MEDIA_F1L1 punpcklqdq, 0 + + +;; +; Media instruction working two half sized input registers (lower half) and a full sized +; destination register (vpunpckh*). +; +; @param 1 The instruction +; +; @param A0 Pointer to the destination register (full sized, output only). +; @param A1 Pointer to the first full sized media source register operand, where we +; will only use the lower half as input - but we'll be loading it in full. +; @param A2 Pointer to the second full sized media source register operand, where we +; will only use the lower half as input - but we'll be loading it in full. +; +%macro IEMIMPL_MEDIA_F1L1L1 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12 + PROLOGUE_3_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu xmm0, [A1] + vmovdqu xmm1, [A2] + %1 xmm0, xmm0, xmm1 + vmovdqu [A0], xmm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 12 + PROLOGUE_3_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu ymm0, [A1] + vmovdqu ymm1, [A2] + %1 ymm0, ymm0, ymm1 + vmovdqu [A0], ymm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u256 +%endmacro + +IEMIMPL_MEDIA_F1L1L1 vpunpcklbw +IEMIMPL_MEDIA_F1L1L1 vpunpcklwd +IEMIMPL_MEDIA_F1L1L1 vpunpckldq +IEMIMPL_MEDIA_F1L1L1 vpunpcklqdq + + +;; +; Media instruction working on one full sized and one half sized register (high half). +; +; @param 1 The instruction +; @param 2 1 if MMX is included, 0 if not. +; +; @param A0 Pointer to the first full sized media register operand (input/output). +; @param A1 Pointer to the second full sized media register operand, where we +; will only use the upper half as input - but we'll load it in full. +; +%macro IEMIMPL_MEDIA_F1H1 2 +IEMIMPL_MEDIA_F1L1 %1, %2 +%endmacro + +IEMIMPL_MEDIA_F1L1 punpckhbw, 1 +IEMIMPL_MEDIA_F1L1 punpckhwd, 1 +IEMIMPL_MEDIA_F1L1 punpckhdq, 1 +IEMIMPL_MEDIA_F1L1 punpckhqdq, 0 + + +;; +; Media instruction working two half sized input registers (high half) and a full sized +; destination register (vpunpckh*). +; +; @param 1 The instruction +; +; @param A0 Pointer to the destination register (full sized, output only). +; @param A1 Pointer to the first full sized media source register operand, where we +; will only use the upper half as input - but we'll be loading it in full. +; @param A2 Pointer to the second full sized media source register operand, where we +; will only use the upper half as input - but we'll be loading it in full. +; +%macro IEMIMPL_MEDIA_F1H1H1 1 +IEMIMPL_MEDIA_F1L1L1 %1 +%endmacro + +IEMIMPL_MEDIA_F1H1H1 vpunpckhbw +IEMIMPL_MEDIA_F1H1H1 vpunpckhwd +IEMIMPL_MEDIA_F1H1H1 vpunpckhdq +IEMIMPL_MEDIA_F1H1H1 vpunpckhqdq + + +; +; Shufflers with evil 8-bit immediates. +; + +BEGINPROC_FASTCALL iemAImpl_pshufw_u64, 16 + PROLOGUE_3_ARGS + IEMIMPL_MMX_PROLOGUE + + movq mm1, [A1] + movq mm0, mm0 ; paranoia! + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*8] ; sizeof(pshufw+ret) == 9 + %else + lea T0, [A2 + A2*4] ; sizeof(pshufw+ret) == 5 + %endif + lea T1, [T1 + T0] + IBT_NOTRACK + call T1 + movq [A0], mm0 + + IEMIMPL_MMX_EPILOGUE + EPILOGUE_3_ARGS +%assign bImm 0 +%rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + pshufw mm0, mm1, bImm + ret + %assign bImm bImm + 1 +%endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500 +ENDPROC iemAImpl_pshufw_u64 + + +%macro IEMIMPL_MEDIA_SSE_PSHUFXX 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm1, [A1] + movdqu xmm0, xmm1 ; paranoia! + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*4] ; sizeof(pshufXX+ret) == 10: A2 * 10 = (A2 * 5) * 2 + %else + lea T0, [A2 + A2*2] ; sizeof(pshufXX+ret) == 6: A2 * 6 = (A2 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + movdqu [A0], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 xmm0, xmm1, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_SSE_PSHUFXX pshufhw +IEMIMPL_MEDIA_SSE_PSHUFXX pshuflw +IEMIMPL_MEDIA_SSE_PSHUFXX pshufd + + +%macro IEMIMPL_MEDIA_AVX_VPSHUFXX 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + vmovdqu ymm1, [A1] + vmovdqu ymm0, ymm1 ; paranoia! + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*4] ; sizeof(pshufXX+ret) == 10: A2 * 10 = (A2 * 5) * 2 + %else + lea T0, [A2 + A2*2] ; sizeof(pshufXX+ret) == 6: A2 * 6 = (A2 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + vmovdqu [A0], ymm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 ymm0, ymm1, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_ %+ %1 %+ _u256 +%endmacro + +IEMIMPL_MEDIA_AVX_VPSHUFXX vpshufhw +IEMIMPL_MEDIA_AVX_VPSHUFXX vpshuflw +IEMIMPL_MEDIA_AVX_VPSHUFXX vpshufd + + +; +; Shifts with evil 8-bit immediates. +; + +%macro IEMIMPL_MEDIA_MMX_PSHIFTXX 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _imm_u64, 16 + PROLOGUE_2_ARGS + IEMIMPL_MMX_PROLOGUE + + movq mm0, [A0] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A1 + A1*8] ; sizeof(psXX+ret) == 9 + %else + lea T0, [A1 + A1*4] ; sizeof(psXX+ret) == 5 + %endif + lea T1, [T1 + T0] + IBT_NOTRACK + call T1 + movq [A0], mm0 + + IEMIMPL_MMX_EPILOGUE + EPILOGUE_2_ARGS +%assign bImm 0 +%rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 mm0, bImm + ret + %assign bImm bImm + 1 +%endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500 +ENDPROC iemAImpl_ %+ %1 %+ _imm_u64 +%endmacro + +IEMIMPL_MEDIA_MMX_PSHIFTXX psllw +IEMIMPL_MEDIA_MMX_PSHIFTXX pslld +IEMIMPL_MEDIA_MMX_PSHIFTXX psllq +IEMIMPL_MEDIA_MMX_PSHIFTXX psrlw +IEMIMPL_MEDIA_MMX_PSHIFTXX psrld +IEMIMPL_MEDIA_MMX_PSHIFTXX psrlq +IEMIMPL_MEDIA_MMX_PSHIFTXX psraw +IEMIMPL_MEDIA_MMX_PSHIFTXX psrad + + +%macro IEMIMPL_MEDIA_SSE_PSHIFTXX 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _imm_u128, 16 + PROLOGUE_2_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A0] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A1 + A1*4] ; sizeof(psXX+ret) == 10: A1 * 10 = (A1 * 5) * 2 + %else + lea T0, [A1 + A1*2] ; sizeof(psXX+ret) == 6: A1 * 6 = (A1 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + movdqu [A0], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_2_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 xmm0, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_ %+ %1 %+ _imm_u128 +%endmacro + +IEMIMPL_MEDIA_SSE_PSHIFTXX psllw +IEMIMPL_MEDIA_SSE_PSHIFTXX pslld +IEMIMPL_MEDIA_SSE_PSHIFTXX psllq +IEMIMPL_MEDIA_SSE_PSHIFTXX psrlw +IEMIMPL_MEDIA_SSE_PSHIFTXX psrld +IEMIMPL_MEDIA_SSE_PSHIFTXX psrlq +IEMIMPL_MEDIA_SSE_PSHIFTXX psraw +IEMIMPL_MEDIA_SSE_PSHIFTXX psrad +IEMIMPL_MEDIA_SSE_PSHIFTXX pslldq +IEMIMPL_MEDIA_SSE_PSHIFTXX psrldq + + +; +; Move byte mask. +; + +BEGINPROC_FASTCALL iemAImpl_pmovmskb_u64, 8 + PROLOGUE_2_ARGS + IEMIMPL_MMX_PROLOGUE + + movq mm1, [A1] + pmovmskb T0, mm1 + mov [A0], T0 +%ifdef RT_ARCH_X86 + mov dword [A0 + 4], 0 +%endif + IEMIMPL_MMX_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_pmovmskb_u64 + +BEGINPROC_FASTCALL iemAImpl_pmovmskb_u128, 8 + PROLOGUE_2_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm1, [A1] + pmovmskb T0, xmm1 + mov [A0], T0 +%ifdef RT_ARCH_X86 + mov dword [A0 + 4], 0 +%endif + IEMIMPL_SSE_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_pmovmskb_u128 + +BEGINPROC_FASTCALL iemAImpl_vpmovmskb_u256, 8 + PROLOGUE_2_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu ymm1, [A1] + vpmovmskb T0, ymm1 + mov [A0], T0 +%ifdef RT_ARCH_X86 + mov dword [A0 + 4], 0 +%endif + IEMIMPL_AVX_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_vpmovmskb_u256 + + +;; +; Media instruction working on two full sized source registers and one destination (AVX). +; +; @param 1 The instruction +; +; @param A0 Pointer to the extended CPU/FPU state (X86XSAVEAREA). +; @param A1 Pointer to the destination media register size operand (output). +; @param A2 Pointer to the first source media register size operand (input). +; @param A3 Pointer to the second source media register size operand (input). +; +%macro IEMIMPL_MEDIA_F3 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu xmm0, [A2] + vmovdqu xmm1, [A3] + %1 xmm0, xmm0, xmm1 + vmovdqu [A1], xmm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu ymm0, [A2] + vmovdqu ymm1, [A3] + %1 ymm0, ymm0, ymm1 + vmovdqu [A1], ymm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u256 +%endmacro + +IEMIMPL_MEDIA_F3 vpshufb +IEMIMPL_MEDIA_F3 vpand +IEMIMPL_MEDIA_F3 vpminub +IEMIMPL_MEDIA_F3 vpminuw +IEMIMPL_MEDIA_F3 vpminud +IEMIMPL_MEDIA_F3 vpminsb +IEMIMPL_MEDIA_F3 vpminsw +IEMIMPL_MEDIA_F3 vpminsd +IEMIMPL_MEDIA_F3 vpmaxub +IEMIMPL_MEDIA_F3 vpmaxuw +IEMIMPL_MEDIA_F3 vpmaxud +IEMIMPL_MEDIA_F3 vpmaxsb +IEMIMPL_MEDIA_F3 vpmaxsw +IEMIMPL_MEDIA_F3 vpmaxsd +IEMIMPL_MEDIA_F3 vpandn +IEMIMPL_MEDIA_F3 vpor +IEMIMPL_MEDIA_F3 vpxor +IEMIMPL_MEDIA_F3 vpcmpeqb +IEMIMPL_MEDIA_F3 vpcmpeqw +IEMIMPL_MEDIA_F3 vpcmpeqd +IEMIMPL_MEDIA_F3 vpcmpeqq +IEMIMPL_MEDIA_F3 vpcmpgtb +IEMIMPL_MEDIA_F3 vpcmpgtw +IEMIMPL_MEDIA_F3 vpcmpgtd +IEMIMPL_MEDIA_F3 vpcmpgtq +IEMIMPL_MEDIA_F3 vpaddb +IEMIMPL_MEDIA_F3 vpaddw +IEMIMPL_MEDIA_F3 vpaddd +IEMIMPL_MEDIA_F3 vpaddq +IEMIMPL_MEDIA_F3 vpsubb +IEMIMPL_MEDIA_F3 vpsubw +IEMIMPL_MEDIA_F3 vpsubd +IEMIMPL_MEDIA_F3 vpsubq + + +;; +; Media instruction working on two full sized source registers and one destination (AVX), +; but no XSAVE state pointer argument. +; +; @param 1 The instruction +; +; @param A0 Pointer to the destination media register size operand (output). +; @param A1 Pointer to the first source media register size operand (input). +; @param A2 Pointer to the second source media register size operand (input). +; +%macro IEMIMPL_MEDIA_OPT_F3 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12 + PROLOGUE_3_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu xmm0, [A1] + vmovdqu xmm1, [A2] + %1 xmm0, xmm0, xmm1 + vmovdqu [A0], xmm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 12 + PROLOGUE_3_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu ymm0, [A1] + vmovdqu ymm1, [A2] + %1 ymm0, ymm0, ymm1 + vmovdqu [A0], ymm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u256 +%endmacro + +IEMIMPL_MEDIA_OPT_F3 vpacksswb +IEMIMPL_MEDIA_OPT_F3 vpackssdw +IEMIMPL_MEDIA_OPT_F3 vpackuswb +IEMIMPL_MEDIA_OPT_F3 vpackusdw +IEMIMPL_MEDIA_OPT_F3 vpmullw +IEMIMPL_MEDIA_OPT_F3 vpmulld +IEMIMPL_MEDIA_OPT_F3 vpmulhw +IEMIMPL_MEDIA_OPT_F3 vpmulhuw +IEMIMPL_MEDIA_OPT_F3 vpavgb +IEMIMPL_MEDIA_OPT_F3 vpavgw +IEMIMPL_MEDIA_OPT_F3 vpsignb +IEMIMPL_MEDIA_OPT_F3 vpsignw +IEMIMPL_MEDIA_OPT_F3 vpsignd +IEMIMPL_MEDIA_OPT_F3 vphaddw +IEMIMPL_MEDIA_OPT_F3 vphaddd +IEMIMPL_MEDIA_OPT_F3 vphsubw +IEMIMPL_MEDIA_OPT_F3 vphsubd +IEMIMPL_MEDIA_OPT_F3 vphaddsw +IEMIMPL_MEDIA_OPT_F3 vphsubsw +IEMIMPL_MEDIA_OPT_F3 vpmaddubsw +IEMIMPL_MEDIA_OPT_F3 vpmulhrsw +IEMIMPL_MEDIA_OPT_F3 vpsadbw +IEMIMPL_MEDIA_OPT_F3 vpmuldq +IEMIMPL_MEDIA_OPT_F3 vpmuludq +IEMIMPL_MEDIA_OPT_F3 vunpcklps +IEMIMPL_MEDIA_OPT_F3 vunpcklpd +IEMIMPL_MEDIA_OPT_F3 vunpckhps +IEMIMPL_MEDIA_OPT_F3 vunpckhpd + +;; +; Media instruction working on one full sized source registers and one destination (AVX), +; but no XSAVE state pointer argument. +; +; @param 1 The instruction +; @param 2 Flag whether the isntruction has a 256-bit (AVX2) variant (1) or not (0). +; +; @param A0 Pointer to the destination media register size operand (output). +; @param A1 Pointer to the source media register size operand (input). +; +%macro IEMIMPL_MEDIA_OPT_F2_AVX 2 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12 + PROLOGUE_2_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu xmm0, [A1] + %1 xmm0, xmm0 + vmovdqu [A0], xmm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 + + %if %2 == 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 12 + PROLOGUE_2_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu ymm0, [A1] + %1 ymm0, ymm0 + vmovdqu [A0], ymm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u256 + %endif +%endmacro + +IEMIMPL_MEDIA_OPT_F2_AVX vpabsb, 1 +IEMIMPL_MEDIA_OPT_F2_AVX vpabsw, 1 +IEMIMPL_MEDIA_OPT_F2_AVX vpabsd, 1 +IEMIMPL_MEDIA_OPT_F2_AVX vphminposuw, 0 + + +; +; The SSE 4.2 crc32 +; +; @param A1 Pointer to the 32-bit destination. +; @param A2 The source operand, sized according to the suffix. +; +BEGINPROC_FASTCALL iemAImpl_crc32_u8, 8 + PROLOGUE_2_ARGS + + mov T0_32, [A0] + crc32 T0_32, A1_8 + mov [A0], T0_32 + + EPILOGUE_2_ARGS +ENDPROC iemAImpl_crc32_u8 + +BEGINPROC_FASTCALL iemAImpl_crc32_u16, 8 + PROLOGUE_2_ARGS + + mov T0_32, [A0] + crc32 T0_32, A1_16 + mov [A0], T0_32 + + EPILOGUE_2_ARGS +ENDPROC iemAImpl_crc32_u16 + +BEGINPROC_FASTCALL iemAImpl_crc32_u32, 8 + PROLOGUE_2_ARGS + + mov T0_32, [A0] + crc32 T0_32, A1_32 + mov [A0], T0_32 + + EPILOGUE_2_ARGS +ENDPROC iemAImpl_crc32_u32 + +%ifdef RT_ARCH_AMD64 +BEGINPROC_FASTCALL iemAImpl_crc32_u64, 8 + PROLOGUE_2_ARGS + + mov T0_32, [A0] + crc32 T0, A1 + mov [A0], T0_32 + + EPILOGUE_2_ARGS +ENDPROC iemAImpl_crc32_u64 +%endif + + +; +; PTEST (SSE 4.1) +; +; @param A0 Pointer to the first source operand (aka readonly destination). +; @param A1 Pointer to the second source operand. +; @param A2 Pointer to the EFLAGS register. +; +BEGINPROC_FASTCALL iemAImpl_ptest_u128, 12 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A0] + movdqu xmm1, [A1] + ptest xmm0, xmm1 + IEM_SAVE_FLAGS A2, X86_EFL_STATUS_BITS, 0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ptest_u128 + +BEGINPROC_FASTCALL iemAImpl_vptest_u256, 12 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + vmovdqu ymm0, [A0] + vmovdqu ymm1, [A1] + vptest ymm0, ymm1 + IEM_SAVE_FLAGS A2, X86_EFL_STATUS_BITS, 0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_vptest_u256 + + +;; +; Template for the [v]pmov{s,z}x* instructions +; +; @param 1 The instruction +; +; @param A0 Pointer to the destination media register size operand (output). +; @param A1 The source operand value (input). +; +%macro IEMIMPL_V_PMOV_SZ_X 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12 + PROLOGUE_2_ARGS + IEMIMPL_SSE_PROLOGUE + + movd xmm0, A1 + %1 xmm0, xmm0 + vmovdqu [A0], xmm0 + + IEMIMPL_SSE_PROLOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 12 + PROLOGUE_2_ARGS + IEMIMPL_AVX_PROLOGUE + + movd xmm0, A1 + v %+ %1 xmm0, xmm0 + vmovdqu [A0], xmm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 12 + PROLOGUE_2_ARGS + IEMIMPL_AVX_PROLOGUE + + movdqu xmm0, [A1] + v %+ %1 ymm0, xmm0 + vmovdqu [A0], ymm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u256 +%endmacro + +IEMIMPL_V_PMOV_SZ_X pmovsxbw +IEMIMPL_V_PMOV_SZ_X pmovsxbd +IEMIMPL_V_PMOV_SZ_X pmovsxbq +IEMIMPL_V_PMOV_SZ_X pmovsxwd +IEMIMPL_V_PMOV_SZ_X pmovsxwq +IEMIMPL_V_PMOV_SZ_X pmovsxdq + +IEMIMPL_V_PMOV_SZ_X pmovzxbw +IEMIMPL_V_PMOV_SZ_X pmovzxbd +IEMIMPL_V_PMOV_SZ_X pmovzxbq +IEMIMPL_V_PMOV_SZ_X pmovzxwd +IEMIMPL_V_PMOV_SZ_X pmovzxwq +IEMIMPL_V_PMOV_SZ_X pmovzxdq + + +;; +; Need to move this as well somewhere better? +; +struc IEMSSERESULT + .uResult resd 4 + .MXCSR resd 1 +endstruc + + +;; +; Need to move this as well somewhere better? +; +struc IEMAVX128RESULT + .uResult resd 4 + .MXCSR resd 1 +endstruc + + +;; +; Need to move this as well somewhere better? +; +struc IEMAVX256RESULT + .uResult resd 8 + .MXCSR resd 1 +endstruc + + +;; +; Initialize the SSE MXCSR register using the guest value partially to +; account for rounding mode. +; +; @uses 4 bytes of stack to save the original value, T0. +; @param 1 Expression giving the address of the FXSTATE of the guest. +; +%macro SSE_LD_FXSTATE_MXCSR 1 + sub xSP, 4 + + stmxcsr [xSP] + mov T0_32, [%1 + X86FXSTATE.MXCSR] + and T0_32, X86_MXCSR_FZ | X86_MXCSR_RC_MASK | X86_MXCSR_DAZ + or T0_32, X86_MXCSR_XCPT_MASK + sub xSP, 4 + mov [xSP], T0_32 + ldmxcsr [xSP] + add xSP, 4 +%endmacro + + +;; +; Restores the SSE MXCSR register with the original value. +; +; @uses 4 bytes of stack to save the content of MXCSR value, T0, T1. +; @param 1 Expression giving the address where to return the MXCSR value. +; @param 2 Expression giving the address of the FXSTATE of the guest. +; +; @note Restores the stack pointer. +; +%macro SSE_ST_FXSTATE_MXCSR 2 + sub xSP, 4 + stmxcsr [xSP] + mov T0_32, [xSP] + add xSP, 4 + ; Merge the status bits into the original MXCSR value. + mov T1_32, [%2 + X86FXSTATE.MXCSR] + and T0_32, X86_MXCSR_XCPT_FLAGS + or T0_32, T1_32 + mov [%1 + IEMSSERESULT.MXCSR], T0_32 + + ldmxcsr [xSP] + add xSP, 4 +%endmacro + + +;; +; Initialize the SSE MXCSR register using the guest value partially to +; account for rounding mode. +; +; @uses 4 bytes of stack to save the original value. +; @param 1 Expression giving the address of the FXSTATE of the guest. +; +%macro AVX_LD_XSAVEAREA_MXCSR 1 + sub xSP, 4 + + stmxcsr [xSP] + mov T0_32, [%1 + X86FXSTATE.MXCSR] + and T0_32, X86_MXCSR_FZ | X86_MXCSR_RC_MASK | X86_MXCSR_DAZ + sub xSP, 4 + mov [xSP], T0_32 + ldmxcsr [xSP] + add xSP, 4 +%endmacro + + +;; +; Restores the AVX128 MXCSR register with the original value. +; +; @param 1 Expression giving the address where to return the MXCSR value. +; +; @note Restores the stack pointer. +; +%macro AVX128_ST_XSAVEAREA_MXCSR 1 + stmxcsr [%1 + IEMAVX128RESULT.MXCSR] + + ldmxcsr [xSP] + add xSP, 4 +%endmacro + + +;; +; Restores the AVX256 MXCSR register with the original value. +; +; @param 1 Expression giving the address where to return the MXCSR value. +; +; @note Restores the stack pointer. +; +%macro AVX256_ST_XSAVEAREA_MXCSR 1 + stmxcsr [%1 + IEMAVX256RESULT.MXCSR] + + ldmxcsr [xSP] + add xSP, 4 +%endmacro + + +;; +; Floating point instruction working on two full sized registers. +; +; @param 1 The instruction +; @param 2 Flag whether the AVX variant of the instruction takes two or three operands, 0 to disable AVX variants +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the result including the MXCSR value. +; @param A2 Pointer to the first media register size operand (input/output). +; @param A3 Pointer to the second media register size operand (input). +; +%macro IEMIMPL_FP_F2 2 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + %1 xmm0, xmm1 + movdqu [A1 + IEMSSERESULT.uResult], xmm0 + + SSE_ST_FXSTATE_MXCSR A1, A0 + IEMIMPL_SSE_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 + + %if %2 == 3 +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 12 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + AVX_LD_XSAVEAREA_MXCSR A0 + + vmovdqu xmm0, [A2] + vmovdqu xmm1, [A3] + v %+ %1 xmm0, xmm0, xmm1 + vmovdqu [A1 + IEMAVX128RESULT.uResult], xmm0 + + AVX128_ST_XSAVEAREA_MXCSR A1 + IEMIMPL_AVX_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 12 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + AVX_LD_XSAVEAREA_MXCSR A0 + + vmovdqu ymm0, [A2] + vmovdqu ymm1, [A3] + v %+ %1 ymm0, ymm0, ymm1 + vmovdqu [A1 + IEMAVX256RESULT.uResult], ymm0 + + AVX256_ST_XSAVEAREA_MXCSR A1 + IEMIMPL_AVX_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u256 + %elif %2 == 2 +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 12 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + AVX_LD_XSAVEAREA_MXCSR A0 + + vmovdqu xmm0, [A2] + vmovdqu xmm1, [A3] + v %+ %1 xmm0, xmm1 + vmovdqu [A1 + IEMAVX128RESULT.uResult], xmm0 + + AVX128_ST_XSAVEAREA_MXCSR A1 + IEMIMPL_AVX_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 12 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + AVX_LD_XSAVEAREA_MXCSR A0 + + vmovdqu ymm0, [A2] + vmovdqu ymm1, [A3] + v %+ %1 ymm0, ymm1 + vmovdqu [A1 + IEMAVX256RESULT.uResult], ymm0 + + AVX256_ST_XSAVEAREA_MXCSR A1 + IEMIMPL_AVX_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u256 + %endif +%endmacro + +IEMIMPL_FP_F2 addps, 3 +IEMIMPL_FP_F2 addpd, 3 +IEMIMPL_FP_F2 mulps, 3 +IEMIMPL_FP_F2 mulpd, 3 +IEMIMPL_FP_F2 subps, 3 +IEMIMPL_FP_F2 subpd, 3 +IEMIMPL_FP_F2 minps, 3 +IEMIMPL_FP_F2 minpd, 3 +IEMIMPL_FP_F2 divps, 3 +IEMIMPL_FP_F2 divpd, 3 +IEMIMPL_FP_F2 maxps, 3 +IEMIMPL_FP_F2 maxpd, 3 +IEMIMPL_FP_F2 haddps, 3 +IEMIMPL_FP_F2 haddpd, 3 +IEMIMPL_FP_F2 hsubps, 3 +IEMIMPL_FP_F2 hsubpd, 3 +IEMIMPL_FP_F2 addsubps, 3 +IEMIMPL_FP_F2 addsubpd, 3 + + +;; +; These are actually unary operations but to keep it simple +; we treat them as binary for now, so the output result is +; always in sync with the register where the result might get written +; to. +IEMIMPL_FP_F2 sqrtps, 2 +IEMIMPL_FP_F2 rsqrtps, 2 +IEMIMPL_FP_F2 sqrtpd, 2 +IEMIMPL_FP_F2 cvtdq2ps, 2 +IEMIMPL_FP_F2 cvtps2dq, 2 +IEMIMPL_FP_F2 cvttps2dq, 2 +IEMIMPL_FP_F2 cvttpd2dq, 0 ; @todo AVX variants due to register size differences missing right now +IEMIMPL_FP_F2 cvtdq2pd, 0 ; @todo AVX variants due to register size differences missing right now +IEMIMPL_FP_F2 cvtpd2dq, 0 ; @todo AVX variants due to register size differences missing right now + + +;; +; Floating point instruction working on a full sized register and a single precision operand. +; +; @param 1 The instruction +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the result including the MXCSR value. +; @param A2 Pointer to the first media register size operand (input/output). +; @param A3 Pointer to the second single precision floating point value (input). +; +%macro IEMIMPL_FP_F2_R32 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128_r32, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + movdqu xmm0, [A2] + movd xmm1, [A3] + %1 xmm0, xmm1 + movdqu [A1 + IEMSSERESULT.uResult], xmm0 + + SSE_ST_FXSTATE_MXCSR A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128_r32 + +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128_r32, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + AVX_LD_XSAVEAREA_MXCSR A0 + + vmovdqu xmm0, [A2] + vmovd xmm1, [A3] + v %+ %1 xmm0, xmm0, xmm1 + vmovdqu [A1 + IEMAVX128RESULT.uResult], xmm0 + + AVX128_ST_XSAVEAREA_MXCSR A1 + IEMIMPL_AVX_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u128_r32 +%endmacro + +IEMIMPL_FP_F2_R32 addss +IEMIMPL_FP_F2_R32 mulss +IEMIMPL_FP_F2_R32 subss +IEMIMPL_FP_F2_R32 minss +IEMIMPL_FP_F2_R32 divss +IEMIMPL_FP_F2_R32 maxss +IEMIMPL_FP_F2_R32 cvtss2sd +IEMIMPL_FP_F2_R32 sqrtss +IEMIMPL_FP_F2_R32 rsqrtss + + +;; +; Floating point instruction working on a full sized register and a double precision operand. +; +; @param 1 The instruction +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the result including the MXCSR value. +; @param A2 Pointer to the first media register size operand (input/output). +; @param A3 Pointer to the second double precision floating point value (input). +; +%macro IEMIMPL_FP_F2_R64 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128_r64, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + movdqu xmm0, [A2] + movq xmm1, [A3] + %1 xmm0, xmm1 + movdqu [A1 + IEMSSERESULT.uResult], xmm0 + + SSE_ST_FXSTATE_MXCSR A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128_r64 + +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128_r64, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + AVX_LD_XSAVEAREA_MXCSR A0 + + vmovdqu xmm0, [A2] + vmovq xmm1, [A3] + v %+ %1 xmm0, xmm0, xmm1 + vmovdqu [A1 + IEMAVX128RESULT.uResult], xmm0 + + AVX128_ST_XSAVEAREA_MXCSR A1 + IEMIMPL_AVX_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u128_r64 +%endmacro + +IEMIMPL_FP_F2_R64 addsd +IEMIMPL_FP_F2_R64 mulsd +IEMIMPL_FP_F2_R64 subsd +IEMIMPL_FP_F2_R64 minsd +IEMIMPL_FP_F2_R64 divsd +IEMIMPL_FP_F2_R64 maxsd +IEMIMPL_FP_F2_R64 cvtsd2ss +IEMIMPL_FP_F2_R64 sqrtsd + + +;; +; Macro for the cvtpd2ps/cvtps2pd instructions. +; +; 1 The instruction name. +; 2 Whether the AVX256 result is 128-bit (0) or 256-bit (1). +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the result including the MXCSR value. +; @param A2 Pointer to the first media register size operand (input/output). +; @param A3 Pointer to the second media register size operand (input). +; +%macro IEMIMPL_CVT_F2 2 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + %1 xmm0, xmm1 + movdqu [A1 + IEMSSERESULT.uResult], xmm0 + + SSE_ST_FXSTATE_MXCSR A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + AVX_LD_XSAVEAREA_MXCSR A0 + + vmovdqu xmm0, [A2] + vmovdqu xmm1, [A3] + v %+ %1 xmm0, xmm1 + vmovdqu [A1 + IEMAVX128RESULT.uResult], xmm0 + + AVX128_ST_XSAVEAREA_MXCSR A1 + IEMIMPL_AVX_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + AVX_LD_XSAVEAREA_MXCSR A0 + + vmovdqu ymm0, [A2] + vmovdqu ymm1, [A3] + %if %2 == 0 + v %+ %1 xmm0, ymm1 + %else + v %+ %1 ymm0, xmm1 + %endif + vmovdqu [A1 + IEMAVX256RESULT.uResult], ymm0 + + AVX256_ST_XSAVEAREA_MXCSR A1 + IEMIMPL_AVX_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_v %+ %1 %+ _u256 +%endmacro + +IEMIMPL_CVT_F2 cvtpd2ps, 0 +IEMIMPL_CVT_F2 cvtps2pd, 1 + + +;; +; shufps instructions with 8-bit immediates. +; +; @param A0 Pointer to the destination media register size operand (input/output). +; @param A1 Pointer to the first source media register size operand (input). +; @param A2 The 8-bit immediate +; +BEGINPROC_FASTCALL iemAImpl_shufps_u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A0] + movdqu xmm1, [A1] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*4] ; sizeof(shufpX+ret+int3) == 10: A2 * 10 = (A2 * 5) * 2 + %else + lea T0, [A2 + A2*2] ; sizeof(shufpX+ret+int3) == 6: A2 * 6 = (A2 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + movdqu [A0], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + shufps xmm0, xmm1, bImm + ret + int3 + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_shufps_u128 + + +;; +; shufpd instruction with 8-bit immediates. +; +; @param A0 Pointer to the destination media register size operand (input/output). +; @param A1 Pointer to the first source media register size operand (input). +; @param A2 The 8-bit immediate +; +BEGINPROC_FASTCALL iemAImpl_shufpd_u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A0] + movdqu xmm1, [A1] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*4] ; sizeof(shufpX+ret) == 10: A2 * 10 = (A2 * 5) * 2 + %else + lea T0, [A2 + A2*2] ; sizeof(shufpX+ret) == 6: A2 * 6 = (A2 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + movdqu [A0], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + shufpd xmm0, xmm1, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_shufpd_u128 + + +;; +; vshufp{s,d} instructions with 8-bit immediates. +; +; @param 1 The instruction name. +; +; @param A0 Pointer to the destination media register size operand (output). +; @param A1 Pointer to the first source media register size operand (input). +; @param A2 Pointer to the second source media register size operand (input). +; @param A3 The 8-bit immediate +; +%macro IEMIMPL_MEDIA_AVX_VSHUFPX 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + + movdqu xmm0, [A1] + movdqu xmm1, [A2] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A3 + A3*4] ; sizeof(vshufpX+ret) == 10: A3 * 10 = (A3 * 5) * 2 + %else + lea T0, [A3 + A3*2] ; sizeof(vshufpX+ret) == 6: A3 * 6 = (A3 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + movdqu [A0], xmm0 + + IEMIMPL_AVX_EPILOGUE + EPILOGUE_4_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 xmm0, xmm0, xmm1, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_ %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu ymm0, [A1] + vmovdqu ymm1, [A2] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A3 + A3*4] ; sizeof(vshufpX+ret) == 10: A3 * 10 = (A3 * 5) * 2 + %else + lea T0, [A3 + A3*2] ; sizeof(vshufpX+ret) == 6: A3 * 6 = (A3 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + vmovdqu [A0], ymm0 + + IEMIMPL_AVX_EPILOGUE + EPILOGUE_4_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 ymm0, ymm0, ymm1, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_ %+ %1 %+ _u256 +%endmacro + +IEMIMPL_MEDIA_AVX_VSHUFPX vshufps +IEMIMPL_MEDIA_AVX_VSHUFPX vshufpd + + +;; +; One of the [p]blendv{b,ps,pd} variants +; +; @param 1 The instruction +; +; @param A0 Pointer to the first media register sized operand (input/output). +; @param A1 Pointer to the second media sized value (input). +; @param A2 Pointer to the media register sized mask value (input). +; +%macro IEMIMPL_P_BLEND 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A2] ; This is implicit + movdqu xmm1, [A0] + movdqu xmm2, [A1] ; @todo Do I need to save the original value here first? + %1 xmm1, xmm2 + movdqu [A0], xmm1 + + IEMIMPL_SSE_PROLOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_P_BLEND pblendvb +IEMIMPL_P_BLEND blendvps +IEMIMPL_P_BLEND blendvpd + + +;; +; One of the v[p]blendv{b,ps,pd} variants +; +; @param 1 The instruction +; +; @param A0 Pointer to the first media register sized operand (output). +; @param A1 Pointer to the first media register sized operand (input). +; @param A2 Pointer to the second media register sized operand (input). +; @param A3 Pointer to the media register sized mask value (input). +%macro IEMIMPL_AVX_P_BLEND 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu xmm0, [A1] + vmovdqu xmm1, [A2] + vmovdqu xmm2, [A3] + %1 xmm0, xmm0, xmm1, xmm2 + vmovdqu [A0], xmm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu ymm0, [A1] + vmovdqu ymm1, [A2] + vmovdqu ymm2, [A3] + %1 ymm0, ymm0, ymm1, ymm2 + vmovdqu [A0], ymm0 + + IEMIMPL_AVX_PROLOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u256 +%endmacro + +IEMIMPL_AVX_P_BLEND vpblendvb +IEMIMPL_AVX_P_BLEND vblendvps +IEMIMPL_AVX_P_BLEND vblendvpd + + +;; +; palignr mm1, mm2/m64 instruction. +; +; @param A0 Pointer to the first media register sized operand (output). +; @param A1 The second register sized operand (input). +; @param A2 The 8-bit immediate. +BEGINPROC_FASTCALL iemAImpl_palignr_u64, 16 + PROLOGUE_3_ARGS + IEMIMPL_MMX_PROLOGUE + + movq mm0, [A0] + movq mm1, A1 + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*4] ; sizeof(endbrxx+palignr+ret) == 10: A2 * 10 = (A2 * 5) * 2 + %else + lea T0, [A2 + A2*2] ; sizeof(palignr+ret) == 6: A2 * 6 = (A2 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + movq [A0], mm0 + + IEMIMPL_MMX_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + palignr mm0, mm1, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_palignr_u64 + + +;; +; SSE instructions with 8-bit immediates of the form +; xxx xmm1, xmm2, imm8. +; where the instruction encoding takes up 6 bytes. +; +; @param 1 The instruction name. +; +; @param A0 Pointer to the first media register size operand (input/output). +; @param A1 Pointer to the second source media register size operand (input). +; @param A2 The 8-bit immediate +; +%macro IEMIMPL_MEDIA_SSE_INSN_IMM8_6 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A0] + movdqu xmm1, [A1] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*2] ; sizeof(endbrxx+insnX+ret+int3) == 12: A2 * 12 = (A2 * 3) * 4 + lea T1, [T1 + T0*4] + %else + lea T1, [T1 + A2*8] ; sizeof(insnX+ret+int3) == 8: A2 * 8 + %endif + IBT_NOTRACK + call T1 + movdqu [A0], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 xmm0, xmm1, bImm + ret + int3 + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800 +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_SSE_INSN_IMM8_6 blendps +IEMIMPL_MEDIA_SSE_INSN_IMM8_6 blendpd +IEMIMPL_MEDIA_SSE_INSN_IMM8_6 pblendw +IEMIMPL_MEDIA_SSE_INSN_IMM8_6 palignr +IEMIMPL_MEDIA_SSE_INSN_IMM8_6 pclmulqdq +IEMIMPL_MEDIA_SSE_INSN_IMM8_6 aeskeygenassist + + +;; +; AVX instructions with 8-bit immediates of the form +; xxx {x,y}mm1, {x,y}mm2, {x,y}mm3, imm8. +; where the instruction encoding takes up 6 bytes. +; +; @param 1 The instruction name. +; @param 2 Whether the instruction has a 256-bit variant (1) or not (0). +; +; @param A0 Pointer to the destination media register size operand (output). +; @param A1 Pointer to the first source media register size operand (input). +; @param A2 Pointer to the second source media register size operand (input). +; @param A3 The 8-bit immediate +; +%macro IEMIMPL_MEDIA_AVX_INSN_IMM8_6 2 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + + movdqu xmm0, [A1] + movdqu xmm1, [A2] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A3 + A3*2] ; sizeof(endbrxx+insnX+ret+int3) == 12: A3 * 12 = (A3 * 3) * 4 + lea T1, [T1 + T0*4] + %else + lea T1, [T1 + A3*8] ; sizeof(insnX+ret+int3) == 8: A3 * 8 + %endif + IBT_NOTRACK + call T1 + movdqu [A0], xmm0 + + IEMIMPL_AVX_EPILOGUE + EPILOGUE_4_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 xmm0, xmm0, xmm1, bImm + ret + int3 + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800 +ENDPROC iemAImpl_ %+ %1 %+ _u128 + + %if %2 == 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16 + PROLOGUE_4_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu ymm0, [A1] + vmovdqu ymm1, [A2] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A3 + A3*2] ; sizeof(endbrxx+insnX+ret+int3) == 12: A3 * 12 = (A3 * 3) * 4 + lea T1, [T1 + T0*4] + %else + lea T1, [T1 + A3*8] ; sizeof(insnX+ret+int3) == 8: A3 * 8 + %endif + IBT_NOTRACK + call T1 + vmovdqu [A0], ymm0 + + IEMIMPL_AVX_EPILOGUE + EPILOGUE_4_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 ymm0, ymm0, ymm1, bImm + ret + int3 + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800 +ENDPROC iemAImpl_ %+ %1 %+ _u256 + %endif +%endmacro + +IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vblendps, 1 +IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vblendpd, 1 +IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vpblendw, 1 +IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vpalignr, 1 +IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vpclmulqdq, 0 + + +;; +; Need to move this as well somewhere better? +; +struc IEMPCMPISTRISRC + .uSrc1 resd 4 + .uSrc2 resd 4 +endstruc + +;; +; The pcmpistri instruction. +; +; @param A0 Pointer to the ECX register to store the result to (output). +; @param A1 Pointer to the EFLAGS register. +; @param A2 Pointer to the structure containing the source operands (input). +; @param A3 The 8-bit immediate +; +BEGINPROC_FASTCALL iemAImpl_pcmpistri_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A2 + IEMPCMPISTRISRC.uSrc1] + movdqu xmm1, [A2 + IEMPCMPISTRISRC.uSrc2] + mov T2, A0 ; A0 can be ecx/rcx in some calling conventions which gets overwritten later (T2 only available on AMD64) + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A3 + A3*2] ; sizeof(endbrxx+insnX+ret) == 12: A3 * 12 = (A3 * 3) * 4 + lea T1, [T1 + T0*4] + %else + lea T1, [T1 + A3*8] ; sizeof(insnX+ret) == 8: A3 * 8 + %endif + IBT_NOTRACK + call T1 + + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + mov [T2], ecx + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + pcmpistri xmm0, xmm1, bImm + ret + int3 + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800 +ENDPROC iemAImpl_pcmpistri_u128 + + +;; +; pinsrw instruction. +; +; @param A0 Pointer to the first media register size operand (input/output). +; @param A1 The 16 bit input operand (input). +; @param A2 The 8-bit immediate +; +BEGINPROC_FASTCALL iemAImpl_pinsrw_u64, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movq mm0, [A0] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*8] ; sizeof(endbrxx+pinsrw+ret) == 9: A2 * 9 + %else + lea T0, [A2 + A2*4] ; sizeof(pinsrw+ret) == 5: A2 * 5 + %endif + lea T1, [T1 + T0] + IBT_NOTRACK + call T1 + movq [A0], mm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + pinsrw mm0, A1_32, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500 +ENDPROC iemAImpl_pinsrw_u64 + +BEGINPROC_FASTCALL iemAImpl_pinsrw_u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A0] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*4] ; sizeof(endbrxx+pinsrw+ret) == 10: A2 * 10 = (A2 * 5) * 2 + %else + lea T0, [A2 + A2*2] ; sizeof(pinsrw+ret) == 6: A2 * 6 = (A2 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + movdqu [A0], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + pinsrw xmm0, A1_32, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_pinsrw_u128 + +;; +; vpinsrw instruction. +; +; @param A0 Pointer to the first media register size operand (output). +; @param A1 Pointer to the source media register size operand (input). +; @param A2 The 16 bit input operand (input). +; @param A3 The 8-bit immediate +; +BEGINPROC_FASTCALL iemAImpl_vpinsrw_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A1] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A3 + A3*4] ; sizeof(endbrxx+vpinsrw+ret) == 10: A3 * 10 = (A3 * 5) * 2 + %else + lea T0, [A3 + A3*2] ; sizeof(vpinsrw+ret) == 6: A3 * 6 = (A3 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + mov A1, A2 ; A2 requires longer encoding on Windows + IBT_NOTRACK + call T1 + movdqu [A0], xmm0 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + vpinsrw xmm0, xmm0, A1_32, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_vpinsrw_u128 + + +;; +; pextrw instruction. +; +; @param A0 Pointer to the 16bit output operand (output). +; @param A1 Pointer to the media register size operand (input). +; @param A2 The 8-bit immediate +; +BEGINPROC_FASTCALL iemAImpl_pextrw_u64, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movq mm0, A1 + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*8] ; sizeof(endbrxx+pextrw+ret) == 9: A2 * 9 + %else + lea T0, [A2 + A2*4] ; sizeof(pextrw+ret) == 5: A2 * 5 + %endif + lea T1, [T1 + T0] + IBT_NOTRACK + call T1 + mov word [A0], T0_16 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + pextrw T0_32, mm0, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500 +ENDPROC iemAImpl_pextrw_u64 + +BEGINPROC_FASTCALL iemAImpl_pextrw_u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A1] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*4] ; sizeof(endbrxx+pextrw+ret) == 10: A2 * 10 = (A2 * 5) * 2 + %else + lea T0, [A2 + A2*2] ; sizeof(pextrw+ret) == 6: A2 * 6 = (A2 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + mov word [A0], T0_16 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + pextrw T0_32, xmm0, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_pextrw_u128 + +;; +; vpextrw instruction. +; +; @param A0 Pointer to the 16bit output operand (output). +; @param A1 Pointer to the source media register size operand (input). +; @param A2 The 8-bit immediate +; +BEGINPROC_FASTCALL iemAImpl_vpextrw_u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A1] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A2 + A2*4] ; sizeof(endbrxx+vpextrw+ret) == 10: A2 * 10 = (A2 * 5) * 2 + %else + lea T0, [A2 + A2*2] ; sizeof(vpextrw+ret) == 6: A2 * 6 = (A2 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + mov word [A0], T0_16 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + vpextrw T0_32, xmm0, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_vpextrw_u128 + + +;; +; movmskp{s,d} SSE instruction template +; +; @param 1 The SSE instruction name. +; @param 2 The AVX instruction name. +; +; @param A0 Pointer to the output register (output/byte sized). +; @param A1 Pointer to the source media register size operand (input). +; +%macro IEMIMPL_MEDIA_MOVMSK_P 2 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_2_ARGS + IEMIMPL_SSE_PROLOGUE + + movdqu xmm0, [A1] + %1 T0, xmm0 + mov byte [A0], T0_8 + + IEMIMPL_SSE_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_ %+ %2 %+ _u128, 16 + PROLOGUE_2_ARGS + IEMIMPL_AVX_PROLOGUE + + movdqu xmm0, [A1] + %2 T0, xmm0 + mov byte [A0], T0_8 + + IEMIMPL_AVX_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %2 %+ _u128 + +BEGINPROC_FASTCALL iemAImpl_ %+ %2 %+ _u256, 16 + PROLOGUE_2_ARGS + IEMIMPL_AVX_PROLOGUE + + vmovdqu ymm0, [A1] + %2 T0, ymm0 + mov byte [A0], T0_8 + + IEMIMPL_AVX_EPILOGUE + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %2 %+ _u256 +%endmacro + +IEMIMPL_MEDIA_MOVMSK_P movmskps, vmovmskps +IEMIMPL_MEDIA_MOVMSK_P movmskpd, vmovmskpd + + +;; +; Restores the SSE MXCSR register with the original value. +; +; @uses 4 bytes of stack to save the content of MXCSR value, T0, T1. +; @param 1 Expression giving the address where to return the MXCSR value - only the MXCSR is stored, no IEMSSERESULT is used. +; @param 2 Expression giving the address of the FXSTATE of the guest. +; +; @note Restores the stack pointer. +; +%macro SSE_ST_FXSTATE_MXCSR_ONLY 2 + sub xSP, 4 + stmxcsr [xSP] + mov T0_32, [xSP] + add xSP, 4 + ; Merge the status bits into the original MXCSR value. + mov T1_32, [%2 + X86FXSTATE.MXCSR] + and T0_32, X86_MXCSR_XCPT_FLAGS + or T0_32, T1_32 + mov [%1], T0_32 + + ldmxcsr [xSP] + add xSP, 4 +%endmacro + + +;; +; cvttsd2si instruction - 32-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvttsd2si_i32_r64, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvttsd2si T0_32, [A3] + mov dword [A2], T0_32 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvttsd2si_i32_r64 + +;; +; cvttsd2si instruction - 64-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvttsd2si_i64_r64, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvttsd2si T0, [A3] + mov qword [A2], T0 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvttsd2si_i64_r64 + + +;; +; cvtsd2si instruction - 32-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvtsd2si_i32_r64, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvtsd2si T0_32, [A3] + mov dword [A2], T0_32 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvtsd2si_i32_r64 + +;; +; cvtsd2si instruction - 64-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvtsd2si_i64_r64, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvtsd2si T0, [A3] + mov qword [A2], T0 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvtsd2si_i64_r64 + + +;; +; cvttss2si instruction - 32-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvttss2si_i32_r32, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvttss2si T0_32, [A3] + mov dword [A2], T0_32 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvttss2si_i32_r32 + +;; +; cvttss2si instruction - 64-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvttss2si_i64_r32, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvttss2si T0, [A3] + mov qword [A2], T0 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvttss2si_i64_r32 + + +;; +; cvtss2si instruction - 32-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvtss2si_i32_r32, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvtss2si T0_32, [A3] + mov dword [A2], T0_32 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvtss2si_i32_r32 + +;; +; cvtss2si instruction - 64-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvtss2si_i64_r32, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvtss2si T0, [A3] + mov qword [A2], T0 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvtss2si_i64_r32 + + +;; +; cvtsi2ss instruction - 32-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvtsi2ss_r32_i32, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvtsi2ss xmm0, dword [A3] + movd dword [A2], xmm0 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvtsi2ss_r32_i32 + +;; +; cvtsi2ss instruction - 64-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvtsi2ss_r32_i64, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvtsi2ss xmm0, qword [A3] + movd dword [A2], xmm0 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvtsi2ss_r32_i64 + + +;; +; cvtsi2sd instruction - 32-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvtsi2sd_r64_i32, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvtsi2sd xmm0, dword [A3] + movq [A2], xmm0 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvtsi2sd_r64_i32 + +;; +; cvtsi2sd instruction - 64-bit variant. +; +; @param A0 FPU context (FXSTATE or XSAVEAREA). +; @param A1 Where to return the MXCSR value. +; @param A2 Pointer to the result operand (output). +; @param A3 Pointer to the second operand (input). +; +BEGINPROC_FASTCALL iemAImpl_cvtsi2sd_r64_i64, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR A0 + + cvtsi2sd xmm0, qword [A3] + movq [A2], xmm0 + + SSE_ST_FXSTATE_MXCSR_ONLY A1, A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_cvtsi2sd_r64_i64 + + +;; +; Initialize the SSE MXCSR register using the guest value partially to +; account for rounding mode. +; +; @uses 4 bytes of stack to save the original value, T0. +; @param 1 Expression giving the address of the MXCSR register of the guest. +; +%macro SSE_LD_FXSTATE_MXCSR_ONLY 1 + sub xSP, 4 + + stmxcsr [xSP] + mov T0_32, [%1] + and T0_32, X86_MXCSR_FZ | X86_MXCSR_RC_MASK | X86_MXCSR_DAZ + or T0_32, X86_MXCSR_XCPT_MASK + sub xSP, 4 + mov [xSP], T0_32 + ldmxcsr [xSP] + add xSP, 4 +%endmacro + + +;; +; Restores the SSE MXCSR register with the original value. +; +; @uses 4 bytes of stack to save the content of MXCSR value, T0, T1. +; @param 1 Expression giving the address where to return the MXCSR value - only the MXCSR is stored, no IEMSSERESULT is used. +; +; @note Restores the stack pointer. +; +%macro SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE 1 + sub xSP, 4 + stmxcsr [xSP] + mov T0_32, [xSP] + add xSP, 4 + ; Merge the status bits into the original MXCSR value. + mov T1_32, [%1] + and T0_32, X86_MXCSR_XCPT_FLAGS + or T0_32, T1_32 + mov [%1], T0_32 + + ldmxcsr [xSP] + add xSP, 4 +%endmacro + + +; +; UCOMISS (SSE) +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the EFLAGS value (input/output). +; @param A2 Pointer to the first source operand (aka readonly destination). +; @param A3 Pointer to the second source operand. +; +BEGINPROC_FASTCALL iemAImpl_ucomiss_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + ucomiss xmm0, xmm1 + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ucomiss_u128 + +BEGINPROC_FASTCALL iemAImpl_vucomiss_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + vucomiss xmm0, xmm1 + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_vucomiss_u128 + + +; +; UCOMISD (SSE) +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the EFLAGS value (input/output). +; @param A2 Pointer to the first source operand (aka readonly destination). +; @param A3 Pointer to the second source operand. +; +BEGINPROC_FASTCALL iemAImpl_ucomisd_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + ucomisd xmm0, xmm1 + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_ucomisd_u128 + +BEGINPROC_FASTCALL iemAImpl_vucomisd_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + vucomisd xmm0, xmm1 + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_vucomisd_u128 + +; +; COMISS (SSE) +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the EFLAGS value (input/output). +; @param A2 Pointer to the first source operand (aka readonly destination). +; @param A3 Pointer to the second source operand. +; +BEGINPROC_FASTCALL iemAImpl_comiss_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + comiss xmm0, xmm1 + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_comiss_u128 + +BEGINPROC_FASTCALL iemAImpl_vcomiss_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + vcomiss xmm0, xmm1 + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_vcomiss_u128 + + +; +; COMISD (SSE) +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the EFLAGS value (input/output). +; @param A2 Pointer to the first source operand (aka readonly destination). +; @param A3 Pointer to the second source operand. +; +BEGINPROC_FASTCALL iemAImpl_comisd_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + comisd xmm0, xmm1 + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_comisd_u128 + +BEGINPROC_FASTCALL iemAImpl_vcomisd_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2] + movdqu xmm1, [A3] + vcomisd xmm0, xmm1 + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS +ENDPROC iemAImpl_vcomisd_u128 + + +;; +; Need to move this as well somewhere better? +; +struc IEMMEDIAF2XMMSRC + .uSrc1 resd 4 + .uSrc2 resd 4 +endstruc + + +; +; CMPPS (SSE) +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the first media register size operand (output). +; @param A2 Pointer to the two media register sized inputs - IEMMEDIAF2XMMSRC (input). +; @param A3 The 8-bit immediate (input). +; +BEGINPROC_FASTCALL iemAImpl_cmpps_u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2 + IEMMEDIAF2XMMSRC.uSrc1] + movdqu xmm1, [A2 + IEMMEDIAF2XMMSRC.uSrc2] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A3 + A3*8] ; sizeof(endbrxx+cmpps+ret) == 9: A3 * 9 + %else + lea T0, [A3 + A3*4] ; sizeof(cmpps+ret) == 5: A3 * 5 + %endif + lea T1, [T1 + T0] + IBT_NOTRACK + call T1 + movdqu [A1], xmm0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + cmpps xmm0, xmm1, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500 +ENDPROC iemAImpl_cmpps_u128 + +;; +; SSE instructions with 8-bit immediates of the form +; xxx xmm1, xmm2, imm8. +; where the instruction encoding takes up 5 bytes and we need to load and save the MXCSR +; register. +; +; @param 1 The instruction name. +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the first media register size operand (output). +; @param A2 Pointer to the two media register sized inputs - IEMMEDIAF2XMMSRC (input). +; @param A3 The 8-bit immediate (input). +; +%macro IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2 + IEMMEDIAF2XMMSRC.uSrc1] + movdqu xmm1, [A2 + IEMMEDIAF2XMMSRC.uSrc2] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A3 + A3*4] ; sizeof(endbrxx+cmpXX+ret) == 10: A3 * 10 = (A3 * 5) * 2 + %else + lea T0, [A3 + A3*2] ; sizeof(cmpXX+ret) == 6: A3 * 6 = (A3 * 3) * 2 + %endif + lea T1, [T1 + T0*2] + IBT_NOTRACK + call T1 + movdqu [A1], xmm0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 xmm0, xmm1, bImm + ret + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600 +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 cmppd +IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 cmpss +IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 cmpsd + +;; +; SSE instructions with 8-bit immediates of the form +; xxx xmm1, xmm2, imm8. +; where the instruction encoding takes up 6 bytes and we need to load and save the MXCSR +; register. +; +; @param 1 The instruction name. +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the first media register size operand (output). +; @param A2 Pointer to the two media register sized inputs - IEMMEDIAF2XMMSRC (input). +; @param A3 The 8-bit immediate (input). +; +%macro IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_4_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2 + IEMMEDIAF2XMMSRC.uSrc1] + movdqu xmm1, [A2 + IEMMEDIAF2XMMSRC.uSrc2] + lea T1, [.imm0 xWrtRIP] + %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK + lea T0, [A3 + A3*2] ; sizeof(endbrxx+insn+ret+int3) == 12: A3 * 12 = (A3 * 3) * 4 + lea T1, [T1 + T0*4] + %else + lea T1, [T1 + A3*8] ; sizeof(insn+ret+int3) == 8: A3 * 8 + %endif + IBT_NOTRACK + call T1 + movdqu [A1], xmm0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_4_ARGS + %assign bImm 0 + %rep 256 +.imm %+ bImm: + IBT_ENDBRxx_WITHOUT_NOTRACK + %1 xmm0, xmm1, bImm + ret + int3 + %assign bImm bImm + 1 + %endrep +.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800 +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundps +IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundpd +IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundss +IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundsd + + +;; +; SSE instructions of the form +; xxx mm, xmm. +; and we need to load and save the MXCSR register. +; +; @param 1 The instruction name. +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the first MMX register sized operand (output). +; @param A2 Pointer to the media register sized operand (input). +; +%macro IEMIMPL_MEDIA_SSE_MXCSR_I64_U128 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A2] + %1 mm0, xmm0 + movq [A1], mm0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_SSE_MXCSR_I64_U128 cvtpd2pi +IEMIMPL_MEDIA_SSE_MXCSR_I64_U128 cvttpd2pi + +;; +; SSE instructions of the form +; xxx xmm, xmm/m64. +; and we need to load and save the MXCSR register. +; +; @param 1 The instruction name. +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the first media register sized operand (input/output). +; @param A2 The 64bit source value from a MMX media register (input) +; +%macro IEMIMPL_MEDIA_SSE_MXCSR_U128_U64 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movdqu xmm0, [A1] + movq mm0, A2 + %1 xmm0, mm0 + movdqu [A1], xmm0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_SSE_MXCSR_U128_U64 cvtpi2ps +IEMIMPL_MEDIA_SSE_MXCSR_U128_U64 cvtpi2pd + +;; +; SSE instructions of the form +; xxx mm, xmm/m64. +; and we need to load and save the MXCSR register. +; +; @param 1 The instruction name. +; +; @param A0 Pointer to the MXCSR value (input/output). +; @param A1 Pointer to the first MMX media register sized operand (output). +; @param A2 The 64bit source value (input). +; +%macro IEMIMPL_MEDIA_SSE_MXCSR_U64_U64 1 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16 + PROLOGUE_3_ARGS + IEMIMPL_SSE_PROLOGUE + SSE_LD_FXSTATE_MXCSR_ONLY A0 + + movq xmm0, A2 + %1 mm0, xmm0 + movq [A1], mm0 + + SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0 + IEMIMPL_SSE_EPILOGUE + EPILOGUE_3_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u128 +%endmacro + +IEMIMPL_MEDIA_SSE_MXCSR_U64_U64 cvtps2pi +IEMIMPL_MEDIA_SSE_MXCSR_U64_U64 cvttps2pi + +; +; All forms of RDRAND and RDSEED +; +; @param A0 Pointer to the destination operand. +; @param A1 Pointer to the EFLAGS value (input/output). +; +%macro IEMIMPL_RDRAND_RDSEED 3 +BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u %+ %3, 8 + PROLOGUE_2_ARGS + + %1 %2 + mov [A0], %2 + IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0 + + EPILOGUE_2_ARGS +ENDPROC iemAImpl_ %+ %1 %+ _u %+ %3 +%endmacro + +IEMIMPL_RDRAND_RDSEED rdrand, ax, 16 +IEMIMPL_RDRAND_RDSEED rdrand, eax, 32 +IEMIMPL_RDRAND_RDSEED rdrand, rax, 64 +IEMIMPL_RDRAND_RDSEED rdseed, ax, 16 +IEMIMPL_RDRAND_RDSEED rdseed, eax, 32 +IEMIMPL_RDRAND_RDSEED rdseed, rax, 64 + diff --git a/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp b/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp new file mode 100644 index 00000000..1196f775 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp @@ -0,0 +1,17407 @@ +/* $Id: IEMAllAImplC.cpp $ */ +/** @file + * IEM - Instruction Implementation in Assembly, portable C variant. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#include "IEMInternal.h" +#include +#include +#include +#include +#include +#include + +RT_C_DECLS_BEGIN +#include +RT_C_DECLS_END + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** @def IEM_WITHOUT_ASSEMBLY + * Enables all the code in this file. + */ +#if !defined(IEM_WITHOUT_ASSEMBLY) +# if defined(RT_ARCH_ARM32) || defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING) +# define IEM_WITHOUT_ASSEMBLY +# endif +#endif +/* IEM_WITH_ASSEMBLY trumps IEM_WITHOUT_ASSEMBLY for tstIEMAImplAsm purposes. */ +#ifdef IEM_WITH_ASSEMBLY +# undef IEM_WITHOUT_ASSEMBLY +#endif + +/** + * Calculates the signed flag value given a result and it's bit width. + * + * The signed flag (SF) is a duplication of the most significant bit in the + * result. + * + * @returns X86_EFL_SF or 0. + * @param a_uResult Unsigned result value. + * @param a_cBitsWidth The width of the result (8, 16, 32, 64). + */ +#define X86_EFL_CALC_SF(a_uResult, a_cBitsWidth) \ + ( (uint32_t)((a_uResult) >> ((a_cBitsWidth) - X86_EFL_SF_BIT - 1)) & X86_EFL_SF ) + +/** + * Calculates the zero flag value given a result. + * + * The zero flag (ZF) indicates whether the result is zero or not. + * + * @returns X86_EFL_ZF or 0. + * @param a_uResult Unsigned result value. + */ +#define X86_EFL_CALC_ZF(a_uResult) \ + ( (uint32_t)((a_uResult) == 0) << X86_EFL_ZF_BIT ) + +/** + * Extracts the OF flag from a OF calculation result. + * + * These are typically used by concating with a bitcount. The problem is that + * 8-bit values needs shifting in the other direction than the others. + */ +#define X86_EFL_GET_OF_8(a_uValue) (((uint32_t)(a_uValue) << (X86_EFL_OF_BIT - 8 + 1)) & X86_EFL_OF) +#define X86_EFL_GET_OF_16(a_uValue) ((uint32_t)((a_uValue) >> (16 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF) +#define X86_EFL_GET_OF_32(a_uValue) ((uint32_t)((a_uValue) >> (32 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF) +#define X86_EFL_GET_OF_64(a_uValue) ((uint32_t)((a_uValue) >> (64 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF) + +/** + * Updates the status bits (CF, PF, AF, ZF, SF, and OF) after arithmetic op. + * + * @returns Status bits. + * @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update. + * @param a_uResult Unsigned result value. + * @param a_uSrc The source value (for AF calc). + * @param a_uDst The original destination value (for AF calc). + * @param a_cBitsWidth The width of the result (8, 16, 32, 64). + * @param a_CfExpr Bool expression for the carry flag (CF). + * @param a_uSrcOf The a_uSrc value to use for overflow calculation. + */ +#define IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(a_pfEFlags, a_uResult, a_uDst, a_uSrc, a_cBitsWidth, a_CfExpr, a_uSrcOf) \ + do { \ + uint32_t fEflTmp = *(a_pfEFlags); \ + fEflTmp &= ~X86_EFL_STATUS_BITS; \ + fEflTmp |= (a_CfExpr) << X86_EFL_CF_BIT; \ + fEflTmp |= g_afParity[(a_uResult) & 0xff]; \ + fEflTmp |= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uSrc) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \ + fEflTmp |= X86_EFL_CALC_ZF(a_uResult); \ + fEflTmp |= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \ + \ + /* Overflow during ADDition happens when both inputs have the same signed \ + bit value and the result has a different sign bit value. \ + \ + Since subtraction can be rewritten as addition: 2 - 1 == 2 + -1, it \ + follows that for SUBtraction the signed bit value must differ between \ + the two inputs and the result's signed bit diff from the first input. \ + Note! Must xor with sign bit to convert, not do (0 - a_uSrc). \ + \ + See also: http://teaching.idallen.com/dat2343/10f/notes/040_overflow.txt */ \ + fEflTmp |= X86_EFL_GET_OF_ ## a_cBitsWidth( ( ((uint ## a_cBitsWidth ## _t)~((a_uDst) ^ (a_uSrcOf))) \ + & RT_BIT_64(a_cBitsWidth - 1)) \ + & ((a_uResult) ^ (a_uDst)) ); \ + *(a_pfEFlags) = fEflTmp; \ + } while (0) + +/** + * Updates the status bits (CF, PF, AF, ZF, SF, and OF) after a logical op. + * + * CF and OF are defined to be 0 by logical operations. AF on the other hand is + * undefined. We do not set AF, as that seems to make the most sense (which + * probably makes it the most wrong in real life). + * + * @returns Status bits. + * @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update. + * @param a_uResult Unsigned result value. + * @param a_cBitsWidth The width of the result (8, 16, 32, 64). + * @param a_fExtra Additional bits to set. + */ +#define IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(a_pfEFlags, a_uResult, a_cBitsWidth, a_fExtra) \ + do { \ + uint32_t fEflTmp = *(a_pfEFlags); \ + fEflTmp &= ~X86_EFL_STATUS_BITS; \ + fEflTmp |= g_afParity[(a_uResult) & 0xff]; \ + fEflTmp |= X86_EFL_CALC_ZF(a_uResult); \ + fEflTmp |= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \ + fEflTmp |= (a_fExtra); \ + *(a_pfEFlags) = fEflTmp; \ + } while (0) + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +/** + * Parity calculation table. + * + * This is also used by iemAllAImpl.asm. + * + * The generator code: + * @code + * #include + * + * int main() + * { + * unsigned b; + * for (b = 0; b < 256; b++) + * { + * int cOnes = ( b & 1) + * + ((b >> 1) & 1) + * + ((b >> 2) & 1) + * + ((b >> 3) & 1) + * + ((b >> 4) & 1) + * + ((b >> 5) & 1) + * + ((b >> 6) & 1) + * + ((b >> 7) & 1); + * printf(" /" "* %#04x = %u%u%u%u%u%u%u%ub *" "/ %s,\n", + * b, + * (b >> 7) & 1, + * (b >> 6) & 1, + * (b >> 5) & 1, + * (b >> 4) & 1, + * (b >> 3) & 1, + * (b >> 2) & 1, + * (b >> 1) & 1, + * b & 1, + * cOnes & 1 ? "0" : "X86_EFL_PF"); + * } + * return 0; + * } + * @endcode + */ +uint8_t const g_afParity[256] = +{ + /* 0000 = 00000000b */ X86_EFL_PF, + /* 0x01 = 00000001b */ 0, + /* 0x02 = 00000010b */ 0, + /* 0x03 = 00000011b */ X86_EFL_PF, + /* 0x04 = 00000100b */ 0, + /* 0x05 = 00000101b */ X86_EFL_PF, + /* 0x06 = 00000110b */ X86_EFL_PF, + /* 0x07 = 00000111b */ 0, + /* 0x08 = 00001000b */ 0, + /* 0x09 = 00001001b */ X86_EFL_PF, + /* 0x0a = 00001010b */ X86_EFL_PF, + /* 0x0b = 00001011b */ 0, + /* 0x0c = 00001100b */ X86_EFL_PF, + /* 0x0d = 00001101b */ 0, + /* 0x0e = 00001110b */ 0, + /* 0x0f = 00001111b */ X86_EFL_PF, + /* 0x10 = 00010000b */ 0, + /* 0x11 = 00010001b */ X86_EFL_PF, + /* 0x12 = 00010010b */ X86_EFL_PF, + /* 0x13 = 00010011b */ 0, + /* 0x14 = 00010100b */ X86_EFL_PF, + /* 0x15 = 00010101b */ 0, + /* 0x16 = 00010110b */ 0, + /* 0x17 = 00010111b */ X86_EFL_PF, + /* 0x18 = 00011000b */ X86_EFL_PF, + /* 0x19 = 00011001b */ 0, + /* 0x1a = 00011010b */ 0, + /* 0x1b = 00011011b */ X86_EFL_PF, + /* 0x1c = 00011100b */ 0, + /* 0x1d = 00011101b */ X86_EFL_PF, + /* 0x1e = 00011110b */ X86_EFL_PF, + /* 0x1f = 00011111b */ 0, + /* 0x20 = 00100000b */ 0, + /* 0x21 = 00100001b */ X86_EFL_PF, + /* 0x22 = 00100010b */ X86_EFL_PF, + /* 0x23 = 00100011b */ 0, + /* 0x24 = 00100100b */ X86_EFL_PF, + /* 0x25 = 00100101b */ 0, + /* 0x26 = 00100110b */ 0, + /* 0x27 = 00100111b */ X86_EFL_PF, + /* 0x28 = 00101000b */ X86_EFL_PF, + /* 0x29 = 00101001b */ 0, + /* 0x2a = 00101010b */ 0, + /* 0x2b = 00101011b */ X86_EFL_PF, + /* 0x2c = 00101100b */ 0, + /* 0x2d = 00101101b */ X86_EFL_PF, + /* 0x2e = 00101110b */ X86_EFL_PF, + /* 0x2f = 00101111b */ 0, + /* 0x30 = 00110000b */ X86_EFL_PF, + /* 0x31 = 00110001b */ 0, + /* 0x32 = 00110010b */ 0, + /* 0x33 = 00110011b */ X86_EFL_PF, + /* 0x34 = 00110100b */ 0, + /* 0x35 = 00110101b */ X86_EFL_PF, + /* 0x36 = 00110110b */ X86_EFL_PF, + /* 0x37 = 00110111b */ 0, + /* 0x38 = 00111000b */ 0, + /* 0x39 = 00111001b */ X86_EFL_PF, + /* 0x3a = 00111010b */ X86_EFL_PF, + /* 0x3b = 00111011b */ 0, + /* 0x3c = 00111100b */ X86_EFL_PF, + /* 0x3d = 00111101b */ 0, + /* 0x3e = 00111110b */ 0, + /* 0x3f = 00111111b */ X86_EFL_PF, + /* 0x40 = 01000000b */ 0, + /* 0x41 = 01000001b */ X86_EFL_PF, + /* 0x42 = 01000010b */ X86_EFL_PF, + /* 0x43 = 01000011b */ 0, + /* 0x44 = 01000100b */ X86_EFL_PF, + /* 0x45 = 01000101b */ 0, + /* 0x46 = 01000110b */ 0, + /* 0x47 = 01000111b */ X86_EFL_PF, + /* 0x48 = 01001000b */ X86_EFL_PF, + /* 0x49 = 01001001b */ 0, + /* 0x4a = 01001010b */ 0, + /* 0x4b = 01001011b */ X86_EFL_PF, + /* 0x4c = 01001100b */ 0, + /* 0x4d = 01001101b */ X86_EFL_PF, + /* 0x4e = 01001110b */ X86_EFL_PF, + /* 0x4f = 01001111b */ 0, + /* 0x50 = 01010000b */ X86_EFL_PF, + /* 0x51 = 01010001b */ 0, + /* 0x52 = 01010010b */ 0, + /* 0x53 = 01010011b */ X86_EFL_PF, + /* 0x54 = 01010100b */ 0, + /* 0x55 = 01010101b */ X86_EFL_PF, + /* 0x56 = 01010110b */ X86_EFL_PF, + /* 0x57 = 01010111b */ 0, + /* 0x58 = 01011000b */ 0, + /* 0x59 = 01011001b */ X86_EFL_PF, + /* 0x5a = 01011010b */ X86_EFL_PF, + /* 0x5b = 01011011b */ 0, + /* 0x5c = 01011100b */ X86_EFL_PF, + /* 0x5d = 01011101b */ 0, + /* 0x5e = 01011110b */ 0, + /* 0x5f = 01011111b */ X86_EFL_PF, + /* 0x60 = 01100000b */ X86_EFL_PF, + /* 0x61 = 01100001b */ 0, + /* 0x62 = 01100010b */ 0, + /* 0x63 = 01100011b */ X86_EFL_PF, + /* 0x64 = 01100100b */ 0, + /* 0x65 = 01100101b */ X86_EFL_PF, + /* 0x66 = 01100110b */ X86_EFL_PF, + /* 0x67 = 01100111b */ 0, + /* 0x68 = 01101000b */ 0, + /* 0x69 = 01101001b */ X86_EFL_PF, + /* 0x6a = 01101010b */ X86_EFL_PF, + /* 0x6b = 01101011b */ 0, + /* 0x6c = 01101100b */ X86_EFL_PF, + /* 0x6d = 01101101b */ 0, + /* 0x6e = 01101110b */ 0, + /* 0x6f = 01101111b */ X86_EFL_PF, + /* 0x70 = 01110000b */ 0, + /* 0x71 = 01110001b */ X86_EFL_PF, + /* 0x72 = 01110010b */ X86_EFL_PF, + /* 0x73 = 01110011b */ 0, + /* 0x74 = 01110100b */ X86_EFL_PF, + /* 0x75 = 01110101b */ 0, + /* 0x76 = 01110110b */ 0, + /* 0x77 = 01110111b */ X86_EFL_PF, + /* 0x78 = 01111000b */ X86_EFL_PF, + /* 0x79 = 01111001b */ 0, + /* 0x7a = 01111010b */ 0, + /* 0x7b = 01111011b */ X86_EFL_PF, + /* 0x7c = 01111100b */ 0, + /* 0x7d = 01111101b */ X86_EFL_PF, + /* 0x7e = 01111110b */ X86_EFL_PF, + /* 0x7f = 01111111b */ 0, + /* 0x80 = 10000000b */ 0, + /* 0x81 = 10000001b */ X86_EFL_PF, + /* 0x82 = 10000010b */ X86_EFL_PF, + /* 0x83 = 10000011b */ 0, + /* 0x84 = 10000100b */ X86_EFL_PF, + /* 0x85 = 10000101b */ 0, + /* 0x86 = 10000110b */ 0, + /* 0x87 = 10000111b */ X86_EFL_PF, + /* 0x88 = 10001000b */ X86_EFL_PF, + /* 0x89 = 10001001b */ 0, + /* 0x8a = 10001010b */ 0, + /* 0x8b = 10001011b */ X86_EFL_PF, + /* 0x8c = 10001100b */ 0, + /* 0x8d = 10001101b */ X86_EFL_PF, + /* 0x8e = 10001110b */ X86_EFL_PF, + /* 0x8f = 10001111b */ 0, + /* 0x90 = 10010000b */ X86_EFL_PF, + /* 0x91 = 10010001b */ 0, + /* 0x92 = 10010010b */ 0, + /* 0x93 = 10010011b */ X86_EFL_PF, + /* 0x94 = 10010100b */ 0, + /* 0x95 = 10010101b */ X86_EFL_PF, + /* 0x96 = 10010110b */ X86_EFL_PF, + /* 0x97 = 10010111b */ 0, + /* 0x98 = 10011000b */ 0, + /* 0x99 = 10011001b */ X86_EFL_PF, + /* 0x9a = 10011010b */ X86_EFL_PF, + /* 0x9b = 10011011b */ 0, + /* 0x9c = 10011100b */ X86_EFL_PF, + /* 0x9d = 10011101b */ 0, + /* 0x9e = 10011110b */ 0, + /* 0x9f = 10011111b */ X86_EFL_PF, + /* 0xa0 = 10100000b */ X86_EFL_PF, + /* 0xa1 = 10100001b */ 0, + /* 0xa2 = 10100010b */ 0, + /* 0xa3 = 10100011b */ X86_EFL_PF, + /* 0xa4 = 10100100b */ 0, + /* 0xa5 = 10100101b */ X86_EFL_PF, + /* 0xa6 = 10100110b */ X86_EFL_PF, + /* 0xa7 = 10100111b */ 0, + /* 0xa8 = 10101000b */ 0, + /* 0xa9 = 10101001b */ X86_EFL_PF, + /* 0xaa = 10101010b */ X86_EFL_PF, + /* 0xab = 10101011b */ 0, + /* 0xac = 10101100b */ X86_EFL_PF, + /* 0xad = 10101101b */ 0, + /* 0xae = 10101110b */ 0, + /* 0xaf = 10101111b */ X86_EFL_PF, + /* 0xb0 = 10110000b */ 0, + /* 0xb1 = 10110001b */ X86_EFL_PF, + /* 0xb2 = 10110010b */ X86_EFL_PF, + /* 0xb3 = 10110011b */ 0, + /* 0xb4 = 10110100b */ X86_EFL_PF, + /* 0xb5 = 10110101b */ 0, + /* 0xb6 = 10110110b */ 0, + /* 0xb7 = 10110111b */ X86_EFL_PF, + /* 0xb8 = 10111000b */ X86_EFL_PF, + /* 0xb9 = 10111001b */ 0, + /* 0xba = 10111010b */ 0, + /* 0xbb = 10111011b */ X86_EFL_PF, + /* 0xbc = 10111100b */ 0, + /* 0xbd = 10111101b */ X86_EFL_PF, + /* 0xbe = 10111110b */ X86_EFL_PF, + /* 0xbf = 10111111b */ 0, + /* 0xc0 = 11000000b */ X86_EFL_PF, + /* 0xc1 = 11000001b */ 0, + /* 0xc2 = 11000010b */ 0, + /* 0xc3 = 11000011b */ X86_EFL_PF, + /* 0xc4 = 11000100b */ 0, + /* 0xc5 = 11000101b */ X86_EFL_PF, + /* 0xc6 = 11000110b */ X86_EFL_PF, + /* 0xc7 = 11000111b */ 0, + /* 0xc8 = 11001000b */ 0, + /* 0xc9 = 11001001b */ X86_EFL_PF, + /* 0xca = 11001010b */ X86_EFL_PF, + /* 0xcb = 11001011b */ 0, + /* 0xcc = 11001100b */ X86_EFL_PF, + /* 0xcd = 11001101b */ 0, + /* 0xce = 11001110b */ 0, + /* 0xcf = 11001111b */ X86_EFL_PF, + /* 0xd0 = 11010000b */ 0, + /* 0xd1 = 11010001b */ X86_EFL_PF, + /* 0xd2 = 11010010b */ X86_EFL_PF, + /* 0xd3 = 11010011b */ 0, + /* 0xd4 = 11010100b */ X86_EFL_PF, + /* 0xd5 = 11010101b */ 0, + /* 0xd6 = 11010110b */ 0, + /* 0xd7 = 11010111b */ X86_EFL_PF, + /* 0xd8 = 11011000b */ X86_EFL_PF, + /* 0xd9 = 11011001b */ 0, + /* 0xda = 11011010b */ 0, + /* 0xdb = 11011011b */ X86_EFL_PF, + /* 0xdc = 11011100b */ 0, + /* 0xdd = 11011101b */ X86_EFL_PF, + /* 0xde = 11011110b */ X86_EFL_PF, + /* 0xdf = 11011111b */ 0, + /* 0xe0 = 11100000b */ 0, + /* 0xe1 = 11100001b */ X86_EFL_PF, + /* 0xe2 = 11100010b */ X86_EFL_PF, + /* 0xe3 = 11100011b */ 0, + /* 0xe4 = 11100100b */ X86_EFL_PF, + /* 0xe5 = 11100101b */ 0, + /* 0xe6 = 11100110b */ 0, + /* 0xe7 = 11100111b */ X86_EFL_PF, + /* 0xe8 = 11101000b */ X86_EFL_PF, + /* 0xe9 = 11101001b */ 0, + /* 0xea = 11101010b */ 0, + /* 0xeb = 11101011b */ X86_EFL_PF, + /* 0xec = 11101100b */ 0, + /* 0xed = 11101101b */ X86_EFL_PF, + /* 0xee = 11101110b */ X86_EFL_PF, + /* 0xef = 11101111b */ 0, + /* 0xf0 = 11110000b */ X86_EFL_PF, + /* 0xf1 = 11110001b */ 0, + /* 0xf2 = 11110010b */ 0, + /* 0xf3 = 11110011b */ X86_EFL_PF, + /* 0xf4 = 11110100b */ 0, + /* 0xf5 = 11110101b */ X86_EFL_PF, + /* 0xf6 = 11110110b */ X86_EFL_PF, + /* 0xf7 = 11110111b */ 0, + /* 0xf8 = 11111000b */ 0, + /* 0xf9 = 11111001b */ X86_EFL_PF, + /* 0xfa = 11111010b */ X86_EFL_PF, + /* 0xfb = 11111011b */ 0, + /* 0xfc = 11111100b */ X86_EFL_PF, + /* 0xfd = 11111101b */ 0, + /* 0xfe = 11111110b */ 0, + /* 0xff = 11111111b */ X86_EFL_PF, +}; + +/* for clang: */ +extern const RTFLOAT32U g_ar32Zero[]; +extern const RTFLOAT64U g_ar64Zero[]; +extern const RTFLOAT80U g_ar80Zero[]; +extern const RTFLOAT80U g_ar80One[]; +extern const RTFLOAT80U g_r80Indefinite; +extern const RTFLOAT32U g_ar32Infinity[]; +extern const RTFLOAT64U g_ar64Infinity[]; +extern const RTFLOAT80U g_ar80Infinity[]; +extern const RTFLOAT128U g_r128Ln2; +extern const RTUINT128U g_u128Ln2Mantissa; +extern const RTUINT128U g_u128Ln2MantissaIntel; +extern const RTFLOAT128U g_ar128F2xm1HornerConsts[]; +extern const RTFLOAT32U g_ar32QNaN[]; +extern const RTFLOAT64U g_ar64QNaN[]; + +/** Zero values (indexed by fSign). */ +RTFLOAT32U const g_ar32Zero[] = { RTFLOAT32U_INIT_ZERO(0), RTFLOAT32U_INIT_ZERO(1) }; +RTFLOAT64U const g_ar64Zero[] = { RTFLOAT64U_INIT_ZERO(0), RTFLOAT64U_INIT_ZERO(1) }; +RTFLOAT80U const g_ar80Zero[] = { RTFLOAT80U_INIT_ZERO(0), RTFLOAT80U_INIT_ZERO(1) }; + +/** One values (indexed by fSign). */ +RTFLOAT80U const g_ar80One[] = +{ RTFLOAT80U_INIT(0, RT_BIT_64(63), RTFLOAT80U_EXP_BIAS), RTFLOAT80U_INIT(1, RT_BIT_64(63), RTFLOAT80U_EXP_BIAS) }; + +/** Indefinite (negative). */ +RTFLOAT80U const g_r80Indefinite = RTFLOAT80U_INIT_INDEFINITE(1); + +/** Infinities (indexed by fSign). */ +RTFLOAT32U const g_ar32Infinity[] = { RTFLOAT32U_INIT_INF(0), RTFLOAT32U_INIT_INF(1) }; +RTFLOAT64U const g_ar64Infinity[] = { RTFLOAT64U_INIT_INF(0), RTFLOAT64U_INIT_INF(1) }; +RTFLOAT80U const g_ar80Infinity[] = { RTFLOAT80U_INIT_INF(0), RTFLOAT80U_INIT_INF(1) }; + +/** Default QNaNs (indexed by fSign). */ +RTFLOAT32U const g_ar32QNaN[] = { RTFLOAT32U_INIT_QNAN(0), RTFLOAT32U_INIT_QNAN(1) }; +RTFLOAT64U const g_ar64QNaN[] = { RTFLOAT64U_INIT_QNAN(0), RTFLOAT64U_INIT_QNAN(1) }; + + +#if 0 +/** 128-bit floating point constant: 2.0 */ +const RTFLOAT128U g_r128Two = RTFLOAT128U_INIT_C(0, 0, 0, RTFLOAT128U_EXP_BIAS + 1); +#endif + + +/* The next section is generated by tools/IEMGenFpuConstants: */ + +/** The ln2 constant as 128-bit floating point value. + * base-10: 6.93147180559945309417232121458176575e-1 + * base-16: b.17217f7d1cf79abc9e3b39803f30@-1 + * base-2 : 1.0110001011100100001011111110111110100011100111101111001101010111100100111100011101100111001100000000011111100110e-1 */ +//const RTFLOAT128U g_r128Ln2 = RTFLOAT128U_INIT_C(0, 0x62e42fefa39e, 0xf35793c7673007e6, 0x3ffe); +const RTFLOAT128U g_r128Ln2 = RTFLOAT128U_INIT_C(0, 0x62e42fefa39e, 0xf357900000000000, 0x3ffe); +/** High precision ln2 value. + * base-10: 6.931471805599453094172321214581765680747e-1 + * base-16: b.17217f7d1cf79abc9e3b39803f2f6af0@-1 + * base-2 : 1.0110001011100100001011111110111110100011100111101111001101010111100100111100011101100111001100000000011111100101111011010101111e-1 */ +const RTUINT128U g_u128Ln2Mantissa = RTUINT128_INIT_C(0xb17217f7d1cf79ab, 0xc9e3b39803f2f6af); +/** High precision ln2 value, compatible with f2xm1 results on intel 10980XE. + * base-10: 6.931471805599453094151379470289064954613e-1 + * base-16: b.17217f7d1cf79abc0000000000000000@-1 + * base-2 : 1.0110001011100100001011111110111110100011100111101111001101010111100000000000000000000000000000000000000000000000000000000000000e-1 */ +const RTUINT128U g_u128Ln2MantissaIntel = RTUINT128_INIT_C(0xb17217f7d1cf79ab, 0xc000000000000000); + +/** Horner constants for f2xm1 */ +const RTFLOAT128U g_ar128F2xm1HornerConsts[] = +{ + /* a0 + * base-10: 1.00000000000000000000000000000000000e0 + * base-16: 1.0000000000000000000000000000@0 + * base-2 : 1.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e0 */ + RTFLOAT128U_INIT_C(0, 0x000000000000, 0x0000000000000000, 0x3fff), + /* a1 + * base-10: 5.00000000000000000000000000000000000e-1 + * base-16: 8.0000000000000000000000000000@-1 + * base-2 : 1.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e-1 */ + RTFLOAT128U_INIT_C(0, 0x000000000000, 0x0000000000000000, 0x3ffe), + /* a2 + * base-10: 1.66666666666666666666666666666666658e-1 + * base-16: 2.aaaaaaaaaaaaaaaaaaaaaaaaaaaa@-1 + * base-2 : 1.0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101e-3 */ + RTFLOAT128U_INIT_C(0, 0x555555555555, 0x5555555555555555, 0x3ffc), + /* a3 + * base-10: 4.16666666666666666666666666666666646e-2 + * base-16: a.aaaaaaaaaaaaaaaaaaaaaaaaaaa8@-2 + * base-2 : 1.0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101e-5 */ + RTFLOAT128U_INIT_C(0, 0x555555555555, 0x5555555555555555, 0x3ffa), + /* a4 + * base-10: 8.33333333333333333333333333333333323e-3 + * base-16: 2.2222222222222222222222222222@-2 + * base-2 : 1.0001000100010001000100010001000100010001000100010001000100010001000100010001000100010001000100010001000100010001e-7 */ + RTFLOAT128U_INIT_C(0, 0x111111111111, 0x1111111111111111, 0x3ff8), + /* a5 + * base-10: 1.38888888888888888888888888888888874e-3 + * base-16: 5.b05b05b05b05b05b05b05b05b058@-3 + * base-2 : 1.0110110000010110110000010110110000010110110000010110110000010110110000010110110000010110110000010110110000010110e-10 */ + RTFLOAT128U_INIT_C(0, 0x6c16c16c16c1, 0x6c16c16c16c16c16, 0x3ff5), + /* a6 + * base-10: 1.98412698412698412698412698412698412e-4 + * base-16: d.00d00d00d00d00d00d00d00d00d0@-4 + * base-2 : 1.1010000000011010000000011010000000011010000000011010000000011010000000011010000000011010000000011010000000011010e-13 */ + RTFLOAT128U_INIT_C(0, 0xa01a01a01a01, 0xa01a01a01a01a01a, 0x3ff2), + /* a7 + * base-10: 2.48015873015873015873015873015873015e-5 + * base-16: 1.a01a01a01a01a01a01a01a01a01a@-4 + * base-2 : 1.1010000000011010000000011010000000011010000000011010000000011010000000011010000000011010000000011010000000011010e-16 */ + RTFLOAT128U_INIT_C(0, 0xa01a01a01a01, 0xa01a01a01a01a01a, 0x3fef), + /* a8 + * base-10: 2.75573192239858906525573192239858902e-6 + * base-16: 2.e3bc74aad8e671f5583911ca002e@-5 + * base-2 : 1.0111000111011110001110100101010101101100011100110011100011111010101011000001110010001000111001010000000000010111e-19 */ + RTFLOAT128U_INIT_C(0, 0x71de3a556c73, 0x38faac1c88e50017, 0x3fec), + /* a9 + * base-10: 2.75573192239858906525573192239858865e-7 + * base-16: 4.9f93edde27d71cbbc05b4fa999e0@-6 + * base-2 : 1.0010011111100100111110110111011110001001111101011100011100101110111100000001011011010011111010100110011001111000e-22 */ + RTFLOAT128U_INIT_C(0, 0x27e4fb7789f5, 0xc72ef016d3ea6678, 0x3fe9), + /* a10 + * base-10: 2.50521083854417187750521083854417184e-8 + * base-16: 6.b99159fd5138e3f9d1f92e0df71c@-7 + * base-2 : 1.1010111001100100010101100111111101010100010011100011100011111110011101000111111001001011100000110111110111000111e-26 */ + RTFLOAT128U_INIT_C(0, 0xae64567f544e, 0x38fe747e4b837dc7, 0x3fe5), + /* a11 + * base-10: 2.08767569878680989792100903212014296e-9 + * base-16: 8.f76c77fc6c4bdaa26d4c3d67f420@-8 + * base-2 : 1.0001111011101101100011101111111110001101100010010111101101010100010011011010100110000111101011001111111010000100e-29 */ + RTFLOAT128U_INIT_C(0, 0x1eed8eff8d89, 0x7b544da987acfe84, 0x3fe2), + /* a12 + * base-10: 1.60590438368216145993923771701549472e-10 + * base-16: b.092309d43684be51c198e91d7b40@-9 + * base-2 : 1.0110000100100100011000010011101010000110110100001001011111001010001110000011001100011101001000111010111101101000e-33 */ + RTFLOAT128U_INIT_C(0, 0x6124613a86d0, 0x97ca38331d23af68, 0x3fde), + /* a13 + * base-10: 1.14707455977297247138516979786821043e-11 + * base-16: c.9cba54603e4e905d6f8a2efd1f20@-10 + * base-2 : 1.1001001110010111010010101000110000000111110010011101001000001011101011011111000101000101110111111010001111100100e-37 */ + RTFLOAT128U_INIT_C(0, 0x93974a8c07c9, 0xd20badf145dfa3e4, 0x3fda), + /* a14 + * base-10: 7.64716373181981647590113198578806964e-13 + * base-16: d.73f9f399dc0f88ec32b587746578@-11 + * base-2 : 1.1010111001111111001111100111001100111011100000011111000100011101100001100101011010110000111011101000110010101111e-41 */ + RTFLOAT128U_INIT_C(0, 0xae7f3e733b81, 0xf11d8656b0ee8caf, 0x3fd6), + /* a15 + * base-10: 4.77947733238738529743820749111754352e-14 + * base-16: d.73f9f399dc0f88ec32b587746578@-12 + * base-2 : 1.1010111001111111001111100111001100111011100000011111000100011101100001100101011010110000111011101000110010101111e-45 */ + RTFLOAT128U_INIT_C(0, 0xae7f3e733b81, 0xf11d8656b0ee8caf, 0x3fd2), + /* a16 + * base-10: 2.81145725434552076319894558301031970e-15 + * base-16: c.a963b81856a53593028cbbb8d7f8@-13 + * base-2 : 1.1001010100101100011101110000001100001010110101001010011010110010011000000101000110010111011101110001101011111111e-49 */ + RTFLOAT128U_INIT_C(0, 0x952c77030ad4, 0xa6b2605197771aff, 0x3fce), + /* a17 + * base-10: 1.56192069685862264622163643500573321e-16 + * base-16: b.413c31dcbecbbdd8024435161550@-14 + * base-2 : 1.0110100000100111100001100011101110010111110110010111011110111011000000000100100010000110101000101100001010101010e-53 */ + RTFLOAT128U_INIT_C(0, 0x6827863b97d9, 0x77bb004886a2c2aa, 0x3fca), + /* a18 + * base-10: 8.22063524662432971695598123687227980e-18 + * base-16: 9.7a4da340a0ab92650f61dbdcb3a0@-15 + * base-2 : 1.0010111101001001101101000110100000010100000101010111001001001100101000011110110000111011011110111001011001110100e-57 */ + RTFLOAT128U_INIT_C(0, 0x2f49b4681415, 0x724ca1ec3b7b9674, 0x3fc6), + /* a19 + * base-10: 4.11031762331216485847799061843614006e-19 + * base-16: 7.950ae900808941ea72b4afe3c2e8@-16 + * base-2 : 1.1110010101000010101110100100000000100000001000100101000001111010100111001010110100101011111110001111000010111010e-62 */ + RTFLOAT128U_INIT_C(0, 0xe542ba402022, 0x507a9cad2bf8f0ba, 0x3fc1), + /* a20 + * base-10: 1.95729410633912612308475743735054143e-20 + * base-16: 5.c6e3bdb73d5c62fbc51bf3b9b8fc@-17 + * base-2 : 1.0111000110111000111011110110110111001111010101110001100010111110111100010100011011111100111011100110111000111111e-66 */ + RTFLOAT128U_INIT_C(0, 0x71b8ef6dcf57, 0x18bef146fcee6e3f, 0x3fbd), + /* a21 + * base-10: 8.89679139245057328674889744250246106e-22 + * base-16: 4.338e5b6dfe14a5143242dfcce3a0@-18 + * base-2 : 1.0000110011100011100101101101101101111111100001010010100101000101000011001001000010110111111100110011100011101000e-70 */ + RTFLOAT128U_INIT_C(0, 0x0ce396db7f85, 0x29450c90b7f338e8, 0x3fb9), +}; + + +/* + * There are a few 64-bit on 32-bit things we'd rather do in C. Actually, doing + * it all in C is probably safer atm., optimize what's necessary later, maybe. + */ +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) + + +/********************************************************************************************************************************* +* Binary Operations * +*********************************************************************************************************************************/ + +/* + * ADD + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_add_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + uint64_t uDst = *puDst; + uint64_t uResult = uDst + uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uResult < uDst, uSrc); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_add_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + uint32_t uDst = *puDst; + uint32_t uResult = uDst + uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uResult < uDst, uSrc); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_add_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + uint16_t uDst = *puDst; + uint16_t uResult = uDst + uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uResult < uDst, uSrc); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_add_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags)) +{ + uint8_t uDst = *puDst; + uint8_t uResult = uDst + uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uResult < uDst, uSrc); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * ADC + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + if (!(*pfEFlags & X86_EFL_CF)) + iemAImpl_add_u64(puDst, uSrc, pfEFlags); + else + { + uint64_t uDst = *puDst; + uint64_t uResult = uDst + uSrc + 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uResult <= uDst, uSrc); + } +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + if (!(*pfEFlags & X86_EFL_CF)) + iemAImpl_add_u32(puDst, uSrc, pfEFlags); + else + { + uint32_t uDst = *puDst; + uint32_t uResult = uDst + uSrc + 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uResult <= uDst, uSrc); + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + if (!(*pfEFlags & X86_EFL_CF)) + iemAImpl_add_u16(puDst, uSrc, pfEFlags); + else + { + uint16_t uDst = *puDst; + uint16_t uResult = uDst + uSrc + 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uResult <= uDst, uSrc); + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags)) +{ + if (!(*pfEFlags & X86_EFL_CF)) + iemAImpl_add_u8(puDst, uSrc, pfEFlags); + else + { + uint8_t uDst = *puDst; + uint8_t uResult = uDst + uSrc + 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uResult <= uDst, uSrc); + } +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * SUB + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + uint64_t uDst = *puDst; + uint64_t uResult = uDst - uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uDst < uSrc, uSrc ^ RT_BIT_64(63)); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + uint32_t uDst = *puDst; + uint32_t uResult = uDst - uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uDst < uSrc, uSrc ^ RT_BIT_32(31)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + uint16_t uDst = *puDst; + uint16_t uResult = uDst - uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uDst < uSrc, uSrc ^ (uint16_t)0x8000); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags)) +{ + uint8_t uDst = *puDst; + uint8_t uResult = uDst - uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uDst < uSrc, uSrc ^ (uint8_t)0x80); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * SBB + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + if (!(*pfEFlags & X86_EFL_CF)) + iemAImpl_sub_u64(puDst, uSrc, pfEFlags); + else + { + uint64_t uDst = *puDst; + uint64_t uResult = uDst - uSrc - 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uDst <= uSrc, uSrc ^ RT_BIT_64(63)); + } +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + if (!(*pfEFlags & X86_EFL_CF)) + iemAImpl_sub_u32(puDst, uSrc, pfEFlags); + else + { + uint32_t uDst = *puDst; + uint32_t uResult = uDst - uSrc - 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uDst <= uSrc, uSrc ^ RT_BIT_32(31)); + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + if (!(*pfEFlags & X86_EFL_CF)) + iemAImpl_sub_u16(puDst, uSrc, pfEFlags); + else + { + uint16_t uDst = *puDst; + uint16_t uResult = uDst - uSrc - 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uDst <= uSrc, uSrc ^ (uint16_t)0x8000); + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags)) +{ + if (!(*pfEFlags & X86_EFL_CF)) + iemAImpl_sub_u8(puDst, uSrc, pfEFlags); + else + { + uint8_t uDst = *puDst; + uint8_t uResult = uDst - uSrc - 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uDst <= uSrc, uSrc ^ (uint8_t)0x80); + } +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * OR + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_or_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + uint64_t uResult = *puDst | uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_or_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + uint32_t uResult = *puDst | uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_or_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + uint16_t uResult = *puDst | uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_or_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags)) +{ + uint8_t uResult = *puDst | uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * XOR + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + uint64_t uResult = *puDst ^ uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + uint32_t uResult = *puDst ^ uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + uint16_t uResult = *puDst ^ uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags)) +{ + uint8_t uResult = *puDst ^ uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * AND + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_and_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + uint64_t const uResult = *puDst & uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_and_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + uint32_t const uResult = *puDst & uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_and_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + uint16_t const uResult = *puDst & uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_and_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags)) +{ + uint8_t const uResult = *puDst & uSrc; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ +#endif /* !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * ANDN (BMI1 instruction) + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_andn_u64_fallback,(uint64_t *puDst, uint64_t uSrc1, uint64_t uSrc2, uint32_t *pfEFlags)) +{ + uint64_t const uResult = ~uSrc1 & uSrc2; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_andn_u32_fallback,(uint32_t *puDst, uint32_t uSrc1, uint32_t uSrc2, uint32_t *pfEFlags)) +{ + uint32_t const uResult = ~uSrc1 & uSrc2; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0); +} + + +#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +IEM_DECL_IMPL_DEF(void, iemAImpl_andn_u64,(uint64_t *puDst, uint64_t uSrc1, uint64_t uSrc2, uint32_t *pfEFlags)) +{ + iemAImpl_andn_u64_fallback(puDst, uSrc1, uSrc2, pfEFlags); +} +#endif + + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +IEM_DECL_IMPL_DEF(void, iemAImpl_andn_u32,(uint32_t *puDst, uint32_t uSrc1, uint32_t uSrc2, uint32_t *pfEFlags)) +{ + iemAImpl_andn_u32_fallback(puDst, uSrc1, uSrc2, pfEFlags); +} +#endif + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) + +/* + * CMP + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + uint64_t uDstTmp = *puDst; + iemAImpl_sub_u64(&uDstTmp, uSrc, pfEFlags); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + uint32_t uDstTmp = *puDst; + iemAImpl_sub_u32(&uDstTmp, uSrc, pfEFlags); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + uint16_t uDstTmp = *puDst; + iemAImpl_sub_u16(&uDstTmp, uSrc, pfEFlags); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags)) +{ + uint8_t uDstTmp = *puDst; + iemAImpl_sub_u8(&uDstTmp, uSrc, pfEFlags); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * TEST + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_test_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + uint64_t uResult = *puDst & uSrc; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_test_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + uint32_t uResult = *puDst & uSrc; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_test_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + uint16_t uResult = *puDst & uSrc; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_test_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags)) +{ + uint8_t uResult = *puDst & uSrc; + IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * LOCK prefixed variants of the above + */ + +/** 64-bit locked binary operand operation. */ +# define DO_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth) \ + do { \ + uint ## a_cBitsWidth ## _t uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \ + uint ## a_cBitsWidth ## _t uTmp; \ + uint32_t fEflTmp; \ + do \ + { \ + uTmp = uOld; \ + fEflTmp = *pfEFlags; \ + iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth(&uTmp, uSrc, &fEflTmp); \ + } while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uTmp, uOld, &uOld)); \ + *pfEFlags = fEflTmp; \ + } while (0) + + +#define EMIT_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth) \ + IEM_DECL_IMPL_DEF(void, iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth ## _locked,(uint ## a_cBitsWidth ## _t *puDst, \ + uint ## a_cBitsWidth ## _t uSrc, \ + uint32_t *pfEFlags)) \ + { \ + DO_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth); \ + } + +EMIT_LOCKED_BIN_OP(add, 64) +EMIT_LOCKED_BIN_OP(adc, 64) +EMIT_LOCKED_BIN_OP(sub, 64) +EMIT_LOCKED_BIN_OP(sbb, 64) +EMIT_LOCKED_BIN_OP(or, 64) +EMIT_LOCKED_BIN_OP(xor, 64) +EMIT_LOCKED_BIN_OP(and, 64) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_LOCKED_BIN_OP(add, 32) +EMIT_LOCKED_BIN_OP(adc, 32) +EMIT_LOCKED_BIN_OP(sub, 32) +EMIT_LOCKED_BIN_OP(sbb, 32) +EMIT_LOCKED_BIN_OP(or, 32) +EMIT_LOCKED_BIN_OP(xor, 32) +EMIT_LOCKED_BIN_OP(and, 32) + +EMIT_LOCKED_BIN_OP(add, 16) +EMIT_LOCKED_BIN_OP(adc, 16) +EMIT_LOCKED_BIN_OP(sub, 16) +EMIT_LOCKED_BIN_OP(sbb, 16) +EMIT_LOCKED_BIN_OP(or, 16) +EMIT_LOCKED_BIN_OP(xor, 16) +EMIT_LOCKED_BIN_OP(and, 16) + +EMIT_LOCKED_BIN_OP(add, 8) +EMIT_LOCKED_BIN_OP(adc, 8) +EMIT_LOCKED_BIN_OP(sub, 8) +EMIT_LOCKED_BIN_OP(sbb, 8) +EMIT_LOCKED_BIN_OP(or, 8) +EMIT_LOCKED_BIN_OP(xor, 8) +EMIT_LOCKED_BIN_OP(and, 8) +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * Bit operations (same signature as above). + */ + +/* + * BT + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're + not modified by either AMD (3990x) or Intel (i9-9980HK). */ + Assert(uSrc < 64); + uint64_t uDst = *puDst; + if (uDst & RT_BIT_64(uSrc)) + *pfEFlags |= X86_EFL_CF; + else + *pfEFlags &= ~X86_EFL_CF; +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're + not modified by either AMD (3990x) or Intel (i9-9980HK). */ + Assert(uSrc < 32); + uint32_t uDst = *puDst; + if (uDst & RT_BIT_32(uSrc)) + *pfEFlags |= X86_EFL_CF; + else + *pfEFlags &= ~X86_EFL_CF; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're + not modified by either AMD (3990x) or Intel (i9-9980HK). */ + Assert(uSrc < 16); + uint16_t uDst = *puDst; + if (uDst & RT_BIT_32(uSrc)) + *pfEFlags |= X86_EFL_CF; + else + *pfEFlags &= ~X86_EFL_CF; +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * BTC + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're + not modified by either AMD (3990x) or Intel (i9-9980HK). */ + Assert(uSrc < 64); + uint64_t fMask = RT_BIT_64(uSrc); + uint64_t uDst = *puDst; + if (uDst & fMask) + { + uDst &= ~fMask; + *puDst = uDst; + *pfEFlags |= X86_EFL_CF; + } + else + { + uDst |= fMask; + *puDst = uDst; + *pfEFlags &= ~X86_EFL_CF; + } +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're + not modified by either AMD (3990x) or Intel (i9-9980HK). */ + Assert(uSrc < 32); + uint32_t fMask = RT_BIT_32(uSrc); + uint32_t uDst = *puDst; + if (uDst & fMask) + { + uDst &= ~fMask; + *puDst = uDst; + *pfEFlags |= X86_EFL_CF; + } + else + { + uDst |= fMask; + *puDst = uDst; + *pfEFlags &= ~X86_EFL_CF; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're + not modified by either AMD (3990x) or Intel (i9-9980HK). */ + Assert(uSrc < 16); + uint16_t fMask = RT_BIT_32(uSrc); + uint16_t uDst = *puDst; + if (uDst & fMask) + { + uDst &= ~fMask; + *puDst = uDst; + *pfEFlags |= X86_EFL_CF; + } + else + { + uDst |= fMask; + *puDst = uDst; + *pfEFlags &= ~X86_EFL_CF; + } +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * BTR + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an + logical operation (AND/OR/whatever). */ + Assert(uSrc < 64); + uint64_t fMask = RT_BIT_64(uSrc); + uint64_t uDst = *puDst; + if (uDst & fMask) + { + uDst &= ~fMask; + *puDst = uDst; + *pfEFlags |= X86_EFL_CF; + } + else + *pfEFlags &= ~X86_EFL_CF; +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an + logical operation (AND/OR/whatever). */ + Assert(uSrc < 32); + uint32_t fMask = RT_BIT_32(uSrc); + uint32_t uDst = *puDst; + if (uDst & fMask) + { + uDst &= ~fMask; + *puDst = uDst; + *pfEFlags |= X86_EFL_CF; + } + else + *pfEFlags &= ~X86_EFL_CF; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an + logical operation (AND/OR/whatever). */ + Assert(uSrc < 16); + uint16_t fMask = RT_BIT_32(uSrc); + uint16_t uDst = *puDst; + if (uDst & fMask) + { + uDst &= ~fMask; + *puDst = uDst; + *pfEFlags |= X86_EFL_CF; + } + else + *pfEFlags &= ~X86_EFL_CF; +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * BTS + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an + logical operation (AND/OR/whatever). */ + Assert(uSrc < 64); + uint64_t fMask = RT_BIT_64(uSrc); + uint64_t uDst = *puDst; + if (uDst & fMask) + *pfEFlags |= X86_EFL_CF; + else + { + uDst |= fMask; + *puDst = uDst; + *pfEFlags &= ~X86_EFL_CF; + } +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an + logical operation (AND/OR/whatever). */ + Assert(uSrc < 32); + uint32_t fMask = RT_BIT_32(uSrc); + uint32_t uDst = *puDst; + if (uDst & fMask) + *pfEFlags |= X86_EFL_CF; + else + { + uDst |= fMask; + *puDst = uDst; + *pfEFlags &= ~X86_EFL_CF; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + /* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an + logical operation (AND/OR/whatever). */ + Assert(uSrc < 16); + uint16_t fMask = RT_BIT_32(uSrc); + uint32_t uDst = *puDst; + if (uDst & fMask) + *pfEFlags |= X86_EFL_CF; + else + { + uDst |= fMask; + *puDst = uDst; + *pfEFlags &= ~X86_EFL_CF; + } +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +EMIT_LOCKED_BIN_OP(btc, 64) +EMIT_LOCKED_BIN_OP(btr, 64) +EMIT_LOCKED_BIN_OP(bts, 64) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_LOCKED_BIN_OP(btc, 32) +EMIT_LOCKED_BIN_OP(btr, 32) +EMIT_LOCKED_BIN_OP(bts, 32) + +EMIT_LOCKED_BIN_OP(btc, 16) +EMIT_LOCKED_BIN_OP(btr, 16) +EMIT_LOCKED_BIN_OP(bts, 16) +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * Helpers for BSR and BSF. + * + * Note! "undefined" flags: OF, SF, AF, PF, CF. + * Intel behavior modelled on 10980xe, AMD on 3990X. Other marchs may + * produce different result (see https://www.sandpile.org/x86/flags.htm), + * but we restrict ourselves to emulating these recent marchs. + */ +#define SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlag, a_iBit) do { \ + unsigned iBit = (a_iBit); \ + uint32_t fEfl = *pfEFlags & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \ + if (iBit) \ + { \ + *puDst = --iBit; \ + fEfl |= g_afParity[iBit]; \ + } \ + else \ + fEfl |= X86_EFL_ZF | X86_EFL_PF; \ + *pfEFlags = fEfl; \ + } while (0) +#define SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlag, a_iBit) do { \ + unsigned const iBit = (a_iBit); \ + if (iBit) \ + { \ + *puDst = iBit - 1; \ + *pfEFlags &= ~X86_EFL_ZF; \ + } \ + else \ + *pfEFlags |= X86_EFL_ZF; \ + } while (0) + + +/* + * BSF - first (least significant) bit set + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU64(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU64(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64_amd,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU64(uSrc)); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU32(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU32(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32_amd,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU32(uSrc)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU16(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU16(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16_amd,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU16(uSrc)); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * BSR - last (most significant) bit set + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU64(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU64(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64_amd,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU64(uSrc)); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU32(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU32(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32_amd,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU32(uSrc)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU16(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU16(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16_amd,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU16(uSrc)); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * Helpers for LZCNT and TZCNT. + */ +#define SET_BIT_CNT_SEARCH_RESULT_INTEL(a_puDst, a_uSrc, a_pfEFlags, a_uResult) do { \ + unsigned const uResult = (a_uResult); \ + *(a_puDst) = uResult; \ + uint32_t fEfl = *(a_pfEFlags) & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \ + if (uResult) \ + fEfl |= g_afParity[uResult]; \ + else \ + fEfl |= X86_EFL_ZF | X86_EFL_PF; \ + if (!a_uSrc) \ + fEfl |= X86_EFL_CF; \ + *(a_pfEFlags) = fEfl; \ + } while (0) +#define SET_BIT_CNT_SEARCH_RESULT_AMD(a_puDst, a_uSrc, a_pfEFlags, a_uResult) do { \ + unsigned const uResult = (a_uResult); \ + *(a_puDst) = uResult; \ + uint32_t fEfl = *(a_pfEFlags) & ~(X86_EFL_ZF | X86_EFL_CF); \ + if (!uResult) \ + fEfl |= X86_EFL_ZF; \ + if (!a_uSrc) \ + fEfl |= X86_EFL_CF; \ + *(a_pfEFlags) = fEfl; \ + } while (0) + + +/* + * LZCNT - count leading zero bits. + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + iemAImpl_lzcnt_u64_intel(puDst, uSrc, pfEFlags); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU64(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u64_amd,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU64(uSrc)); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + iemAImpl_lzcnt_u32_intel(puDst, uSrc, pfEFlags); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU32(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u32_amd,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU32(uSrc)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + iemAImpl_lzcnt_u16_intel(puDst, uSrc, pfEFlags); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU16(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u16_amd,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU16(uSrc)); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * TZCNT - count leading zero bits. + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + iemAImpl_tzcnt_u64_intel(puDst, uSrc, pfEFlags); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU64(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u64_amd,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU64(uSrc)); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + iemAImpl_tzcnt_u32_intel(puDst, uSrc, pfEFlags); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU32(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u32_amd,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU32(uSrc)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + iemAImpl_tzcnt_u16_intel(puDst, uSrc, pfEFlags); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU16(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u16_amd,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags)) +{ + SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU16(uSrc)); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ +#endif /* !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * BEXTR (BMI1 instruction) + */ +#define EMIT_BEXTR(a_cBits, a_Type, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_bextr_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc1, \ + a_Type uSrc2, uint32_t *pfEFlags)) \ +{ \ + /* uSrc1 is considered virtually zero extended to 512 bits width. */ \ + uint32_t fEfl = *pfEFlags & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \ + a_Type uResult; \ + uint8_t const iFirstBit = (uint8_t)uSrc2; \ + if (iFirstBit < a_cBits) \ + { \ + uResult = uSrc1 >> iFirstBit; \ + uint8_t const cBits = (uint8_t)(uSrc2 >> 8); \ + if (cBits < a_cBits) \ + uResult &= RT_CONCAT(RT_BIT_,a_cBits)(cBits) - 1; \ + *puDst = uResult; \ + if (!uResult) \ + fEfl |= X86_EFL_ZF; \ + } \ + else \ + { \ + *puDst = uResult = 0; \ + fEfl |= X86_EFL_ZF; \ + } \ + /** @todo complete flag calculations. */ \ + *pfEFlags = fEfl; \ +} + +EMIT_BEXTR(64, uint64_t, _fallback) +EMIT_BEXTR(32, uint32_t, _fallback) +#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BEXTR(64, uint64_t, RT_NOTHING) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BEXTR(32, uint32_t, RT_NOTHING) +#endif + +/* + * BLSR (BMI1 instruction) + */ +#define EMIT_BLSR(a_cBits, a_Type, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_blsr_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc, uint32_t *pfEFlags)) \ +{ \ + uint32_t fEfl1 = *pfEFlags; \ + uint32_t fEfl2 = fEfl1; \ + *puDst = uSrc; \ + iemAImpl_sub_u ## a_cBits(&uSrc, 1, &fEfl1); \ + iemAImpl_and_u ## a_cBits(puDst, uSrc, &fEfl2); \ + \ + /* AMD: The carry flag is from the SUB operation. */ \ + /* 10890xe: PF always cleared? */ \ + fEfl2 &= ~(X86_EFL_CF | X86_EFL_PF); \ + fEfl2 |= fEfl1 & X86_EFL_CF; \ + *pfEFlags = fEfl2; \ +} + +EMIT_BLSR(64, uint64_t, _fallback) +EMIT_BLSR(32, uint32_t, _fallback) +#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BLSR(64, uint64_t, RT_NOTHING) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BLSR(32, uint32_t, RT_NOTHING) +#endif + +/* + * BLSMSK (BMI1 instruction) + */ +#define EMIT_BLSMSK(a_cBits, a_Type, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_blsmsk_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc, uint32_t *pfEFlags)) \ +{ \ + uint32_t fEfl1 = *pfEFlags; \ + uint32_t fEfl2 = fEfl1; \ + *puDst = uSrc; \ + iemAImpl_sub_u ## a_cBits(&uSrc, 1, &fEfl1); \ + iemAImpl_xor_u ## a_cBits(puDst, uSrc, &fEfl2); \ + \ + /* AMD: The carry flag is from the SUB operation. */ \ + /* 10890xe: PF always cleared? */ \ + fEfl2 &= ~(X86_EFL_CF | X86_EFL_PF); \ + fEfl2 |= fEfl1 & X86_EFL_CF; \ + *pfEFlags = fEfl2; \ +} + +EMIT_BLSMSK(64, uint64_t, _fallback) +EMIT_BLSMSK(32, uint32_t, _fallback) +#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BLSMSK(64, uint64_t, RT_NOTHING) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BLSMSK(32, uint32_t, RT_NOTHING) +#endif + +/* + * BLSI (BMI1 instruction) + */ +#define EMIT_BLSI(a_cBits, a_Type, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_blsi_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc, uint32_t *pfEFlags)) \ +{ \ + uint32_t fEfl1 = *pfEFlags; \ + uint32_t fEfl2 = fEfl1; \ + *puDst = uSrc; \ + iemAImpl_neg_u ## a_cBits(&uSrc, &fEfl1); \ + iemAImpl_and_u ## a_cBits(puDst, uSrc, &fEfl2); \ + \ + /* AMD: The carry flag is from the SUB operation. */ \ + /* 10890xe: PF always cleared? */ \ + fEfl2 &= ~(X86_EFL_CF | X86_EFL_PF); \ + fEfl2 |= fEfl1 & X86_EFL_CF; \ + *pfEFlags = fEfl2; \ +} + +EMIT_BLSI(64, uint64_t, _fallback) +EMIT_BLSI(32, uint32_t, _fallback) +#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BLSI(64, uint64_t, RT_NOTHING) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BLSI(32, uint32_t, RT_NOTHING) +#endif + +/* + * BZHI (BMI2 instruction) + */ +#define EMIT_BZHI(a_cBits, a_Type, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_bzhi_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc1, \ + a_Type uSrc2, uint32_t *pfEFlags)) \ +{ \ + uint32_t fEfl = *pfEFlags & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \ + a_Type uResult; \ + uint8_t const iFirstBit = (uint8_t)uSrc2; \ + if (iFirstBit < a_cBits) \ + uResult = uSrc1 & (((a_Type)1 << iFirstBit) - 1); \ + else \ + { \ + uResult = uSrc1; \ + fEfl |= X86_EFL_CF; \ + } \ + *puDst = uResult; \ + fEfl |= X86_EFL_CALC_ZF(uResult); \ + fEfl |= X86_EFL_CALC_SF(uResult, a_cBits); \ + *pfEFlags = fEfl; \ +} + +EMIT_BZHI(64, uint64_t, _fallback) +EMIT_BZHI(32, uint32_t, _fallback) +#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BZHI(64, uint64_t, RT_NOTHING) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_BZHI(32, uint32_t, RT_NOTHING) +#endif + +/* + * POPCNT + */ +RT_ALIGNAS_VAR(64) static uint8_t const g_abBitCounts6[64] = +{ + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, +}; + +/** @todo Use native popcount where possible and employ some more efficient + * algorithm here (or in asm.h fallback)! */ + +DECLINLINE(uint8_t) iemPopCountU16(uint16_t u16) +{ + return g_abBitCounts6[ u16 & 0x3f] + + g_abBitCounts6[(u16 >> 6) & 0x3f] + + g_abBitCounts6[(u16 >> 12) & 0x3f]; +} + +DECLINLINE(uint8_t) iemPopCountU32(uint32_t u32) +{ + return g_abBitCounts6[ u32 & 0x3f] + + g_abBitCounts6[(u32 >> 6) & 0x3f] + + g_abBitCounts6[(u32 >> 12) & 0x3f] + + g_abBitCounts6[(u32 >> 18) & 0x3f] + + g_abBitCounts6[(u32 >> 24) & 0x3f] + + g_abBitCounts6[(u32 >> 30) & 0x3f]; +} + +DECLINLINE(uint8_t) iemPopCountU64(uint64_t u64) +{ + return g_abBitCounts6[ u64 & 0x3f] + + g_abBitCounts6[(u64 >> 6) & 0x3f] + + g_abBitCounts6[(u64 >> 12) & 0x3f] + + g_abBitCounts6[(u64 >> 18) & 0x3f] + + g_abBitCounts6[(u64 >> 24) & 0x3f] + + g_abBitCounts6[(u64 >> 30) & 0x3f] + + g_abBitCounts6[(u64 >> 36) & 0x3f] + + g_abBitCounts6[(u64 >> 42) & 0x3f] + + g_abBitCounts6[(u64 >> 48) & 0x3f] + + g_abBitCounts6[(u64 >> 54) & 0x3f] + + g_abBitCounts6[(u64 >> 60) & 0x3f]; +} + +#define EMIT_POPCNT(a_cBits, a_Type, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_popcnt_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc, uint32_t *pfEFlags)) \ +{ \ + uint32_t fEfl = *pfEFlags & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \ + a_Type uResult; \ + if (uSrc) \ + uResult = iemPopCountU ## a_cBits(uSrc); \ + else \ + { \ + fEfl |= X86_EFL_ZF; \ + uResult = 0; \ + } \ + *puDst = uResult; \ + *pfEFlags = fEfl; \ +} + +EMIT_POPCNT(64, uint64_t, _fallback) +EMIT_POPCNT(32, uint32_t, _fallback) +EMIT_POPCNT(16, uint16_t, _fallback) +#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_POPCNT(64, uint64_t, RT_NOTHING) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_POPCNT(32, uint32_t, RT_NOTHING) +EMIT_POPCNT(16, uint16_t, RT_NOTHING) +#endif + + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) + +/* + * XCHG + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_locked,(uint64_t *puMem, uint64_t *puReg)) +{ +#if ARCH_BITS >= 64 + *puReg = ASMAtomicXchgU64(puMem, *puReg); +#else + uint64_t uOldMem = *puMem; + while (!ASMAtomicCmpXchgExU64(puMem, *puReg, uOldMem, &uOldMem)) + ASMNopPause(); + *puReg = uOldMem; +#endif +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_locked,(uint32_t *puMem, uint32_t *puReg)) +{ + *puReg = ASMAtomicXchgU32(puMem, *puReg); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_locked,(uint16_t *puMem, uint16_t *puReg)) +{ + *puReg = ASMAtomicXchgU16(puMem, *puReg); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_locked,(uint8_t *puMem, uint8_t *puReg)) +{ + *puReg = ASMAtomicXchgU8(puMem, *puReg); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* Unlocked variants for fDisregardLock mode: */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_unlocked,(uint64_t *puMem, uint64_t *puReg)) +{ + uint64_t const uOld = *puMem; + *puMem = *puReg; + *puReg = uOld; +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_unlocked,(uint32_t *puMem, uint32_t *puReg)) +{ + uint32_t const uOld = *puMem; + *puMem = *puReg; + *puReg = uOld; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_unlocked,(uint16_t *puMem, uint16_t *puReg)) +{ + uint16_t const uOld = *puMem; + *puMem = *puReg; + *puReg = uOld; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_unlocked,(uint8_t *puMem, uint8_t *puReg)) +{ + uint8_t const uOld = *puMem; + *puMem = *puReg; + *puReg = uOld; +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * XADD and LOCK XADD. + */ +#define EMIT_XADD(a_cBitsWidth, a_Type) \ +IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u ## a_cBitsWidth,(a_Type *puDst, a_Type *puReg, uint32_t *pfEFlags)) \ +{ \ + a_Type uDst = *puDst; \ + a_Type uResult = uDst; \ + iemAImpl_add_u ## a_cBitsWidth(&uResult, *puReg, pfEFlags); \ + *puDst = uResult; \ + *puReg = uDst; \ +} \ +\ +IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u ## a_cBitsWidth ## _locked,(a_Type *puDst, a_Type *puReg, uint32_t *pfEFlags)) \ +{ \ + a_Type uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \ + a_Type uResult; \ + uint32_t fEflTmp; \ + do \ + { \ + uResult = uOld; \ + fEflTmp = *pfEFlags; \ + iemAImpl_add_u ## a_cBitsWidth(&uResult, *puReg, &fEflTmp); \ + } while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uResult, uOld, &uOld)); \ + *puReg = uOld; \ + *pfEFlags = fEflTmp; \ +} +EMIT_XADD(64, uint64_t) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_XADD(32, uint32_t) +EMIT_XADD(16, uint16_t) +EMIT_XADD(8, uint8_t) +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +#endif + +/* + * CMPXCHG, CMPXCHG8B, CMPXCHG16B + * + * Note! We don't have non-locking/atomic cmpxchg primitives, so all cmpxchg + * instructions are emulated as locked. + */ +#if defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8_locked, (uint8_t *pu8Dst, uint8_t *puAl, uint8_t uSrcReg, uint32_t *pEFlags)) +{ + uint8_t uOld = *puAl; + if (ASMAtomicCmpXchgExU8(pu8Dst, uSrcReg, uOld, puAl)) + Assert(*puAl == uOld); + iemAImpl_cmp_u8(&uOld, *puAl, pEFlags); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16_locked,(uint16_t *pu16Dst, uint16_t *puAx, uint16_t uSrcReg, uint32_t *pEFlags)) +{ + uint16_t uOld = *puAx; + if (ASMAtomicCmpXchgExU16(pu16Dst, uSrcReg, uOld, puAx)) + Assert(*puAx == uOld); + iemAImpl_cmp_u16(&uOld, *puAx, pEFlags); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32_locked,(uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags)) +{ + uint32_t uOld = *puEax; + if (ASMAtomicCmpXchgExU32(pu32Dst, uSrcReg, uOld, puEax)) + Assert(*puEax == uOld); + iemAImpl_cmp_u32(&uOld, *puEax, pEFlags); +} + + +# if ARCH_BITS == 32 +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags)) +# else +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags)) +# endif +{ +# if ARCH_BITS == 32 + uint64_t const uSrcReg = *puSrcReg; +# endif + uint64_t uOld = *puRax; + if (ASMAtomicCmpXchgExU64(pu64Dst, uSrcReg, uOld, puRax)) + Assert(*puRax == uOld); + iemAImpl_cmp_u64(&uOld, *puRax, pEFlags); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b_locked,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx, + uint32_t *pEFlags)) +{ + uint64_t const uNew = pu64EbxEcx->u; + uint64_t const uOld = pu64EaxEdx->u; + if (ASMAtomicCmpXchgExU64(pu64Dst, uNew, uOld, &pu64EaxEdx->u)) + { + Assert(pu64EaxEdx->u == uOld); + *pEFlags |= X86_EFL_ZF; + } + else + *pEFlags &= ~X86_EFL_ZF; +} + + +# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_ARM64) +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_locked,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx, + uint32_t *pEFlags)) +{ +# ifdef VBOX_STRICT + RTUINT128U const uOld = *pu128RaxRdx; +# endif +# if defined(RT_ARCH_AMD64) + if (ASMAtomicCmpXchgU128v2(&pu128Dst->u, pu128RbxRcx->s.Hi, pu128RbxRcx->s.Lo, pu128RaxRdx->s.Hi, pu128RaxRdx->s.Lo, + &pu128RaxRdx->u)) +# else + if (ASMAtomicCmpXchgU128(&pu128Dst->u, pu128RbxRcx->u, pu128RaxRdx->u, &pu128RaxRdx->u)) +# endif + { + Assert(pu128RaxRdx->s.Lo == uOld.s.Lo && pu128RaxRdx->s.Hi == uOld.s.Hi); + *pEFlags |= X86_EFL_ZF; + } + else + *pEFlags &= ~X86_EFL_ZF; +} +# endif + +#endif /* defined(IEM_WITHOUT_ASSEMBLY) */ + +# if !defined(RT_ARCH_ARM64) /** @todo may need this for unaligned accesses... */ +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, + PRTUINT128U pu128RbxRcx, uint32_t *pEFlags)) +{ + RTUINT128U u128Tmp = *pu128Dst; + if ( u128Tmp.s.Lo == pu128RaxRdx->s.Lo + && u128Tmp.s.Hi == pu128RaxRdx->s.Hi) + { + *pu128Dst = *pu128RbxRcx; + *pEFlags |= X86_EFL_ZF; + } + else + { + *pu128RaxRdx = u128Tmp; + *pEFlags &= ~X86_EFL_ZF; + } +} +#endif /* !RT_ARCH_ARM64 */ + +#if defined(IEM_WITHOUT_ASSEMBLY) + +/* Unlocked versions mapped to the locked ones: */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8, (uint8_t *pu8Dst, uint8_t *puAl, uint8_t uSrcReg, uint32_t *pEFlags)) +{ + iemAImpl_cmpxchg_u8_locked(pu8Dst, puAl, uSrcReg, pEFlags); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16, (uint16_t *pu16Dst, uint16_t *puAx, uint16_t uSrcReg, uint32_t *pEFlags)) +{ + iemAImpl_cmpxchg_u16_locked(pu16Dst, puAx, uSrcReg, pEFlags); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32, (uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags)) +{ + iemAImpl_cmpxchg_u32_locked(pu32Dst, puEax, uSrcReg, pEFlags); +} + + +# if ARCH_BITS == 32 +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags)) +{ + iemAImpl_cmpxchg_u64_locked(pu64Dst, puRax, puSrcReg, pEFlags); +} +# else +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags)) +{ + iemAImpl_cmpxchg_u64_locked(pu64Dst, puRax, uSrcReg, pEFlags); +} +# endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx, uint32_t *pEFlags)) +{ + iemAImpl_cmpxchg8b_locked(pu64Dst, pu64EaxEdx, pu64EbxEcx, pEFlags); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx, + uint32_t *pEFlags)) +{ + iemAImpl_cmpxchg16b_locked(pu128Dst, pu128RaxRdx, pu128RbxRcx, pEFlags); +} + +#endif /* defined(IEM_WITHOUT_ASSEMBLY) */ + +#if (!defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)) \ + && !defined(DOXYGEN_RUNNING) /* Doxygen has some groking issues here and ends up mixing up input. Not worth tracking down now. */ + +/* + * MUL, IMUL, DIV and IDIV helpers. + * + * - The U64 versions must use 128-bit intermediates, so we need to abstract the + * division step so we can select between using C operators and + * RTUInt128DivRem/RTUInt128MulU64ByU64. + * + * - The U8 versions work returns output in AL + AH instead of xDX + xAX, with the + * IDIV/DIV taking all the input in AX too. This means we have to abstract some + * input loads and the result storing. + */ + +DECLINLINE(void) RTUInt128DivRemByU64(PRTUINT128U pQuotient, PRTUINT128U pRemainder, PCRTUINT128U pDividend, uint64_t u64Divisor) +{ +# ifdef __GNUC__ /* GCC maybe really annoying in function. */ + pQuotient->s.Lo = 0; + pQuotient->s.Hi = 0; +# endif + RTUINT128U Divisor; + Divisor.s.Lo = u64Divisor; + Divisor.s.Hi = 0; + RTUInt128DivRem(pQuotient, pRemainder, pDividend, &Divisor); +} + +# define DIV_LOAD(a_Dividend) \ + a_Dividend.s.Lo = *puA, a_Dividend.s.Hi = *puD +# define DIV_LOAD_U8(a_Dividend) \ + a_Dividend.u = *puAX + +# define DIV_STORE(a_Quotient, a_uReminder) *puA = (a_Quotient), *puD = (a_uReminder) +# define DIV_STORE_U8(a_Quotient, a_uReminder) *puAX = (uint8_t)(a_Quotient) | ((uint16_t)(a_uReminder) << 8) + +# define MUL_LOAD_F1() *puA +# define MUL_LOAD_F1_U8() ((uint8_t)*puAX) + +# define MUL_STORE(a_Result) *puA = (a_Result).s.Lo, *puD = (a_Result).s.Hi +# define MUL_STORE_U8(a_Result) *puAX = a_Result.u + +# define MULDIV_NEG(a_Value, a_cBitsWidth2x) \ + (a_Value).u = UINT ## a_cBitsWidth2x ## _C(0) - (a_Value).u +# define MULDIV_NEG_U128(a_Value, a_cBitsWidth2x) \ + RTUInt128AssignNeg(&(a_Value)) + +# define MULDIV_MUL(a_Result, a_Factor1, a_Factor2, a_cBitsWidth2x) \ + (a_Result).u = (uint ## a_cBitsWidth2x ## _t)(a_Factor1) * (a_Factor2) +# define MULDIV_MUL_U128(a_Result, a_Factor1, a_Factor2, a_cBitsWidth2x) \ + RTUInt128MulU64ByU64(&(a_Result), a_Factor1, a_Factor2); + +# define MULDIV_MODDIV(a_Quotient, a_Remainder, a_Dividend, a_uDivisor) \ + a_Quotient.u = (a_Dividend).u / (a_uDivisor), \ + a_Remainder.u = (a_Dividend).u % (a_uDivisor) +# define MULDIV_MODDIV_U128(a_Quotient, a_Remainder, a_Dividend, a_uDivisor) \ + RTUInt128DivRemByU64(&a_Quotient, &a_Remainder, &a_Dividend, a_uDivisor) + + +/* + * MUL + */ +# define EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_mul_u,a_cBitsWidth,a_Suffix), a_Args) \ +{ \ + RTUINT ## a_cBitsWidth2x ## U Result; \ + a_fnMul(Result, a_fnLoadF1(), uFactor, a_cBitsWidth2x); \ + a_fnStore(Result); \ + \ + /* Calc EFLAGS: */ \ + uint32_t fEfl = *pfEFlags; \ + if (a_fIntelFlags) \ + { /* Intel: 6700K and 10980XE behavior */ \ + fEfl &= ~(X86_EFL_SF | X86_EFL_CF | X86_EFL_OF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_PF); \ + if (Result.s.Lo & RT_BIT_64(a_cBitsWidth - 1)) \ + fEfl |= X86_EFL_SF; \ + fEfl |= g_afParity[Result.s.Lo & 0xff]; \ + if (Result.s.Hi != 0) \ + fEfl |= X86_EFL_CF | X86_EFL_OF; \ + } \ + else \ + { /* AMD: 3990X */ \ + if (Result.s.Hi != 0) \ + fEfl |= X86_EFL_CF | X86_EFL_OF; \ + else \ + fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \ + } \ + *pfEFlags = fEfl; \ + return 0; \ +} \ + +# define EMIT_MUL(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul) \ + EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, RT_NOTHING, 1) \ + EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, _intel, 1) \ + EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, _amd, 0) \ + +# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */ +EMIT_MUL(64, 128, (uint64_t *puA, uint64_t *puD, uint64_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags), + MUL_LOAD_F1, MUL_STORE, MULDIV_MUL_U128) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_MUL(32, 64, (uint32_t *puA, uint32_t *puD, uint32_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags), + MUL_LOAD_F1, MUL_STORE, MULDIV_MUL) +EMIT_MUL(16, 32, (uint16_t *puA, uint16_t *puD, uint16_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags), + MUL_LOAD_F1, MUL_STORE, MULDIV_MUL) +EMIT_MUL(8, 16, (uint16_t *puAX, uint8_t uFactor, uint32_t *pfEFlags), (puAX, uFactor, pfEFlags), + MUL_LOAD_F1_U8, MUL_STORE_U8, MULDIV_MUL) +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ +# endif /* !DOXYGEN_RUNNING */ + +/* + * MULX + */ +# define EMIT_MULX(a_cBitsWidth, a_cBitsWidth2x, a_uType, a_fnMul, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_mulx_u,a_cBitsWidth,a_Suffix), \ + (a_uType *puDst1, a_uType *puDst2, a_uType uSrc1, a_uType uSrc2)) \ +{ \ + RTUINT ## a_cBitsWidth2x ## U Result; \ + a_fnMul(Result, uSrc1, uSrc2, a_cBitsWidth2x); \ + *puDst2 = Result.s.Lo; /* Lower part first, as we should return the high part when puDst2 == puDst1. */ \ + *puDst1 = Result.s.Hi; \ +} \ + +# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */ +EMIT_MULX(64, 128, uint64_t, MULDIV_MUL_U128, RT_NOTHING) +EMIT_MULX(64, 128, uint64_t, MULDIV_MUL_U128, _fallback) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_MULX(32, 64, uint32_t, MULDIV_MUL, RT_NOTHING) +EMIT_MULX(32, 64, uint32_t, MULDIV_MUL, _fallback) +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ +# endif /* !DOXYGEN_RUNNING */ + + +/* + * IMUL + * + * The SF, ZF, AF and PF flags are "undefined". AMD (3990x) leaves these + * flags as is. Whereas Intel skylake (6700K and 10980X (Cascade Lake)) always + * clear AF and ZF and calculates SF and PF as per the lower half of the result. + */ +# define EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, \ + a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_imul_u,a_cBitsWidth,a_Suffix),a_Args) \ +{ \ + RTUINT ## a_cBitsWidth2x ## U Result; \ + uint32_t fEfl = *pfEFlags & ~(X86_EFL_CF | X86_EFL_OF); \ + \ + uint ## a_cBitsWidth ## _t const uFactor1 = a_fnLoadF1(); \ + if (!(uFactor1 & RT_BIT_64(a_cBitsWidth - 1))) \ + { \ + if (!(uFactor2 & RT_BIT_64(a_cBitsWidth - 1))) \ + { \ + a_fnMul(Result, uFactor1, uFactor2, a_cBitsWidth2x); \ + if (Result.s.Hi != 0 || Result.s.Lo >= RT_BIT_64(a_cBitsWidth - 1)) \ + fEfl |= X86_EFL_CF | X86_EFL_OF; \ + } \ + else \ + { \ + uint ## a_cBitsWidth ## _t const uPositiveFactor2 = UINT ## a_cBitsWidth ## _C(0) - uFactor2; \ + a_fnMul(Result, uFactor1, uPositiveFactor2, a_cBitsWidth2x); \ + if (Result.s.Hi != 0 || Result.s.Lo > RT_BIT_64(a_cBitsWidth - 1)) \ + fEfl |= X86_EFL_CF | X86_EFL_OF; \ + a_fnNeg(Result, a_cBitsWidth2x); \ + } \ + } \ + else \ + { \ + if (!(uFactor2 & RT_BIT_64(a_cBitsWidth - 1))) \ + { \ + uint ## a_cBitsWidth ## _t const uPositiveFactor1 = UINT ## a_cBitsWidth ## _C(0) - uFactor1; \ + a_fnMul(Result, uPositiveFactor1, uFactor2, a_cBitsWidth2x); \ + if (Result.s.Hi != 0 || Result.s.Lo > RT_BIT_64(a_cBitsWidth - 1)) \ + fEfl |= X86_EFL_CF | X86_EFL_OF; \ + a_fnNeg(Result, a_cBitsWidth2x); \ + } \ + else \ + { \ + uint ## a_cBitsWidth ## _t const uPositiveFactor1 = UINT ## a_cBitsWidth ## _C(0) - uFactor1; \ + uint ## a_cBitsWidth ## _t const uPositiveFactor2 = UINT ## a_cBitsWidth ## _C(0) - uFactor2; \ + a_fnMul(Result, uPositiveFactor1, uPositiveFactor2, a_cBitsWidth2x); \ + if (Result.s.Hi != 0 || Result.s.Lo >= RT_BIT_64(a_cBitsWidth - 1)) \ + fEfl |= X86_EFL_CF | X86_EFL_OF; \ + } \ + } \ + a_fnStore(Result); \ + \ + if (a_fIntelFlags) \ + { \ + fEfl &= ~(X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF | X86_EFL_PF); \ + if (Result.s.Lo & RT_BIT_64(a_cBitsWidth - 1)) \ + fEfl |= X86_EFL_SF; \ + fEfl |= g_afParity[Result.s.Lo & 0xff]; \ + } \ + *pfEFlags = fEfl; \ + return 0; \ +} +# define EMIT_IMUL(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul) \ + EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, RT_NOTHING, 1) \ + EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, _intel, 1) \ + EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, _amd, 0) + +# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */ +EMIT_IMUL(64, 128, (uint64_t *puA, uint64_t *puD, uint64_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags), + MUL_LOAD_F1, MUL_STORE, MULDIV_NEG_U128, MULDIV_MUL_U128) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_IMUL(32, 64, (uint32_t *puA, uint32_t *puD, uint32_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags), + MUL_LOAD_F1, MUL_STORE, MULDIV_NEG, MULDIV_MUL) +EMIT_IMUL(16, 32, (uint16_t *puA, uint16_t *puD, uint16_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags), + MUL_LOAD_F1, MUL_STORE, MULDIV_NEG, MULDIV_MUL) +EMIT_IMUL(8, 16, (uint16_t *puAX, uint8_t uFactor2, uint32_t *pfEFlags), (puAX, uFactor2, pfEFlags), + MUL_LOAD_F1_U8, MUL_STORE_U8, MULDIV_NEG, MULDIV_MUL) +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ +# endif /* !DOXYGEN_RUNNING */ + + +/* + * IMUL with two operands are mapped onto the three operand variant, ignoring + * the high part of the product. + */ +# define EMIT_IMUL_TWO(a_cBits, a_uType) \ +IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits,(a_uType *puDst, a_uType uSrc, uint32_t *pfEFlags)) \ +{ \ + a_uType uIgn; \ + iemAImpl_imul_u ## a_cBits(puDst, &uIgn, uSrc, pfEFlags); \ +} \ +\ +IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits ## _intel,(a_uType *puDst, a_uType uSrc, uint32_t *pfEFlags)) \ +{ \ + a_uType uIgn; \ + iemAImpl_imul_u ## a_cBits ## _intel(puDst, &uIgn, uSrc, pfEFlags); \ +} \ +\ +IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits ## _amd,(a_uType *puDst, a_uType uSrc, uint32_t *pfEFlags)) \ +{ \ + a_uType uIgn; \ + iemAImpl_imul_u ## a_cBits ## _amd(puDst, &uIgn, uSrc, pfEFlags); \ +} + +EMIT_IMUL_TWO(64, uint64_t) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_IMUL_TWO(32, uint32_t) +EMIT_IMUL_TWO(16, uint16_t) +# endif + + +/* + * DIV + */ +# define EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, \ + a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_div_u,a_cBitsWidth,a_Suffix),a_Args) \ +{ \ + RTUINT ## a_cBitsWidth2x ## U Dividend; \ + a_fnLoad(Dividend); \ + if ( uDivisor != 0 \ + && Dividend.s.Hi < uDivisor) \ + { \ + RTUINT ## a_cBitsWidth2x ## U Remainder, Quotient; \ + a_fnDivRem(Quotient, Remainder, Dividend, uDivisor); \ + a_fnStore(Quotient.s.Lo, Remainder.s.Lo); \ + \ + /* Calc EFLAGS: Intel 6700K and 10980XE leaves them alone. AMD 3990X sets AF and clears PF, ZF and SF. */ \ + if (!a_fIntelFlags) \ + *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \ + return 0; \ + } \ + /* #DE */ \ + return -1; \ +} +# define EMIT_DIV(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem) \ + EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, RT_NOTHING, 1) \ + EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, _intel, 1) \ + EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, _amd, 0) + +# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */ +EMIT_DIV(64,128,(uint64_t *puA, uint64_t *puD, uint64_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags), + DIV_LOAD, DIV_STORE, MULDIV_MODDIV_U128) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_DIV(32,64, (uint32_t *puA, uint32_t *puD, uint32_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags), + DIV_LOAD, DIV_STORE, MULDIV_MODDIV) +EMIT_DIV(16,32, (uint16_t *puA, uint16_t *puD, uint16_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags), + DIV_LOAD, DIV_STORE, MULDIV_MODDIV) +EMIT_DIV(8,16, (uint16_t *puAX, uint8_t uDivisor, uint32_t *pfEFlags), (puAX, uDivisor, pfEFlags), + DIV_LOAD_U8, DIV_STORE_U8, MULDIV_MODDIV) +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ +# endif /* !DOXYGEN_RUNNING */ + + +/* + * IDIV + * + * EFLAGS are ignored and left as-is by Intel 6700K and 10980XE. AMD 3990X will + * set AF and clear PF, ZF and SF just like it does for DIV. + * + */ +# define EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, \ + a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_idiv_u,a_cBitsWidth,a_Suffix),a_Args) \ +{ \ + /* Note! Skylake leaves all flags alone. */ \ + \ + /** @todo overflow checks */ \ + if (uDivisor != 0) \ + { \ + /* \ + * Convert to unsigned division. \ + */ \ + RTUINT ## a_cBitsWidth2x ## U Dividend; \ + a_fnLoad(Dividend); \ + bool const fSignedDividend = RT_BOOL(Dividend.s.Hi & RT_BIT_64(a_cBitsWidth - 1)); \ + if (fSignedDividend) \ + a_fnNeg(Dividend, a_cBitsWidth2x); \ + \ + uint ## a_cBitsWidth ## _t uDivisorPositive; \ + if (!(uDivisor & RT_BIT_64(a_cBitsWidth - 1))) \ + uDivisorPositive = uDivisor; \ + else \ + uDivisorPositive = UINT ## a_cBitsWidth ## _C(0) - uDivisor; \ + \ + RTUINT ## a_cBitsWidth2x ## U Remainder, Quotient; \ + a_fnDivRem(Quotient, Remainder, Dividend, uDivisorPositive); \ + \ + /* \ + * Setup the result, checking for overflows. \ + */ \ + if (!(uDivisor & RT_BIT_64(a_cBitsWidth - 1))) \ + { \ + if (!fSignedDividend) \ + { \ + /* Positive divisor, positive dividend => result positive. */ \ + if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint ## a_cBitsWidth ## _t)INT ## a_cBitsWidth ## _MAX) \ + { \ + a_fnStore(Quotient.s.Lo, Remainder.s.Lo); \ + if (!a_fIntelFlags) \ + *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \ + return 0; \ + } \ + } \ + else \ + { \ + /* Positive divisor, negative dividend => result negative. */ \ + if (Quotient.s.Hi == 0 && Quotient.s.Lo <= RT_BIT_64(a_cBitsWidth - 1)) \ + { \ + a_fnStore(UINT ## a_cBitsWidth ## _C(0) - Quotient.s.Lo, UINT ## a_cBitsWidth ## _C(0) - Remainder.s.Lo); \ + if (!a_fIntelFlags) \ + *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \ + return 0; \ + } \ + } \ + } \ + else \ + { \ + if (!fSignedDividend) \ + { \ + /* Negative divisor, positive dividend => negative quotient, positive remainder. */ \ + if (Quotient.s.Hi == 0 && Quotient.s.Lo <= RT_BIT_64(a_cBitsWidth - 1)) \ + { \ + a_fnStore(UINT ## a_cBitsWidth ## _C(0) - Quotient.s.Lo, Remainder.s.Lo); \ + if (!a_fIntelFlags) \ + *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \ + return 0; \ + } \ + } \ + else \ + { \ + /* Negative divisor, negative dividend => positive quotient, negative remainder. */ \ + if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint ## a_cBitsWidth ## _t)INT ## a_cBitsWidth ## _MAX) \ + { \ + a_fnStore(Quotient.s.Lo, UINT ## a_cBitsWidth ## _C(0) - Remainder.s.Lo); \ + if (!a_fIntelFlags) \ + *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \ + return 0; \ + } \ + } \ + } \ + } \ + /* #DE */ \ + return -1; \ +} +# define EMIT_IDIV(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem) \ + EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, RT_NOTHING, 1) \ + EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, _intel, 1) \ + EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, _amd, 0) + +# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */ +EMIT_IDIV(64,128,(uint64_t *puA, uint64_t *puD, uint64_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags), + DIV_LOAD, DIV_STORE, MULDIV_NEG_U128, MULDIV_MODDIV_U128) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_IDIV(32,64,(uint32_t *puA, uint32_t *puD, uint32_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags), + DIV_LOAD, DIV_STORE, MULDIV_NEG, MULDIV_MODDIV) +EMIT_IDIV(16,32,(uint16_t *puA, uint16_t *puD, uint16_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags), + DIV_LOAD, DIV_STORE, MULDIV_NEG, MULDIV_MODDIV) +EMIT_IDIV(8,16,(uint16_t *puAX, uint8_t uDivisor, uint32_t *pfEFlags), (puAX, uDivisor, pfEFlags), + DIV_LOAD_U8, DIV_STORE_U8, MULDIV_NEG, MULDIV_MODDIV) +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ +# endif /* !DOXYGEN_RUNNING */ + +#endif /* (!defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)) && !defined(DOXYGEN_RUNNING) */ + + +/********************************************************************************************************************************* +* Unary operations. * +*********************************************************************************************************************************/ +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) + +/** @def IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC + * Updates the status bits (CF, PF, AF, ZF, SF, and OF) for an INC or DEC instruction. + * + * CF is NOT modified for hysterical raisins (allegedly for carrying and + * borrowing in arithmetic loops on intel 8008). + * + * @returns Status bits. + * @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update. + * @param a_uResult Unsigned result value. + * @param a_uDst The original destination value (for AF calc). + * @param a_cBitsWidth The width of the result (8, 16, 32, 64). + * @param a_OfMethod 0 for INC-style, 1 for DEC-style. + */ +#define IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(a_pfEFlags, a_uResult, a_uDst, a_cBitsWidth, a_OfMethod) \ + do { \ + uint32_t fEflTmp = *(a_pfEFlags); \ + fEflTmp &= ~X86_EFL_STATUS_BITS | X86_EFL_CF; \ + fEflTmp |= g_afParity[(a_uResult) & 0xff]; \ + fEflTmp |= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \ + fEflTmp |= X86_EFL_CALC_ZF(a_uResult); \ + fEflTmp |= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \ + fEflTmp |= X86_EFL_GET_OF_ ## a_cBitsWidth(a_OfMethod == 0 ? (((a_uDst) ^ RT_BIT_64(a_cBitsWidth - 1)) & (a_uResult)) \ + : ((a_uDst) & ((a_uResult) ^ RT_BIT_64(a_cBitsWidth - 1))) ); \ + *(a_pfEFlags) = fEflTmp; \ + } while (0) + +/* + * INC + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u64,(uint64_t *puDst, uint32_t *pfEFlags)) +{ + uint64_t uDst = *puDst; + uint64_t uResult = uDst + 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 64, 0 /*INC*/); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u32,(uint32_t *puDst, uint32_t *pfEFlags)) +{ + uint32_t uDst = *puDst; + uint32_t uResult = uDst + 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 32, 0 /*INC*/); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u16,(uint16_t *puDst, uint32_t *pfEFlags)) +{ + uint16_t uDst = *puDst; + uint16_t uResult = uDst + 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 16, 0 /*INC*/); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u8,(uint8_t *puDst, uint32_t *pfEFlags)) +{ + uint8_t uDst = *puDst; + uint8_t uResult = uDst + 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 8, 0 /*INC*/); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * DEC + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u64,(uint64_t *puDst, uint32_t *pfEFlags)) +{ + uint64_t uDst = *puDst; + uint64_t uResult = uDst - 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 64, 1 /*INC*/); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u32,(uint32_t *puDst, uint32_t *pfEFlags)) +{ + uint32_t uDst = *puDst; + uint32_t uResult = uDst - 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 32, 1 /*INC*/); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u16,(uint16_t *puDst, uint32_t *pfEFlags)) +{ + uint16_t uDst = *puDst; + uint16_t uResult = uDst - 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 16, 1 /*INC*/); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u8,(uint8_t *puDst, uint32_t *pfEFlags)) +{ + uint8_t uDst = *puDst; + uint8_t uResult = uDst - 1; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 8, 1 /*INC*/); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * NOT + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_not_u64,(uint64_t *puDst, uint32_t *pfEFlags)) +{ + uint64_t uDst = *puDst; + uint64_t uResult = ~uDst; + *puDst = uResult; + /* EFLAGS are not modified. */ + RT_NOREF_PV(pfEFlags); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_not_u32,(uint32_t *puDst, uint32_t *pfEFlags)) +{ + uint32_t uDst = *puDst; + uint32_t uResult = ~uDst; + *puDst = uResult; + /* EFLAGS are not modified. */ + RT_NOREF_PV(pfEFlags); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_not_u16,(uint16_t *puDst, uint32_t *pfEFlags)) +{ + uint16_t uDst = *puDst; + uint16_t uResult = ~uDst; + *puDst = uResult; + /* EFLAGS are not modified. */ + RT_NOREF_PV(pfEFlags); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_not_u8,(uint8_t *puDst, uint32_t *pfEFlags)) +{ + uint8_t uDst = *puDst; + uint8_t uResult = ~uDst; + *puDst = uResult; + /* EFLAGS are not modified. */ + RT_NOREF_PV(pfEFlags); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/* + * NEG + */ + +/** + * Updates the status bits (CF, PF, AF, ZF, SF, and OF) for an NEG instruction. + * + * @returns Status bits. + * @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update. + * @param a_uResult Unsigned result value. + * @param a_uDst The original destination value (for AF calc). + * @param a_cBitsWidth The width of the result (8, 16, 32, 64). + */ +#define IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(a_pfEFlags, a_uResult, a_uDst, a_cBitsWidth) \ + do { \ + uint32_t fEflTmp = *(a_pfEFlags); \ + fEflTmp &= ~X86_EFL_STATUS_BITS & ~X86_EFL_CF; \ + fEflTmp |= ((a_uDst) != 0) << X86_EFL_CF_BIT; \ + fEflTmp |= g_afParity[(a_uResult) & 0xff]; \ + fEflTmp |= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \ + fEflTmp |= X86_EFL_CALC_ZF(a_uResult); \ + fEflTmp |= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \ + fEflTmp |= X86_EFL_GET_OF_ ## a_cBitsWidth((a_uDst) & (a_uResult)); \ + *(a_pfEFlags) = fEflTmp; \ + } while (0) + +IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u64,(uint64_t *puDst, uint32_t *pfEFlags)) +{ + uint64_t uDst = *puDst; + uint64_t uResult = (uint64_t)0 - uDst; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 64); +} + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u32,(uint32_t *puDst, uint32_t *pfEFlags)) +{ + uint32_t uDst = *puDst; + uint32_t uResult = (uint32_t)0 - uDst; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 32); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u16,(uint16_t *puDst, uint32_t *pfEFlags)) +{ + uint16_t uDst = *puDst; + uint16_t uResult = (uint16_t)0 - uDst; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 16); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u8,(uint8_t *puDst, uint32_t *pfEFlags)) +{ + uint8_t uDst = *puDst; + uint8_t uResult = (uint8_t)0 - uDst; + *puDst = uResult; + IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 8); +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + +/* + * Locked variants. + */ + +/** Emit a function for doing a locked unary operand operation. */ +# define EMIT_LOCKED_UNARY_OP(a_Mnemonic, a_cBitsWidth) \ + IEM_DECL_IMPL_DEF(void, iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth ## _locked,(uint ## a_cBitsWidth ## _t *puDst, \ + uint32_t *pfEFlags)) \ + { \ + uint ## a_cBitsWidth ## _t uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \ + uint ## a_cBitsWidth ## _t uTmp; \ + uint32_t fEflTmp; \ + do \ + { \ + uTmp = uOld; \ + fEflTmp = *pfEFlags; \ + iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth(&uTmp, &fEflTmp); \ + } while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uTmp, uOld, &uOld)); \ + *pfEFlags = fEflTmp; \ + } + +EMIT_LOCKED_UNARY_OP(inc, 64) +EMIT_LOCKED_UNARY_OP(dec, 64) +EMIT_LOCKED_UNARY_OP(not, 64) +EMIT_LOCKED_UNARY_OP(neg, 64) +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_LOCKED_UNARY_OP(inc, 32) +EMIT_LOCKED_UNARY_OP(dec, 32) +EMIT_LOCKED_UNARY_OP(not, 32) +EMIT_LOCKED_UNARY_OP(neg, 32) + +EMIT_LOCKED_UNARY_OP(inc, 16) +EMIT_LOCKED_UNARY_OP(dec, 16) +EMIT_LOCKED_UNARY_OP(not, 16) +EMIT_LOCKED_UNARY_OP(neg, 16) + +EMIT_LOCKED_UNARY_OP(inc, 8) +EMIT_LOCKED_UNARY_OP(dec, 8) +EMIT_LOCKED_UNARY_OP(not, 8) +EMIT_LOCKED_UNARY_OP(neg, 8) +# endif + +#endif /* !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) */ + + +/********************************************************************************************************************************* +* Shifting and Rotating * +*********************************************************************************************************************************/ + +/* + * ROL + */ +#define EMIT_ROL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags, a_fnHlp) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rol_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \ + if (cShift) \ + { \ + if (a_cBitsWidth < 32) \ + cShift &= a_cBitsWidth - 1; \ + a_uType const uDst = *puDst; \ + a_uType const uResult = a_fnHlp(uDst, cShift); \ + *puDst = uResult; \ + \ + /* Calc EFLAGS. The OF bit is undefined if cShift > 1, we implement \ + it the same way as for 1 bit shifts. */ \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + uint32_t fEfl = *pfEFlags; \ + fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \ + uint32_t const fCarry = (uResult & X86_EFL_CF); \ + fEfl |= fCarry; \ + if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \ + fEfl |= ((uResult >> (a_cBitsWidth - 1)) ^ fCarry) << X86_EFL_OF_BIT; \ + else /* Intel 10980XE: According to the first sub-shift: */ \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \ + *pfEFlags = fEfl; \ + } \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_ROL(64, uint64_t, RT_NOTHING, 1, ASMRotateLeftU64) +#endif +EMIT_ROL(64, uint64_t, _intel, 1, ASMRotateLeftU64) +EMIT_ROL(64, uint64_t, _amd, 0, ASMRotateLeftU64) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_ROL(32, uint32_t, RT_NOTHING, 1, ASMRotateLeftU32) +#endif +EMIT_ROL(32, uint32_t, _intel, 1, ASMRotateLeftU32) +EMIT_ROL(32, uint32_t, _amd, 0, ASMRotateLeftU32) + +DECL_FORCE_INLINE(uint16_t) iemAImpl_rol_u16_hlp(uint16_t uValue, uint8_t cShift) +{ + return (uValue << cShift) | (uValue >> (16 - cShift)); +} +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_ROL(16, uint16_t, RT_NOTHING, 1, iemAImpl_rol_u16_hlp) +#endif +EMIT_ROL(16, uint16_t, _intel, 1, iemAImpl_rol_u16_hlp) +EMIT_ROL(16, uint16_t, _amd, 0, iemAImpl_rol_u16_hlp) + +DECL_FORCE_INLINE(uint8_t) iemAImpl_rol_u8_hlp(uint8_t uValue, uint8_t cShift) +{ + return (uValue << cShift) | (uValue >> (8 - cShift)); +} +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_ROL(8, uint8_t, RT_NOTHING, 1, iemAImpl_rol_u8_hlp) +#endif +EMIT_ROL(8, uint8_t, _intel, 1, iemAImpl_rol_u8_hlp) +EMIT_ROL(8, uint8_t, _amd, 0, iemAImpl_rol_u8_hlp) + + +/* + * ROR + */ +#define EMIT_ROR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags, a_fnHlp) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_ror_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \ + if (cShift) \ + { \ + if (a_cBitsWidth < 32) \ + cShift &= a_cBitsWidth - 1; \ + a_uType const uDst = *puDst; \ + a_uType const uResult = a_fnHlp(uDst, cShift); \ + *puDst = uResult; \ + \ + /* Calc EFLAGS: */ \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + uint32_t fEfl = *pfEFlags; \ + fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \ + uint32_t const fCarry = (uResult >> ((a_cBitsWidth) - 1)) & X86_EFL_CF; \ + fEfl |= fCarry; \ + if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \ + fEfl |= (((uResult >> ((a_cBitsWidth) - 2)) ^ fCarry) & 1) << X86_EFL_OF_BIT; \ + else /* Intel 10980XE: According to the first sub-shift: */ \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << (a_cBitsWidth - 1))); \ + *pfEFlags = fEfl; \ + } \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_ROR(64, uint64_t, RT_NOTHING, 1, ASMRotateRightU64) +#endif +EMIT_ROR(64, uint64_t, _intel, 1, ASMRotateRightU64) +EMIT_ROR(64, uint64_t, _amd, 0, ASMRotateRightU64) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_ROR(32, uint32_t, RT_NOTHING, 1, ASMRotateRightU32) +#endif +EMIT_ROR(32, uint32_t, _intel, 1, ASMRotateRightU32) +EMIT_ROR(32, uint32_t, _amd, 0, ASMRotateRightU32) + +DECL_FORCE_INLINE(uint16_t) iemAImpl_ror_u16_hlp(uint16_t uValue, uint8_t cShift) +{ + return (uValue >> cShift) | (uValue << (16 - cShift)); +} +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_ROR(16, uint16_t, RT_NOTHING, 1, iemAImpl_ror_u16_hlp) +#endif +EMIT_ROR(16, uint16_t, _intel, 1, iemAImpl_ror_u16_hlp) +EMIT_ROR(16, uint16_t, _amd, 0, iemAImpl_ror_u16_hlp) + +DECL_FORCE_INLINE(uint8_t) iemAImpl_ror_u8_hlp(uint8_t uValue, uint8_t cShift) +{ + return (uValue >> cShift) | (uValue << (8 - cShift)); +} +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_ROR(8, uint8_t, RT_NOTHING, 1, iemAImpl_ror_u8_hlp) +#endif +EMIT_ROR(8, uint8_t, _intel, 1, iemAImpl_ror_u8_hlp) +EMIT_ROR(8, uint8_t, _amd, 0, iemAImpl_ror_u8_hlp) + + +/* + * RCL + */ +#define EMIT_RCL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rcl_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \ + if (a_cBitsWidth < 32 && a_fIntelFlags) \ + cShift %= a_cBitsWidth + 1; \ + if (cShift) \ + { \ + if (a_cBitsWidth < 32 && !a_fIntelFlags) \ + cShift %= a_cBitsWidth + 1; \ + a_uType const uDst = *puDst; \ + a_uType uResult = uDst << cShift; \ + if (cShift > 1) \ + uResult |= uDst >> (a_cBitsWidth + 1 - cShift); \ + \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + uint32_t fEfl = *pfEFlags; \ + uint32_t fInCarry = fEfl & X86_EFL_CF; \ + uResult |= (a_uType)fInCarry << (cShift - 1); \ + \ + *puDst = uResult; \ + \ + /* Calc EFLAGS. */ \ + fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \ + uint32_t const fOutCarry = a_cBitsWidth >= 32 || a_fIntelFlags || cShift \ + ? (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF : fInCarry; \ + fEfl |= fOutCarry; \ + if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \ + fEfl |= ((uResult >> (a_cBitsWidth - 1)) ^ fOutCarry) << X86_EFL_OF_BIT; \ + else /* Intel 10980XE: According to the first sub-shift: */ \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \ + *pfEFlags = fEfl; \ + } \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RCL(64, uint64_t, RT_NOTHING, 1) +#endif +EMIT_RCL(64, uint64_t, _intel, 1) +EMIT_RCL(64, uint64_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RCL(32, uint32_t, RT_NOTHING, 1) +#endif +EMIT_RCL(32, uint32_t, _intel, 1) +EMIT_RCL(32, uint32_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RCL(16, uint16_t, RT_NOTHING, 1) +#endif +EMIT_RCL(16, uint16_t, _intel, 1) +EMIT_RCL(16, uint16_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RCL(8, uint8_t, RT_NOTHING, 1) +#endif +EMIT_RCL(8, uint8_t, _intel, 1) +EMIT_RCL(8, uint8_t, _amd, 0) + + +/* + * RCR + */ +#define EMIT_RCR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rcr_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \ + if (a_cBitsWidth < 32 && a_fIntelFlags) \ + cShift %= a_cBitsWidth + 1; \ + if (cShift) \ + { \ + if (a_cBitsWidth < 32 && !a_fIntelFlags) \ + cShift %= a_cBitsWidth + 1; \ + a_uType const uDst = *puDst; \ + a_uType uResult = uDst >> cShift; \ + if (cShift > 1) \ + uResult |= uDst << (a_cBitsWidth + 1 - cShift); \ + \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + uint32_t fEfl = *pfEFlags; \ + uint32_t fInCarry = fEfl & X86_EFL_CF; \ + uResult |= (a_uType)fInCarry << (a_cBitsWidth - cShift); \ + *puDst = uResult; \ + \ + /* Calc EFLAGS. The OF bit is undefined if cShift > 1, we implement \ + it the same way as for 1 bit shifts. */ \ + fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \ + uint32_t const fOutCarry = a_cBitsWidth >= 32 || a_fIntelFlags || cShift \ + ? (uDst >> (cShift - 1)) & X86_EFL_CF : fInCarry; \ + fEfl |= fOutCarry; \ + if (!a_fIntelFlags) /* AMD 3990X: XOR two most signficant bits of the result: */ \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uResult ^ (uResult << 1)); \ + else /* Intel 10980XE: same as AMD, but only for the first sub-shift: */ \ + fEfl |= (fInCarry ^ (uint32_t)(uDst >> (a_cBitsWidth - 1))) << X86_EFL_OF_BIT; \ + *pfEFlags = fEfl; \ + } \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RCR(64, uint64_t, RT_NOTHING, 1) +#endif +EMIT_RCR(64, uint64_t, _intel, 1) +EMIT_RCR(64, uint64_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RCR(32, uint32_t, RT_NOTHING, 1) +#endif +EMIT_RCR(32, uint32_t, _intel, 1) +EMIT_RCR(32, uint32_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RCR(16, uint16_t, RT_NOTHING, 1) +#endif +EMIT_RCR(16, uint16_t, _intel, 1) +EMIT_RCR(16, uint16_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RCR(8, uint8_t, RT_NOTHING, 1) +#endif +EMIT_RCR(8, uint8_t, _intel, 1) +EMIT_RCR(8, uint8_t, _amd, 0) + + +/* + * SHL + */ +#define EMIT_SHL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shl_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \ + if (cShift) \ + { \ + a_uType const uDst = *puDst; \ + a_uType uResult = uDst << cShift; \ + *puDst = uResult; \ + \ + /* Calc EFLAGS. */ \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \ + uint32_t fCarry = (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF; \ + fEfl |= fCarry; \ + if (!a_fIntelFlags) \ + fEfl |= ((uResult >> (a_cBitsWidth - 1)) ^ fCarry) << X86_EFL_OF_BIT; /* AMD 3990X: Last shift result. */ \ + else \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); /* Intel 10980XE: First shift result. */ \ + fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \ + fEfl |= X86_EFL_CALC_ZF(uResult); \ + fEfl |= g_afParity[uResult & 0xff]; \ + if (!a_fIntelFlags) \ + fEfl |= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \ + *pfEFlags = fEfl; \ + } \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHL(64, uint64_t, RT_NOTHING, 1) +#endif +EMIT_SHL(64, uint64_t, _intel, 1) +EMIT_SHL(64, uint64_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHL(32, uint32_t, RT_NOTHING, 1) +#endif +EMIT_SHL(32, uint32_t, _intel, 1) +EMIT_SHL(32, uint32_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHL(16, uint16_t, RT_NOTHING, 1) +#endif +EMIT_SHL(16, uint16_t, _intel, 1) +EMIT_SHL(16, uint16_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHL(8, uint8_t, RT_NOTHING, 1) +#endif +EMIT_SHL(8, uint8_t, _intel, 1) +EMIT_SHL(8, uint8_t, _amd, 0) + + +/* + * SHR + */ +#define EMIT_SHR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shr_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \ + if (cShift) \ + { \ + a_uType const uDst = *puDst; \ + a_uType uResult = uDst >> cShift; \ + *puDst = uResult; \ + \ + /* Calc EFLAGS. */ \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \ + fEfl |= (uDst >> (cShift - 1)) & X86_EFL_CF; \ + if (a_fIntelFlags || cShift == 1) /* AMD 3990x does what intel documents; Intel 10980XE does this for all shift counts. */ \ + fEfl |= (uDst >> (a_cBitsWidth - 1)) << X86_EFL_OF_BIT; \ + fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \ + fEfl |= X86_EFL_CALC_ZF(uResult); \ + fEfl |= g_afParity[uResult & 0xff]; \ + if (!a_fIntelFlags) \ + fEfl |= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \ + *pfEFlags = fEfl; \ + } \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHR(64, uint64_t, RT_NOTHING, 1) +#endif +EMIT_SHR(64, uint64_t, _intel, 1) +EMIT_SHR(64, uint64_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHR(32, uint32_t, RT_NOTHING, 1) +#endif +EMIT_SHR(32, uint32_t, _intel, 1) +EMIT_SHR(32, uint32_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHR(16, uint16_t, RT_NOTHING, 1) +#endif +EMIT_SHR(16, uint16_t, _intel, 1) +EMIT_SHR(16, uint16_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHR(8, uint8_t, RT_NOTHING, 1) +#endif +EMIT_SHR(8, uint8_t, _intel, 1) +EMIT_SHR(8, uint8_t, _amd, 0) + + +/* + * SAR + */ +#define EMIT_SAR(a_cBitsWidth, a_uType, a_iType, a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_sar_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \ + if (cShift) \ + { \ + a_iType const iDst = (a_iType)*puDst; \ + a_uType uResult = iDst >> cShift; \ + *puDst = uResult; \ + \ + /* Calc EFLAGS. \ + Note! The OF flag is always zero because the result never differs from the input. */ \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \ + fEfl |= (iDst >> (cShift - 1)) & X86_EFL_CF; \ + fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \ + fEfl |= X86_EFL_CALC_ZF(uResult); \ + fEfl |= g_afParity[uResult & 0xff]; \ + if (!a_fIntelFlags) \ + fEfl |= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \ + *pfEFlags = fEfl; \ + } \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SAR(64, uint64_t, int64_t, RT_NOTHING, 1) +#endif +EMIT_SAR(64, uint64_t, int64_t, _intel, 1) +EMIT_SAR(64, uint64_t, int64_t, _amd, 0) + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SAR(32, uint32_t, int32_t, RT_NOTHING, 1) +#endif +EMIT_SAR(32, uint32_t, int32_t, _intel, 1) +EMIT_SAR(32, uint32_t, int32_t, _amd, 0) + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SAR(16, uint16_t, int16_t, RT_NOTHING, 1) +#endif +EMIT_SAR(16, uint16_t, int16_t, _intel, 1) +EMIT_SAR(16, uint16_t, int16_t, _amd, 0) + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SAR(8, uint8_t, int8_t, RT_NOTHING, 1) +#endif +EMIT_SAR(8, uint8_t, int8_t, _intel, 1) +EMIT_SAR(8, uint8_t, int8_t, _amd, 0) + + +/* + * SHLD + * + * - CF is the last bit shifted out of puDst. + * - AF is always cleared by Intel 10980XE. + * - AF is always set by AMD 3990X. + * - OF is set according to the first shift on Intel 10980XE, it seems. + * - OF is set according to the last sub-shift on AMD 3990X. + * - ZF, SF and PF are calculated according to the result by both vendors. + * + * For 16-bit shifts the count mask isn't 15, but 31, and the CPU will + * pick either the source register or the destination register for input bits + * when going beyond 16. According to https://www.sandpile.org/x86/flags.htm + * intel has changed behaviour here several times. We implement what current + * skylake based does for now, we can extend this later as needed. + */ +#define EMIT_SHLD(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shld_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, uint8_t cShift, \ + uint32_t *pfEFlags)) \ +{ \ + cShift &= a_cBitsWidth - 1; \ + if (cShift) \ + { \ + a_uType const uDst = *puDst; \ + a_uType uResult = uDst << cShift; \ + uResult |= uSrc >> (a_cBitsWidth - cShift); \ + *puDst = uResult; \ + \ + /* CALC EFLAGS: */ \ + uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \ + if (a_fIntelFlags) \ + /* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \ + else \ + { /* AMD 3990X: Set according to last shift. AF always set. */ \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth((uDst << (cShift - 1)) ^ uResult); \ + fEfl |= X86_EFL_AF; \ + } \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + fEfl |= (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF; /* CF = last bit shifted out */ \ + fEfl |= g_afParity[uResult & 0xff]; \ + fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \ + fEfl |= X86_EFL_CALC_ZF(uResult); \ + *pfEFlags = fEfl; \ + } \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHLD(64, uint64_t, RT_NOTHING, 1) +#endif +EMIT_SHLD(64, uint64_t, _intel, 1) +EMIT_SHLD(64, uint64_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHLD(32, uint32_t, RT_NOTHING, 1) +#endif +EMIT_SHLD(32, uint32_t, _intel, 1) +EMIT_SHLD(32, uint32_t, _amd, 0) + +#define EMIT_SHLD_16(a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_shld_u16,a_Suffix),(uint16_t *puDst, uint16_t uSrc, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= 31; \ + if (cShift) \ + { \ + uint16_t const uDst = *puDst; \ + uint64_t const uTmp = a_fIntelFlags \ + ? ((uint64_t)uDst << 32) | ((uint32_t)uSrc << 16) | uDst \ + : ((uint64_t)uDst << 32) | ((uint32_t)uSrc << 16) | uSrc; \ + uint16_t const uResult = (uint16_t)((uTmp << cShift) >> 32); \ + *puDst = uResult; \ + \ + /* CALC EFLAGS: */ \ + uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + if (a_fIntelFlags) \ + { \ + fEfl |= (uTmp >> (48 - cShift)) & X86_EFL_CF; /* CF = last bit shifted out of the combined operand */ \ + /* Intel 6700K & 10980XE: OF is et according to the first shift. AF always cleared. */ \ + fEfl |= X86_EFL_GET_OF_16(uDst ^ (uDst << 1)); \ + } \ + else \ + { \ + /* AMD 3990X: OF is set according to last shift, with some weirdness. AF always set. CF = last bit shifted out of uDst. */ \ + if (cShift < 16) \ + { \ + fEfl |= (uDst >> (16 - cShift)) & X86_EFL_CF; \ + fEfl |= X86_EFL_GET_OF_16((uDst << (cShift - 1)) ^ uResult); \ + } \ + else \ + { \ + if (cShift == 16) \ + fEfl |= uDst & X86_EFL_CF; \ + fEfl |= X86_EFL_GET_OF_16((uDst << (cShift - 1)) ^ 0); \ + } \ + fEfl |= X86_EFL_AF; \ + } \ + fEfl |= g_afParity[uResult & 0xff]; \ + fEfl |= X86_EFL_CALC_SF(uResult, 16); \ + fEfl |= X86_EFL_CALC_ZF(uResult); \ + *pfEFlags = fEfl; \ + } \ +} + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHLD_16(RT_NOTHING, 1) +#endif +EMIT_SHLD_16(_intel, 1) +EMIT_SHLD_16(_amd, 0) + + +/* + * SHRD + * + * EFLAGS behaviour seems to be the same as with SHLD: + * - CF is the last bit shifted out of puDst. + * - AF is always cleared by Intel 10980XE. + * - AF is always set by AMD 3990X. + * - OF is set according to the first shift on Intel 10980XE, it seems. + * - OF is set according to the last sub-shift on AMD 3990X. + * - ZF, SF and PF are calculated according to the result by both vendors. + * + * For 16-bit shifts the count mask isn't 15, but 31, and the CPU will + * pick either the source register or the destination register for input bits + * when going beyond 16. According to https://www.sandpile.org/x86/flags.htm + * intel has changed behaviour here several times. We implement what current + * skylake based does for now, we can extend this later as needed. + */ +#define EMIT_SHRD(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shrd_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= a_cBitsWidth - 1; \ + if (cShift) \ + { \ + a_uType const uDst = *puDst; \ + a_uType uResult = uDst >> cShift; \ + uResult |= uSrc << (a_cBitsWidth - cShift); \ + *puDst = uResult; \ + \ + uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + fEfl |= (uDst >> (cShift - 1)) & X86_EFL_CF; \ + if (a_fIntelFlags) \ + /* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uSrc << (a_cBitsWidth - 1))); \ + else \ + { /* AMD 3990X: Set according to last shift. AF always set. */ \ + if (cShift > 1) /* Set according to last shift. */ \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth((uSrc << (a_cBitsWidth - cShift + 1)) ^ uResult); \ + else \ + fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ uResult); \ + fEfl |= X86_EFL_AF; \ + } \ + fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \ + fEfl |= X86_EFL_CALC_ZF(uResult); \ + fEfl |= g_afParity[uResult & 0xff]; \ + *pfEFlags = fEfl; \ + } \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHRD(64, uint64_t, RT_NOTHING, 1) +#endif +EMIT_SHRD(64, uint64_t, _intel, 1) +EMIT_SHRD(64, uint64_t, _amd, 0) + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHRD(32, uint32_t, RT_NOTHING, 1) +#endif +EMIT_SHRD(32, uint32_t, _intel, 1) +EMIT_SHRD(32, uint32_t, _amd, 0) + +#define EMIT_SHRD_16(a_Suffix, a_fIntelFlags) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_shrd_u16,a_Suffix),(uint16_t *puDst, uint16_t uSrc, uint8_t cShift, uint32_t *pfEFlags)) \ +{ \ + cShift &= 31; \ + if (cShift) \ + { \ + uint16_t const uDst = *puDst; \ + uint64_t const uTmp = a_fIntelFlags \ + ? uDst | ((uint32_t)uSrc << 16) | ((uint64_t)uDst << 32) \ + : uDst | ((uint32_t)uSrc << 16) | ((uint64_t)uSrc << 32); \ + uint16_t const uResult = (uint16_t)(uTmp >> cShift); \ + *puDst = uResult; \ + \ + uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \ + AssertCompile(X86_EFL_CF_BIT == 0); \ + if (a_fIntelFlags) \ + { \ + /* Intel 10980XE: The CF is the last shifted out of the combined uTmp operand. */ \ + fEfl |= (uTmp >> (cShift - 1)) & X86_EFL_CF; \ + /* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \ + fEfl |= X86_EFL_GET_OF_16(uDst ^ (uSrc << 15)); \ + } \ + else \ + { \ + /* AMD 3990X: CF flag seems to be last bit shifted out of uDst, not the combined uSrc:uSrc:uDst operand. */ \ + fEfl |= (uDst >> (cShift - 1)) & X86_EFL_CF; \ + /* AMD 3990X: Set according to last shift. AF always set. */ \ + if (cShift > 1) /* Set according to last shift. */ \ + fEfl |= X86_EFL_GET_OF_16((uint16_t)(uTmp >> (cShift - 1)) ^ uResult); \ + else \ + fEfl |= X86_EFL_GET_OF_16(uDst ^ uResult); \ + fEfl |= X86_EFL_AF; \ + } \ + fEfl |= X86_EFL_CALC_SF(uResult, 16); \ + fEfl |= X86_EFL_CALC_ZF(uResult); \ + fEfl |= g_afParity[uResult & 0xff]; \ + *pfEFlags = fEfl; \ + } \ +} + +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHRD_16(RT_NOTHING, 1) +#endif +EMIT_SHRD_16(_intel, 1) +EMIT_SHRD_16(_amd, 0) + + +/* + * RORX (BMI2) + */ +#define EMIT_RORX(a_cBitsWidth, a_uType, a_fnHlp) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_rorx_u,a_cBitsWidth),(a_uType *puDst, a_uType uSrc, a_uType cShift)) \ +{ \ + *puDst = a_fnHlp(uSrc, cShift & (a_cBitsWidth - 1)); \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RORX(64, uint64_t, ASMRotateRightU64) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_RORX(32, uint32_t, ASMRotateRightU32) +#endif + + +/* + * SHLX (BMI2) + */ +#define EMIT_SHLX(a_cBitsWidth, a_uType, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shlx_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType cShift)) \ +{ \ + cShift &= a_cBitsWidth - 1; \ + *puDst = uSrc << cShift; \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHLX(64, uint64_t, RT_NOTHING) +EMIT_SHLX(64, uint64_t, _fallback) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHLX(32, uint32_t, RT_NOTHING) +EMIT_SHLX(32, uint32_t, _fallback) +#endif + + +/* + * SHRX (BMI2) + */ +#define EMIT_SHRX(a_cBitsWidth, a_uType, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shrx_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType cShift)) \ +{ \ + cShift &= a_cBitsWidth - 1; \ + *puDst = uSrc >> cShift; \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHRX(64, uint64_t, RT_NOTHING) +EMIT_SHRX(64, uint64_t, _fallback) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SHRX(32, uint32_t, RT_NOTHING) +EMIT_SHRX(32, uint32_t, _fallback) +#endif + + +/* + * SARX (BMI2) + */ +#define EMIT_SARX(a_cBitsWidth, a_uType, a_iType, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_sarx_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType cShift)) \ +{ \ + cShift &= a_cBitsWidth - 1; \ + *puDst = (a_iType)uSrc >> cShift; \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SARX(64, uint64_t, int64_t, RT_NOTHING) +EMIT_SARX(64, uint64_t, int64_t, _fallback) +#endif +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_SARX(32, uint32_t, int32_t, RT_NOTHING) +EMIT_SARX(32, uint32_t, int32_t, _fallback) +#endif + + +/* + * PDEP (BMI2) + */ +#define EMIT_PDEP(a_cBitsWidth, a_uType, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_pdep_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType fMask)) \ +{ \ + a_uType uResult = 0; \ + for (unsigned iMaskBit = 0, iBit = 0; iMaskBit < a_cBitsWidth; iMaskBit++) \ + if (fMask & ((a_uType)1 << iMaskBit)) \ + { \ + uResult |= ((uSrc >> iBit) & 1) << iMaskBit; \ + iBit++; \ + } \ + *puDst = uResult; \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_PDEP(64, uint64_t, RT_NOTHING) +#endif +EMIT_PDEP(64, uint64_t, _fallback) +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_PDEP(32, uint32_t, RT_NOTHING) +#endif +EMIT_PDEP(32, uint32_t, _fallback) + +/* + * PEXT (BMI2) + */ +#define EMIT_PEXT(a_cBitsWidth, a_uType, a_Suffix) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_pext_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType fMask)) \ +{ \ + a_uType uResult = 0; \ + for (unsigned iMaskBit = 0, iBit = 0; iMaskBit < a_cBitsWidth; iMaskBit++) \ + if (fMask & ((a_uType)1 << iMaskBit)) \ + { \ + uResult |= ((uSrc >> iMaskBit) & 1) << iBit; \ + iBit++; \ + } \ + *puDst = uResult; \ +} + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_PEXT(64, uint64_t, RT_NOTHING) +#endif +EMIT_PEXT(64, uint64_t, _fallback) +#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY) +EMIT_PEXT(32, uint32_t, RT_NOTHING) +#endif +EMIT_PEXT(32, uint32_t, _fallback) + + +#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) + +# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) +/* + * BSWAP + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u64,(uint64_t *puDst)) +{ + *puDst = ASMByteSwapU64(*puDst); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u32,(uint32_t *puDst)) +{ + *puDst = ASMByteSwapU32(*puDst); +} + + +/* Note! undocument, so 32-bit arg */ +IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u16,(uint32_t *puDst)) +{ +#if 0 + *(uint16_t *)puDst = ASMByteSwapU16(*(uint16_t *)puDst); +#else + /* This is the behaviour AMD 3990x (64-bit mode): */ + *(uint16_t *)puDst = 0; +#endif +} + +# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */ + + + +# if defined(IEM_WITHOUT_ASSEMBLY) + +/* + * LFENCE, SFENCE & MFENCE. + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_lfence,(void)) +{ + ASMReadFence(); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_sfence,(void)) +{ + ASMWriteFence(); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_mfence,(void)) +{ + ASMMemoryFence(); +} + + +# ifndef RT_ARCH_ARM64 +IEM_DECL_IMPL_DEF(void, iemAImpl_alt_mem_fence,(void)) +{ + ASMMemoryFence(); +} +# endif + +# endif + +#endif /* !RT_ARCH_AMD64 || IEM_WITHOUT_ASSEMBLY */ + + +IEM_DECL_IMPL_DEF(void, iemAImpl_arpl,(uint16_t *pu16Dst, uint16_t u16Src, uint32_t *pfEFlags)) +{ + if ((*pu16Dst & X86_SEL_RPL) < (u16Src & X86_SEL_RPL)) + { + *pu16Dst &= X86_SEL_MASK_OFF_RPL; + *pu16Dst |= u16Src & X86_SEL_RPL; + + *pfEFlags |= X86_EFL_ZF; + } + else + *pfEFlags &= ~X86_EFL_ZF; +} + + +#if defined(IEM_WITHOUT_ASSEMBLY) + +/********************************************************************************************************************************* +* x87 FPU Loads * +*********************************************************************************************************************************/ + +IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT32U pr32Val)) +{ + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ + if (RTFLOAT32U_IS_NORMAL(pr32Val)) + { + pFpuRes->r80Result.sj64.fSign = pr32Val->s.fSign; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = (uint64_t)pr32Val->s.uFraction + << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS); + pFpuRes->r80Result.sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS; + Assert(RTFLOAT80U_IS_NORMAL(&pFpuRes->r80Result)); + } + else if (RTFLOAT32U_IS_ZERO(pr32Val)) + { + pFpuRes->r80Result.s.fSign = pr32Val->s.fSign; + pFpuRes->r80Result.s.uExponent = 0; + pFpuRes->r80Result.s.uMantissa = 0; + Assert(RTFLOAT80U_IS_ZERO(&pFpuRes->r80Result)); + } + else if (RTFLOAT32U_IS_SUBNORMAL(pr32Val)) + { + /* Subnormal values gets normalized. */ + pFpuRes->r80Result.sj64.fSign = pr32Val->s.fSign; + pFpuRes->r80Result.sj64.fInteger = 1; + unsigned const cExtraShift = RTFLOAT32U_FRACTION_BITS - ASMBitLastSetU32(pr32Val->s.uFraction); + pFpuRes->r80Result.sj64.uFraction = (uint64_t)pr32Val->s.uFraction + << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS + cExtraShift + 1); + pFpuRes->r80Result.sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift; + pFpuRes->FSW |= X86_FSW_DE; + if (!(pFpuState->FCW & X86_FCW_DM)) + pFpuRes->FSW |= X86_FSW_ES | X86_FSW_B; /* The value is still pushed. */ + } + else if (RTFLOAT32U_IS_INF(pr32Val)) + { + pFpuRes->r80Result.s.fSign = pr32Val->s.fSign; + pFpuRes->r80Result.s.uExponent = RTFLOAT80U_EXP_MAX; + pFpuRes->r80Result.s.uMantissa = RT_BIT_64(63); + Assert(RTFLOAT80U_IS_INF(&pFpuRes->r80Result)); + } + else + { + /* Signalling and quiet NaNs, both turn into quiet ones when loaded (weird). */ + Assert(RTFLOAT32U_IS_NAN(pr32Val)); + pFpuRes->r80Result.sj64.fSign = pr32Val->s.fSign; + pFpuRes->r80Result.sj64.uExponent = RTFLOAT80U_EXP_MAX; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = (uint64_t)pr32Val->s.uFraction + << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS); + if (RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val)) + { + pFpuRes->r80Result.sj64.uFraction |= RT_BIT_64(62); /* make quiet */ + Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result)); + pFpuRes->FSW |= X86_FSW_IE; + + if (!(pFpuState->FCW & X86_FCW_IM)) + { + /* The value is not pushed. */ + pFpuRes->FSW &= ~X86_FSW_TOP_MASK; + pFpuRes->FSW |= X86_FSW_ES | X86_FSW_B; + pFpuRes->r80Result.au64[0] = 0; + pFpuRes->r80Result.au16[4] = 0; + } + } + else + Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result)); + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT64U pr64Val)) +{ + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ + if (RTFLOAT64U_IS_NORMAL(pr64Val)) + { + pFpuRes->r80Result.sj64.fSign = pr64Val->s.fSign; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS); + pFpuRes->r80Result.sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS; + Assert(RTFLOAT80U_IS_NORMAL(&pFpuRes->r80Result)); + } + else if (RTFLOAT64U_IS_ZERO(pr64Val)) + { + pFpuRes->r80Result.s.fSign = pr64Val->s.fSign; + pFpuRes->r80Result.s.uExponent = 0; + pFpuRes->r80Result.s.uMantissa = 0; + Assert(RTFLOAT80U_IS_ZERO(&pFpuRes->r80Result)); + } + else if (RTFLOAT64U_IS_SUBNORMAL(pr64Val)) + { + /* Subnormal values gets normalized. */ + pFpuRes->r80Result.sj64.fSign = pr64Val->s.fSign; + pFpuRes->r80Result.sj64.fInteger = 1; + unsigned const cExtraShift = RTFLOAT64U_FRACTION_BITS - ASMBitLastSetU64(pr64Val->s64.uFraction); + pFpuRes->r80Result.sj64.uFraction = pr64Val->s64.uFraction + << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS + cExtraShift + 1); + pFpuRes->r80Result.sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift; + pFpuRes->FSW |= X86_FSW_DE; + if (!(pFpuState->FCW & X86_FCW_DM)) + pFpuRes->FSW |= X86_FSW_ES | X86_FSW_B; /* The value is still pushed. */ + } + else if (RTFLOAT64U_IS_INF(pr64Val)) + { + pFpuRes->r80Result.s.fSign = pr64Val->s.fSign; + pFpuRes->r80Result.s.uExponent = RTFLOAT80U_EXP_MAX; + pFpuRes->r80Result.s.uMantissa = RT_BIT_64(63); + Assert(RTFLOAT80U_IS_INF(&pFpuRes->r80Result)); + } + else + { + /* Signalling and quiet NaNs, both turn into quiet ones when loaded (weird). */ + Assert(RTFLOAT64U_IS_NAN(pr64Val)); + pFpuRes->r80Result.sj64.fSign = pr64Val->s.fSign; + pFpuRes->r80Result.sj64.uExponent = RTFLOAT80U_EXP_MAX; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS); + if (RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val)) + { + pFpuRes->r80Result.sj64.uFraction |= RT_BIT_64(62); /* make quiet */ + Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result)); + pFpuRes->FSW |= X86_FSW_IE; + + if (!(pFpuState->FCW & X86_FCW_IM)) + { + /* The value is not pushed. */ + pFpuRes->FSW &= ~X86_FSW_TOP_MASK; + pFpuRes->FSW |= X86_FSW_ES | X86_FSW_B; + pFpuRes->r80Result.au64[0] = 0; + pFpuRes->r80Result.au16[4] = 0; + } + } + else + Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result)); + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + pFpuRes->r80Result.au64[0] = pr80Val->au64[0]; + pFpuRes->r80Result.au16[4] = pr80Val->au16[4]; + /* Raises no exceptions. */ + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fld1,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes)) +{ + pFpuRes->r80Result.sj64.fSign = 0; + pFpuRes->r80Result.sj64.uExponent = 0 + 16383; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = 0; + + /* + * FPU status word: + * - TOP is irrelevant, but we must match x86 assembly version. + * - C1 is always cleared as we don't have any stack overflows. + * - C0, C2, and C3 are undefined and Intel 10980XE does not touch them. + */ + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fldl2e,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes)) +{ + pFpuRes->r80Result.sj64.fSign = 0; + pFpuRes->r80Result.sj64.uExponent = 0 + 16383; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST + || (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP + ? UINT64_C(0x38aa3b295c17f0bc) : UINT64_C(0x38aa3b295c17f0bb); + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fldl2t,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes)) +{ + pFpuRes->r80Result.sj64.fSign = 0; + pFpuRes->r80Result.sj64.uExponent = 1 + 16383; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) != X86_FCW_RC_UP + ? UINT64_C(0x549a784bcd1b8afe) : UINT64_C(0x549a784bcd1b8aff); + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fldlg2,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes)) +{ + pFpuRes->r80Result.sj64.fSign = 0; + pFpuRes->r80Result.sj64.uExponent = -2 + 16383; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST + || (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP + ? UINT64_C(0x1a209a84fbcff799) : UINT64_C(0x1a209a84fbcff798); + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fldln2,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes)) +{ + pFpuRes->r80Result.sj64.fSign = 0; + pFpuRes->r80Result.sj64.uExponent = -1 + 16383; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST + || (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP + ? UINT64_C(0x317217f7d1cf79ac) : UINT64_C(0x317217f7d1cf79ab); + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fldpi,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes)) +{ + pFpuRes->r80Result.sj64.fSign = 0; + pFpuRes->r80Result.sj64.uExponent = 1 + 16383; + pFpuRes->r80Result.sj64.fInteger = 1; + pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST + || (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP + ? UINT64_C(0x490fdaa22168c235) : UINT64_C(0x490fdaa22168c234); + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fldz,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes)) +{ + pFpuRes->r80Result.s.fSign = 0; + pFpuRes->r80Result.s.uExponent = 0; + pFpuRes->r80Result.s.uMantissa = 0; + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ +} + +#define EMIT_FILD(a_cBits) \ +IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i ## a_cBits,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, \ + int ## a_cBits ## _t const *piVal)) \ +{ \ + int ## a_cBits ## _t iVal = *piVal; \ + if (iVal == 0) \ + { \ + pFpuRes->r80Result.s.fSign = 0; \ + pFpuRes->r80Result.s.uExponent = 0; \ + pFpuRes->r80Result.s.uMantissa = 0; \ + } \ + else \ + { \ + if (iVal > 0) \ + pFpuRes->r80Result.s.fSign = 0; \ + else \ + { \ + pFpuRes->r80Result.s.fSign = 1; \ + iVal = -iVal; \ + } \ + unsigned const cBits = ASMBitLastSetU ## a_cBits((uint ## a_cBits ## _t)iVal); \ + pFpuRes->r80Result.s.uExponent = cBits - 1 + RTFLOAT80U_EXP_BIAS; \ + pFpuRes->r80Result.s.uMantissa = (uint64_t)iVal << (RTFLOAT80U_FRACTION_BITS + 1 - cBits); \ + } \ + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ \ +} +EMIT_FILD(16) +EMIT_FILD(32) +EMIT_FILD(64) + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_d80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTPBCD80U pd80Val)) +{ + pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ + if ( pd80Val->s.abPairs[0] == 0 + && pd80Val->s.abPairs[1] == 0 + && pd80Val->s.abPairs[2] == 0 + && pd80Val->s.abPairs[3] == 0 + && pd80Val->s.abPairs[4] == 0 + && pd80Val->s.abPairs[5] == 0 + && pd80Val->s.abPairs[6] == 0 + && pd80Val->s.abPairs[7] == 0 + && pd80Val->s.abPairs[8] == 0) + { + pFpuRes->r80Result.s.fSign = pd80Val->s.fSign; + pFpuRes->r80Result.s.uExponent = 0; + pFpuRes->r80Result.s.uMantissa = 0; + } + else + { + pFpuRes->r80Result.s.fSign = pd80Val->s.fSign; + + size_t cPairs = RT_ELEMENTS(pd80Val->s.abPairs); + while (cPairs > 0 && pd80Val->s.abPairs[cPairs - 1] == 0) + cPairs--; + + uint64_t uVal = 0; + uint64_t uFactor = 1; + for (size_t iPair = 0; iPair < cPairs; iPair++, uFactor *= 100) + uVal += RTPBCD80U_LO_DIGIT(pd80Val->s.abPairs[iPair]) * uFactor + + RTPBCD80U_HI_DIGIT(pd80Val->s.abPairs[iPair]) * uFactor * 10; + + unsigned const cBits = ASMBitLastSetU64(uVal); + pFpuRes->r80Result.s.uExponent = cBits - 1 + RTFLOAT80U_EXP_BIAS; + pFpuRes->r80Result.s.uMantissa = uVal << (RTFLOAT80U_FRACTION_BITS + 1 - cBits); + } +} + + +/********************************************************************************************************************************* +* x87 FPU Stores * +*********************************************************************************************************************************/ + +/** + * Helper for storing a deconstructed and normal R80 value as a 64-bit one. + * + * This uses the rounding rules indicated by fFcw and returns updated fFsw. + * + * @returns Updated FPU status word value. + * @param fSignIn Incoming sign indicator. + * @param uMantissaIn Incoming mantissa (dot between bit 63 and 62). + * @param iExponentIn Unbiased exponent. + * @param fFcw The FPU control word. + * @param fFsw Prepped FPU status word, i.e. exceptions and C1 clear. + * @param pr32Dst Where to return the output value, if one should be + * returned. + * + * @note Tailored as a helper for iemAImpl_fst_r80_to_r32 right now. + * @note Exact same logic as iemAImpl_StoreNormalR80AsR64. + */ +static uint16_t iemAImpl_StoreNormalR80AsR32(bool fSignIn, uint64_t uMantissaIn, int32_t iExponentIn, + uint16_t fFcw, uint16_t fFsw, PRTFLOAT32U pr32Dst) +{ + uint64_t const fRoundingOffMask = RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS) - 1; /* 0x7ff */ + uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST + ? RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS - 1) /* 0x400 */ + : (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP) + ? fRoundingOffMask + : 0; + uint64_t fRoundedOff = uMantissaIn & fRoundingOffMask; + + /* + * Deal with potential overflows/underflows first, optimizing for none. + * 0 and MAX are used for special values; MAX-1 may be rounded up to MAX. + */ + int32_t iExponentOut = (int32_t)iExponentIn + RTFLOAT32U_EXP_BIAS; + if ((uint32_t)iExponentOut - 1 < (uint32_t)(RTFLOAT32U_EXP_MAX - 3)) + { /* likely? */ } + /* + * Underflow if the exponent zero or negative. This is attempted mapped + * to a subnormal number when possible, with some additional trickery ofc. + */ + else if (iExponentOut <= 0) + { + bool const fIsTiny = iExponentOut < 0 + || UINT64_MAX - uMantissaIn > uRoundingAdd; + if (!(fFcw & X86_FCW_UM) && fIsTiny) + /* Note! 754-1985 sec 7.4 has something about bias adjust of 192 here, not in 2008 & 2019. Perhaps only 8087 & 287? */ + return fFsw | X86_FSW_UE | X86_FSW_ES | X86_FSW_B; + + if (iExponentOut <= 0) + { + uMantissaIn = iExponentOut <= -63 + ? uMantissaIn != 0 + : (uMantissaIn >> (-iExponentOut + 1)) | ((uMantissaIn & (RT_BIT_64(-iExponentOut + 1) - 1)) != 0); + fRoundedOff = uMantissaIn & fRoundingOffMask; + if (fRoundedOff && fIsTiny) + fFsw |= X86_FSW_UE; + iExponentOut = 0; + } + } + /* + * Overflow if at or above max exponent value or if we will reach max + * when rounding. Will return +/-zero or +/-max value depending on + * whether we're rounding or not. + */ + else if ( iExponentOut >= RTFLOAT32U_EXP_MAX + || ( iExponentOut == RTFLOAT32U_EXP_MAX - 1 + && UINT64_MAX - uMantissaIn <= uRoundingAdd)) + { + fFsw |= X86_FSW_OE; + if (!(fFcw & X86_FCW_OM)) + return fFsw | X86_FSW_ES | X86_FSW_B; + fFsw |= X86_FSW_PE; + if (uRoundingAdd) + fFsw |= X86_FSW_C1; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + + pr32Dst->s.fSign = fSignIn; + if (uRoundingAdd) + { /* Zero */ + pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX; + pr32Dst->s.uFraction = 0; + } + else + { /* Max */ + pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX - 1; + pr32Dst->s.uFraction = RT_BIT_32(RTFLOAT32U_FRACTION_BITS) - 1; + } + return fFsw; + } + + /* + * Normal or subnormal number. + */ + /* Do rounding - just truncate in near mode when midway on an even outcome. */ + uint64_t uMantissaOut = uMantissaIn; + if ( (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST + || (uMantissaIn & RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS)) + || fRoundedOff != uRoundingAdd) + { + uMantissaOut = uMantissaIn + uRoundingAdd; + if (uMantissaOut >= uMantissaIn) + { /* likely */ } + else + { + uMantissaOut >>= 1; /* (We don't need to add bit 63 here (the integer bit), as it will be chopped off below.) */ + iExponentOut++; + Assert(iExponentOut < RTFLOAT32U_EXP_MAX); /* checked above */ + fFsw |= X86_FSW_C1; + } + } + else + uMantissaOut = uMantissaIn; + + /* Truncate the mantissa and set the return value. */ + uMantissaOut >>= RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS; + + pr32Dst->s.uFraction = (uint32_t)uMantissaOut; /* Note! too big for bitfield if normal. */ + pr32Dst->s.uExponent = iExponentOut; + pr32Dst->s.fSign = fSignIn; + + /* Set status flags realted to rounding. */ + if (fRoundedOff) + { + fFsw |= X86_FSW_PE; + if (uMantissaOut > (uMantissaIn >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS))) + fFsw |= X86_FSW_C1; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + return fFsw; +} + + +/** + * @note Exact same logic as iemAImpl_fst_r80_to_r64. + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, + PRTFLOAT32U pr32Dst, PCRTFLOAT80U pr80Src)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); + if (RTFLOAT80U_IS_NORMAL(pr80Src)) + fFsw = iemAImpl_StoreNormalR80AsR32(pr80Src->s.fSign, pr80Src->s.uMantissa, + (int32_t)pr80Src->s.uExponent - RTFLOAT80U_EXP_BIAS, fFcw, fFsw, pr32Dst); + else if (RTFLOAT80U_IS_ZERO(pr80Src)) + { + pr32Dst->s.fSign = pr80Src->s.fSign; + pr32Dst->s.uExponent = 0; + pr32Dst->s.uFraction = 0; + Assert(RTFLOAT32U_IS_ZERO(pr32Dst)); + } + else if (RTFLOAT80U_IS_INF(pr80Src)) + { + pr32Dst->s.fSign = pr80Src->s.fSign; + pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX; + pr32Dst->s.uFraction = 0; + Assert(RTFLOAT32U_IS_INF(pr32Dst)); + } + else if (RTFLOAT80U_IS_INDEFINITE(pr80Src)) + { + /* Mapped to +/-QNaN */ + pr32Dst->s.fSign = pr80Src->s.fSign; + pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX; + pr32Dst->s.uFraction = RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1); + } + else if (RTFLOAT80U_IS_PSEUDO_INF(pr80Src) || RTFLOAT80U_IS_UNNORMAL(pr80Src) || RTFLOAT80U_IS_PSEUDO_NAN(pr80Src)) + { + /* Pseudo-Inf / Pseudo-Nan / Unnormal -> QNaN (during load, probably) */ + if (fFcw & X86_FCW_IM) + { + pr32Dst->s.fSign = 1; + pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX; + pr32Dst->s.uFraction = RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1); + fFsw |= X86_FSW_IE; + } + else + fFsw |= X86_FSW_IE | X86_FSW_ES | X86_FSW_B;; + } + else if (RTFLOAT80U_IS_NAN(pr80Src)) + { + /* IM applies to signalled NaN input only. Everything is converted to quiet NaN. */ + if ((fFcw & X86_FCW_IM) || !RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src)) + { + pr32Dst->s.fSign = pr80Src->s.fSign; + pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX; + pr32Dst->s.uFraction = (uint32_t)(pr80Src->sj64.uFraction >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS)); + pr32Dst->s.uFraction |= RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1); + if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src)) + fFsw |= X86_FSW_IE; + } + else + fFsw |= X86_FSW_IE | X86_FSW_ES | X86_FSW_B; + } + else + { + /* Denormal values causes both an underflow and precision exception. */ + Assert(RTFLOAT80U_IS_DENORMAL(pr80Src) || RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Src)); + if (fFcw & X86_FCW_UM) + { + pr32Dst->s.fSign = pr80Src->s.fSign; + pr32Dst->s.uExponent = 0; + if ((fFcw & X86_FCW_RC_MASK) == (!pr80Src->s.fSign ? X86_FCW_RC_UP : X86_FCW_RC_DOWN)) + { + pr32Dst->s.uFraction = 1; + fFsw |= X86_FSW_UE | X86_FSW_PE | X86_FSW_C1; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + else + { + pr32Dst->s.uFraction = 0; + fFsw |= X86_FSW_UE | X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else + fFsw |= X86_FSW_UE | X86_FSW_ES | X86_FSW_B; + } + *pu16FSW = fFsw; +} + + +/** + * Helper for storing a deconstructed and normal R80 value as a 64-bit one. + * + * This uses the rounding rules indicated by fFcw and returns updated fFsw. + * + * @returns Updated FPU status word value. + * @param fSignIn Incoming sign indicator. + * @param uMantissaIn Incoming mantissa (dot between bit 63 and 62). + * @param iExponentIn Unbiased exponent. + * @param fFcw The FPU control word. + * @param fFsw Prepped FPU status word, i.e. exceptions and C1 clear. + * @param pr64Dst Where to return the output value, if one should be + * returned. + * + * @note Tailored as a helper for iemAImpl_fst_r80_to_r64 right now. + * @note Exact same logic as iemAImpl_StoreNormalR80AsR32. + */ +static uint16_t iemAImpl_StoreNormalR80AsR64(bool fSignIn, uint64_t uMantissaIn, int32_t iExponentIn, + uint16_t fFcw, uint16_t fFsw, PRTFLOAT64U pr64Dst) +{ + uint64_t const fRoundingOffMask = RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS) - 1; /* 0x7ff */ + uint32_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST + ? RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS - 1) /* 0x400 */ + : (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP) + ? fRoundingOffMask + : 0; + uint32_t fRoundedOff = uMantissaIn & fRoundingOffMask; + + /* + * Deal with potential overflows/underflows first, optimizing for none. + * 0 and MAX are used for special values; MAX-1 may be rounded up to MAX. + */ + int32_t iExponentOut = (int32_t)iExponentIn + RTFLOAT64U_EXP_BIAS; + if ((uint32_t)iExponentOut - 1 < (uint32_t)(RTFLOAT64U_EXP_MAX - 3)) + { /* likely? */ } + /* + * Underflow if the exponent zero or negative. This is attempted mapped + * to a subnormal number when possible, with some additional trickery ofc. + */ + else if (iExponentOut <= 0) + { + bool const fIsTiny = iExponentOut < 0 + || UINT64_MAX - uMantissaIn > uRoundingAdd; + if (!(fFcw & X86_FCW_UM) && fIsTiny) + /* Note! 754-1985 sec 7.4 has something about bias adjust of 1536 here, not in 2008 & 2019. Perhaps only 8087 & 287? */ + return fFsw | X86_FSW_UE | X86_FSW_ES | X86_FSW_B; + + if (iExponentOut <= 0) + { + uMantissaIn = iExponentOut <= -63 + ? uMantissaIn != 0 + : (uMantissaIn >> (-iExponentOut + 1)) | ((uMantissaIn & (RT_BIT_64(-iExponentOut + 1) - 1)) != 0); + fRoundedOff = uMantissaIn & fRoundingOffMask; + if (fRoundedOff && fIsTiny) + fFsw |= X86_FSW_UE; + iExponentOut = 0; + } + } + /* + * Overflow if at or above max exponent value or if we will reach max + * when rounding. Will return +/-zero or +/-max value depending on + * whether we're rounding or not. + */ + else if ( iExponentOut >= RTFLOAT64U_EXP_MAX + || ( iExponentOut == RTFLOAT64U_EXP_MAX - 1 + && UINT64_MAX - uMantissaIn <= uRoundingAdd)) + { + fFsw |= X86_FSW_OE; + if (!(fFcw & X86_FCW_OM)) + return fFsw | X86_FSW_ES | X86_FSW_B; + fFsw |= X86_FSW_PE; + if (uRoundingAdd) + fFsw |= X86_FSW_C1; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + + pr64Dst->s64.fSign = fSignIn; + if (uRoundingAdd) + { /* Zero */ + pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX; + pr64Dst->s64.uFraction = 0; + } + else + { /* Max */ + pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX - 1; + pr64Dst->s64.uFraction = RT_BIT_64(RTFLOAT64U_FRACTION_BITS) - 1; + } + return fFsw; + } + + /* + * Normal or subnormal number. + */ + /* Do rounding - just truncate in near mode when midway on an even outcome. */ + uint64_t uMantissaOut = uMantissaIn; + if ( (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST + || (uMantissaIn & RT_BIT_32(RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS)) + || fRoundedOff != uRoundingAdd) + { + uMantissaOut = uMantissaIn + uRoundingAdd; + if (uMantissaOut >= uMantissaIn) + { /* likely */ } + else + { + uMantissaOut >>= 1; /* (We don't need to add bit 63 here (the integer bit), as it will be chopped off below.) */ + iExponentOut++; + Assert(iExponentOut < RTFLOAT64U_EXP_MAX); /* checked above */ + fFsw |= X86_FSW_C1; + } + } + else + uMantissaOut = uMantissaIn; + + /* Truncate the mantissa and set the return value. */ + uMantissaOut >>= RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS; + + pr64Dst->s64.uFraction = uMantissaOut; /* Note! too big for bitfield if normal. */ + pr64Dst->s64.uExponent = iExponentOut; + pr64Dst->s64.fSign = fSignIn; + + /* Set status flags realted to rounding. */ + if (fRoundedOff) + { + fFsw |= X86_FSW_PE; + if (uMantissaOut > (uMantissaIn >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS))) + fFsw |= X86_FSW_C1; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + return fFsw; +} + + +/** + * @note Exact same logic as iemAImpl_fst_r80_to_r32. + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, + PRTFLOAT64U pr64Dst, PCRTFLOAT80U pr80Src)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); + if (RTFLOAT80U_IS_NORMAL(pr80Src)) + fFsw = iemAImpl_StoreNormalR80AsR64(pr80Src->s.fSign, pr80Src->s.uMantissa, + (int32_t)pr80Src->s.uExponent - RTFLOAT80U_EXP_BIAS, fFcw, fFsw, pr64Dst); + else if (RTFLOAT80U_IS_ZERO(pr80Src)) + { + pr64Dst->s64.fSign = pr80Src->s.fSign; + pr64Dst->s64.uExponent = 0; + pr64Dst->s64.uFraction = 0; + Assert(RTFLOAT64U_IS_ZERO(pr64Dst)); + } + else if (RTFLOAT80U_IS_INF(pr80Src)) + { + pr64Dst->s64.fSign = pr80Src->s.fSign; + pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX; + pr64Dst->s64.uFraction = 0; + Assert(RTFLOAT64U_IS_INF(pr64Dst)); + } + else if (RTFLOAT80U_IS_INDEFINITE(pr80Src)) + { + /* Mapped to +/-QNaN */ + pr64Dst->s64.fSign = pr80Src->s.fSign; + pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX; + pr64Dst->s64.uFraction = RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1); + } + else if (RTFLOAT80U_IS_PSEUDO_INF(pr80Src) || RTFLOAT80U_IS_UNNORMAL(pr80Src) || RTFLOAT80U_IS_PSEUDO_NAN(pr80Src)) + { + /* Pseudo-Inf / Pseudo-Nan / Unnormal -> QNaN (during load, probably) */ + if (fFcw & X86_FCW_IM) + { + pr64Dst->s64.fSign = 1; + pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX; + pr64Dst->s64.uFraction = RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1); + fFsw |= X86_FSW_IE; + } + else + fFsw |= X86_FSW_IE | X86_FSW_ES | X86_FSW_B;; + } + else if (RTFLOAT80U_IS_NAN(pr80Src)) + { + /* IM applies to signalled NaN input only. Everything is converted to quiet NaN. */ + if ((fFcw & X86_FCW_IM) || !RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src)) + { + pr64Dst->s64.fSign = pr80Src->s.fSign; + pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX; + pr64Dst->s64.uFraction = pr80Src->sj64.uFraction >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS); + pr64Dst->s64.uFraction |= RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1); + if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src)) + fFsw |= X86_FSW_IE; + } + else + fFsw |= X86_FSW_IE | X86_FSW_ES | X86_FSW_B; + } + else + { + /* Denormal values causes both an underflow and precision exception. */ + Assert(RTFLOAT80U_IS_DENORMAL(pr80Src) || RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Src)); + if (fFcw & X86_FCW_UM) + { + pr64Dst->s64.fSign = pr80Src->s.fSign; + pr64Dst->s64.uExponent = 0; + if ((fFcw & X86_FCW_RC_MASK) == (!pr80Src->s.fSign ? X86_FCW_RC_UP : X86_FCW_RC_DOWN)) + { + pr64Dst->s64.uFraction = 1; + fFsw |= X86_FSW_UE | X86_FSW_PE | X86_FSW_C1; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + else + { + pr64Dst->s64.uFraction = 0; + fFsw |= X86_FSW_UE | X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else + fFsw |= X86_FSW_UE | X86_FSW_ES | X86_FSW_B; + } + *pu16FSW = fFsw; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, + PRTFLOAT80U pr80Dst, PCRTFLOAT80U pr80Src)) +{ + /* + * FPU status word: + * - TOP is irrelevant, but we must match x86 assembly version (0). + * - C1 is always cleared as we don't have any stack overflows. + * - C0, C2, and C3 are undefined and Intel 10980XE does not touch them. + */ + *pu16FSW = pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3); /* see iemAImpl_fld1 */ + *pr80Dst = *pr80Src; +} + + +/* + * + * Mantissa: + * 63 56 48 40 32 24 16 8 0 + * v v v v v v v v v + * 1[.]111 0000 1111 0000 1111 0000 1111 0000 1111 0000 1111 0000 1111 0000 1111 0000 + * \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ + * Exp: 0 4 8 12 16 20 24 28 32 36 40 44 48 52 56 60 + * + * int64_t has the same width, only bit 63 is the sign bit. So, the max we can map over + * are bits 1 thru 63, dropping off bit 0, with an exponent of 62. The number of bits we + * drop off from the mantissa increases with decreasing exponent, till an exponent of 0 + * where we'll drop off all but bit 63. + */ +#define EMIT_FIST(a_cBits, a_iType, a_iTypeMin, a_iTypeIndefinite) \ +IEM_DECL_IMPL_DEF(void, iemAImpl_fist_r80_to_i ## a_cBits,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, \ + a_iType *piDst, PCRTFLOAT80U pr80Val)) \ +{ \ + uint16_t const fFcw = pFpuState->FCW; \ + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); \ + bool const fSignIn = pr80Val->s.fSign; \ + \ + /* \ + * Deal with normal numbers first. \ + */ \ + if (RTFLOAT80U_IS_NORMAL(pr80Val)) \ + { \ + uint64_t uMantissa = pr80Val->s.uMantissa; \ + int32_t iExponent = (int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS; \ + \ + if ((uint32_t)iExponent <= a_cBits - 2) \ + { \ + unsigned const cShiftOff = 63 - iExponent; \ + uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1; \ + uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST \ + ? RT_BIT_64(cShiftOff - 1) \ + : (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP) \ + ? fRoundingOffMask \ + : 0; \ + uint64_t fRoundedOff = uMantissa & fRoundingOffMask; \ + \ + uMantissa >>= cShiftOff; \ + uint64_t const uRounding = (fRoundedOff + uRoundingAdd) >> cShiftOff; \ + uMantissa += uRounding; \ + if (!(uMantissa & RT_BIT_64(a_cBits - 1))) \ + { \ + if (fRoundedOff) \ + { \ + if ((uMantissa & 1) && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST && fRoundedOff == uRoundingAdd) \ + uMantissa &= ~(uint64_t)1; /* round to even number if equal distance between up/down. */ \ + else if (uRounding) \ + fFsw |= X86_FSW_C1; \ + fFsw |= X86_FSW_PE; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + \ + if (!fSignIn) \ + *piDst = (a_iType)uMantissa; \ + else \ + *piDst = -(a_iType)uMantissa; \ + } \ + else \ + { \ + /* overflowed after rounding. */ \ + AssertMsg(iExponent == a_cBits - 2 && uMantissa == RT_BIT_64(a_cBits - 1), \ + ("e=%d m=%#RX64 (org %#RX64) s=%d; shift=%d ro=%#RX64 rm=%#RX64 ra=%#RX64\n", iExponent, uMantissa, \ + pr80Val->s.uMantissa, fSignIn, cShiftOff, fRoundedOff, fRoundingOffMask, uRoundingAdd)); \ + \ + /* Special case for the integer minimum value. */ \ + if (fSignIn) \ + { \ + *piDst = a_iTypeMin; \ + fFsw |= X86_FSW_PE | X86_FSW_C1; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + else \ + { \ + fFsw |= X86_FSW_IE; \ + if (fFcw & X86_FCW_IM) \ + *piDst = a_iTypeMin; \ + else \ + fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \ + } \ + } \ + } \ + /* \ + * Tiny sub-zero numbers. \ + */ \ + else if (iExponent < 0) \ + { \ + if (!fSignIn) \ + { \ + if ( (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP \ + || (iExponent == -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST)) \ + { \ + *piDst = 1; \ + fFsw |= X86_FSW_C1; \ + } \ + else \ + *piDst = 0; \ + } \ + else \ + { \ + if ( (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP \ + || (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_ZERO \ + || (iExponent < -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST)) \ + *piDst = 0; \ + else \ + { \ + *piDst = -1; \ + fFsw |= X86_FSW_C1; \ + } \ + } \ + fFsw |= X86_FSW_PE; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + /* \ + * Special MIN case. \ + */ \ + else if ( fSignIn && iExponent == a_cBits - 1 \ + && ( a_cBits < 64 && (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_DOWN \ + ? uMantissa < (RT_BIT_64(63) | RT_BIT_64(65 - a_cBits)) \ + : uMantissa == RT_BIT_64(63))) \ + { \ + *piDst = a_iTypeMin; \ + if (uMantissa & (RT_BIT_64(64 - a_cBits + 1) - 1)) \ + { \ + fFsw |= X86_FSW_PE; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + } \ + /* \ + * Too large/small number outside the target integer range. \ + */ \ + else \ + { \ + fFsw |= X86_FSW_IE; \ + if (fFcw & X86_FCW_IM) \ + *piDst = a_iTypeIndefinite; \ + else \ + fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \ + } \ + } \ + /* \ + * Map both +0 and -0 to integer zero (signless/+). \ + */ \ + else if (RTFLOAT80U_IS_ZERO(pr80Val)) \ + *piDst = 0; \ + /* \ + * Denormals are just really tiny sub-zero numbers that are either rounded \ + * to zero, 1 or -1 depending on sign and rounding control. \ + */ \ + else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val) || RTFLOAT80U_IS_DENORMAL(pr80Val)) \ + { \ + if ((fFcw & X86_FCW_RC_MASK) != (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)) \ + *piDst = 0; \ + else \ + { \ + *piDst = fSignIn ? -1 : 1; \ + fFsw |= X86_FSW_C1; \ + } \ + fFsw |= X86_FSW_PE; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + /* \ + * All other special values are considered invalid arguments and result \ + * in an IE exception and indefinite value if masked. \ + */ \ + else \ + { \ + fFsw |= X86_FSW_IE; \ + if (fFcw & X86_FCW_IM) \ + *piDst = a_iTypeIndefinite; \ + else \ + fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \ + } \ + *pu16FSW = fFsw; \ +} +EMIT_FIST(64, int64_t, INT64_MIN, X86_FPU_INT64_INDEFINITE) +EMIT_FIST(32, int32_t, INT32_MIN, X86_FPU_INT32_INDEFINITE) +EMIT_FIST(16, int16_t, INT16_MIN, X86_FPU_INT16_INDEFINITE) + +#endif /*IEM_WITHOUT_ASSEMBLY */ + + +/* + * The FISTT instruction was added with SSE3 and are a lot simpler than FIST. + * + * The 16-bit version is a bit peculiar, though, as it seems to be raising IE + * as if it was the 32-bit version (i.e. starting with exp 31 instead of 15), + * thus the @a a_cBitsIn. + */ +#define EMIT_FISTT(a_cBits, a_cBitsIn, a_iType, a_iTypeMin, a_iTypeMax, a_iTypeIndefinite, a_Suffix, a_fIntelVersion) \ +IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_fistt_r80_to_i,a_cBits,a_Suffix),(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, \ + a_iType *piDst, PCRTFLOAT80U pr80Val)) \ +{ \ + uint16_t const fFcw = pFpuState->FCW; \ + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); \ + bool const fSignIn = pr80Val->s.fSign; \ + \ + /* \ + * Deal with normal numbers first. \ + */ \ + if (RTFLOAT80U_IS_NORMAL(pr80Val)) \ + { \ + uint64_t uMantissa = pr80Val->s.uMantissa; \ + int32_t iExponent = (int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS; \ + \ + if ((uint32_t)iExponent <= a_cBitsIn - 2) \ + { \ + unsigned const cShiftOff = 63 - iExponent; \ + uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1; \ + uint64_t const fRoundedOff = uMantissa & fRoundingOffMask; \ + uMantissa >>= cShiftOff; \ + /*Assert(!(uMantissa & RT_BIT_64(a_cBits - 1)));*/ \ + if (!fSignIn) \ + *piDst = (a_iType)uMantissa; \ + else \ + *piDst = -(a_iType)uMantissa; \ + \ + if (fRoundedOff) \ + { \ + fFsw |= X86_FSW_PE; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + } \ + /* \ + * Tiny sub-zero numbers. \ + */ \ + else if (iExponent < 0) \ + { \ + *piDst = 0; \ + fFsw |= X86_FSW_PE; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + /* \ + * Special MIN case. \ + */ \ + else if ( fSignIn && iExponent == a_cBits - 1 \ + && (a_cBits < 64 \ + ? uMantissa < (RT_BIT_64(63) | RT_BIT_64(65 - a_cBits)) \ + : uMantissa == RT_BIT_64(63)) ) \ + { \ + *piDst = a_iTypeMin; \ + if (uMantissa & (RT_BIT_64(64 - a_cBits + 1) - 1)) \ + { \ + fFsw |= X86_FSW_PE; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + } \ + /* \ + * Figure this weirdness. \ + */ \ + else if (0 /* huh? gone? */ && a_cBits == 16 && fSignIn && iExponent == 31 && uMantissa < UINT64_C(0x8000100000000000) ) \ + { \ + *piDst = 0; \ + if (uMantissa & (RT_BIT_64(64 - a_cBits + 1) - 1)) \ + { \ + fFsw |= X86_FSW_PE; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + } \ + /* \ + * Too large/small number outside the target integer range. \ + */ \ + else \ + { \ + fFsw |= X86_FSW_IE; \ + if (fFcw & X86_FCW_IM) \ + *piDst = a_iTypeIndefinite; \ + else \ + fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \ + } \ + } \ + /* \ + * Map both +0 and -0 to integer zero (signless/+). \ + */ \ + else if (RTFLOAT80U_IS_ZERO(pr80Val)) \ + *piDst = 0; \ + /* \ + * Denormals are just really tiny sub-zero numbers that are trucated to zero. \ + */ \ + else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val) || RTFLOAT80U_IS_DENORMAL(pr80Val)) \ + { \ + *piDst = 0; \ + fFsw |= X86_FSW_PE; \ + if (!(fFcw & X86_FCW_PM)) \ + fFsw |= X86_FSW_ES | X86_FSW_B; \ + } \ + /* \ + * All other special values are considered invalid arguments and result \ + * in an IE exception and indefinite value if masked. \ + */ \ + else \ + { \ + fFsw |= X86_FSW_IE; \ + if (fFcw & X86_FCW_IM) \ + *piDst = a_iTypeIndefinite; \ + else \ + fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \ + } \ + *pu16FSW = fFsw; \ +} +#if defined(IEM_WITHOUT_ASSEMBLY) +EMIT_FISTT(64, 64, int64_t, INT64_MIN, INT64_MAX, X86_FPU_INT64_INDEFINITE, RT_NOTHING, 1) +EMIT_FISTT(32, 32, int32_t, INT32_MIN, INT32_MAX, X86_FPU_INT32_INDEFINITE, RT_NOTHING, 1) +EMIT_FISTT(16, 16, int16_t, INT16_MIN, INT16_MAX, X86_FPU_INT16_INDEFINITE, RT_NOTHING, 1) +#endif +EMIT_FISTT(16, 16, int16_t, INT16_MIN, INT16_MAX, X86_FPU_INT16_INDEFINITE, _intel, 1) +EMIT_FISTT(16, 16, int16_t, INT16_MIN, INT16_MAX, X86_FPU_INT16_INDEFINITE, _amd, 0) + + +#if defined(IEM_WITHOUT_ASSEMBLY) + +IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_d80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, + PRTPBCD80U pd80Dst, PCRTFLOAT80U pr80Src)) +{ + /*static RTPBCD80U const s_ad80MaxMin[2] = { RTPBCD80U_INIT_MAX(), RTPBCD80U_INIT_MIN() };*/ + static RTPBCD80U const s_ad80Zeros[2] = { RTPBCD80U_INIT_ZERO(0), RTPBCD80U_INIT_ZERO(1) }; + static RTPBCD80U const s_ad80One[2] = { RTPBCD80U_INIT_C(0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,1), + RTPBCD80U_INIT_C(1, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,1) }; + static RTPBCD80U const s_d80Indefinite = RTPBCD80U_INIT_INDEFINITE(); + + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); + bool const fSignIn = pr80Src->s.fSign; + + /* + * Deal with normal numbers first. + */ + if (RTFLOAT80U_IS_NORMAL(pr80Src)) + { + uint64_t uMantissa = pr80Src->s.uMantissa; + int32_t iExponent = (int32_t)pr80Src->s.uExponent - RTFLOAT80U_EXP_BIAS; + if ( (uint32_t)iExponent <= 58 + || ((uint32_t)iExponent == 59 && uMantissa <= UINT64_C(0xde0b6b3a763fffff)) ) + { + unsigned const cShiftOff = 63 - iExponent; + uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1; + uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST + ? RT_BIT_64(cShiftOff - 1) + : (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP) + ? fRoundingOffMask + : 0; + uint64_t fRoundedOff = uMantissa & fRoundingOffMask; + + uMantissa >>= cShiftOff; + uint64_t const uRounding = (fRoundedOff + uRoundingAdd) >> cShiftOff; + uMantissa += uRounding; + if (uMantissa <= (uint64_t)RTPBCD80U_MAX) + { + if (fRoundedOff) + { + if ((uMantissa & 1) && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST && fRoundedOff == uRoundingAdd) + uMantissa &= ~(uint64_t)1; /* round to even number if equal distance between up/down. */ + else if (uRounding) + fFsw |= X86_FSW_C1; + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + pd80Dst->s.fSign = fSignIn; + pd80Dst->s.uPad = 0; + for (size_t iPair = 0; iPair < RT_ELEMENTS(pd80Dst->s.abPairs); iPair++) + { + unsigned const uDigits = uMantissa % 100; + uMantissa /= 100; + uint8_t const bLo = uDigits % 10; + uint8_t const bHi = uDigits / 10; + pd80Dst->s.abPairs[iPair] = RTPBCD80U_MAKE_PAIR(bHi, bLo); + } + } + else + { + /* overflowed after rounding. */ + fFsw |= X86_FSW_IE; + if (fFcw & X86_FCW_IM) + *pd80Dst = s_d80Indefinite; + else + fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); + } + } + /* + * Tiny sub-zero numbers. + */ + else if (iExponent < 0) + { + if (!fSignIn) + { + if ( (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP + || (iExponent == -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST)) + { + *pd80Dst = s_ad80One[fSignIn]; + fFsw |= X86_FSW_C1; + } + else + *pd80Dst = s_ad80Zeros[fSignIn]; + } + else + { + if ( (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP + || (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_ZERO + || (iExponent < -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST)) + *pd80Dst = s_ad80Zeros[fSignIn]; + else + { + *pd80Dst = s_ad80One[fSignIn]; + fFsw |= X86_FSW_C1; + } + } + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + /* + * Too large/small number outside the target integer range. + */ + else + { + fFsw |= X86_FSW_IE; + if (fFcw & X86_FCW_IM) + *pd80Dst = s_d80Indefinite; + else + fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); + } + } + /* + * Map both +0 and -0 to integer zero (signless/+). + */ + else if (RTFLOAT80U_IS_ZERO(pr80Src)) + *pd80Dst = s_ad80Zeros[fSignIn]; + /* + * Denormals are just really tiny sub-zero numbers that are either rounded + * to zero, 1 or -1 depending on sign and rounding control. + */ + else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Src) || RTFLOAT80U_IS_DENORMAL(pr80Src)) + { + if ((fFcw & X86_FCW_RC_MASK) != (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)) + *pd80Dst = s_ad80Zeros[fSignIn]; + else + { + *pd80Dst = s_ad80One[fSignIn]; + fFsw |= X86_FSW_C1; + } + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + /* + * All other special values are considered invalid arguments and result + * in an IE exception and indefinite value if masked. + */ + else + { + fFsw |= X86_FSW_IE; + if (fFcw & X86_FCW_IM) + *pd80Dst = s_d80Indefinite; + else + fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); + } + *pu16FSW = fFsw; +} + + +/********************************************************************************************************************************* +* FPU Helpers * +*********************************************************************************************************************************/ +AssertCompileSize(RTFLOAT128U, 16); +AssertCompileSize(RTFLOAT80U, 10); +AssertCompileSize(RTFLOAT64U, 8); +AssertCompileSize(RTFLOAT32U, 4); + +/** + * Normalizes a possible pseudo-normal value. + * + * Psuedo-normal values are some oddities from the 8087 & 287 days. They are + * denormals with the J-bit set, so they can simply be rewritten as 2**-16382, + * i.e. changing uExponent from 0 to 1. + * + * This macro will declare a RTFLOAT80U with the name given by + * @a a_r80ValNormalized and update the @a a_pr80Val variable to point to it if + * a normalization was performed. + * + * @note This must be applied before calling SoftFloat with a value that couldbe + * a pseudo-denormal, as SoftFloat doesn't handle pseudo-denormals + * correctly. + */ +#define IEM_NORMALIZE_PSEUDO_DENORMAL(a_pr80Val, a_r80ValNormalized) \ + RTFLOAT80U a_r80ValNormalized; \ + if (RTFLOAT80U_IS_PSEUDO_DENORMAL(a_pr80Val)) \ + { \ + a_r80ValNormalized = *a_pr80Val; \ + a_r80ValNormalized.s.uExponent = 1; \ + a_pr80Val = &a_r80ValNormalized; \ + } else do {} while (0) + +#ifdef IEM_WITH_FLOAT128_FOR_FPU + +DECLINLINE(int) iemFpuF128SetRounding(uint16_t fFcw) +{ + int fNew; + switch (fFcw & X86_FCW_RC_MASK) + { + default: + case X86_FCW_RC_NEAREST: fNew = FE_TONEAREST; break; + case X86_FCW_RC_ZERO: fNew = FE_TOWARDZERO; break; + case X86_FCW_RC_UP: fNew = FE_UPWARD; break; + case X86_FCW_RC_DOWN: fNew = FE_DOWNWARD; break; + } + int fOld = fegetround(); + fesetround(fNew); + return fOld; +} + + +DECLINLINE(void) iemFpuF128RestoreRounding(int fOld) +{ + fesetround(fOld); +} + +DECLINLINE(_Float128) iemFpuF128FromFloat80(PCRTFLOAT80U pr80Val, uint16_t fFcw) +{ + RT_NOREF(fFcw); + RTFLOAT128U Tmp; + Tmp.s2.uSignAndExponent = pr80Val->s2.uSignAndExponent; + Tmp.s2.uFractionHigh = (uint16_t)((pr80Val->s2.uMantissa & (RT_BIT_64(63) - 1)) >> 48); + Tmp.s2.uFractionMid = (uint32_t)((pr80Val->s2.uMantissa & UINT32_MAX) >> 16); + Tmp.s2.uFractionLow = pr80Val->s2.uMantissa << 48; + if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val)) + { + Assert(Tmp.s.uExponent == 0); + Tmp.s2.uSignAndExponent++; + } + return *(_Float128 *)&Tmp; +} + + +DECLINLINE(uint16_t) iemFpuF128ToFloat80(PRTFLOAT80U pr80Dst, _Float128 rd128ValSrc, uint16_t fFcw, uint16_t fFsw) +{ + RT_NOREF(fFcw); + RTFLOAT128U Tmp; + *(_Float128 *)&Tmp = rd128ValSrc; + ASMCompilerBarrier(); + if (RTFLOAT128U_IS_NORMAL(&Tmp)) + { + pr80Dst->s.fSign = Tmp.s64.fSign; + pr80Dst->s.uExponent = Tmp.s64.uExponent; + uint64_t uFraction = Tmp.s64.uFractionHi << (63 - 48) + | Tmp.s64.uFractionLo >> (64 - 15); + + /* Do rounding - just truncate in near mode when midway on an even outcome. */ + unsigned const cShiftOff = 64 - 15; + uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1; + uint64_t const uRoundedOff = Tmp.s64.uFractionLo & fRoundingOffMask; + if (uRoundedOff) + { + uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST + ? RT_BIT_64(cShiftOff - 1) + : (fFcw & X86_FCW_RC_MASK) == (Tmp.s64.fSign ? X86_FCW_RC_DOWN : X86_FCW_RC_UP) + ? fRoundingOffMask + : 0; + if ( (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST + || (Tmp.s64.uFractionLo & RT_BIT_64(cShiftOff)) + || uRoundedOff != uRoundingAdd) + { + if ((uRoundedOff + uRoundingAdd) >> cShiftOff) + { + uFraction += 1; + if (!(uFraction & RT_BIT_64(63))) + { /* likely */ } + else + { + uFraction >>= 1; + pr80Dst->s.uExponent++; + if (pr80Dst->s.uExponent == RTFLOAT64U_EXP_MAX) + return fFsw; + } + fFsw |= X86_FSW_C1; + } + } + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + pr80Dst->s.uMantissa = RT_BIT_64(63) | uFraction; + } + else if (RTFLOAT128U_IS_ZERO(&Tmp)) + { + pr80Dst->s.fSign = Tmp.s64.fSign; + pr80Dst->s.uExponent = 0; + pr80Dst->s.uMantissa = 0; + } + else if (RTFLOAT128U_IS_INF(&Tmp)) + { + pr80Dst->s.fSign = Tmp.s64.fSign; + pr80Dst->s.uExponent = 0; + pr80Dst->s.uMantissa = 0; + } + return fFsw; +} + + +#else /* !IEM_WITH_FLOAT128_FOR_FPU - SoftFloat */ + +/** Initializer for the SoftFloat state structure. */ +# define IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(a_fFcw) \ + { \ + softfloat_tininess_afterRounding, \ + ((a_fFcw) & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST ? (uint8_t)softfloat_round_near_even \ + : ((a_fFcw) & X86_FCW_RC_MASK) == X86_FCW_RC_UP ? (uint8_t)softfloat_round_max \ + : ((a_fFcw) & X86_FCW_RC_MASK) == X86_FCW_RC_DOWN ? (uint8_t)softfloat_round_min \ + : (uint8_t)softfloat_round_minMag, \ + 0, \ + (uint8_t)((a_fFcw) & X86_FCW_XCPT_MASK), \ + ((a_fFcw) & X86_FCW_PC_MASK) == X86_FCW_PC_53 ? (uint8_t)64 \ + : ((a_fFcw) & X86_FCW_PC_MASK) == X86_FCW_PC_24 ? (uint8_t)32 : (uint8_t)80 \ + } + +/** Returns updated FSW from a SoftFloat state and exception mask (FCW). */ +# define IEM_SOFTFLOAT_STATE_TO_FSW(a_fFsw, a_pSoftState, a_fFcw) \ + ( (a_fFsw) \ + | (uint16_t)(((a_pSoftState)->exceptionFlags & softfloat_flag_c1) << 2) \ + | ((a_pSoftState)->exceptionFlags & X86_FSW_XCPT_MASK) \ + | ( ((a_pSoftState)->exceptionFlags & X86_FSW_XCPT_MASK) & (~(a_fFcw) & X86_FSW_XCPT_MASK) \ + ? X86_FSW_ES | X86_FSW_B : 0) ) + + +DECLINLINE(float128_t) iemFpuSoftF128Precision(float128_t r128, unsigned cBits, uint16_t fFcw = X86_FCW_RC_NEAREST) +{ + RT_NOREF(fFcw); + Assert(cBits > 64); +# if 0 /* rounding does not seem to help */ + uint64_t off = r128.v[0] & (RT_BIT_64(1 + 112 - cBits) - 1); + r128.v[0] &= ~(RT_BIT_64(1 + 112 - cBits) - 1); + if (off >= RT_BIT_64(1 + 112 - cBits - 1) + && (r128.v[0] & RT_BIT_64(1 + 112 - cBits))) + { + uint64_t uOld = r128.v[0]; + r128.v[0] += RT_BIT_64(1 + 112 - cBits); + if (r128.v[0] < uOld) + r128.v[1] += 1; + } +# else + r128.v[0] &= ~(RT_BIT_64(1 + 112 - cBits) - 1); +# endif + return r128; +} + + +DECLINLINE(float128_t) iemFpuSoftF128PrecisionIprt(PCRTFLOAT128U pr128, unsigned cBits, uint16_t fFcw = X86_FCW_RC_NEAREST) +{ + RT_NOREF(fFcw); + Assert(cBits > 64); +# if 0 /* rounding does not seem to help, not even on constants */ + float128_t r128 = { pr128->au64[0], pr128->au64[1] }; + uint64_t off = r128.v[0] & (RT_BIT_64(1 + 112 - cBits) - 1); + r128.v[0] &= ~(RT_BIT_64(1 + 112 - cBits) - 1); + if (off >= RT_BIT_64(1 + 112 - cBits - 1) + && (r128.v[0] & RT_BIT_64(1 + 112 - cBits))) + { + uint64_t uOld = r128.v[0]; + r128.v[0] += RT_BIT_64(1 + 112 - cBits); + if (r128.v[0] < uOld) + r128.v[1] += 1; + } + return r128; +# else + float128_t r128 = { { pr128->au64[0] & ~(RT_BIT_64(1 + 112 - cBits) - 1), pr128->au64[1] } }; + return r128; +# endif +} + + +# if 0 /* unused */ +DECLINLINE(float128_t) iemFpuSoftF128FromIprt(PCRTFLOAT128U pr128) +{ + float128_t r128 = { { pr128->au64[0], pr128->au64[1] } }; + return r128; +} +# endif + + +/** Converts a 80-bit floating point value to SoftFloat 128-bit floating point. */ +DECLINLINE(float128_t) iemFpuSoftF128FromFloat80(PCRTFLOAT80U pr80Val) +{ + extFloat80_t Tmp; + Tmp.signExp = pr80Val->s2.uSignAndExponent; + Tmp.signif = pr80Val->s2.uMantissa; + softfloat_state_t Ignored = SOFTFLOAT_STATE_INIT_DEFAULTS(); + return extF80_to_f128(Tmp, &Ignored); +} + + +/** + * Converts from the packed IPRT 80-bit floating point (RTFLOAT80U) format to + * the SoftFloat extended 80-bit floating point format (extFloat80_t). + * + * This is only a structure format conversion, nothing else. + */ +DECLINLINE(extFloat80_t) iemFpuSoftF80FromIprt(PCRTFLOAT80U pr80Val) +{ + extFloat80_t Tmp; + Tmp.signExp = pr80Val->s2.uSignAndExponent; + Tmp.signif = pr80Val->s2.uMantissa; + return Tmp; +} + + +/** + * Converts from SoftFloat extended 80-bit floating point format (extFloat80_t) + * to the packed IPRT 80-bit floating point (RTFLOAT80U) format. + * + * This is only a structure format conversion, nothing else. + */ +DECLINLINE(PRTFLOAT80U) iemFpuSoftF80ToIprt(PRTFLOAT80U pr80Dst, extFloat80_t const r80XSrc) +{ + pr80Dst->s2.uSignAndExponent = r80XSrc.signExp; + pr80Dst->s2.uMantissa = r80XSrc.signif; + return pr80Dst; +} + + +DECLINLINE(uint16_t) iemFpuSoftF128ToFloat80(PRTFLOAT80U pr80Dst, float128_t r128Src, uint16_t fFcw, uint16_t fFsw) +{ + RT_NOREF(fFcw); + RTFLOAT128U Tmp; + *(float128_t *)&Tmp = r128Src; + ASMCompilerBarrier(); + + if (RTFLOAT128U_IS_NORMAL(&Tmp)) + { + pr80Dst->s.fSign = Tmp.s64.fSign; + pr80Dst->s.uExponent = Tmp.s64.uExponent; + uint64_t uFraction = Tmp.s64.uFractionHi << (63 - 48) + | Tmp.s64.uFractionLo >> (64 - 15); + + /* Do rounding - just truncate in near mode when midway on an even outcome. */ + unsigned const cShiftOff = 64 - 15; + uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1; + uint64_t const uRoundedOff = Tmp.s64.uFractionLo & fRoundingOffMask; + if (uRoundedOff) + { + uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST + ? RT_BIT_64(cShiftOff - 1) + : (fFcw & X86_FCW_RC_MASK) == (Tmp.s64.fSign ? X86_FCW_RC_DOWN : X86_FCW_RC_UP) + ? fRoundingOffMask + : 0; + if ( (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST + || (Tmp.s64.uFractionLo & RT_BIT_64(cShiftOff)) + || uRoundedOff != uRoundingAdd) + { + if ((uRoundedOff + uRoundingAdd) >> cShiftOff) + { + uFraction += 1; + if (!(uFraction & RT_BIT_64(63))) + { /* likely */ } + else + { + uFraction >>= 1; + pr80Dst->s.uExponent++; + if (pr80Dst->s.uExponent == RTFLOAT64U_EXP_MAX) + return fFsw; + } + fFsw |= X86_FSW_C1; + } + } + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + pr80Dst->s.uMantissa = RT_BIT_64(63) | uFraction; + } + else if (RTFLOAT128U_IS_ZERO(&Tmp)) + { + pr80Dst->s.fSign = Tmp.s64.fSign; + pr80Dst->s.uExponent = 0; + pr80Dst->s.uMantissa = 0; + } + else if (RTFLOAT128U_IS_INF(&Tmp)) + { + pr80Dst->s.fSign = Tmp.s64.fSign; + pr80Dst->s.uExponent = 0x7fff; + pr80Dst->s.uMantissa = 0; + } + return fFsw; +} + + +/** + * Helper for transfering exception and C1 to FSW and setting the result value + * accordingly. + * + * @returns Updated FSW. + * @param pSoftState The SoftFloat state following the operation. + * @param r80XResult The result of the SoftFloat operation. + * @param pr80Result Where to store the result for IEM. + * @param fFcw The FPU control word. + * @param fFsw The FSW before the operation, with necessary bits + * cleared and such. + * @param pr80XcptResult Alternative return value for use an unmasked \#IE is + * raised. + */ +DECLINLINE(uint16_t) iemFpuSoftStateAndF80ToFswAndIprtResult(softfloat_state_t const *pSoftState, extFloat80_t r80XResult, + PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw, + PCRTFLOAT80U pr80XcptResult) +{ + fFsw |= (pSoftState->exceptionFlags & X86_FSW_XCPT_MASK) + | (uint16_t)((pSoftState->exceptionFlags & softfloat_flag_c1) << 2); + if (fFsw & ~fFcw & X86_FSW_XCPT_MASK) + fFsw |= X86_FSW_ES | X86_FSW_B; + + if (!(fFsw & ~fFcw & (X86_FSW_IE | X86_FSW_DE))) + iemFpuSoftF80ToIprt(pr80Result, r80XResult); + else + { + fFsw &= ~(X86_FSW_OE | X86_FSW_UE | X86_FSW_PE | X86_FSW_ZE | X86_FSW_C1); + *pr80Result = *pr80XcptResult; + } + return fFsw; +} + + +/** + * Helper doing polynomial evaluation using Horner's method. + * + * See https://en.wikipedia.org/wiki/Horner%27s_method for details. + */ +float128_t iemFpuSoftF128HornerPoly(float128_t z, PCRTFLOAT128U g_par128HornerConsts, size_t cHornerConsts, + unsigned cPrecision, softfloat_state_t *pSoftState) +{ + Assert(cHornerConsts > 1); + size_t i = cHornerConsts - 1; + float128_t r128Result = iemFpuSoftF128PrecisionIprt(&g_par128HornerConsts[i], cPrecision); + while (i-- > 0) + { + r128Result = iemFpuSoftF128Precision(f128_mul(r128Result, z, pSoftState), cPrecision); + r128Result = f128_add(r128Result, iemFpuSoftF128PrecisionIprt(&g_par128HornerConsts[i], cPrecision), pSoftState); + r128Result = iemFpuSoftF128Precision(r128Result, cPrecision); + } + return r128Result; +} + +#endif /* !IEM_WITH_FLOAT128_FOR_FPU - SoftFloat */ + + +/** + * Composes a normalized and rounded RTFLOAT80U result from a 192 bit wide + * mantissa, exponent and sign. + * + * @returns Updated FSW. + * @param pr80Dst Where to return the composed value. + * @param fSign The sign. + * @param puMantissa The mantissa, 256-bit type but the to 64-bits are + * ignored and should be zero. This will probably be + * modified during normalization and rounding. + * @param iExponent Unbiased exponent. + * @param fFcw The FPU control word. + * @param fFsw The FPU status word. + */ +static uint16_t iemFpuFloat80RoundAndComposeFrom192(PRTFLOAT80U pr80Dst, bool fSign, PRTUINT256U puMantissa, + int32_t iExponent, uint16_t fFcw, uint16_t fFsw) +{ + AssertStmt(puMantissa->QWords.qw3 == 0, puMantissa->QWords.qw3 = 0); + + iExponent += RTFLOAT80U_EXP_BIAS; + + /* Do normalization if necessary and possible. */ + if (!(puMantissa->QWords.qw2 & RT_BIT_64(63))) + { + int cShift = 192 - RTUInt256BitCount(puMantissa); + if (iExponent > cShift) + iExponent -= cShift; + else + { + if (fFcw & X86_FCW_UM) + { + if (iExponent > 0) + cShift = --iExponent; + else + cShift = 0; + } + iExponent -= cShift; + } + RTUInt256AssignShiftLeft(puMantissa, cShift); + } + + /* Do rounding. */ + uint64_t uMantissa = puMantissa->QWords.qw2; + if (puMantissa->QWords.qw1 || puMantissa->QWords.qw0) + { + bool fAdd; + switch (fFcw & X86_FCW_RC_MASK) + { + default: /* (for the simple-minded MSC which otherwise things fAdd would be used uninitialized) */ + case X86_FCW_RC_NEAREST: + if (puMantissa->QWords.qw1 & RT_BIT_64(63)) + { + if ( (uMantissa & 1) + || puMantissa->QWords.qw0 != 0 + || puMantissa->QWords.qw1 != RT_BIT_64(63)) + { + fAdd = true; + break; + } + uMantissa &= ~(uint64_t)1; + } + fAdd = false; + break; + case X86_FCW_RC_ZERO: + fAdd = false; + break; + case X86_FCW_RC_UP: + fAdd = !fSign; + break; + case X86_FCW_RC_DOWN: + fAdd = fSign; + break; + } + if (fAdd) + { + uint64_t const uTmp = uMantissa; + uMantissa = uTmp + 1; + if (uMantissa < uTmp) + { + uMantissa >>= 1; + uMantissa |= RT_BIT_64(63); + iExponent++; + } + fFsw |= X86_FSW_C1; + } + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + /* Check for underflow (denormals). */ + if (iExponent <= 0) + { + if (fFcw & X86_FCW_UM) + { + if (uMantissa & RT_BIT_64(63)) + uMantissa >>= 1; + iExponent = 0; + } + else + { + iExponent += RTFLOAT80U_EXP_BIAS_ADJUST; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_UE; + } + /* Check for overflow */ + else if (iExponent >= RTFLOAT80U_EXP_MAX) + { + Assert(iExponent < RTFLOAT80U_EXP_MAX); + } + + /* Compose the result. */ + pr80Dst->s.uMantissa = uMantissa; + pr80Dst->s.uExponent = iExponent; + pr80Dst->s.fSign = fSign; + return fFsw; +} + + +/** + * See also iemAImpl_fld_r80_from_r32 + */ +static uint16_t iemAImplConvertR32ToR80(PCRTFLOAT32U pr32Val, PRTFLOAT80U pr80Dst) +{ + uint16_t fFsw = 0; + if (RTFLOAT32U_IS_NORMAL(pr32Val)) + { + pr80Dst->sj64.fSign = pr32Val->s.fSign; + pr80Dst->sj64.fInteger = 1; + pr80Dst->sj64.uFraction = (uint64_t)pr32Val->s.uFraction + << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS); + pr80Dst->sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS; + Assert(RTFLOAT80U_IS_NORMAL(pr80Dst)); + } + else if (RTFLOAT32U_IS_ZERO(pr32Val)) + { + pr80Dst->s.fSign = pr32Val->s.fSign; + pr80Dst->s.uExponent = 0; + pr80Dst->s.uMantissa = 0; + Assert(RTFLOAT80U_IS_ZERO(pr80Dst)); + } + else if (RTFLOAT32U_IS_SUBNORMAL(pr32Val)) + { + /* Subnormal -> normalized + X86_FSW_DE return. */ + pr80Dst->sj64.fSign = pr32Val->s.fSign; + pr80Dst->sj64.fInteger = 1; + unsigned const cExtraShift = RTFLOAT32U_FRACTION_BITS - ASMBitLastSetU32(pr32Val->s.uFraction); + pr80Dst->sj64.uFraction = (uint64_t)pr32Val->s.uFraction + << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS + cExtraShift + 1); + pr80Dst->sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift; + fFsw = X86_FSW_DE; + } + else if (RTFLOAT32U_IS_INF(pr32Val)) + { + pr80Dst->s.fSign = pr32Val->s.fSign; + pr80Dst->s.uExponent = RTFLOAT80U_EXP_MAX; + pr80Dst->s.uMantissa = RT_BIT_64(63); + Assert(RTFLOAT80U_IS_INF(pr80Dst)); + } + else + { + Assert(RTFLOAT32U_IS_NAN(pr32Val)); + pr80Dst->sj64.fSign = pr32Val->s.fSign; + pr80Dst->sj64.uExponent = RTFLOAT80U_EXP_MAX; + pr80Dst->sj64.fInteger = 1; + pr80Dst->sj64.uFraction = (uint64_t)pr32Val->s.uFraction + << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS); + Assert(RTFLOAT80U_IS_NAN(pr80Dst)); + Assert(RTFLOAT80U_IS_SIGNALLING_NAN(pr80Dst) == RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val)); + } + return fFsw; +} + + +/** + * See also iemAImpl_fld_r80_from_r64 + */ +static uint16_t iemAImplConvertR64ToR80(PCRTFLOAT64U pr64Val, PRTFLOAT80U pr80Dst) +{ + uint16_t fFsw = 0; + if (RTFLOAT64U_IS_NORMAL(pr64Val)) + { + pr80Dst->sj64.fSign = pr64Val->s.fSign; + pr80Dst->sj64.fInteger = 1; + pr80Dst->sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS); + pr80Dst->sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS; + Assert(RTFLOAT80U_IS_NORMAL(pr80Dst)); + } + else if (RTFLOAT64U_IS_ZERO(pr64Val)) + { + pr80Dst->s.fSign = pr64Val->s.fSign; + pr80Dst->s.uExponent = 0; + pr80Dst->s.uMantissa = 0; + Assert(RTFLOAT80U_IS_ZERO(pr80Dst)); + } + else if (RTFLOAT64U_IS_SUBNORMAL(pr64Val)) + { + /* Subnormal values gets normalized. */ + pr80Dst->sj64.fSign = pr64Val->s.fSign; + pr80Dst->sj64.fInteger = 1; + unsigned const cExtraShift = RTFLOAT64U_FRACTION_BITS - ASMBitLastSetU64(pr64Val->s64.uFraction); + pr80Dst->sj64.uFraction = pr64Val->s64.uFraction + << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS + cExtraShift + 1); + pr80Dst->sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift; + fFsw = X86_FSW_DE; + } + else if (RTFLOAT64U_IS_INF(pr64Val)) + { + pr80Dst->s.fSign = pr64Val->s.fSign; + pr80Dst->s.uExponent = RTFLOAT80U_EXP_MAX; + pr80Dst->s.uMantissa = RT_BIT_64(63); + Assert(RTFLOAT80U_IS_INF(pr80Dst)); + } + else + { + /* Signalling and quiet NaNs, both turn into quiet ones when loaded (weird). */ + Assert(RTFLOAT64U_IS_NAN(pr64Val)); + pr80Dst->sj64.fSign = pr64Val->s.fSign; + pr80Dst->sj64.uExponent = RTFLOAT80U_EXP_MAX; + pr80Dst->sj64.fInteger = 1; + pr80Dst->sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS); + Assert(RTFLOAT80U_IS_NAN(pr80Dst)); + Assert(RTFLOAT80U_IS_SIGNALLING_NAN(pr80Dst) == RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val)); + } + return fFsw; +} + + +/** + * See also EMIT_FILD. + */ +#define EMIT_CONVERT_IXX_TO_R80(a_cBits) \ +static PRTFLOAT80U iemAImplConvertI ## a_cBits ## ToR80(int ## a_cBits ## _t iVal, PRTFLOAT80U pr80Dst) \ +{ \ + if (iVal == 0) \ + { \ + pr80Dst->s.fSign = 0; \ + pr80Dst->s.uExponent = 0; \ + pr80Dst->s.uMantissa = 0; \ + } \ + else \ + { \ + if (iVal > 0) \ + pr80Dst->s.fSign = 0; \ + else \ + { \ + pr80Dst->s.fSign = 1; \ + iVal = -iVal; \ + } \ + unsigned const cBits = ASMBitLastSetU ## a_cBits((uint ## a_cBits ## _t)iVal); \ + pr80Dst->s.uExponent = cBits - 1 + RTFLOAT80U_EXP_BIAS; \ + pr80Dst->s.uMantissa = (uint64_t)iVal << (RTFLOAT80U_FRACTION_BITS + 1 - cBits); \ + } \ + return pr80Dst; \ +} +EMIT_CONVERT_IXX_TO_R80(16) +EMIT_CONVERT_IXX_TO_R80(32) +//EMIT_CONVERT_IXX_TO_R80(64) + +/** For implementing iemAImpl_fmul_r80_by_r64 and such. */ +#define EMIT_R80_BY_R64(a_Name, a_fnR80ByR80, a_DenormalException) \ +IEM_DECL_IMPL_DEF(void, a_Name,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2)) \ +{ \ + RTFLOAT80U r80Val2; \ + uint16_t fFsw = iemAImplConvertR64ToR80(pr64Val2, &r80Val2); \ + Assert(!fFsw || fFsw == X86_FSW_DE); \ + if (fFsw) \ + { \ + if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_NAN(pr80Val1) || (a_DenormalException)) \ + fFsw = 0; \ + else if (!(pFpuState->FCW & X86_FCW_DM)) \ + { \ + pFpuRes->r80Result = *pr80Val1; \ + pFpuRes->FSW = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT) \ + | X86_FSW_DE | X86_FSW_ES | X86_FSW_B; \ + return; \ + } \ + } \ + a_fnR80ByR80(pFpuState, pFpuRes, pr80Val1, &r80Val2); \ + pFpuRes->FSW = (pFpuRes->FSW & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT) | fFsw; \ +} + +/** For implementing iemAImpl_fmul_r80_by_r32 and such. */ +#define EMIT_R80_BY_R32(a_Name, a_fnR80ByR80, a_DenormalException) \ +IEM_DECL_IMPL_DEF(void, a_Name,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2)) \ +{ \ + RTFLOAT80U r80Val2; \ + uint16_t fFsw = iemAImplConvertR32ToR80(pr32Val2, &r80Val2); \ + Assert(!fFsw || fFsw == X86_FSW_DE); \ + if (fFsw) \ + { \ + if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_NAN(pr80Val1) || (a_DenormalException)) \ + fFsw = 0; \ + else if (!(pFpuState->FCW & X86_FCW_DM)) \ + { \ + pFpuRes->r80Result = *pr80Val1; \ + pFpuRes->FSW = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT) \ + | X86_FSW_DE | X86_FSW_ES | X86_FSW_B; \ + return; \ + } \ + } \ + a_fnR80ByR80(pFpuState, pFpuRes, pr80Val1, &r80Val2); \ + pFpuRes->FSW = (pFpuRes->FSW & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT) | fFsw; \ +} + +/** For implementing iemAImpl_fimul_r80_by_i32 and such. */ +#define EMIT_R80_BY_I32(a_Name, a_fnR80ByR80) \ +IEM_DECL_IMPL_DEF(void, a_Name,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2)) \ +{ \ + RTFLOAT80U r80Val2; \ + a_fnR80ByR80(pFpuState, pFpuRes, pr80Val1, iemAImplConvertI32ToR80(*pi32Val2, &r80Val2)); \ + pFpuRes->FSW = (pFpuRes->FSW & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT); \ +} + +/** For implementing iemAImpl_fimul_r80_by_i16 and such. */ +#define EMIT_R80_BY_I16(a_Name, a_fnR80ByR80) \ +IEM_DECL_IMPL_DEF(void, a_Name,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2)) \ +{ \ + RTFLOAT80U r80Val2; \ + a_fnR80ByR80(pFpuState, pFpuRes, pr80Val1, iemAImplConvertI16ToR80(*pi16Val2, &r80Val2)); \ + pFpuRes->FSW = (pFpuRes->FSW & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT); \ +} + + + +/********************************************************************************************************************************* +* x86 FPU Division Operations * +*********************************************************************************************************************************/ + +/** Worker for iemAImpl_fdiv_r80_by_r80 & iemAImpl_fdivr_r80_by_r80. */ +static uint16_t iemAImpl_fdiv_f80_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result, + uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org) +{ + if (!RTFLOAT80U_IS_ZERO(pr80Val2) || RTFLOAT80U_IS_NAN(pr80Val1) || RTFLOAT80U_IS_INF(pr80Val1)) + { + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw); + extFloat80_t r80XResult = extF80_div(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState); + return iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org); + } + if (!RTFLOAT80U_IS_ZERO(pr80Val1)) + { /* Div by zero. */ + if (fFcw & X86_FCW_ZM) + *pr80Result = g_ar80Infinity[pr80Val1->s.fSign != pr80Val2->s.fSign]; + else + { + *pr80Result = *pr80Val1Org; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_ZE; + } + else + { /* Invalid operand */ + if (fFcw & X86_FCW_IM) + *pr80Result = g_r80Indefinite; + else + { + *pr80Result = *pr80Val1Org; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + return fFsw; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fdiv_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + + /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */ + if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2)) + { + if (fFcw & X86_FCW_IM) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + /* SoftFloat does not check for denormals and certainly not report them to us. NaNs & /0 trumps denormals. */ + else if ( (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2) && !RTFLOAT80U_IS_ZERO(pr80Val2)) + || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) ) + { + if (fFcw & X86_FCW_DM) + { + PCRTFLOAT80U const pr80Val1Org = pr80Val1; + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized); + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized); + fFsw = iemAImpl_fdiv_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org); + } + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_DE; + } + /* SoftFloat can handle the rest: */ + else + fFsw = iemAImpl_fdiv_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1); + + pFpuRes->FSW = fFsw; +} + + +EMIT_R80_BY_R64(iemAImpl_fdiv_r80_by_r64, iemAImpl_fdiv_r80_by_r80, 0) +EMIT_R80_BY_R32(iemAImpl_fdiv_r80_by_r32, iemAImpl_fdiv_r80_by_r80, 0) +EMIT_R80_BY_I32(iemAImpl_fidiv_r80_by_i32, iemAImpl_fdiv_r80_by_r80) +EMIT_R80_BY_I16(iemAImpl_fidiv_r80_by_i16, iemAImpl_fdiv_r80_by_r80) + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fdivr_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + + /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */ + if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2)) + { + if (fFcw & X86_FCW_IM) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + /* SoftFloat does not check for denormals and certainly not report them to us. NaNs & /0 trumps denormals. */ + else if ( (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2)) + || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1) && !RTFLOAT80U_IS_ZERO(pr80Val1)) ) + { + if (fFcw & X86_FCW_DM) + { + PCRTFLOAT80U const pr80Val1Org = pr80Val1; + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized); + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized); + fFsw = iemAImpl_fdiv_f80_r80_worker(pr80Val2, pr80Val1, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org); + } + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_DE; + } + /* SoftFloat can handle the rest: */ + else + fFsw = iemAImpl_fdiv_f80_r80_worker(pr80Val2, pr80Val1, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1); + + pFpuRes->FSW = fFsw; +} + + +EMIT_R80_BY_R64(iemAImpl_fdivr_r80_by_r64, iemAImpl_fdivr_r80_by_r80, RTFLOAT80U_IS_ZERO(pr80Val1)) +EMIT_R80_BY_R32(iemAImpl_fdivr_r80_by_r32, iemAImpl_fdivr_r80_by_r80, RTFLOAT80U_IS_ZERO(pr80Val1)) +EMIT_R80_BY_I32(iemAImpl_fidivr_r80_by_i32, iemAImpl_fdivr_r80_by_r80) +EMIT_R80_BY_I16(iemAImpl_fidivr_r80_by_i16, iemAImpl_fdivr_r80_by_r80) + + +/** Worker for iemAImpl_fprem_r80_by_r80 & iemAImpl_fprem1_r80_by_r80. */ +static uint16_t iemAImpl_fprem_fprem1_r80_by_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result, + uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org, bool fLegacyInstr) +{ + if (!RTFLOAT80U_IS_ZERO(pr80Val2) || RTFLOAT80U_IS_NAN(pr80Val1) || RTFLOAT80U_IS_INF(pr80Val1)) + { + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw); + uint16_t fCxFlags = 0; + extFloat80_t r80XResult = extF80_partialRem(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), + fLegacyInstr ? softfloat_round_minMag : softfloat_round_near_even, + &fCxFlags, &SoftState); + Assert(!(fCxFlags & ~X86_FSW_C_MASK)); + fFsw = iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org); + if ( !(fFsw & X86_FSW_IE) + && !RTFLOAT80U_IS_NAN(pr80Result) + && !RTFLOAT80U_IS_INDEFINITE(pr80Result)) + { + fFsw &= ~(uint16_t)X86_FSW_C_MASK; + fFsw |= fCxFlags & X86_FSW_C_MASK; + } + return fFsw; + } + + /* Invalid operand */ + if (fFcw & X86_FCW_IM) + *pr80Result = g_r80Indefinite; + else + { + *pr80Result = *pr80Val1Org; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + return fFsw | X86_FSW_IE; +} + + +static void iemAImpl_fprem_fprem1_r80_by_r80(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, bool fLegacyInstr) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 /*| X86_FSW_C2*/ | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + + /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. + In addition, we'd like to handle zero ST(1) now as SoftFloat returns Inf instead + of Indefinite. (Note! There is no #Z like the footnotes to tables 3-31 and 3-32 + for the FPREM1 & FPREM1 instructions in the intel reference manual claims!) */ + if ( RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2) + || (RTFLOAT80U_IS_ZERO(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1) && !RTFLOAT80U_IS_INDEFINITE(pr80Val1))) + { + if (fFcw & X86_FCW_IM) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + /* SoftFloat does not check for denormals and certainly not report them to us. NaNs & /0 trumps denormals. */ + else if ( (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2) && !RTFLOAT80U_IS_ZERO(pr80Val2)) + || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1) && !RTFLOAT80U_IS_INF(pr80Val1)) ) + { + if (fFcw & X86_FCW_DM) + { + PCRTFLOAT80U const pr80Val1Org = pr80Val1; + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized); + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized); + fFsw = iemAImpl_fprem_fprem1_r80_by_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, + pr80Val1Org, fLegacyInstr); + } + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_DE; + } + /* SoftFloat can handle the rest: */ + else + fFsw = iemAImpl_fprem_fprem1_r80_by_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, + pr80Val1, fLegacyInstr); + + pFpuRes->FSW = fFsw; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fprem_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + iemAImpl_fprem_fprem1_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2, true /*fLegacyInstr*/); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fprem1_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + iemAImpl_fprem_fprem1_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2, false /*fLegacyInstr*/); +} + + +/********************************************************************************************************************************* +* x87 FPU Multiplication Operations * +*********************************************************************************************************************************/ + +/** Worker for iemAImpl_fmul_r80_by_r80. */ +static uint16_t iemAImpl_fmul_f80_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result, + uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw); + extFloat80_t r80XResult = extF80_mul(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState); + return iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fmul_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + + /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */ + if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2)) + { + if (fFcw & X86_FCW_IM) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + /* SoftFloat does not check for denormals and certainly not report them to us. NaNs trumps denormals. */ + else if ( (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2)) + || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) ) + { + if (fFcw & X86_FCW_DM) + { + PCRTFLOAT80U const pr80Val1Org = pr80Val1; + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized); + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized); + fFsw = iemAImpl_fmul_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org); + } + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_DE; + } + /* SoftFloat can handle the rest: */ + else + fFsw = iemAImpl_fmul_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1); + + pFpuRes->FSW = fFsw; +} + + +EMIT_R80_BY_R64(iemAImpl_fmul_r80_by_r64, iemAImpl_fmul_r80_by_r80, 0) +EMIT_R80_BY_R32(iemAImpl_fmul_r80_by_r32, iemAImpl_fmul_r80_by_r80, 0) +EMIT_R80_BY_I32(iemAImpl_fimul_r80_by_i32, iemAImpl_fmul_r80_by_r80) +EMIT_R80_BY_I16(iemAImpl_fimul_r80_by_i16, iemAImpl_fmul_r80_by_r80) + + +/********************************************************************************************************************************* +* x87 FPU Addition * +*********************************************************************************************************************************/ + +/** Worker for iemAImpl_fadd_r80_by_r80. */ +static uint16_t iemAImpl_fadd_f80_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result, + uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw); + extFloat80_t r80XResult = extF80_add(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState); + return iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fadd_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + + /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */ + if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2)) + { + if (fFcw & X86_FCW_IM) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + /* SoftFloat does not check for denormals and certainly not report them to us. NaNs trumps denormals. */ + else if ( (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2)) + || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) ) + { + if (fFcw & X86_FCW_DM) + { + PCRTFLOAT80U const pr80Val1Org = pr80Val1; + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized); + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized); + fFsw = iemAImpl_fadd_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org); + } + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_DE; + } + /* SoftFloat can handle the rest: */ + else + fFsw = iemAImpl_fadd_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1); + + pFpuRes->FSW = fFsw; +} + + +EMIT_R80_BY_R64(iemAImpl_fadd_r80_by_r64, iemAImpl_fadd_r80_by_r80, 0) +EMIT_R80_BY_R32(iemAImpl_fadd_r80_by_r32, iemAImpl_fadd_r80_by_r80, 0) +EMIT_R80_BY_I32(iemAImpl_fiadd_r80_by_i32, iemAImpl_fadd_r80_by_r80) +EMIT_R80_BY_I16(iemAImpl_fiadd_r80_by_i16, iemAImpl_fadd_r80_by_r80) + + +/********************************************************************************************************************************* +* x87 FPU Subtraction * +*********************************************************************************************************************************/ + +/** Worker for iemAImpl_fsub_r80_by_r80 and iemAImpl_fsubr_r80_by_r80. */ +static uint16_t iemAImpl_fsub_f80_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result, + uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw); + extFloat80_t r80XResult = extF80_sub(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState); + return iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fsub_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + + /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */ + if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2)) + { + if (fFcw & X86_FCW_IM) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + /* SoftFloat does not check for denormals and certainly not report them to us. NaNs trumps denormals. */ + else if ( (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2)) + || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) ) + { + if (fFcw & X86_FCW_DM) + { + PCRTFLOAT80U const pr80Val1Org = pr80Val1; + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized); + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized); + fFsw = iemAImpl_fsub_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org); + } + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_DE; + } + /* SoftFloat can handle the rest: */ + else + fFsw = iemAImpl_fsub_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1); + + pFpuRes->FSW = fFsw; +} + + +EMIT_R80_BY_R64(iemAImpl_fsub_r80_by_r64, iemAImpl_fsub_r80_by_r80, 0) +EMIT_R80_BY_R32(iemAImpl_fsub_r80_by_r32, iemAImpl_fsub_r80_by_r80, 0) +EMIT_R80_BY_I32(iemAImpl_fisub_r80_by_i32, iemAImpl_fsub_r80_by_r80) +EMIT_R80_BY_I16(iemAImpl_fisub_r80_by_i16, iemAImpl_fsub_r80_by_r80) + + +/* Same as iemAImpl_fsub_r80_by_r80, but with input operands switched. */ +IEM_DECL_IMPL_DEF(void, iemAImpl_fsubr_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + + /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */ + if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2)) + { + if (fFcw & X86_FCW_IM) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + /* SoftFloat does not check for denormals and certainly not report them to us. NaNs trumps denormals. */ + else if ( (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2)) + || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) ) + { + if (fFcw & X86_FCW_DM) + { + PCRTFLOAT80U const pr80Val1Org = pr80Val1; + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized); + IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized); + fFsw = iemAImpl_fsub_f80_r80_worker(pr80Val2, pr80Val1, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org); + } + else + { + pFpuRes->r80Result = *pr80Val1; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_DE; + } + /* SoftFloat can handle the rest: */ + else + fFsw = iemAImpl_fsub_f80_r80_worker(pr80Val2, pr80Val1, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1); + + pFpuRes->FSW = fFsw; +} + + +EMIT_R80_BY_R64(iemAImpl_fsubr_r80_by_r64, iemAImpl_fsubr_r80_by_r80, 0) +EMIT_R80_BY_R32(iemAImpl_fsubr_r80_by_r32, iemAImpl_fsubr_r80_by_r80, 0) +EMIT_R80_BY_I32(iemAImpl_fisubr_r80_by_i32, iemAImpl_fsubr_r80_by_r80) +EMIT_R80_BY_I16(iemAImpl_fisubr_r80_by_i16, iemAImpl_fsubr_r80_by_r80) + + +/********************************************************************************************************************************* +* x87 FPU Trigometric Operations * +*********************************************************************************************************************************/ +static uint16_t iemAImpl_fpatan_r80_by_r80_normal(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PIEMFPURESULT pFpuRes, uint16_t fFcw, uint16_t fFsw) +{ + softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS(); + extFloat80_t y = iemFpuSoftF80FromIprt(pr80Val1); + extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val2); + extFloat80_t v; + (void)fFcw; + + v = extF80_atan2(y, x, &SoftState); + + iemFpuSoftF80ToIprt(&pFpuRes->r80Result, v); + return fFsw; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fpatan_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3); + + if (RTFLOAT80U_IS_NORMAL(pr80Val1) && RTFLOAT80U_IS_NORMAL(pr80Val2)) + { + fFsw = iemAImpl_fpatan_r80_by_r80_normal(pr80Val1, pr80Val2, pFpuRes, fFcw, fFsw); + + fFsw |= X86_FSW_PE | (7 << X86_FSW_TOP_SHIFT); + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + else + { + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + { + pFpuRes->r80Result = *pr80Val2; + fFsw |= X86_FSW_ES | X86_FSW_B | (6 << X86_FSW_TOP_SHIFT); + } + else + { + pFpuRes->r80Result = g_r80Indefinite; + fFsw |= (7 << X86_FSW_TOP_SHIFT); + } + } + + pFpuRes->FSW = fFsw; +} +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_fpatan_r80_by_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + iemAImpl_fpatan_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fpatan_r80_by_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + iemAImpl_fpatan_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2); +} + + +#if defined(IEM_WITHOUT_ASSEMBLY) +static uint16_t iemAImpl_fptan_r80_r80_normal(PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val, uint16_t fFcw, uint16_t fFsw) +{ + softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS(); + extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val); + extFloat80_t v; + (void)fFcw; + + v = extF80_tan(x, &SoftState); + + iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result1, v); + return fFsw; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fptan_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | /*X86_FSW_C2 |*/ X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + + if (RTFLOAT80U_IS_ZERO(pr80Val)) + { + pFpuResTwo->r80Result1 = *pr80Val; + pFpuResTwo->r80Result2 = g_ar80One[0]; + } + else if (RTFLOAT80U_IS_NORMAL(pr80Val)) + { + if (pr80Val->s.uExponent >= RTFLOAT80U_EXP_BIAS + 63) + { + fFsw |= X86_FSW_C2 | (7 << X86_FSW_TOP_SHIFT); + pFpuResTwo->r80Result1 = *pr80Val; + } + else + { + if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 63) + { + pFpuResTwo->r80Result1 = *pr80Val; + } + else + { + fFsw = iemAImpl_fptan_r80_r80_normal(pFpuResTwo, pr80Val, fFcw, fFsw); + } + + pFpuResTwo->r80Result2 = g_ar80One[0]; + + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else + { + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); + } + + pFpuResTwo->FSW = fFsw; +} +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_fptan_r80_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_fptan_r80_r80(pFpuState, pFpuResTwo, pr80Val); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fptan_r80_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_fptan_r80_r80(pFpuState, pFpuResTwo, pr80Val); +} + +#ifdef IEM_WITHOUT_ASSEMBLY + +static uint16_t iemAImpl_fsin_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw) +{ + softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS(); + extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val); + extFloat80_t v; + (void)fFcw; + + v = extF80_sin(x, &SoftState); + + iemFpuSoftF80ToIprt(pr80Result, v); + + return fFsw; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fsin_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | /*X86_FSW_C2 |*/ X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT); + + if (RTFLOAT80U_IS_ZERO(pr80Val)) + { + pFpuRes->r80Result = *pr80Val; + } + else if (RTFLOAT80U_IS_NORMAL(pr80Val)) + { + if (pr80Val->s.uExponent >= RTFLOAT80U_EXP_BIAS + 63) + { + fFsw |= X86_FSW_C2; + pFpuRes->r80Result = *pr80Val; + } + else + { + if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 63) + { + pFpuRes->r80Result = *pr80Val; + } + else + { + fFsw = iemAImpl_fsin_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw); + } + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else if (RTFLOAT80U_IS_INF(pr80Val)) + { + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + { + fFsw |= X86_FSW_ES | X86_FSW_B; + pFpuRes->r80Result = *pr80Val; + } + else + { + pFpuRes->r80Result = g_r80Indefinite; + } + } + else if (RTFLOAT80U_IS_DENORMAL(pr80Val)) + { + fFsw |= X86_FSW_DE; + + if (fFcw & X86_FCW_DM) + { + if (fFcw & X86_FCW_UM) + { + pFpuRes->r80Result = *pr80Val; + } + else + { + /* Underflow signalling as described at 7.4 section of 1985 IEEE 754*/ + uint64_t uMantissa = pr80Val->s.uMantissa; + uint32_t uExponent = ASMBitLastSetU64(uMantissa); + + uExponent = 64 - uExponent; + uMantissa <<= uExponent; + uExponent = RTFLOAT128U_EXP_BIAS_ADJUST - uExponent + 1; + + pFpuRes->r80Result.s.fSign = pr80Val->s.fSign; + pFpuRes->r80Result.s.uMantissa = uMantissa; + pFpuRes->r80Result.s.uExponent = uExponent; + } + + fFsw |= X86_FSW_UE | X86_FSW_PE; + + if ((fFcw & X86_FCW_UM) && (fFcw & X86_FCW_PM)) + { + /* All the exceptions are masked. */ + } + else + { + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else + { + pFpuRes->r80Result = *pr80Val; + + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val)) + { + pFpuRes->r80Result = *pr80Val; + fFsw |= X86_FSW_DE; + + if (fFcw & X86_FCW_DM) + { + if (fFcw & X86_FCW_PM) + { + fFsw |= X86_FSW_PE; + } + else + { + fFsw |= X86_FSW_ES | X86_FSW_B | X86_FSW_PE; + } + + pFpuRes->r80Result.sj64.uExponent = 1; + } + else + { + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } else if ( RTFLOAT80U_IS_QUIET_NAN(pr80Val) + || RTFLOAT80U_IS_INDEFINITE(pr80Val)) + { + pFpuRes->r80Result = *pr80Val; + } else { + if ( ( RTFLOAT80U_IS_UNNORMAL(pr80Val) + || RTFLOAT80U_IS_PSEUDO_NAN(pr80Val)) + && (fFcw & X86_FCW_IM)) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val; + if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val) && (fFcw & X86_FCW_IM)) + pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */ + } + + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + pFpuRes->FSW = fFsw; +} +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_fsin_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_fsin_r80(pFpuState, pFpuRes, pr80Val); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fsin_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_fsin_r80(pFpuState, pFpuRes, pr80Val); +} + +#ifdef IEM_WITHOUT_ASSEMBLY + +static uint16_t iemAImpl_fcos_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw) +{ + softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS(); + extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val); + extFloat80_t v; + (void)fFcw; + + v = extF80_cos(x, &SoftState); + + iemFpuSoftF80ToIprt(pr80Result, v); + + return fFsw; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fcos_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | /*X86_FSW_C2 |*/ X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT); + + if (RTFLOAT80U_IS_ZERO(pr80Val)) + { + pFpuRes->r80Result = g_ar80One[0]; + } + else if (RTFLOAT80U_IS_NORMAL(pr80Val)) + { + if (pr80Val->s.uExponent >= RTFLOAT80U_EXP_BIAS + 63) + { + fFsw |= X86_FSW_C2; + pFpuRes->r80Result = *pr80Val; + } + else + { + if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 63) + { + pFpuRes->r80Result = g_ar80One[0]; + + } + else + { + fFsw = iemAImpl_fcos_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw); + fFsw |= X86_FSW_C1; // TBD: If the inexact result was rounded up (C1 is set) or “not rounded up” (C1 is cleared). + } + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else if (RTFLOAT80U_IS_INF(pr80Val)) + { + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + { + fFsw |= X86_FSW_ES | X86_FSW_B; + pFpuRes->r80Result = *pr80Val; + } + else + { + pFpuRes->r80Result = g_r80Indefinite; + } + } + else if (RTFLOAT80U_IS_DENORMAL(pr80Val) || RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val)) + { + fFsw |= X86_FSW_DE; + + if (fFcw & X86_FCW_DM) + { + pFpuRes->r80Result = g_ar80One[0]; + + if (fFcw & X86_FCW_PM) + { + fFsw |= X86_FSW_PE; + } + else + { + fFsw |= X86_FSW_PE | X86_FSW_ES | X86_FSW_B; + } + } + else + { + pFpuRes->r80Result = *pr80Val; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } else if ( RTFLOAT80U_IS_QUIET_NAN(pr80Val) + || RTFLOAT80U_IS_INDEFINITE(pr80Val)) + { + pFpuRes->r80Result = *pr80Val; + } else { + if ( ( RTFLOAT80U_IS_UNNORMAL(pr80Val) + || RTFLOAT80U_IS_PSEUDO_NAN(pr80Val)) + && (fFcw & X86_FCW_IM)) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val; + if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val) && (fFcw & X86_FCW_IM)) + pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */ + } + + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + pFpuRes->FSW = fFsw; +} +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_fcos_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_fcos_r80(pFpuState, pFpuRes, pr80Val); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fcos_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_fcos_r80(pFpuState, pFpuRes, pr80Val); +} + +#ifdef IEM_WITHOUT_ASSEMBLY + +static uint16_t iemAImpl_fsincos_r80_r80_normal(PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val, uint16_t fFcw, uint16_t fFsw) +{ + softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS(); + extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val); + extFloat80_t r80Sin, r80Cos; + (void)fFcw; + + extF80_sincos(x, &r80Sin, &r80Cos, &SoftState); + + iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result1, r80Sin); + iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result2, r80Cos); + + return fFsw; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fsincos_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | /*X86_FSW_C2 |*/ X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT); + + if (RTFLOAT80U_IS_ZERO(pr80Val)) + { + pFpuResTwo->r80Result1 = *pr80Val; + pFpuResTwo->r80Result2 = g_ar80One[0]; + fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT); + } + else if (RTFLOAT80U_IS_NORMAL(pr80Val)) + { + if (pr80Val->s.uExponent >= RTFLOAT80U_EXP_BIAS + 63) + { + fFsw |= X86_FSW_C2; + + if (fFcw & X86_FCW_IM) + { + pFpuResTwo->r80Result1 = g_r80Indefinite; + } + else + { + pFpuResTwo->r80Result1 = g_ar80Zero[0]; + } + + pFpuResTwo->r80Result2 = *pr80Val; + } + else + { + fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT); + + if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 63) + { + pFpuResTwo->r80Result1 = *pr80Val; + pFpuResTwo->r80Result2 = g_ar80One[0]; + } + else + { + fFsw = iemAImpl_fsincos_r80_r80_normal(pFpuResTwo, pr80Val, fFcw, fFsw); + fFsw |= X86_FSW_C1; // TBD: If the inexact result was rounded up (C1 is set) or “not rounded up” (C1 is cleared). + } + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val)) + { + fFsw |= X86_FSW_DE; + + if (fFcw & X86_FCW_DM) + { + pFpuResTwo->r80Result1 = *pr80Val; + pFpuResTwo->r80Result2 = g_ar80One[0]; + fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT); + + if (fFcw & X86_FCW_PM) + { + fFsw |= X86_FSW_PE; + } + else + { + fFsw |= X86_FSW_PE | X86_FSW_ES | X86_FSW_B; + } + + pFpuResTwo->r80Result1.sj64.uExponent = 1; + } + else + { + pFpuResTwo->r80Result1 = g_ar80Zero[0]; + pFpuResTwo->r80Result2 = *pr80Val; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else if (RTFLOAT80U_IS_DENORMAL(pr80Val)) + { + fFsw |= X86_FSW_DE; + + if (fFcw & X86_FCW_DM) + { + pFpuResTwo->r80Result2 = g_ar80One[0]; + + if (fFcw & X86_FCW_UM) + { + pFpuResTwo->r80Result1 = *pr80Val; + } + else + { + /* Underflow signalling as described at 7.4 section of 1985 IEEE 754*/ + uint64_t uMantissa = pr80Val->s.uMantissa; + uint32_t uExponent = ASMBitLastSetU64(uMantissa); + + uExponent = 64 - uExponent; + uMantissa <<= uExponent; + uExponent = RTFLOAT128U_EXP_BIAS_ADJUST - uExponent + 1; + + pFpuResTwo->r80Result1.s.fSign = pr80Val->s.fSign; + pFpuResTwo->r80Result1.s.uMantissa = uMantissa; + pFpuResTwo->r80Result1.s.uExponent = uExponent; + } + + fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT); + fFsw |= X86_FSW_UE | X86_FSW_PE; + + if ((fFcw & X86_FCW_UM) && (fFcw & X86_FCW_PM)) + { + /* All the exceptions are masked. */ + } + else + { + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else + { + pFpuResTwo->r80Result1 = g_ar80Zero[0]; + pFpuResTwo->r80Result2 = *pr80Val; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else if (RTFLOAT80U_IS_QUIET_NAN(pr80Val) || RTFLOAT80U_IS_INDEFINITE(pr80Val)) + { + pFpuResTwo->r80Result1 = *pr80Val; + pFpuResTwo->r80Result2 = *pr80Val; + fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT); + } + else if (RTFLOAT80U_IS_UNNORMAL(pr80Val) || RTFLOAT80U_IS_PSEUDO_NAN(pr80Val)) + { + if (fFcw & X86_FCW_IM) + { + pFpuResTwo->r80Result1 = g_r80Indefinite; + pFpuResTwo->r80Result2 = g_r80Indefinite; + fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT); + } + else + { + pFpuResTwo->r80Result1 = g_ar80Zero[0]; + pFpuResTwo->r80Result2 = *pr80Val; + } + + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + else if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val)) + { + pFpuResTwo->r80Result1 = *pr80Val; + pFpuResTwo->r80Result2 = *pr80Val; + + if (fFcw & X86_FCW_IM) + { + pFpuResTwo->r80Result1.s.uMantissa |= RT_BIT_64(62); /* make it quiet */ + pFpuResTwo->r80Result2.s.uMantissa |= RT_BIT_64(62); + fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT); + } + else + { + pFpuResTwo->r80Result1 = g_ar80Zero[0]; + pFpuResTwo->r80Result2 = *pr80Val; + } + + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + else if (RTFLOAT80U_IS_INF(pr80Val)) + { + if (fFcw & X86_FCW_IM) + { + pFpuResTwo->r80Result1 = g_r80Indefinite; + pFpuResTwo->r80Result2 = g_r80Indefinite; + fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT); + } + else + { + pFpuResTwo->r80Result1 = g_ar80Zero[0]; + pFpuResTwo->r80Result2 = *pr80Val; + } + + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + pFpuResTwo->FSW = fFsw; +} +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_fsincos_r80_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_fsincos_r80_r80(pFpuState, pFpuResTwo, pr80Val); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fsincos_r80_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_fsincos_r80_r80(pFpuState, pFpuResTwo, pr80Val); +} + +#ifdef IEM_WITHOUT_ASSEMBLY + + +/********************************************************************************************************************************* +* x87 FPU Compare and Testing Operations * +*********************************************************************************************************************************/ + +IEM_DECL_IMPL_DEF(void, iemAImpl_ftst_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val)) +{ + uint16_t fFsw = (7 << X86_FSW_TOP_SHIFT); + + if (RTFLOAT80U_IS_ZERO(pr80Val)) + fFsw |= X86_FSW_C3; + else if (RTFLOAT80U_IS_NORMAL(pr80Val) || RTFLOAT80U_IS_INF(pr80Val)) + fFsw |= pr80Val->s.fSign ? X86_FSW_C0 : 0; + else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val)) + { + fFsw |= pr80Val->s.fSign ? X86_FSW_C0 | X86_FSW_DE : X86_FSW_DE; + if (!(pFpuState->FCW & X86_FCW_DM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + else + { + fFsw |= X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3 | X86_FSW_IE; + if (!(pFpuState->FCW & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + *pu16Fsw = fFsw; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fxam_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val)) +{ + RT_NOREF(pFpuState); + uint16_t fFsw = (7 << X86_FSW_TOP_SHIFT); + + /* C1 = sign bit (always, even if empty Intel says). */ + if (pr80Val->s.fSign) + fFsw |= X86_FSW_C1; + + /* Classify the value in C0, C2, C3. */ + if (!(pFpuState->FTW & RT_BIT_32(X86_FSW_TOP_GET(pFpuState->FSW)))) + fFsw |= X86_FSW_C0 | X86_FSW_C3; /* empty */ + else if (RTFLOAT80U_IS_NORMAL(pr80Val)) + fFsw |= X86_FSW_C2; + else if (RTFLOAT80U_IS_ZERO(pr80Val)) + fFsw |= X86_FSW_C3; + else if (RTFLOAT80U_IS_QUIET_OR_SIGNALLING_NAN(pr80Val)) + fFsw |= X86_FSW_C0; + else if (RTFLOAT80U_IS_INF(pr80Val)) + fFsw |= X86_FSW_C0 | X86_FSW_C2; + else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val)) + fFsw |= X86_FSW_C2 | X86_FSW_C3; + /* whatever else: 0 */ + + *pu16Fsw = fFsw; +} + + +/** + * Worker for fcom, fucom, and friends. + */ +static uint16_t iemAImpl_fcom_r80_by_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, + uint16_t fFcw, uint16_t fFsw, bool fIeOnAllNaNs) +{ + /* + * Unpack the values. + */ + bool const fSign1 = pr80Val1->s.fSign; + int32_t iExponent1 = pr80Val1->s.uExponent; + uint64_t uMantissa1 = pr80Val1->s.uMantissa; + + bool const fSign2 = pr80Val2->s.fSign; + int32_t iExponent2 = pr80Val2->s.uExponent; + uint64_t uMantissa2 = pr80Val2->s.uMantissa; + + /* + * Check for invalid inputs. + */ + if ( RTFLOAT80U_IS_387_INVALID_EX(uMantissa1, iExponent1) + || RTFLOAT80U_IS_387_INVALID_EX(uMantissa2, iExponent2)) + { + if (!(fFcw & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + return fFsw | X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3 | X86_FSW_IE; + } + + /* + * Check for NaNs and indefinites, they are all unordered and trumps #DE. + */ + if ( RTFLOAT80U_IS_INDEFINITE_OR_QUIET_OR_SIGNALLING_NAN_EX(uMantissa1, iExponent1) + || RTFLOAT80U_IS_INDEFINITE_OR_QUIET_OR_SIGNALLING_NAN_EX(uMantissa2, iExponent2)) + { + if ( fIeOnAllNaNs + || RTFLOAT80U_IS_SIGNALLING_NAN_EX(uMantissa1, iExponent1) + || RTFLOAT80U_IS_SIGNALLING_NAN_EX(uMantissa2, iExponent2)) + { + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + return fFsw | X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3; + } + + /* + * Normalize the values. + */ + if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL_EX(uMantissa1, iExponent1)) + { + if (RTFLOAT80U_IS_PSEUDO_DENORMAL_EX(uMantissa1, iExponent1)) + iExponent1 = 1; + else + { + iExponent1 = 64 - ASMBitLastSetU64(uMantissa1); + uMantissa1 <<= iExponent1; + iExponent1 = 1 - iExponent1; + } + fFsw |= X86_FSW_DE; + if (!(fFcw & X86_FCW_DM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL_EX(uMantissa2, iExponent2)) + { + if (RTFLOAT80U_IS_PSEUDO_DENORMAL_EX(uMantissa2, iExponent2)) + iExponent2 = 1; + else + { + iExponent2 = 64 - ASMBitLastSetU64(uMantissa2); + uMantissa2 <<= iExponent2; + iExponent2 = 1 - iExponent2; + } + fFsw |= X86_FSW_DE; + if (!(fFcw & X86_FCW_DM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + + /* + * Test if equal (val1 == val2): + */ + if ( uMantissa1 == uMantissa2 + && iExponent1 == iExponent2 + && ( fSign1 == fSign2 + || (uMantissa1 == 0 && iExponent1 == 0) /* ignore sign for zero */ ) ) + fFsw |= X86_FSW_C3; + /* + * Test if less than (val1 < val2): + */ + else if (fSign1 && !fSign2) + fFsw |= X86_FSW_C0; + else if (fSign1 == fSign2) + { + /* Zeros are problematic, however at the most one can be zero here. */ + if (RTFLOAT80U_IS_ZERO_EX(uMantissa1, iExponent1)) + return !fSign1 ? fFsw | X86_FSW_C0 : fFsw; + if (RTFLOAT80U_IS_ZERO_EX(uMantissa2, iExponent2)) + return fSign1 ? fFsw | X86_FSW_C0 : fFsw; + + if ( fSign1 + ^ ( iExponent1 < iExponent2 + || ( iExponent1 == iExponent2 + && uMantissa1 < uMantissa2 ) ) ) + fFsw |= X86_FSW_C0; + } + /* else: No flags set if greater. */ + + return fFsw; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pfFsw, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + *pfFsw = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, pr80Val2, pFpuState->FCW, 6 << X86_FSW_TOP_SHIFT, true /*fIeOnAllNaNs*/); +} + + + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fucom_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pfFsw, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + *pfFsw = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, pr80Val2, pFpuState->FCW, 6 << X86_FSW_TOP_SHIFT, false /*fIeOnAllNaNs*/); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r64,(PCX86FXSTATE pFpuState, uint16_t *pfFsw, + PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2)) +{ + RTFLOAT80U r80Val2; + uint16_t fFsw = iemAImplConvertR64ToR80(pr64Val2, &r80Val2); + Assert(!fFsw || fFsw == X86_FSW_DE); + *pfFsw = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, &r80Val2, pFpuState->FCW, 7 << X86_FSW_TOP_SHIFT, true /*fIeOnAllNaNs*/); + if (fFsw != 0 && !(*pfFsw & X86_FSW_IE)) + { + if (!(pFpuState->FCW & X86_FCW_DM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + *pfFsw |= fFsw; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r32,(PCX86FXSTATE pFpuState, uint16_t *pfFsw, + PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2)) +{ + RTFLOAT80U r80Val2; + uint16_t fFsw = iemAImplConvertR32ToR80(pr32Val2, &r80Val2); + Assert(!fFsw || fFsw == X86_FSW_DE); + *pfFsw = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, &r80Val2, pFpuState->FCW, 7 << X86_FSW_TOP_SHIFT, true /*fIeOnAllNaNs*/); + if (fFsw != 0 && !(*pfFsw & X86_FSW_IE)) + { + if (!(pFpuState->FCW & X86_FCW_DM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + *pfFsw |= fFsw; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i32,(PCX86FXSTATE pFpuState, uint16_t *pfFsw, + PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2)) +{ + RTFLOAT80U r80Val2; + iemAImpl_fcom_r80_by_r80(pFpuState, pfFsw, pr80Val1, iemAImplConvertI32ToR80(*pi32Val2, &r80Val2)); + *pfFsw = (*pfFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i16,(PCX86FXSTATE pFpuState, uint16_t *pfFsw, + PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2)) +{ + RTFLOAT80U r80Val2; + iemAImpl_fcom_r80_by_r80(pFpuState, pfFsw, pr80Val1, iemAImplConvertI16ToR80(*pi16Val2, &r80Val2)); + *pfFsw = (*pfFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT); +} + + +/** + * Worker for fcomi & fucomi. + */ +static uint32_t iemAImpl_fcomi_r80_by_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, + uint16_t fFcw, uint16_t fFswIn, bool fIeOnAllNaNs, uint16_t *pfFsw) +{ + uint16_t fFsw = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, pr80Val2, fFcw, 6 << X86_FSW_TOP_SHIFT, fIeOnAllNaNs); + uint32_t fEflags = ((fFsw & X86_FSW_C3) >> (X86_FSW_C3_BIT - X86_EFL_ZF_BIT)) + | ((fFsw & X86_FSW_C2) >> (X86_FSW_C2_BIT - X86_EFL_PF_BIT)) + | ((fFsw & X86_FSW_C0) >> (X86_FSW_C0_BIT - X86_EFL_CF_BIT)); + + /* Note! C1 is not cleared as per docs! Everything is preserved. */ + *pfFsw = (fFsw & ~X86_FSW_C_MASK) | (fFswIn & X86_FSW_C_MASK); + return fEflags | X86_EFL_IF | X86_EFL_RA1_MASK; +} + + +IEM_DECL_IMPL_DEF(uint32_t, iemAImpl_fcomi_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pfFsw, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + return iemAImpl_fcomi_r80_by_r80_worker(pr80Val1, pr80Val2, pFpuState->FCW, pFpuState->FSW, true /*fIeOnAllNaNs*/, pfFsw); +} + + +IEM_DECL_IMPL_DEF(uint32_t, iemAImpl_fucomi_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pfFsw, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + return iemAImpl_fcomi_r80_by_r80_worker(pr80Val1, pr80Val2, pFpuState->FCW, pFpuState->FSW, false /*fIeOnAllNaNs*/, pfFsw); +} + + +/********************************************************************************************************************************* +* x87 FPU Other Operations * +*********************************************************************************************************************************/ + +/** + * Helper for iemAImpl_frndint_r80, called both on normal and denormal numbers. + */ +static uint16_t iemAImpl_frndint_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw); + iemFpuSoftF80ToIprt(pr80Result, extF80_roundToInt(iemFpuSoftF80FromIprt(pr80Val), SoftState.roundingMode, + true /*exact / generate #PE */, &SoftState)); + return IEM_SOFTFLOAT_STATE_TO_FSW(fFsw, &SoftState, fFcw); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_frndint_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT); + + if (RTFLOAT80U_IS_NORMAL(pr80Val)) + fFsw = iemAImpl_frndint_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw); + else if ( RTFLOAT80U_IS_ZERO(pr80Val) + || RTFLOAT80U_IS_QUIET_NAN(pr80Val) + || RTFLOAT80U_IS_INDEFINITE(pr80Val) + || RTFLOAT80U_IS_INF(pr80Val)) + pFpuRes->r80Result = *pr80Val; + else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val)) + { + fFsw |= X86_FSW_DE; + if (fFcw & X86_FCW_DM) + fFsw = iemAImpl_frndint_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw); + else + { + pFpuRes->r80Result = *pr80Val; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else + { + if (fFcw & X86_FCW_IM) + { + if (!RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val)) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val; + pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */ + } + } + else + { + pFpuRes->r80Result = *pr80Val; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + pFpuRes->FSW = fFsw; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fscale_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + /* The SoftFloat worker function extF80_scale_extF80 is of our creation, so + it does everything we need it to do. */ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw); + extFloat80_t r80XResult = extF80_scale_extF80(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState); + pFpuRes->FSW = iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1); +} + + +/** + * Helper for iemAImpl_fsqrt_r80, called both on normal and denormal numbers. + */ +static uint16_t iemAImpl_fsqrt_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw) +{ + Assert(!pr80Val->s.fSign); + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw); + iemFpuSoftF80ToIprt(pr80Result, extF80_sqrt(iemFpuSoftF80FromIprt(pr80Val), &SoftState)); + return IEM_SOFTFLOAT_STATE_TO_FSW(fFsw, &SoftState, fFcw); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fsqrt_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT); + + if (RTFLOAT80U_IS_NORMAL(pr80Val) && !pr80Val->s.fSign) + fFsw = iemAImpl_fsqrt_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw); + else if ( RTFLOAT80U_IS_ZERO(pr80Val) + || RTFLOAT80U_IS_QUIET_NAN(pr80Val) + || RTFLOAT80U_IS_INDEFINITE(pr80Val) + || (RTFLOAT80U_IS_INF(pr80Val) && !pr80Val->s.fSign)) + pFpuRes->r80Result = *pr80Val; + else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val) && !pr80Val->s.fSign) /* Negative denormals only generate #IE! */ + { + fFsw |= X86_FSW_DE; + if (fFcw & X86_FCW_DM) + fFsw = iemAImpl_fsqrt_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw); + else + { + pFpuRes->r80Result = *pr80Val; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else + { + if (fFcw & X86_FCW_IM) + { + if (!RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val)) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val; + pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */ + } + } + else + { + pFpuRes->r80Result = *pr80Val; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + fFsw |= X86_FSW_IE; + } + pFpuRes->FSW = fFsw; +} + + +/** + * @code{.unparsed} + * x x * ln2 + * f(x) = 2 - 1 = e - 1 + * + * @endcode + * + * We can approximate e^x by a Taylor/Maclaurin series (see + * https://en.wikipedia.org/wiki/Taylor_series#Exponential_function): + * @code{.unparsed} + * n 0 1 2 3 4 + * inf x x x x x x + * SUM ----- = --- + --- + --- + --- + --- + ... + * n=0 n! 0! 1! 2! 3! 4! + * + * 2 3 4 + * x x x + * = 1 + x + --- + --- + --- + ... + * 2! 3! 4! + * @endcode + * + * Given z = x * ln2, we get: + * @code{.unparsed} + * 2 3 4 n + * z z z z z + * e - 1 = z + --- + --- + --- + ... + --- + * 2! 3! 4! n! + * @endcode + * + * Wanting to use Horner's method, we move one z outside and get: + * @code{.unparsed} + * 2 3 (n-1) + * z z z z + * = z ( 1 + --- + --- + --- + ... + ------- ) + * 2! 3! 4! n! + * @endcode + * + * The constants we need for using Horner's methods are 1 and 1 / n!. + * + * For very tiny x values, we can get away with f(x) = x * ln 2, because + * because we don't have the necessary precision to represent 1.0 + z/3 + ... + * and can approximate it to be 1.0. For a visual demonstration of this + * check out https://www.desmos.com/calculator/vidcdxizd9 (for as long + * as it valid), plotting f(x) = 2^x - 1 and f(x) = x * ln2. + * + * + * As constant accuracy goes, figure 0.1 "80387 Block Diagram" in the "80387 + * Data Sheet" (order 231920-002; Appendix E in 80387 PRM 231917-001; Military + * i387SX 271166-002), indicates that constants are 67-bit (constant rom block) + * and the internal mantissa size is 68-bit (mantissa adder & barrel shifter + * blocks). (The one bit difference is probably an implicit one missing from + * the constant ROM.) A paper on division and sqrt on the AMD-K7 by Stuart F. + * Oberman states that it internally used a 68 bit mantissa with a 18-bit + * exponent. + * + * However, even when sticking to 67 constants / 68 mantissas, I have not yet + * successfully reproduced the exact results from an Intel 10980XE, there is + * always a portition of rounding differences. Not going to spend too much time + * on getting this 100% the same, at least not now. + * + * P.S. If someone are really curious about 8087 and its contstants: + * http://www.righto.com/2020/05/extracting-rom-constants-from-8087-math.html + * + * + * @param pr80Val The exponent value (x), less than 1.0, greater than + * -1.0 and not zero. This can be a normal, denormal + * or pseudo-denormal value. + * @param pr80Result Where to return the result. + * @param fFcw FPU control word. + * @param fFsw FPU status word. + */ +static uint16_t iemAImpl_f2xm1_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw) +{ + /* As mentioned above, we can skip the expensive polynomial calculation + as it will be close enough to 1.0 that it makes no difference. + + The cutoff point for intel 10980XE is exponents >= -69. Intel + also seems to be using a 67-bit or 68-bit constant value, and we get + a smattering of rounding differences if we go for higher precision. */ + if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 69) + { + RTUINT256U u256; + RTUInt128MulByU64Ex(&u256, &g_u128Ln2MantissaIntel, pr80Val->s.uMantissa); + u256.QWords.qw0 |= 1; /* force #PE */ + fFsw = iemFpuFloat80RoundAndComposeFrom192(pr80Result, pr80Val->s.fSign, &u256, + !RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val) && !RTFLOAT80U_IS_DENORMAL(pr80Val) + ? (int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS + : 1 - RTFLOAT80U_EXP_BIAS, + fFcw, fFsw); + } + else + { +#ifdef IEM_WITH_FLOAT128_FOR_FPU + /* This approach is not good enough for small values, we end up with zero. */ + int const fOldRounding = iemFpuF128SetRounding(fFcw); + _Float128 rd128Val = iemFpuF128FromFloat80(pr80Val, fFcw); + _Float128 rd128Result = powf128(2.0L, rd128Val); + rd128Result -= 1.0L; + fFsw = iemFpuF128ToFloat80(pr80Result, rd128Result, fFcw, fFsw); + iemFpuF128RestoreRounding(fOldRounding); + +# else + softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS(); + float128_t const x = iemFpuSoftF128FromFloat80(pr80Val); + + /* As mentioned above, enforce 68-bit internal mantissa width to better + match the Intel 10980XE results. */ + unsigned const cPrecision = 68; + + /* first calculate z = x * ln2 */ + float128_t z = iemFpuSoftF128Precision(f128_mul(x, iemFpuSoftF128PrecisionIprt(&g_r128Ln2, cPrecision), &SoftState), + cPrecision); + + /* Then do the polynomial evaluation. */ + float128_t r = iemFpuSoftF128HornerPoly(z, g_ar128F2xm1HornerConsts, RT_ELEMENTS(g_ar128F2xm1HornerConsts), + cPrecision, &SoftState); + r = f128_mul(z, r, &SoftState); + + /* Output the result. */ + fFsw = iemFpuSoftF128ToFloat80(pr80Result, r, fFcw, fFsw); +# endif + } + return fFsw; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_f2xm1_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT); + + if (RTFLOAT80U_IS_NORMAL(pr80Val)) + { + if (pr80Val->s.uExponent < RTFLOAT80U_EXP_BIAS) + fFsw = iemAImpl_f2xm1_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw); + else + { + /* Special case: + 2^+1.0 - 1.0 = 1.0 + 2^-1.0 - 1.0 = -0.5 */ + if ( pr80Val->s.uExponent == RTFLOAT80U_EXP_BIAS + && pr80Val->s.uMantissa == RT_BIT_64(63)) + { + pFpuRes->r80Result.s.uMantissa = RT_BIT_64(63); + pFpuRes->r80Result.s.uExponent = RTFLOAT80U_EXP_BIAS - pr80Val->s.fSign; + pFpuRes->r80Result.s.fSign = pr80Val->s.fSign; + } + /* ST(0) > 1.0 || ST(0) < -1.0: undefined behavior */ + /** @todo 287 is documented to only accept values 0 <= ST(0) <= 0.5. */ + else + pFpuRes->r80Result = *pr80Val; + fFsw |= X86_FSW_PE; + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else if ( RTFLOAT80U_IS_ZERO(pr80Val) + || RTFLOAT80U_IS_QUIET_NAN(pr80Val) + || RTFLOAT80U_IS_INDEFINITE(pr80Val)) + pFpuRes->r80Result = *pr80Val; + else if (RTFLOAT80U_IS_INF(pr80Val)) + pFpuRes->r80Result = pr80Val->s.fSign ? g_ar80One[1] : *pr80Val; + else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val)) + { + fFsw |= X86_FSW_DE; + if (fFcw & X86_FCW_DM) + fFsw = iemAImpl_f2xm1_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw); + else + { + pFpuRes->r80Result = *pr80Val; + fFsw |= X86_FSW_ES | X86_FSW_B; + } + } + else + { + if ( ( RTFLOAT80U_IS_UNNORMAL(pr80Val) + || RTFLOAT80U_IS_PSEUDO_NAN(pr80Val)) + && (fFcw & X86_FCW_IM)) + pFpuRes->r80Result = g_r80Indefinite; + else + { + pFpuRes->r80Result = *pr80Val; + if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val) && (fFcw & X86_FCW_IM)) + pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */ + } + fFsw |= X86_FSW_IE; + if (!(fFcw & X86_FCW_IM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + pFpuRes->FSW = fFsw; +} + +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_f2xm1_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_f2xm1_r80(pFpuState, pFpuRes, pr80Val); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_f2xm1_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + iemAImpl_f2xm1_r80(pFpuState, pFpuRes, pr80Val); +} + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_fabs_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + pFpuRes->FSW = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT); + pFpuRes->r80Result = *pr80Val; + pFpuRes->r80Result.s.fSign = 0; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fchs_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val)) +{ + pFpuRes->FSW = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT); + pFpuRes->r80Result = *pr80Val; + pFpuRes->r80Result.s.fSign = !pr80Val->s.fSign; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_fxtract_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT); + + if (RTFLOAT80U_IS_NORMAL(pr80Val)) + { + softfloat_state_t Ignored = SOFTFLOAT_STATE_INIT_DEFAULTS(); + iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result1, i32_to_extF80((int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS, &Ignored)); + + pFpuResTwo->r80Result2.s.fSign = pr80Val->s.fSign; + pFpuResTwo->r80Result2.s.uExponent = RTFLOAT80U_EXP_BIAS; + pFpuResTwo->r80Result2.s.uMantissa = pr80Val->s.uMantissa; + } + else if (RTFLOAT80U_IS_ZERO(pr80Val)) + { + fFsw |= X86_FSW_ZE; + if (fFcw & X86_FCW_ZM) + { + pFpuResTwo->r80Result1 = g_ar80Infinity[1]; + pFpuResTwo->r80Result2 = *pr80Val; + } + else + { + pFpuResTwo->r80Result2 = *pr80Val; + fFsw = X86_FSW_ES | X86_FSW_B | (fFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT); + } + } + else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val)) + { + fFsw |= X86_FSW_DE; + if (fFcw & X86_FCW_DM) + { + pFpuResTwo->r80Result2.s.fSign = pr80Val->s.fSign; + pFpuResTwo->r80Result2.s.uExponent = RTFLOAT80U_EXP_BIAS; + pFpuResTwo->r80Result2.s.uMantissa = pr80Val->s.uMantissa; + int32_t iExponent = -16382; + while (!(pFpuResTwo->r80Result2.s.uMantissa & RT_BIT_64(63))) + { + pFpuResTwo->r80Result2.s.uMantissa <<= 1; + iExponent--; + } + + softfloat_state_t Ignored = SOFTFLOAT_STATE_INIT_DEFAULTS(); + iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result1, i32_to_extF80(iExponent, &Ignored)); + } + else + { + pFpuResTwo->r80Result2 = *pr80Val; + fFsw = X86_FSW_ES | X86_FSW_B | (fFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT); + } + } + else if ( RTFLOAT80U_IS_QUIET_NAN(pr80Val) + || RTFLOAT80U_IS_INDEFINITE(pr80Val)) + { + pFpuResTwo->r80Result1 = *pr80Val; + pFpuResTwo->r80Result2 = *pr80Val; + } + else if (RTFLOAT80U_IS_INF(pr80Val)) + { + pFpuResTwo->r80Result1 = g_ar80Infinity[0]; + pFpuResTwo->r80Result2 = *pr80Val; + } + else + { + if (fFcw & X86_FCW_IM) + { + if (!RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val)) + pFpuResTwo->r80Result1 = g_r80Indefinite; + else + { + pFpuResTwo->r80Result1 = *pr80Val; + pFpuResTwo->r80Result1.s.uMantissa |= RT_BIT_64(62); /* make it quiet */ + } + pFpuResTwo->r80Result2 = pFpuResTwo->r80Result1; + } + else + { + pFpuResTwo->r80Result2 = *pr80Val; + fFsw = X86_FSW_ES | X86_FSW_B | (fFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT); + } + fFsw |= X86_FSW_IE; + } + pFpuResTwo->FSW = fFsw; +} +#endif /* IEM_WITHOUT_ASSEMBLY */ + +#if defined(IEM_WITHOUT_ASSEMBLY) + +static uint16_t iemAImpl_fyl2x_r80_by_r80_normal(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw) +{ + softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS(); + extFloat80_t y = iemFpuSoftF80FromIprt(pr80Val1); + extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val2); + extFloat80_t v; + (void)fFcw; + + v = extF80_ylog2x(y, x, &SoftState); + iemFpuSoftF80ToIprt(pr80Result, v); + + return fFsw; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2x_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3); + + if (RTFLOAT80U_IS_NORMAL(pr80Val1) && RTFLOAT80U_IS_NORMAL(pr80Val2) && !pr80Val2->s.fSign) + { + fFsw |= iemAImpl_fyl2x_r80_by_r80_normal(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw); + + fFsw |= X86_FSW_PE | (7 << X86_FSW_TOP_SHIFT); + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + else + { + fFsw |= X86_FSW_IE; + + if (!(fFcw & X86_FCW_IM)) + { + pFpuRes->r80Result = *pr80Val2; + fFsw |= X86_FSW_ES | X86_FSW_B | (6 << X86_FSW_TOP_SHIFT); + } + else + { + pFpuRes->r80Result = g_r80Indefinite; + fFsw |= (7 << X86_FSW_TOP_SHIFT); + } + } + + pFpuRes->FSW = fFsw; +} +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2x_r80_by_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + iemAImpl_fyl2x_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2x_r80_by_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + iemAImpl_fyl2x_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2); +} + +#if defined(IEM_WITHOUT_ASSEMBLY) + +static uint16_t iemAImpl_fyl2xp1_r80_by_r80_normal(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw) +{ + softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS(); + extFloat80_t y = iemFpuSoftF80FromIprt(pr80Val1); + extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val2); + extFloat80_t v; + (void)fFcw; + + v = extF80_ylog2xp1(y, x, &SoftState); + iemFpuSoftF80ToIprt(pr80Result, v); + + return fFsw; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2xp1_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + uint16_t const fFcw = pFpuState->FCW; + uint16_t fFsw = pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3); + + if (RTFLOAT80U_IS_NORMAL(pr80Val1) && RTFLOAT80U_IS_NORMAL(pr80Val2) && pr80Val2->s.uExponent < RTFLOAT80U_EXP_BIAS) + { + fFsw = iemAImpl_fyl2xp1_r80_by_r80_normal(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw); + + fFsw |= X86_FSW_PE | (7 << X86_FSW_TOP_SHIFT); + if (!(fFcw & X86_FCW_PM)) + fFsw |= X86_FSW_ES | X86_FSW_B; + } + else + { + fFsw |= X86_FSW_IE; + + if (!(fFcw & X86_FCW_IM)) + { + pFpuRes->r80Result = *pr80Val2; + fFsw |= X86_FSW_ES | X86_FSW_B | (6 << X86_FSW_TOP_SHIFT); + } + else + { + pFpuRes->r80Result = g_r80Indefinite; + fFsw |= (7 << X86_FSW_TOP_SHIFT); + } + } + + pFpuRes->FSW = fFsw; +} + +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2xp1_r80_by_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + iemAImpl_fyl2xp1_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2xp1_r80_by_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, + PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2)) +{ + iemAImpl_fyl2xp1_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2); +} + + +/********************************************************************************************************************************* +* MMX, SSE & AVX * +*********************************************************************************************************************************/ + +#ifdef IEM_WITH_VEX + +/* + * VMOVSLDUP + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc)) +{ + pXState->x87.aXMM[iYRegDst].au32[0] = pXState->x87.aXMM[iYRegSrc].au32[0]; + pXState->x87.aXMM[iYRegDst].au32[1] = pXState->x87.aXMM[iYRegSrc].au32[0]; + pXState->x87.aXMM[iYRegDst].au32[2] = pXState->x87.aXMM[iYRegSrc].au32[2]; + pXState->x87.aXMM[iYRegDst].au32[3] = pXState->x87.aXMM[iYRegSrc].au32[2]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc)) +{ + pXState->x87.aXMM[iYRegDst].au32[0] = pSrc->au32[0]; + pXState->x87.aXMM[iYRegDst].au32[1] = pSrc->au32[0]; + pXState->x87.aXMM[iYRegDst].au32[2] = pSrc->au32[2]; + pXState->x87.aXMM[iYRegDst].au32[3] = pSrc->au32[2]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pSrc->au32[4]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pSrc->au32[4]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pSrc->au32[6]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pSrc->au32[6]; +} + +#endif /* IEM_WITH_VEX */ + + +#ifdef IEM_WITH_VEX + +/* + * VMOVSHDUP + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc)) +{ + pXState->x87.aXMM[iYRegDst].au32[0] = pXState->x87.aXMM[iYRegSrc].au32[1]; + pXState->x87.aXMM[iYRegDst].au32[1] = pXState->x87.aXMM[iYRegSrc].au32[1]; + pXState->x87.aXMM[iYRegDst].au32[2] = pXState->x87.aXMM[iYRegSrc].au32[3]; + pXState->x87.aXMM[iYRegDst].au32[3] = pXState->x87.aXMM[iYRegSrc].au32[3]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[1]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[1]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[3]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc)) +{ + pXState->x87.aXMM[iYRegDst].au32[0] = pSrc->au32[1]; + pXState->x87.aXMM[iYRegDst].au32[1] = pSrc->au32[1]; + pXState->x87.aXMM[iYRegDst].au32[2] = pSrc->au32[3]; + pXState->x87.aXMM[iYRegDst].au32[3] = pSrc->au32[3]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pSrc->au32[5]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pSrc->au32[5]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pSrc->au32[7]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pSrc->au32[7]; +} + +#endif /* IEM_WITH_VEX */ + + +#ifdef IEM_WITH_VEX + +/* + * VMOVDDUP + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc)) +{ + pXState->x87.aXMM[iYRegDst].au64[0] = pXState->x87.aXMM[iYRegSrc].au64[0]; + pXState->x87.aXMM[iYRegDst].au64[1] = pXState->x87.aXMM[iYRegSrc].au64[0]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc)) +{ + pXState->x87.aXMM[iYRegDst].au64[0] = pSrc->au64[0]; + pXState->x87.aXMM[iYRegDst].au64[1] = pSrc->au64[0]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pSrc->au64[2]; + pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pSrc->au64[2]; +} + +#endif /* IEM_WITH_VEX */ + + +/* + * PAND / VPAND / PANDPS / VPANDPS / PANDPD / VPANDPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pand_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + *puDst &= *puSrc; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pand_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + puDst->au64[0] &= puSrc->au64[0]; + puDst->au64[1] &= puSrc->au64[1]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpand_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] & puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] & puSrc2->au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpand_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] & puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] & puSrc2->au64[1]; + puDst->au64[2] = puSrc1->au64[2] & puSrc2->au64[2]; + puDst->au64[3] = puSrc1->au64[3] & puSrc2->au64[3]; +} + + +/* + * PANDN / VPANDN / PANDNPS / VPANDNPS / PANDNPD / VPANDNPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pandn_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + *puDst = ~*puDst & *puSrc; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pandn_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + puDst->au64[0] = ~puDst->au64[0] & puSrc->au64[0]; + puDst->au64[1] = ~puDst->au64[1] & puSrc->au64[1]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpandn_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = ~puSrc1->au64[0] & puSrc2->au64[0]; + puDst->au64[1] = ~puSrc1->au64[1] & puSrc2->au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpandn_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = ~puSrc1->au64[0] & puSrc2->au64[0]; + puDst->au64[1] = ~puSrc1->au64[1] & puSrc2->au64[1]; + puDst->au64[2] = ~puSrc1->au64[2] & puSrc2->au64[2]; + puDst->au64[3] = ~puSrc1->au64[3] & puSrc2->au64[3]; +} + + +/* + * POR / VPOR / PORPS / VPORPS / PORPD / VPORPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_por_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + *puDst |= *puSrc; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_por_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + puDst->au64[0] |= puSrc->au64[0]; + puDst->au64[1] |= puSrc->au64[1]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpor_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] | puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] | puSrc2->au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpor_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] | puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] | puSrc2->au64[1]; + puDst->au64[2] = puSrc1->au64[2] | puSrc2->au64[2]; + puDst->au64[3] = puSrc1->au64[3] | puSrc2->au64[3]; +} + + +/* + * PXOR / VPXOR / PXORPS / VPXORPS / PXORPD / VPXORPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pxor_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + *puDst ^= *puSrc; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pxor_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + puDst->au64[0] ^= puSrc->au64[0]; + puDst->au64[1] ^= puSrc->au64[1]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpxor_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] ^ puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] ^ puSrc2->au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpxor_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] ^ puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] ^ puSrc2->au64[1]; + puDst->au64[2] = puSrc1->au64[2] ^ puSrc2->au64[2]; + puDst->au64[3] = puSrc1->au64[3] ^ puSrc2->au64[3]; +} + + +/* + * PCMPEQB / VPCMPEQB + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au8[0] = uSrc1.au8[0] == uSrc2.au8[0] ? 0xff : 0; + uDst.au8[1] = uSrc1.au8[1] == uSrc2.au8[1] ? 0xff : 0; + uDst.au8[2] = uSrc1.au8[2] == uSrc2.au8[2] ? 0xff : 0; + uDst.au8[3] = uSrc1.au8[3] == uSrc2.au8[3] ? 0xff : 0; + uDst.au8[4] = uSrc1.au8[4] == uSrc2.au8[4] ? 0xff : 0; + uDst.au8[5] = uSrc1.au8[5] == uSrc2.au8[5] ? 0xff : 0; + uDst.au8[6] = uSrc1.au8[6] == uSrc2.au8[6] ? 0xff : 0; + uDst.au8[7] = uSrc1.au8[7] == uSrc2.au8[7] ? 0xff : 0; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au8[0] = uSrc1.au8[0] == puSrc->au8[0] ? UINT8_MAX : 0; + puDst->au8[1] = uSrc1.au8[1] == puSrc->au8[1] ? UINT8_MAX : 0; + puDst->au8[2] = uSrc1.au8[2] == puSrc->au8[2] ? UINT8_MAX : 0; + puDst->au8[3] = uSrc1.au8[3] == puSrc->au8[3] ? UINT8_MAX : 0; + puDst->au8[4] = uSrc1.au8[4] == puSrc->au8[4] ? UINT8_MAX : 0; + puDst->au8[5] = uSrc1.au8[5] == puSrc->au8[5] ? UINT8_MAX : 0; + puDst->au8[6] = uSrc1.au8[6] == puSrc->au8[6] ? UINT8_MAX : 0; + puDst->au8[7] = uSrc1.au8[7] == puSrc->au8[7] ? UINT8_MAX : 0; + puDst->au8[8] = uSrc1.au8[8] == puSrc->au8[8] ? UINT8_MAX : 0; + puDst->au8[9] = uSrc1.au8[9] == puSrc->au8[9] ? UINT8_MAX : 0; + puDst->au8[10] = uSrc1.au8[10] == puSrc->au8[10] ? UINT8_MAX : 0; + puDst->au8[11] = uSrc1.au8[11] == puSrc->au8[11] ? UINT8_MAX : 0; + puDst->au8[12] = uSrc1.au8[12] == puSrc->au8[12] ? UINT8_MAX : 0; + puDst->au8[13] = uSrc1.au8[13] == puSrc->au8[13] ? UINT8_MAX : 0; + puDst->au8[14] = uSrc1.au8[14] == puSrc->au8[14] ? UINT8_MAX : 0; + puDst->au8[15] = uSrc1.au8[15] == puSrc->au8[15] ? UINT8_MAX : 0; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au8[0] = puSrc1->au8[0] == puSrc2->au8[0] ? UINT8_MAX : 0; + puDst->au8[1] = puSrc1->au8[1] == puSrc2->au8[1] ? UINT8_MAX : 0; + puDst->au8[2] = puSrc1->au8[2] == puSrc2->au8[2] ? UINT8_MAX : 0; + puDst->au8[3] = puSrc1->au8[3] == puSrc2->au8[3] ? UINT8_MAX : 0; + puDst->au8[4] = puSrc1->au8[4] == puSrc2->au8[4] ? UINT8_MAX : 0; + puDst->au8[5] = puSrc1->au8[5] == puSrc2->au8[5] ? UINT8_MAX : 0; + puDst->au8[6] = puSrc1->au8[6] == puSrc2->au8[6] ? UINT8_MAX : 0; + puDst->au8[7] = puSrc1->au8[7] == puSrc2->au8[7] ? UINT8_MAX : 0; + puDst->au8[8] = puSrc1->au8[8] == puSrc2->au8[8] ? UINT8_MAX : 0; + puDst->au8[9] = puSrc1->au8[9] == puSrc2->au8[9] ? UINT8_MAX : 0; + puDst->au8[10] = puSrc1->au8[10] == puSrc2->au8[10] ? UINT8_MAX : 0; + puDst->au8[11] = puSrc1->au8[11] == puSrc2->au8[11] ? UINT8_MAX : 0; + puDst->au8[12] = puSrc1->au8[12] == puSrc2->au8[12] ? UINT8_MAX : 0; + puDst->au8[13] = puSrc1->au8[13] == puSrc2->au8[13] ? UINT8_MAX : 0; + puDst->au8[14] = puSrc1->au8[14] == puSrc2->au8[14] ? UINT8_MAX : 0; + puDst->au8[15] = puSrc1->au8[15] == puSrc2->au8[15] ? UINT8_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au8[0] = puSrc1->au8[0] == puSrc2->au8[0] ? UINT8_MAX : 0; + puDst->au8[1] = puSrc1->au8[1] == puSrc2->au8[1] ? UINT8_MAX : 0; + puDst->au8[2] = puSrc1->au8[2] == puSrc2->au8[2] ? UINT8_MAX : 0; + puDst->au8[3] = puSrc1->au8[3] == puSrc2->au8[3] ? UINT8_MAX : 0; + puDst->au8[4] = puSrc1->au8[4] == puSrc2->au8[4] ? UINT8_MAX : 0; + puDst->au8[5] = puSrc1->au8[5] == puSrc2->au8[5] ? UINT8_MAX : 0; + puDst->au8[6] = puSrc1->au8[6] == puSrc2->au8[6] ? UINT8_MAX : 0; + puDst->au8[7] = puSrc1->au8[7] == puSrc2->au8[7] ? UINT8_MAX : 0; + puDst->au8[8] = puSrc1->au8[8] == puSrc2->au8[8] ? UINT8_MAX : 0; + puDst->au8[9] = puSrc1->au8[9] == puSrc2->au8[9] ? UINT8_MAX : 0; + puDst->au8[10] = puSrc1->au8[10] == puSrc2->au8[10] ? UINT8_MAX : 0; + puDst->au8[11] = puSrc1->au8[11] == puSrc2->au8[11] ? UINT8_MAX : 0; + puDst->au8[12] = puSrc1->au8[12] == puSrc2->au8[12] ? UINT8_MAX : 0; + puDst->au8[13] = puSrc1->au8[13] == puSrc2->au8[13] ? UINT8_MAX : 0; + puDst->au8[14] = puSrc1->au8[14] == puSrc2->au8[14] ? UINT8_MAX : 0; + puDst->au8[15] = puSrc1->au8[15] == puSrc2->au8[15] ? UINT8_MAX : 0; + puDst->au8[16] = puSrc1->au8[16] == puSrc2->au8[16] ? UINT8_MAX : 0; + puDst->au8[17] = puSrc1->au8[17] == puSrc2->au8[17] ? UINT8_MAX : 0; + puDst->au8[18] = puSrc1->au8[18] == puSrc2->au8[18] ? UINT8_MAX : 0; + puDst->au8[19] = puSrc1->au8[19] == puSrc2->au8[19] ? UINT8_MAX : 0; + puDst->au8[20] = puSrc1->au8[20] == puSrc2->au8[20] ? UINT8_MAX : 0; + puDst->au8[21] = puSrc1->au8[21] == puSrc2->au8[21] ? UINT8_MAX : 0; + puDst->au8[22] = puSrc1->au8[22] == puSrc2->au8[22] ? UINT8_MAX : 0; + puDst->au8[23] = puSrc1->au8[23] == puSrc2->au8[23] ? UINT8_MAX : 0; + puDst->au8[24] = puSrc1->au8[24] == puSrc2->au8[24] ? UINT8_MAX : 0; + puDst->au8[25] = puSrc1->au8[25] == puSrc2->au8[25] ? UINT8_MAX : 0; + puDst->au8[26] = puSrc1->au8[26] == puSrc2->au8[26] ? UINT8_MAX : 0; + puDst->au8[27] = puSrc1->au8[27] == puSrc2->au8[27] ? UINT8_MAX : 0; + puDst->au8[28] = puSrc1->au8[28] == puSrc2->au8[28] ? UINT8_MAX : 0; + puDst->au8[29] = puSrc1->au8[29] == puSrc2->au8[29] ? UINT8_MAX : 0; + puDst->au8[30] = puSrc1->au8[30] == puSrc2->au8[30] ? UINT8_MAX : 0; + puDst->au8[31] = puSrc1->au8[31] == puSrc2->au8[31] ? UINT8_MAX : 0; +} + + +/* + * PCMPEQW / VPCMPEQW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au16[0] = uSrc1.au16[0] == uSrc2.au16[0] ? UINT16_MAX : 0; + uDst.au16[1] = uSrc1.au16[1] == uSrc2.au16[1] ? UINT16_MAX : 0; + uDst.au16[2] = uSrc1.au16[2] == uSrc2.au16[2] ? UINT16_MAX : 0; + uDst.au16[3] = uSrc1.au16[3] == uSrc2.au16[3] ? UINT16_MAX : 0; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au16[0] = uSrc1.au16[0] == puSrc->au16[0] ? UINT16_MAX : 0; + puDst->au16[1] = uSrc1.au16[1] == puSrc->au16[1] ? UINT16_MAX : 0; + puDst->au16[2] = uSrc1.au16[2] == puSrc->au16[2] ? UINT16_MAX : 0; + puDst->au16[3] = uSrc1.au16[3] == puSrc->au16[3] ? UINT16_MAX : 0; + puDst->au16[4] = uSrc1.au16[4] == puSrc->au16[4] ? UINT16_MAX : 0; + puDst->au16[5] = uSrc1.au16[5] == puSrc->au16[5] ? UINT16_MAX : 0; + puDst->au16[6] = uSrc1.au16[6] == puSrc->au16[6] ? UINT16_MAX : 0; + puDst->au16[7] = uSrc1.au16[7] == puSrc->au16[7] ? UINT16_MAX : 0; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au16[0] = puSrc1->au16[0] == puSrc2->au16[0] ? UINT16_MAX : 0; + puDst->au16[1] = puSrc1->au16[1] == puSrc2->au16[1] ? UINT16_MAX : 0; + puDst->au16[2] = puSrc1->au16[2] == puSrc2->au16[2] ? UINT16_MAX : 0; + puDst->au16[3] = puSrc1->au16[3] == puSrc2->au16[3] ? UINT16_MAX : 0; + puDst->au16[4] = puSrc1->au16[4] == puSrc2->au16[4] ? UINT16_MAX : 0; + puDst->au16[5] = puSrc1->au16[5] == puSrc2->au16[5] ? UINT16_MAX : 0; + puDst->au16[6] = puSrc1->au16[6] == puSrc2->au16[6] ? UINT16_MAX : 0; + puDst->au16[7] = puSrc1->au16[7] == puSrc2->au16[7] ? UINT16_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au16[0] = puSrc1->au16[0] == puSrc2->au16[0] ? UINT16_MAX : 0; + puDst->au16[1] = puSrc1->au16[1] == puSrc2->au16[1] ? UINT16_MAX : 0; + puDst->au16[2] = puSrc1->au16[2] == puSrc2->au16[2] ? UINT16_MAX : 0; + puDst->au16[3] = puSrc1->au16[3] == puSrc2->au16[3] ? UINT16_MAX : 0; + puDst->au16[4] = puSrc1->au16[4] == puSrc2->au16[4] ? UINT16_MAX : 0; + puDst->au16[5] = puSrc1->au16[5] == puSrc2->au16[5] ? UINT16_MAX : 0; + puDst->au16[6] = puSrc1->au16[6] == puSrc2->au16[6] ? UINT16_MAX : 0; + puDst->au16[7] = puSrc1->au16[7] == puSrc2->au16[7] ? UINT16_MAX : 0; + puDst->au16[8] = puSrc1->au16[8] == puSrc2->au16[8] ? UINT16_MAX : 0; + puDst->au16[9] = puSrc1->au16[9] == puSrc2->au16[9] ? UINT16_MAX : 0; + puDst->au16[10] = puSrc1->au16[10] == puSrc2->au16[10] ? UINT16_MAX : 0; + puDst->au16[11] = puSrc1->au16[11] == puSrc2->au16[11] ? UINT16_MAX : 0; + puDst->au16[12] = puSrc1->au16[12] == puSrc2->au16[12] ? UINT16_MAX : 0; + puDst->au16[13] = puSrc1->au16[13] == puSrc2->au16[13] ? UINT16_MAX : 0; + puDst->au16[14] = puSrc1->au16[14] == puSrc2->au16[14] ? UINT16_MAX : 0; + puDst->au16[15] = puSrc1->au16[15] == puSrc2->au16[15] ? UINT16_MAX : 0; +} + + +/* + * PCMPEQD / VPCMPEQD. + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au32[0] = uSrc1.au32[0] == uSrc2.au32[0] ? UINT32_MAX : 0; + uDst.au32[1] = uSrc1.au32[1] == uSrc2.au32[1] ? UINT32_MAX : 0; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au32[0] = uSrc1.au32[0] == puSrc->au32[0] ? UINT32_MAX : 0; + puDst->au32[1] = uSrc1.au32[1] == puSrc->au32[1] ? UINT32_MAX : 0; + puDst->au32[2] = uSrc1.au32[2] == puSrc->au32[2] ? UINT32_MAX : 0; + puDst->au32[3] = uSrc1.au32[3] == puSrc->au32[3] ? UINT32_MAX : 0; +} + +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au32[0] = puSrc1->au32[0] == puSrc2->au32[0] ? UINT32_MAX : 0; + puDst->au32[1] = puSrc1->au32[1] == puSrc2->au32[1] ? UINT32_MAX : 0; + puDst->au32[2] = puSrc1->au32[2] == puSrc2->au32[2] ? UINT32_MAX : 0; + puDst->au32[3] = puSrc1->au32[3] == puSrc2->au32[3] ? UINT32_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au32[0] = puSrc1->au32[0] == puSrc2->au32[0] ? UINT32_MAX : 0; + puDst->au32[1] = puSrc1->au32[1] == puSrc2->au32[1] ? UINT32_MAX : 0; + puDst->au32[2] = puSrc1->au32[2] == puSrc2->au32[2] ? UINT32_MAX : 0; + puDst->au32[3] = puSrc1->au32[3] == puSrc2->au32[3] ? UINT32_MAX : 0; + puDst->au32[4] = puSrc1->au32[4] == puSrc2->au32[4] ? UINT32_MAX : 0; + puDst->au32[5] = puSrc1->au32[5] == puSrc2->au32[5] ? UINT32_MAX : 0; + puDst->au32[6] = puSrc1->au32[6] == puSrc2->au32[6] ? UINT32_MAX : 0; + puDst->au32[7] = puSrc1->au32[7] == puSrc2->au32[7] ? UINT32_MAX : 0; +} + + +/* + * PCMPEQQ / VPCMPEQQ. + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqq_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au64[0] = uSrc1.au64[0] == puSrc->au64[0] ? UINT64_MAX : 0; + puDst->au64[1] = uSrc1.au64[1] == puSrc->au64[1] ? UINT64_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqq_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] == puSrc2->au64[0] ? UINT64_MAX : 0; + puDst->au64[1] = puSrc1->au64[1] == puSrc2->au64[1] ? UINT64_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqq_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] == puSrc2->au64[0] ? UINT64_MAX : 0; + puDst->au64[1] = puSrc1->au64[1] == puSrc2->au64[1] ? UINT64_MAX : 0; + puDst->au64[2] = puSrc1->au64[2] == puSrc2->au64[2] ? UINT64_MAX : 0; + puDst->au64[3] = puSrc1->au64[3] == puSrc2->au64[3] ? UINT64_MAX : 0; +} + + +/* + * PCMPGTB / VPCMPGTB + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au8[0] = uSrc1.ai8[0] > uSrc2.ai8[0] ? UINT8_MAX : 0; + uDst.au8[1] = uSrc1.ai8[1] > uSrc2.ai8[1] ? UINT8_MAX : 0; + uDst.au8[2] = uSrc1.ai8[2] > uSrc2.ai8[2] ? UINT8_MAX : 0; + uDst.au8[3] = uSrc1.ai8[3] > uSrc2.ai8[3] ? UINT8_MAX : 0; + uDst.au8[4] = uSrc1.ai8[4] > uSrc2.ai8[4] ? UINT8_MAX : 0; + uDst.au8[5] = uSrc1.ai8[5] > uSrc2.ai8[5] ? UINT8_MAX : 0; + uDst.au8[6] = uSrc1.ai8[6] > uSrc2.ai8[6] ? UINT8_MAX : 0; + uDst.au8[7] = uSrc1.ai8[7] > uSrc2.ai8[7] ? UINT8_MAX : 0; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au8[0] = uSrc1.ai8[0] > puSrc->ai8[0] ? UINT8_MAX : 0; + puDst->au8[1] = uSrc1.ai8[1] > puSrc->ai8[1] ? UINT8_MAX : 0; + puDst->au8[2] = uSrc1.ai8[2] > puSrc->ai8[2] ? UINT8_MAX : 0; + puDst->au8[3] = uSrc1.ai8[3] > puSrc->ai8[3] ? UINT8_MAX : 0; + puDst->au8[4] = uSrc1.ai8[4] > puSrc->ai8[4] ? UINT8_MAX : 0; + puDst->au8[5] = uSrc1.ai8[5] > puSrc->ai8[5] ? UINT8_MAX : 0; + puDst->au8[6] = uSrc1.ai8[6] > puSrc->ai8[6] ? UINT8_MAX : 0; + puDst->au8[7] = uSrc1.ai8[7] > puSrc->ai8[7] ? UINT8_MAX : 0; + puDst->au8[8] = uSrc1.ai8[8] > puSrc->ai8[8] ? UINT8_MAX : 0; + puDst->au8[9] = uSrc1.ai8[9] > puSrc->ai8[9] ? UINT8_MAX : 0; + puDst->au8[10] = uSrc1.ai8[10] > puSrc->ai8[10] ? UINT8_MAX : 0; + puDst->au8[11] = uSrc1.ai8[11] > puSrc->ai8[11] ? UINT8_MAX : 0; + puDst->au8[12] = uSrc1.ai8[12] > puSrc->ai8[12] ? UINT8_MAX : 0; + puDst->au8[13] = uSrc1.ai8[13] > puSrc->ai8[13] ? UINT8_MAX : 0; + puDst->au8[14] = uSrc1.ai8[14] > puSrc->ai8[14] ? UINT8_MAX : 0; + puDst->au8[15] = uSrc1.ai8[15] > puSrc->ai8[15] ? UINT8_MAX : 0; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au8[0] = puSrc1->ai8[0] > puSrc2->ai8[0] ? UINT8_MAX : 0; + puDst->au8[1] = puSrc1->ai8[1] > puSrc2->ai8[1] ? UINT8_MAX : 0; + puDst->au8[2] = puSrc1->ai8[2] > puSrc2->ai8[2] ? UINT8_MAX : 0; + puDst->au8[3] = puSrc1->ai8[3] > puSrc2->ai8[3] ? UINT8_MAX : 0; + puDst->au8[4] = puSrc1->ai8[4] > puSrc2->ai8[4] ? UINT8_MAX : 0; + puDst->au8[5] = puSrc1->ai8[5] > puSrc2->ai8[5] ? UINT8_MAX : 0; + puDst->au8[6] = puSrc1->ai8[6] > puSrc2->ai8[6] ? UINT8_MAX : 0; + puDst->au8[7] = puSrc1->ai8[7] > puSrc2->ai8[7] ? UINT8_MAX : 0; + puDst->au8[8] = puSrc1->ai8[8] > puSrc2->ai8[8] ? UINT8_MAX : 0; + puDst->au8[9] = puSrc1->ai8[9] > puSrc2->ai8[9] ? UINT8_MAX : 0; + puDst->au8[10] = puSrc1->ai8[10] > puSrc2->ai8[10] ? UINT8_MAX : 0; + puDst->au8[11] = puSrc1->ai8[11] > puSrc2->ai8[11] ? UINT8_MAX : 0; + puDst->au8[12] = puSrc1->ai8[12] > puSrc2->ai8[12] ? UINT8_MAX : 0; + puDst->au8[13] = puSrc1->ai8[13] > puSrc2->ai8[13] ? UINT8_MAX : 0; + puDst->au8[14] = puSrc1->ai8[14] > puSrc2->ai8[14] ? UINT8_MAX : 0; + puDst->au8[15] = puSrc1->ai8[15] > puSrc2->ai8[15] ? UINT8_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au8[0] = puSrc1->ai8[0] > puSrc2->ai8[0] ? UINT8_MAX : 0; + puDst->au8[1] = puSrc1->ai8[1] > puSrc2->ai8[1] ? UINT8_MAX : 0; + puDst->au8[2] = puSrc1->ai8[2] > puSrc2->ai8[2] ? UINT8_MAX : 0; + puDst->au8[3] = puSrc1->ai8[3] > puSrc2->ai8[3] ? UINT8_MAX : 0; + puDst->au8[4] = puSrc1->ai8[4] > puSrc2->ai8[4] ? UINT8_MAX : 0; + puDst->au8[5] = puSrc1->ai8[5] > puSrc2->ai8[5] ? UINT8_MAX : 0; + puDst->au8[6] = puSrc1->ai8[6] > puSrc2->ai8[6] ? UINT8_MAX : 0; + puDst->au8[7] = puSrc1->ai8[7] > puSrc2->ai8[7] ? UINT8_MAX : 0; + puDst->au8[8] = puSrc1->ai8[8] > puSrc2->ai8[8] ? UINT8_MAX : 0; + puDst->au8[9] = puSrc1->ai8[9] > puSrc2->ai8[9] ? UINT8_MAX : 0; + puDst->au8[10] = puSrc1->ai8[10] > puSrc2->ai8[10] ? UINT8_MAX : 0; + puDst->au8[11] = puSrc1->ai8[11] > puSrc2->ai8[11] ? UINT8_MAX : 0; + puDst->au8[12] = puSrc1->ai8[12] > puSrc2->ai8[12] ? UINT8_MAX : 0; + puDst->au8[13] = puSrc1->ai8[13] > puSrc2->ai8[13] ? UINT8_MAX : 0; + puDst->au8[14] = puSrc1->ai8[14] > puSrc2->ai8[14] ? UINT8_MAX : 0; + puDst->au8[15] = puSrc1->ai8[15] > puSrc2->ai8[15] ? UINT8_MAX : 0; + puDst->au8[16] = puSrc1->ai8[16] > puSrc2->ai8[16] ? UINT8_MAX : 0; + puDst->au8[17] = puSrc1->ai8[17] > puSrc2->ai8[17] ? UINT8_MAX : 0; + puDst->au8[18] = puSrc1->ai8[18] > puSrc2->ai8[18] ? UINT8_MAX : 0; + puDst->au8[19] = puSrc1->ai8[19] > puSrc2->ai8[19] ? UINT8_MAX : 0; + puDst->au8[20] = puSrc1->ai8[20] > puSrc2->ai8[20] ? UINT8_MAX : 0; + puDst->au8[21] = puSrc1->ai8[21] > puSrc2->ai8[21] ? UINT8_MAX : 0; + puDst->au8[22] = puSrc1->ai8[22] > puSrc2->ai8[22] ? UINT8_MAX : 0; + puDst->au8[23] = puSrc1->ai8[23] > puSrc2->ai8[23] ? UINT8_MAX : 0; + puDst->au8[24] = puSrc1->ai8[24] > puSrc2->ai8[24] ? UINT8_MAX : 0; + puDst->au8[25] = puSrc1->ai8[25] > puSrc2->ai8[25] ? UINT8_MAX : 0; + puDst->au8[26] = puSrc1->ai8[26] > puSrc2->ai8[26] ? UINT8_MAX : 0; + puDst->au8[27] = puSrc1->ai8[27] > puSrc2->ai8[27] ? UINT8_MAX : 0; + puDst->au8[28] = puSrc1->ai8[28] > puSrc2->ai8[28] ? UINT8_MAX : 0; + puDst->au8[29] = puSrc1->ai8[29] > puSrc2->ai8[29] ? UINT8_MAX : 0; + puDst->au8[30] = puSrc1->ai8[30] > puSrc2->ai8[30] ? UINT8_MAX : 0; + puDst->au8[31] = puSrc1->ai8[31] > puSrc2->ai8[31] ? UINT8_MAX : 0; +} + + +/* + * PCMPGTW / VPCMPGTW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au16[0] = uSrc1.ai16[0] > uSrc2.ai16[0] ? UINT16_MAX : 0; + uDst.au16[1] = uSrc1.ai16[1] > uSrc2.ai16[1] ? UINT16_MAX : 0; + uDst.au16[2] = uSrc1.ai16[2] > uSrc2.ai16[2] ? UINT16_MAX : 0; + uDst.au16[3] = uSrc1.ai16[3] > uSrc2.ai16[3] ? UINT16_MAX : 0; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au16[0] = uSrc1.ai16[0] > puSrc->ai16[0] ? UINT16_MAX : 0; + puDst->au16[1] = uSrc1.ai16[1] > puSrc->ai16[1] ? UINT16_MAX : 0; + puDst->au16[2] = uSrc1.ai16[2] > puSrc->ai16[2] ? UINT16_MAX : 0; + puDst->au16[3] = uSrc1.ai16[3] > puSrc->ai16[3] ? UINT16_MAX : 0; + puDst->au16[4] = uSrc1.ai16[4] > puSrc->ai16[4] ? UINT16_MAX : 0; + puDst->au16[5] = uSrc1.ai16[5] > puSrc->ai16[5] ? UINT16_MAX : 0; + puDst->au16[6] = uSrc1.ai16[6] > puSrc->ai16[6] ? UINT16_MAX : 0; + puDst->au16[7] = uSrc1.ai16[7] > puSrc->ai16[7] ? UINT16_MAX : 0; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au16[0] = puSrc1->ai16[0] > puSrc2->ai16[0] ? UINT16_MAX : 0; + puDst->au16[1] = puSrc1->ai16[1] > puSrc2->ai16[1] ? UINT16_MAX : 0; + puDst->au16[2] = puSrc1->ai16[2] > puSrc2->ai16[2] ? UINT16_MAX : 0; + puDst->au16[3] = puSrc1->ai16[3] > puSrc2->ai16[3] ? UINT16_MAX : 0; + puDst->au16[4] = puSrc1->ai16[4] > puSrc2->ai16[4] ? UINT16_MAX : 0; + puDst->au16[5] = puSrc1->ai16[5] > puSrc2->ai16[5] ? UINT16_MAX : 0; + puDst->au16[6] = puSrc1->ai16[6] > puSrc2->ai16[6] ? UINT16_MAX : 0; + puDst->au16[7] = puSrc1->ai16[7] > puSrc2->ai16[7] ? UINT16_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au16[0] = puSrc1->ai16[0] > puSrc2->ai16[0] ? UINT16_MAX : 0; + puDst->au16[1] = puSrc1->ai16[1] > puSrc2->ai16[1] ? UINT16_MAX : 0; + puDst->au16[2] = puSrc1->ai16[2] > puSrc2->ai16[2] ? UINT16_MAX : 0; + puDst->au16[3] = puSrc1->ai16[3] > puSrc2->ai16[3] ? UINT16_MAX : 0; + puDst->au16[4] = puSrc1->ai16[4] > puSrc2->ai16[4] ? UINT16_MAX : 0; + puDst->au16[5] = puSrc1->ai16[5] > puSrc2->ai16[5] ? UINT16_MAX : 0; + puDst->au16[6] = puSrc1->ai16[6] > puSrc2->ai16[6] ? UINT16_MAX : 0; + puDst->au16[7] = puSrc1->ai16[7] > puSrc2->ai16[7] ? UINT16_MAX : 0; + puDst->au16[8] = puSrc1->ai16[8] > puSrc2->ai16[8] ? UINT16_MAX : 0; + puDst->au16[9] = puSrc1->ai16[9] > puSrc2->ai16[9] ? UINT16_MAX : 0; + puDst->au16[10] = puSrc1->ai16[10] > puSrc2->ai16[10] ? UINT16_MAX : 0; + puDst->au16[11] = puSrc1->ai16[11] > puSrc2->ai16[11] ? UINT16_MAX : 0; + puDst->au16[12] = puSrc1->ai16[12] > puSrc2->ai16[12] ? UINT16_MAX : 0; + puDst->au16[13] = puSrc1->ai16[13] > puSrc2->ai16[13] ? UINT16_MAX : 0; + puDst->au16[14] = puSrc1->ai16[14] > puSrc2->ai16[14] ? UINT16_MAX : 0; + puDst->au16[15] = puSrc1->ai16[15] > puSrc2->ai16[15] ? UINT16_MAX : 0; +} + + +/* + * PCMPGTD / VPCMPGTD. + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au32[0] = uSrc1.ai32[0] > uSrc2.ai32[0] ? UINT32_MAX : 0; + uDst.au32[1] = uSrc1.ai32[1] > uSrc2.ai32[1] ? UINT32_MAX : 0; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au32[0] = uSrc1.ai32[0] > puSrc->ai32[0] ? UINT32_MAX : 0; + puDst->au32[1] = uSrc1.ai32[1] > puSrc->ai32[1] ? UINT32_MAX : 0; + puDst->au32[2] = uSrc1.ai32[2] > puSrc->ai32[2] ? UINT32_MAX : 0; + puDst->au32[3] = uSrc1.ai32[3] > puSrc->ai32[3] ? UINT32_MAX : 0; +} + +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au32[0] = puSrc1->ai32[0] > puSrc2->ai32[0] ? UINT32_MAX : 0; + puDst->au32[1] = puSrc1->ai32[1] > puSrc2->ai32[1] ? UINT32_MAX : 0; + puDst->au32[2] = puSrc1->ai32[2] > puSrc2->ai32[2] ? UINT32_MAX : 0; + puDst->au32[3] = puSrc1->ai32[3] > puSrc2->ai32[3] ? UINT32_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au32[0] = puSrc1->ai32[0] > puSrc2->ai32[0] ? UINT32_MAX : 0; + puDst->au32[1] = puSrc1->ai32[1] > puSrc2->ai32[1] ? UINT32_MAX : 0; + puDst->au32[2] = puSrc1->ai32[2] > puSrc2->ai32[2] ? UINT32_MAX : 0; + puDst->au32[3] = puSrc1->ai32[3] > puSrc2->ai32[3] ? UINT32_MAX : 0; + puDst->au32[4] = puSrc1->ai32[4] > puSrc2->ai32[4] ? UINT32_MAX : 0; + puDst->au32[5] = puSrc1->ai32[5] > puSrc2->ai32[5] ? UINT32_MAX : 0; + puDst->au32[6] = puSrc1->ai32[6] > puSrc2->ai32[6] ? UINT32_MAX : 0; + puDst->au32[7] = puSrc1->ai32[7] > puSrc2->ai32[7] ? UINT32_MAX : 0; +} + + +/* + * PCMPGTQ / VPCMPGTQ. + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtq_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au64[0] = uSrc1.ai64[0] > puSrc->ai64[0] ? UINT64_MAX : 0; + puDst->au64[1] = uSrc1.ai64[1] > puSrc->ai64[1] ? UINT64_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtq_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->ai64[0] > puSrc2->ai64[0] ? UINT64_MAX : 0; + puDst->au64[1] = puSrc1->ai64[1] > puSrc2->ai64[1] ? UINT64_MAX : 0; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtq_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->ai64[0] > puSrc2->ai64[0] ? UINT64_MAX : 0; + puDst->au64[1] = puSrc1->ai64[1] > puSrc2->ai64[1] ? UINT64_MAX : 0; + puDst->au64[2] = puSrc1->ai64[2] > puSrc2->ai64[2] ? UINT64_MAX : 0; + puDst->au64[3] = puSrc1->ai64[3] > puSrc2->ai64[3] ? UINT64_MAX : 0; +} + + +/* + * PADDB / VPADDB + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au8[0] = uSrc1.au8[0] + uSrc2.au8[0]; + uDst.au8[1] = uSrc1.au8[1] + uSrc2.au8[1]; + uDst.au8[2] = uSrc1.au8[2] + uSrc2.au8[2]; + uDst.au8[3] = uSrc1.au8[3] + uSrc2.au8[3]; + uDst.au8[4] = uSrc1.au8[4] + uSrc2.au8[4]; + uDst.au8[5] = uSrc1.au8[5] + uSrc2.au8[5]; + uDst.au8[6] = uSrc1.au8[6] + uSrc2.au8[6]; + uDst.au8[7] = uSrc1.au8[7] + uSrc2.au8[7]; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au8[0] = uSrc1.au8[0] + puSrc->au8[0]; + puDst->au8[1] = uSrc1.au8[1] + puSrc->au8[1]; + puDst->au8[2] = uSrc1.au8[2] + puSrc->au8[2]; + puDst->au8[3] = uSrc1.au8[3] + puSrc->au8[3]; + puDst->au8[4] = uSrc1.au8[4] + puSrc->au8[4]; + puDst->au8[5] = uSrc1.au8[5] + puSrc->au8[5]; + puDst->au8[6] = uSrc1.au8[6] + puSrc->au8[6]; + puDst->au8[7] = uSrc1.au8[7] + puSrc->au8[7]; + puDst->au8[8] = uSrc1.au8[8] + puSrc->au8[8]; + puDst->au8[9] = uSrc1.au8[9] + puSrc->au8[9]; + puDst->au8[10] = uSrc1.au8[10] + puSrc->au8[10]; + puDst->au8[11] = uSrc1.au8[11] + puSrc->au8[11]; + puDst->au8[12] = uSrc1.au8[12] + puSrc->au8[12]; + puDst->au8[13] = uSrc1.au8[13] + puSrc->au8[13]; + puDst->au8[14] = uSrc1.au8[14] + puSrc->au8[14]; + puDst->au8[15] = uSrc1.au8[15] + puSrc->au8[15]; +} + +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au8[0] = puSrc1->au8[0] + puSrc2->au8[0]; + puDst->au8[1] = puSrc1->au8[1] + puSrc2->au8[1]; + puDst->au8[2] = puSrc1->au8[2] + puSrc2->au8[2]; + puDst->au8[3] = puSrc1->au8[3] + puSrc2->au8[3]; + puDst->au8[4] = puSrc1->au8[4] + puSrc2->au8[4]; + puDst->au8[5] = puSrc1->au8[5] + puSrc2->au8[5]; + puDst->au8[6] = puSrc1->au8[6] + puSrc2->au8[6]; + puDst->au8[7] = puSrc1->au8[7] + puSrc2->au8[7]; + puDst->au8[8] = puSrc1->au8[8] + puSrc2->au8[8]; + puDst->au8[9] = puSrc1->au8[9] + puSrc2->au8[9]; + puDst->au8[10] = puSrc1->au8[10] + puSrc2->au8[10]; + puDst->au8[11] = puSrc1->au8[11] + puSrc2->au8[11]; + puDst->au8[12] = puSrc1->au8[12] + puSrc2->au8[12]; + puDst->au8[13] = puSrc1->au8[13] + puSrc2->au8[13]; + puDst->au8[14] = puSrc1->au8[14] + puSrc2->au8[14]; + puDst->au8[15] = puSrc1->au8[15] + puSrc2->au8[15]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au8[0] = puSrc1->au8[0] + puSrc2->au8[0]; + puDst->au8[1] = puSrc1->au8[1] + puSrc2->au8[1]; + puDst->au8[2] = puSrc1->au8[2] + puSrc2->au8[2]; + puDst->au8[3] = puSrc1->au8[3] + puSrc2->au8[3]; + puDst->au8[4] = puSrc1->au8[4] + puSrc2->au8[4]; + puDst->au8[5] = puSrc1->au8[5] + puSrc2->au8[5]; + puDst->au8[6] = puSrc1->au8[6] + puSrc2->au8[6]; + puDst->au8[7] = puSrc1->au8[7] + puSrc2->au8[7]; + puDst->au8[8] = puSrc1->au8[8] + puSrc2->au8[8]; + puDst->au8[9] = puSrc1->au8[9] + puSrc2->au8[9]; + puDst->au8[10] = puSrc1->au8[10] + puSrc2->au8[10]; + puDst->au8[11] = puSrc1->au8[11] + puSrc2->au8[11]; + puDst->au8[12] = puSrc1->au8[12] + puSrc2->au8[12]; + puDst->au8[13] = puSrc1->au8[13] + puSrc2->au8[13]; + puDst->au8[14] = puSrc1->au8[14] + puSrc2->au8[14]; + puDst->au8[15] = puSrc1->au8[15] + puSrc2->au8[15]; + puDst->au8[16] = puSrc1->au8[16] + puSrc2->au8[16]; + puDst->au8[17] = puSrc1->au8[17] + puSrc2->au8[17]; + puDst->au8[18] = puSrc1->au8[18] + puSrc2->au8[18]; + puDst->au8[19] = puSrc1->au8[19] + puSrc2->au8[19]; + puDst->au8[20] = puSrc1->au8[20] + puSrc2->au8[20]; + puDst->au8[21] = puSrc1->au8[21] + puSrc2->au8[21]; + puDst->au8[22] = puSrc1->au8[22] + puSrc2->au8[22]; + puDst->au8[23] = puSrc1->au8[23] + puSrc2->au8[23]; + puDst->au8[24] = puSrc1->au8[24] + puSrc2->au8[24]; + puDst->au8[25] = puSrc1->au8[25] + puSrc2->au8[25]; + puDst->au8[26] = puSrc1->au8[26] + puSrc2->au8[26]; + puDst->au8[27] = puSrc1->au8[27] + puSrc2->au8[27]; + puDst->au8[28] = puSrc1->au8[28] + puSrc2->au8[28]; + puDst->au8[29] = puSrc1->au8[29] + puSrc2->au8[29]; + puDst->au8[30] = puSrc1->au8[30] + puSrc2->au8[30]; + puDst->au8[31] = puSrc1->au8[31] + puSrc2->au8[31]; +} + + +/* + * PADDSB / VPADDSB + */ +#define SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(a_iWord) \ + ( (uint16_t)((a_iWord) + 0x80) <= (uint16_t)0xff \ + ? (uint8_t)(a_iWord) \ + : (uint8_t)0x7f + (uint8_t)(((a_iWord) >> 15) & 1) ) /* 0x7f = INT8_MAX; 0x80 = INT8_MIN; source bit 15 = sign */ + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddsb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au8[0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[0] + uSrc2.ai8[0]); + uDst.au8[1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[1] + uSrc2.ai8[1]); + uDst.au8[2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[2] + uSrc2.ai8[2]); + uDst.au8[3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[3] + uSrc2.ai8[3]); + uDst.au8[4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[4] + uSrc2.ai8[4]); + uDst.au8[5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[5] + uSrc2.ai8[5]); + uDst.au8[6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[6] + uSrc2.ai8[6]); + uDst.au8[7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[7] + uSrc2.ai8[7]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddsb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au8[0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[0] + puSrc->ai8[0]); + puDst->au8[1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[1] + puSrc->ai8[1]); + puDst->au8[2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[2] + puSrc->ai8[2]); + puDst->au8[3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[3] + puSrc->ai8[3]); + puDst->au8[4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[4] + puSrc->ai8[4]); + puDst->au8[5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[5] + puSrc->ai8[5]); + puDst->au8[6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[6] + puSrc->ai8[6]); + puDst->au8[7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[7] + puSrc->ai8[7]); + puDst->au8[8] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[8] + puSrc->ai8[8]); + puDst->au8[9] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[9] + puSrc->ai8[9]); + puDst->au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[10] + puSrc->ai8[10]); + puDst->au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[11] + puSrc->ai8[11]); + puDst->au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[12] + puSrc->ai8[12]); + puDst->au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[13] + puSrc->ai8[13]); + puDst->au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[14] + puSrc->ai8[14]); + puDst->au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[15] + puSrc->ai8[15]); +} + +#endif + + +/* + * PADDSB / VPADDSB + */ +#define SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(a_uWord) \ + ( (uint16_t)(a_uWord) <= (uint16_t)0xff \ + ? (uint8_t)(a_uWord) \ + : (uint8_t)0xff ) /* 0xff = UINT8_MAX */ + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddusb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au8[0] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[0] + uSrc2.au8[0]); + uDst.au8[1] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[1] + uSrc2.au8[1]); + uDst.au8[2] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[2] + uSrc2.au8[2]); + uDst.au8[3] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[3] + uSrc2.au8[3]); + uDst.au8[4] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[4] + uSrc2.au8[4]); + uDst.au8[5] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[5] + uSrc2.au8[5]); + uDst.au8[6] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[6] + uSrc2.au8[6]); + uDst.au8[7] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[7] + uSrc2.au8[7]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddusb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au8[0] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[0] + puSrc->au8[0]); + puDst->au8[1] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[1] + puSrc->au8[1]); + puDst->au8[2] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[2] + puSrc->au8[2]); + puDst->au8[3] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[3] + puSrc->au8[3]); + puDst->au8[4] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[4] + puSrc->au8[4]); + puDst->au8[5] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[5] + puSrc->au8[5]); + puDst->au8[6] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[6] + puSrc->au8[6]); + puDst->au8[7] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[7] + puSrc->au8[7]); + puDst->au8[8] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[8] + puSrc->au8[8]); + puDst->au8[9] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[9] + puSrc->au8[9]); + puDst->au8[10] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[10] + puSrc->au8[10]); + puDst->au8[11] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[11] + puSrc->au8[11]); + puDst->au8[12] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[12] + puSrc->au8[12]); + puDst->au8[13] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[13] + puSrc->au8[13]); + puDst->au8[14] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[14] + puSrc->au8[14]); + puDst->au8[15] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[15] + puSrc->au8[15]); +} + +#endif + + +/* + * PADDW / VPADDW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au16[0] = uSrc1.au16[0] + uSrc2.au16[0]; + uDst.au16[1] = uSrc1.au16[1] + uSrc2.au16[1]; + uDst.au16[2] = uSrc1.au16[2] + uSrc2.au16[2]; + uDst.au16[3] = uSrc1.au16[3] + uSrc2.au16[3]; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au16[0] = uSrc1.au16[0] + puSrc->au16[0]; + puDst->au16[1] = uSrc1.au16[1] + puSrc->au16[1]; + puDst->au16[2] = uSrc1.au16[2] + puSrc->au16[2]; + puDst->au16[3] = uSrc1.au16[3] + puSrc->au16[3]; + puDst->au16[4] = uSrc1.au16[4] + puSrc->au16[4]; + puDst->au16[5] = uSrc1.au16[5] + puSrc->au16[5]; + puDst->au16[6] = uSrc1.au16[6] + puSrc->au16[6]; + puDst->au16[7] = uSrc1.au16[7] + puSrc->au16[7]; +} + +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au16[0] = puSrc1->au16[0] + puSrc2->au16[0]; + puDst->au16[1] = puSrc1->au16[1] + puSrc2->au16[1]; + puDst->au16[2] = puSrc1->au16[2] + puSrc2->au16[2]; + puDst->au16[3] = puSrc1->au16[3] + puSrc2->au16[3]; + puDst->au16[4] = puSrc1->au16[4] + puSrc2->au16[4]; + puDst->au16[5] = puSrc1->au16[5] + puSrc2->au16[5]; + puDst->au16[6] = puSrc1->au16[6] + puSrc2->au16[6]; + puDst->au16[7] = puSrc1->au16[7] + puSrc2->au16[7]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au16[0] = puSrc1->au16[0] + puSrc2->au16[0]; + puDst->au16[1] = puSrc1->au16[1] + puSrc2->au16[1]; + puDst->au16[2] = puSrc1->au16[2] + puSrc2->au16[2]; + puDst->au16[3] = puSrc1->au16[3] + puSrc2->au16[3]; + puDst->au16[4] = puSrc1->au16[4] + puSrc2->au16[4]; + puDst->au16[5] = puSrc1->au16[5] + puSrc2->au16[5]; + puDst->au16[6] = puSrc1->au16[6] + puSrc2->au16[6]; + puDst->au16[7] = puSrc1->au16[7] + puSrc2->au16[7]; + puDst->au16[8] = puSrc1->au16[8] + puSrc2->au16[8]; + puDst->au16[9] = puSrc1->au16[9] + puSrc2->au16[9]; + puDst->au16[10] = puSrc1->au16[10] + puSrc2->au16[10]; + puDst->au16[11] = puSrc1->au16[11] + puSrc2->au16[11]; + puDst->au16[12] = puSrc1->au16[12] + puSrc2->au16[12]; + puDst->au16[13] = puSrc1->au16[13] + puSrc2->au16[13]; + puDst->au16[14] = puSrc1->au16[14] + puSrc2->au16[14]; + puDst->au16[15] = puSrc1->au16[15] + puSrc2->au16[15]; +} + + +/* + * PADDSW / VPADDSW + */ +#define SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(a_iDword) \ + ( (uint32_t)((a_iDword) + 0x8000) <= (uint16_t)0xffff \ + ? (uint16_t)(a_iDword) \ + : (uint16_t)0x7fff + (uint16_t)(((a_iDword) >> 31) & 1) ) /* 0x7fff = INT16_MAX; 0x8000 = INT16_MIN; source bit 31 = sign */ + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddsw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] + uSrc2.ai16[0]); + uDst.au16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[1] + uSrc2.ai16[1]); + uDst.au16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] + uSrc2.ai16[2]); + uDst.au16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[3] + uSrc2.ai16[3]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddsw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] + puSrc->ai16[0]); + puDst->au16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[1] + puSrc->ai16[1]); + puDst->au16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] + puSrc->ai16[2]); + puDst->au16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[3] + puSrc->ai16[3]); + puDst->au16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[4] + puSrc->ai16[4]); + puDst->au16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[5] + puSrc->ai16[5]); + puDst->au16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[6] + puSrc->ai16[6]); + puDst->au16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[7] + puSrc->ai16[7]); +} + +#endif + + +/* + * PADDUSW / VPADDUSW + */ +#define SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(a_uDword) \ + ( (uint32_t)(a_uDword) <= (uint16_t)0xffff \ + ? (uint16_t)(a_uDword) \ + : (uint16_t)0xffff ) /* 0xffff = UINT16_MAX */ + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddusw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au16[0] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[0] + uSrc2.au16[0]); + uDst.au16[1] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[1] + uSrc2.au16[1]); + uDst.au16[2] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[2] + uSrc2.au16[2]); + uDst.au16[3] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[3] + uSrc2.au16[3]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddusw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au16[0] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[0] + puSrc->au16[0]); + puDst->au16[1] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[1] + puSrc->au16[1]); + puDst->au16[2] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[2] + puSrc->au16[2]); + puDst->au16[3] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[3] + puSrc->au16[3]); + puDst->au16[4] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[4] + puSrc->au16[4]); + puDst->au16[5] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[5] + puSrc->au16[5]); + puDst->au16[6] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[6] + puSrc->au16[6]); + puDst->au16[7] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[7] + puSrc->au16[7]); +} + +#endif + + +/* + * PADDD / VPADDD. + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au32[0] = uSrc1.au32[0] + uSrc2.au32[0]; + uDst.au32[1] = uSrc1.au32[1] + uSrc2.au32[1]; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au32[0] = uSrc1.au32[0] + puSrc->au32[0]; + puDst->au32[1] = uSrc1.au32[1] + puSrc->au32[1]; + puDst->au32[2] = uSrc1.au32[2] + puSrc->au32[2]; + puDst->au32[3] = uSrc1.au32[3] + puSrc->au32[3]; +} + +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au32[0] = puSrc1->au32[0] + puSrc2->au32[0]; + puDst->au32[1] = puSrc1->au32[1] + puSrc2->au32[1]; + puDst->au32[2] = puSrc1->au32[2] + puSrc2->au32[2]; + puDst->au32[3] = puSrc1->au32[3] + puSrc2->au32[3]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au32[0] = puSrc1->au32[0] + puSrc2->au32[0]; + puDst->au32[1] = puSrc1->au32[1] + puSrc2->au32[1]; + puDst->au32[2] = puSrc1->au32[2] + puSrc2->au32[2]; + puDst->au32[3] = puSrc1->au32[3] + puSrc2->au32[3]; + puDst->au32[4] = puSrc1->au32[4] + puSrc2->au32[4]; + puDst->au32[5] = puSrc1->au32[5] + puSrc2->au32[5]; + puDst->au32[6] = puSrc1->au32[6] + puSrc2->au32[6]; + puDst->au32[7] = puSrc1->au32[7] + puSrc2->au32[7]; +} + + +/* + * PADDQ / VPADDQ. + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddq_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + *puDst = *puDst + *puSrc; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_paddq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au64[0] = uSrc1.au64[0] + puSrc->au64[0]; + puDst->au64[1] = uSrc1.au64[1] + puSrc->au64[1]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddq_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] + puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] + puSrc2->au64[1]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddq_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] + puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] + puSrc2->au64[1]; + puDst->au64[2] = puSrc1->au64[2] + puSrc2->au64[2]; + puDst->au64[3] = puSrc1->au64[3] + puSrc2->au64[3]; +} + + +/* + * PSUBB / VPSUBB + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au8[0] = uSrc1.au8[0] - uSrc2.au8[0]; + uDst.au8[1] = uSrc1.au8[1] - uSrc2.au8[1]; + uDst.au8[2] = uSrc1.au8[2] - uSrc2.au8[2]; + uDst.au8[3] = uSrc1.au8[3] - uSrc2.au8[3]; + uDst.au8[4] = uSrc1.au8[4] - uSrc2.au8[4]; + uDst.au8[5] = uSrc1.au8[5] - uSrc2.au8[5]; + uDst.au8[6] = uSrc1.au8[6] - uSrc2.au8[6]; + uDst.au8[7] = uSrc1.au8[7] - uSrc2.au8[7]; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au8[0] = uSrc1.au8[0] - puSrc->au8[0]; + puDst->au8[1] = uSrc1.au8[1] - puSrc->au8[1]; + puDst->au8[2] = uSrc1.au8[2] - puSrc->au8[2]; + puDst->au8[3] = uSrc1.au8[3] - puSrc->au8[3]; + puDst->au8[4] = uSrc1.au8[4] - puSrc->au8[4]; + puDst->au8[5] = uSrc1.au8[5] - puSrc->au8[5]; + puDst->au8[6] = uSrc1.au8[6] - puSrc->au8[6]; + puDst->au8[7] = uSrc1.au8[7] - puSrc->au8[7]; + puDst->au8[8] = uSrc1.au8[8] - puSrc->au8[8]; + puDst->au8[9] = uSrc1.au8[9] - puSrc->au8[9]; + puDst->au8[10] = uSrc1.au8[10] - puSrc->au8[10]; + puDst->au8[11] = uSrc1.au8[11] - puSrc->au8[11]; + puDst->au8[12] = uSrc1.au8[12] - puSrc->au8[12]; + puDst->au8[13] = uSrc1.au8[13] - puSrc->au8[13]; + puDst->au8[14] = uSrc1.au8[14] - puSrc->au8[14]; + puDst->au8[15] = uSrc1.au8[15] - puSrc->au8[15]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au8[0] = puSrc1->au8[0] - puSrc2->au8[0]; + puDst->au8[1] = puSrc1->au8[1] - puSrc2->au8[1]; + puDst->au8[2] = puSrc1->au8[2] - puSrc2->au8[2]; + puDst->au8[3] = puSrc1->au8[3] - puSrc2->au8[3]; + puDst->au8[4] = puSrc1->au8[4] - puSrc2->au8[4]; + puDst->au8[5] = puSrc1->au8[5] - puSrc2->au8[5]; + puDst->au8[6] = puSrc1->au8[6] - puSrc2->au8[6]; + puDst->au8[7] = puSrc1->au8[7] - puSrc2->au8[7]; + puDst->au8[8] = puSrc1->au8[8] - puSrc2->au8[8]; + puDst->au8[9] = puSrc1->au8[9] - puSrc2->au8[9]; + puDst->au8[10] = puSrc1->au8[10] - puSrc2->au8[10]; + puDst->au8[11] = puSrc1->au8[11] - puSrc2->au8[11]; + puDst->au8[12] = puSrc1->au8[12] - puSrc2->au8[12]; + puDst->au8[13] = puSrc1->au8[13] - puSrc2->au8[13]; + puDst->au8[14] = puSrc1->au8[14] - puSrc2->au8[14]; + puDst->au8[15] = puSrc1->au8[15] - puSrc2->au8[15]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au8[0] = puSrc1->au8[0] - puSrc2->au8[0]; + puDst->au8[1] = puSrc1->au8[1] - puSrc2->au8[1]; + puDst->au8[2] = puSrc1->au8[2] - puSrc2->au8[2]; + puDst->au8[3] = puSrc1->au8[3] - puSrc2->au8[3]; + puDst->au8[4] = puSrc1->au8[4] - puSrc2->au8[4]; + puDst->au8[5] = puSrc1->au8[5] - puSrc2->au8[5]; + puDst->au8[6] = puSrc1->au8[6] - puSrc2->au8[6]; + puDst->au8[7] = puSrc1->au8[7] - puSrc2->au8[7]; + puDst->au8[8] = puSrc1->au8[8] - puSrc2->au8[8]; + puDst->au8[9] = puSrc1->au8[9] - puSrc2->au8[9]; + puDst->au8[10] = puSrc1->au8[10] - puSrc2->au8[10]; + puDst->au8[11] = puSrc1->au8[11] - puSrc2->au8[11]; + puDst->au8[12] = puSrc1->au8[12] - puSrc2->au8[12]; + puDst->au8[13] = puSrc1->au8[13] - puSrc2->au8[13]; + puDst->au8[14] = puSrc1->au8[14] - puSrc2->au8[14]; + puDst->au8[15] = puSrc1->au8[15] - puSrc2->au8[15]; + puDst->au8[16] = puSrc1->au8[16] - puSrc2->au8[16]; + puDst->au8[17] = puSrc1->au8[17] - puSrc2->au8[17]; + puDst->au8[18] = puSrc1->au8[18] - puSrc2->au8[18]; + puDst->au8[19] = puSrc1->au8[19] - puSrc2->au8[19]; + puDst->au8[20] = puSrc1->au8[20] - puSrc2->au8[20]; + puDst->au8[21] = puSrc1->au8[21] - puSrc2->au8[21]; + puDst->au8[22] = puSrc1->au8[22] - puSrc2->au8[22]; + puDst->au8[23] = puSrc1->au8[23] - puSrc2->au8[23]; + puDst->au8[24] = puSrc1->au8[24] - puSrc2->au8[24]; + puDst->au8[25] = puSrc1->au8[25] - puSrc2->au8[25]; + puDst->au8[26] = puSrc1->au8[26] - puSrc2->au8[26]; + puDst->au8[27] = puSrc1->au8[27] - puSrc2->au8[27]; + puDst->au8[28] = puSrc1->au8[28] - puSrc2->au8[28]; + puDst->au8[29] = puSrc1->au8[29] - puSrc2->au8[29]; + puDst->au8[30] = puSrc1->au8[30] - puSrc2->au8[30]; + puDst->au8[31] = puSrc1->au8[31] - puSrc2->au8[31]; +} + + +/* + * PSUBSB / VSUBSB + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubsb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au8[0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[0] - uSrc2.ai8[0]); + uDst.au8[1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[1] - uSrc2.ai8[1]); + uDst.au8[2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[2] - uSrc2.ai8[2]); + uDst.au8[3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[3] - uSrc2.ai8[3]); + uDst.au8[4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[4] - uSrc2.ai8[4]); + uDst.au8[5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[5] - uSrc2.ai8[5]); + uDst.au8[6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[6] - uSrc2.ai8[6]); + uDst.au8[7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[7] - uSrc2.ai8[7]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubsb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au8[0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[0] - puSrc->ai8[0]); + puDst->au8[1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[1] - puSrc->ai8[1]); + puDst->au8[2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[2] - puSrc->ai8[2]); + puDst->au8[3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[3] - puSrc->ai8[3]); + puDst->au8[4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[4] - puSrc->ai8[4]); + puDst->au8[5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[5] - puSrc->ai8[5]); + puDst->au8[6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[6] - puSrc->ai8[6]); + puDst->au8[7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[7] - puSrc->ai8[7]); + puDst->au8[8] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[8] - puSrc->ai8[8]); + puDst->au8[9] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[9] - puSrc->ai8[9]); + puDst->au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[10] - puSrc->ai8[10]); + puDst->au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[11] - puSrc->ai8[11]); + puDst->au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[12] - puSrc->ai8[12]); + puDst->au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[13] - puSrc->ai8[13]); + puDst->au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[14] - puSrc->ai8[14]); + puDst->au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[15] - puSrc->ai8[15]); +} + +#endif + + +/* + * PADDSB / VPADDSB + */ +#define SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(a_uWord) \ + ( (uint16_t)(a_uWord) <= (uint16_t)0xff \ + ? (uint8_t)(a_uWord) \ + : (uint8_t)0 ) + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubusb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au8[0] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[0] - uSrc2.au8[0]); + uDst.au8[1] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[1] - uSrc2.au8[1]); + uDst.au8[2] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[2] - uSrc2.au8[2]); + uDst.au8[3] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[3] - uSrc2.au8[3]); + uDst.au8[4] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[4] - uSrc2.au8[4]); + uDst.au8[5] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[5] - uSrc2.au8[5]); + uDst.au8[6] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[6] - uSrc2.au8[6]); + uDst.au8[7] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[7] - uSrc2.au8[7]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubusb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au8[0] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[0] - puSrc->au8[0]); + puDst->au8[1] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[1] - puSrc->au8[1]); + puDst->au8[2] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[2] - puSrc->au8[2]); + puDst->au8[3] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[3] - puSrc->au8[3]); + puDst->au8[4] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[4] - puSrc->au8[4]); + puDst->au8[5] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[5] - puSrc->au8[5]); + puDst->au8[6] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[6] - puSrc->au8[6]); + puDst->au8[7] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[7] - puSrc->au8[7]); + puDst->au8[8] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[8] - puSrc->au8[8]); + puDst->au8[9] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[9] - puSrc->au8[9]); + puDst->au8[10] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[10] - puSrc->au8[10]); + puDst->au8[11] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[11] - puSrc->au8[11]); + puDst->au8[12] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[12] - puSrc->au8[12]); + puDst->au8[13] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[13] - puSrc->au8[13]); + puDst->au8[14] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[14] - puSrc->au8[14]); + puDst->au8[15] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[15] - puSrc->au8[15]); +} + +#endif + + +/* + * PSUBW / VPSUBW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au16[0] = uSrc1.au16[0] - uSrc2.au16[0]; + uDst.au16[1] = uSrc1.au16[1] - uSrc2.au16[1]; + uDst.au16[2] = uSrc1.au16[2] - uSrc2.au16[2]; + uDst.au16[3] = uSrc1.au16[3] - uSrc2.au16[3]; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au16[0] = uSrc1.au16[0] - puSrc->au16[0]; + puDst->au16[1] = uSrc1.au16[1] - puSrc->au16[1]; + puDst->au16[2] = uSrc1.au16[2] - puSrc->au16[2]; + puDst->au16[3] = uSrc1.au16[3] - puSrc->au16[3]; + puDst->au16[4] = uSrc1.au16[4] - puSrc->au16[4]; + puDst->au16[5] = uSrc1.au16[5] - puSrc->au16[5]; + puDst->au16[6] = uSrc1.au16[6] - puSrc->au16[6]; + puDst->au16[7] = uSrc1.au16[7] - puSrc->au16[7]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au16[0] = puSrc1->au16[0] - puSrc2->au16[0]; + puDst->au16[1] = puSrc1->au16[1] - puSrc2->au16[1]; + puDst->au16[2] = puSrc1->au16[2] - puSrc2->au16[2]; + puDst->au16[3] = puSrc1->au16[3] - puSrc2->au16[3]; + puDst->au16[4] = puSrc1->au16[4] - puSrc2->au16[4]; + puDst->au16[5] = puSrc1->au16[5] - puSrc2->au16[5]; + puDst->au16[6] = puSrc1->au16[6] - puSrc2->au16[6]; + puDst->au16[7] = puSrc1->au16[7] - puSrc2->au16[7]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au16[0] = puSrc1->au16[0] - puSrc2->au16[0]; + puDst->au16[1] = puSrc1->au16[1] - puSrc2->au16[1]; + puDst->au16[2] = puSrc1->au16[2] - puSrc2->au16[2]; + puDst->au16[3] = puSrc1->au16[3] - puSrc2->au16[3]; + puDst->au16[4] = puSrc1->au16[4] - puSrc2->au16[4]; + puDst->au16[5] = puSrc1->au16[5] - puSrc2->au16[5]; + puDst->au16[6] = puSrc1->au16[6] - puSrc2->au16[6]; + puDst->au16[7] = puSrc1->au16[7] - puSrc2->au16[7]; + puDst->au16[8] = puSrc1->au16[8] - puSrc2->au16[8]; + puDst->au16[9] = puSrc1->au16[9] - puSrc2->au16[9]; + puDst->au16[10] = puSrc1->au16[10] - puSrc2->au16[10]; + puDst->au16[11] = puSrc1->au16[11] - puSrc2->au16[11]; + puDst->au16[12] = puSrc1->au16[12] - puSrc2->au16[12]; + puDst->au16[13] = puSrc1->au16[13] - puSrc2->au16[13]; + puDst->au16[14] = puSrc1->au16[14] - puSrc2->au16[14]; + puDst->au16[15] = puSrc1->au16[15] - puSrc2->au16[15]; +} + + +/* + * PSUBSW / VPSUBSW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubsw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] - uSrc2.ai16[0]); + uDst.au16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[1] - uSrc2.ai16[1]); + uDst.au16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] - uSrc2.ai16[2]); + uDst.au16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[3] - uSrc2.ai16[3]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubsw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] - puSrc->ai16[0]); + puDst->au16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[1] - puSrc->ai16[1]); + puDst->au16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] - puSrc->ai16[2]); + puDst->au16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[3] - puSrc->ai16[3]); + puDst->au16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[4] - puSrc->ai16[4]); + puDst->au16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[5] - puSrc->ai16[5]); + puDst->au16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[6] - puSrc->ai16[6]); + puDst->au16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[7] - puSrc->ai16[7]); +} + +#endif + + +/* + * PSUBUSW / VPSUBUSW + */ +#define SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(a_uDword) \ + ( (uint32_t)(a_uDword) <= (uint16_t)0xffff \ + ? (uint16_t)(a_uDword) \ + : (uint16_t)0 ) + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubusw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au16[0] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[0] - uSrc2.au16[0]); + uDst.au16[1] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[1] - uSrc2.au16[1]); + uDst.au16[2] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[2] - uSrc2.au16[2]); + uDst.au16[3] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[3] - uSrc2.au16[3]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubusw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au16[0] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[0] - puSrc->au16[0]); + puDst->au16[1] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[1] - puSrc->au16[1]); + puDst->au16[2] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[2] - puSrc->au16[2]); + puDst->au16[3] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[3] - puSrc->au16[3]); + puDst->au16[4] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[4] - puSrc->au16[4]); + puDst->au16[5] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[5] - puSrc->au16[5]); + puDst->au16[6] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[6] - puSrc->au16[6]); + puDst->au16[7] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[7] - puSrc->au16[7]); +} + +#endif + + +/* + * PSUBD / VPSUBD. + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au32[0] = uSrc1.au32[0] - uSrc2.au32[0]; + uDst.au32[1] = uSrc1.au32[1] - uSrc2.au32[1]; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au32[0] = uSrc1.au32[0] - puSrc->au32[0]; + puDst->au32[1] = uSrc1.au32[1] - puSrc->au32[1]; + puDst->au32[2] = uSrc1.au32[2] - puSrc->au32[2]; + puDst->au32[3] = uSrc1.au32[3] - puSrc->au32[3]; +} + +#endif /* IEM_WITHOUT_ASSEMBLY */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au32[0] = puSrc1->au32[0] - puSrc2->au32[0]; + puDst->au32[1] = puSrc1->au32[1] - puSrc2->au32[1]; + puDst->au32[2] = puSrc1->au32[2] - puSrc2->au32[2]; + puDst->au32[3] = puSrc1->au32[3] - puSrc2->au32[3]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au32[0] = puSrc1->au32[0] - puSrc2->au32[0]; + puDst->au32[1] = puSrc1->au32[1] - puSrc2->au32[1]; + puDst->au32[2] = puSrc1->au32[2] - puSrc2->au32[2]; + puDst->au32[3] = puSrc1->au32[3] - puSrc2->au32[3]; + puDst->au32[4] = puSrc1->au32[4] - puSrc2->au32[4]; + puDst->au32[5] = puSrc1->au32[5] - puSrc2->au32[5]; + puDst->au32[6] = puSrc1->au32[6] - puSrc2->au32[6]; + puDst->au32[7] = puSrc1->au32[7] - puSrc2->au32[7]; +} + + +/* + * PSUBQ / VPSUBQ. + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubq_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + *puDst = *puDst - *puSrc; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_psubq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->au64[0] = uSrc1.au64[0] - puSrc->au64[0]; + puDst->au64[1] = uSrc1.au64[1] - puSrc->au64[1]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubq_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] - puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] - puSrc2->au64[1]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubq_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RT_NOREF(pExtState); + puDst->au64[0] = puSrc1->au64[0] - puSrc2->au64[0]; + puDst->au64[1] = puSrc1->au64[1] - puSrc2->au64[1]; + puDst->au64[2] = puSrc1->au64[2] - puSrc2->au64[2]; + puDst->au64[3] = puSrc1->au64[3] - puSrc2->au64[3]; +} + + + +/* + * PMULLW / VPMULLW / PMULLD / VPMULLD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmullw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.ai16[0] = uSrc1.ai16[0] * uSrc2.ai16[0]; + uDst.ai16[1] = uSrc1.ai16[1] * uSrc2.ai16[1]; + uDst.ai16[2] = uSrc1.ai16[2] * uSrc2.ai16[2]; + uDst.ai16[3] = uSrc1.ai16[3] * uSrc2.ai16[3]; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmullw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->ai16[0] = uSrc1.ai16[0] * puSrc->ai16[0]; + puDst->ai16[1] = uSrc1.ai16[1] * puSrc->ai16[1]; + puDst->ai16[2] = uSrc1.ai16[2] * puSrc->ai16[2]; + puDst->ai16[3] = uSrc1.ai16[3] * puSrc->ai16[3]; + puDst->ai16[4] = uSrc1.ai16[4] * puSrc->ai16[4]; + puDst->ai16[5] = uSrc1.ai16[5] * puSrc->ai16[5]; + puDst->ai16[6] = uSrc1.ai16[6] * puSrc->ai16[6]; + puDst->ai16[7] = uSrc1.ai16[7] * puSrc->ai16[7]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmulld_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai32[0] = uSrc1.ai32[0] * puSrc->ai32[0]; + puDst->ai32[1] = uSrc1.ai32[1] * puSrc->ai32[1]; + puDst->ai32[2] = uSrc1.ai32[2] * puSrc->ai32[2]; + puDst->ai32[3] = uSrc1.ai32[3] * puSrc->ai32[3]; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmullw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->ai16[0] = puSrc1->ai16[0] * puSrc2->ai16[0]; + puDst->ai16[1] = puSrc1->ai16[1] * puSrc2->ai16[1]; + puDst->ai16[2] = puSrc1->ai16[2] * puSrc2->ai16[2]; + puDst->ai16[3] = puSrc1->ai16[3] * puSrc2->ai16[3]; + puDst->ai16[4] = puSrc1->ai16[4] * puSrc2->ai16[4]; + puDst->ai16[5] = puSrc1->ai16[5] * puSrc2->ai16[5]; + puDst->ai16[6] = puSrc1->ai16[6] * puSrc2->ai16[6]; + puDst->ai16[7] = puSrc1->ai16[7] * puSrc2->ai16[7]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmullw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->ai16[ 0] = puSrc1->ai16[ 0] * puSrc2->ai16[ 0]; + puDst->ai16[ 1] = puSrc1->ai16[ 1] * puSrc2->ai16[ 1]; + puDst->ai16[ 2] = puSrc1->ai16[ 2] * puSrc2->ai16[ 2]; + puDst->ai16[ 3] = puSrc1->ai16[ 3] * puSrc2->ai16[ 3]; + puDst->ai16[ 4] = puSrc1->ai16[ 4] * puSrc2->ai16[ 4]; + puDst->ai16[ 5] = puSrc1->ai16[ 5] * puSrc2->ai16[ 5]; + puDst->ai16[ 6] = puSrc1->ai16[ 6] * puSrc2->ai16[ 6]; + puDst->ai16[ 7] = puSrc1->ai16[ 7] * puSrc2->ai16[ 7]; + puDst->ai16[ 8] = puSrc1->ai16[ 8] * puSrc2->ai16[ 8]; + puDst->ai16[ 9] = puSrc1->ai16[ 9] * puSrc2->ai16[ 9]; + puDst->ai16[10] = puSrc1->ai16[10] * puSrc2->ai16[10]; + puDst->ai16[11] = puSrc1->ai16[11] * puSrc2->ai16[11]; + puDst->ai16[12] = puSrc1->ai16[12] * puSrc2->ai16[12]; + puDst->ai16[13] = puSrc1->ai16[13] * puSrc2->ai16[13]; + puDst->ai16[14] = puSrc1->ai16[14] * puSrc2->ai16[14]; + puDst->ai16[15] = puSrc1->ai16[15] * puSrc2->ai16[15]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulld_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->ai32[0] = puSrc1->ai32[0] * puSrc2->ai32[0]; + puDst->ai32[1] = puSrc1->ai32[1] * puSrc2->ai32[1]; + puDst->ai32[2] = puSrc1->ai32[2] * puSrc2->ai32[2]; + puDst->ai32[3] = puSrc1->ai32[3] * puSrc2->ai32[3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulld_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->ai32[0] = puSrc1->ai32[0] * puSrc2->ai32[0]; + puDst->ai32[1] = puSrc1->ai32[1] * puSrc2->ai32[1]; + puDst->ai32[2] = puSrc1->ai32[2] * puSrc2->ai32[2]; + puDst->ai32[3] = puSrc1->ai32[3] * puSrc2->ai32[3]; + puDst->ai32[4] = puSrc1->ai32[4] * puSrc2->ai32[4]; + puDst->ai32[5] = puSrc1->ai32[5] * puSrc2->ai32[5]; + puDst->ai32[6] = puSrc1->ai32[6] * puSrc2->ai32[6]; + puDst->ai32[7] = puSrc1->ai32[7] * puSrc2->ai32[7]; +} + + +/* + * PMULHW / VPMULHW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.ai16[0] = RT_HIWORD(uSrc1.ai16[0] * uSrc2.ai16[0]); + uDst.ai16[1] = RT_HIWORD(uSrc1.ai16[1] * uSrc2.ai16[1]); + uDst.ai16[2] = RT_HIWORD(uSrc1.ai16[2] * uSrc2.ai16[2]); + uDst.ai16[3] = RT_HIWORD(uSrc1.ai16[3] * uSrc2.ai16[3]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RT_NOREF(pFpuState); + RTUINT128U uSrc1 = *puDst; + puDst->ai16[0] = RT_HIWORD(uSrc1.ai16[0] * puSrc->ai16[0]); + puDst->ai16[1] = RT_HIWORD(uSrc1.ai16[1] * puSrc->ai16[1]); + puDst->ai16[2] = RT_HIWORD(uSrc1.ai16[2] * puSrc->ai16[2]); + puDst->ai16[3] = RT_HIWORD(uSrc1.ai16[3] * puSrc->ai16[3]); + puDst->ai16[4] = RT_HIWORD(uSrc1.ai16[4] * puSrc->ai16[4]); + puDst->ai16[5] = RT_HIWORD(uSrc1.ai16[5] * puSrc->ai16[5]); + puDst->ai16[6] = RT_HIWORD(uSrc1.ai16[6] * puSrc->ai16[6]); + puDst->ai16[7] = RT_HIWORD(uSrc1.ai16[7] * puSrc->ai16[7]); +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->ai16[0] = RT_HIWORD(puSrc1->ai16[0] * puSrc2->ai16[0]); + puDst->ai16[1] = RT_HIWORD(puSrc1->ai16[1] * puSrc2->ai16[1]); + puDst->ai16[2] = RT_HIWORD(puSrc1->ai16[2] * puSrc2->ai16[2]); + puDst->ai16[3] = RT_HIWORD(puSrc1->ai16[3] * puSrc2->ai16[3]); + puDst->ai16[4] = RT_HIWORD(puSrc1->ai16[4] * puSrc2->ai16[4]); + puDst->ai16[5] = RT_HIWORD(puSrc1->ai16[5] * puSrc2->ai16[5]); + puDst->ai16[6] = RT_HIWORD(puSrc1->ai16[6] * puSrc2->ai16[6]); + puDst->ai16[7] = RT_HIWORD(puSrc1->ai16[7] * puSrc2->ai16[7]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->ai16[ 0] = RT_HIWORD(puSrc1->ai16[ 0] * puSrc2->ai16[ 0]); + puDst->ai16[ 1] = RT_HIWORD(puSrc1->ai16[ 1] * puSrc2->ai16[ 1]); + puDst->ai16[ 2] = RT_HIWORD(puSrc1->ai16[ 2] * puSrc2->ai16[ 2]); + puDst->ai16[ 3] = RT_HIWORD(puSrc1->ai16[ 3] * puSrc2->ai16[ 3]); + puDst->ai16[ 4] = RT_HIWORD(puSrc1->ai16[ 4] * puSrc2->ai16[ 4]); + puDst->ai16[ 5] = RT_HIWORD(puSrc1->ai16[ 5] * puSrc2->ai16[ 5]); + puDst->ai16[ 6] = RT_HIWORD(puSrc1->ai16[ 6] * puSrc2->ai16[ 6]); + puDst->ai16[ 7] = RT_HIWORD(puSrc1->ai16[ 7] * puSrc2->ai16[ 7]); + puDst->ai16[ 8] = RT_HIWORD(puSrc1->ai16[ 8] * puSrc2->ai16[ 8]); + puDst->ai16[ 9] = RT_HIWORD(puSrc1->ai16[ 9] * puSrc2->ai16[ 9]); + puDst->ai16[10] = RT_HIWORD(puSrc1->ai16[10] * puSrc2->ai16[10]); + puDst->ai16[11] = RT_HIWORD(puSrc1->ai16[11] * puSrc2->ai16[11]); + puDst->ai16[12] = RT_HIWORD(puSrc1->ai16[12] * puSrc2->ai16[12]); + puDst->ai16[13] = RT_HIWORD(puSrc1->ai16[13] * puSrc2->ai16[13]); + puDst->ai16[14] = RT_HIWORD(puSrc1->ai16[14] * puSrc2->ai16[14]); + puDst->ai16[15] = RT_HIWORD(puSrc1->ai16[15] * puSrc2->ai16[15]); +} + + +/* + * PMULHUW / VPMULHUW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhuw_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uDst.au16[0] = RT_HIWORD(uSrc1.au16[0] * uSrc2.au16[0]); + uDst.au16[1] = RT_HIWORD(uSrc1.au16[1] * uSrc2.au16[1]); + uDst.au16[2] = RT_HIWORD(uSrc1.au16[2] * uSrc2.au16[2]); + uDst.au16[3] = RT_HIWORD(uSrc1.au16[3] * uSrc2.au16[3]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhuw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + puDst->au16[0] = RT_HIWORD(uSrc1.au16[0] * puSrc->au16[0]); + puDst->au16[1] = RT_HIWORD(uSrc1.au16[1] * puSrc->au16[1]); + puDst->au16[2] = RT_HIWORD(uSrc1.au16[2] * puSrc->au16[2]); + puDst->au16[3] = RT_HIWORD(uSrc1.au16[3] * puSrc->au16[3]); + puDst->au16[4] = RT_HIWORD(uSrc1.au16[4] * puSrc->au16[4]); + puDst->au16[5] = RT_HIWORD(uSrc1.au16[5] * puSrc->au16[5]); + puDst->au16[6] = RT_HIWORD(uSrc1.au16[6] * puSrc->au16[6]); + puDst->au16[7] = RT_HIWORD(uSrc1.au16[7] * puSrc->au16[7]); +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhuw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->au16[0] = RT_HIWORD(puSrc1->au16[0] * puSrc2->au16[0]); + puDst->au16[1] = RT_HIWORD(puSrc1->au16[1] * puSrc2->au16[1]); + puDst->au16[2] = RT_HIWORD(puSrc1->au16[2] * puSrc2->au16[2]); + puDst->au16[3] = RT_HIWORD(puSrc1->au16[3] * puSrc2->au16[3]); + puDst->au16[4] = RT_HIWORD(puSrc1->au16[4] * puSrc2->au16[4]); + puDst->au16[5] = RT_HIWORD(puSrc1->au16[5] * puSrc2->au16[5]); + puDst->au16[6] = RT_HIWORD(puSrc1->au16[6] * puSrc2->au16[6]); + puDst->au16[7] = RT_HIWORD(puSrc1->au16[7] * puSrc2->au16[7]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhuw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->au16[ 0] = RT_HIWORD(puSrc1->au16[ 0] * puSrc2->au16[ 0]); + puDst->au16[ 1] = RT_HIWORD(puSrc1->au16[ 1] * puSrc2->au16[ 1]); + puDst->au16[ 2] = RT_HIWORD(puSrc1->au16[ 2] * puSrc2->au16[ 2]); + puDst->au16[ 3] = RT_HIWORD(puSrc1->au16[ 3] * puSrc2->au16[ 3]); + puDst->au16[ 4] = RT_HIWORD(puSrc1->au16[ 4] * puSrc2->au16[ 4]); + puDst->au16[ 5] = RT_HIWORD(puSrc1->au16[ 5] * puSrc2->au16[ 5]); + puDst->au16[ 6] = RT_HIWORD(puSrc1->au16[ 6] * puSrc2->au16[ 6]); + puDst->au16[ 7] = RT_HIWORD(puSrc1->au16[ 7] * puSrc2->au16[ 7]); + puDst->au16[ 8] = RT_HIWORD(puSrc1->au16[ 8] * puSrc2->au16[ 8]); + puDst->au16[ 9] = RT_HIWORD(puSrc1->au16[ 9] * puSrc2->au16[ 9]); + puDst->au16[10] = RT_HIWORD(puSrc1->au16[10] * puSrc2->au16[10]); + puDst->au16[11] = RT_HIWORD(puSrc1->au16[11] * puSrc2->au16[11]); + puDst->au16[12] = RT_HIWORD(puSrc1->au16[12] * puSrc2->au16[12]); + puDst->au16[13] = RT_HIWORD(puSrc1->au16[13] * puSrc2->au16[13]); + puDst->au16[14] = RT_HIWORD(puSrc1->au16[14] * puSrc2->au16[14]); + puDst->au16[15] = RT_HIWORD(puSrc1->au16[15] * puSrc2->au16[15]); +} + + +/* + * PSRLW / VPSRLW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrlw_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + if (uSrc2.au64[0] <= 15) + { + uDst.au16[0] = uSrc1.au16[0] >> uSrc2.au8[0]; + uDst.au16[1] = uSrc1.au16[1] >> uSrc2.au8[0]; + uDst.au16[2] = uSrc1.au16[2] >> uSrc2.au8[0]; + uDst.au16[3] = uSrc1.au16[3] >> uSrc2.au8[0]; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrlw_imm_u64,(uint64_t *puDst, uint8_t uShift)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uDst; + + if (uShift <= 15) + { + uDst.au16[0] = uSrc1.au16[0] >> uShift; + uDst.au16[1] = uSrc1.au16[1] >> uShift; + uDst.au16[2] = uSrc1.au16[2] >> uShift; + uDst.au16[3] = uSrc1.au16[3] >> uShift; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrlw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + if (puSrc->au64[0] <= 15) + { + puDst->au16[0] = uSrc1.au16[0] >> puSrc->au8[0]; + puDst->au16[1] = uSrc1.au16[1] >> puSrc->au8[0]; + puDst->au16[2] = uSrc1.au16[2] >> puSrc->au8[0]; + puDst->au16[3] = uSrc1.au16[3] >> puSrc->au8[0]; + puDst->au16[4] = uSrc1.au16[4] >> puSrc->au8[0]; + puDst->au16[5] = uSrc1.au16[5] >> puSrc->au8[0]; + puDst->au16[6] = uSrc1.au16[6] >> puSrc->au8[0]; + puDst->au16[7] = uSrc1.au16[7] >> puSrc->au8[0]; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrlw_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift <= 15) + { + puDst->au16[0] = uSrc1.au16[0] >> uShift; + puDst->au16[1] = uSrc1.au16[1] >> uShift; + puDst->au16[2] = uSrc1.au16[2] >> uShift; + puDst->au16[3] = uSrc1.au16[3] >> uShift; + puDst->au16[4] = uSrc1.au16[4] >> uShift; + puDst->au16[5] = uSrc1.au16[5] >> uShift; + puDst->au16[6] = uSrc1.au16[6] >> uShift; + puDst->au16[7] = uSrc1.au16[7] >> uShift; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PSRAW / VPSRAW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psraw_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + if (uSrc2.au64[0] <= 15) + { + uDst.ai16[0] = uSrc1.ai16[0] >> uSrc2.au8[0]; + uDst.ai16[1] = uSrc1.ai16[1] >> uSrc2.au8[0]; + uDst.ai16[2] = uSrc1.ai16[2] >> uSrc2.au8[0]; + uDst.ai16[3] = uSrc1.ai16[3] >> uSrc2.au8[0]; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psraw_imm_u64,(uint64_t *puDst, uint8_t uShift)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uDst; + + if (uShift <= 15) + { + uDst.ai16[0] = uSrc1.ai16[0] >> uShift; + uDst.ai16[1] = uSrc1.ai16[1] >> uShift; + uDst.ai16[2] = uSrc1.ai16[2] >> uShift; + uDst.ai16[3] = uSrc1.ai16[3] >> uShift; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psraw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + if (puSrc->au64[0] <= 15) + { + puDst->ai16[0] = uSrc1.ai16[0] >> puSrc->au8[0]; + puDst->ai16[1] = uSrc1.ai16[1] >> puSrc->au8[0]; + puDst->ai16[2] = uSrc1.ai16[2] >> puSrc->au8[0]; + puDst->ai16[3] = uSrc1.ai16[3] >> puSrc->au8[0]; + puDst->ai16[4] = uSrc1.ai16[4] >> puSrc->au8[0]; + puDst->ai16[5] = uSrc1.ai16[5] >> puSrc->au8[0]; + puDst->ai16[6] = uSrc1.ai16[6] >> puSrc->au8[0]; + puDst->ai16[7] = uSrc1.ai16[7] >> puSrc->au8[0]; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_psraw_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift <= 15) + { + puDst->ai16[0] = uSrc1.ai16[0] >> uShift; + puDst->ai16[1] = uSrc1.ai16[1] >> uShift; + puDst->ai16[2] = uSrc1.ai16[2] >> uShift; + puDst->ai16[3] = uSrc1.ai16[3] >> uShift; + puDst->ai16[4] = uSrc1.ai16[4] >> uShift; + puDst->ai16[5] = uSrc1.ai16[5] >> uShift; + puDst->ai16[6] = uSrc1.ai16[6] >> uShift; + puDst->ai16[7] = uSrc1.ai16[7] >> uShift; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PSLLW / VPSLLW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psllw_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + if (uSrc2.au64[0] <= 15) + { + uDst.au16[0] = uSrc1.au16[0] << uSrc2.au8[0]; + uDst.au16[1] = uSrc1.au16[1] << uSrc2.au8[0]; + uDst.au16[2] = uSrc1.au16[2] << uSrc2.au8[0]; + uDst.au16[3] = uSrc1.au16[3] << uSrc2.au8[0]; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psllw_imm_u64,(uint64_t *puDst, uint8_t uShift)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uDst; + + if (uShift <= 15) + { + uDst.au16[0] = uSrc1.au16[0] << uShift; + uDst.au16[1] = uSrc1.au16[1] << uShift; + uDst.au16[2] = uSrc1.au16[2] << uShift; + uDst.au16[3] = uSrc1.au16[3] << uShift; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psllw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + if (puSrc->au64[0] <= 15) + { + puDst->au16[0] = uSrc1.au16[0] << puSrc->au8[0]; + puDst->au16[1] = uSrc1.au16[1] << puSrc->au8[0]; + puDst->au16[2] = uSrc1.au16[2] << puSrc->au8[0]; + puDst->au16[3] = uSrc1.au16[3] << puSrc->au8[0]; + puDst->au16[4] = uSrc1.au16[4] << puSrc->au8[0]; + puDst->au16[5] = uSrc1.au16[5] << puSrc->au8[0]; + puDst->au16[6] = uSrc1.au16[6] << puSrc->au8[0]; + puDst->au16[7] = uSrc1.au16[7] << puSrc->au8[0]; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_psllw_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift <= 15) + { + puDst->au16[0] = uSrc1.au16[0] << uShift; + puDst->au16[1] = uSrc1.au16[1] << uShift; + puDst->au16[2] = uSrc1.au16[2] << uShift; + puDst->au16[3] = uSrc1.au16[3] << uShift; + puDst->au16[4] = uSrc1.au16[4] << uShift; + puDst->au16[5] = uSrc1.au16[5] << uShift; + puDst->au16[6] = uSrc1.au16[6] << uShift; + puDst->au16[7] = uSrc1.au16[7] << uShift; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PSRLD / VPSRLD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrld_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + if (uSrc2.au64[0] <= 31) + { + uDst.au32[0] = uSrc1.au32[0] >> uSrc2.au8[0]; + uDst.au32[1] = uSrc1.au32[1] >> uSrc2.au8[0]; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrld_imm_u64,(uint64_t *puDst, uint8_t uShift)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uDst; + + if (uShift <= 31) + { + uDst.au32[0] = uSrc1.au32[0] >> uShift; + uDst.au32[1] = uSrc1.au32[1] >> uShift; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrld_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + if (puSrc->au64[0] <= 31) + { + puDst->au32[0] = uSrc1.au32[0] >> puSrc->au8[0]; + puDst->au32[1] = uSrc1.au32[1] >> puSrc->au8[0]; + puDst->au32[2] = uSrc1.au32[2] >> puSrc->au8[0]; + puDst->au32[3] = uSrc1.au32[3] >> puSrc->au8[0]; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrld_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift <= 31) + { + puDst->au32[0] = uSrc1.au32[0] >> uShift; + puDst->au32[1] = uSrc1.au32[1] >> uShift; + puDst->au32[2] = uSrc1.au32[2] >> uShift; + puDst->au32[3] = uSrc1.au32[3] >> uShift; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PSRAD / VPSRAD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrad_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + if (uSrc2.au64[0] <= 31) + { + uDst.ai32[0] = uSrc1.ai32[0] >> uSrc2.au8[0]; + uDst.ai32[1] = uSrc1.ai32[1] >> uSrc2.au8[0]; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrad_imm_u64,(uint64_t *puDst, uint8_t uShift)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uDst; + + if (uShift <= 31) + { + uDst.ai32[0] = uSrc1.ai32[0] >> uShift; + uDst.ai32[1] = uSrc1.ai32[1] >> uShift; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrad_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + if (puSrc->au64[0] <= 31) + { + puDst->ai32[0] = uSrc1.ai32[0] >> puSrc->au8[0]; + puDst->ai32[1] = uSrc1.ai32[1] >> puSrc->au8[0]; + puDst->ai32[2] = uSrc1.ai32[2] >> puSrc->au8[0]; + puDst->ai32[3] = uSrc1.ai32[3] >> puSrc->au8[0]; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrad_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift <= 31) + { + puDst->ai32[0] = uSrc1.ai32[0] >> uShift; + puDst->ai32[1] = uSrc1.ai32[1] >> uShift; + puDst->ai32[2] = uSrc1.ai32[2] >> uShift; + puDst->ai32[3] = uSrc1.ai32[3] >> uShift; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PSLLD / VPSLLD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pslld_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + if (uSrc2.au64[0] <= 31) + { + uDst.au32[0] = uSrc1.au32[0] << uSrc2.au8[0]; + uDst.au32[1] = uSrc1.au32[1] << uSrc2.au8[0]; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pslld_imm_u64,(uint64_t *puDst, uint8_t uShift)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uDst; + + if (uShift <= 31) + { + uDst.au32[0] = uSrc1.au32[0] << uShift; + uDst.au32[1] = uSrc1.au32[1] << uShift; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pslld_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + if (puSrc->au64[0] <= 31) + { + puDst->au32[0] = uSrc1.au32[0] << puSrc->au8[0]; + puDst->au32[1] = uSrc1.au32[1] << puSrc->au8[0]; + puDst->au32[2] = uSrc1.au32[2] << puSrc->au8[0]; + puDst->au32[3] = uSrc1.au32[3] << puSrc->au8[0]; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_pslld_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift <= 31) + { + puDst->au32[0] = uSrc1.au32[0] << uShift; + puDst->au32[1] = uSrc1.au32[1] << uShift; + puDst->au32[2] = uSrc1.au32[2] << uShift; + puDst->au32[3] = uSrc1.au32[3] << uShift; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PSRLQ / VPSRLQ + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrlq_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + if (uSrc2.au64[0] <= 63) + { + uDst.au64[0] = uSrc1.au64[0] >> uSrc2.au8[0]; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrlq_imm_u64,(uint64_t *puDst, uint8_t uShift)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uDst; + + if (uShift <= 63) + { + uDst.au64[0] = uSrc1.au64[0] >> uShift; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrlq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + if (puSrc->au64[0] <= 63) + { + puDst->au64[0] = uSrc1.au64[0] >> puSrc->au8[0]; + puDst->au64[1] = uSrc1.au64[1] >> puSrc->au8[0]; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrlq_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift <= 63) + { + puDst->au64[0] = uSrc1.au64[0] >> uShift; + puDst->au64[1] = uSrc1.au64[1] >> uShift; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PSLLQ / VPSLLQ + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psllq_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + if (uSrc2.au64[0] <= 63) + { + uDst.au64[0] = uSrc1.au64[0] << uSrc2.au8[0]; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psllq_imm_u64,(uint64_t *puDst, uint8_t uShift)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uDst; + + if (uShift <= 63) + { + uDst.au64[0] = uSrc1.au64[0] << uShift; + } + else + { + uDst.au64[0] = 0; + } + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psllq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + if (puSrc->au64[0] <= 63) + { + puDst->au64[0] = uSrc1.au64[0] << puSrc->au8[0]; + puDst->au64[1] = uSrc1.au64[1] << puSrc->au8[0]; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_psllq_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift <= 63) + { + puDst->au64[0] = uSrc1.au64[0] << uShift; + puDst->au64[1] = uSrc1.au64[1] << uShift; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PSRLDQ / VPSRLDQ + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psrldq_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift < 16) + { + int i; + + for (i = 0; i < 16 - uShift; ++i) + puDst->au8[i] = uSrc1.au8[i + uShift]; + for (i = 16 - uShift; i < 16; ++i) + puDst->au8[i] = 0; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PSLLDQ / VPSLLDQ + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pslldq_imm_u128,(PRTUINT128U puDst, uint8_t uShift)) +{ + RTUINT128U uSrc1 = *puDst; + + if (uShift < 16) + { + int i; + + for (i = 0; i < uShift; ++i) + puDst->au8[i] = 0; + for (i = uShift; i < 16; ++i) + puDst->au8[i] = uSrc1.au8[i - uShift]; + } + else + { + puDst->au64[0] = 0; + puDst->au64[1] = 0; + } +} + +#endif + + +/* + * PMADDWD / VPMADDWD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaddwd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + uDst.ai32[0] = (int32_t)uSrc1.ai16[0] * uSrc2.ai16[0] + (int32_t)uSrc1.ai16[1] * uSrc2.ai16[1]; + uDst.ai32[1] = (int32_t)uSrc1.ai16[2] * uSrc2.ai16[2] + (int32_t)uSrc1.ai16[3] * uSrc2.ai16[3]; + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaddwd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai32[0] = (int32_t)uSrc1.ai16[0] * puSrc->ai16[0] + (int32_t)uSrc1.ai16[1] * puSrc->ai16[1]; + puDst->ai32[1] = (int32_t)uSrc1.ai16[2] * puSrc->ai16[2] + (int32_t)uSrc1.ai16[3] * puSrc->ai16[3]; + puDst->ai32[2] = (int32_t)uSrc1.ai16[4] * puSrc->ai16[4] + (int32_t)uSrc1.ai16[5] * puSrc->ai16[5]; + puDst->ai32[3] = (int32_t)uSrc1.ai16[6] * puSrc->ai16[6] + (int32_t)uSrc1.ai16[7] * puSrc->ai16[7]; + RT_NOREF(pFpuState); +} + +#endif + + +/* + * PMAXUB / VPMAXUB / PMAXUW / VPMAXUW / PMAXUD / VPMAXUD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxub_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + uDst.au8[0] = RT_MAX(uSrc1.au8[0], uSrc2.au8[0]); + uDst.au8[1] = RT_MAX(uSrc1.au8[1], uSrc2.au8[1]); + uDst.au8[2] = RT_MAX(uSrc1.au8[2], uSrc2.au8[2]); + uDst.au8[3] = RT_MAX(uSrc1.au8[3], uSrc2.au8[3]); + uDst.au8[4] = RT_MAX(uSrc1.au8[4], uSrc2.au8[4]); + uDst.au8[5] = RT_MAX(uSrc1.au8[5], uSrc2.au8[5]); + uDst.au8[6] = RT_MAX(uSrc1.au8[6], uSrc2.au8[6]); + uDst.au8[7] = RT_MAX(uSrc1.au8[7], uSrc2.au8[7]); + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxub_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au8[ 0] = RT_MAX(uSrc1.au8[ 0], puSrc->au8[ 0]); + puDst->au8[ 1] = RT_MAX(uSrc1.au8[ 1], puSrc->au8[ 1]); + puDst->au8[ 2] = RT_MAX(uSrc1.au8[ 2], puSrc->au8[ 2]); + puDst->au8[ 3] = RT_MAX(uSrc1.au8[ 3], puSrc->au8[ 3]); + puDst->au8[ 4] = RT_MAX(uSrc1.au8[ 4], puSrc->au8[ 4]); + puDst->au8[ 5] = RT_MAX(uSrc1.au8[ 5], puSrc->au8[ 5]); + puDst->au8[ 6] = RT_MAX(uSrc1.au8[ 6], puSrc->au8[ 6]); + puDst->au8[ 7] = RT_MAX(uSrc1.au8[ 7], puSrc->au8[ 7]); + puDst->au8[ 8] = RT_MAX(uSrc1.au8[ 8], puSrc->au8[ 8]); + puDst->au8[ 9] = RT_MAX(uSrc1.au8[ 9], puSrc->au8[ 9]); + puDst->au8[10] = RT_MAX(uSrc1.au8[10], puSrc->au8[10]); + puDst->au8[11] = RT_MAX(uSrc1.au8[11], puSrc->au8[11]); + puDst->au8[12] = RT_MAX(uSrc1.au8[12], puSrc->au8[12]); + puDst->au8[13] = RT_MAX(uSrc1.au8[13], puSrc->au8[13]); + puDst->au8[14] = RT_MAX(uSrc1.au8[14], puSrc->au8[14]); + puDst->au8[15] = RT_MAX(uSrc1.au8[15], puSrc->au8[15]); + RT_NOREF(pFpuState); +} + +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxuw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au16[ 0] = RT_MAX(uSrc1.au16[ 0], puSrc->au16[ 0]); + puDst->au16[ 1] = RT_MAX(uSrc1.au16[ 1], puSrc->au16[ 1]); + puDst->au16[ 2] = RT_MAX(uSrc1.au16[ 2], puSrc->au16[ 2]); + puDst->au16[ 3] = RT_MAX(uSrc1.au16[ 3], puSrc->au16[ 3]); + puDst->au16[ 4] = RT_MAX(uSrc1.au16[ 4], puSrc->au16[ 4]); + puDst->au16[ 5] = RT_MAX(uSrc1.au16[ 5], puSrc->au16[ 5]); + puDst->au16[ 6] = RT_MAX(uSrc1.au16[ 6], puSrc->au16[ 6]); + puDst->au16[ 7] = RT_MAX(uSrc1.au16[ 7], puSrc->au16[ 7]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxud_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au32[ 0] = RT_MAX(uSrc1.au32[ 0], puSrc->au32[ 0]); + puDst->au32[ 1] = RT_MAX(uSrc1.au32[ 1], puSrc->au32[ 1]); + puDst->au32[ 2] = RT_MAX(uSrc1.au32[ 2], puSrc->au32[ 2]); + puDst->au32[ 3] = RT_MAX(uSrc1.au32[ 3], puSrc->au32[ 3]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxub_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->au8[ 0] = RT_MAX(puSrc1->au8[ 0], puSrc2->au8[ 0]); + puDst->au8[ 1] = RT_MAX(puSrc1->au8[ 1], puSrc2->au8[ 1]); + puDst->au8[ 2] = RT_MAX(puSrc1->au8[ 2], puSrc2->au8[ 2]); + puDst->au8[ 3] = RT_MAX(puSrc1->au8[ 3], puSrc2->au8[ 3]); + puDst->au8[ 4] = RT_MAX(puSrc1->au8[ 4], puSrc2->au8[ 4]); + puDst->au8[ 5] = RT_MAX(puSrc1->au8[ 5], puSrc2->au8[ 5]); + puDst->au8[ 6] = RT_MAX(puSrc1->au8[ 6], puSrc2->au8[ 6]); + puDst->au8[ 7] = RT_MAX(puSrc1->au8[ 7], puSrc2->au8[ 7]); + puDst->au8[ 8] = RT_MAX(puSrc1->au8[ 8], puSrc2->au8[ 8]); + puDst->au8[ 9] = RT_MAX(puSrc1->au8[ 9], puSrc2->au8[ 9]); + puDst->au8[10] = RT_MAX(puSrc1->au8[10], puSrc2->au8[10]); + puDst->au8[11] = RT_MAX(puSrc1->au8[11], puSrc2->au8[11]); + puDst->au8[12] = RT_MAX(puSrc1->au8[12], puSrc2->au8[12]); + puDst->au8[13] = RT_MAX(puSrc1->au8[13], puSrc2->au8[13]); + puDst->au8[14] = RT_MAX(puSrc1->au8[14], puSrc2->au8[14]); + puDst->au8[15] = RT_MAX(puSrc1->au8[15], puSrc2->au8[15]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxub_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->au8[ 0] = RT_MAX(puSrc1->au8[ 0], puSrc2->au8[ 0]); + puDst->au8[ 1] = RT_MAX(puSrc1->au8[ 1], puSrc2->au8[ 1]); + puDst->au8[ 2] = RT_MAX(puSrc1->au8[ 2], puSrc2->au8[ 2]); + puDst->au8[ 3] = RT_MAX(puSrc1->au8[ 3], puSrc2->au8[ 3]); + puDst->au8[ 4] = RT_MAX(puSrc1->au8[ 4], puSrc2->au8[ 4]); + puDst->au8[ 5] = RT_MAX(puSrc1->au8[ 5], puSrc2->au8[ 5]); + puDst->au8[ 6] = RT_MAX(puSrc1->au8[ 6], puSrc2->au8[ 6]); + puDst->au8[ 7] = RT_MAX(puSrc1->au8[ 7], puSrc2->au8[ 7]); + puDst->au8[ 8] = RT_MAX(puSrc1->au8[ 8], puSrc2->au8[ 8]); + puDst->au8[ 9] = RT_MAX(puSrc1->au8[ 9], puSrc2->au8[ 9]); + puDst->au8[10] = RT_MAX(puSrc1->au8[10], puSrc2->au8[10]); + puDst->au8[11] = RT_MAX(puSrc1->au8[11], puSrc2->au8[11]); + puDst->au8[12] = RT_MAX(puSrc1->au8[12], puSrc2->au8[12]); + puDst->au8[13] = RT_MAX(puSrc1->au8[13], puSrc2->au8[13]); + puDst->au8[14] = RT_MAX(puSrc1->au8[14], puSrc2->au8[14]); + puDst->au8[15] = RT_MAX(puSrc1->au8[15], puSrc2->au8[15]); + puDst->au8[16] = RT_MAX(puSrc1->au8[16], puSrc2->au8[16]); + puDst->au8[17] = RT_MAX(puSrc1->au8[17], puSrc2->au8[17]); + puDst->au8[18] = RT_MAX(puSrc1->au8[18], puSrc2->au8[18]); + puDst->au8[19] = RT_MAX(puSrc1->au8[19], puSrc2->au8[19]); + puDst->au8[20] = RT_MAX(puSrc1->au8[20], puSrc2->au8[20]); + puDst->au8[21] = RT_MAX(puSrc1->au8[21], puSrc2->au8[21]); + puDst->au8[22] = RT_MAX(puSrc1->au8[22], puSrc2->au8[22]); + puDst->au8[23] = RT_MAX(puSrc1->au8[23], puSrc2->au8[23]); + puDst->au8[24] = RT_MAX(puSrc1->au8[24], puSrc2->au8[24]); + puDst->au8[25] = RT_MAX(puSrc1->au8[25], puSrc2->au8[25]); + puDst->au8[26] = RT_MAX(puSrc1->au8[26], puSrc2->au8[26]); + puDst->au8[27] = RT_MAX(puSrc1->au8[27], puSrc2->au8[27]); + puDst->au8[28] = RT_MAX(puSrc1->au8[28], puSrc2->au8[28]); + puDst->au8[29] = RT_MAX(puSrc1->au8[29], puSrc2->au8[29]); + puDst->au8[30] = RT_MAX(puSrc1->au8[30], puSrc2->au8[30]); + puDst->au8[31] = RT_MAX(puSrc1->au8[31], puSrc2->au8[31]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxuw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->au16[ 0] = RT_MAX(puSrc1->au16[ 0], puSrc2->au16[ 0]); + puDst->au16[ 1] = RT_MAX(puSrc1->au16[ 1], puSrc2->au16[ 1]); + puDst->au16[ 2] = RT_MAX(puSrc1->au16[ 2], puSrc2->au16[ 2]); + puDst->au16[ 3] = RT_MAX(puSrc1->au16[ 3], puSrc2->au16[ 3]); + puDst->au16[ 4] = RT_MAX(puSrc1->au16[ 4], puSrc2->au16[ 4]); + puDst->au16[ 5] = RT_MAX(puSrc1->au16[ 5], puSrc2->au16[ 5]); + puDst->au16[ 6] = RT_MAX(puSrc1->au16[ 6], puSrc2->au16[ 6]); + puDst->au16[ 7] = RT_MAX(puSrc1->au16[ 7], puSrc2->au16[ 7]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxuw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->au16[ 0] = RT_MAX(puSrc1->au16[ 0], puSrc2->au16[ 0]); + puDst->au16[ 1] = RT_MAX(puSrc1->au16[ 1], puSrc2->au16[ 1]); + puDst->au16[ 2] = RT_MAX(puSrc1->au16[ 2], puSrc2->au16[ 2]); + puDst->au16[ 3] = RT_MAX(puSrc1->au16[ 3], puSrc2->au16[ 3]); + puDst->au16[ 4] = RT_MAX(puSrc1->au16[ 4], puSrc2->au16[ 4]); + puDst->au16[ 5] = RT_MAX(puSrc1->au16[ 5], puSrc2->au16[ 5]); + puDst->au16[ 6] = RT_MAX(puSrc1->au16[ 6], puSrc2->au16[ 6]); + puDst->au16[ 7] = RT_MAX(puSrc1->au16[ 7], puSrc2->au16[ 7]); + puDst->au16[ 8] = RT_MAX(puSrc1->au16[ 8], puSrc2->au16[ 8]); + puDst->au16[ 9] = RT_MAX(puSrc1->au16[ 9], puSrc2->au16[ 9]); + puDst->au16[10] = RT_MAX(puSrc1->au16[10], puSrc2->au16[10]); + puDst->au16[11] = RT_MAX(puSrc1->au16[11], puSrc2->au16[11]); + puDst->au16[12] = RT_MAX(puSrc1->au16[12], puSrc2->au16[12]); + puDst->au16[13] = RT_MAX(puSrc1->au16[13], puSrc2->au16[13]); + puDst->au16[14] = RT_MAX(puSrc1->au16[14], puSrc2->au16[14]); + puDst->au16[15] = RT_MAX(puSrc1->au16[15], puSrc2->au16[15]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxud_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->au32[ 0] = RT_MAX(puSrc1->au32[ 0], puSrc2->au32[ 0]); + puDst->au32[ 1] = RT_MAX(puSrc1->au32[ 1], puSrc2->au32[ 1]); + puDst->au32[ 2] = RT_MAX(puSrc1->au32[ 2], puSrc2->au32[ 2]); + puDst->au32[ 3] = RT_MAX(puSrc1->au32[ 3], puSrc2->au32[ 3]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxud_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->au32[ 0] = RT_MAX(puSrc1->au32[ 0], puSrc2->au32[ 0]); + puDst->au32[ 1] = RT_MAX(puSrc1->au32[ 1], puSrc2->au32[ 1]); + puDst->au32[ 2] = RT_MAX(puSrc1->au32[ 2], puSrc2->au32[ 2]); + puDst->au32[ 3] = RT_MAX(puSrc1->au32[ 3], puSrc2->au32[ 3]); + puDst->au32[ 4] = RT_MAX(puSrc1->au32[ 4], puSrc2->au32[ 4]); + puDst->au32[ 5] = RT_MAX(puSrc1->au32[ 5], puSrc2->au32[ 5]); + puDst->au32[ 6] = RT_MAX(puSrc1->au32[ 6], puSrc2->au32[ 6]); + puDst->au32[ 7] = RT_MAX(puSrc1->au32[ 7], puSrc2->au32[ 7]); + RT_NOREF(pExtState); +} + + +/* + * PMAXSB / VPMAXSB / PMAXSW / VPMAXSW / PMAXSD / VPMAXSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxsw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + uDst.ai16[0] = RT_MAX(uSrc1.ai16[0], uSrc2.ai16[0]); + uDst.ai16[1] = RT_MAX(uSrc1.ai16[1], uSrc2.ai16[1]); + uDst.ai16[2] = RT_MAX(uSrc1.ai16[2], uSrc2.ai16[2]); + uDst.ai16[3] = RT_MAX(uSrc1.ai16[3], uSrc2.ai16[3]); + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxsw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai16[ 0] = RT_MAX(uSrc1.ai16[ 0], puSrc->ai16[ 0]); + puDst->ai16[ 1] = RT_MAX(uSrc1.ai16[ 1], puSrc->ai16[ 1]); + puDst->ai16[ 2] = RT_MAX(uSrc1.ai16[ 2], puSrc->ai16[ 2]); + puDst->ai16[ 3] = RT_MAX(uSrc1.ai16[ 3], puSrc->ai16[ 3]); + puDst->ai16[ 4] = RT_MAX(uSrc1.ai16[ 4], puSrc->ai16[ 4]); + puDst->ai16[ 5] = RT_MAX(uSrc1.ai16[ 5], puSrc->ai16[ 5]); + puDst->ai16[ 6] = RT_MAX(uSrc1.ai16[ 6], puSrc->ai16[ 6]); + puDst->ai16[ 7] = RT_MAX(uSrc1.ai16[ 7], puSrc->ai16[ 7]); + RT_NOREF(pFpuState); +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxsb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai8[ 0] = RT_MAX(uSrc1.ai8[ 0], puSrc->ai8[ 0]); + puDst->ai8[ 1] = RT_MAX(uSrc1.ai8[ 1], puSrc->ai8[ 1]); + puDst->ai8[ 2] = RT_MAX(uSrc1.ai8[ 2], puSrc->ai8[ 2]); + puDst->ai8[ 3] = RT_MAX(uSrc1.ai8[ 3], puSrc->ai8[ 3]); + puDst->ai8[ 4] = RT_MAX(uSrc1.ai8[ 4], puSrc->ai8[ 4]); + puDst->ai8[ 5] = RT_MAX(uSrc1.ai8[ 5], puSrc->ai8[ 5]); + puDst->ai8[ 6] = RT_MAX(uSrc1.ai8[ 6], puSrc->ai8[ 6]); + puDst->ai8[ 7] = RT_MAX(uSrc1.ai8[ 7], puSrc->ai8[ 7]); + puDst->ai8[ 8] = RT_MAX(uSrc1.ai8[ 8], puSrc->ai8[ 8]); + puDst->ai8[ 9] = RT_MAX(uSrc1.ai8[ 9], puSrc->ai8[ 9]); + puDst->ai8[10] = RT_MAX(uSrc1.ai8[10], puSrc->ai8[10]); + puDst->ai8[11] = RT_MAX(uSrc1.ai8[11], puSrc->ai8[11]); + puDst->ai8[12] = RT_MAX(uSrc1.ai8[12], puSrc->ai8[12]); + puDst->ai8[13] = RT_MAX(uSrc1.ai8[13], puSrc->ai8[13]); + puDst->ai8[14] = RT_MAX(uSrc1.ai8[14], puSrc->ai8[14]); + puDst->ai8[15] = RT_MAX(uSrc1.ai8[15], puSrc->ai8[15]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxsd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai32[ 0] = RT_MAX(uSrc1.ai32[ 0], puSrc->ai32[ 0]); + puDst->ai32[ 1] = RT_MAX(uSrc1.ai32[ 1], puSrc->ai32[ 1]); + puDst->ai32[ 2] = RT_MAX(uSrc1.ai32[ 2], puSrc->ai32[ 2]); + puDst->ai32[ 3] = RT_MAX(uSrc1.ai32[ 3], puSrc->ai32[ 3]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->ai8[ 0] = RT_MAX(puSrc1->ai8[ 0], puSrc2->ai8[ 0]); + puDst->ai8[ 1] = RT_MAX(puSrc1->ai8[ 1], puSrc2->ai8[ 1]); + puDst->ai8[ 2] = RT_MAX(puSrc1->ai8[ 2], puSrc2->ai8[ 2]); + puDst->ai8[ 3] = RT_MAX(puSrc1->ai8[ 3], puSrc2->ai8[ 3]); + puDst->ai8[ 4] = RT_MAX(puSrc1->ai8[ 4], puSrc2->ai8[ 4]); + puDst->ai8[ 5] = RT_MAX(puSrc1->ai8[ 5], puSrc2->ai8[ 5]); + puDst->ai8[ 6] = RT_MAX(puSrc1->ai8[ 6], puSrc2->ai8[ 6]); + puDst->ai8[ 7] = RT_MAX(puSrc1->ai8[ 7], puSrc2->ai8[ 7]); + puDst->ai8[ 8] = RT_MAX(puSrc1->ai8[ 8], puSrc2->ai8[ 8]); + puDst->ai8[ 9] = RT_MAX(puSrc1->ai8[ 9], puSrc2->ai8[ 9]); + puDst->ai8[10] = RT_MAX(puSrc1->ai8[10], puSrc2->ai8[10]); + puDst->ai8[11] = RT_MAX(puSrc1->ai8[11], puSrc2->ai8[11]); + puDst->ai8[12] = RT_MAX(puSrc1->ai8[12], puSrc2->ai8[12]); + puDst->ai8[13] = RT_MAX(puSrc1->ai8[13], puSrc2->ai8[13]); + puDst->ai8[14] = RT_MAX(puSrc1->ai8[14], puSrc2->ai8[14]); + puDst->ai8[15] = RT_MAX(puSrc1->ai8[15], puSrc2->ai8[15]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->ai8[ 0] = RT_MAX(puSrc1->ai8[ 0], puSrc2->ai8[ 0]); + puDst->ai8[ 1] = RT_MAX(puSrc1->ai8[ 1], puSrc2->ai8[ 1]); + puDst->ai8[ 2] = RT_MAX(puSrc1->ai8[ 2], puSrc2->ai8[ 2]); + puDst->ai8[ 3] = RT_MAX(puSrc1->ai8[ 3], puSrc2->ai8[ 3]); + puDst->ai8[ 4] = RT_MAX(puSrc1->ai8[ 4], puSrc2->ai8[ 4]); + puDst->ai8[ 5] = RT_MAX(puSrc1->ai8[ 5], puSrc2->ai8[ 5]); + puDst->ai8[ 6] = RT_MAX(puSrc1->ai8[ 6], puSrc2->ai8[ 6]); + puDst->ai8[ 7] = RT_MAX(puSrc1->ai8[ 7], puSrc2->ai8[ 7]); + puDst->ai8[ 8] = RT_MAX(puSrc1->ai8[ 8], puSrc2->ai8[ 8]); + puDst->ai8[ 9] = RT_MAX(puSrc1->ai8[ 9], puSrc2->ai8[ 9]); + puDst->ai8[10] = RT_MAX(puSrc1->ai8[10], puSrc2->ai8[10]); + puDst->ai8[11] = RT_MAX(puSrc1->ai8[11], puSrc2->ai8[11]); + puDst->ai8[12] = RT_MAX(puSrc1->ai8[12], puSrc2->ai8[12]); + puDst->ai8[13] = RT_MAX(puSrc1->ai8[13], puSrc2->ai8[13]); + puDst->ai8[14] = RT_MAX(puSrc1->ai8[14], puSrc2->ai8[14]); + puDst->ai8[15] = RT_MAX(puSrc1->ai8[15], puSrc2->ai8[15]); + puDst->ai8[16] = RT_MAX(puSrc1->ai8[16], puSrc2->ai8[16]); + puDst->ai8[17] = RT_MAX(puSrc1->ai8[17], puSrc2->ai8[17]); + puDst->ai8[18] = RT_MAX(puSrc1->ai8[18], puSrc2->ai8[18]); + puDst->ai8[19] = RT_MAX(puSrc1->ai8[19], puSrc2->ai8[19]); + puDst->ai8[20] = RT_MAX(puSrc1->ai8[20], puSrc2->ai8[20]); + puDst->ai8[21] = RT_MAX(puSrc1->ai8[21], puSrc2->ai8[21]); + puDst->ai8[22] = RT_MAX(puSrc1->ai8[22], puSrc2->ai8[22]); + puDst->ai8[23] = RT_MAX(puSrc1->ai8[23], puSrc2->ai8[23]); + puDst->ai8[24] = RT_MAX(puSrc1->ai8[24], puSrc2->ai8[24]); + puDst->ai8[25] = RT_MAX(puSrc1->ai8[25], puSrc2->ai8[25]); + puDst->ai8[26] = RT_MAX(puSrc1->ai8[26], puSrc2->ai8[26]); + puDst->ai8[27] = RT_MAX(puSrc1->ai8[27], puSrc2->ai8[27]); + puDst->ai8[28] = RT_MAX(puSrc1->ai8[28], puSrc2->ai8[28]); + puDst->ai8[29] = RT_MAX(puSrc1->ai8[29], puSrc2->ai8[29]); + puDst->ai8[30] = RT_MAX(puSrc1->ai8[30], puSrc2->ai8[30]); + puDst->ai8[31] = RT_MAX(puSrc1->ai8[31], puSrc2->ai8[31]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->ai16[ 0] = RT_MAX(puSrc1->ai16[ 0], puSrc2->ai16[ 0]); + puDst->ai16[ 1] = RT_MAX(puSrc1->ai16[ 1], puSrc2->ai16[ 1]); + puDst->ai16[ 2] = RT_MAX(puSrc1->ai16[ 2], puSrc2->ai16[ 2]); + puDst->ai16[ 3] = RT_MAX(puSrc1->ai16[ 3], puSrc2->ai16[ 3]); + puDst->ai16[ 4] = RT_MAX(puSrc1->ai16[ 4], puSrc2->ai16[ 4]); + puDst->ai16[ 5] = RT_MAX(puSrc1->ai16[ 5], puSrc2->ai16[ 5]); + puDst->ai16[ 6] = RT_MAX(puSrc1->ai16[ 6], puSrc2->ai16[ 6]); + puDst->ai16[ 7] = RT_MAX(puSrc1->ai16[ 7], puSrc2->ai16[ 7]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->ai16[ 0] = RT_MAX(puSrc1->ai16[ 0], puSrc2->ai16[ 0]); + puDst->ai16[ 1] = RT_MAX(puSrc1->ai16[ 1], puSrc2->ai16[ 1]); + puDst->ai16[ 2] = RT_MAX(puSrc1->ai16[ 2], puSrc2->ai16[ 2]); + puDst->ai16[ 3] = RT_MAX(puSrc1->ai16[ 3], puSrc2->ai16[ 3]); + puDst->ai16[ 4] = RT_MAX(puSrc1->ai16[ 4], puSrc2->ai16[ 4]); + puDst->ai16[ 5] = RT_MAX(puSrc1->ai16[ 5], puSrc2->ai16[ 5]); + puDst->ai16[ 6] = RT_MAX(puSrc1->ai16[ 6], puSrc2->ai16[ 6]); + puDst->ai16[ 7] = RT_MAX(puSrc1->ai16[ 7], puSrc2->ai16[ 7]); + puDst->ai16[ 8] = RT_MAX(puSrc1->ai16[ 8], puSrc2->ai16[ 8]); + puDst->ai16[ 9] = RT_MAX(puSrc1->ai16[ 9], puSrc2->ai16[ 9]); + puDst->ai16[10] = RT_MAX(puSrc1->ai16[10], puSrc2->ai16[10]); + puDst->ai16[11] = RT_MAX(puSrc1->ai16[11], puSrc2->ai16[11]); + puDst->ai16[12] = RT_MAX(puSrc1->ai16[12], puSrc2->ai16[12]); + puDst->ai16[13] = RT_MAX(puSrc1->ai16[13], puSrc2->ai16[13]); + puDst->ai16[14] = RT_MAX(puSrc1->ai16[14], puSrc2->ai16[14]); + puDst->ai16[15] = RT_MAX(puSrc1->ai16[15], puSrc2->ai16[15]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->ai32[ 0] = RT_MAX(puSrc1->ai32[ 0], puSrc2->ai32[ 0]); + puDst->ai32[ 1] = RT_MAX(puSrc1->ai32[ 1], puSrc2->ai32[ 1]); + puDst->ai32[ 2] = RT_MAX(puSrc1->ai32[ 2], puSrc2->ai32[ 2]); + puDst->ai32[ 3] = RT_MAX(puSrc1->ai32[ 3], puSrc2->ai32[ 3]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->ai32[ 0] = RT_MAX(puSrc1->ai32[ 0], puSrc2->ai32[ 0]); + puDst->ai32[ 1] = RT_MAX(puSrc1->ai32[ 1], puSrc2->ai32[ 1]); + puDst->ai32[ 2] = RT_MAX(puSrc1->ai32[ 2], puSrc2->ai32[ 2]); + puDst->ai32[ 3] = RT_MAX(puSrc1->ai32[ 3], puSrc2->ai32[ 3]); + puDst->ai32[ 4] = RT_MAX(puSrc1->ai32[ 4], puSrc2->ai32[ 4]); + puDst->ai32[ 5] = RT_MAX(puSrc1->ai32[ 5], puSrc2->ai32[ 5]); + puDst->ai32[ 6] = RT_MAX(puSrc1->ai32[ 6], puSrc2->ai32[ 6]); + puDst->ai32[ 7] = RT_MAX(puSrc1->ai32[ 7], puSrc2->ai32[ 7]); + RT_NOREF(pExtState); +} + + +/* + * PMINUB / VPMINUB / PMINUW / VPMINUW / PMINUD / VPMINUD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pminub_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + uDst.au8[0] = RT_MIN(uSrc1.au8[0], uSrc2.au8[0]); + uDst.au8[1] = RT_MIN(uSrc1.au8[1], uSrc2.au8[1]); + uDst.au8[2] = RT_MIN(uSrc1.au8[2], uSrc2.au8[2]); + uDst.au8[3] = RT_MIN(uSrc1.au8[3], uSrc2.au8[3]); + uDst.au8[4] = RT_MIN(uSrc1.au8[4], uSrc2.au8[4]); + uDst.au8[5] = RT_MIN(uSrc1.au8[5], uSrc2.au8[5]); + uDst.au8[6] = RT_MIN(uSrc1.au8[6], uSrc2.au8[6]); + uDst.au8[7] = RT_MIN(uSrc1.au8[7], uSrc2.au8[7]); + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pminub_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au8[ 0] = RT_MIN(uSrc1.au8[ 0], puSrc->au8[ 0]); + puDst->au8[ 1] = RT_MIN(uSrc1.au8[ 1], puSrc->au8[ 1]); + puDst->au8[ 2] = RT_MIN(uSrc1.au8[ 2], puSrc->au8[ 2]); + puDst->au8[ 3] = RT_MIN(uSrc1.au8[ 3], puSrc->au8[ 3]); + puDst->au8[ 4] = RT_MIN(uSrc1.au8[ 4], puSrc->au8[ 4]); + puDst->au8[ 5] = RT_MIN(uSrc1.au8[ 5], puSrc->au8[ 5]); + puDst->au8[ 6] = RT_MIN(uSrc1.au8[ 6], puSrc->au8[ 6]); + puDst->au8[ 7] = RT_MIN(uSrc1.au8[ 7], puSrc->au8[ 7]); + puDst->au8[ 8] = RT_MIN(uSrc1.au8[ 8], puSrc->au8[ 8]); + puDst->au8[ 9] = RT_MIN(uSrc1.au8[ 9], puSrc->au8[ 9]); + puDst->au8[10] = RT_MIN(uSrc1.au8[10], puSrc->au8[10]); + puDst->au8[11] = RT_MIN(uSrc1.au8[11], puSrc->au8[11]); + puDst->au8[12] = RT_MIN(uSrc1.au8[12], puSrc->au8[12]); + puDst->au8[13] = RT_MIN(uSrc1.au8[13], puSrc->au8[13]); + puDst->au8[14] = RT_MIN(uSrc1.au8[14], puSrc->au8[14]); + puDst->au8[15] = RT_MIN(uSrc1.au8[15], puSrc->au8[15]); + RT_NOREF(pFpuState); +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_pminuw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au16[ 0] = RT_MIN(uSrc1.au16[ 0], puSrc->au16[ 0]); + puDst->au16[ 1] = RT_MIN(uSrc1.au16[ 1], puSrc->au16[ 1]); + puDst->au16[ 2] = RT_MIN(uSrc1.au16[ 2], puSrc->au16[ 2]); + puDst->au16[ 3] = RT_MIN(uSrc1.au16[ 3], puSrc->au16[ 3]); + puDst->au16[ 4] = RT_MIN(uSrc1.au16[ 4], puSrc->au16[ 4]); + puDst->au16[ 5] = RT_MIN(uSrc1.au16[ 5], puSrc->au16[ 5]); + puDst->au16[ 6] = RT_MIN(uSrc1.au16[ 6], puSrc->au16[ 6]); + puDst->au16[ 7] = RT_MIN(uSrc1.au16[ 7], puSrc->au16[ 7]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pminud_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au32[ 0] = RT_MIN(uSrc1.au32[ 0], puSrc->au32[ 0]); + puDst->au32[ 1] = RT_MIN(uSrc1.au32[ 1], puSrc->au32[ 1]); + puDst->au32[ 2] = RT_MIN(uSrc1.au32[ 2], puSrc->au32[ 2]); + puDst->au32[ 3] = RT_MIN(uSrc1.au32[ 3], puSrc->au32[ 3]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminub_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->au8[ 0] = RT_MIN(puSrc1->au8[ 0], puSrc2->au8[ 0]); + puDst->au8[ 1] = RT_MIN(puSrc1->au8[ 1], puSrc2->au8[ 1]); + puDst->au8[ 2] = RT_MIN(puSrc1->au8[ 2], puSrc2->au8[ 2]); + puDst->au8[ 3] = RT_MIN(puSrc1->au8[ 3], puSrc2->au8[ 3]); + puDst->au8[ 4] = RT_MIN(puSrc1->au8[ 4], puSrc2->au8[ 4]); + puDst->au8[ 5] = RT_MIN(puSrc1->au8[ 5], puSrc2->au8[ 5]); + puDst->au8[ 6] = RT_MIN(puSrc1->au8[ 6], puSrc2->au8[ 6]); + puDst->au8[ 7] = RT_MIN(puSrc1->au8[ 7], puSrc2->au8[ 7]); + puDst->au8[ 8] = RT_MIN(puSrc1->au8[ 8], puSrc2->au8[ 8]); + puDst->au8[ 9] = RT_MIN(puSrc1->au8[ 9], puSrc2->au8[ 9]); + puDst->au8[10] = RT_MIN(puSrc1->au8[10], puSrc2->au8[10]); + puDst->au8[11] = RT_MIN(puSrc1->au8[11], puSrc2->au8[11]); + puDst->au8[12] = RT_MIN(puSrc1->au8[12], puSrc2->au8[12]); + puDst->au8[13] = RT_MIN(puSrc1->au8[13], puSrc2->au8[13]); + puDst->au8[14] = RT_MIN(puSrc1->au8[14], puSrc2->au8[14]); + puDst->au8[15] = RT_MIN(puSrc1->au8[15], puSrc2->au8[15]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminub_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->au8[ 0] = RT_MIN(puSrc1->au8[ 0], puSrc2->au8[ 0]); + puDst->au8[ 1] = RT_MIN(puSrc1->au8[ 1], puSrc2->au8[ 1]); + puDst->au8[ 2] = RT_MIN(puSrc1->au8[ 2], puSrc2->au8[ 2]); + puDst->au8[ 3] = RT_MIN(puSrc1->au8[ 3], puSrc2->au8[ 3]); + puDst->au8[ 4] = RT_MIN(puSrc1->au8[ 4], puSrc2->au8[ 4]); + puDst->au8[ 5] = RT_MIN(puSrc1->au8[ 5], puSrc2->au8[ 5]); + puDst->au8[ 6] = RT_MIN(puSrc1->au8[ 6], puSrc2->au8[ 6]); + puDst->au8[ 7] = RT_MIN(puSrc1->au8[ 7], puSrc2->au8[ 7]); + puDst->au8[ 8] = RT_MIN(puSrc1->au8[ 8], puSrc2->au8[ 8]); + puDst->au8[ 9] = RT_MIN(puSrc1->au8[ 9], puSrc2->au8[ 9]); + puDst->au8[10] = RT_MIN(puSrc1->au8[10], puSrc2->au8[10]); + puDst->au8[11] = RT_MIN(puSrc1->au8[11], puSrc2->au8[11]); + puDst->au8[12] = RT_MIN(puSrc1->au8[12], puSrc2->au8[12]); + puDst->au8[13] = RT_MIN(puSrc1->au8[13], puSrc2->au8[13]); + puDst->au8[14] = RT_MIN(puSrc1->au8[14], puSrc2->au8[14]); + puDst->au8[15] = RT_MIN(puSrc1->au8[15], puSrc2->au8[15]); + puDst->au8[16] = RT_MIN(puSrc1->au8[16], puSrc2->au8[16]); + puDst->au8[17] = RT_MIN(puSrc1->au8[17], puSrc2->au8[17]); + puDst->au8[18] = RT_MIN(puSrc1->au8[18], puSrc2->au8[18]); + puDst->au8[19] = RT_MIN(puSrc1->au8[19], puSrc2->au8[19]); + puDst->au8[20] = RT_MIN(puSrc1->au8[20], puSrc2->au8[20]); + puDst->au8[21] = RT_MIN(puSrc1->au8[21], puSrc2->au8[21]); + puDst->au8[22] = RT_MIN(puSrc1->au8[22], puSrc2->au8[22]); + puDst->au8[23] = RT_MIN(puSrc1->au8[23], puSrc2->au8[23]); + puDst->au8[24] = RT_MIN(puSrc1->au8[24], puSrc2->au8[24]); + puDst->au8[25] = RT_MIN(puSrc1->au8[25], puSrc2->au8[25]); + puDst->au8[26] = RT_MIN(puSrc1->au8[26], puSrc2->au8[26]); + puDst->au8[27] = RT_MIN(puSrc1->au8[27], puSrc2->au8[27]); + puDst->au8[28] = RT_MIN(puSrc1->au8[28], puSrc2->au8[28]); + puDst->au8[29] = RT_MIN(puSrc1->au8[29], puSrc2->au8[29]); + puDst->au8[30] = RT_MIN(puSrc1->au8[30], puSrc2->au8[30]); + puDst->au8[31] = RT_MIN(puSrc1->au8[31], puSrc2->au8[31]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminuw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->au16[ 0] = RT_MIN(puSrc1->au16[ 0], puSrc2->au16[ 0]); + puDst->au16[ 1] = RT_MIN(puSrc1->au16[ 1], puSrc2->au16[ 1]); + puDst->au16[ 2] = RT_MIN(puSrc1->au16[ 2], puSrc2->au16[ 2]); + puDst->au16[ 3] = RT_MIN(puSrc1->au16[ 3], puSrc2->au16[ 3]); + puDst->au16[ 4] = RT_MIN(puSrc1->au16[ 4], puSrc2->au16[ 4]); + puDst->au16[ 5] = RT_MIN(puSrc1->au16[ 5], puSrc2->au16[ 5]); + puDst->au16[ 6] = RT_MIN(puSrc1->au16[ 6], puSrc2->au16[ 6]); + puDst->au16[ 7] = RT_MIN(puSrc1->au16[ 7], puSrc2->au16[ 7]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminuw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->au16[ 0] = RT_MIN(puSrc1->au16[ 0], puSrc2->au16[ 0]); + puDst->au16[ 1] = RT_MIN(puSrc1->au16[ 1], puSrc2->au16[ 1]); + puDst->au16[ 2] = RT_MIN(puSrc1->au16[ 2], puSrc2->au16[ 2]); + puDst->au16[ 3] = RT_MIN(puSrc1->au16[ 3], puSrc2->au16[ 3]); + puDst->au16[ 4] = RT_MIN(puSrc1->au16[ 4], puSrc2->au16[ 4]); + puDst->au16[ 5] = RT_MIN(puSrc1->au16[ 5], puSrc2->au16[ 5]); + puDst->au16[ 6] = RT_MIN(puSrc1->au16[ 6], puSrc2->au16[ 6]); + puDst->au16[ 7] = RT_MIN(puSrc1->au16[ 7], puSrc2->au16[ 7]); + puDst->au16[ 8] = RT_MIN(puSrc1->au16[ 8], puSrc2->au16[ 8]); + puDst->au16[ 9] = RT_MIN(puSrc1->au16[ 9], puSrc2->au16[ 9]); + puDst->au16[10] = RT_MIN(puSrc1->au16[10], puSrc2->au16[10]); + puDst->au16[11] = RT_MIN(puSrc1->au16[11], puSrc2->au16[11]); + puDst->au16[12] = RT_MIN(puSrc1->au16[12], puSrc2->au16[12]); + puDst->au16[13] = RT_MIN(puSrc1->au16[13], puSrc2->au16[13]); + puDst->au16[14] = RT_MIN(puSrc1->au16[14], puSrc2->au16[14]); + puDst->au16[15] = RT_MIN(puSrc1->au16[15], puSrc2->au16[15]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminud_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->au32[ 0] = RT_MIN(puSrc1->au32[ 0], puSrc2->au32[ 0]); + puDst->au32[ 1] = RT_MIN(puSrc1->au32[ 1], puSrc2->au32[ 1]); + puDst->au32[ 2] = RT_MIN(puSrc1->au32[ 2], puSrc2->au32[ 2]); + puDst->au32[ 3] = RT_MIN(puSrc1->au32[ 3], puSrc2->au32[ 3]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminud_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->au32[ 0] = RT_MIN(puSrc1->au32[ 0], puSrc2->au32[ 0]); + puDst->au32[ 1] = RT_MIN(puSrc1->au32[ 1], puSrc2->au32[ 1]); + puDst->au32[ 2] = RT_MIN(puSrc1->au32[ 2], puSrc2->au32[ 2]); + puDst->au32[ 3] = RT_MIN(puSrc1->au32[ 3], puSrc2->au32[ 3]); + puDst->au32[ 4] = RT_MIN(puSrc1->au32[ 4], puSrc2->au32[ 4]); + puDst->au32[ 5] = RT_MIN(puSrc1->au32[ 5], puSrc2->au32[ 5]); + puDst->au32[ 6] = RT_MIN(puSrc1->au32[ 6], puSrc2->au32[ 6]); + puDst->au32[ 7] = RT_MIN(puSrc1->au32[ 7], puSrc2->au32[ 7]); + RT_NOREF(pExtState); +} + + +/* + * PMINSB / VPMINSB / PMINSW / VPMINSW / PMINSD / VPMINSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pminsw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + uDst.ai16[0] = RT_MIN(uSrc1.ai16[0], uSrc2.ai16[0]); + uDst.ai16[1] = RT_MIN(uSrc1.ai16[1], uSrc2.ai16[1]); + uDst.ai16[2] = RT_MIN(uSrc1.ai16[2], uSrc2.ai16[2]); + uDst.ai16[3] = RT_MIN(uSrc1.ai16[3], uSrc2.ai16[3]); + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pminsw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai16[ 0] = RT_MIN(uSrc1.ai16[ 0], puSrc->ai16[ 0]); + puDst->ai16[ 1] = RT_MIN(uSrc1.ai16[ 1], puSrc->ai16[ 1]); + puDst->ai16[ 2] = RT_MIN(uSrc1.ai16[ 2], puSrc->ai16[ 2]); + puDst->ai16[ 3] = RT_MIN(uSrc1.ai16[ 3], puSrc->ai16[ 3]); + puDst->ai16[ 4] = RT_MIN(uSrc1.ai16[ 4], puSrc->ai16[ 4]); + puDst->ai16[ 5] = RT_MIN(uSrc1.ai16[ 5], puSrc->ai16[ 5]); + puDst->ai16[ 6] = RT_MIN(uSrc1.ai16[ 6], puSrc->ai16[ 6]); + puDst->ai16[ 7] = RT_MIN(uSrc1.ai16[ 7], puSrc->ai16[ 7]); + RT_NOREF(pFpuState); +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_pminsb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai8[ 0] = RT_MIN(uSrc1.ai8[ 0], puSrc->ai8[ 0]); + puDst->ai8[ 1] = RT_MIN(uSrc1.ai8[ 1], puSrc->ai8[ 1]); + puDst->ai8[ 2] = RT_MIN(uSrc1.ai8[ 2], puSrc->ai8[ 2]); + puDst->ai8[ 3] = RT_MIN(uSrc1.ai8[ 3], puSrc->ai8[ 3]); + puDst->ai8[ 4] = RT_MIN(uSrc1.ai8[ 4], puSrc->ai8[ 4]); + puDst->ai8[ 5] = RT_MIN(uSrc1.ai8[ 5], puSrc->ai8[ 5]); + puDst->ai8[ 6] = RT_MIN(uSrc1.ai8[ 6], puSrc->ai8[ 6]); + puDst->ai8[ 7] = RT_MIN(uSrc1.ai8[ 7], puSrc->ai8[ 7]); + puDst->ai8[ 8] = RT_MIN(uSrc1.ai8[ 8], puSrc->ai8[ 8]); + puDst->ai8[ 9] = RT_MIN(uSrc1.ai8[ 9], puSrc->ai8[ 9]); + puDst->ai8[10] = RT_MIN(uSrc1.ai8[10], puSrc->ai8[10]); + puDst->ai8[11] = RT_MIN(uSrc1.ai8[11], puSrc->ai8[11]); + puDst->ai8[12] = RT_MIN(uSrc1.ai8[12], puSrc->ai8[12]); + puDst->ai8[13] = RT_MIN(uSrc1.ai8[13], puSrc->ai8[13]); + puDst->ai8[14] = RT_MIN(uSrc1.ai8[14], puSrc->ai8[14]); + puDst->ai8[15] = RT_MIN(uSrc1.ai8[15], puSrc->ai8[15]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pminsd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai32[ 0] = RT_MIN(uSrc1.ai32[ 0], puSrc->ai32[ 0]); + puDst->ai32[ 1] = RT_MIN(uSrc1.ai32[ 1], puSrc->ai32[ 1]); + puDst->ai32[ 2] = RT_MIN(uSrc1.ai32[ 2], puSrc->ai32[ 2]); + puDst->ai32[ 3] = RT_MIN(uSrc1.ai32[ 3], puSrc->ai32[ 3]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->ai8[ 0] = RT_MIN(puSrc1->ai8[ 0], puSrc2->ai8[ 0]); + puDst->ai8[ 1] = RT_MIN(puSrc1->ai8[ 1], puSrc2->ai8[ 1]); + puDst->ai8[ 2] = RT_MIN(puSrc1->ai8[ 2], puSrc2->ai8[ 2]); + puDst->ai8[ 3] = RT_MIN(puSrc1->ai8[ 3], puSrc2->ai8[ 3]); + puDst->ai8[ 4] = RT_MIN(puSrc1->ai8[ 4], puSrc2->ai8[ 4]); + puDst->ai8[ 5] = RT_MIN(puSrc1->ai8[ 5], puSrc2->ai8[ 5]); + puDst->ai8[ 6] = RT_MIN(puSrc1->ai8[ 6], puSrc2->ai8[ 6]); + puDst->ai8[ 7] = RT_MIN(puSrc1->ai8[ 7], puSrc2->ai8[ 7]); + puDst->ai8[ 8] = RT_MIN(puSrc1->ai8[ 8], puSrc2->ai8[ 8]); + puDst->ai8[ 9] = RT_MIN(puSrc1->ai8[ 9], puSrc2->ai8[ 9]); + puDst->ai8[10] = RT_MIN(puSrc1->ai8[10], puSrc2->ai8[10]); + puDst->ai8[11] = RT_MIN(puSrc1->ai8[11], puSrc2->ai8[11]); + puDst->ai8[12] = RT_MIN(puSrc1->ai8[12], puSrc2->ai8[12]); + puDst->ai8[13] = RT_MIN(puSrc1->ai8[13], puSrc2->ai8[13]); + puDst->ai8[14] = RT_MIN(puSrc1->ai8[14], puSrc2->ai8[14]); + puDst->ai8[15] = RT_MIN(puSrc1->ai8[15], puSrc2->ai8[15]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->ai8[ 0] = RT_MIN(puSrc1->ai8[ 0], puSrc2->ai8[ 0]); + puDst->ai8[ 1] = RT_MIN(puSrc1->ai8[ 1], puSrc2->ai8[ 1]); + puDst->ai8[ 2] = RT_MIN(puSrc1->ai8[ 2], puSrc2->ai8[ 2]); + puDst->ai8[ 3] = RT_MIN(puSrc1->ai8[ 3], puSrc2->ai8[ 3]); + puDst->ai8[ 4] = RT_MIN(puSrc1->ai8[ 4], puSrc2->ai8[ 4]); + puDst->ai8[ 5] = RT_MIN(puSrc1->ai8[ 5], puSrc2->ai8[ 5]); + puDst->ai8[ 6] = RT_MIN(puSrc1->ai8[ 6], puSrc2->ai8[ 6]); + puDst->ai8[ 7] = RT_MIN(puSrc1->ai8[ 7], puSrc2->ai8[ 7]); + puDst->ai8[ 8] = RT_MIN(puSrc1->ai8[ 8], puSrc2->ai8[ 8]); + puDst->ai8[ 9] = RT_MIN(puSrc1->ai8[ 9], puSrc2->ai8[ 9]); + puDst->ai8[10] = RT_MIN(puSrc1->ai8[10], puSrc2->ai8[10]); + puDst->ai8[11] = RT_MIN(puSrc1->ai8[11], puSrc2->ai8[11]); + puDst->ai8[12] = RT_MIN(puSrc1->ai8[12], puSrc2->ai8[12]); + puDst->ai8[13] = RT_MIN(puSrc1->ai8[13], puSrc2->ai8[13]); + puDst->ai8[14] = RT_MIN(puSrc1->ai8[14], puSrc2->ai8[14]); + puDst->ai8[15] = RT_MIN(puSrc1->ai8[15], puSrc2->ai8[15]); + puDst->ai8[16] = RT_MIN(puSrc1->ai8[16], puSrc2->ai8[16]); + puDst->ai8[17] = RT_MIN(puSrc1->ai8[17], puSrc2->ai8[17]); + puDst->ai8[18] = RT_MIN(puSrc1->ai8[18], puSrc2->ai8[18]); + puDst->ai8[19] = RT_MIN(puSrc1->ai8[19], puSrc2->ai8[19]); + puDst->ai8[20] = RT_MIN(puSrc1->ai8[20], puSrc2->ai8[20]); + puDst->ai8[21] = RT_MIN(puSrc1->ai8[21], puSrc2->ai8[21]); + puDst->ai8[22] = RT_MIN(puSrc1->ai8[22], puSrc2->ai8[22]); + puDst->ai8[23] = RT_MIN(puSrc1->ai8[23], puSrc2->ai8[23]); + puDst->ai8[24] = RT_MIN(puSrc1->ai8[24], puSrc2->ai8[24]); + puDst->ai8[25] = RT_MIN(puSrc1->ai8[25], puSrc2->ai8[25]); + puDst->ai8[26] = RT_MIN(puSrc1->ai8[26], puSrc2->ai8[26]); + puDst->ai8[27] = RT_MIN(puSrc1->ai8[27], puSrc2->ai8[27]); + puDst->ai8[28] = RT_MIN(puSrc1->ai8[28], puSrc2->ai8[28]); + puDst->ai8[29] = RT_MIN(puSrc1->ai8[29], puSrc2->ai8[29]); + puDst->ai8[30] = RT_MIN(puSrc1->ai8[30], puSrc2->ai8[30]); + puDst->ai8[31] = RT_MIN(puSrc1->ai8[31], puSrc2->ai8[31]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->ai16[ 0] = RT_MIN(puSrc1->ai16[ 0], puSrc2->ai16[ 0]); + puDst->ai16[ 1] = RT_MIN(puSrc1->ai16[ 1], puSrc2->ai16[ 1]); + puDst->ai16[ 2] = RT_MIN(puSrc1->ai16[ 2], puSrc2->ai16[ 2]); + puDst->ai16[ 3] = RT_MIN(puSrc1->ai16[ 3], puSrc2->ai16[ 3]); + puDst->ai16[ 4] = RT_MIN(puSrc1->ai16[ 4], puSrc2->ai16[ 4]); + puDst->ai16[ 5] = RT_MIN(puSrc1->ai16[ 5], puSrc2->ai16[ 5]); + puDst->ai16[ 6] = RT_MIN(puSrc1->ai16[ 6], puSrc2->ai16[ 6]); + puDst->ai16[ 7] = RT_MIN(puSrc1->ai16[ 7], puSrc2->ai16[ 7]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->ai16[ 0] = RT_MIN(puSrc1->ai16[ 0], puSrc2->ai16[ 0]); + puDst->ai16[ 1] = RT_MIN(puSrc1->ai16[ 1], puSrc2->ai16[ 1]); + puDst->ai16[ 2] = RT_MIN(puSrc1->ai16[ 2], puSrc2->ai16[ 2]); + puDst->ai16[ 3] = RT_MIN(puSrc1->ai16[ 3], puSrc2->ai16[ 3]); + puDst->ai16[ 4] = RT_MIN(puSrc1->ai16[ 4], puSrc2->ai16[ 4]); + puDst->ai16[ 5] = RT_MIN(puSrc1->ai16[ 5], puSrc2->ai16[ 5]); + puDst->ai16[ 6] = RT_MIN(puSrc1->ai16[ 6], puSrc2->ai16[ 6]); + puDst->ai16[ 7] = RT_MIN(puSrc1->ai16[ 7], puSrc2->ai16[ 7]); + puDst->ai16[ 8] = RT_MIN(puSrc1->ai16[ 8], puSrc2->ai16[ 8]); + puDst->ai16[ 9] = RT_MIN(puSrc1->ai16[ 9], puSrc2->ai16[ 9]); + puDst->ai16[10] = RT_MIN(puSrc1->ai16[10], puSrc2->ai16[10]); + puDst->ai16[11] = RT_MIN(puSrc1->ai16[11], puSrc2->ai16[11]); + puDst->ai16[12] = RT_MIN(puSrc1->ai16[12], puSrc2->ai16[12]); + puDst->ai16[13] = RT_MIN(puSrc1->ai16[13], puSrc2->ai16[13]); + puDst->ai16[14] = RT_MIN(puSrc1->ai16[14], puSrc2->ai16[14]); + puDst->ai16[15] = RT_MIN(puSrc1->ai16[15], puSrc2->ai16[15]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->ai32[ 0] = RT_MIN(puSrc1->ai32[ 0], puSrc2->ai32[ 0]); + puDst->ai32[ 1] = RT_MIN(puSrc1->ai32[ 1], puSrc2->ai32[ 1]); + puDst->ai32[ 2] = RT_MIN(puSrc1->ai32[ 2], puSrc2->ai32[ 2]); + puDst->ai32[ 3] = RT_MIN(puSrc1->ai32[ 3], puSrc2->ai32[ 3]); + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->ai32[ 0] = RT_MIN(puSrc1->ai32[ 0], puSrc2->ai32[ 0]); + puDst->ai32[ 1] = RT_MIN(puSrc1->ai32[ 1], puSrc2->ai32[ 1]); + puDst->ai32[ 2] = RT_MIN(puSrc1->ai32[ 2], puSrc2->ai32[ 2]); + puDst->ai32[ 3] = RT_MIN(puSrc1->ai32[ 3], puSrc2->ai32[ 3]); + puDst->ai32[ 4] = RT_MIN(puSrc1->ai32[ 4], puSrc2->ai32[ 4]); + puDst->ai32[ 5] = RT_MIN(puSrc1->ai32[ 5], puSrc2->ai32[ 5]); + puDst->ai32[ 6] = RT_MIN(puSrc1->ai32[ 6], puSrc2->ai32[ 6]); + puDst->ai32[ 7] = RT_MIN(puSrc1->ai32[ 7], puSrc2->ai32[ 7]); + RT_NOREF(pExtState); +} + + +/* + * PAVGB / VPAVGB / PAVGW / VPAVGW + */ +#define PAVGB_EXEC(a_Src1, a_Src2) ((uint8_t)(((uint16_t)(a_Src1) + (a_Src2) + 1) >> 1)) +#define PAVGW_EXEC(a_Src1, a_Src2) ((uint16_t)(((uint32_t)(a_Src1) + (a_Src2) + 1) >> 1)) + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pavgb_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + uDst.au8[0] = PAVGB_EXEC(uSrc1.au8[0], uSrc2.au8[0]); + uDst.au8[1] = PAVGB_EXEC(uSrc1.au8[1], uSrc2.au8[1]); + uDst.au8[2] = PAVGB_EXEC(uSrc1.au8[2], uSrc2.au8[2]); + uDst.au8[3] = PAVGB_EXEC(uSrc1.au8[3], uSrc2.au8[3]); + uDst.au8[4] = PAVGB_EXEC(uSrc1.au8[4], uSrc2.au8[4]); + uDst.au8[5] = PAVGB_EXEC(uSrc1.au8[5], uSrc2.au8[5]); + uDst.au8[6] = PAVGB_EXEC(uSrc1.au8[6], uSrc2.au8[6]); + uDst.au8[7] = PAVGB_EXEC(uSrc1.au8[7], uSrc2.au8[7]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pavgb_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au8[ 0] = PAVGB_EXEC(uSrc1.au8[ 0], puSrc->au8[ 0]); + puDst->au8[ 1] = PAVGB_EXEC(uSrc1.au8[ 1], puSrc->au8[ 1]); + puDst->au8[ 2] = PAVGB_EXEC(uSrc1.au8[ 2], puSrc->au8[ 2]); + puDst->au8[ 3] = PAVGB_EXEC(uSrc1.au8[ 3], puSrc->au8[ 3]); + puDst->au8[ 4] = PAVGB_EXEC(uSrc1.au8[ 4], puSrc->au8[ 4]); + puDst->au8[ 5] = PAVGB_EXEC(uSrc1.au8[ 5], puSrc->au8[ 5]); + puDst->au8[ 6] = PAVGB_EXEC(uSrc1.au8[ 6], puSrc->au8[ 6]); + puDst->au8[ 7] = PAVGB_EXEC(uSrc1.au8[ 7], puSrc->au8[ 7]); + puDst->au8[ 8] = PAVGB_EXEC(uSrc1.au8[ 8], puSrc->au8[ 8]); + puDst->au8[ 9] = PAVGB_EXEC(uSrc1.au8[ 9], puSrc->au8[ 9]); + puDst->au8[10] = PAVGB_EXEC(uSrc1.au8[10], puSrc->au8[10]); + puDst->au8[11] = PAVGB_EXEC(uSrc1.au8[11], puSrc->au8[11]); + puDst->au8[12] = PAVGB_EXEC(uSrc1.au8[12], puSrc->au8[12]); + puDst->au8[13] = PAVGB_EXEC(uSrc1.au8[13], puSrc->au8[13]); + puDst->au8[14] = PAVGB_EXEC(uSrc1.au8[14], puSrc->au8[14]); + puDst->au8[15] = PAVGB_EXEC(uSrc1.au8[15], puSrc->au8[15]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pavgw_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + uDst.au16[0] = PAVGW_EXEC(uSrc1.au16[0], uSrc2.au16[0]); + uDst.au16[1] = PAVGW_EXEC(uSrc1.au16[1], uSrc2.au16[1]); + uDst.au16[2] = PAVGW_EXEC(uSrc1.au16[2], uSrc2.au16[2]); + uDst.au16[3] = PAVGW_EXEC(uSrc1.au16[3], uSrc2.au16[3]); + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pavgw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au16[0] = PAVGW_EXEC(uSrc1.au16[0], puSrc->au16[0]); + puDst->au16[1] = PAVGW_EXEC(uSrc1.au16[1], puSrc->au16[1]); + puDst->au16[2] = PAVGW_EXEC(uSrc1.au16[2], puSrc->au16[2]); + puDst->au16[3] = PAVGW_EXEC(uSrc1.au16[3], puSrc->au16[3]); + puDst->au16[4] = PAVGW_EXEC(uSrc1.au16[4], puSrc->au16[4]); + puDst->au16[5] = PAVGW_EXEC(uSrc1.au16[5], puSrc->au16[5]); + puDst->au16[6] = PAVGW_EXEC(uSrc1.au16[6], puSrc->au16[6]); + puDst->au16[7] = PAVGW_EXEC(uSrc1.au16[7], puSrc->au16[7]); +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_pavgb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au8[ 0] = PAVGB_EXEC(uSrc1.au8[ 0], puSrc->au8[ 0]); + puDst->au8[ 1] = PAVGB_EXEC(uSrc1.au8[ 1], puSrc->au8[ 1]); + puDst->au8[ 2] = PAVGB_EXEC(uSrc1.au8[ 2], puSrc->au8[ 2]); + puDst->au8[ 3] = PAVGB_EXEC(uSrc1.au8[ 3], puSrc->au8[ 3]); + puDst->au8[ 4] = PAVGB_EXEC(uSrc1.au8[ 4], puSrc->au8[ 4]); + puDst->au8[ 5] = PAVGB_EXEC(uSrc1.au8[ 5], puSrc->au8[ 5]); + puDst->au8[ 6] = PAVGB_EXEC(uSrc1.au8[ 6], puSrc->au8[ 6]); + puDst->au8[ 7] = PAVGB_EXEC(uSrc1.au8[ 7], puSrc->au8[ 7]); + puDst->au8[ 8] = PAVGB_EXEC(uSrc1.au8[ 8], puSrc->au8[ 8]); + puDst->au8[ 9] = PAVGB_EXEC(uSrc1.au8[ 9], puSrc->au8[ 9]); + puDst->au8[10] = PAVGB_EXEC(uSrc1.au8[10], puSrc->au8[10]); + puDst->au8[11] = PAVGB_EXEC(uSrc1.au8[11], puSrc->au8[11]); + puDst->au8[12] = PAVGB_EXEC(uSrc1.au8[12], puSrc->au8[12]); + puDst->au8[13] = PAVGB_EXEC(uSrc1.au8[13], puSrc->au8[13]); + puDst->au8[14] = PAVGB_EXEC(uSrc1.au8[14], puSrc->au8[14]); + puDst->au8[15] = PAVGB_EXEC(uSrc1.au8[15], puSrc->au8[15]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pavgw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au8[ 0] = PAVGW_EXEC(uSrc1.au8[ 0], puSrc->au8[ 0]); + puDst->au8[ 1] = PAVGW_EXEC(uSrc1.au8[ 1], puSrc->au8[ 1]); + puDst->au8[ 2] = PAVGW_EXEC(uSrc1.au8[ 2], puSrc->au8[ 2]); + puDst->au8[ 3] = PAVGW_EXEC(uSrc1.au8[ 3], puSrc->au8[ 3]); + puDst->au8[ 4] = PAVGW_EXEC(uSrc1.au8[ 4], puSrc->au8[ 4]); + puDst->au8[ 5] = PAVGW_EXEC(uSrc1.au8[ 5], puSrc->au8[ 5]); + puDst->au8[ 6] = PAVGW_EXEC(uSrc1.au8[ 6], puSrc->au8[ 6]); + puDst->au8[ 7] = PAVGW_EXEC(uSrc1.au8[ 7], puSrc->au8[ 7]); + puDst->au8[ 8] = PAVGW_EXEC(uSrc1.au8[ 8], puSrc->au8[ 8]); + puDst->au8[ 9] = PAVGW_EXEC(uSrc1.au8[ 9], puSrc->au8[ 9]); + puDst->au8[10] = PAVGW_EXEC(uSrc1.au8[10], puSrc->au8[10]); + puDst->au8[11] = PAVGW_EXEC(uSrc1.au8[11], puSrc->au8[11]); + puDst->au8[12] = PAVGW_EXEC(uSrc1.au8[12], puSrc->au8[12]); + puDst->au8[13] = PAVGW_EXEC(uSrc1.au8[13], puSrc->au8[13]); + puDst->au8[14] = PAVGW_EXEC(uSrc1.au8[14], puSrc->au8[14]); + puDst->au8[15] = PAVGW_EXEC(uSrc1.au8[15], puSrc->au8[15]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpavgb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->au8[ 0] = PAVGB_EXEC(puSrc1->au8[ 0], puSrc2->au8[ 0]); + puDst->au8[ 1] = PAVGB_EXEC(puSrc1->au8[ 1], puSrc2->au8[ 1]); + puDst->au8[ 2] = PAVGB_EXEC(puSrc1->au8[ 2], puSrc2->au8[ 2]); + puDst->au8[ 3] = PAVGB_EXEC(puSrc1->au8[ 3], puSrc2->au8[ 3]); + puDst->au8[ 4] = PAVGB_EXEC(puSrc1->au8[ 4], puSrc2->au8[ 4]); + puDst->au8[ 5] = PAVGB_EXEC(puSrc1->au8[ 5], puSrc2->au8[ 5]); + puDst->au8[ 6] = PAVGB_EXEC(puSrc1->au8[ 6], puSrc2->au8[ 6]); + puDst->au8[ 7] = PAVGB_EXEC(puSrc1->au8[ 7], puSrc2->au8[ 7]); + puDst->au8[ 8] = PAVGB_EXEC(puSrc1->au8[ 8], puSrc2->au8[ 8]); + puDst->au8[ 9] = PAVGB_EXEC(puSrc1->au8[ 9], puSrc2->au8[ 9]); + puDst->au8[10] = PAVGB_EXEC(puSrc1->au8[10], puSrc2->au8[10]); + puDst->au8[11] = PAVGB_EXEC(puSrc1->au8[11], puSrc2->au8[11]); + puDst->au8[12] = PAVGB_EXEC(puSrc1->au8[12], puSrc2->au8[12]); + puDst->au8[13] = PAVGB_EXEC(puSrc1->au8[13], puSrc2->au8[13]); + puDst->au8[14] = PAVGB_EXEC(puSrc1->au8[14], puSrc2->au8[14]); + puDst->au8[15] = PAVGB_EXEC(puSrc1->au8[15], puSrc2->au8[15]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpavgb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->au8[ 0] = PAVGB_EXEC(puSrc1->au8[ 0], puSrc2->au8[ 0]); + puDst->au8[ 1] = PAVGB_EXEC(puSrc1->au8[ 1], puSrc2->au8[ 1]); + puDst->au8[ 2] = PAVGB_EXEC(puSrc1->au8[ 2], puSrc2->au8[ 2]); + puDst->au8[ 3] = PAVGB_EXEC(puSrc1->au8[ 3], puSrc2->au8[ 3]); + puDst->au8[ 4] = PAVGB_EXEC(puSrc1->au8[ 4], puSrc2->au8[ 4]); + puDst->au8[ 5] = PAVGB_EXEC(puSrc1->au8[ 5], puSrc2->au8[ 5]); + puDst->au8[ 6] = PAVGB_EXEC(puSrc1->au8[ 6], puSrc2->au8[ 6]); + puDst->au8[ 7] = PAVGB_EXEC(puSrc1->au8[ 7], puSrc2->au8[ 7]); + puDst->au8[ 8] = PAVGB_EXEC(puSrc1->au8[ 8], puSrc2->au8[ 8]); + puDst->au8[ 9] = PAVGB_EXEC(puSrc1->au8[ 9], puSrc2->au8[ 9]); + puDst->au8[10] = PAVGB_EXEC(puSrc1->au8[10], puSrc2->au8[10]); + puDst->au8[11] = PAVGB_EXEC(puSrc1->au8[11], puSrc2->au8[11]); + puDst->au8[12] = PAVGB_EXEC(puSrc1->au8[12], puSrc2->au8[12]); + puDst->au8[13] = PAVGB_EXEC(puSrc1->au8[13], puSrc2->au8[13]); + puDst->au8[14] = PAVGB_EXEC(puSrc1->au8[14], puSrc2->au8[14]); + puDst->au8[15] = PAVGB_EXEC(puSrc1->au8[15], puSrc2->au8[15]); + puDst->au8[16] = PAVGB_EXEC(puSrc1->au8[16], puSrc2->au8[16]); + puDst->au8[17] = PAVGB_EXEC(puSrc1->au8[17], puSrc2->au8[17]); + puDst->au8[18] = PAVGB_EXEC(puSrc1->au8[18], puSrc2->au8[18]); + puDst->au8[19] = PAVGB_EXEC(puSrc1->au8[19], puSrc2->au8[19]); + puDst->au8[20] = PAVGB_EXEC(puSrc1->au8[20], puSrc2->au8[20]); + puDst->au8[21] = PAVGB_EXEC(puSrc1->au8[21], puSrc2->au8[21]); + puDst->au8[22] = PAVGB_EXEC(puSrc1->au8[22], puSrc2->au8[22]); + puDst->au8[23] = PAVGB_EXEC(puSrc1->au8[23], puSrc2->au8[23]); + puDst->au8[24] = PAVGB_EXEC(puSrc1->au8[24], puSrc2->au8[24]); + puDst->au8[25] = PAVGB_EXEC(puSrc1->au8[25], puSrc2->au8[25]); + puDst->au8[26] = PAVGB_EXEC(puSrc1->au8[26], puSrc2->au8[26]); + puDst->au8[27] = PAVGB_EXEC(puSrc1->au8[27], puSrc2->au8[27]); + puDst->au8[28] = PAVGB_EXEC(puSrc1->au8[28], puSrc2->au8[28]); + puDst->au8[29] = PAVGB_EXEC(puSrc1->au8[29], puSrc2->au8[29]); + puDst->au8[30] = PAVGB_EXEC(puSrc1->au8[30], puSrc2->au8[30]); + puDst->au8[31] = PAVGB_EXEC(puSrc1->au8[31], puSrc2->au8[31]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpavgw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + puDst->au16[ 0] = PAVGW_EXEC(puSrc1->au16[ 0], puSrc2->au16[ 0]); + puDst->au16[ 1] = PAVGW_EXEC(puSrc1->au16[ 1], puSrc2->au16[ 1]); + puDst->au16[ 2] = PAVGW_EXEC(puSrc1->au16[ 2], puSrc2->au16[ 2]); + puDst->au16[ 3] = PAVGW_EXEC(puSrc1->au16[ 3], puSrc2->au16[ 3]); + puDst->au16[ 4] = PAVGW_EXEC(puSrc1->au16[ 4], puSrc2->au16[ 4]); + puDst->au16[ 5] = PAVGW_EXEC(puSrc1->au16[ 5], puSrc2->au16[ 5]); + puDst->au16[ 6] = PAVGW_EXEC(puSrc1->au16[ 6], puSrc2->au16[ 6]); + puDst->au16[ 7] = PAVGW_EXEC(puSrc1->au16[ 7], puSrc2->au16[ 7]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpavgw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + puDst->au16[ 0] = PAVGW_EXEC(puSrc1->au16[ 0], puSrc2->au16[ 0]); + puDst->au16[ 1] = PAVGW_EXEC(puSrc1->au16[ 1], puSrc2->au16[ 1]); + puDst->au16[ 2] = PAVGW_EXEC(puSrc1->au16[ 2], puSrc2->au16[ 2]); + puDst->au16[ 3] = PAVGW_EXEC(puSrc1->au16[ 3], puSrc2->au16[ 3]); + puDst->au16[ 4] = PAVGW_EXEC(puSrc1->au16[ 4], puSrc2->au16[ 4]); + puDst->au16[ 5] = PAVGW_EXEC(puSrc1->au16[ 5], puSrc2->au16[ 5]); + puDst->au16[ 6] = PAVGW_EXEC(puSrc1->au16[ 6], puSrc2->au16[ 6]); + puDst->au16[ 7] = PAVGW_EXEC(puSrc1->au16[ 7], puSrc2->au16[ 7]); + puDst->au16[ 8] = PAVGW_EXEC(puSrc1->au16[ 8], puSrc2->au16[ 8]); + puDst->au16[ 9] = PAVGW_EXEC(puSrc1->au16[ 9], puSrc2->au16[ 9]); + puDst->au16[10] = PAVGW_EXEC(puSrc1->au16[10], puSrc2->au16[10]); + puDst->au16[11] = PAVGW_EXEC(puSrc1->au16[11], puSrc2->au16[11]); + puDst->au16[12] = PAVGW_EXEC(puSrc1->au16[12], puSrc2->au16[12]); + puDst->au16[13] = PAVGW_EXEC(puSrc1->au16[13], puSrc2->au16[13]); + puDst->au16[14] = PAVGW_EXEC(puSrc1->au16[14], puSrc2->au16[14]); + puDst->au16[15] = PAVGW_EXEC(puSrc1->au16[15], puSrc2->au16[15]); +} + +#undef PAVGB_EXEC +#undef PAVGW_EXEC + + +/* + * PMOVMSKB / VPMOVMSKB + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u64,(uint64_t *pu64Dst, uint64_t const *pu64Src)) +{ + /* The the most signficant bit from each byte and store them in the given general purpose register. */ + uint64_t const uSrc = *pu64Src; + *pu64Dst = ((uSrc >> ( 7-0)) & RT_BIT_64(0)) + | ((uSrc >> (15-1)) & RT_BIT_64(1)) + | ((uSrc >> (23-2)) & RT_BIT_64(2)) + | ((uSrc >> (31-3)) & RT_BIT_64(3)) + | ((uSrc >> (39-4)) & RT_BIT_64(4)) + | ((uSrc >> (47-5)) & RT_BIT_64(5)) + | ((uSrc >> (55-6)) & RT_BIT_64(6)) + | ((uSrc >> (63-7)) & RT_BIT_64(7)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u128,(uint64_t *pu64Dst, PCRTUINT128U pu128Src)) +{ + /* The the most signficant bit from each byte and store them in the given general purpose register. */ + uint64_t const uSrc0 = pu128Src->QWords.qw0; + uint64_t const uSrc1 = pu128Src->QWords.qw1; + *pu64Dst = ((uSrc0 >> ( 7-0)) & RT_BIT_64(0)) + | ((uSrc0 >> (15-1)) & RT_BIT_64(1)) + | ((uSrc0 >> (23-2)) & RT_BIT_64(2)) + | ((uSrc0 >> (31-3)) & RT_BIT_64(3)) + | ((uSrc0 >> (39-4)) & RT_BIT_64(4)) + | ((uSrc0 >> (47-5)) & RT_BIT_64(5)) + | ((uSrc0 >> (55-6)) & RT_BIT_64(6)) + | ((uSrc0 >> (63-7)) & RT_BIT_64(7)) + | ((uSrc1 << (1 /*7-8*/)) & RT_BIT_64(8)) + | ((uSrc1 >> (15-9)) & RT_BIT_64(9)) + | ((uSrc1 >> (23-10)) & RT_BIT_64(10)) + | ((uSrc1 >> (31-11)) & RT_BIT_64(11)) + | ((uSrc1 >> (39-12)) & RT_BIT_64(12)) + | ((uSrc1 >> (47-13)) & RT_BIT_64(13)) + | ((uSrc1 >> (55-14)) & RT_BIT_64(14)) + | ((uSrc1 >> (63-15)) & RT_BIT_64(15)); +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovmskb_u256_fallback,(uint64_t *pu64Dst, PCRTUINT256U puSrc)) +{ + /* The the most signficant bit from each byte and store them in the given general purpose register. */ + uint64_t const uSrc0 = puSrc->QWords.qw0; + uint64_t const uSrc1 = puSrc->QWords.qw1; + uint64_t const uSrc2 = puSrc->QWords.qw2; + uint64_t const uSrc3 = puSrc->QWords.qw3; + *pu64Dst = ((uSrc0 >> ( 7-0)) & RT_BIT_64(0)) + | ((uSrc0 >> (15-1)) & RT_BIT_64(1)) + | ((uSrc0 >> (23-2)) & RT_BIT_64(2)) + | ((uSrc0 >> (31-3)) & RT_BIT_64(3)) + | ((uSrc0 >> (39-4)) & RT_BIT_64(4)) + | ((uSrc0 >> (47-5)) & RT_BIT_64(5)) + | ((uSrc0 >> (55-6)) & RT_BIT_64(6)) + | ((uSrc0 >> (63-7)) & RT_BIT_64(7)) + | ((uSrc1 << (1 /*7-8*/)) & RT_BIT_64(8)) + | ((uSrc1 >> (15-9)) & RT_BIT_64(9)) + | ((uSrc1 >> (23-10)) & RT_BIT_64(10)) + | ((uSrc1 >> (31-11)) & RT_BIT_64(11)) + | ((uSrc1 >> (39-12)) & RT_BIT_64(12)) + | ((uSrc1 >> (47-13)) & RT_BIT_64(13)) + | ((uSrc1 >> (55-14)) & RT_BIT_64(14)) + | ((uSrc1 >> (63-15)) & RT_BIT_64(15)) + | ((uSrc2 << (9 /* 7-16*/)) & RT_BIT_64(16)) + | ((uSrc2 << (2 /*15-17*/)) & RT_BIT_64(17)) + | ((uSrc2 >> (23-18)) & RT_BIT_64(18)) + | ((uSrc2 >> (31-19)) & RT_BIT_64(19)) + | ((uSrc2 >> (39-20)) & RT_BIT_64(20)) + | ((uSrc2 >> (47-21)) & RT_BIT_64(21)) + | ((uSrc2 >> (55-22)) & RT_BIT_64(22)) + | ((uSrc2 >> (63-23)) & RT_BIT_64(23)) + | ((uSrc3 << (17 /* 7-24*/)) & RT_BIT_64(24)) + | ((uSrc3 << (10 /*15-25*/)) & RT_BIT_64(25)) + | ((uSrc3 << (3 /*23-26*/)) & RT_BIT_64(26)) + | ((uSrc3 >> (31-27)) & RT_BIT_64(27)) + | ((uSrc3 >> (39-28)) & RT_BIT_64(28)) + | ((uSrc3 >> (47-29)) & RT_BIT_64(29)) + | ((uSrc3 >> (55-30)) & RT_BIT_64(30)) + | ((uSrc3 >> (63-31)) & RT_BIT_64(31)); +} + + +/* + * [V]PSHUFB + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_pshufb_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc = { *puSrc }; + RTUINT64U const uDstIn = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut = { 0 }; + for (unsigned iByte = 0; iByte < RT_ELEMENTS(uDstIn.au8); iByte++) + { + uint8_t idxSrc = uSrc.au8[iByte]; + if (!(idxSrc & 0x80)) + uDstOut.au8[iByte] = uDstIn.au8[idxSrc & 7]; + } + *puDst = uDstOut.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pshufb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc = *puSrc; + RTUINT128U const uDstIn = *puDst; + ASMCompilerBarrier(); + puDst->au64[0] = 0; + puDst->au64[1] = 0; + for (unsigned iByte = 0; iByte < RT_ELEMENTS(puDst->au8); iByte++) + { + uint8_t idxSrc = uSrc.au8[iByte]; + if (!(idxSrc & 0x80)) + puDst->au8[iByte] = uDstIn.au8[idxSrc & 15]; + } + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpshufb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst, + PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc1 = *puSrc1; /* could be same as puDst */ + RTUINT128U const uSrc2 = *puSrc2; /* could be same as puDst */ + ASMCompilerBarrier(); + puDst->au64[0] = 0; + puDst->au64[1] = 0; + for (unsigned iByte = 0; iByte < 16; iByte++) + { + uint8_t idxSrc = uSrc2.au8[iByte]; + if (!(idxSrc & 0x80)) + puDst->au8[iByte] = uSrc1.au8[(idxSrc & 15)]; + } + RT_NOREF(pExtState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpshufb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst, + PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc1 = *puSrc1; /* could be same as puDst */ + RTUINT256U const uSrc2 = *puSrc2; /* could be same as puDst */ + ASMCompilerBarrier(); + puDst->au64[0] = 0; + puDst->au64[1] = 0; + puDst->au64[2] = 0; + puDst->au64[3] = 0; + for (unsigned iByte = 0; iByte < 16; iByte++) + { + uint8_t idxSrc = uSrc2.au8[iByte]; + if (!(idxSrc & 0x80)) + puDst->au8[iByte] = uSrc1.au8[(idxSrc & 15)]; + } + for (unsigned iByte = 16; iByte < RT_ELEMENTS(puDst->au8); iByte++) + { + uint8_t idxSrc = uSrc2.au8[iByte]; + if (!(idxSrc & 0x80)) + puDst->au8[iByte] = uSrc1.au8[(idxSrc & 15) + 16]; /* baka intel */ + } + RT_NOREF(pExtState); +} + + +/* + * PSHUFW, [V]PSHUFHW, [V]PSHUFLW, [V]PSHUFD + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pshufw_u64,(uint64_t *puDst, uint64_t const *puSrc, uint8_t bEvil)) +{ + uint64_t const uSrc = *puSrc; + ASMCompilerBarrier(); + *puDst = RT_MAKE_U64_FROM_U16(uSrc >> (( bEvil & 3) * 16), + uSrc >> (((bEvil >> 2) & 3) * 16), + uSrc >> (((bEvil >> 4) & 3) * 16), + uSrc >> (((bEvil >> 6) & 3) * 16)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pshufhw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + puDst->QWords.qw0 = puSrc->QWords.qw0; + uint64_t const uSrc = puSrc->QWords.qw1; + ASMCompilerBarrier(); + puDst->QWords.qw1 = RT_MAKE_U64_FROM_U16(uSrc >> (( bEvil & 3) * 16), + uSrc >> (((bEvil >> 2) & 3) * 16), + uSrc >> (((bEvil >> 4) & 3) * 16), + uSrc >> (((bEvil >> 6) & 3) * 16)); +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpshufhw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc, uint8_t bEvil)) +{ + puDst->QWords.qw0 = puSrc->QWords.qw0; + uint64_t const uSrc1 = puSrc->QWords.qw1; + puDst->QWords.qw2 = puSrc->QWords.qw2; + uint64_t const uSrc3 = puSrc->QWords.qw3; + ASMCompilerBarrier(); + puDst->QWords.qw1 = RT_MAKE_U64_FROM_U16(uSrc1 >> (( bEvil & 3) * 16), + uSrc1 >> (((bEvil >> 2) & 3) * 16), + uSrc1 >> (((bEvil >> 4) & 3) * 16), + uSrc1 >> (((bEvil >> 6) & 3) * 16)); + puDst->QWords.qw3 = RT_MAKE_U64_FROM_U16(uSrc3 >> (( bEvil & 3) * 16), + uSrc3 >> (((bEvil >> 2) & 3) * 16), + uSrc3 >> (((bEvil >> 4) & 3) * 16), + uSrc3 >> (((bEvil >> 6) & 3) * 16)); +} + +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_pshuflw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + puDst->QWords.qw1 = puSrc->QWords.qw1; + uint64_t const uSrc = puSrc->QWords.qw0; + ASMCompilerBarrier(); + puDst->QWords.qw0 = RT_MAKE_U64_FROM_U16(uSrc >> (( bEvil & 3) * 16), + uSrc >> (((bEvil >> 2) & 3) * 16), + uSrc >> (((bEvil >> 4) & 3) * 16), + uSrc >> (((bEvil >> 6) & 3) * 16)); + +} +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpshuflw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc, uint8_t bEvil)) +{ + puDst->QWords.qw3 = puSrc->QWords.qw3; + uint64_t const uSrc2 = puSrc->QWords.qw2; + puDst->QWords.qw1 = puSrc->QWords.qw1; + uint64_t const uSrc0 = puSrc->QWords.qw0; + ASMCompilerBarrier(); + puDst->QWords.qw0 = RT_MAKE_U64_FROM_U16(uSrc0 >> (( bEvil & 3) * 16), + uSrc0 >> (((bEvil >> 2) & 3) * 16), + uSrc0 >> (((bEvil >> 4) & 3) * 16), + uSrc0 >> (((bEvil >> 6) & 3) * 16)); + puDst->QWords.qw2 = RT_MAKE_U64_FROM_U16(uSrc2 >> (( bEvil & 3) * 16), + uSrc2 >> (((bEvil >> 2) & 3) * 16), + uSrc2 >> (((bEvil >> 4) & 3) * 16), + uSrc2 >> (((bEvil >> 6) & 3) * 16)); + +} + + +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_pshufd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + RTUINT128U const uSrc = *puSrc; + ASMCompilerBarrier(); + puDst->au32[0] = uSrc.au32[bEvil & 3]; + puDst->au32[1] = uSrc.au32[(bEvil >> 2) & 3]; + puDst->au32[2] = uSrc.au32[(bEvil >> 4) & 3]; + puDst->au32[3] = uSrc.au32[(bEvil >> 6) & 3]; +} +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpshufd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc, uint8_t bEvil)) +{ + RTUINT256U const uSrc = *puSrc; + ASMCompilerBarrier(); + puDst->au128[0].au32[0] = uSrc.au128[0].au32[bEvil & 3]; + puDst->au128[0].au32[1] = uSrc.au128[0].au32[(bEvil >> 2) & 3]; + puDst->au128[0].au32[2] = uSrc.au128[0].au32[(bEvil >> 4) & 3]; + puDst->au128[0].au32[3] = uSrc.au128[0].au32[(bEvil >> 6) & 3]; + puDst->au128[1].au32[0] = uSrc.au128[1].au32[bEvil & 3]; + puDst->au128[1].au32[1] = uSrc.au128[1].au32[(bEvil >> 2) & 3]; + puDst->au128[1].au32[2] = uSrc.au128[1].au32[(bEvil >> 4) & 3]; + puDst->au128[1].au32[3] = uSrc.au128[1].au32[(bEvil >> 6) & 3]; +} + + +/* + * PUNPCKHBW - high bytes -> words + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhbw_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc2 = { *puSrc }; + RTUINT64U const uSrc1 = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut; + uDstOut.au8[0] = uSrc1.au8[4]; + uDstOut.au8[1] = uSrc2.au8[4]; + uDstOut.au8[2] = uSrc1.au8[5]; + uDstOut.au8[3] = uSrc2.au8[5]; + uDstOut.au8[4] = uSrc1.au8[6]; + uDstOut.au8[5] = uSrc2.au8[6]; + uDstOut.au8[6] = uSrc1.au8[7]; + uDstOut.au8[7] = uSrc2.au8[7]; + *puDst = uDstOut.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhbw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au8[ 0] = uSrc1.au8[ 8]; + uDstOut.au8[ 1] = uSrc2.au8[ 8]; + uDstOut.au8[ 2] = uSrc1.au8[ 9]; + uDstOut.au8[ 3] = uSrc2.au8[ 9]; + uDstOut.au8[ 4] = uSrc1.au8[10]; + uDstOut.au8[ 5] = uSrc2.au8[10]; + uDstOut.au8[ 6] = uSrc1.au8[11]; + uDstOut.au8[ 7] = uSrc2.au8[11]; + uDstOut.au8[ 8] = uSrc1.au8[12]; + uDstOut.au8[ 9] = uSrc2.au8[12]; + uDstOut.au8[10] = uSrc1.au8[13]; + uDstOut.au8[11] = uSrc2.au8[13]; + uDstOut.au8[12] = uSrc1.au8[14]; + uDstOut.au8[13] = uSrc2.au8[14]; + uDstOut.au8[14] = uSrc1.au8[15]; + uDstOut.au8[15] = uSrc2.au8[15]; + *puDst = uDstOut; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhbw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au8[ 0] = uSrc1.au8[ 8]; + uDstOut.au8[ 1] = uSrc2.au8[ 8]; + uDstOut.au8[ 2] = uSrc1.au8[ 9]; + uDstOut.au8[ 3] = uSrc2.au8[ 9]; + uDstOut.au8[ 4] = uSrc1.au8[10]; + uDstOut.au8[ 5] = uSrc2.au8[10]; + uDstOut.au8[ 6] = uSrc1.au8[11]; + uDstOut.au8[ 7] = uSrc2.au8[11]; + uDstOut.au8[ 8] = uSrc1.au8[12]; + uDstOut.au8[ 9] = uSrc2.au8[12]; + uDstOut.au8[10] = uSrc1.au8[13]; + uDstOut.au8[11] = uSrc2.au8[13]; + uDstOut.au8[12] = uSrc1.au8[14]; + uDstOut.au8[13] = uSrc2.au8[14]; + uDstOut.au8[14] = uSrc1.au8[15]; + uDstOut.au8[15] = uSrc2.au8[15]; + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au8[ 0] = uSrc1.au8[ 8]; + uDstOut.au8[ 1] = uSrc2.au8[ 8]; + uDstOut.au8[ 2] = uSrc1.au8[ 9]; + uDstOut.au8[ 3] = uSrc2.au8[ 9]; + uDstOut.au8[ 4] = uSrc1.au8[10]; + uDstOut.au8[ 5] = uSrc2.au8[10]; + uDstOut.au8[ 6] = uSrc1.au8[11]; + uDstOut.au8[ 7] = uSrc2.au8[11]; + uDstOut.au8[ 8] = uSrc1.au8[12]; + uDstOut.au8[ 9] = uSrc2.au8[12]; + uDstOut.au8[10] = uSrc1.au8[13]; + uDstOut.au8[11] = uSrc2.au8[13]; + uDstOut.au8[12] = uSrc1.au8[14]; + uDstOut.au8[13] = uSrc2.au8[14]; + uDstOut.au8[14] = uSrc1.au8[15]; + uDstOut.au8[15] = uSrc2.au8[15]; + /* As usual, the upper 128-bits are treated like a parallel register to the lower half. */ + uDstOut.au8[16] = uSrc1.au8[24]; + uDstOut.au8[17] = uSrc2.au8[24]; + uDstOut.au8[18] = uSrc1.au8[25]; + uDstOut.au8[19] = uSrc2.au8[25]; + uDstOut.au8[20] = uSrc1.au8[26]; + uDstOut.au8[21] = uSrc2.au8[26]; + uDstOut.au8[22] = uSrc1.au8[27]; + uDstOut.au8[23] = uSrc2.au8[27]; + uDstOut.au8[24] = uSrc1.au8[28]; + uDstOut.au8[25] = uSrc2.au8[28]; + uDstOut.au8[26] = uSrc1.au8[29]; + uDstOut.au8[27] = uSrc2.au8[29]; + uDstOut.au8[28] = uSrc1.au8[30]; + uDstOut.au8[29] = uSrc2.au8[30]; + uDstOut.au8[30] = uSrc1.au8[31]; + uDstOut.au8[31] = uSrc2.au8[31]; + *puDst = uDstOut; +} + + +/* + * PUNPCKHBW - high words -> dwords + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhwd_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc2 = { *puSrc }; + RTUINT64U const uSrc1 = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut; + uDstOut.au16[0] = uSrc1.au16[2]; + uDstOut.au16[1] = uSrc2.au16[2]; + uDstOut.au16[2] = uSrc1.au16[3]; + uDstOut.au16[3] = uSrc2.au16[3]; + *puDst = uDstOut.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhwd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au16[0] = uSrc1.au16[4]; + uDstOut.au16[1] = uSrc2.au16[4]; + uDstOut.au16[2] = uSrc1.au16[5]; + uDstOut.au16[3] = uSrc2.au16[5]; + uDstOut.au16[4] = uSrc1.au16[6]; + uDstOut.au16[5] = uSrc2.au16[6]; + uDstOut.au16[6] = uSrc1.au16[7]; + uDstOut.au16[7] = uSrc2.au16[7]; + *puDst = uDstOut; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhwd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au16[0] = uSrc1.au16[4]; + uDstOut.au16[1] = uSrc2.au16[4]; + uDstOut.au16[2] = uSrc1.au16[5]; + uDstOut.au16[3] = uSrc2.au16[5]; + uDstOut.au16[4] = uSrc1.au16[6]; + uDstOut.au16[5] = uSrc2.au16[6]; + uDstOut.au16[6] = uSrc1.au16[7]; + uDstOut.au16[7] = uSrc2.au16[7]; + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au16[0] = uSrc1.au16[4]; + uDstOut.au16[1] = uSrc2.au16[4]; + uDstOut.au16[2] = uSrc1.au16[5]; + uDstOut.au16[3] = uSrc2.au16[5]; + uDstOut.au16[4] = uSrc1.au16[6]; + uDstOut.au16[5] = uSrc2.au16[6]; + uDstOut.au16[6] = uSrc1.au16[7]; + uDstOut.au16[7] = uSrc2.au16[7]; + + uDstOut.au16[8] = uSrc1.au16[12]; + uDstOut.au16[9] = uSrc2.au16[12]; + uDstOut.au16[10] = uSrc1.au16[13]; + uDstOut.au16[11] = uSrc2.au16[13]; + uDstOut.au16[12] = uSrc1.au16[14]; + uDstOut.au16[13] = uSrc2.au16[14]; + uDstOut.au16[14] = uSrc1.au16[15]; + uDstOut.au16[15] = uSrc2.au16[15]; + *puDst = uDstOut; +} + + +/* + * PUNPCKHBW - high dwords -> qword(s) + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhdq_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc2 = { *puSrc }; + RTUINT64U const uSrc1 = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut; + uDstOut.au32[0] = uSrc1.au32[1]; + uDstOut.au32[1] = uSrc2.au32[1]; + *puDst = uDstOut.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhdq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au32[0] = uSrc1.au32[2]; + uDstOut.au32[1] = uSrc2.au32[2]; + uDstOut.au32[2] = uSrc1.au32[3]; + uDstOut.au32[3] = uSrc2.au32[3]; + *puDst = uDstOut; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au32[0] = uSrc1.au32[2]; + uDstOut.au32[1] = uSrc2.au32[2]; + uDstOut.au32[2] = uSrc1.au32[3]; + uDstOut.au32[3] = uSrc2.au32[3]; + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au32[0] = uSrc1.au32[2]; + uDstOut.au32[1] = uSrc2.au32[2]; + uDstOut.au32[2] = uSrc1.au32[3]; + uDstOut.au32[3] = uSrc2.au32[3]; + + uDstOut.au32[4] = uSrc1.au32[6]; + uDstOut.au32[5] = uSrc2.au32[6]; + uDstOut.au32[6] = uSrc1.au32[7]; + uDstOut.au32[7] = uSrc2.au32[7]; + *puDst = uDstOut; +} + + +/* + * PUNPCKHQDQ -> High qwords -> double qword(s). + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhqdq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au64[0] = uSrc1.au64[1]; + uDstOut.au64[1] = uSrc2.au64[1]; + *puDst = uDstOut; +} +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhqdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au64[0] = uSrc1.au64[1]; + uDstOut.au64[1] = uSrc2.au64[1]; + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhqdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au64[0] = uSrc1.au64[1]; + uDstOut.au64[1] = uSrc2.au64[1]; + + uDstOut.au64[2] = uSrc1.au64[3]; + uDstOut.au64[3] = uSrc2.au64[3]; + *puDst = uDstOut; +} + + +/* + * PUNPCKLBW - low bytes -> words + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklbw_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc2 = { *puSrc }; + RTUINT64U const uSrc1 = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut; + uDstOut.au8[0] = uSrc1.au8[0]; + uDstOut.au8[1] = uSrc2.au8[0]; + uDstOut.au8[2] = uSrc1.au8[1]; + uDstOut.au8[3] = uSrc2.au8[1]; + uDstOut.au8[4] = uSrc1.au8[2]; + uDstOut.au8[5] = uSrc2.au8[2]; + uDstOut.au8[6] = uSrc1.au8[3]; + uDstOut.au8[7] = uSrc2.au8[3]; + *puDst = uDstOut.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklbw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au8[ 0] = uSrc1.au8[0]; + uDstOut.au8[ 1] = uSrc2.au8[0]; + uDstOut.au8[ 2] = uSrc1.au8[1]; + uDstOut.au8[ 3] = uSrc2.au8[1]; + uDstOut.au8[ 4] = uSrc1.au8[2]; + uDstOut.au8[ 5] = uSrc2.au8[2]; + uDstOut.au8[ 6] = uSrc1.au8[3]; + uDstOut.au8[ 7] = uSrc2.au8[3]; + uDstOut.au8[ 8] = uSrc1.au8[4]; + uDstOut.au8[ 9] = uSrc2.au8[4]; + uDstOut.au8[10] = uSrc1.au8[5]; + uDstOut.au8[11] = uSrc2.au8[5]; + uDstOut.au8[12] = uSrc1.au8[6]; + uDstOut.au8[13] = uSrc2.au8[6]; + uDstOut.au8[14] = uSrc1.au8[7]; + uDstOut.au8[15] = uSrc2.au8[7]; + *puDst = uDstOut; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklbw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au8[ 0] = uSrc1.au8[0]; + uDstOut.au8[ 1] = uSrc2.au8[0]; + uDstOut.au8[ 2] = uSrc1.au8[1]; + uDstOut.au8[ 3] = uSrc2.au8[1]; + uDstOut.au8[ 4] = uSrc1.au8[2]; + uDstOut.au8[ 5] = uSrc2.au8[2]; + uDstOut.au8[ 6] = uSrc1.au8[3]; + uDstOut.au8[ 7] = uSrc2.au8[3]; + uDstOut.au8[ 8] = uSrc1.au8[4]; + uDstOut.au8[ 9] = uSrc2.au8[4]; + uDstOut.au8[10] = uSrc1.au8[5]; + uDstOut.au8[11] = uSrc2.au8[5]; + uDstOut.au8[12] = uSrc1.au8[6]; + uDstOut.au8[13] = uSrc2.au8[6]; + uDstOut.au8[14] = uSrc1.au8[7]; + uDstOut.au8[15] = uSrc2.au8[7]; + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au8[ 0] = uSrc1.au8[0]; + uDstOut.au8[ 1] = uSrc2.au8[0]; + uDstOut.au8[ 2] = uSrc1.au8[1]; + uDstOut.au8[ 3] = uSrc2.au8[1]; + uDstOut.au8[ 4] = uSrc1.au8[2]; + uDstOut.au8[ 5] = uSrc2.au8[2]; + uDstOut.au8[ 6] = uSrc1.au8[3]; + uDstOut.au8[ 7] = uSrc2.au8[3]; + uDstOut.au8[ 8] = uSrc1.au8[4]; + uDstOut.au8[ 9] = uSrc2.au8[4]; + uDstOut.au8[10] = uSrc1.au8[5]; + uDstOut.au8[11] = uSrc2.au8[5]; + uDstOut.au8[12] = uSrc1.au8[6]; + uDstOut.au8[13] = uSrc2.au8[6]; + uDstOut.au8[14] = uSrc1.au8[7]; + uDstOut.au8[15] = uSrc2.au8[7]; + /* As usual, the upper 128-bits are treated like a parallel register to the lower half. */ + uDstOut.au8[16] = uSrc1.au8[16]; + uDstOut.au8[17] = uSrc2.au8[16]; + uDstOut.au8[18] = uSrc1.au8[17]; + uDstOut.au8[19] = uSrc2.au8[17]; + uDstOut.au8[20] = uSrc1.au8[18]; + uDstOut.au8[21] = uSrc2.au8[18]; + uDstOut.au8[22] = uSrc1.au8[19]; + uDstOut.au8[23] = uSrc2.au8[19]; + uDstOut.au8[24] = uSrc1.au8[20]; + uDstOut.au8[25] = uSrc2.au8[20]; + uDstOut.au8[26] = uSrc1.au8[21]; + uDstOut.au8[27] = uSrc2.au8[21]; + uDstOut.au8[28] = uSrc1.au8[22]; + uDstOut.au8[29] = uSrc2.au8[22]; + uDstOut.au8[30] = uSrc1.au8[23]; + uDstOut.au8[31] = uSrc2.au8[23]; + *puDst = uDstOut; +} + + +/* + * PUNPCKLBW - low words -> dwords + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklwd_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc2 = { *puSrc }; + RTUINT64U const uSrc1 = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut; + uDstOut.au16[0] = uSrc1.au16[0]; + uDstOut.au16[1] = uSrc2.au16[0]; + uDstOut.au16[2] = uSrc1.au16[1]; + uDstOut.au16[3] = uSrc2.au16[1]; + *puDst = uDstOut.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklwd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au16[0] = uSrc1.au16[0]; + uDstOut.au16[1] = uSrc2.au16[0]; + uDstOut.au16[2] = uSrc1.au16[1]; + uDstOut.au16[3] = uSrc2.au16[1]; + uDstOut.au16[4] = uSrc1.au16[2]; + uDstOut.au16[5] = uSrc2.au16[2]; + uDstOut.au16[6] = uSrc1.au16[3]; + uDstOut.au16[7] = uSrc2.au16[3]; + *puDst = uDstOut; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklwd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au16[0] = uSrc1.au16[0]; + uDstOut.au16[1] = uSrc2.au16[0]; + uDstOut.au16[2] = uSrc1.au16[1]; + uDstOut.au16[3] = uSrc2.au16[1]; + uDstOut.au16[4] = uSrc1.au16[2]; + uDstOut.au16[5] = uSrc2.au16[2]; + uDstOut.au16[6] = uSrc1.au16[3]; + uDstOut.au16[7] = uSrc2.au16[3]; + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au16[0] = uSrc1.au16[0]; + uDstOut.au16[1] = uSrc2.au16[0]; + uDstOut.au16[2] = uSrc1.au16[1]; + uDstOut.au16[3] = uSrc2.au16[1]; + uDstOut.au16[4] = uSrc1.au16[2]; + uDstOut.au16[5] = uSrc2.au16[2]; + uDstOut.au16[6] = uSrc1.au16[3]; + uDstOut.au16[7] = uSrc2.au16[3]; + + uDstOut.au16[8] = uSrc1.au16[8]; + uDstOut.au16[9] = uSrc2.au16[8]; + uDstOut.au16[10] = uSrc1.au16[9]; + uDstOut.au16[11] = uSrc2.au16[9]; + uDstOut.au16[12] = uSrc1.au16[10]; + uDstOut.au16[13] = uSrc2.au16[10]; + uDstOut.au16[14] = uSrc1.au16[11]; + uDstOut.au16[15] = uSrc2.au16[11]; + *puDst = uDstOut; +} + + +/* + * PUNPCKLBW - low dwords -> qword(s) + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpckldq_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc2 = { *puSrc }; + RTUINT64U const uSrc1 = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut; + uDstOut.au32[0] = uSrc1.au32[0]; + uDstOut.au32[1] = uSrc2.au32[0]; + *puDst = uDstOut.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_punpckldq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au32[0] = uSrc1.au32[0]; + uDstOut.au32[1] = uSrc2.au32[0]; + uDstOut.au32[2] = uSrc1.au32[1]; + uDstOut.au32[3] = uSrc2.au32[1]; + *puDst = uDstOut; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckldq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au32[0] = uSrc1.au32[0]; + uDstOut.au32[1] = uSrc2.au32[0]; + uDstOut.au32[2] = uSrc1.au32[1]; + uDstOut.au32[3] = uSrc2.au32[1]; + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckldq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au32[0] = uSrc1.au32[0]; + uDstOut.au32[1] = uSrc2.au32[0]; + uDstOut.au32[2] = uSrc1.au32[1]; + uDstOut.au32[3] = uSrc2.au32[1]; + + uDstOut.au32[4] = uSrc1.au32[4]; + uDstOut.au32[5] = uSrc2.au32[4]; + uDstOut.au32[6] = uSrc1.au32[5]; + uDstOut.au32[7] = uSrc2.au32[5]; + *puDst = uDstOut; +} + + +/* + * PUNPCKLQDQ -> Low qwords -> double qword(s). + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklqdq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au64[0] = uSrc1.au64[0]; + uDstOut.au64[1] = uSrc2.au64[0]; + *puDst = uDstOut; +} +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklqdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au64[0] = uSrc1.au64[0]; + uDstOut.au64[1] = uSrc2.au64[0]; + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklqdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au64[0] = uSrc1.au64[0]; + uDstOut.au64[1] = uSrc2.au64[0]; + + uDstOut.au64[2] = uSrc1.au64[2]; + uDstOut.au64[3] = uSrc2.au64[2]; + *puDst = uDstOut; +} + + +/* + * PACKSSWB - signed words -> signed bytes + */ + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_packsswb_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc2 = { *puSrc }; + RTUINT64U const uSrc1 = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut; + uDstOut.au8[0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[0]); + uDstOut.au8[1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[1]); + uDstOut.au8[2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[2]); + uDstOut.au8[3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[3]); + uDstOut.au8[4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[0]); + uDstOut.au8[5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[1]); + uDstOut.au8[6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[2]); + uDstOut.au8[7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[3]); + *puDst = uDstOut.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_packsswb_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[0]); + uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[1]); + uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[2]); + uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[3]); + uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[4]); + uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[5]); + uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[6]); + uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[7]); + uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[0]); + uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[1]); + uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[2]); + uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[3]); + uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[4]); + uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[5]); + uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[6]); + uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[7]); + *puDst = uDstOut; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpacksswb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[0]); + uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[1]); + uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[2]); + uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[3]); + uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[4]); + uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[5]); + uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[6]); + uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[7]); + uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[0]); + uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[1]); + uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[2]); + uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[3]); + uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[4]); + uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[5]); + uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[6]); + uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[7]); + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpacksswb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[0]); + uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[1]); + uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[2]); + uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[3]); + uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[4]); + uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[5]); + uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[6]); + uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[7]); + uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[0]); + uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[1]); + uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[2]); + uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[3]); + uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[4]); + uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[5]); + uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[6]); + uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[7]); + + uDstOut.au8[16] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[ 8]); + uDstOut.au8[17] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[ 9]); + uDstOut.au8[18] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[10]); + uDstOut.au8[19] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[11]); + uDstOut.au8[20] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[12]); + uDstOut.au8[21] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[13]); + uDstOut.au8[22] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[14]); + uDstOut.au8[23] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[15]); + uDstOut.au8[24] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[ 8]); + uDstOut.au8[25] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[ 9]); + uDstOut.au8[26] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[10]); + uDstOut.au8[27] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[11]); + uDstOut.au8[28] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[12]); + uDstOut.au8[29] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[13]); + uDstOut.au8[30] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[14]); + uDstOut.au8[31] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[15]); + *puDst = uDstOut; +} + + +/* + * PACKUSWB - signed words -> unsigned bytes + */ +#define SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(a_iWord) \ + ( (uint16_t)(a_iWord) <= (uint16_t)0xff \ + ? (uint8_t)(a_iWord) \ + : (uint8_t)0xff * (uint8_t)((((a_iWord) >> 15) & 1) ^ 1) ) /* 0xff = UINT8_MAX; 0x00 == UINT8_MIN; source bit 15 = sign */ + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_packuswb_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc2 = { *puSrc }; + RTUINT64U const uSrc1 = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut; + uDstOut.au8[0] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[0]); + uDstOut.au8[1] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[1]); + uDstOut.au8[2] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[2]); + uDstOut.au8[3] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[3]); + uDstOut.au8[4] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[0]); + uDstOut.au8[5] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[1]); + uDstOut.au8[6] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[2]); + uDstOut.au8[7] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[3]); + *puDst = uDstOut.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_packuswb_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[0]); + uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[1]); + uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[2]); + uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[3]); + uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[4]); + uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[5]); + uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[6]); + uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[7]); + uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[0]); + uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[1]); + uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[2]); + uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[3]); + uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[4]); + uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[5]); + uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[6]); + uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[7]); + *puDst = uDstOut; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpackuswb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[0]); + uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[1]); + uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[2]); + uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[3]); + uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[4]); + uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[5]); + uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[6]); + uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[7]); + uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[0]); + uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[1]); + uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[2]); + uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[3]); + uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[4]); + uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[5]); + uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[6]); + uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[7]); + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpackuswb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[0]); + uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[1]); + uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[2]); + uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[3]); + uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[4]); + uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[5]); + uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[6]); + uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[7]); + uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[0]); + uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[1]); + uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[2]); + uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[3]); + uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[4]); + uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[5]); + uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[6]); + uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[7]); + + uDstOut.au8[16] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[ 8]); + uDstOut.au8[17] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[ 9]); + uDstOut.au8[18] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[10]); + uDstOut.au8[19] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[11]); + uDstOut.au8[20] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[12]); + uDstOut.au8[21] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[13]); + uDstOut.au8[22] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[14]); + uDstOut.au8[23] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[15]); + uDstOut.au8[24] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[ 8]); + uDstOut.au8[25] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[ 9]); + uDstOut.au8[26] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[10]); + uDstOut.au8[27] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[11]); + uDstOut.au8[28] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[12]); + uDstOut.au8[29] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[13]); + uDstOut.au8[30] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[14]); + uDstOut.au8[31] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[15]); + *puDst = uDstOut; +} + + +/* + * PACKSSDW - signed dwords -> signed words + */ + +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_packssdw_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc2 = { *puSrc }; + RTUINT64U const uSrc1 = { *puDst }; + ASMCompilerBarrier(); + RTUINT64U uDstOut; + uDstOut.au16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[0]); + uDstOut.au16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[1]); + uDstOut.au16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[0]); + uDstOut.au16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[1]); + *puDst = uDstOut.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_packssdw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[0]); + uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[1]); + uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[2]); + uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[3]); + uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[0]); + uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[1]); + uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[2]); + uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[3]); + *puDst = uDstOut; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpackssdw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[0]); + uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[1]); + uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[2]); + uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[3]); + uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[0]); + uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[1]); + uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[2]); + uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[3]); + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpackssdw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[0]); + uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[1]); + uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[2]); + uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[3]); + uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[0]); + uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[1]); + uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[2]); + uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[3]); + + uDstOut.au16[ 8] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[4]); + uDstOut.au16[ 9] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[5]); + uDstOut.au16[10] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[6]); + uDstOut.au16[11] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[7]); + uDstOut.au16[12] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[4]); + uDstOut.au16[13] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[5]); + uDstOut.au16[14] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[6]); + uDstOut.au16[15] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[7]); + *puDst = uDstOut; +} + + +/* + * PACKUSDW - signed dwords -> unsigned words + */ +#define SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(a_iDword) \ + ( (uint32_t)(a_iDword) <= (uint16_t)0xffff \ + ? (uint16_t)(a_iDword) \ + : (uint16_t)0xffff * (uint16_t)((((a_iDword) >> 31) & 1) ^ 1) ) /* 0xffff = UINT16_MAX; source bit 31 = sign */ + +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_packusdw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U const uSrc2 = *puSrc; + RTUINT128U const uSrc1 = *puDst; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[0]); + uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[1]); + uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[2]); + uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[3]); + uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[0]); + uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[1]); + uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[2]); + uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[3]); + *puDst = uDstOut; +} +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpackusdw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U const uSrc2 = *puSrc2; + RTUINT128U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT128U uDstOut; + uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[0]); + uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[1]); + uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[2]); + uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[3]); + uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[0]); + uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[1]); + uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[2]); + uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[3]); + *puDst = uDstOut; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpackusdw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U const uSrc2 = *puSrc2; + RTUINT256U const uSrc1 = *puSrc1; + ASMCompilerBarrier(); + RTUINT256U uDstOut; + uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[0]); + uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[1]); + uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[2]); + uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[3]); + uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[0]); + uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[1]); + uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[2]); + uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[3]); + + uDstOut.au16[ 8] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[4]); + uDstOut.au16[ 9] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[5]); + uDstOut.au16[10] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[6]); + uDstOut.au16[11] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[7]); + uDstOut.au16[12] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[4]); + uDstOut.au16[13] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[5]); + uDstOut.au16[14] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[6]); + uDstOut.au16[15] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[7]); + *puDst = uDstOut; +} + + +/* + * [V]PABSB / [V]PABSW / [V]PABSD + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_pabsb_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc = { *puSrc }; + RTUINT64U uDstOut = { 0 }; + + uDstOut.au8[0] = RT_ABS(uSrc.ai8[0]); + uDstOut.au8[1] = RT_ABS(uSrc.ai8[1]); + uDstOut.au8[2] = RT_ABS(uSrc.ai8[2]); + uDstOut.au8[3] = RT_ABS(uSrc.ai8[3]); + uDstOut.au8[4] = RT_ABS(uSrc.ai8[4]); + uDstOut.au8[5] = RT_ABS(uSrc.ai8[5]); + uDstOut.au8[6] = RT_ABS(uSrc.ai8[6]); + uDstOut.au8[7] = RT_ABS(uSrc.ai8[7]); + *puDst = uDstOut.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pabsb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + puDst->au8[ 0] = RT_ABS(puSrc->ai8[ 0]); + puDst->au8[ 1] = RT_ABS(puSrc->ai8[ 1]); + puDst->au8[ 2] = RT_ABS(puSrc->ai8[ 2]); + puDst->au8[ 3] = RT_ABS(puSrc->ai8[ 3]); + puDst->au8[ 4] = RT_ABS(puSrc->ai8[ 4]); + puDst->au8[ 5] = RT_ABS(puSrc->ai8[ 5]); + puDst->au8[ 6] = RT_ABS(puSrc->ai8[ 6]); + puDst->au8[ 7] = RT_ABS(puSrc->ai8[ 7]); + puDst->au8[ 8] = RT_ABS(puSrc->ai8[ 8]); + puDst->au8[ 9] = RT_ABS(puSrc->ai8[ 9]); + puDst->au8[10] = RT_ABS(puSrc->ai8[10]); + puDst->au8[11] = RT_ABS(puSrc->ai8[11]); + puDst->au8[12] = RT_ABS(puSrc->ai8[12]); + puDst->au8[13] = RT_ABS(puSrc->ai8[13]); + puDst->au8[14] = RT_ABS(puSrc->ai8[14]); + puDst->au8[15] = RT_ABS(puSrc->ai8[15]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pabsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc = { *puSrc }; + RTUINT64U uDstOut = { 0 }; + + uDstOut.au16[0] = RT_ABS(uSrc.ai16[0]); + uDstOut.au16[1] = RT_ABS(uSrc.ai16[1]); + uDstOut.au16[2] = RT_ABS(uSrc.ai16[2]); + uDstOut.au16[3] = RT_ABS(uSrc.ai16[3]); + *puDst = uDstOut.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pabsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + puDst->au16[ 0] = RT_ABS(puSrc->ai16[ 0]); + puDst->au16[ 1] = RT_ABS(puSrc->ai16[ 1]); + puDst->au16[ 2] = RT_ABS(puSrc->ai16[ 2]); + puDst->au16[ 3] = RT_ABS(puSrc->ai16[ 3]); + puDst->au16[ 4] = RT_ABS(puSrc->ai16[ 4]); + puDst->au16[ 5] = RT_ABS(puSrc->ai16[ 5]); + puDst->au16[ 6] = RT_ABS(puSrc->ai16[ 6]); + puDst->au16[ 7] = RT_ABS(puSrc->ai16[ 7]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pabsd_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U const uSrc = { *puSrc }; + RTUINT64U uDstOut = { 0 }; + + uDstOut.au32[0] = RT_ABS(uSrc.ai32[0]); + uDstOut.au32[1] = RT_ABS(uSrc.ai32[1]); + *puDst = uDstOut.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pabsd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + puDst->au32[ 0] = RT_ABS(puSrc->ai32[ 0]); + puDst->au32[ 1] = RT_ABS(puSrc->ai32[ 1]); + puDst->au32[ 2] = RT_ABS(puSrc->ai32[ 2]); + puDst->au32[ 3] = RT_ABS(puSrc->ai32[ 3]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + puDst->au8[ 0] = RT_ABS(puSrc->ai8[ 0]); + puDst->au8[ 1] = RT_ABS(puSrc->ai8[ 1]); + puDst->au8[ 2] = RT_ABS(puSrc->ai8[ 2]); + puDst->au8[ 3] = RT_ABS(puSrc->ai8[ 3]); + puDst->au8[ 4] = RT_ABS(puSrc->ai8[ 4]); + puDst->au8[ 5] = RT_ABS(puSrc->ai8[ 5]); + puDst->au8[ 6] = RT_ABS(puSrc->ai8[ 6]); + puDst->au8[ 7] = RT_ABS(puSrc->ai8[ 7]); + puDst->au8[ 8] = RT_ABS(puSrc->ai8[ 8]); + puDst->au8[ 9] = RT_ABS(puSrc->ai8[ 9]); + puDst->au8[10] = RT_ABS(puSrc->ai8[10]); + puDst->au8[11] = RT_ABS(puSrc->ai8[11]); + puDst->au8[12] = RT_ABS(puSrc->ai8[12]); + puDst->au8[13] = RT_ABS(puSrc->ai8[13]); + puDst->au8[14] = RT_ABS(puSrc->ai8[14]); + puDst->au8[15] = RT_ABS(puSrc->ai8[15]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc)) +{ + puDst->au8[ 0] = RT_ABS(puSrc->ai8[ 0]); + puDst->au8[ 1] = RT_ABS(puSrc->ai8[ 1]); + puDst->au8[ 2] = RT_ABS(puSrc->ai8[ 2]); + puDst->au8[ 3] = RT_ABS(puSrc->ai8[ 3]); + puDst->au8[ 4] = RT_ABS(puSrc->ai8[ 4]); + puDst->au8[ 5] = RT_ABS(puSrc->ai8[ 5]); + puDst->au8[ 6] = RT_ABS(puSrc->ai8[ 6]); + puDst->au8[ 7] = RT_ABS(puSrc->ai8[ 7]); + puDst->au8[ 8] = RT_ABS(puSrc->ai8[ 8]); + puDst->au8[ 9] = RT_ABS(puSrc->ai8[ 9]); + puDst->au8[10] = RT_ABS(puSrc->ai8[10]); + puDst->au8[11] = RT_ABS(puSrc->ai8[11]); + puDst->au8[12] = RT_ABS(puSrc->ai8[12]); + puDst->au8[13] = RT_ABS(puSrc->ai8[13]); + puDst->au8[14] = RT_ABS(puSrc->ai8[14]); + puDst->au8[15] = RT_ABS(puSrc->ai8[15]); + puDst->au8[16] = RT_ABS(puSrc->ai8[16]); + puDst->au8[17] = RT_ABS(puSrc->ai8[17]); + puDst->au8[18] = RT_ABS(puSrc->ai8[18]); + puDst->au8[19] = RT_ABS(puSrc->ai8[19]); + puDst->au8[20] = RT_ABS(puSrc->ai8[20]); + puDst->au8[21] = RT_ABS(puSrc->ai8[21]); + puDst->au8[22] = RT_ABS(puSrc->ai8[22]); + puDst->au8[23] = RT_ABS(puSrc->ai8[23]); + puDst->au8[24] = RT_ABS(puSrc->ai8[24]); + puDst->au8[25] = RT_ABS(puSrc->ai8[25]); + puDst->au8[26] = RT_ABS(puSrc->ai8[26]); + puDst->au8[27] = RT_ABS(puSrc->ai8[27]); + puDst->au8[28] = RT_ABS(puSrc->ai8[28]); + puDst->au8[29] = RT_ABS(puSrc->ai8[29]); + puDst->au8[30] = RT_ABS(puSrc->ai8[30]); + puDst->au8[31] = RT_ABS(puSrc->ai8[31]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + puDst->au16[ 0] = RT_ABS(puSrc->ai16[ 0]); + puDst->au16[ 1] = RT_ABS(puSrc->ai16[ 1]); + puDst->au16[ 2] = RT_ABS(puSrc->ai16[ 2]); + puDst->au16[ 3] = RT_ABS(puSrc->ai16[ 3]); + puDst->au16[ 4] = RT_ABS(puSrc->ai16[ 4]); + puDst->au16[ 5] = RT_ABS(puSrc->ai16[ 5]); + puDst->au16[ 6] = RT_ABS(puSrc->ai16[ 6]); + puDst->au16[ 7] = RT_ABS(puSrc->ai16[ 7]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc)) +{ + puDst->au16[ 0] = RT_ABS(puSrc->ai16[ 0]); + puDst->au16[ 1] = RT_ABS(puSrc->ai16[ 1]); + puDst->au16[ 2] = RT_ABS(puSrc->ai16[ 2]); + puDst->au16[ 3] = RT_ABS(puSrc->ai16[ 3]); + puDst->au16[ 4] = RT_ABS(puSrc->ai16[ 4]); + puDst->au16[ 5] = RT_ABS(puSrc->ai16[ 5]); + puDst->au16[ 6] = RT_ABS(puSrc->ai16[ 6]); + puDst->au16[ 7] = RT_ABS(puSrc->ai16[ 7]); + puDst->au16[ 8] = RT_ABS(puSrc->ai16[ 8]); + puDst->au16[ 9] = RT_ABS(puSrc->ai16[ 9]); + puDst->au16[10] = RT_ABS(puSrc->ai16[10]); + puDst->au16[11] = RT_ABS(puSrc->ai16[11]); + puDst->au16[12] = RT_ABS(puSrc->ai16[12]); + puDst->au16[13] = RT_ABS(puSrc->ai16[13]); + puDst->au16[14] = RT_ABS(puSrc->ai16[14]); + puDst->au16[15] = RT_ABS(puSrc->ai16[15]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + puDst->au32[ 0] = RT_ABS(puSrc->ai32[ 0]); + puDst->au32[ 1] = RT_ABS(puSrc->ai32[ 1]); + puDst->au32[ 2] = RT_ABS(puSrc->ai32[ 2]); + puDst->au32[ 3] = RT_ABS(puSrc->ai32[ 3]); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc)) +{ + puDst->au32[ 0] = RT_ABS(puSrc->ai32[ 0]); + puDst->au32[ 1] = RT_ABS(puSrc->ai32[ 1]); + puDst->au32[ 2] = RT_ABS(puSrc->ai32[ 2]); + puDst->au32[ 3] = RT_ABS(puSrc->ai32[ 3]); + puDst->au32[ 4] = RT_ABS(puSrc->ai32[ 4]); + puDst->au32[ 5] = RT_ABS(puSrc->ai32[ 5]); + puDst->au32[ 6] = RT_ABS(puSrc->ai32[ 6]); + puDst->au32[ 7] = RT_ABS(puSrc->ai32[ 7]); +} + + +/* + * PSIGNB / VPSIGNB / PSIGNW / VPSIGNW / PSIGND / VPSIGND + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_psignb_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + for (uint32_t i = 0; i < RT_ELEMENTS(uDst.ai8); i++) + { + if (uSrc2.ai8[i] < 0) + uDst.ai8[i] = -uSrc1.ai8[i]; + else if (uSrc2.ai8[i] == 0) + uDst.ai8[i] = 0; + else /* uSrc2.ai8[i] > 0 */ + uDst.ai8[i] = uSrc1.ai8[i]; + } + + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psignb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai8); i++) + { + if (puSrc->ai8[i] < 0) + puDst->ai8[i] = -uSrc1.ai8[i]; + else if (puSrc->ai8[i] == 0) + puDst->ai8[i] = 0; + else /* puSrc->ai8[i] > 0 */ + puDst->ai8[i] = uSrc1.ai8[i]; + } + + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psignw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + for (uint32_t i = 0; i < RT_ELEMENTS(uDst.ai16); i++) + { + if (uSrc2.ai16[i] < 0) + uDst.ai16[i] = -uSrc1.ai16[i]; + else if (uSrc2.ai16[i] == 0) + uDst.ai16[i] = 0; + else /* uSrc2.ai16[i] > 0 */ + uDst.ai16[i] = uSrc1.ai16[i]; + } + + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psignw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai16); i++) + { + if (puSrc->ai16[i] < 0) + puDst->ai16[i] = -uSrc1.ai16[i]; + else if (puSrc->ai16[i] == 0) + puDst->ai16[i] = 0; + else /* puSrc->ai16[i] > 0 */ + puDst->ai16[i] = uSrc1.ai16[i]; + } + + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psignd_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + for (uint32_t i = 0; i < RT_ELEMENTS(uDst.ai32); i++) + { + if (uSrc2.ai32[i] < 0) + uDst.ai32[i] = -uSrc1.ai32[i]; + else if (uSrc2.ai32[i] == 0) + uDst.ai32[i] = 0; + else /* uSrc2.ai32[i] > 0 */ + uDst.ai32[i] = uSrc1.ai32[i]; + } + + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psignd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai32); i++) + { + if (puSrc->ai32[i] < 0) + puDst->ai32[i] = -uSrc1.ai32[i]; + else if (puSrc->ai32[i] == 0) + puDst->ai32[i] = 0; + else /* puSrc->ai32[i] > 0 */ + puDst->ai32[i] = uSrc1.ai32[i]; + } + + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai8); i++) + { + if (puSrc2->ai8[i] < 0) + puDst->ai8[i] = -puSrc1->ai8[i]; + else if (puSrc2->ai8[i] == 0) + puDst->ai8[i] = 0; + else /* puSrc2->ai8[i] > 0 */ + puDst->ai8[i] = puSrc1->ai8[i]; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai8); i++) + { + if (puSrc2->ai8[i] < 0) + puDst->ai8[i] = -puSrc1->ai8[i]; + else if (puSrc2->ai8[i] == 0) + puDst->ai8[i] = 0; + else /* puSrc2->ai8[i] > 0 */ + puDst->ai8[i] = puSrc1->ai8[i]; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai16); i++) + { + if (puSrc2->ai16[i] < 0) + puDst->ai16[i] = -puSrc1->ai16[i]; + else if (puSrc2->ai16[i] == 0) + puDst->ai16[i] = 0; + else /* puSrc2->ai16[i] > 0 */ + puDst->ai16[i] = puSrc1->ai16[i]; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai16); i++) + { + if (puSrc2->ai16[i] < 0) + puDst->ai16[i] = -puSrc1->ai16[i]; + else if (puSrc2->ai16[i] == 0) + puDst->ai16[i] = 0; + else /* puSrc2->ai16[i] > 0 */ + puDst->ai16[i] = puSrc1->ai16[i]; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai32); i++) + { + if (puSrc2->ai32[i] < 0) + puDst->ai32[i] = -puSrc1->ai32[i]; + else if (puSrc2->ai32[i] == 0) + puDst->ai32[i] = 0; + else /* puSrc2->ai32[i] > 0 */ + puDst->ai32[i] = puSrc1->ai32[i]; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai32); i++) + { + if (puSrc2->ai32[i] < 0) + puDst->ai32[i] = -puSrc1->ai32[i]; + else if (puSrc2->ai32[i] == 0) + puDst->ai32[i] = 0; + else /* puSrc2->ai32[i] > 0 */ + puDst->ai32[i] = puSrc1->ai32[i]; + } +} + + +/* + * PHADDW / VPHADDW / PHADDD / VPHADDD + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_phaddw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + uDst.ai16[0] = uSrc1.ai16[0] + uSrc1.ai16[1]; + uDst.ai16[1] = uSrc1.ai16[2] + uSrc1.ai16[3]; + uDst.ai16[2] = uSrc2.ai16[0] + uSrc2.ai16[1]; + uDst.ai16[3] = uSrc2.ai16[2] + uSrc2.ai16[3]; + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_phaddw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai16[0] = uSrc1.ai16[0] + uSrc1.ai16[1]; + puDst->ai16[1] = uSrc1.ai16[2] + uSrc1.ai16[3]; + puDst->ai16[2] = uSrc1.ai16[4] + uSrc1.ai16[5]; + puDst->ai16[3] = uSrc1.ai16[6] + uSrc1.ai16[7]; + + puDst->ai16[4] = puSrc->ai16[0] + puSrc->ai16[1]; + puDst->ai16[5] = puSrc->ai16[2] + puSrc->ai16[3]; + puDst->ai16[6] = puSrc->ai16[4] + puSrc->ai16[5]; + puDst->ai16[7] = puSrc->ai16[6] + puSrc->ai16[7]; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_phaddd_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + uDst.ai32[0] = uSrc1.ai32[0] + uSrc1.ai32[1]; + uDst.ai32[1] = uSrc2.ai32[0] + uSrc2.ai32[1]; + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_phaddd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai32[0] = uSrc1.ai32[0] + uSrc1.ai32[1]; + puDst->ai32[1] = uSrc1.ai32[2] + uSrc1.ai32[3]; + + puDst->ai32[2] = puSrc->ai32[0] + puSrc->ai32[1]; + puDst->ai32[3] = puSrc->ai32[2] + puSrc->ai32[3]; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[0] = puSrc1->ai16[0] + puSrc1->ai16[1]; + uDst.ai16[1] = puSrc1->ai16[2] + puSrc1->ai16[3]; + uDst.ai16[2] = puSrc1->ai16[4] + puSrc1->ai16[5]; + uDst.ai16[3] = puSrc1->ai16[6] + puSrc1->ai16[7]; + + uDst.ai16[4] = puSrc2->ai16[0] + puSrc2->ai16[1]; + uDst.ai16[5] = puSrc2->ai16[2] + puSrc2->ai16[3]; + uDst.ai16[6] = puSrc2->ai16[4] + puSrc2->ai16[5]; + uDst.ai16[7] = puSrc2->ai16[6] + puSrc2->ai16[7]; + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[ 0] = puSrc1->ai16[ 0] + puSrc1->ai16[ 1]; + uDst.ai16[ 1] = puSrc1->ai16[ 2] + puSrc1->ai16[ 3]; + uDst.ai16[ 2] = puSrc1->ai16[ 4] + puSrc1->ai16[ 5]; + uDst.ai16[ 3] = puSrc1->ai16[ 6] + puSrc1->ai16[ 7]; + uDst.ai16[ 4] = puSrc2->ai16[ 0] + puSrc2->ai16[ 1]; + uDst.ai16[ 5] = puSrc2->ai16[ 2] + puSrc2->ai16[ 3]; + uDst.ai16[ 6] = puSrc2->ai16[ 4] + puSrc2->ai16[ 5]; + uDst.ai16[ 7] = puSrc2->ai16[ 6] + puSrc2->ai16[ 7]; + + uDst.ai16[ 8] = puSrc1->ai16[ 8] + puSrc1->ai16[ 9]; + uDst.ai16[ 9] = puSrc1->ai16[10] + puSrc1->ai16[11]; + uDst.ai16[10] = puSrc1->ai16[12] + puSrc1->ai16[13]; + uDst.ai16[11] = puSrc1->ai16[14] + puSrc1->ai16[15]; + uDst.ai16[12] = puSrc2->ai16[ 8] + puSrc2->ai16[ 9]; + uDst.ai16[13] = puSrc2->ai16[10] + puSrc2->ai16[11]; + uDst.ai16[14] = puSrc2->ai16[12] + puSrc2->ai16[13]; + uDst.ai16[15] = puSrc2->ai16[14] + puSrc2->ai16[15]; + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; + puDst->au64[2] = uDst.au64[2]; + puDst->au64[3] = uDst.au64[3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai32[0] = puSrc1->ai32[0] + puSrc1->ai32[1]; + uDst.ai32[1] = puSrc1->ai32[2] + puSrc1->ai32[3]; + + uDst.ai32[2] = puSrc2->ai32[0] + puSrc2->ai32[1]; + uDst.ai32[3] = puSrc2->ai32[2] + puSrc2->ai32[3]; + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai32[0] = puSrc1->ai32[ 0] + puSrc1->ai32[ 1]; + uDst.ai32[1] = puSrc1->ai32[ 2] + puSrc1->ai32[ 3]; + uDst.ai32[2] = puSrc2->ai32[ 0] + puSrc2->ai32[ 1]; + uDst.ai32[3] = puSrc2->ai32[ 2] + puSrc2->ai32[ 3]; + + uDst.ai32[4] = puSrc1->ai32[ 4] + puSrc1->ai32[ 5]; + uDst.ai32[5] = puSrc1->ai32[ 6] + puSrc1->ai32[ 7]; + uDst.ai32[6] = puSrc2->ai32[ 4] + puSrc2->ai32[ 5]; + uDst.ai32[7] = puSrc2->ai32[ 6] + puSrc2->ai32[ 7]; + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; + puDst->au64[2] = uDst.au64[2]; + puDst->au64[3] = uDst.au64[3]; +} + + +/* + * PHSUBW / VPHSUBW / PHSUBD / VPHSUBD + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_phsubw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + uDst.ai16[0] = uSrc1.ai16[0] - uSrc1.ai16[1]; + uDst.ai16[1] = uSrc1.ai16[2] - uSrc1.ai16[3]; + uDst.ai16[2] = uSrc2.ai16[0] - uSrc2.ai16[1]; + uDst.ai16[3] = uSrc2.ai16[2] - uSrc2.ai16[3]; + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_phsubw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai16[0] = uSrc1.ai16[0] - uSrc1.ai16[1]; + puDst->ai16[1] = uSrc1.ai16[2] - uSrc1.ai16[3]; + puDst->ai16[2] = uSrc1.ai16[4] - uSrc1.ai16[5]; + puDst->ai16[3] = uSrc1.ai16[6] - uSrc1.ai16[7]; + + puDst->ai16[4] = puSrc->ai16[0] - puSrc->ai16[1]; + puDst->ai16[5] = puSrc->ai16[2] - puSrc->ai16[3]; + puDst->ai16[6] = puSrc->ai16[4] - puSrc->ai16[5]; + puDst->ai16[7] = puSrc->ai16[6] - puSrc->ai16[7]; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_phsubd_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + uDst.ai32[0] = uSrc1.ai32[0] - uSrc1.ai32[1]; + uDst.ai32[1] = uSrc2.ai32[0] - uSrc2.ai32[1]; + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_phsubd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai32[0] = uSrc1.ai32[0] - uSrc1.ai32[1]; + puDst->ai32[1] = uSrc1.ai32[2] - uSrc1.ai32[3]; + + puDst->ai32[2] = puSrc->ai32[0] - puSrc->ai32[1]; + puDst->ai32[3] = puSrc->ai32[2] - puSrc->ai32[3]; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[0] = puSrc1->ai16[0] - puSrc1->ai16[1]; + uDst.ai16[1] = puSrc1->ai16[2] - puSrc1->ai16[3]; + uDst.ai16[2] = puSrc1->ai16[4] - puSrc1->ai16[5]; + uDst.ai16[3] = puSrc1->ai16[6] - puSrc1->ai16[7]; + + uDst.ai16[4] = puSrc2->ai16[0] - puSrc2->ai16[1]; + uDst.ai16[5] = puSrc2->ai16[2] - puSrc2->ai16[3]; + uDst.ai16[6] = puSrc2->ai16[4] - puSrc2->ai16[5]; + uDst.ai16[7] = puSrc2->ai16[6] - puSrc2->ai16[7]; + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[ 0] = puSrc1->ai16[ 0] - puSrc1->ai16[ 1]; + uDst.ai16[ 1] = puSrc1->ai16[ 2] - puSrc1->ai16[ 3]; + uDst.ai16[ 2] = puSrc1->ai16[ 4] - puSrc1->ai16[ 5]; + uDst.ai16[ 3] = puSrc1->ai16[ 6] - puSrc1->ai16[ 7]; + uDst.ai16[ 4] = puSrc2->ai16[ 0] - puSrc2->ai16[ 1]; + uDst.ai16[ 5] = puSrc2->ai16[ 2] - puSrc2->ai16[ 3]; + uDst.ai16[ 6] = puSrc2->ai16[ 4] - puSrc2->ai16[ 5]; + uDst.ai16[ 7] = puSrc2->ai16[ 6] - puSrc2->ai16[ 7]; + + uDst.ai16[ 8] = puSrc1->ai16[ 8] - puSrc1->ai16[ 9]; + uDst.ai16[ 9] = puSrc1->ai16[10] - puSrc1->ai16[11]; + uDst.ai16[10] = puSrc1->ai16[12] - puSrc1->ai16[13]; + uDst.ai16[11] = puSrc1->ai16[14] - puSrc1->ai16[15]; + uDst.ai16[12] = puSrc2->ai16[ 8] - puSrc2->ai16[ 9]; + uDst.ai16[13] = puSrc2->ai16[10] - puSrc2->ai16[11]; + uDst.ai16[14] = puSrc2->ai16[12] - puSrc2->ai16[13]; + uDst.ai16[15] = puSrc2->ai16[14] - puSrc2->ai16[15]; + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; + puDst->au64[2] = uDst.au64[2]; + puDst->au64[3] = uDst.au64[3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai32[0] = puSrc1->ai32[0] - puSrc1->ai32[1]; + uDst.ai32[1] = puSrc1->ai32[2] - puSrc1->ai32[3]; + + uDst.ai32[2] = puSrc2->ai32[0] - puSrc2->ai32[1]; + uDst.ai32[3] = puSrc2->ai32[2] - puSrc2->ai32[3]; + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai32[0] = puSrc1->ai32[ 0] - puSrc1->ai32[ 1]; + uDst.ai32[1] = puSrc1->ai32[ 2] - puSrc1->ai32[ 3]; + uDst.ai32[2] = puSrc2->ai32[ 0] - puSrc2->ai32[ 1]; + uDst.ai32[3] = puSrc2->ai32[ 2] - puSrc2->ai32[ 3]; + + uDst.ai32[4] = puSrc1->ai32[ 4] - puSrc1->ai32[ 5]; + uDst.ai32[5] = puSrc1->ai32[ 6] - puSrc1->ai32[ 7]; + uDst.ai32[6] = puSrc2->ai32[ 4] - puSrc2->ai32[ 5]; + uDst.ai32[7] = puSrc2->ai32[ 6] - puSrc2->ai32[ 7]; + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; + puDst->au64[2] = uDst.au64[2]; + puDst->au64[3] = uDst.au64[3]; +} + + +/* + * PHADDSW / VPHADDSW + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_phaddsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] + uSrc1.ai16[1]); + uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] + uSrc1.ai16[3]); + uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.ai16[0] + uSrc2.ai16[1]); + uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.ai16[2] + uSrc2.ai16[3]); + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_phaddsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] + uSrc1.ai16[1]); + puDst->ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] + uSrc1.ai16[3]); + puDst->ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[4] + uSrc1.ai16[5]); + puDst->ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[6] + uSrc1.ai16[7]); + + puDst->ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[0] + puSrc->ai16[1]); + puDst->ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[2] + puSrc->ai16[3]); + puDst->ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[4] + puSrc->ai16[5]); + puDst->ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[6] + puSrc->ai16[7]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[0] + puSrc1->ai16[1]); + uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[2] + puSrc1->ai16[3]); + uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[4] + puSrc1->ai16[5]); + uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[6] + puSrc1->ai16[7]); + + uDst.ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[0] + puSrc2->ai16[1]); + uDst.ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[2] + puSrc2->ai16[3]); + uDst.ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[4] + puSrc2->ai16[5]); + uDst.ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[6] + puSrc2->ai16[7]); + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 0] + puSrc1->ai16[ 1]); + uDst.ai16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 2] + puSrc1->ai16[ 3]); + uDst.ai16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 4] + puSrc1->ai16[ 5]); + uDst.ai16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 6] + puSrc1->ai16[ 7]); + uDst.ai16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 0] + puSrc2->ai16[ 1]); + uDst.ai16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 2] + puSrc2->ai16[ 3]); + uDst.ai16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 4] + puSrc2->ai16[ 5]); + uDst.ai16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 6] + puSrc2->ai16[ 7]); + + uDst.ai16[ 8] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 8] + puSrc1->ai16[ 9]); + uDst.ai16[ 9] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[10] + puSrc1->ai16[11]); + uDst.ai16[10] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[12] + puSrc1->ai16[13]); + uDst.ai16[11] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[14] + puSrc1->ai16[15]); + uDst.ai16[12] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 8] + puSrc2->ai16[ 9]); + uDst.ai16[13] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[10] + puSrc2->ai16[11]); + uDst.ai16[14] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[12] + puSrc2->ai16[13]); + uDst.ai16[15] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[14] + puSrc2->ai16[15]); + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; + puDst->au64[2] = uDst.au64[2]; + puDst->au64[3] = uDst.au64[3]; +} + + +/* + * PHSUBSW / VPHSUBSW + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_phsubsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] - uSrc1.ai16[1]); + uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] - uSrc1.ai16[3]); + uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.ai16[0] - uSrc2.ai16[1]); + uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.ai16[2] - uSrc2.ai16[3]); + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_phsubsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] - uSrc1.ai16[1]); + puDst->ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] - uSrc1.ai16[3]); + puDst->ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[4] - uSrc1.ai16[5]); + puDst->ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[6] - uSrc1.ai16[7]); + + puDst->ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[0] - puSrc->ai16[1]); + puDst->ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[2] - puSrc->ai16[3]); + puDst->ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[4] - puSrc->ai16[5]); + puDst->ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[6] - puSrc->ai16[7]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[0] - puSrc1->ai16[1]); + uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[2] - puSrc1->ai16[3]); + uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[4] - puSrc1->ai16[5]); + uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[6] - puSrc1->ai16[7]); + + uDst.ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[0] - puSrc2->ai16[1]); + uDst.ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[2] - puSrc2->ai16[3]); + uDst.ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[4] - puSrc2->ai16[5]); + uDst.ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[6] - puSrc2->ai16[7]); + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 0] - puSrc1->ai16[ 1]); + uDst.ai16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 2] - puSrc1->ai16[ 3]); + uDst.ai16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 4] - puSrc1->ai16[ 5]); + uDst.ai16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 6] - puSrc1->ai16[ 7]); + uDst.ai16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 0] - puSrc2->ai16[ 1]); + uDst.ai16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 2] - puSrc2->ai16[ 3]); + uDst.ai16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 4] - puSrc2->ai16[ 5]); + uDst.ai16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 6] - puSrc2->ai16[ 7]); + + uDst.ai16[ 8] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 8] - puSrc1->ai16[ 9]); + uDst.ai16[ 9] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[10] - puSrc1->ai16[11]); + uDst.ai16[10] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[12] - puSrc1->ai16[13]); + uDst.ai16[11] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[14] - puSrc1->ai16[15]); + uDst.ai16[12] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 8] - puSrc2->ai16[ 9]); + uDst.ai16[13] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[10] - puSrc2->ai16[11]); + uDst.ai16[14] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[12] - puSrc2->ai16[13]); + uDst.ai16[15] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[14] - puSrc2->ai16[15]); + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; + puDst->au64[2] = uDst.au64[2]; + puDst->au64[3] = uDst.au64[3]; +} + + +/* + * PMADDUBSW / VPMADDUBSW + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaddubsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst = { 0 }; /* Shut up MSVC. */ + + uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[0] * uSrc2.ai8[0] + (uint16_t)uSrc1.au8[1] * uSrc2.ai8[1]); + uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[2] * uSrc2.ai8[2] + (uint16_t)uSrc1.au8[3] * uSrc2.ai8[3]); + uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[4] * uSrc2.ai8[4] + (uint16_t)uSrc1.au8[5] * uSrc2.ai8[5]); + uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[6] * uSrc2.ai8[6] + (uint16_t)uSrc1.au8[7] * uSrc2.ai8[7]); + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmaddubsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 0] * puSrc->ai8[ 0] + (uint16_t)uSrc1.au8[ 1] * puSrc->ai8[ 1]); + puDst->ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 2] * puSrc->ai8[ 2] + (uint16_t)uSrc1.au8[ 3] * puSrc->ai8[ 3]); + puDst->ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 4] * puSrc->ai8[ 4] + (uint16_t)uSrc1.au8[ 5] * puSrc->ai8[ 5]); + puDst->ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 6] * puSrc->ai8[ 6] + (uint16_t)uSrc1.au8[ 7] * puSrc->ai8[ 7]); + puDst->ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 8] * puSrc->ai8[ 8] + (uint16_t)uSrc1.au8[ 9] * puSrc->ai8[ 9]); + puDst->ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[10] * puSrc->ai8[10] + (uint16_t)uSrc1.au8[11] * puSrc->ai8[11]); + puDst->ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[12] * puSrc->ai8[12] + (uint16_t)uSrc1.au8[13] * puSrc->ai8[13]); + puDst->ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[14] * puSrc->ai8[14] + (uint16_t)uSrc1.au8[15] * puSrc->ai8[15]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaddubsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 0] * puSrc2->ai8[ 0] + (uint16_t)puSrc1->au8[ 1] * puSrc2->ai8[ 1]); + uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 2] * puSrc2->ai8[ 2] + (uint16_t)puSrc1->au8[ 3] * puSrc2->ai8[ 3]); + uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 4] * puSrc2->ai8[ 4] + (uint16_t)puSrc1->au8[ 5] * puSrc2->ai8[ 5]); + uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 6] * puSrc2->ai8[ 6] + (uint16_t)puSrc1->au8[ 7] * puSrc2->ai8[ 7]); + uDst.ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 8] * puSrc2->ai8[ 8] + (uint16_t)puSrc1->au8[ 9] * puSrc2->ai8[ 9]); + uDst.ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[10] * puSrc2->ai8[10] + (uint16_t)puSrc1->au8[11] * puSrc2->ai8[11]); + uDst.ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[12] * puSrc2->ai8[12] + (uint16_t)puSrc1->au8[13] * puSrc2->ai8[13]); + uDst.ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[14] * puSrc2->ai8[14] + (uint16_t)puSrc1->au8[15] * puSrc2->ai8[15]); + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaddubsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 0] * puSrc2->ai8[ 0] + (uint16_t)puSrc1->au8[ 1] * puSrc2->ai8[ 1]); + uDst.ai16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 2] * puSrc2->ai8[ 2] + (uint16_t)puSrc1->au8[ 3] * puSrc2->ai8[ 3]); + uDst.ai16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 4] * puSrc2->ai8[ 4] + (uint16_t)puSrc1->au8[ 5] * puSrc2->ai8[ 5]); + uDst.ai16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 6] * puSrc2->ai8[ 6] + (uint16_t)puSrc1->au8[ 7] * puSrc2->ai8[ 7]); + uDst.ai16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 8] * puSrc2->ai8[ 8] + (uint16_t)puSrc1->au8[ 9] * puSrc2->ai8[ 9]); + uDst.ai16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[10] * puSrc2->ai8[10] + (uint16_t)puSrc1->au8[11] * puSrc2->ai8[11]); + uDst.ai16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[12] * puSrc2->ai8[12] + (uint16_t)puSrc1->au8[13] * puSrc2->ai8[13]); + uDst.ai16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[14] * puSrc2->ai8[14] + (uint16_t)puSrc1->au8[15] * puSrc2->ai8[15]); + uDst.ai16[ 8] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[16] * puSrc2->ai8[16] + (uint16_t)puSrc1->au8[17] * puSrc2->ai8[17]); + uDst.ai16[ 9] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[18] * puSrc2->ai8[18] + (uint16_t)puSrc1->au8[19] * puSrc2->ai8[19]); + uDst.ai16[10] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[20] * puSrc2->ai8[20] + (uint16_t)puSrc1->au8[21] * puSrc2->ai8[21]); + uDst.ai16[11] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[22] * puSrc2->ai8[22] + (uint16_t)puSrc1->au8[23] * puSrc2->ai8[23]); + uDst.ai16[12] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[24] * puSrc2->ai8[24] + (uint16_t)puSrc1->au8[25] * puSrc2->ai8[25]); + uDst.ai16[13] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[26] * puSrc2->ai8[26] + (uint16_t)puSrc1->au8[27] * puSrc2->ai8[27]); + uDst.ai16[14] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[28] * puSrc2->ai8[28] + (uint16_t)puSrc1->au8[29] * puSrc2->ai8[29]); + uDst.ai16[15] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[30] * puSrc2->ai8[30] + (uint16_t)puSrc1->au8[31] * puSrc2->ai8[31]); + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; + puDst->au64[2] = uDst.au64[2]; + puDst->au64[3] = uDst.au64[3]; +} + + +/* + * PMULHRSW / VPMULHRSW + */ +#define DO_PMULHRSW(a_Src1, a_Src2) \ + (uint16_t)(((((int32_t)(a_Src1) * (a_Src2)) >> 14 ) + 1) >> 1) + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhrsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + + uDst.au16[0] = DO_PMULHRSW(uSrc1.ai16[0], uSrc2.ai16[0]); + uDst.au16[1] = DO_PMULHRSW(uSrc1.ai16[1], uSrc2.ai16[1]); + uDst.au16[2] = DO_PMULHRSW(uSrc1.ai16[2], uSrc2.ai16[2]); + uDst.au16[3] = DO_PMULHRSW(uSrc1.ai16[3], uSrc2.ai16[3]); + *puDst = uDst.u; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhrsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->ai16[0] = DO_PMULHRSW(uSrc1.ai16[0], puSrc->ai16[0]); + puDst->ai16[1] = DO_PMULHRSW(uSrc1.ai16[1], puSrc->ai16[1]); + puDst->ai16[2] = DO_PMULHRSW(uSrc1.ai16[2], puSrc->ai16[2]); + puDst->ai16[3] = DO_PMULHRSW(uSrc1.ai16[3], puSrc->ai16[3]); + puDst->ai16[4] = DO_PMULHRSW(uSrc1.ai16[4], puSrc->ai16[4]); + puDst->ai16[5] = DO_PMULHRSW(uSrc1.ai16[5], puSrc->ai16[5]); + puDst->ai16[6] = DO_PMULHRSW(uSrc1.ai16[6], puSrc->ai16[6]); + puDst->ai16[7] = DO_PMULHRSW(uSrc1.ai16[7], puSrc->ai16[7]); + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhrsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[0] = DO_PMULHRSW(puSrc1->ai16[0], puSrc2->ai16[0]); + uDst.ai16[1] = DO_PMULHRSW(puSrc1->ai16[1], puSrc2->ai16[1]); + uDst.ai16[2] = DO_PMULHRSW(puSrc1->ai16[2], puSrc2->ai16[2]); + uDst.ai16[3] = DO_PMULHRSW(puSrc1->ai16[3], puSrc2->ai16[3]); + uDst.ai16[4] = DO_PMULHRSW(puSrc1->ai16[4], puSrc2->ai16[4]); + uDst.ai16[5] = DO_PMULHRSW(puSrc1->ai16[5], puSrc2->ai16[5]); + uDst.ai16[6] = DO_PMULHRSW(puSrc1->ai16[6], puSrc2->ai16[6]); + uDst.ai16[7] = DO_PMULHRSW(puSrc1->ai16[7], puSrc2->ai16[7]); + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhrsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uDst; /* puDst can be the same as one of the source operands. */ + + uDst.ai16[ 0] = DO_PMULHRSW(puSrc1->ai16[ 0], puSrc2->ai16[ 0]); + uDst.ai16[ 1] = DO_PMULHRSW(puSrc1->ai16[ 1], puSrc2->ai16[ 1]); + uDst.ai16[ 2] = DO_PMULHRSW(puSrc1->ai16[ 2], puSrc2->ai16[ 2]); + uDst.ai16[ 3] = DO_PMULHRSW(puSrc1->ai16[ 3], puSrc2->ai16[ 3]); + uDst.ai16[ 4] = DO_PMULHRSW(puSrc1->ai16[ 4], puSrc2->ai16[ 4]); + uDst.ai16[ 5] = DO_PMULHRSW(puSrc1->ai16[ 5], puSrc2->ai16[ 5]); + uDst.ai16[ 6] = DO_PMULHRSW(puSrc1->ai16[ 6], puSrc2->ai16[ 6]); + uDst.ai16[ 7] = DO_PMULHRSW(puSrc1->ai16[ 7], puSrc2->ai16[ 7]); + uDst.ai16[ 8] = DO_PMULHRSW(puSrc1->ai16[ 8], puSrc2->ai16[ 8]); + uDst.ai16[ 9] = DO_PMULHRSW(puSrc1->ai16[ 9], puSrc2->ai16[ 9]); + uDst.ai16[10] = DO_PMULHRSW(puSrc1->ai16[10], puSrc2->ai16[10]); + uDst.ai16[11] = DO_PMULHRSW(puSrc1->ai16[11], puSrc2->ai16[11]); + uDst.ai16[12] = DO_PMULHRSW(puSrc1->ai16[12], puSrc2->ai16[12]); + uDst.ai16[13] = DO_PMULHRSW(puSrc1->ai16[13], puSrc2->ai16[13]); + uDst.ai16[14] = DO_PMULHRSW(puSrc1->ai16[14], puSrc2->ai16[14]); + uDst.ai16[15] = DO_PMULHRSW(puSrc1->ai16[15], puSrc2->ai16[15]); + + puDst->au64[0] = uDst.au64[0]; + puDst->au64[1] = uDst.au64[1]; + puDst->au64[2] = uDst.au64[2]; + puDst->au64[3] = uDst.au64[3]; +} + + +/* + * PSADBW / VPSADBW + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_psadbw_u64,(uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + RTUINT64U uDst; + uint16_t uSum = RT_ABS((int16_t)uSrc1.au8[0] - uSrc2.au8[0]); + uSum += RT_ABS((int16_t)uSrc1.au8[1] - uSrc2.au8[1]); + uSum += RT_ABS((int16_t)uSrc1.au8[2] - uSrc2.au8[2]); + uSum += RT_ABS((int16_t)uSrc1.au8[3] - uSrc2.au8[3]); + uSum += RT_ABS((int16_t)uSrc1.au8[4] - uSrc2.au8[4]); + uSum += RT_ABS((int16_t)uSrc1.au8[5] - uSrc2.au8[5]); + uSum += RT_ABS((int16_t)uSrc1.au8[6] - uSrc2.au8[6]); + uSum += RT_ABS((int16_t)uSrc1.au8[7] - uSrc2.au8[7]); + + uDst.au64[0] = 0; + uDst.au16[0] = uSum; + *puDst = uDst.u; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_psadbw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au64[0] = 0; + puDst->au64[1] = 0; + + uint16_t uSum = RT_ABS((int16_t)uSrc1.ai8[0] - puSrc->ai8[0]); + uSum += RT_ABS((int16_t)uSrc1.au8[1] - puSrc->au8[1]); + uSum += RT_ABS((int16_t)uSrc1.au8[2] - puSrc->au8[2]); + uSum += RT_ABS((int16_t)uSrc1.au8[3] - puSrc->au8[3]); + uSum += RT_ABS((int16_t)uSrc1.au8[4] - puSrc->au8[4]); + uSum += RT_ABS((int16_t)uSrc1.au8[5] - puSrc->au8[5]); + uSum += RT_ABS((int16_t)uSrc1.au8[6] - puSrc->au8[6]); + uSum += RT_ABS((int16_t)uSrc1.au8[7] - puSrc->au8[7]); + puDst->au16[0] = uSum; + + uSum = RT_ABS((int16_t)uSrc1.au8[ 8] - puSrc->au8[ 8]); + uSum += RT_ABS((int16_t)uSrc1.au8[ 9] - puSrc->au8[ 9]); + uSum += RT_ABS((int16_t)uSrc1.au8[10] - puSrc->au8[10]); + uSum += RT_ABS((int16_t)uSrc1.au8[11] - puSrc->au8[11]); + uSum += RT_ABS((int16_t)uSrc1.au8[12] - puSrc->au8[12]); + uSum += RT_ABS((int16_t)uSrc1.au8[13] - puSrc->au8[13]); + uSum += RT_ABS((int16_t)uSrc1.au8[14] - puSrc->au8[14]); + uSum += RT_ABS((int16_t)uSrc1.au8[15] - puSrc->au8[15]); + puDst->au16[4] = uSum; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsadbw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uSrc1 = *puSrc1; + RTUINT128U uSrc2 = *puSrc2; + + puDst->au64[0] = 0; + puDst->au64[1] = 0; + + uint16_t uSum = RT_ABS((int16_t)uSrc1.ai8[0] - uSrc2.ai8[0]); + uSum += RT_ABS((int16_t)uSrc1.au8[1] - uSrc2.au8[1]); + uSum += RT_ABS((int16_t)uSrc1.au8[2] - uSrc2.au8[2]); + uSum += RT_ABS((int16_t)uSrc1.au8[3] - uSrc2.au8[3]); + uSum += RT_ABS((int16_t)uSrc1.au8[4] - uSrc2.au8[4]); + uSum += RT_ABS((int16_t)uSrc1.au8[5] - uSrc2.au8[5]); + uSum += RT_ABS((int16_t)uSrc1.au8[6] - uSrc2.au8[6]); + uSum += RT_ABS((int16_t)uSrc1.au8[7] - uSrc2.au8[7]); + puDst->au16[0] = uSum; + + uSum = RT_ABS((int16_t)uSrc1.au8[ 8] - uSrc2.au8[ 8]); + uSum += RT_ABS((int16_t)uSrc1.au8[ 9] - uSrc2.au8[ 9]); + uSum += RT_ABS((int16_t)uSrc1.au8[10] - uSrc2.au8[10]); + uSum += RT_ABS((int16_t)uSrc1.au8[11] - uSrc2.au8[11]); + uSum += RT_ABS((int16_t)uSrc1.au8[12] - uSrc2.au8[12]); + uSum += RT_ABS((int16_t)uSrc1.au8[13] - uSrc2.au8[13]); + uSum += RT_ABS((int16_t)uSrc1.au8[14] - uSrc2.au8[14]); + uSum += RT_ABS((int16_t)uSrc1.au8[15] - uSrc2.au8[15]); + puDst->au16[4] = uSum; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpsadbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uSrc1 = *puSrc1; + RTUINT256U uSrc2 = *puSrc2; + + puDst->au64[0] = 0; + puDst->au64[1] = 0; + puDst->au64[2] = 0; + puDst->au64[3] = 0; + + uint16_t uSum = RT_ABS((int16_t)uSrc1.au8[0] - uSrc2.au8[0]); + uSum += RT_ABS((int16_t)uSrc1.au8[1] - uSrc2.au8[1]); + uSum += RT_ABS((int16_t)uSrc1.au8[2] - uSrc2.au8[2]); + uSum += RT_ABS((int16_t)uSrc1.au8[3] - uSrc2.au8[3]); + uSum += RT_ABS((int16_t)uSrc1.au8[4] - uSrc2.au8[4]); + uSum += RT_ABS((int16_t)uSrc1.au8[5] - uSrc2.au8[5]); + uSum += RT_ABS((int16_t)uSrc1.au8[6] - uSrc2.au8[6]); + uSum += RT_ABS((int16_t)uSrc1.au8[7] - uSrc2.au8[7]); + puDst->au16[0] = uSum; + + uSum = RT_ABS((int16_t)uSrc1.au8[ 8] - uSrc2.au8[ 8]); + uSum += RT_ABS((int16_t)uSrc1.au8[ 9] - uSrc2.au8[ 9]); + uSum += RT_ABS((int16_t)uSrc1.au8[10] - uSrc2.au8[10]); + uSum += RT_ABS((int16_t)uSrc1.au8[11] - uSrc2.au8[11]); + uSum += RT_ABS((int16_t)uSrc1.au8[12] - uSrc2.au8[12]); + uSum += RT_ABS((int16_t)uSrc1.au8[13] - uSrc2.au8[13]); + uSum += RT_ABS((int16_t)uSrc1.au8[14] - uSrc2.au8[14]); + uSum += RT_ABS((int16_t)uSrc1.au8[15] - uSrc2.au8[15]); + puDst->au16[4] = uSum; + + uSum = RT_ABS((int16_t)uSrc1.au8[16] - uSrc2.au8[16]); + uSum += RT_ABS((int16_t)uSrc1.au8[17] - uSrc2.au8[17]); + uSum += RT_ABS((int16_t)uSrc1.au8[18] - uSrc2.au8[18]); + uSum += RT_ABS((int16_t)uSrc1.au8[19] - uSrc2.au8[19]); + uSum += RT_ABS((int16_t)uSrc1.au8[20] - uSrc2.au8[20]); + uSum += RT_ABS((int16_t)uSrc1.au8[21] - uSrc2.au8[21]); + uSum += RT_ABS((int16_t)uSrc1.au8[22] - uSrc2.au8[22]); + uSum += RT_ABS((int16_t)uSrc1.au8[23] - uSrc2.au8[23]); + puDst->au16[8] = uSum; + + uSum = RT_ABS((int16_t)uSrc1.au8[24] - uSrc2.au8[24]); + uSum += RT_ABS((int16_t)uSrc1.au8[25] - uSrc2.au8[25]); + uSum += RT_ABS((int16_t)uSrc1.au8[26] - uSrc2.au8[26]); + uSum += RT_ABS((int16_t)uSrc1.au8[27] - uSrc2.au8[27]); + uSum += RT_ABS((int16_t)uSrc1.au8[28] - uSrc2.au8[28]); + uSum += RT_ABS((int16_t)uSrc1.au8[29] - uSrc2.au8[29]); + uSum += RT_ABS((int16_t)uSrc1.au8[30] - uSrc2.au8[30]); + uSum += RT_ABS((int16_t)uSrc1.au8[31] - uSrc2.au8[31]); + puDst->au16[12] = uSum; +} + + +/* + * PMULDQ / VPMULDQ + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_pmuldq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + + puDst->au64[0] = (int64_t)uSrc1.ai32[0] * puSrc->ai32[0]; + puDst->au64[1] = (int64_t)uSrc1.ai32[2] * puSrc->ai32[2]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmuldq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uSrc1 = *puSrc1; + RTUINT128U uSrc2 = *puSrc2; + + puDst->au64[0] = (int64_t)uSrc1.ai32[0] * uSrc2.ai32[0]; + puDst->au64[1] = (int64_t)uSrc1.ai32[2] * uSrc2.ai32[2]; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmuldq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uSrc1 = *puSrc1; + RTUINT256U uSrc2 = *puSrc2; + + puDst->au64[0] = (int64_t)uSrc1.ai32[0] * uSrc2.ai32[0]; + puDst->au64[1] = (int64_t)uSrc1.ai32[2] * uSrc2.ai32[2]; + puDst->au64[2] = (int64_t)uSrc1.ai32[4] * uSrc2.ai32[4]; + puDst->au64[3] = (int64_t)uSrc1.ai32[6] * uSrc2.ai32[6]; +} + + +/* + * PMULUDQ / VPMULUDQ + */ +#ifdef IEM_WITHOUT_ASSEMBLY + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmuludq_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc)) +{ + RTUINT64U uSrc1 = { *puDst }; + RTUINT64U uSrc2 = { *puSrc }; + ASMCompilerBarrier(); + *puDst = (uint64_t)uSrc1.au32[0] * uSrc2.au32[0]; + RT_NOREF(pFpuState); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pmuludq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + RTUINT128U uSrc2 = *puSrc; + ASMCompilerBarrier(); + puDst->au64[0] = (uint64_t)uSrc1.au32[0] * uSrc2.au32[0]; + puDst->au64[1] = (uint64_t)uSrc1.au32[2] * uSrc2.au32[2]; + RT_NOREF(pFpuState); +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmuludq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au64[0] = (uint64_t)uSrc1.au32[0] * uSrc2.au32[0]; + puDst->au64[1] = (uint64_t)uSrc1.au32[2] * uSrc2.au32[2]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmuludq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au64[0] = (uint64_t)uSrc1.au32[0] * uSrc2.au32[0]; + puDst->au64[1] = (uint64_t)uSrc1.au32[2] * uSrc2.au32[2]; + puDst->au64[2] = (uint64_t)uSrc1.au32[4] * uSrc2.au32[4]; + puDst->au64[3] = (uint64_t)uSrc1.au32[6] * uSrc2.au32[6]; +} + + +/* + * UNPCKLPS / VUNPCKLPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_unpcklps_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + RTUINT128U uSrc2 = *puSrc; + ASMCompilerBarrier(); + puDst->au32[0] = uSrc1.au32[0]; + puDst->au32[1] = uSrc2.au32[0]; + puDst->au32[2] = uSrc1.au32[1]; + puDst->au32[3] = uSrc2.au32[1]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vunpcklps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au32[0] = uSrc1.au32[0]; + puDst->au32[1] = uSrc2.au32[0]; + puDst->au32[2] = uSrc1.au32[1]; + puDst->au32[3] = uSrc2.au32[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vunpcklps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au32[0] = uSrc1.au32[0]; + puDst->au32[1] = uSrc2.au32[0]; + puDst->au32[2] = uSrc1.au32[1]; + puDst->au32[3] = uSrc2.au32[1]; + + puDst->au32[4] = uSrc1.au32[4]; + puDst->au32[5] = uSrc2.au32[4]; + puDst->au32[6] = uSrc1.au32[5]; + puDst->au32[7] = uSrc2.au32[5]; +} + + +/* + * UNPCKLPD / VUNPCKLPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_unpcklpd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + RTUINT128U uSrc2 = *puSrc; + ASMCompilerBarrier(); + puDst->au64[0] = uSrc1.au64[0]; + puDst->au64[1] = uSrc2.au64[0]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vunpcklpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au64[0] = uSrc1.au64[0]; + puDst->au64[1] = uSrc2.au64[0]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vunpcklpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au64[0] = uSrc1.au64[0]; + puDst->au64[1] = uSrc2.au64[0]; + puDst->au64[2] = uSrc1.au64[2]; + puDst->au64[3] = uSrc2.au64[2]; +} + + +/* + * UNPCKHPS / VUNPCKHPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_unpckhps_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + RTUINT128U uSrc2 = *puSrc; + ASMCompilerBarrier(); + puDst->au32[0] = uSrc1.au32[2]; + puDst->au32[1] = uSrc2.au32[2]; + puDst->au32[2] = uSrc1.au32[3]; + puDst->au32[3] = uSrc2.au32[3]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vunpckhps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au32[0] = uSrc1.au32[2]; + puDst->au32[1] = uSrc2.au32[2]; + puDst->au32[2] = uSrc1.au32[3]; + puDst->au32[3] = uSrc2.au32[3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vunpckhps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au32[0] = uSrc1.au32[2]; + puDst->au32[1] = uSrc2.au32[2]; + puDst->au32[2] = uSrc1.au32[3]; + puDst->au32[3] = uSrc2.au32[3]; + + puDst->au32[4] = uSrc1.au32[6]; + puDst->au32[5] = uSrc2.au32[6]; + puDst->au32[6] = uSrc1.au32[7]; + puDst->au32[7] = uSrc2.au32[7]; +} + + +/* + * UNPCKHPD / VUNPCKHPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_unpckhpd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puDst; + RTUINT128U uSrc2 = *puSrc; + ASMCompilerBarrier(); + puDst->au64[0] = uSrc1.au64[1]; + puDst->au64[1] = uSrc2.au64[1]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vunpckhpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2)) +{ + RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au64[0] = uSrc1.au64[1]; + puDst->au64[1] = uSrc2.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vunpckhpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2)) +{ + RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */ + RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */ + ASMCompilerBarrier(); + puDst->au64[0] = uSrc1.au64[1]; + puDst->au64[1] = uSrc2.au64[1]; + puDst->au64[2] = uSrc1.au64[3]; + puDst->au64[3] = uSrc2.au64[3]; +} + + +/* + * CRC32 (SEE 4.2). + */ + +IEM_DECL_IMPL_DEF(void, iemAImpl_crc32_u8_fallback,(uint32_t *puDst, uint8_t uSrc)) +{ + *puDst = RTCrc32CProcess(*puDst, &uSrc, sizeof(uSrc)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_crc32_u16_fallback,(uint32_t *puDst, uint16_t uSrc)) +{ + *puDst = RTCrc32CProcess(*puDst, &uSrc, sizeof(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_crc32_u32_fallback,(uint32_t *puDst, uint32_t uSrc)) +{ + *puDst = RTCrc32CProcess(*puDst, &uSrc, sizeof(uSrc)); +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_crc32_u64_fallback,(uint32_t *puDst, uint64_t uSrc)) +{ + *puDst = RTCrc32CProcess(*puDst, &uSrc, sizeof(uSrc)); +} + + +/* + * PTEST (SSE 4.1) - special as it output only EFLAGS. + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_ptest_u128,(PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint32_t *pfEFlags)) +{ + uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; + if ( (puSrc1->au64[0] & puSrc2->au64[0]) == 0 + && (puSrc1->au64[1] & puSrc2->au64[1]) == 0) + fEfl |= X86_EFL_ZF; + if ( (~puSrc1->au64[0] & puSrc2->au64[0]) == 0 + && (~puSrc1->au64[1] & puSrc2->au64[1]) == 0) + fEfl |= X86_EFL_CF; + *pfEFlags = fEfl; +} +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vptest_u256_fallback,(PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint32_t *pfEFlags)) +{ + uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; + if ( (puSrc1->au64[0] & puSrc2->au64[0]) == 0 + && (puSrc1->au64[1] & puSrc2->au64[1]) == 0 + && (puSrc1->au64[2] & puSrc2->au64[2]) == 0 + && (puSrc1->au64[3] & puSrc2->au64[3]) == 0) + fEfl |= X86_EFL_ZF; + if ( (~puSrc1->au64[0] & puSrc2->au64[0]) == 0 + && (~puSrc1->au64[1] & puSrc2->au64[1]) == 0 + && (~puSrc1->au64[2] & puSrc2->au64[2]) == 0 + && (~puSrc1->au64[3] & puSrc2->au64[3]) == 0) + fEfl |= X86_EFL_CF; + *pfEFlags = fEfl; +} + + +/* + * PMOVSXBW / VPMOVSXBW + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbw_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc)) +{ + RTUINT64U uSrc1 = { uSrc }; + puDst->ai16[0] = uSrc1.ai8[0]; + puDst->ai16[1] = uSrc1.ai8[1]; + puDst->ai16[2] = uSrc1.ai8[2]; + puDst->ai16[3] = uSrc1.ai8[3]; + puDst->ai16[4] = uSrc1.ai8[4]; + puDst->ai16[5] = uSrc1.ai8[5]; + puDst->ai16[6] = uSrc1.ai8[6]; + puDst->ai16[7] = uSrc1.ai8[7]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->ai16[ 0] = uSrc1.ai8[ 0]; + puDst->ai16[ 1] = uSrc1.ai8[ 1]; + puDst->ai16[ 2] = uSrc1.ai8[ 2]; + puDst->ai16[ 3] = uSrc1.ai8[ 3]; + puDst->ai16[ 4] = uSrc1.ai8[ 4]; + puDst->ai16[ 5] = uSrc1.ai8[ 5]; + puDst->ai16[ 6] = uSrc1.ai8[ 6]; + puDst->ai16[ 7] = uSrc1.ai8[ 7]; + puDst->ai16[ 8] = uSrc1.ai8[ 8]; + puDst->ai16[ 9] = uSrc1.ai8[ 9]; + puDst->ai16[10] = uSrc1.ai8[10]; + puDst->ai16[11] = uSrc1.ai8[11]; + puDst->ai16[12] = uSrc1.ai8[12]; + puDst->ai16[13] = uSrc1.ai8[13]; + puDst->ai16[14] = uSrc1.ai8[14]; + puDst->ai16[15] = uSrc1.ai8[15]; +} + + +/* + * PMOVSXBD / VPMOVSXBD + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbd_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc)) +{ + RTUINT32U uSrc1 = { uSrc }; + puDst->ai32[0] = uSrc1.ai8[0]; + puDst->ai32[1] = uSrc1.ai8[1]; + puDst->ai32[2] = uSrc1.ai8[2]; + puDst->ai32[3] = uSrc1.ai8[3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->ai32[0] = uSrc1.ai8[0]; + puDst->ai32[1] = uSrc1.ai8[1]; + puDst->ai32[2] = uSrc1.ai8[2]; + puDst->ai32[3] = uSrc1.ai8[3]; + puDst->ai32[4] = uSrc1.ai8[4]; + puDst->ai32[5] = uSrc1.ai8[5]; + puDst->ai32[6] = uSrc1.ai8[6]; + puDst->ai32[7] = uSrc1.ai8[7]; +} + + +/* + * PMOVSXBQ / VPMOVSXBQ + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbq_u128_fallback,(PRTUINT128U puDst, uint16_t uSrc)) +{ + RTUINT16U uSrc1 = { uSrc }; + puDst->ai64[0] = uSrc1.ai8[0]; + puDst->ai64[1] = uSrc1.ai8[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->ai64[0] = uSrc1.ai8[0]; + puDst->ai64[1] = uSrc1.ai8[1]; + puDst->ai64[2] = uSrc1.ai8[2]; + puDst->ai64[3] = uSrc1.ai8[3]; +} + + +/* + * PMOVSXWD / VPMOVSXWD + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwd_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc)) +{ + RTUINT64U uSrc1 = { uSrc }; + puDst->ai32[0] = uSrc1.ai16[0]; + puDst->ai32[1] = uSrc1.ai16[1]; + puDst->ai32[2] = uSrc1.ai16[2]; + puDst->ai32[3] = uSrc1.ai16[3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->ai32[0] = uSrc1.ai16[0]; + puDst->ai32[1] = uSrc1.ai16[1]; + puDst->ai32[2] = uSrc1.ai16[2]; + puDst->ai32[3] = uSrc1.ai16[3]; + puDst->ai32[4] = uSrc1.ai16[4]; + puDst->ai32[5] = uSrc1.ai16[5]; + puDst->ai32[6] = uSrc1.ai16[6]; + puDst->ai32[7] = uSrc1.ai16[7]; +} + + +/* + * PMOVSXWQ / VPMOVSXWQ + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwq_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc)) +{ + RTUINT32U uSrc1 = { uSrc }; + puDst->ai64[0] = uSrc1.ai16[0]; + puDst->ai64[1] = uSrc1.ai16[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->ai64[0] = uSrc1.ai16[0]; + puDst->ai64[1] = uSrc1.ai16[1]; + puDst->ai64[2] = uSrc1.ai16[2]; + puDst->ai64[3] = uSrc1.ai16[3]; +} + + +/* + * PMOVSXDQ / VPMOVSXDQ + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxdq_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc)) +{ + RTUINT64U uSrc1 = { uSrc }; + puDst->ai64[0] = uSrc1.ai32[0]; + puDst->ai64[1] = uSrc1.ai32[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->ai64[0] = uSrc1.ai32[0]; + puDst->ai64[1] = uSrc1.ai32[1]; + puDst->ai64[2] = uSrc1.ai32[2]; + puDst->ai64[3] = uSrc1.ai32[3]; +} + + +/* + * PMOVZXBW / VPMOVZXBW + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbw_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc)) +{ + RTUINT64U uSrc1 = { uSrc }; + puDst->au16[0] = uSrc1.au8[0]; + puDst->au16[1] = uSrc1.au8[1]; + puDst->au16[2] = uSrc1.au8[2]; + puDst->au16[3] = uSrc1.au8[3]; + puDst->au16[4] = uSrc1.au8[4]; + puDst->au16[5] = uSrc1.au8[5]; + puDst->au16[6] = uSrc1.au8[6]; + puDst->au16[7] = uSrc1.au8[7]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->au16[ 0] = uSrc1.au8[ 0]; + puDst->au16[ 1] = uSrc1.au8[ 1]; + puDst->au16[ 2] = uSrc1.au8[ 2]; + puDst->au16[ 3] = uSrc1.au8[ 3]; + puDst->au16[ 4] = uSrc1.au8[ 4]; + puDst->au16[ 5] = uSrc1.au8[ 5]; + puDst->au16[ 6] = uSrc1.au8[ 6]; + puDst->au16[ 7] = uSrc1.au8[ 7]; + puDst->au16[ 8] = uSrc1.au8[ 8]; + puDst->au16[ 9] = uSrc1.au8[ 9]; + puDst->au16[10] = uSrc1.au8[10]; + puDst->au16[11] = uSrc1.au8[11]; + puDst->au16[12] = uSrc1.au8[12]; + puDst->au16[13] = uSrc1.au8[13]; + puDst->au16[14] = uSrc1.au8[14]; + puDst->au16[15] = uSrc1.au8[15]; +} + + +/* + * PMOVZXBD / VPMOVZXBD + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbd_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc)) +{ + RTUINT32U uSrc1 = { uSrc }; + puDst->au32[0] = uSrc1.au8[0]; + puDst->au32[1] = uSrc1.au8[1]; + puDst->au32[2] = uSrc1.au8[2]; + puDst->au32[3] = uSrc1.au8[3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->au32[0] = uSrc1.au8[0]; + puDst->au32[1] = uSrc1.au8[1]; + puDst->au32[2] = uSrc1.au8[2]; + puDst->au32[3] = uSrc1.au8[3]; + puDst->au32[4] = uSrc1.au8[4]; + puDst->au32[5] = uSrc1.au8[5]; + puDst->au32[6] = uSrc1.au8[6]; + puDst->au32[7] = uSrc1.au8[7]; +} + + +/* + * PMOVZXBQ / VPMOVZXBQ + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbq_u128_fallback,(PRTUINT128U puDst, uint16_t uSrc)) +{ + RTUINT16U uSrc1 = { uSrc }; + puDst->au64[0] = uSrc1.au8[0]; + puDst->au64[1] = uSrc1.au8[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->au64[0] = uSrc1.au8[0]; + puDst->au64[1] = uSrc1.au8[1]; + puDst->au64[2] = uSrc1.au8[2]; + puDst->au64[3] = uSrc1.au8[3]; +} + + +/* + * PMOVZXWD / VPMOVZXWD + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwd_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc)) +{ + RTUINT64U uSrc1 = { uSrc }; + puDst->au32[0] = uSrc1.au16[0]; + puDst->au32[1] = uSrc1.au16[1]; + puDst->au32[2] = uSrc1.au16[2]; + puDst->au32[3] = uSrc1.au16[3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->au32[0] = uSrc1.au16[0]; + puDst->au32[1] = uSrc1.au16[1]; + puDst->au32[2] = uSrc1.au16[2]; + puDst->au32[3] = uSrc1.au16[3]; + puDst->au32[4] = uSrc1.au16[4]; + puDst->au32[5] = uSrc1.au16[5]; + puDst->au32[6] = uSrc1.au16[6]; + puDst->au32[7] = uSrc1.au16[7]; +} + + +/* + * PMOVZXWQ / VPMOVZXWQ + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwq_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc)) +{ + RTUINT32U uSrc1 = { uSrc }; + puDst->au64[0] = uSrc1.au16[0]; + puDst->au64[1] = uSrc1.au16[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->au64[0] = uSrc1.au16[0]; + puDst->au64[1] = uSrc1.au16[1]; + puDst->au64[2] = uSrc1.au16[2]; + puDst->au64[3] = uSrc1.au16[3]; +} + + +/* + * PMOVZXDQ / VPMOVZXDQ + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxdq_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc)) +{ + RTUINT64U uSrc1 = { uSrc }; + puDst->au64[0] = uSrc1.au32[0]; + puDst->au64[1] = uSrc1.au32[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */ + puDst->au64[0] = uSrc1.au32[0]; + puDst->au64[1] = uSrc1.au32[1]; + puDst->au64[2] = uSrc1.au32[2]; + puDst->au64[3] = uSrc1.au32[3]; +} + +/** + * Converts from the packed IPRT 32-bit (single precision) floating point format to + * the SoftFloat 32-bit floating point format (float32_t). + * + * This is only a structure format conversion, nothing else. + */ +DECLINLINE(float32_t) iemFpSoftF32FromIprt(PCRTFLOAT32U pr32Val) +{ + float32_t Tmp; + Tmp.v = pr32Val->u; + return Tmp; +} + + +/** + * Converts from SoftFloat 32-bit floating point format (float32_t) + * to the packed IPRT 32-bit floating point (RTFLOAT32U) format. + * + * This is only a structure format conversion, nothing else. + */ +DECLINLINE(PRTFLOAT32U) iemFpSoftF32ToIprt(PRTFLOAT32U pr32Dst, float32_t const r32XSrc) +{ + pr32Dst->u = r32XSrc.v; + return pr32Dst; +} + + +/** + * Converts from the packed IPRT 64-bit (single precision) floating point format to + * the SoftFloat 64-bit floating point format (float64_t). + * + * This is only a structure format conversion, nothing else. + */ +DECLINLINE(float64_t) iemFpSoftF64FromIprt(PCRTFLOAT64U pr64Val) +{ + float64_t Tmp; + Tmp.v = pr64Val->u; + return Tmp; +} + + +/** + * Converts from SoftFloat 64-bit floating point format (float64_t) + * to the packed IPRT 64-bit floating point (RTFLOAT64U) format. + * + * This is only a structure format conversion, nothing else. + */ +DECLINLINE(PRTFLOAT64U) iemFpSoftF64ToIprt(PRTFLOAT64U pr64Dst, float64_t const r64XSrc) +{ + pr64Dst->u = r64XSrc.v; + return pr64Dst; +} + + +/** Initializer for the SoftFloat state structure. */ +# define IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(a_Mxcsr) \ + { \ + softfloat_tininess_afterRounding, \ + ((a_Mxcsr) & X86_MXCSR_RC_MASK) == X86_MXCSR_RC_NEAREST ? (uint8_t)softfloat_round_near_even \ + : ((a_Mxcsr) & X86_MXCSR_RC_MASK) == X86_MXCSR_RC_UP ? (uint8_t)softfloat_round_max \ + : ((a_Mxcsr) & X86_MXCSR_RC_MASK) == X86_MXCSR_RC_DOWN ? (uint8_t)softfloat_round_min \ + : (uint8_t)softfloat_round_minMag, \ + 0, \ + (uint8_t)(((a_Mxcsr) & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT), /* Matches X86_FSW_?E */\ + 32 /* Rounding precision, not relevant for SIMD. */ \ + } + +#ifdef IEM_WITHOUT_ASSEMBLY + +/** + * Helper for transfering exception to MXCSR and setting the result value + * accordingly. + * + * @returns Updated MXCSR. + * @param pSoftState The SoftFloat state following the operation. + * @param r32Result The result of the SoftFloat operation. + * @param pr32Result Where to store the result for IEM. + * @param fMxcsr The original MXCSR value. + */ +DECLINLINE(uint32_t) iemSseSoftStateAndR32ToMxcsrAndIprtResult(softfloat_state_t const *pSoftState, float32_t r32Result, + PRTFLOAT32U pr32Result, uint32_t fMxcsr) +{ + iemFpSoftF32ToIprt(pr32Result, r32Result); + + uint8_t fXcpt = pSoftState->exceptionFlags; + if ( (fMxcsr & X86_MXCSR_FZ) + && RTFLOAT32U_IS_SUBNORMAL(pr32Result)) + { + /* Underflow masked and flush to zero is set. */ + pr32Result->s.uFraction = 0; + pr32Result->s.uExponent = 0; + fXcpt |= X86_MXCSR_UE | X86_MXCSR_PE; + } + + /* If DAZ is set \#DE is never set. */ + if ( fMxcsr & X86_MXCSR_DAZ + || ( (fXcpt & X86_MXCSR_DE) /* Softfloat sets DE for sub-normal values. */ + && (RTFLOAT32U_IS_SUBNORMAL(pr32Result)))) + fXcpt &= ~X86_MXCSR_DE; + + return fMxcsr | (fXcpt & X86_MXCSR_XCPT_FLAGS); +} + + +/** + * Helper for transfering exception to MXCSR and setting the result value + * accordingly - ignores Flush-to-Zero. + * + * @returns Updated MXCSR. + * @param pSoftState The SoftFloat state following the operation. + * @param r32Result The result of the SoftFloat operation. + * @param pr32Result Where to store the result for IEM. + * @param fMxcsr The original MXCSR value. + */ +DECLINLINE(uint32_t) iemSseSoftStateAndR32ToMxcsrAndIprtResultNoFz(softfloat_state_t const *pSoftState, float32_t r32Result, + PRTFLOAT32U pr32Result, uint32_t fMxcsr) +{ + iemFpSoftF32ToIprt(pr32Result, r32Result); + + uint8_t fXcpt = pSoftState->exceptionFlags; + /* If DAZ is set \#DE is never set. */ + if ( fMxcsr & X86_MXCSR_DAZ + || ( (fXcpt & X86_MXCSR_DE) /* Softfloat sets DE for sub-normal values. */ + && (RTFLOAT32U_IS_SUBNORMAL(pr32Result)))) + fXcpt &= ~X86_MXCSR_DE; + + return fMxcsr | (fXcpt & X86_MXCSR_XCPT_FLAGS); +} + + +/** + * Helper for transfering exception to MXCSR and setting the result value + * accordingly. + * + * @returns Updated MXCSR. + * @param pSoftState The SoftFloat state following the operation. + * @param r64Result The result of the SoftFloat operation. + * @param pr64Result Where to store the result for IEM. + * @param fMxcsr The original MXCSR value. + */ +DECLINLINE(uint32_t) iemSseSoftStateAndR64ToMxcsrAndIprtResult(softfloat_state_t const *pSoftState, float64_t r64Result, + PRTFLOAT64U pr64Result, uint32_t fMxcsr) +{ + iemFpSoftF64ToIprt(pr64Result, r64Result); + uint8_t fXcpt = pSoftState->exceptionFlags; + if ( (fMxcsr & X86_MXCSR_FZ) + && RTFLOAT64U_IS_SUBNORMAL(pr64Result)) + { + /* Underflow masked and flush to zero is set. */ + iemFpSoftF64ToIprt(pr64Result, r64Result); + pr64Result->s.uFractionHigh = 0; + pr64Result->s.uFractionLow = 0; + pr64Result->s.uExponent = 0; + fXcpt |= X86_MXCSR_UE | X86_MXCSR_PE; + } + + /* If DAZ is set \#DE is never set. */ + if ( fMxcsr & X86_MXCSR_DAZ + || ( (fXcpt & X86_MXCSR_DE) /* Softfloat sets DE for sub-normal values. */ + && (RTFLOAT64U_IS_SUBNORMAL(pr64Result)))) + fXcpt &= ~X86_MXCSR_DE; + + return fMxcsr | (fXcpt & X86_MXCSR_XCPT_FLAGS); +} + + +/** + * Helper for transfering exception to MXCSR and setting the result value + * accordingly - ignores Flush-to-Zero. + * + * @returns Updated MXCSR. + * @param pSoftState The SoftFloat state following the operation. + * @param r64Result The result of the SoftFloat operation. + * @param pr64Result Where to store the result for IEM. + * @param fMxcsr The original MXCSR value. + */ +DECLINLINE(uint32_t) iemSseSoftStateAndR64ToMxcsrAndIprtResultNoFz(softfloat_state_t const *pSoftState, float64_t r64Result, + PRTFLOAT64U pr64Result, uint32_t fMxcsr) +{ + iemFpSoftF64ToIprt(pr64Result, r64Result); + + uint8_t fXcpt = pSoftState->exceptionFlags; + /* If DAZ is set \#DE is never set. */ + if ( fMxcsr & X86_MXCSR_DAZ + || ( (fXcpt & X86_MXCSR_DE) /* Softfloat sets DE for sub-normal values. */ + && (RTFLOAT64U_IS_SUBNORMAL(pr64Result)))) + fXcpt &= ~X86_MXCSR_DE; + + return fMxcsr | (fXcpt & X86_MXCSR_XCPT_FLAGS); +} + +#endif /* IEM_WITHOUT_ASSEMBLY */ + + +/** + * Sets the given single precision floating point input value to the given output taking the Denormals-as-zero flag + * in MXCSR into account. + * + * @returns The output MXCSR De-normal flag if the input is a de-normal and the DAZ flag is not set. + * @param pr32Val Where to store the result. + * @param fMxcsr The input MXCSR value. + * @param pr32Src The value to use. + */ +DECLINLINE(uint32_t) iemSsePrepareValueR32(PRTFLOAT32U pr32Val, uint32_t fMxcsr, PCRTFLOAT32U pr32Src) +{ + if (RTFLOAT32U_IS_SUBNORMAL(pr32Src)) + { + if (fMxcsr & X86_MXCSR_DAZ) + { + /* De-normals are changed to 0. */ + pr32Val->s.fSign = pr32Src->s.fSign; + pr32Val->s.uFraction = 0; + pr32Val->s.uExponent = 0; + return 0; + } + + *pr32Val = *pr32Src; + return X86_MXCSR_DE; + } + + *pr32Val = *pr32Src; + return 0; +} + + +/** + * Sets the given double precision floating point input value to the given output taking the Denormals-as-zero flag + * in MXCSR into account. + * + * @returns The output MXCSR De-normal flag if the input is a de-normal and the DAZ flag is not set. + * @param pr64Val Where to store the result. + * @param fMxcsr The input MXCSR value. + * @param pr64Src The value to use. + */ +DECLINLINE(uint32_t) iemSsePrepareValueR64(PRTFLOAT64U pr64Val, uint32_t fMxcsr, PCRTFLOAT64U pr64Src) +{ + if (RTFLOAT64U_IS_SUBNORMAL(pr64Src)) + { + if (fMxcsr & X86_MXCSR_DAZ) + { + /* De-normals are changed to 0. */ + pr64Val->s64.fSign = pr64Src->s.fSign; + pr64Val->s64.uFraction = 0; + pr64Val->s64.uExponent = 0; + return 0; + } + + *pr64Val = *pr64Src; + return X86_MXCSR_DE; + } + + *pr64Val = *pr64Src; + return 0; +} + +#ifdef IEM_WITHOUT_ASSEMBLY + +/** + * Validates the given input operands returning whether the operation can continue or whether one + * of the source operands contains a NaN value, setting the output accordingly. + * + * @returns Flag whether the operation can continue (false) or whether a NaN value was detected in one of the operands (true). + * @param pr32Res Where to store the result in case the operation can't continue. + * @param pr32Val1 The first input operand. + * @param pr32Val2 The second input operand. + * @param pfMxcsr Where to return the modified MXCSR state when false is returned. + */ +DECLINLINE(bool) iemSseBinaryValIsNaNR32(PRTFLOAT32U pr32Res, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2, uint32_t *pfMxcsr) +{ + uint8_t const cQNan = RTFLOAT32U_IS_QUIET_NAN(pr32Val1) + RTFLOAT32U_IS_QUIET_NAN(pr32Val2); + uint8_t const cSNan = RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val1) + RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val2); + if (cSNan + cQNan == 2) + { + /* Both values are either SNan or QNan, first operand is placed into the result and converted to a QNan. */ + *pr32Res = *pr32Val1; + pr32Res->s.uFraction |= RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1); + *pfMxcsr |= (cSNan ? X86_MXCSR_IE : 0); + return true; + } + if (cSNan) + { + /* One operand is an SNan and placed into the result, converting it to a QNan. */ + *pr32Res = RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val1) ? *pr32Val1 : *pr32Val2; + pr32Res->s.uFraction |= RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1); + *pfMxcsr |= X86_MXCSR_IE; + return true; + } + if (cQNan) + { + /* The QNan operand is placed into the result. */ + *pr32Res = RTFLOAT32U_IS_QUIET_NAN(pr32Val1) ? *pr32Val1 : *pr32Val2; + return true; + } + + Assert(!cQNan && !cSNan); + return false; +} + + +/** + * Validates the given double precision input operands returning whether the operation can continue or whether one + * of the source operands contains a NaN value, setting the output accordingly. + * + * @returns Flag whether the operation can continue (false) or whether a NaN value was detected in one of the operands (true). + * @param pr64Res Where to store the result in case the operation can't continue. + * @param pr64Val1 The first input operand. + * @param pr64Val2 The second input operand. + * @param pfMxcsr Where to return the modified MXCSR state when false is returned. + */ +DECLINLINE(bool) iemSseBinaryValIsNaNR64(PRTFLOAT64U pr64Res, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2, uint32_t *pfMxcsr) +{ + uint8_t const cQNan = RTFLOAT64U_IS_QUIET_NAN(pr64Val1) + RTFLOAT64U_IS_QUIET_NAN(pr64Val2); + uint8_t const cSNan = RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val1) + RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val2); + if (cSNan + cQNan == 2) + { + /* Both values are either SNan or QNan, first operand is placed into the result and converted to a QNan. */ + *pr64Res = *pr64Val1; + pr64Res->s64.uFraction |= RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1); + *pfMxcsr |= (cSNan ? X86_MXCSR_IE : 0); + return true; + } + if (cSNan) + { + /* One operand is an SNan and placed into the result, converting it to a QNan. */ + *pr64Res = RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val1) ? *pr64Val1 : *pr64Val2; + pr64Res->s64.uFraction |= RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1); + *pfMxcsr |= X86_MXCSR_IE; + return true; + } + if (cQNan) + { + /* The QNan operand is placed into the result. */ + *pr64Res = RTFLOAT64U_IS_QUIET_NAN(pr64Val1) ? *pr64Val1 : *pr64Val2; + return true; + } + + Assert(!cQNan && !cSNan); + return false; +} + + +/** + * Validates the given single input operand returning whether the operation can continue or whether + * contains a NaN value, setting the output accordingly. + * + * @returns Flag whether the operation can continue (false) or whether a NaN value was detected in the operand (true). + * @param pr32Res Where to store the result in case the operation can't continue. + * @param pr32Val The input operand. + * @param pfMxcsr Where to return the modified MXCSR state when false is returned. + */ +DECLINLINE(bool) iemSseUnaryValIsNaNR32(PRTFLOAT32U pr32Res, PCRTFLOAT32U pr32Val, uint32_t *pfMxcsr) +{ + if (RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val)) + { + /* One operand is an SNan and placed into the result, converting it to a QNan. */ + *pr32Res = *pr32Val; + pr32Res->s.uFraction |= RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1); + *pfMxcsr |= X86_MXCSR_IE; + return true; + } + if (RTFLOAT32U_IS_QUIET_NAN(pr32Val)) + { + /* The QNan operand is placed into the result. */ + *pr32Res = *pr32Val; + return true; + } + + return false; +} + + +/** + * Validates the given double input operand returning whether the operation can continue or whether + * contains a NaN value, setting the output accordingly. + * + * @returns Flag whether the operation can continue (false) or whether a NaN value was detected in the operand (true). + * @param pr64Res Where to store the result in case the operation can't continue. + * @param pr64Val The input operand. + * @param pfMxcsr Where to return the modified MXCSR state when false is returned. + */ +DECLINLINE(bool) iemSseUnaryValIsNaNR64(PRTFLOAT64U pr64Res, PCRTFLOAT64U pr64Val, uint32_t *pfMxcsr) +{ + if (RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val)) + { + /* One operand is an SNan and placed into the result, converting it to a QNan. */ + *pr64Res = *pr64Val; + pr64Res->s64.uFraction |= RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1); + *pfMxcsr |= X86_MXCSR_IE; + return true; + } + if (RTFLOAT64U_IS_QUIET_NAN(pr64Val)) + { + /* The QNan operand is placed into the result. */ + *pr64Res = *pr64Val; + return true; + } + + return false; +} + +#endif /* IEM_WITHOUT_ASSEMBLY */ + +/** + * ADDPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_addps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2) +{ + if (iemSseBinaryValIsNaNR32(pr32Res, pr32Val1, pr32Val2, &fMxcsr)) + return fMxcsr; + + RTFLOAT32U r32Src1, r32Src2; + fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1); + fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2); + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float32_t r32Result = f32_add(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_addps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_addps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]); +} +#endif + + +/** + * ADDSS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_addss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2)) +{ + pResult->MXCSR = iemAImpl_addps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2); + pResult->uResult.ar32[1] = puSrc1->ar32[1]; + pResult->uResult.ar32[2] = puSrc1->ar32[2]; + pResult->uResult.ar32[3] = puSrc1->ar32[3]; +} +#endif + + +/** + * ADDPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_addpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2) +{ + if (iemSseBinaryValIsNaNR64(pr64Res, pr64Val1, pr64Val2, &fMxcsr)) + return fMxcsr; + + RTFLOAT64U r64Src1, r64Src2; + fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1); + fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2); + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float64_t r64Result = f64_add(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_addpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]); +} +#endif + + +/** + * ADDSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_addsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2)) +{ + pResult->MXCSR = iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2); + pResult->uResult.ar64[1] = puSrc1->ar64[1]; +} +#endif + + +/** + * MULPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_mulps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2) +{ + if (iemSseBinaryValIsNaNR32(pr32Res, pr32Val1, pr32Val2, &fMxcsr)) + return fMxcsr; + + RTFLOAT32U r32Src1, r32Src2; + fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1); + fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2); + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float32_t r32Result = f32_mul(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_mulps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]); +} +#endif + + +/** + * MULSS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_mulss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2)) +{ + pResult->MXCSR = iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2); + pResult->uResult.ar32[1] = puSrc1->ar32[1]; + pResult->uResult.ar32[2] = puSrc1->ar32[2]; + pResult->uResult.ar32[3] = puSrc1->ar32[3]; +} +#endif + + +/** + * MULPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_mulpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2) +{ + if (iemSseBinaryValIsNaNR64(pr64Res, pr64Val1, pr64Val2, &fMxcsr)) + return fMxcsr; + + RTFLOAT64U r64Src1, r64Src2; + fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1); + fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2); + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float64_t r64Result = f64_mul(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_mulpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_mulpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_mulpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]); +} +#endif + + +/** + * MULSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_mulsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2)) +{ + pResult->MXCSR = iemAImpl_mulpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2); + pResult->uResult.ar64[1] = puSrc1->ar64[1]; +} +#endif + + +/** + * SUBPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_subps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2) +{ + if (iemSseBinaryValIsNaNR32(pr32Res, pr32Val1, pr32Val2, &fMxcsr)) + return fMxcsr; + + RTFLOAT32U r32Src1, r32Src2; + fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1); + fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2); + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float32_t r32Result = f32_sub(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_subps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_subps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]); +} +#endif + + +/** + * SUBSS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_subss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2)) +{ + pResult->MXCSR = iemAImpl_subps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2); + pResult->uResult.ar32[1] = puSrc1->ar32[1]; + pResult->uResult.ar32[2] = puSrc1->ar32[2]; + pResult->uResult.ar32[3] = puSrc1->ar32[3]; +} +#endif + + +/** + * SUBPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_subpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2) +{ + if (iemSseBinaryValIsNaNR64(pr64Res, pr64Val1, pr64Val2, &fMxcsr)) + return fMxcsr; + + RTFLOAT64U r64Src1, r64Src2; + fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1); + fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2); + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float64_t r64Result = f64_sub(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_subpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]); +} +#endif + + +/** + * SUBSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_subsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2)) +{ + pResult->MXCSR = iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2); + pResult->uResult.ar64[1] = puSrc1->ar64[1]; +} +#endif + + +/** + * MINPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_minps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2) +{ + if (RTFLOAT32U_IS_NAN(pr32Val1) || RTFLOAT32U_IS_NAN(pr32Val2)) + { + /* The DAZ flag gets honored but the DE flag will not get set because \#IE has higher priority. */ + iemSsePrepareValueR32(pr32Res, fMxcsr, pr32Val2); + return fMxcsr | X86_MXCSR_IE; + } + + RTFLOAT32U r32Src1, r32Src2; + fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1); + fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2); + if (RTFLOAT32U_IS_ZERO(&r32Src1) && RTFLOAT32U_IS_ZERO(&r32Src2)) + { + *pr32Res = r32Src2; + return fMxcsr; + } + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + bool fLe = f32_le(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResultNoFz(&SoftState, + fLe + ? iemFpSoftF32FromIprt(&r32Src1) + : iemFpSoftF32FromIprt(&r32Src2), + pr32Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_minps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_minps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_minps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_minps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_minps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]); +} +#endif + + +/** + * MINSS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_minss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2)) +{ + pResult->MXCSR = iemAImpl_minps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2); + pResult->uResult.ar32[1] = puSrc1->ar32[1]; + pResult->uResult.ar32[2] = puSrc1->ar32[2]; + pResult->uResult.ar32[3] = puSrc1->ar32[3]; +} +#endif + + +/** + * MINPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_minpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2) +{ + if (RTFLOAT64U_IS_NAN(pr64Val1) || RTFLOAT64U_IS_NAN(pr64Val2)) + { + /* The DAZ flag gets honored but the DE flag will not get set because \#IE has higher priority. */ + iemSsePrepareValueR64(pr64Res, fMxcsr, pr64Val2); + return fMxcsr | X86_MXCSR_IE; + } + + RTFLOAT64U r64Src1, r64Src2; + fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1); + fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2); + if (RTFLOAT64U_IS_ZERO(&r64Src1) && RTFLOAT64U_IS_ZERO(&r64Src2)) + { + *pr64Res = r64Src2; + return fMxcsr; + } + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + bool fLe = f64_le(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResultNoFz(&SoftState, + fLe + ? iemFpSoftF64FromIprt(&r64Src1) + : iemFpSoftF64FromIprt(&r64Src2), + pr64Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_minpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_minpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_minpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]); +} +#endif + + +/** + * MINSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_minsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2)) +{ + pResult->MXCSR = iemAImpl_minpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2); + pResult->uResult.ar64[1] = puSrc1->ar64[1]; +} +#endif + + +/** + * DIVPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_divps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2) +{ + if (iemSseBinaryValIsNaNR32(pr32Res, pr32Val1, pr32Val2, &fMxcsr)) + return fMxcsr; + + RTFLOAT32U r32Src1, r32Src2; + uint32_t fDe = iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1); + fDe |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2); + if (RTFLOAT32U_IS_ZERO(&r32Src2)) + { + if ( RTFLOAT32U_IS_ZERO(&r32Src1) + || RTFLOAT32U_IS_QUIET_NAN(&r32Src1)) + { + *pr32Res = g_ar32QNaN[1]; + return fMxcsr | X86_MXCSR_IE; + } + else if (RTFLOAT32U_IS_INF(&r32Src1)) + { + *pr32Res = g_ar32Infinity[r32Src1.s.fSign != r32Src2.s.fSign]; + return fMxcsr; + } + else + { + *pr32Res = g_ar32Infinity[r32Src1.s.fSign != r32Src2.s.fSign]; + return fMxcsr | X86_MXCSR_ZE; + } + } + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float32_t r32Result = f32_div(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr | fDe); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_divps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_divps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_divps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_divps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_divps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]); +} +#endif + + +/** + * DIVSS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_divss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2)) +{ + pResult->MXCSR = iemAImpl_divps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2); + pResult->uResult.ar32[1] = puSrc1->ar32[1]; + pResult->uResult.ar32[2] = puSrc1->ar32[2]; + pResult->uResult.ar32[3] = puSrc1->ar32[3]; +} +#endif + + +/** + * DIVPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_divpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2) +{ + if (iemSseBinaryValIsNaNR64(pr64Res, pr64Val1, pr64Val2, &fMxcsr)) + return fMxcsr; + + RTFLOAT64U r64Src1, r64Src2; + uint32_t fDe = iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1); + fDe |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2); + if (RTFLOAT64U_IS_ZERO(&r64Src2)) + { + if ( RTFLOAT64U_IS_ZERO(&r64Src1) + || RTFLOAT64U_IS_QUIET_NAN(&r64Src1)) + { + *pr64Res = g_ar64QNaN[1]; + return fMxcsr | X86_MXCSR_IE; + } + else if (RTFLOAT64U_IS_INF(&r64Src1)) + { + *pr64Res = g_ar64Infinity[r64Src1.s.fSign != r64Src2.s.fSign]; + return fMxcsr; + } + else + { + *pr64Res = g_ar64Infinity[r64Src1.s.fSign != r64Src2.s.fSign]; + return fMxcsr | X86_MXCSR_ZE; + } + } + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float64_t r64Result = f64_div(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr | fDe); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_divpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_divpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_divpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]); +} +#endif + + +/** + * DIVSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_divsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2)) +{ + pResult->MXCSR = iemAImpl_divpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2); + pResult->uResult.ar64[1] = puSrc1->ar64[1]; +} +#endif + + +/** + * MAXPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_maxps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2) +{ + if (RTFLOAT32U_IS_NAN(pr32Val1) || RTFLOAT32U_IS_NAN(pr32Val2)) + { + /* The DAZ flag gets honored but the DE flag will not get set because \#IE has higher priority. */ + iemSsePrepareValueR32(pr32Res, fMxcsr, pr32Val2); + return fMxcsr | X86_MXCSR_IE; + } + + RTFLOAT32U r32Src1, r32Src2; + fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1); + fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2); + if (RTFLOAT32U_IS_ZERO(&r32Src1) && RTFLOAT32U_IS_ZERO(&r32Src2)) + { + *pr32Res = r32Src2; + return fMxcsr; + } + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + bool fLe = f32_le(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResultNoFz(&SoftState, + fLe + ? iemFpSoftF32FromIprt(&r32Src2) + : iemFpSoftF32FromIprt(&r32Src1), + pr32Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_maxps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]); +} +#endif + + +/** + * MAXSS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_maxss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2)) +{ + pResult->MXCSR = iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2); + pResult->uResult.ar32[1] = puSrc1->ar32[1]; + pResult->uResult.ar32[2] = puSrc1->ar32[2]; + pResult->uResult.ar32[3] = puSrc1->ar32[3]; +} +#endif + + +/** + * MAXPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_maxpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2) +{ + if (RTFLOAT64U_IS_NAN(pr64Val1) || RTFLOAT64U_IS_NAN(pr64Val2)) + { + /* The DAZ flag gets honored but the DE flag will not get set because \#IE has higher priority. */ + iemSsePrepareValueR64(pr64Res, fMxcsr, pr64Val2); + return fMxcsr | X86_MXCSR_IE; + } + + RTFLOAT64U r64Src1, r64Src2; + fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1); + fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2); + if (RTFLOAT64U_IS_ZERO(&r64Src1) && RTFLOAT64U_IS_ZERO(&r64Src2)) + { + *pr64Res = r64Src2; + return fMxcsr; + } + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + bool fLe = f64_le(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResultNoFz(&SoftState, + fLe + ? iemFpSoftF64FromIprt(&r64Src2) + : iemFpSoftF64FromIprt(&r64Src1), + pr64Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_maxpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_maxpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_maxpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]); +} +#endif + + +/** + * MAXSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_maxsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2)) +{ + pResult->MXCSR = iemAImpl_maxpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2); + pResult->uResult.ar64[1] = puSrc1->ar64[1]; +} +#endif + + +/** + * CVTSS2SD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtss2sd_u128_r32_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1) +{ + RTFLOAT32U r32Src1; + fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1); + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float64_t r64Result = f32_to_f64(iemFpSoftF32FromIprt(&r32Src1), &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtss2sd_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2)) +{ + pResult->MXCSR = iemAImpl_cvtss2sd_u128_r32_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, pr32Src2); + pResult->uResult.ar64[1] = puSrc1->ar64[1]; +} +#endif + + +/** + * CVTSD2SS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtsd2ss_u128_r64_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1) +{ + RTFLOAT64U r64Src1; + fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1); + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float32_t r32Result = f64_to_f32(iemFpSoftF64FromIprt(&r64Src1), &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsd2ss_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2)) +{ + pResult->MXCSR = iemAImpl_cvtsd2ss_u128_r64_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, pr64Src2); + pResult->uResult.ar32[1] = puSrc1->ar32[1]; + pResult->uResult.ar32[2] = puSrc1->ar32[2]; + pResult->uResult.ar32[3] = puSrc1->ar32[3]; +} +#endif + + +/** + * HADDPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_haddps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_addps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc1->ar32[1]); + pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc1->ar32[3]); + pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc2->ar32[0], &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc2->ar32[2], &puSrc2->ar32[3]); +} +#endif + + +/** + * HADDPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_haddpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc1->ar64[1]); + pResult->MXCSR |= iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc2->ar64[0], &puSrc2->ar64[1]); +} +#endif + + +/** + * HSUBPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_hsubps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_subps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc1->ar32[1]); + pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc1->ar32[3]); + pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc2->ar32[0], &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc2->ar32[2], &puSrc2->ar32[3]); +} +#endif + + +/** + * HSUBPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_hsubpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + pResult->MXCSR = iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc1->ar64[1]); + pResult->MXCSR |= iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc2->ar64[0], &puSrc2->ar64[1]); +} +#endif + + +/** + * SQRTPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_sqrtps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val) +{ + if (iemSseUnaryValIsNaNR32(pr32Res, pr32Val, &fMxcsr)) + return fMxcsr; + + RTFLOAT32U r32Src; + uint32_t fDe = iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Val); + if (RTFLOAT32U_IS_ZERO(&r32Src)) + { + *pr32Res = r32Src; + return fMxcsr; + } + else if (r32Src.s.fSign) + { + *pr32Res = g_ar32QNaN[1]; + return fMxcsr | X86_MXCSR_IE; + } + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float32_t r32Result = f32_sqrt(iemFpSoftF32FromIprt(&r32Src), &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr | fDe); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_sqrtps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc2->ar32[3]); +} +#endif + + +/** + * SQRTSS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_sqrtss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2)) +{ + pResult->MXCSR = iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, pr32Src2); + pResult->uResult.ar32[1] = puSrc1->ar32[1]; + pResult->uResult.ar32[2] = puSrc1->ar32[2]; + pResult->uResult.ar32[3] = puSrc1->ar32[3]; +} +#endif + + +/** + * SQRTPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_sqrtpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val) +{ + if (iemSseUnaryValIsNaNR64(pr64Res, pr64Val, &fMxcsr)) + return fMxcsr; + + RTFLOAT64U r64Src; + uint32_t fDe = iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Val); + if (RTFLOAT64U_IS_ZERO(&r64Src)) + { + *pr64Res = r64Src; + return fMxcsr; + } + else if (r64Src.s.fSign) + { + *pr64Res = g_ar64QNaN[1]; + return fMxcsr | X86_MXCSR_IE; + } + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float64_t r64Result = f64_sqrt(iemFpSoftF64FromIprt(&r64Src), &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr | fDe); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_sqrtpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_sqrtpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_sqrtpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc2->ar64[1]); +} +#endif + + +/** + * SQRTSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_sqrtsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2)) +{ + pResult->MXCSR = iemAImpl_sqrtpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, pr64Src2); + pResult->uResult.ar64[1] = puSrc1->ar64[1]; +} +#endif + + +#ifdef IEM_WITHOUT_ASSEMBLY +/** + * RSQRTPS + */ +static uint32_t iemAImpl_rsqrt_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val) +{ + RT_NOREF(pr32Res); + RT_NOREF(pr32Val); + AssertReleaseFailed(); + return fMxcsr; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_rsqrtps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_rsqrt_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_rsqrt_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_rsqrt_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_rsqrt_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc2->ar32[3]); +} + + +/** + * RSQRTSS + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_rsqrtss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2)) +{ + pResult->MXCSR = iemAImpl_rsqrt_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, pr32Src2); + pResult->uResult.ar32[1] = puSrc1->ar32[1]; + pResult->uResult.ar32[2] = puSrc1->ar32[2]; + pResult->uResult.ar32[3] = puSrc1->ar32[3]; +} +#endif + + +/** + * ADDSUBPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_addsubps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_subps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]); +} +#endif + + +/** + * ADDSUBPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_addsubpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]); +} +#endif + + +/** + * CVTPD2PS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtpd2ps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1) +{ + RTFLOAT64U r64Src1; + fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1); + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float32_t r32Result = f64_to_f32(iemFpSoftF64FromIprt(&r64Src1), &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpd2ps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_cvtpd2ps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_cvtpd2ps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc2->ar64[1]); + pResult->uResult.au32[2] = 0; + pResult->uResult.au32[3] = 0; +} +#endif + + +/** + * CVTPS2PD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtps2pd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1) +{ + RTFLOAT32U r32Src1; + fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1); + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float64_t r64Result = f32_to_f64(iemFpSoftF32FromIprt(&r32Src1), &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtps2pd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_cvtps2pd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_cvtps2pd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc2->ar32[1]); +} +#endif + + +/** + * CVTDQ2PS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtdq2ps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, int32_t i32Val) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float32_t r32Result = i32_to_f32(i32Val, &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtdq2ps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_cvtdq2ps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, puSrc2->ai32[0]); + pResult->MXCSR |= iemAImpl_cvtdq2ps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, puSrc2->ai32[1]); + pResult->MXCSR |= iemAImpl_cvtdq2ps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, puSrc2->ai32[2]); + pResult->MXCSR |= iemAImpl_cvtdq2ps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, puSrc2->ai32[3]); +} +#endif + + +/** + * CVTPS2DQ + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtps2dq_u128_worker(int32_t *pi32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Src) +{ + RTFLOAT32U r32Src; + iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Src); /* De-normal seems to be ignored. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + *pi32Res = f32_to_i32(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, true /*exact*/, &SoftState); + return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtps2dq_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_cvtps2dq_u128_worker(&pResult->uResult.ai32[0], pFpuState->MXCSR, &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_cvtps2dq_u128_worker(&pResult->uResult.ai32[1], pFpuState->MXCSR, &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_cvtps2dq_u128_worker(&pResult->uResult.ai32[2], pFpuState->MXCSR, &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_cvtps2dq_u128_worker(&pResult->uResult.ai32[3], pFpuState->MXCSR, &puSrc2->ar32[3]); +} +#endif + + +/** + * CVTTPS2DQ + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvttps2dq_u128_worker(int32_t *pi32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Src) +{ + RTFLOAT32U r32Src; + iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Src); /* De-normal seems to be ignored. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + SoftState.roundingMode = softfloat_round_minMag; + *pi32Res = f32_to_i32_r_minMag(iemFpSoftF32FromIprt(&r32Src), true /*exact*/, &SoftState); + return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvttps2dq_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_cvttps2dq_u128_worker(&pResult->uResult.ai32[0], pFpuState->MXCSR, &puSrc2->ar32[0]); + pResult->MXCSR |= iemAImpl_cvttps2dq_u128_worker(&pResult->uResult.ai32[1], pFpuState->MXCSR, &puSrc2->ar32[1]); + pResult->MXCSR |= iemAImpl_cvttps2dq_u128_worker(&pResult->uResult.ai32[2], pFpuState->MXCSR, &puSrc2->ar32[2]); + pResult->MXCSR |= iemAImpl_cvttps2dq_u128_worker(&pResult->uResult.ai32[3], pFpuState->MXCSR, &puSrc2->ar32[3]); +} +#endif + + +/** + * CVTTPD2DQ + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvttpd2dq_u128_worker(int32_t *pi32Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Src) +{ + RTFLOAT64U r64Src; + iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Src); /* De-normal seems to be ignored. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + SoftState.roundingMode = softfloat_round_minMag; + *pi32Res = f64_to_i32(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState); + return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvttpd2dq_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_cvttpd2dq_u128_worker(&pResult->uResult.ai32[0], pFpuState->MXCSR, &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_cvttpd2dq_u128_worker(&pResult->uResult.ai32[1], pFpuState->MXCSR, &puSrc2->ar64[1]); + pResult->uResult.au64[1] = 0; +} +#endif + + +/** + * CVTDQ2PD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtdq2pd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, int32_t i32Val) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float64_t r64Result = i32_to_f64(i32Val, &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtdq2pd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_cvtdq2pd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, puSrc2->ai32[0]); + pResult->MXCSR |= iemAImpl_cvtdq2pd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, puSrc2->ai32[1]); +} +#endif + + +/** + * CVTPD2DQ + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtpd2dq_u128_worker(int32_t *pi32Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Src) +{ + RTFLOAT64U r64Src; + iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Src); /* De-normal seems to be ignored. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + *pi32Res = f64_to_i32(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState); + return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpd2dq_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + RT_NOREF(puSrc1); + + pResult->MXCSR = iemAImpl_cvtpd2dq_u128_worker(&pResult->uResult.ai32[0], pFpuState->MXCSR, &puSrc2->ar64[0]); + pResult->MXCSR |= iemAImpl_cvtpd2dq_u128_worker(&pResult->uResult.ai32[1], pFpuState->MXCSR, &puSrc2->ar64[1]); + pResult->uResult.au64[1] = 0; +} +#endif + + +/** + * [V]SHUFPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_shufps_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + RTUINT128U const uSrc1 = *puDst; + RTUINT128U const uSrc2 = *puSrc; + ASMCompilerBarrier(); + puDst->au32[0] = uSrc1.au32[bEvil & 0x3]; + puDst->au32[1] = uSrc1.au32[(bEvil >> 2) & 0x3]; + puDst->au32[2] = uSrc2.au32[(bEvil >> 4) & 0x3]; + puDst->au32[3] = uSrc2.au32[(bEvil >> 6) & 0x3]; +} +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vshufps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil)) +{ + RTUINT128U const uSrc1 = *puSrc1; + RTUINT128U const uSrc2 = *puSrc2; + ASMCompilerBarrier(); + puDst->au32[0] = uSrc1.au32[bEvil & 0x3]; + puDst->au32[1] = uSrc1.au32[(bEvil >> 2) & 0x3]; + puDst->au32[2] = uSrc2.au32[(bEvil >> 4) & 0x3]; + puDst->au32[3] = uSrc2.au32[(bEvil >> 6) & 0x3]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vshufps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil)) +{ + RTUINT256U const uSrc1 = *puSrc1; + RTUINT256U const uSrc2 = *puSrc2; + ASMCompilerBarrier(); + puDst->au32[0] = uSrc1.au32[bEvil & 0x3]; + puDst->au32[1] = uSrc1.au32[(bEvil >> 2) & 0x3]; + puDst->au32[2] = uSrc2.au32[(bEvil >> 4) & 0x3]; + puDst->au32[3] = uSrc2.au32[(bEvil >> 6) & 0x3]; + + puDst->au32[4] = uSrc1.au32[4 + (bEvil & 0x3)]; + puDst->au32[5] = uSrc1.au32[4 + ((bEvil >> 2) & 0x3)]; + puDst->au32[6] = uSrc2.au32[4 + ((bEvil >> 4) & 0x3)]; + puDst->au32[7] = uSrc2.au32[4 + ((bEvil >> 6) & 0x3)]; +} + + +/** + * [V]SHUFPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_shufpd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + RTUINT128U const uSrc1 = *puDst; + RTUINT128U const uSrc2 = *puSrc; + ASMCompilerBarrier(); + puDst->au64[0] = (bEvil & RT_BIT(0)) ? uSrc1.au64[1] : uSrc1.au64[0]; + puDst->au64[1] = (bEvil & RT_BIT(1)) ? uSrc2.au64[1] : uSrc2.au64[0]; +} +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vshufpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil)) +{ + RTUINT128U const uSrc1 = *puSrc1; + RTUINT128U const uSrc2 = *puSrc2; + ASMCompilerBarrier(); + puDst->au64[0] = (bEvil & RT_BIT(0)) ? uSrc1.au64[1] : uSrc1.au64[0]; + puDst->au64[1] = (bEvil & RT_BIT(1)) ? uSrc2.au64[1] : uSrc2.au64[0]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vshufpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil)) +{ + RTUINT256U const uSrc1 = *puSrc1; + RTUINT256U const uSrc2 = *puSrc2; + ASMCompilerBarrier(); + puDst->au64[0] = (bEvil & RT_BIT(0)) ? uSrc1.au64[1] : uSrc1.au64[0]; + puDst->au64[1] = (bEvil & RT_BIT(1)) ? uSrc2.au64[1] : uSrc2.au64[0]; + puDst->au64[2] = (bEvil & RT_BIT(2)) ? uSrc1.au64[3] : uSrc1.au64[2]; + puDst->au64[3] = (bEvil & RT_BIT(3)) ? uSrc2.au64[3] : uSrc2.au64[2]; +} + + +/* + * PHMINPOSUW / VPHMINPOSUW + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_phminposuw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + uint16_t u16Min = puSrc->au16[0]; + uint8_t idxMin = 0; + + for (uint8_t i = 1; i < RT_ELEMENTS(puSrc->au16); i++) + if (puSrc->au16[i] < u16Min) + { + u16Min = puSrc->au16[i]; + idxMin = i; + } + + puDst->au64[0] = 0; + puDst->au64[1] = 0; + puDst->au16[0] = u16Min; + puDst->au16[1] = idxMin; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vphminposuw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + iemAImpl_phminposuw_u128_fallback(puDst, puSrc); +} + + +/* + * [V]PBLENDVB + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_pblendvb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, PCRTUINT128U puMask)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8); i++) + if (puMask->au8[i] & RT_BIT(7)) + puDst->au8[i] = puSrc->au8[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpblendvb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, PCRTUINT128U puMask)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8); i++) + puDst->au8[i] = puMask->au8[i] & RT_BIT(7) ? puSrc2->au8[i] : puSrc1->au8[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpblendvb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, PCRTUINT256U puMask)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8); i++) + puDst->au8[i] = puMask->au8[i] & RT_BIT(7) ? puSrc2->au8[i] : puSrc1->au8[i]; +} + + +/* + * [V]BLENDVPS + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_blendvps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, PCRTUINT128U puMask)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++) + if (puMask->au32[i] & RT_BIT_32(31)) + puDst->au32[i] = puSrc->au32[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vblendvps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, PCRTUINT128U puMask)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++) + puDst->au32[i] = (puMask->au32[i] & RT_BIT_32(31)) ? puSrc2->au32[i] : puSrc1->au32[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vblendvps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, PCRTUINT256U puMask)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++) + puDst->au32[i] = (puMask->au32[i] & RT_BIT_32(31)) ? puSrc2->au32[i] : puSrc1->au32[i]; +} + + +/* + * [V]BLENDVPD + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_blendvpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, PCRTUINT128U puMask)) +{ + if (puMask->au64[0] & RT_BIT_64(63)) puDst->au64[0] = puSrc->au64[0]; + if (puMask->au64[1] & RT_BIT_64(63)) puDst->au64[1] = puSrc->au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vblendvpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, PCRTUINT128U puMask)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++) + puDst->au64[i] = (puMask->au64[i] & RT_BIT_64(63)) ? puSrc2->au64[i] : puSrc1->au64[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vblendvpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, PCRTUINT256U puMask)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++) + puDst->au64[i] = (puMask->au64[i] & RT_BIT_64(63)) ? puSrc2->au64[i] : puSrc1->au64[i]; +} + + +/** + * [V]PALIGNR + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_palignr_u64_fallback,(uint64_t *pu64Dst, uint64_t u64Src2, uint8_t bEvil)) +{ + uint64_t const u64Src1 = *pu64Dst; + ASMCompilerBarrier(); + + if (bEvil >= 16) + *pu64Dst = 0; + else if (bEvil >= 8) + *pu64Dst = u64Src1 >> ((bEvil - 8) * 8); + else + { + uint8_t cShift = bEvil * 8; + *pu64Dst = ((u64Src1 & (RT_BIT_64(cShift) - 1)) << ((8 - bEvil) * 8)) + | (u64Src2 >> cShift); + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_palignr_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + RTUINT128U const uSrc1 = *puDst; + RTUINT128U const uSrc2 = *puSrc; + ASMCompilerBarrier(); + + puDst->au64[0] = 0; + puDst->au64[1] = 0; + if (bEvil >= 32) + { /* Everything stays 0. */ } + else if (bEvil >= 16) + { + bEvil -= 16; + for (uint8_t i = bEvil; i < RT_ELEMENTS(puDst->au8); i++) + puDst->au8[i - bEvil] = uSrc1.au8[i]; + } + else + { + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8) - bEvil; i++) + puDst->au8[i] = uSrc2.au8[i + bEvil]; + for (uint8_t i = 0; i < bEvil; i++) + puDst->au8[i + RT_ELEMENTS(puDst->au8) - bEvil] = uSrc1.au8[i]; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpalignr_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil)) +{ + RTUINT128U const uSrc1 = *puSrc1; /* Might overlap with destination. */ + RTUINT128U const uSrc2 = *puSrc2; + ASMCompilerBarrier(); + + puDst->au64[0] = 0; + puDst->au64[1] = 0; + if (bEvil >= 32) + { /* Everything stays 0. */ } + else if (bEvil >= 16) + { + bEvil -= 16; + for (uint8_t i = bEvil; i < RT_ELEMENTS(puDst->au8); i++) + puDst->au8[i - bEvil] = uSrc1.au8[i]; + } + else + { + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8) - bEvil; i++) + puDst->au8[i] = uSrc2.au8[i + bEvil]; + for (uint8_t i = 0; i < bEvil; i++) + puDst->au8[i + RT_ELEMENTS(puDst->au8) - bEvil] = uSrc1.au8[i]; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpalignr_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil)) +{ + RTUINT256U const uSrc1 = *puSrc1; /* Might overlap with destination. */ + RTUINT256U const uSrc2 = *puSrc2; + ASMCompilerBarrier(); + + iemAImpl_vpalignr_u128_fallback(&puDst->au128[0], &uSrc1.au128[0], &uSrc2.au128[0], bEvil); + iemAImpl_vpalignr_u128_fallback(&puDst->au128[1], &uSrc1.au128[1], &uSrc2.au128[1], bEvil); +} + + +/** + * [V]PBLENDW + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_pblendw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au16); i++) + if (bEvil & RT_BIT(i)) + puDst->au16[i] = puSrc->au16[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpblendw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au16); i++) + if (bEvil & RT_BIT(i)) + puDst->au16[i] = puSrc2->au16[i]; + else + puDst->au16[i] = puSrc1->au16[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpblendw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < 8; i++) + if (bEvil & RT_BIT(i)) + { + puDst->au16[ i] = puSrc2->au16[ i]; + puDst->au16[8 + i] = puSrc2->au16[8 + i]; + } + else + { + puDst->au16[ i] = puSrc1->au16[ i]; + puDst->au16[8 + i] = puSrc1->au16[8 + i]; + } +} + + +/** + * [V]BLENDPS + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_blendps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++) + if (bEvil & RT_BIT(i)) + puDst->au32[i] = puSrc->au32[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vblendps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++) + if (bEvil & RT_BIT(i)) + puDst->au32[i] = puSrc2->au32[i]; + else + puDst->au32[i] = puSrc1->au32[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vblendps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++) + if (bEvil & RT_BIT(i)) + puDst->au32[i] = puSrc2->au32[i]; + else + puDst->au32[i] = puSrc1->au32[i]; +} + + +/** + * [V]BLENDPD + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_blendpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++) + if (bEvil & RT_BIT(i)) + puDst->au64[i] = puSrc->au64[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vblendpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++) + if (bEvil & RT_BIT(i)) + puDst->au64[i] = puSrc2->au64[i]; + else + puDst->au64[i] = puSrc1->au64[i]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vblendpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++) + if (bEvil & RT_BIT(i)) + puDst->au64[i] = puSrc2->au64[i]; + else + puDst->au64[i] = puSrc1->au64[i]; +} + + +/** + * AES tables and helper routines. Tables from Intel AES-NI whitepaper. + */ + +static uint8_t iemAImpl_aes_sbox[] = { + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 +}; + +/* The InvS-Box lookup table. */ +static uint8_t iemAImpl_aes_inv_sbox[] = { + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d +}; + +/* The ShiftRows lookup table. */ +static uint8_t iemAImpl_aes_shift_rows_tbl[] = { + 0, 5, 10, 15, 4, 9, 14, 3, 8, 13, 2, 7, 12, 1, 6, 11 +}; + +/* The InvShiftRows lookup table. */ +static uint8_t iemAImpl_aes_inv_shift_rows_tbl[] = { + 0, 13, 10, 7, 4, 1, 14, 11, 8, 5, 2, 15, 12, 9, 6, 3 +}; + +static inline RTUINT128U iemAImpl_aes_sub_bytes(PCRTUINT128U puSrc, uint8_t abSubst[256]) +{ + RTUINT128U uVal; + int i; + + for (i = 0; i < 16; ++i) + uVal.au8[i] = abSubst[puSrc->au8[i]]; + + return uVal; +} + +static inline uint8_t iemAImpl_aes_xtime(uint8_t u) +{ + return (u << 1) ^ (((u >> 7) & 1) * 27); +} + +static RTUINT128U iemAImpl_aes_mix_col(PCRTUINT128U puSrc) +{ + RTUINT128U uVal; + int i; + uint8_t tmp; + + for (i = 0; i < 16; i += 4) { + tmp = puSrc->au8[i+0] ^ puSrc->au8[i+1] ^ puSrc->au8[i+2] ^ puSrc->au8[i+3]; + uVal.au8[i+0] = puSrc->au8[i+0] ^ tmp ^ iemAImpl_aes_xtime(puSrc->au8[i+0] ^ puSrc->au8[i+1]); + uVal.au8[i+1] = puSrc->au8[i+1] ^ tmp ^ iemAImpl_aes_xtime(puSrc->au8[i+1] ^ puSrc->au8[i+2]); + uVal.au8[i+2] = puSrc->au8[i+2] ^ tmp ^ iemAImpl_aes_xtime(puSrc->au8[i+2] ^ puSrc->au8[i+3]); + uVal.au8[i+3] = puSrc->au8[i+3] ^ tmp ^ iemAImpl_aes_xtime(puSrc->au8[i+3] ^ puSrc->au8[i+0]); + } + + return uVal; +} + +static inline RTUINT128U iemAImpl_aes_shift_rows(PCRTUINT128U puSrc, uint8_t abShift[16]) +{ + RTUINT128U uVal; + int i; + + for (i = 0; i < 16; ++i) + uVal.au8[i] = puSrc->au8[abShift[i]]; + + return uVal; +} + +static uint8_t iemAImpl_aes_clmul(uint8_t a, uint8_t b) +{ + uint8_t val; + + val = ((b >> 0) & 1) * a; + val ^= ((b >> 1) & 1) * iemAImpl_aes_xtime(a); + val ^= ((b >> 2) & 1) * iemAImpl_aes_xtime(iemAImpl_aes_xtime(a)); + val ^= ((b >> 3) & 1) * iemAImpl_aes_xtime(iemAImpl_aes_xtime(iemAImpl_aes_xtime(a))); + val ^= ((b >> 4) & 1) * iemAImpl_aes_xtime(iemAImpl_aes_xtime(iemAImpl_aes_xtime(iemAImpl_aes_xtime(a)))); + + return val; +} + +static RTUINT128U iemAImpl_aes_inv_mix_col(PCRTUINT128U puSrc) +{ + RTUINT128U uVal; + int i; + + for (i = 0; i < 16; i += 4) { + uVal.au8[i+0] = iemAImpl_aes_clmul(puSrc->au8[i+0], 0x0e) ^ iemAImpl_aes_clmul(puSrc->au8[i+1], 0x0b)^ iemAImpl_aes_clmul(puSrc->au8[i+2], 0x0d) ^ iemAImpl_aes_clmul(puSrc->au8[i+3], 0x09); + uVal.au8[i+1] = iemAImpl_aes_clmul(puSrc->au8[i+0], 0x09) ^ iemAImpl_aes_clmul(puSrc->au8[i+1], 0x0e)^ iemAImpl_aes_clmul(puSrc->au8[i+2], 0x0b) ^ iemAImpl_aes_clmul(puSrc->au8[i+3], 0x0d); + uVal.au8[i+2] = iemAImpl_aes_clmul(puSrc->au8[i+0], 0x0d) ^ iemAImpl_aes_clmul(puSrc->au8[i+1], 0x09)^ iemAImpl_aes_clmul(puSrc->au8[i+2], 0x0e) ^ iemAImpl_aes_clmul(puSrc->au8[i+3], 0x0b); + uVal.au8[i+3] = iemAImpl_aes_clmul(puSrc->au8[i+0], 0x0b) ^ iemAImpl_aes_clmul(puSrc->au8[i+1], 0x0d)^ iemAImpl_aes_clmul(puSrc->au8[i+2], 0x09) ^ iemAImpl_aes_clmul(puSrc->au8[i+3], 0x0e); + } + + return uVal; +} + +static inline uint32_t iemAImpl_aes_sub_word(uint32_t w) +{ + RTUINT32U uTmp; + + uTmp.au32[0] = w; + uTmp.au8[0] = iemAImpl_aes_sbox[uTmp.au8[0]]; + uTmp.au8[1] = iemAImpl_aes_sbox[uTmp.au8[1]]; + uTmp.au8[2] = iemAImpl_aes_sbox[uTmp.au8[2]]; + uTmp.au8[3] = iemAImpl_aes_sbox[uTmp.au8[3]]; + + return uTmp.au32[0]; +} + +static inline uint32_t iemAImpl_aes_rot_word(uint32_t w) +{ + return (w << 24) | (w >> 8); +} + +/** + * [V]AESKEYGENASSIST + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_aeskeygenassist_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bImm)) +{ + RTUINT128U uTmp; + uint32_t uRCon = bImm; /* Round constant. */ + + uTmp.au32[0] = iemAImpl_aes_sub_word(puSrc->au32[1]); /* puSrc = KeyGen. */ + uTmp.au32[1] = iemAImpl_aes_rot_word(iemAImpl_aes_sub_word(puSrc->au32[1])) ^ uRCon; + uTmp.au32[2] = iemAImpl_aes_sub_word(puSrc->au32[3]); + uTmp.au32[3] = iemAImpl_aes_rot_word(iemAImpl_aes_sub_word(puSrc->au32[3])) ^ uRCon; + + *puDst = uTmp; +} + + +/** + * [V]AESIMC + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_aesimc_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + *puDst = iemAImpl_aes_inv_mix_col(puSrc); /* Src = Key. */ +} + + +/** + * [V]AESENC + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_aesenc_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uTmp; + + uTmp = iemAImpl_aes_shift_rows(puDst, iemAImpl_aes_shift_rows_tbl); /* Dst = state. */ + uTmp = iemAImpl_aes_sub_bytes(&uTmp, iemAImpl_aes_sbox); + uTmp = iemAImpl_aes_mix_col(&uTmp); + uTmp.au64[0] ^= puSrc->au64[0]; /* Src = Round Key. */ + uTmp.au64[1] ^= puSrc->au64[1]; + + *puDst = uTmp; +} + + +/** + * [V]AESENCLAST + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_aesenclast_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uTmp; + + uTmp = iemAImpl_aes_shift_rows(puDst, iemAImpl_aes_shift_rows_tbl); /* Dst = state. */ + uTmp = iemAImpl_aes_sub_bytes(&uTmp, iemAImpl_aes_sbox); + uTmp.au64[0] ^= puSrc->au64[0]; /* Src = Round Key. */ + uTmp.au64[1] ^= puSrc->au64[1]; + + *puDst = uTmp; +} + + +/** + * [V]AESDEC + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_aesdec_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uTmp; + + uTmp = iemAImpl_aes_shift_rows(puDst, iemAImpl_aes_inv_shift_rows_tbl); /* Dst = state. */ + uTmp = iemAImpl_aes_sub_bytes(&uTmp, iemAImpl_aes_inv_sbox); + uTmp = iemAImpl_aes_inv_mix_col(&uTmp); + uTmp.au64[0] ^= puSrc->au64[0]; /* Src = Round Key. */ + uTmp.au64[1] ^= puSrc->au64[1]; + + *puDst = uTmp; +} + + +/** + * [V]AESDECLAST + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_aesdeclast_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc)) +{ + RTUINT128U uTmp; + + uTmp = iemAImpl_aes_shift_rows(puDst, iemAImpl_aes_inv_shift_rows_tbl); /* Dst = state. */ + uTmp = iemAImpl_aes_sub_bytes(&uTmp, iemAImpl_aes_inv_sbox); + uTmp.au64[0] ^= puSrc->au64[0]; /* Src = Round Key. */ + uTmp.au64[1] ^= puSrc->au64[1]; + + *puDst = uTmp; +} + + +/** + * [V]PCMPISTRI + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpistri_u128_fallback,(uint32_t *pu32Ecx, uint32_t *pEFlags, PCIEMPCMPISTRISRC pSrc, uint8_t bEvil)) +{ + RT_NOREF(pu32Ecx, pEFlags, pSrc, bEvil); + AssertReleaseFailed(); +} + + +/* + * [V]PCLMULQDQ + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_pclmulqdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + iemAImpl_vpclmulqdq_u128_fallback(puDst, puDst, puSrc, bEvil); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpclmulqdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil)) +{ + uint64_t uSrc1 = puSrc1->au64[bEvil & 0x1]; + uint64_t uSrc2 = puSrc2->au64[(bEvil >> 4) & 0x1]; + + puDst->au64[0] = 0; + puDst->au64[1] = 0; + + /* + * See https://en.wikipedia.org/wiki/Carry-less_product#Example (as of 2022-09-08) for the algorithm. + * Do the first round outside the loop to avoid ASAN complaining about shift exponent being too large (64) + * and squeeze out some optimizations. + */ + if (uSrc1 & 0x1) + puDst->au64[0] = uSrc2; + + uSrc1 >>= 1; + + uint8_t iDigit = 1; + while (uSrc1) + { + if (uSrc1 & 0x1) + { + puDst->au64[0] ^= (uSrc2 << iDigit); + puDst->au64[1] ^= uSrc2 >> (64 - iDigit); + } + + uSrc1 >>= 1; + iDigit++; + } +} + + +/** + * [V]PINSRW + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_pinsrw_u64,(uint64_t *pu64Dst, uint16_t u16Src, uint8_t bEvil)) +{ + uint8_t cShift = (bEvil & 0x3) * 16; + *pu64Dst = (*pu64Dst & ~(UINT64_C(0xffff) << cShift)) | ((uint64_t)u16Src << cShift); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pinsrw_u128,(PRTUINT128U puDst, uint16_t u16Src, uint8_t bEvil)) +{ + puDst->au16[bEvil & 0x7] = u16Src; +} +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpinsrw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint16_t u16Src, uint8_t bEvil)) +{ + *puDst = *puSrc; + puDst->au16[bEvil & 0x7] = u16Src; +} + + +/** + * [V]PEXTRW + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_pextrw_u64,(uint16_t *pu16Dst, uint64_t u64Src, uint8_t bEvil)) +{ + *pu16Dst = (uint16_t)(u64Src >> ((bEvil & 0x3) * 16)); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_pextrw_u128,(uint16_t *pu16Dst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + *pu16Dst = puSrc->au16[bEvil & 0x7]; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vpextrw_u128_fallback,(uint16_t *pu16Dst, PCRTUINT128U puSrc, uint8_t bEvil)) +{ + *pu16Dst = puSrc->au16[bEvil & 0x7]; +} + + +/** + * [V]MOVMSKPS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_movmskps_u128,(uint8_t *pu8Dst, PCRTUINT128U puSrc)) +{ + *pu8Dst = puSrc->au32[0] >> 31; + *pu8Dst |= (puSrc->au32[1] >> 31) << 1; + *pu8Dst |= (puSrc->au32[2] >> 31) << 2; + *pu8Dst |= (puSrc->au32[3] >> 31) << 3; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovmskps_u128_fallback,(uint8_t *pu8Dst, PCRTUINT128U puSrc)) +{ + *pu8Dst = puSrc->au32[0] >> 31; + *pu8Dst |= (puSrc->au32[1] >> 31) << 1; + *pu8Dst |= (puSrc->au32[2] >> 31) << 2; + *pu8Dst |= (puSrc->au32[3] >> 31) << 3; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovmskps_u256_fallback,(uint8_t *pu8Dst, PCRTUINT256U puSrc)) +{ + *pu8Dst = puSrc->au32[0] >> 31; + *pu8Dst |= (puSrc->au32[1] >> 31) << 1; + *pu8Dst |= (puSrc->au32[2] >> 31) << 2; + *pu8Dst |= (puSrc->au32[3] >> 31) << 3; + *pu8Dst |= (puSrc->au32[4] >> 31) << 4; + *pu8Dst |= (puSrc->au32[5] >> 31) << 5; + *pu8Dst |= (puSrc->au32[6] >> 31) << 6; + *pu8Dst |= (puSrc->au32[7] >> 31) << 7; +} + + +/** + * [V]MOVMSKPD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_movmskpd_u128,(uint8_t *pu8Dst, PCRTUINT128U puSrc)) +{ + *pu8Dst = puSrc->au64[0] >> 63; + *pu8Dst |= (puSrc->au64[1] >> 63) << 1; +} + +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovmskpd_u128_fallback,(uint8_t *pu8Dst, PCRTUINT128U puSrc)) +{ + *pu8Dst = puSrc->au64[0] >> 63; + *pu8Dst |= (puSrc->au64[1] >> 63) << 1; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vmovmskpd_u256_fallback,(uint8_t *pu8Dst, PCRTUINT256U puSrc)) +{ + *pu8Dst = puSrc->au64[0] >> 63; + *pu8Dst |= (puSrc->au64[1] >> 63) << 1; + *pu8Dst |= (puSrc->au64[2] >> 63) << 2; + *pu8Dst |= (puSrc->au64[3] >> 63) << 3; +} + + +/** + * CVTTSD2SI + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_cvttsd2si_i32_r64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int32_t *pi32Dst, const uint64_t *pu64Src)) +{ + RTFLOAT64U r64Src; + + r64Src.u = *pu64Src; + iemSsePrepareValueR64(&r64Src, pFpuState->MXCSR, &r64Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + *pi32Dst = f64_to_i32_r_minMag(iemFpSoftF64FromIprt(&r64Src), true /*exact*/, &SoftState); + *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvttsd2si_i64_r64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int64_t *pi64Dst, const uint64_t *pu64Src)) +{ + RTFLOAT64U r64Src; + + r64Src.u = *pu64Src; + iemSsePrepareValueR64(&r64Src, pFpuState->MXCSR, &r64Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + *pi64Dst = f64_to_i64_r_minMag(iemFpSoftF64FromIprt(&r64Src), true /*exact*/, &SoftState); + *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} +#endif + + +/** + * CVTSD2SI + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsd2si_i32_r64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int32_t *pi32Dst, const uint64_t *pu64Src)) +{ + RTFLOAT64U r64Src; + + r64Src.u = *pu64Src; + iemSsePrepareValueR64(&r64Src, pFpuState->MXCSR, &r64Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + *pi32Dst = f64_to_i32(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState); + *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsd2si_i64_r64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int64_t *pi64Dst, const uint64_t *pu64Src)) +{ + RTFLOAT64U r64Src; + + r64Src.u = *pu64Src; + iemSsePrepareValueR64(&r64Src, pFpuState->MXCSR, &r64Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + *pi64Dst = f64_to_i64(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState); + *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} +#endif + + +/** + * CVTTSS2SI + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_cvttss2si_i32_r32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int32_t *pi32Dst, const uint32_t *pu32Src)) +{ + RTFLOAT32U r32Src; + + r32Src.u = *pu32Src; + iemSsePrepareValueR32(&r32Src, pFpuState->MXCSR, &r32Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + *pi32Dst = f32_to_i32_r_minMag(iemFpSoftF32FromIprt(&r32Src), true /*exact*/, &SoftState); + *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvttss2si_i64_r32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int64_t *pi64Dst, const uint32_t *pu32Src)) +{ + RTFLOAT32U r32Src; + + r32Src.u = *pu32Src; + iemSsePrepareValueR32(&r32Src, pFpuState->MXCSR, &r32Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + *pi64Dst = f32_to_i64_r_minMag(iemFpSoftF32FromIprt(&r32Src), true /*exact*/, &SoftState); + *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} +#endif + + +/** + * CVTSS2SI + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtss2si_i32_r32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int32_t *pi32Dst, const uint32_t *pu32Src)) +{ + RTFLOAT32U r32Src; + + r32Src.u = *pu32Src; + iemSsePrepareValueR32(&r32Src, pFpuState->MXCSR, &r32Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + *pi32Dst = f32_to_i32(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, true /*exact*/, &SoftState); + *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtss2si_i64_r32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int64_t *pi64Dst, const uint32_t *pu32Src)) +{ + RTFLOAT32U r32Src; + + r32Src.u = *pu32Src; + iemSsePrepareValueR32(&r32Src, pFpuState->MXCSR, &r32Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + *pi64Dst = f32_to_i64(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, true /*exact*/, &SoftState); + *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} +#endif + + +/** + * CVTSI2SD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsi2sd_r64_i32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, PRTFLOAT64U pr64Dst, const int32_t *pi32Src)) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + float64_t r64Res = i32_to_f64(*pi32Src, &SoftState); + *pfMxcsr = iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Res, pr64Dst, pFpuState->MXCSR); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsi2sd_r64_i64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, PRTFLOAT64U pr64Dst, const int64_t *pi64Src)) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + float64_t r64Res = i64_to_f64(*pi64Src, &SoftState); + *pfMxcsr = iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Res, pr64Dst, pFpuState->MXCSR); +} +#endif + + +/** + * CVTSI2SS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsi2ss_r32_i32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, PRTFLOAT32U pr32Dst, const int32_t *pi32Src)) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + float32_t r32Res = i32_to_f32(*pi32Src, &SoftState); + *pfMxcsr = iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Res, pr32Dst, pFpuState->MXCSR); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsi2ss_r32_i64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, PRTFLOAT32U pr32Dst, const int64_t *pi64Src)) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR); + float32_t r32Res = i64_to_f32(*pi64Src, &SoftState); + *pfMxcsr = iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Res, pr32Dst, pFpuState->MXCSR); +} +#endif + + +/** + * [V]UCOMISS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_ucomiss_u128,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + uint32_t fEFlagsNew = *pfEFlags & ~X86_EFL_STATUS_BITS; + + if (RTFLOAT32U_IS_SIGNALLING_NAN(&puSrc1->ar32[0]) || RTFLOAT32U_IS_SIGNALLING_NAN(&puSrc2->ar32[0])) + { + *pfMxcsr |= X86_MXCSR_IE; + fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */ + } + else if (RTFLOAT32U_IS_QUIET_NAN(&puSrc1->ar32[0]) || RTFLOAT32U_IS_QUIET_NAN(&puSrc2->ar32[0])) + { + /* ucomiss doesn't raise \#IE for quiet NaNs. */ + fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */ + } + else + { + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr); + + RTFLOAT32U r32Src1, r32Src2; + uint32_t fDe = iemSsePrepareValueR32(&r32Src1, *pfMxcsr, &puSrc1->ar32[0]); + fDe |= iemSsePrepareValueR32(&r32Src2, *pfMxcsr, &puSrc2->ar32[0]); + + float32_t f32Src1 = iemFpSoftF32FromIprt(&r32Src1); + float32_t f32Src2 = iemFpSoftF32FromIprt(&r32Src2); + if (f32_eq(f32Src1, f32Src2, &SoftState)) + fEFlagsNew |= X86_EFL_ZF; /* EQUAL 100 */ + else if (f32_lt(f32Src1, f32Src2, &SoftState)) + fEFlagsNew |= X86_EFL_CF; /* LESS_THAN 001 */ + /* else: GREATER_THAN 000 */ + + *pfMxcsr |= fDe; + } + + *pfEFlags = fEFlagsNew; +} +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vucomiss_u128_fallback,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + iemAImpl_ucomiss_u128(pfMxcsr, pfEFlags, puSrc1, puSrc2); +} + + +/** + * [V]UCOMISD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_ucomisd_u128,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + uint32_t fEFlagsNew = *pfEFlags & ~X86_EFL_STATUS_BITS; + + if (RTFLOAT64U_IS_SIGNALLING_NAN(&puSrc1->ar64[0]) || RTFLOAT64U_IS_SIGNALLING_NAN(&puSrc2->ar64[0])) + { + *pfMxcsr |= X86_MXCSR_IE; + fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */ + } + else if (RTFLOAT64U_IS_QUIET_NAN(&puSrc1->ar64[0]) || RTFLOAT64U_IS_QUIET_NAN(&puSrc2->ar64[0])) + { + /* ucomiss doesn't raise \#IE for quiet NaNs. */ + fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */ + } + else + { + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr); + + RTFLOAT64U r64Src1, r64Src2; + uint32_t fDe = iemSsePrepareValueR64(&r64Src1, *pfMxcsr, &puSrc1->ar64[0]); + fDe |= iemSsePrepareValueR64(&r64Src2, *pfMxcsr, &puSrc2->ar64[0]); + + float64_t f64Src1 = iemFpSoftF64FromIprt(&r64Src1); + float64_t f64Src2 = iemFpSoftF64FromIprt(&r64Src2); + if (f64_eq(f64Src1, f64Src2, &SoftState)) + fEFlagsNew |= X86_EFL_ZF; /* EQUAL 100 */ + else if (f64_lt(f64Src1, f64Src2, &SoftState)) + fEFlagsNew |= X86_EFL_CF; /* LESS_THAN 001 */ + /* else: GREATER_THAN 000 */ + + *pfMxcsr |= fDe; + } + + *pfEFlags = fEFlagsNew; +} +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vucomisd_u128_fallback,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + iemAImpl_ucomisd_u128(pfMxcsr, pfEFlags, puSrc1, puSrc2); +} + + +/** + * [V]COMISS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_comiss_u128,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + uint32_t fEFlagsNew = *pfEFlags & ~X86_EFL_STATUS_BITS; + + if ( RTFLOAT32U_IS_SIGNALLING_NAN(&puSrc1->ar32[0]) || RTFLOAT32U_IS_SIGNALLING_NAN(&puSrc2->ar32[0]) + || RTFLOAT32U_IS_QUIET_NAN(&puSrc1->ar32[0]) || RTFLOAT32U_IS_QUIET_NAN(&puSrc2->ar32[0])) + { + *pfMxcsr |= X86_MXCSR_IE; + fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */ + } + else + { + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr); + + RTFLOAT32U r32Src1, r32Src2; + uint32_t fDe = iemSsePrepareValueR32(&r32Src1, *pfMxcsr, &puSrc1->ar32[0]); + fDe |= iemSsePrepareValueR32(&r32Src2, *pfMxcsr, &puSrc2->ar32[0]); + + float32_t f32Src1 = iemFpSoftF32FromIprt(&r32Src1); + float32_t f32Src2 = iemFpSoftF32FromIprt(&r32Src2); + if (f32_eq(f32Src1, f32Src2, &SoftState)) + fEFlagsNew |= X86_EFL_ZF; /* EQUAL 100 */ + else if (f32_lt(f32Src1, f32Src2, &SoftState)) + fEFlagsNew |= X86_EFL_CF; /* LESS_THAN 001 */ + /* else: GREATER_THAN 000 */ + + *pfMxcsr |= fDe; + } + + *pfEFlags = fEFlagsNew; +} +#endif + + +IEM_DECL_IMPL_DEF(void, iemAImpl_vcomiss_u128_fallback,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + iemAImpl_comiss_u128(pfMxcsr, pfEFlags, puSrc1, puSrc2); +} + + +/** + * [V]COMISD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_comisd_u128,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + uint32_t fEFlagsNew = *pfEFlags & ~X86_EFL_STATUS_BITS; + + if ( RTFLOAT64U_IS_SIGNALLING_NAN(&puSrc1->ar64[0]) || RTFLOAT64U_IS_SIGNALLING_NAN(&puSrc2->ar64[0]) + || RTFLOAT64U_IS_QUIET_NAN(&puSrc1->ar64[0]) || RTFLOAT64U_IS_QUIET_NAN(&puSrc2->ar64[0])) + { + *pfMxcsr |= X86_MXCSR_IE; + fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */ + } + else + { + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr); + + RTFLOAT64U r64Src1, r64Src2; + uint32_t fDe = iemSsePrepareValueR64(&r64Src1, *pfMxcsr, &puSrc1->ar64[0]); + fDe |= iemSsePrepareValueR64(&r64Src2, *pfMxcsr, &puSrc2->ar64[0]); + + float64_t f64Src1 = iemFpSoftF64FromIprt(&r64Src1); + float64_t f64Src2 = iemFpSoftF64FromIprt(&r64Src2); + if (f64_eq(f64Src1, f64Src2, &SoftState)) + fEFlagsNew |= X86_EFL_ZF; /* EQUAL 100 */ + else if (f64_lt(f64Src1, f64Src2, &SoftState)) + fEFlagsNew |= X86_EFL_CF; /* LESS_THAN 001 */ + /* else: GREATER_THAN 000 */ + + *pfMxcsr |= fDe; + } + + *pfEFlags = fEFlagsNew; +} +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_vcomisd_u128_fallback,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2)) +{ + iemAImpl_comisd_u128(pfMxcsr, pfEFlags, puSrc1, puSrc2); +} + + +/** + * CMPPS / CMPPD / CMPSS / CMPSD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +/** + * A compare truth table entry. + */ +typedef struct CMPTRUTHTBLENTRY +{ + /** Flag whether the \#IA is signalled when one of the source oeprans is a QNaN */ + bool fSignalsOnQNan; + /** The boolean result when the input operands are unordered. */ + bool fUnordered; + /** The boolean result when A = B. */ + bool fEqual; + /** The boolean result when A < B. */ + bool fLowerThan; + /** The boolean result when A > B. */ + bool fGreaterThan; +} CMPTRUTHTBLENTRY; +/** Pointer to a const truth table entry. */ +typedef const CMPTRUTHTBLENTRY *PCCMPTRUTHTBLENTRY; + + +/** The compare truth table (indexed by immediate). */ +static const CMPTRUTHTBLENTRY g_aCmpTbl[] = +{ + /* fSignalsOnQNan fUnordered fEqual fLowerThan fGreaterThan */ + /* 00H (EQ_OQ) */ { false, false, true, false, false }, + /* 01H (LT_OS) */ { true, false, false, true, false }, + /* 02H (LE_OS) */ { true, false, true, true, false }, + /* 03H (UNORD_Q) */ { false, true, false, false, false }, + /* 04H (NEQ_UQ) */ { false, true, false, true, true }, + /* 05H (NLT_US) */ { true, true, true, false, true }, + /* 06H (NLE_US) */ { true, true, false, false, true }, + /* 07H (ORQ_Q) */ { false, false, true, true, true }, + /** @todo AVX variants. */ +}; + + +static bool iemAImpl_cmp_worker_r32(uint32_t *pfMxcsr, PCRTFLOAT32U pr32Src1, PCRTFLOAT32U pr32Src2, uint8_t bEvil) +{ + bool fRes; + AssertRelease(bEvil < RT_ELEMENTS(g_aCmpTbl)); + + if (RTFLOAT32U_IS_SIGNALLING_NAN(pr32Src1) || RTFLOAT32U_IS_SIGNALLING_NAN(pr32Src2)) + { + *pfMxcsr |= X86_MXCSR_IE; + fRes = g_aCmpTbl[bEvil].fUnordered; + } + else if (RTFLOAT32U_IS_QUIET_NAN(pr32Src1) || RTFLOAT32U_IS_QUIET_NAN(pr32Src2)) + { + if (g_aCmpTbl[bEvil].fSignalsOnQNan) + *pfMxcsr |= X86_MXCSR_IE; + fRes = g_aCmpTbl[bEvil].fUnordered; + } + else + { + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr); + + RTFLOAT32U r32Src1, r32Src2; + uint32_t fDe = iemSsePrepareValueR32(&r32Src1, *pfMxcsr, pr32Src1); + fDe |= iemSsePrepareValueR32(&r32Src2, *pfMxcsr, pr32Src2); + + *pfMxcsr |= fDe; + float32_t f32Src1 = iemFpSoftF32FromIprt(&r32Src1); + float32_t f32Src2 = iemFpSoftF32FromIprt(&r32Src2); + if (f32_eq(f32Src1, f32Src2, &SoftState)) + fRes = g_aCmpTbl[bEvil].fEqual; + else if (f32_lt(f32Src1, f32Src2, &SoftState)) + fRes = g_aCmpTbl[bEvil].fLowerThan; + else + fRes = g_aCmpTbl[bEvil].fGreaterThan; + } + + return fRes; +} + + +static bool iemAImpl_cmp_worker_r64(uint32_t *pfMxcsr, PCRTFLOAT64U pr64Src1, PCRTFLOAT64U pr64Src2, uint8_t bEvil) +{ + bool fRes; + AssertRelease(bEvil < RT_ELEMENTS(g_aCmpTbl)); + + if (RTFLOAT64U_IS_SIGNALLING_NAN(pr64Src1) || RTFLOAT64U_IS_SIGNALLING_NAN(pr64Src2)) + { + *pfMxcsr |= X86_MXCSR_IE; + fRes = g_aCmpTbl[bEvil].fUnordered; + } + else if (RTFLOAT64U_IS_QUIET_NAN(pr64Src1) || RTFLOAT64U_IS_QUIET_NAN(pr64Src2)) + { + if (g_aCmpTbl[bEvil].fSignalsOnQNan) + *pfMxcsr |= X86_MXCSR_IE; + fRes = g_aCmpTbl[bEvil].fUnordered; + } + else + { + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr); + + RTFLOAT64U r64Src1, r64Src2; + uint32_t fDe = iemSsePrepareValueR64(&r64Src1, *pfMxcsr, pr64Src1) + | iemSsePrepareValueR64(&r64Src2, *pfMxcsr, pr64Src2); + + *pfMxcsr |= fDe; + float64_t f64Src1 = iemFpSoftF64FromIprt(&r64Src1); + float64_t f64Src2 = iemFpSoftF64FromIprt(&r64Src2); + if (f64_eq(f64Src1, f64Src2, &SoftState)) + fRes = g_aCmpTbl[bEvil].fEqual; + else if (f64_lt(f64Src1, f64Src2, &SoftState)) + fRes = g_aCmpTbl[bEvil].fLowerThan; + else + fRes = g_aCmpTbl[bEvil].fGreaterThan; + } + + return fRes; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpps_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->ar32); i++) + { + if (iemAImpl_cmp_worker_r32(pfMxcsr, &pSrc->uSrc1.ar32[i], &pSrc->uSrc2.ar32[i], bEvil & 0x7)) + puDst->au32[i] = UINT32_MAX; + else + puDst->au32[i] = 0; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmppd_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bEvil)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->ar64); i++) + { + if (iemAImpl_cmp_worker_r64(pfMxcsr, &pSrc->uSrc1.ar64[i], &pSrc->uSrc2.ar64[i], bEvil & 0x7)) + puDst->au64[i] = UINT64_MAX; + else + puDst->au64[i] = 0; + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpss_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bEvil)) +{ + if (iemAImpl_cmp_worker_r32(pfMxcsr, &pSrc->uSrc1.ar32[0], &pSrc->uSrc2.ar32[0], bEvil & 0x7)) + puDst->au32[0] = UINT32_MAX; + else + puDst->au32[0] = 0; + + puDst->au32[1] = pSrc->uSrc1.au32[1]; + puDst->au64[1] = pSrc->uSrc1.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cmpsd_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bEvil)) +{ + if (iemAImpl_cmp_worker_r64(pfMxcsr, &pSrc->uSrc1.ar64[0], &pSrc->uSrc2.ar64[0], bEvil & 0x7)) + puDst->au64[0] = UINT64_MAX; + else + puDst->au64[0] = 0; + + puDst->au64[1] = pSrc->uSrc1.au64[1]; +} +#endif + + +/** + * ROUNDPS / ROUNDPD / ROUNDSS / ROUNDSD + */ + +#define X86_SSE_ROUNDXX_IMM_RC_MASK UINT32_C(0x0003) +#define X86_SSE_ROUNDXX_IMM_ROUND_SEL UINT32_C(0x0004) +#define X86_SSE_ROUNDXX_IMM_PRECISION UINT32_C(0x0008) + +DECLINLINE(softfloat_state_t) iemSseRoundXXMxcsrAndImmToSoftState(uint32_t fMxcsr, uint8_t bImm) +{ + if (bImm & X86_SSE_ROUNDXX_IMM_ROUND_SEL) + return IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + + fMxcsr &= ~X86_MXCSR_RC_MASK; + fMxcsr |= (bImm & X86_SSE_ROUNDXX_IMM_RC_MASK) << X86_MXCSR_RC_SHIFT; + return IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); +} + +static RTFLOAT32U iemAImpl_round_worker_r32(uint32_t *pfMxcsr, PCRTFLOAT32U pr32Src, uint8_t bImm) +{ + RTFLOAT32U r32Src, r32Dst; + float32_t f32Src; + softfloat_state_t SoftState = iemSseRoundXXMxcsrAndImmToSoftState(*pfMxcsr, bImm); + bool fExact = !RT_BOOL(bImm & X86_SSE_ROUNDXX_IMM_PRECISION); + + iemSsePrepareValueR32(&r32Src, *pfMxcsr, pr32Src); + f32Src = f32_roundToInt(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, fExact, &SoftState); + + iemFpSoftF32ToIprt(&r32Dst, f32Src); + return r32Dst; +} + +static RTFLOAT64U iemAImpl_round_worker_r64(uint32_t *pfMxcsr, PCRTFLOAT64U pr64Src, uint8_t bImm) +{ + RTFLOAT64U r64Src, r64Dst; + float64_t f64Src; + softfloat_state_t SoftState = iemSseRoundXXMxcsrAndImmToSoftState(*pfMxcsr, bImm); + bool fExact = !RT_BOOL(bImm & X86_SSE_ROUNDXX_IMM_PRECISION); + + iemSsePrepareValueR64(&r64Src, *pfMxcsr, pr64Src); + f64Src = f64_roundToInt(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, fExact, &SoftState); + + iemFpSoftF64ToIprt(&r64Dst, f64Src); + return r64Dst; +} + +#ifdef IEM_WITHOUT_ASSEMBLY +IEM_DECL_IMPL_DEF(void, iemAImpl_roundss_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bImm)) +{ + puDst->ar32[0] = iemAImpl_round_worker_r32(pfMxcsr, &pSrc->uSrc2.ar32[0], bImm & 0x7); + puDst->au32[1] = pSrc->uSrc1.au32[1]; + puDst->au64[1] = pSrc->uSrc1.au64[1]; +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_roundsd_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bImm)) +{ + puDst->ar64[0] = iemAImpl_round_worker_r64(pfMxcsr, &pSrc->uSrc2.ar64[0], bImm & 0x7); + puDst->au64[1] = pSrc->uSrc1.au64[1]; +} +#endif + +IEM_DECL_IMPL_DEF(void, iemAImpl_roundps_u128_fallback,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bImm)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->ar32); i++) + { + puDst->ar32[i] = iemAImpl_round_worker_r32(pfMxcsr, &pSrc->uSrc2.ar32[i], bImm & 0x7); + } +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_roundpd_u128_fallback,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bImm)) +{ + for (uint8_t i = 0; i < RT_ELEMENTS(puDst->ar64); i++) + { + puDst->ar64[i] = iemAImpl_round_worker_r64(pfMxcsr, &pSrc->uSrc2.ar64[i], bImm & 0x7); + } +} + +/** + * CVTPD2PI + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtpd2pi_u128_worker(uint32_t fMxcsr, int32_t *pi32Dst, PCRTFLOAT64U pr64Src) +{ + RTFLOAT64U r64Src; + iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + *pi32Dst = f64_to_i32(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState); + return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpd2pi_u128,(uint32_t *pfMxcsr, uint64_t *pu64Dst, PCX86XMMREG pSrc)) +{ + RTUINT64U u64Res; + uint32_t fMxcsrOut = iemAImpl_cvtpd2pi_u128_worker(*pfMxcsr, &u64Res.ai32[0], &pSrc->ar64[0]); + fMxcsrOut |= iemAImpl_cvtpd2pi_u128_worker(*pfMxcsr, &u64Res.ai32[1], &pSrc->ar64[1]); + + *pu64Dst = u64Res.u; + *pfMxcsr = fMxcsrOut; +} +#endif + + +/** + * CVTTPD2PI + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvttpd2pi_u128_worker(uint32_t fMxcsr, int32_t *pi32Dst, PCRTFLOAT64U pr64Src) +{ + RTFLOAT64U r64Src; + iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + *pi32Dst = f64_to_i32_r_minMag(iemFpSoftF64FromIprt(&r64Src), true /*exact*/, &SoftState); + return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvttpd2pi_u128,(uint32_t *pfMxcsr, uint64_t *pu64Dst, PCX86XMMREG pSrc)) +{ + RTUINT64U u64Res; + uint32_t fMxcsrOut = iemAImpl_cvttpd2pi_u128_worker(*pfMxcsr, &u64Res.ai32[0], &pSrc->ar64[0]); + fMxcsrOut |= iemAImpl_cvttpd2pi_u128_worker(*pfMxcsr, &u64Res.ai32[1], &pSrc->ar64[1]); + + *pu64Dst = u64Res.u; + *pfMxcsr = fMxcsrOut; +} +#endif + + +/** + * CVTPI2PS + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtpi2ps_u128_worker(uint32_t fMxcsr, PRTFLOAT32U pr32Dst, int32_t i32Src) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float32_t r32Res = i32_to_f32(i32Src, &SoftState); + return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Res, pr32Dst, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpi2ps_u128,(uint32_t *pfMxcsr, PX86XMMREG pDst, uint64_t u64Src)) +{ + RTUINT64U uSrc = { u64Src }; + uint32_t fMxcsrOut = iemAImpl_cvtpi2ps_u128_worker(*pfMxcsr, &pDst->ar32[0], uSrc.ai32[0]); + fMxcsrOut |= iemAImpl_cvtpi2ps_u128_worker(*pfMxcsr, &pDst->ar32[1], uSrc.ai32[1]); + *pfMxcsr = fMxcsrOut; +} +#endif + + +/** + * CVTPI2PD + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtpi2pd_u128_worker(uint32_t fMxcsr, PRTFLOAT64U pr64Dst, int32_t i32Src) +{ + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + float64_t r64Res = i32_to_f64(i32Src, &SoftState); + return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Res, pr64Dst, fMxcsr); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpi2pd_u128,(uint32_t *pfMxcsr, PX86XMMREG pDst, uint64_t u64Src)) +{ + RTUINT64U uSrc = { u64Src }; + uint32_t fMxcsrOut = iemAImpl_cvtpi2pd_u128_worker(*pfMxcsr, &pDst->ar64[0], uSrc.ai32[0]); + fMxcsrOut |= iemAImpl_cvtpi2pd_u128_worker(*pfMxcsr, &pDst->ar64[1], uSrc.ai32[1]); + *pfMxcsr = fMxcsrOut; +} +#endif + + +/** + * CVTPS2PI + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvtps2pi_u128_worker(uint32_t fMxcsr, int32_t *pi32Dst, PCRTFLOAT32U pr32Src) +{ + RTFLOAT32U r32Src; + iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + *pi32Dst = f32_to_i32(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, true /*exact*/, &SoftState); + return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvtps2pi_u128,(uint32_t *pfMxcsr, uint64_t *pu64Dst, uint64_t u64Src)) +{ + RTUINT64U uDst; + RTUINT64U uSrc = { u64Src }; + uint32_t fMxcsrOut = iemAImpl_cvtps2pi_u128_worker(*pfMxcsr, &uDst.ai32[0], (PCRTFLOAT32U)&uSrc.au32[0]); + fMxcsrOut |= iemAImpl_cvtps2pi_u128_worker(*pfMxcsr, &uDst.ai32[1], (PCRTFLOAT32U)&uSrc.au32[1]); + *pu64Dst = uDst.u; + *pfMxcsr = fMxcsrOut; +} +#endif + + +/** + * CVTTPS2PI + */ +#ifdef IEM_WITHOUT_ASSEMBLY +static uint32_t iemAImpl_cvttps2pi_u128_worker(uint32_t fMxcsr, int32_t *pi32Dst, PCRTFLOAT32U pr32Src) +{ + RTFLOAT32U r32Src; + iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Src); /* The de-normal flag is not set. */ + + softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr); + *pi32Dst = f32_to_i32_r_minMag(iemFpSoftF32FromIprt(&r32Src), true /*exact*/, &SoftState); + return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS); +} + + +IEM_DECL_IMPL_DEF(void, iemAImpl_cvttps2pi_u128,(uint32_t *pfMxcsr, uint64_t *pu64Dst, uint64_t u64Src)) +{ + RTUINT64U uDst; + RTUINT64U uSrc = { u64Src }; + uint32_t fMxcsrOut = iemAImpl_cvttps2pi_u128_worker(*pfMxcsr, &uDst.ai32[0], (PCRTFLOAT32U)&uSrc.au32[0]); + fMxcsrOut |= iemAImpl_cvttps2pi_u128_worker(*pfMxcsr, &uDst.ai32[1], (PCRTFLOAT32U)&uSrc.au32[1]); + *pu64Dst = uDst.u; + *pfMxcsr = fMxcsrOut; +} +#endif + +/** + * RDRAND + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_rdrand_u16_fallback,(uint16_t *puDst, uint32_t *pEFlags)) +{ + *puDst = 0; + *pEFlags &= ~X86_EFL_STATUS_BITS; + *pEFlags |= X86_EFL_CF; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_rdrand_u32_fallback,(uint32_t *puDst, uint32_t *pEFlags)) +{ + *puDst = 0; + *pEFlags &= ~X86_EFL_STATUS_BITS; + *pEFlags |= X86_EFL_CF; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_rdrand_u64_fallback,(uint64_t *puDst, uint32_t *pEFlags)) +{ + *puDst = 0; + *pEFlags &= ~X86_EFL_STATUS_BITS; + *pEFlags |= X86_EFL_CF; +} + +/** + * RDSEED + */ +IEM_DECL_IMPL_DEF(void, iemAImpl_rdseed_u16_fallback,(uint16_t *puDst, uint32_t *pEFlags)) +{ + *puDst = 0; + *pEFlags &= ~X86_EFL_STATUS_BITS; + *pEFlags |= X86_EFL_CF; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_rdseed_u32_fallback,(uint32_t *puDst, uint32_t *pEFlags)) +{ + *puDst = 0; + *pEFlags &= ~X86_EFL_STATUS_BITS; + *pEFlags |= X86_EFL_CF; +} + +IEM_DECL_IMPL_DEF(void, iemAImpl_rdseed_u64_fallback,(uint64_t *puDst, uint32_t *pEFlags)) +{ + *puDst = 0; + *pEFlags &= ~X86_EFL_STATUS_BITS; + *pEFlags |= X86_EFL_CF; +} + diff --git a/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp b/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp new file mode 100644 index 00000000..c1a4d8d8 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp @@ -0,0 +1,9671 @@ +/* $Id: IEMAllCImpl.cpp $ */ +/** @file + * IEM - Instruction Implementation in C/C++ (code include). + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_IEM +#define VMCPU_INCL_CPUM_GST_CTX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +# include +# include +#endif +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +# include +#endif +#ifndef VBOX_WITHOUT_CPUID_HOST_CALL +# include +#endif +#include +#include +#include +#include "IEMInternal.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "IEMInline.h" + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** + * Flushes the prefetch buffer, light version. + */ +#ifndef IEM_WITH_CODE_TLB +# define IEM_FLUSH_PREFETCH_LIGHT(a_pVCpu, a_cbInstr) do { (a_pVCpu)->iem.s.cbOpcode = (a_cbInstr); } while (0) +#else +# define IEM_FLUSH_PREFETCH_LIGHT(a_pVCpu, a_cbInstr) do { } while (0) +#endif + +/** + * Flushes the prefetch buffer, heavy version. + */ +#ifndef IEM_WITH_CODE_TLB +# define IEM_FLUSH_PREFETCH_HEAVY(a_pVCpu, a_cbInstr) do { (a_pVCpu)->iem.s.cbOpcode = (a_cbInstr); } while (0) +#else +# if 1 +# define IEM_FLUSH_PREFETCH_HEAVY(a_pVCpu, a_cbInstr) do { (a_pVCpu)->iem.s.pbInstrBuf = NULL; } while (0) +# else +# define IEM_FLUSH_PREFETCH_HEAVY(a_pVCpu, a_cbInstr) do { } while (0) +# endif +#endif + + + +/** @name Misc Helpers + * @{ + */ + + +/** + * Worker function for iemHlpCheckPortIOPermission, don't call directly. + * + * @returns Strict VBox status code. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16Port The port number. + * @param cbOperand The operand size. + */ +static VBOXSTRICTRC iemHlpCheckPortIOPermissionBitmap(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand) +{ + /* The TSS bits we're interested in are the same on 386 and AMD64. */ + AssertCompile(AMD64_SEL_TYPE_SYS_TSS_BUSY == X86_SEL_TYPE_SYS_386_TSS_BUSY); + AssertCompile(AMD64_SEL_TYPE_SYS_TSS_AVAIL == X86_SEL_TYPE_SYS_386_TSS_AVAIL); + AssertCompileMembersAtSameOffset(X86TSS32, offIoBitmap, X86TSS64, offIoBitmap); + AssertCompile(sizeof(X86TSS32) == sizeof(X86TSS64)); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR); + + /* + * Check the TSS type, 16-bit TSSes doesn't have any I/O permission bitmap. + */ + Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType); + if (RT_UNLIKELY( pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_BUSY + && pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_AVAIL)) + { + Log(("iemHlpCheckPortIOPermissionBitmap: Port=%#x cb=%d - TSS type %#x (attr=%#x) has no I/O bitmap -> #GP(0)\n", + u16Port, cbOperand, pVCpu->cpum.GstCtx.tr.Attr.n.u4Type, pVCpu->cpum.GstCtx.tr.Attr.u)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Read the bitmap offset (may #PF). + */ + uint16_t offBitmap; + VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &offBitmap, UINT8_MAX, + pVCpu->cpum.GstCtx.tr.u64Base + RT_UOFFSETOF(X86TSS64, offIoBitmap)); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemHlpCheckPortIOPermissionBitmap: Error reading offIoBitmap (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* + * The bit range from u16Port to (u16Port + cbOperand - 1), however intel + * describes the CPU actually reading two bytes regardless of whether the + * bit range crosses a byte boundrary. Thus the + 1 in the test below. + */ + uint32_t offFirstBit = (uint32_t)u16Port / 8 + offBitmap; + /** @todo check if real CPUs ensures that offBitmap has a minimum value of + * for instance sizeof(X86TSS32). */ + if (offFirstBit + 1 > pVCpu->cpum.GstCtx.tr.u32Limit) /* the limit is inclusive */ + { + Log(("iemHlpCheckPortIOPermissionBitmap: offFirstBit=%#x + 1 is beyond u32Limit=%#x -> #GP(0)\n", + offFirstBit, pVCpu->cpum.GstCtx.tr.u32Limit)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Read the necessary bits. + */ + /** @todo Test the assertion in the intel manual that the CPU reads two + * bytes. The question is how this works wrt to \#PF and \#GP on the + * 2nd byte when it's not required. */ + uint16_t bmBytes = UINT16_MAX; + rcStrict = iemMemFetchSysU16(pVCpu, &bmBytes, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + offFirstBit); + if (rcStrict != VINF_SUCCESS) + { + Log(("iemHlpCheckPortIOPermissionBitmap: Error reading I/O bitmap @%#x (%Rrc)\n", offFirstBit, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* + * Perform the check. + */ + uint16_t fPortMask = (1 << cbOperand) - 1; + bmBytes >>= (u16Port & 7); + if (bmBytes & fPortMask) + { + Log(("iemHlpCheckPortIOPermissionBitmap: u16Port=%#x LB %u - access denied (bm=%#x mask=%#x) -> #GP(0)\n", + u16Port, cbOperand, bmBytes, fPortMask)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + return VINF_SUCCESS; +} + + +/** + * Checks if we are allowed to access the given I/O port, raising the + * appropriate exceptions if we aren't (or if the I/O bitmap is not + * accessible). + * + * @returns Strict VBox status code. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16Port The port number. + * @param cbOperand The operand size. + */ +DECLINLINE(VBOXSTRICTRC) iemHlpCheckPortIOPermission(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand) +{ + X86EFLAGS Efl; + Efl.u = IEMMISC_GET_EFL(pVCpu); + if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) + && ( pVCpu->iem.s.uCpl > Efl.Bits.u2IOPL + || Efl.Bits.u1VM) ) + return iemHlpCheckPortIOPermissionBitmap(pVCpu, u16Port, cbOperand); + return VINF_SUCCESS; +} + + +#if 0 +/** + * Calculates the parity bit. + * + * @returns true if the bit is set, false if not. + * @param u8Result The least significant byte of the result. + */ +static bool iemHlpCalcParityFlag(uint8_t u8Result) +{ + /* + * Parity is set if the number of bits in the least significant byte of + * the result is even. + */ + uint8_t cBits; + cBits = u8Result & 1; /* 0 */ + u8Result >>= 1; + cBits += u8Result & 1; + u8Result >>= 1; + cBits += u8Result & 1; + u8Result >>= 1; + cBits += u8Result & 1; + u8Result >>= 1; + cBits += u8Result & 1; /* 4 */ + u8Result >>= 1; + cBits += u8Result & 1; + u8Result >>= 1; + cBits += u8Result & 1; + u8Result >>= 1; + cBits += u8Result & 1; + return !(cBits & 1); +} +#endif /* not used */ + + +/** + * Updates the specified flags according to a 8-bit result. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u8Result The result to set the flags according to. + * @param fToUpdate The flags to update. + * @param fUndefined The flags that are specified as undefined. + */ +static void iemHlpUpdateArithEFlagsU8(PVMCPUCC pVCpu, uint8_t u8Result, uint32_t fToUpdate, uint32_t fUndefined) +{ + uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u; + iemAImpl_test_u8(&u8Result, u8Result, &fEFlags); + pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate | fUndefined); + pVCpu->cpum.GstCtx.eflags.u |= (fToUpdate | fUndefined) & fEFlags; +} + + +/** + * Updates the specified flags according to a 16-bit result. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16Result The result to set the flags according to. + * @param fToUpdate The flags to update. + * @param fUndefined The flags that are specified as undefined. + */ +static void iemHlpUpdateArithEFlagsU16(PVMCPUCC pVCpu, uint16_t u16Result, uint32_t fToUpdate, uint32_t fUndefined) +{ + uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u; + iemAImpl_test_u16(&u16Result, u16Result, &fEFlags); + pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate | fUndefined); + pVCpu->cpum.GstCtx.eflags.u |= (fToUpdate | fUndefined) & fEFlags; +} + + +/** + * Helper used by iret. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param uCpl The new CPL. + * @param pSReg Pointer to the segment register. + */ +static void iemHlpAdjustSelectorForNewCpl(PVMCPUCC pVCpu, uint8_t uCpl, PCPUMSELREG pSReg) +{ + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_MASK); + + if ( uCpl > pSReg->Attr.n.u2Dpl + && pSReg->Attr.n.u1DescType /* code or data, not system */ + && (pSReg->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) + != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) /* not conforming code */ + iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, 0); +} + + +/** + * Indicates that we have modified the FPU state. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemHlpUsedFpu(PVMCPUCC pVCpu) +{ + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM); +} + +/** @} */ + +/** @name C Implementations + * @{ + */ + +/** + * Implements a 16-bit popa. + */ +IEM_CIMPL_DEF_0(iemCImpl_popa_16) +{ + RTGCPTR GCPtrStart = iemRegGetEffRsp(pVCpu); + RTGCPTR GCPtrLast = GCPtrStart + 15; + VBOXSTRICTRC rcStrict; + + /* + * The docs are a bit hard to comprehend here, but it looks like we wrap + * around in real mode as long as none of the individual "popa" crosses the + * end of the stack segment. In protected mode we check the whole access + * in one go. For efficiency, only do the word-by-word thing if we're in + * danger of wrapping around. + */ + /** @todo do popa boundary / wrap-around checks. */ + if (RT_UNLIKELY( IEM_IS_REAL_OR_V86_MODE(pVCpu) + && (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */ + { + /* word-by-word */ + RTUINT64U TmpRsp; + TmpRsp.u = pVCpu->cpum.GstCtx.rsp; + rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.di, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.si, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bp, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + { + iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */ + rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bx, &TmpRsp); + } + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.dx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.cx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.ax, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rsp = TmpRsp.u; + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + else + { + uint16_t const *pa16Mem = NULL; + rcStrict = iemMemMap(pVCpu, (void **)&pa16Mem, 16, X86_SREG_SS, GCPtrStart, IEM_ACCESS_STACK_R, sizeof(*pa16Mem) - 1); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.di = pa16Mem[7 - X86_GREG_xDI]; + pVCpu->cpum.GstCtx.si = pa16Mem[7 - X86_GREG_xSI]; + pVCpu->cpum.GstCtx.bp = pa16Mem[7 - X86_GREG_xBP]; + /* skip sp */ + pVCpu->cpum.GstCtx.bx = pa16Mem[7 - X86_GREG_xBX]; + pVCpu->cpum.GstCtx.dx = pa16Mem[7 - X86_GREG_xDX]; + pVCpu->cpum.GstCtx.cx = pa16Mem[7 - X86_GREG_xCX]; + pVCpu->cpum.GstCtx.ax = pa16Mem[7 - X86_GREG_xAX]; + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa16Mem, IEM_ACCESS_STACK_R); + if (rcStrict == VINF_SUCCESS) + { + iemRegAddToRsp(pVCpu, 16); + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + } + return rcStrict; +} + + +/** + * Implements a 32-bit popa. + */ +IEM_CIMPL_DEF_0(iemCImpl_popa_32) +{ + RTGCPTR GCPtrStart = iemRegGetEffRsp(pVCpu); + RTGCPTR GCPtrLast = GCPtrStart + 31; + VBOXSTRICTRC rcStrict; + + /* + * The docs are a bit hard to comprehend here, but it looks like we wrap + * around in real mode as long as none of the individual "popa" crosses the + * end of the stack segment. In protected mode we check the whole access + * in one go. For efficiency, only do the word-by-word thing if we're in + * danger of wrapping around. + */ + /** @todo do popa boundary / wrap-around checks. */ + if (RT_UNLIKELY( IEM_IS_REAL_OR_V86_MODE(pVCpu) + && (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */ + { + /* word-by-word */ + RTUINT64U TmpRsp; + TmpRsp.u = pVCpu->cpum.GstCtx.rsp; + rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edi, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.esi, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebp, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + { + iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */ + rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebx, &TmpRsp); + } + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ecx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.eax, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + { +#if 1 /** @todo what actually happens with the high bits when we're in 16-bit mode? */ + pVCpu->cpum.GstCtx.rdi &= UINT32_MAX; + pVCpu->cpum.GstCtx.rsi &= UINT32_MAX; + pVCpu->cpum.GstCtx.rbp &= UINT32_MAX; + pVCpu->cpum.GstCtx.rbx &= UINT32_MAX; + pVCpu->cpum.GstCtx.rdx &= UINT32_MAX; + pVCpu->cpum.GstCtx.rcx &= UINT32_MAX; + pVCpu->cpum.GstCtx.rax &= UINT32_MAX; +#endif + pVCpu->cpum.GstCtx.rsp = TmpRsp.u; + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + else + { + uint32_t const *pa32Mem; + rcStrict = iemMemMap(pVCpu, (void **)&pa32Mem, 32, X86_SREG_SS, GCPtrStart, IEM_ACCESS_STACK_R, sizeof(*pa32Mem) - 1); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rdi = pa32Mem[7 - X86_GREG_xDI]; + pVCpu->cpum.GstCtx.rsi = pa32Mem[7 - X86_GREG_xSI]; + pVCpu->cpum.GstCtx.rbp = pa32Mem[7 - X86_GREG_xBP]; + /* skip esp */ + pVCpu->cpum.GstCtx.rbx = pa32Mem[7 - X86_GREG_xBX]; + pVCpu->cpum.GstCtx.rdx = pa32Mem[7 - X86_GREG_xDX]; + pVCpu->cpum.GstCtx.rcx = pa32Mem[7 - X86_GREG_xCX]; + pVCpu->cpum.GstCtx.rax = pa32Mem[7 - X86_GREG_xAX]; + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa32Mem, IEM_ACCESS_STACK_R); + if (rcStrict == VINF_SUCCESS) + { + iemRegAddToRsp(pVCpu, 32); + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + } + return rcStrict; +} + + +/** + * Implements a 16-bit pusha. + */ +IEM_CIMPL_DEF_0(iemCImpl_pusha_16) +{ + RTGCPTR GCPtrTop = iemRegGetEffRsp(pVCpu); + RTGCPTR GCPtrBottom = GCPtrTop - 15; + VBOXSTRICTRC rcStrict; + + /* + * The docs are a bit hard to comprehend here, but it looks like we wrap + * around in real mode as long as none of the individual "pushd" crosses the + * end of the stack segment. In protected mode we check the whole access + * in one go. For efficiency, only do the word-by-word thing if we're in + * danger of wrapping around. + */ + /** @todo do pusha boundary / wrap-around checks. */ + if (RT_UNLIKELY( GCPtrBottom > GCPtrTop + && IEM_IS_REAL_OR_V86_MODE(pVCpu) ) ) + { + /* word-by-word */ + RTUINT64U TmpRsp; + TmpRsp.u = pVCpu->cpum.GstCtx.rsp; + rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.ax, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.cx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.dx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.sp, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bp, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.si, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.di, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rsp = TmpRsp.u; + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + else + { + GCPtrBottom--; + uint16_t *pa16Mem = NULL; + rcStrict = iemMemMap(pVCpu, (void **)&pa16Mem, 16, X86_SREG_SS, GCPtrBottom, IEM_ACCESS_STACK_W, sizeof(*pa16Mem) - 1); + if (rcStrict == VINF_SUCCESS) + { + pa16Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.di; + pa16Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.si; + pa16Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.bp; + pa16Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.sp; + pa16Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.bx; + pa16Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.dx; + pa16Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.cx; + pa16Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.ax; + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa16Mem, IEM_ACCESS_STACK_W); + if (rcStrict == VINF_SUCCESS) + { + iemRegSubFromRsp(pVCpu, 16); + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + } + return rcStrict; +} + + +/** + * Implements a 32-bit pusha. + */ +IEM_CIMPL_DEF_0(iemCImpl_pusha_32) +{ + RTGCPTR GCPtrTop = iemRegGetEffRsp(pVCpu); + RTGCPTR GCPtrBottom = GCPtrTop - 31; + VBOXSTRICTRC rcStrict; + + /* + * The docs are a bit hard to comprehend here, but it looks like we wrap + * around in real mode as long as none of the individual "pusha" crosses the + * end of the stack segment. In protected mode we check the whole access + * in one go. For efficiency, only do the word-by-word thing if we're in + * danger of wrapping around. + */ + /** @todo do pusha boundary / wrap-around checks. */ + if (RT_UNLIKELY( GCPtrBottom > GCPtrTop + && IEM_IS_REAL_OR_V86_MODE(pVCpu) ) ) + { + /* word-by-word */ + RTUINT64U TmpRsp; + TmpRsp.u = pVCpu->cpum.GstCtx.rsp; + rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.eax, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ecx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebx, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esp, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebp, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esi, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edi, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rsp = TmpRsp.u; + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + else + { + GCPtrBottom--; + uint32_t *pa32Mem; + rcStrict = iemMemMap(pVCpu, (void **)&pa32Mem, 32, X86_SREG_SS, GCPtrBottom, IEM_ACCESS_STACK_W, sizeof(*pa32Mem) - 1); + if (rcStrict == VINF_SUCCESS) + { + pa32Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.edi; + pa32Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.esi; + pa32Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.ebp; + pa32Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.esp; + pa32Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.ebx; + pa32Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.edx; + pa32Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.ecx; + pa32Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.eax; + rcStrict = iemMemCommitAndUnmap(pVCpu, pa32Mem, IEM_ACCESS_STACK_W); + if (rcStrict == VINF_SUCCESS) + { + iemRegSubFromRsp(pVCpu, 32); + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + } + return rcStrict; +} + + +/** + * Implements pushf. + * + * + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_1(iemCImpl_pushf, IEMMODE, enmEffOpSize) +{ + VBOXSTRICTRC rcStrict; + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_PUSHF)) + { + Log2(("pushf: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_PUSHF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * If we're in V8086 mode some care is required (which is why we're in + * doing this in a C implementation). + */ + uint32_t fEfl = IEMMISC_GET_EFL(pVCpu); + if ( (fEfl & X86_EFL_VM) + && X86_EFL_GET_IOPL(fEfl) != 3 ) + { + Assert(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE); + if ( enmEffOpSize != IEMMODE_16BIT + || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME)) + return iemRaiseGeneralProtectionFault0(pVCpu); + fEfl &= ~X86_EFL_IF; /* (RF and VM are out of range) */ + fEfl |= (fEfl & X86_EFL_VIF) >> (19 - 9); + rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl); + } + else + { + + /* + * Ok, clear RF and VM, adjust for ancient CPUs, and push the flags. + */ + fEfl &= ~(X86_EFL_RF | X86_EFL_VM); + + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186); + if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_186) + fEfl |= UINT16_C(0xf000); + rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl); + break; + case IEMMODE_32BIT: + rcStrict = iemMemStackPushU32(pVCpu, fEfl); + break; + case IEMMODE_64BIT: + rcStrict = iemMemStackPushU64(pVCpu, fEfl); + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + + if (rcStrict == VINF_SUCCESS) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return rcStrict; +} + + +/** + * Implements popf. + * + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_1(iemCImpl_popf, IEMMODE, enmEffOpSize) +{ + uint32_t const fEflOld = IEMMISC_GET_EFL(pVCpu); + VBOXSTRICTRC rcStrict; + uint32_t fEflNew; + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_POPF)) + { + Log2(("popf: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_POPF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * V8086 is special as usual. + */ + if (fEflOld & X86_EFL_VM) + { + /* + * Almost anything goes if IOPL is 3. + */ + if (X86_EFL_GET_IOPL(fEflOld) == 3) + { + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t u16Value; + rcStrict = iemMemStackPopU16(pVCpu, &u16Value); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + fEflNew = u16Value | (fEflOld & UINT32_C(0xffff0000)); + break; + } + case IEMMODE_32BIT: + rcStrict = iemMemStackPopU32(pVCpu, &fEflNew); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386 + ? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386; + fEflNew &= fPopfBits & ~(X86_EFL_IOPL); + fEflNew |= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld; + } + /* + * Interrupt flag virtualization with CR4.VME=1. + */ + else if ( enmEffOpSize == IEMMODE_16BIT + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) ) + { + uint16_t u16Value; + RTUINT64U TmpRsp; + TmpRsp.u = pVCpu->cpum.GstCtx.rsp; + rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Value, &TmpRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /** @todo Is the popf VME \#GP(0) delivered after updating RSP+RIP + * or before? */ + if ( ( (u16Value & X86_EFL_IF) + && (fEflOld & X86_EFL_VIP)) + || (u16Value & X86_EFL_TF) ) + return iemRaiseGeneralProtectionFault0(pVCpu); + + fEflNew = u16Value | (fEflOld & UINT32_C(0xffff0000) & ~X86_EFL_VIF); + fEflNew |= (fEflNew & X86_EFL_IF) << (19 - 9); + fEflNew &= X86_EFL_POPF_BITS & ~(X86_EFL_IOPL | X86_EFL_IF); + fEflNew |= ~(X86_EFL_POPF_BITS & ~(X86_EFL_IOPL | X86_EFL_IF)) & fEflOld; + + pVCpu->cpum.GstCtx.rsp = TmpRsp.u; + } + else + return iemRaiseGeneralProtectionFault0(pVCpu); + + } + /* + * Not in V8086 mode. + */ + else + { + /* Pop the flags. */ + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t u16Value; + rcStrict = iemMemStackPopU16(pVCpu, &u16Value); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + fEflNew = u16Value | (fEflOld & UINT32_C(0xffff0000)); + + /* + * Ancient CPU adjustments: + * - 8086, 80186, V20/30: + * Fixed bits 15:12 bits are not kept correctly internally, mostly for + * practical reasons (masking below). We add them when pushing flags. + * - 80286: + * The NT and IOPL flags cannot be popped from real mode and are + * therefore always zero (since a 286 can never exit from PM and + * their initial value is zero). This changed on a 386 and can + * therefore be used to detect 286 or 386 CPU in real mode. + */ + if ( IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286 + && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) ) + fEflNew &= ~(X86_EFL_NT | X86_EFL_IOPL); + break; + } + case IEMMODE_32BIT: + rcStrict = iemMemStackPopU32(pVCpu, &fEflNew); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + break; + case IEMMODE_64BIT: + { + uint64_t u64Value; + rcStrict = iemMemStackPopU64(pVCpu, &u64Value); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + fEflNew = u64Value; /** @todo testcase: Check exactly what happens if high bits are set. */ + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + /* Merge them with the current flags. */ + const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386 + ? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386; + if ( (fEflNew & (X86_EFL_IOPL | X86_EFL_IF)) == (fEflOld & (X86_EFL_IOPL | X86_EFL_IF)) + || pVCpu->iem.s.uCpl == 0) + { + fEflNew &= fPopfBits; + fEflNew |= ~fPopfBits & fEflOld; + } + else if (pVCpu->iem.s.uCpl <= X86_EFL_GET_IOPL(fEflOld)) + { + fEflNew &= fPopfBits & ~(X86_EFL_IOPL); + fEflNew |= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld; + } + else + { + fEflNew &= fPopfBits & ~(X86_EFL_IOPL | X86_EFL_IF); + fEflNew |= ~(fPopfBits & ~(X86_EFL_IOPL | X86_EFL_IF)) & fEflOld; + } + } + + /* + * Commit the flags. + */ + Assert(fEflNew & RT_BIT_32(1)); + IEMMISC_SET_EFL(pVCpu, fEflNew); + return iemRegAddToRipAndFinishingClearingRfEx(pVCpu, cbInstr, fEflOld); +} + + +/** + * Implements an indirect call. + * + * @param uNewPC The new program counter (RIP) value (loaded from the + * operand). + */ +IEM_CIMPL_DEF_1(iemCImpl_call_16, uint16_t, uNewPC) +{ + uint16_t const uOldPC = pVCpu->cpum.GstCtx.ip + cbInstr; + if (uNewPC <= pVCpu->cpum.GstCtx.cs.u32Limit) + { + VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldPC); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rip = uNewPC; + IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr); + return iemRegFinishClearingRF(pVCpu); + } + return rcStrict; + } + return iemRaiseGeneralProtectionFault0(pVCpu); +} + + +/** + * Implements a 16-bit relative call. + * + * @param offDisp The displacment offset. + */ +IEM_CIMPL_DEF_1(iemCImpl_call_rel_16, int16_t, offDisp) +{ + uint16_t const uOldPC = pVCpu->cpum.GstCtx.ip + cbInstr; + uint16_t const uNewPC = uOldPC + offDisp; + if (uNewPC <= pVCpu->cpum.GstCtx.cs.u32Limit) + { + VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldPC); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rip = uNewPC; + IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr); + return iemRegFinishClearingRF(pVCpu); + } + return rcStrict; + } + return iemRaiseGeneralProtectionFault0(pVCpu); +} + + +/** + * Implements a 32-bit indirect call. + * + * @param uNewPC The new program counter (RIP) value (loaded from the + * operand). + */ +IEM_CIMPL_DEF_1(iemCImpl_call_32, uint32_t, uNewPC) +{ + uint32_t const uOldPC = pVCpu->cpum.GstCtx.eip + cbInstr; + if (uNewPC <= pVCpu->cpum.GstCtx.cs.u32Limit) + { + VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldPC); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rip = uNewPC; + IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr); + return iemRegFinishClearingRF(pVCpu); + } + return rcStrict; + } + return iemRaiseGeneralProtectionFault0(pVCpu); +} + + +/** + * Implements a 32-bit relative call. + * + * @param offDisp The displacment offset. + */ +IEM_CIMPL_DEF_1(iemCImpl_call_rel_32, int32_t, offDisp) +{ + uint32_t const uOldPC = pVCpu->cpum.GstCtx.eip + cbInstr; + uint32_t const uNewPC = uOldPC + offDisp; + if (uNewPC <= pVCpu->cpum.GstCtx.cs.u32Limit) + { + VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldPC); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rip = uNewPC; + IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr); + return iemRegFinishClearingRF(pVCpu); + } + return rcStrict; + } + return iemRaiseGeneralProtectionFault0(pVCpu); +} + + +/** + * Implements a 64-bit indirect call. + * + * @param uNewPC The new program counter (RIP) value (loaded from the + * operand). + */ +IEM_CIMPL_DEF_1(iemCImpl_call_64, uint64_t, uNewPC) +{ + uint64_t const uOldPC = pVCpu->cpum.GstCtx.rip + cbInstr; + if (IEM_IS_CANONICAL(uNewPC)) + { + VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldPC); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rip = uNewPC; + IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr); + return iemRegFinishClearingRF(pVCpu); + } + return rcStrict; + } + return iemRaiseGeneralProtectionFault0(pVCpu); +} + + +/** + * Implements a 64-bit relative call. + * + * @param offDisp The displacment offset. + */ +IEM_CIMPL_DEF_1(iemCImpl_call_rel_64, int64_t, offDisp) +{ + uint64_t const uOldPC = pVCpu->cpum.GstCtx.rip + cbInstr; + uint64_t const uNewPC = uOldPC + offDisp; + if (IEM_IS_CANONICAL(uNewPC)) + { + VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldPC); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rip = uNewPC; + IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr); + return iemRegFinishClearingRF(pVCpu); + } + return rcStrict; + } + return iemRaiseNotCanonical(pVCpu); +} + + +/** + * Implements far jumps and calls thru task segments (TSS). + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param cbInstr The current instruction length. + * @param uSel The selector. + * @param enmBranch The kind of branching we're performing. + * @param enmEffOpSize The effective operand size. + * @param pDesc The descriptor corresponding to @a uSel. The type is + * task gate. + */ +static VBOXSTRICTRC iemCImpl_BranchTaskSegment(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch, + IEMMODE enmEffOpSize, PIEMSELDESC pDesc) +{ +#ifndef IEM_IMPLEMENTS_TASKSWITCH + IEM_RETURN_ASPECT_NOT_IMPLEMENTED(); +#else + Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL); + Assert( pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL + || pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL); + RT_NOREF_PV(enmEffOpSize); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + + if ( pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl + || pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL)) + { + Log(("BranchTaskSegment invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl, + pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL))); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL); + } + + /** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not + * far calls (see iemCImpl_callf). Most likely in both cases it should be + * checked here, need testcases. */ + if (!pDesc->Legacy.Gen.u1Present) + { + Log(("BranchTaskSegment TSS not present uSel=%04x -> #NP\n", uSel)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL); + } + + uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr; + return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL, + uNextEip, 0 /* fFlags */, 0 /* uErr */, 0 /* uCr2 */, uSel, pDesc); +#endif +} + + +/** + * Implements far jumps and calls thru task gates. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param cbInstr The current instruction length. + * @param uSel The selector. + * @param enmBranch The kind of branching we're performing. + * @param enmEffOpSize The effective operand size. + * @param pDesc The descriptor corresponding to @a uSel. The type is + * task gate. + */ +static VBOXSTRICTRC iemCImpl_BranchTaskGate(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch, + IEMMODE enmEffOpSize, PIEMSELDESC pDesc) +{ +#ifndef IEM_IMPLEMENTS_TASKSWITCH + IEM_RETURN_ASPECT_NOT_IMPLEMENTED(); +#else + Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL); + RT_NOREF_PV(enmEffOpSize); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + + if ( pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl + || pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL)) + { + Log(("BranchTaskGate invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl, + pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL))); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL); + } + + /** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not + * far calls (see iemCImpl_callf). Most likely in both cases it should be + * checked here, need testcases. */ + if (!pDesc->Legacy.Gen.u1Present) + { + Log(("BranchTaskSegment segment not present uSel=%04x -> #NP\n", uSel)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL); + } + + /* + * Fetch the new TSS descriptor from the GDT. + */ + RTSEL uSelTss = pDesc->Legacy.Gate.u16Sel; + if (uSelTss & X86_SEL_LDT) + { + Log(("BranchTaskGate TSS is in LDT. uSel=%04x uSelTss=%04x -> #GP\n", uSel, uSelTss)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL); + } + + IEMSELDESC TssDesc; + VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelTss, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + if (TssDesc.Legacy.Gate.u4Type & X86_SEL_TYPE_SYS_TSS_BUSY_MASK) + { + Log(("BranchTaskGate TSS is busy. uSel=%04x uSelTss=%04x DescType=%#x -> #GP\n", uSel, uSelTss, + TssDesc.Legacy.Gate.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL); + } + + if (!TssDesc.Legacy.Gate.u1Present) + { + Log(("BranchTaskGate TSS is not present. uSel=%04x uSelTss=%04x -> #NP\n", uSel, uSelTss)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelTss & X86_SEL_MASK_OFF_RPL); + } + + uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr; + return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL, + uNextEip, 0 /* fFlags */, 0 /* uErr */, 0 /* uCr2 */, uSelTss, &TssDesc); +#endif +} + + +/** + * Implements far jumps and calls thru call gates. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param cbInstr The current instruction length. + * @param uSel The selector. + * @param enmBranch The kind of branching we're performing. + * @param enmEffOpSize The effective operand size. + * @param pDesc The descriptor corresponding to @a uSel. The type is + * call gate. + */ +static VBOXSTRICTRC iemCImpl_BranchCallGate(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch, + IEMMODE enmEffOpSize, PIEMSELDESC pDesc) +{ +#define IEM_IMPLEMENTS_CALLGATE +#ifndef IEM_IMPLEMENTS_CALLGATE + IEM_RETURN_ASPECT_NOT_IMPLEMENTED(); +#else + RT_NOREF_PV(enmEffOpSize); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + + /* NB: Far jumps can only do intra-privilege transfers. Far calls support + * inter-privilege calls and are much more complex. + * + * NB: 64-bit call gate has the same type as a 32-bit call gate! If + * EFER.LMA=1, the gate must be 64-bit. Conversely if EFER.LMA=0, the gate + * must be 16-bit or 32-bit. + */ + /** @todo effective operand size is probably irrelevant here, only the + * call gate bitness matters?? + */ + VBOXSTRICTRC rcStrict; + RTPTRUNION uPtrRet; + uint64_t uNewRsp; + uint64_t uNewRip; + uint64_t u64Base; + uint32_t cbLimit; + RTSEL uNewCS; + IEMSELDESC DescCS; + + AssertCompile(X86_SEL_TYPE_SYS_386_CALL_GATE == AMD64_SEL_TYPE_SYS_CALL_GATE); + Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL); + Assert( pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE + || pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE); + + /* Determine the new instruction pointer from the gate descriptor. */ + uNewRip = pDesc->Legacy.Gate.u16OffsetLow + | ((uint32_t)pDesc->Legacy.Gate.u16OffsetHigh << 16) + | ((uint64_t)pDesc->Long.Gate.u32OffsetTop << 32); + + /* Perform DPL checks on the gate descriptor. */ + if ( pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl + || pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL)) + { + Log(("BranchCallGate invalid priv. uSel=%04x Gate DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl, + pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL))); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + + /** @todo does this catch NULL selectors, too? */ + if (!pDesc->Legacy.Gen.u1Present) + { + Log(("BranchCallGate Gate not present uSel=%04x -> #NP\n", uSel)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel); + } + + /* + * Fetch the target CS descriptor from the GDT or LDT. + */ + uNewCS = pDesc->Legacy.Gate.u16Sel; + rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Target CS must be a code selector. */ + if ( !DescCS.Legacy.Gen.u1DescType + || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) ) + { + Log(("BranchCallGate %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n", + uNewCS, uNewRip, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS); + } + + /* Privilege checks on target CS. */ + if (enmBranch == IEMBRANCH_JUMP) + { + if (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) + { + if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl) + { + Log(("BranchCallGate jump (conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n", + uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS); + } + } + else + { + if (DescCS.Legacy.Gen.u2Dpl != pVCpu->iem.s.uCpl) + { + Log(("BranchCallGate jump (non-conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n", + uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS); + } + } + } + else + { + Assert(enmBranch == IEMBRANCH_CALL); + if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl) + { + Log(("BranchCallGate call invalid priv. uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n", + uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL); + } + } + + /* Additional long mode checks. */ + if (IEM_IS_LONG_MODE(pVCpu)) + { + if (!DescCS.Legacy.Gen.u1Long) + { + Log(("BranchCallGate uNewCS %04x -> not a 64-bit code segment.\n", uNewCS)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS); + } + + /* L vs D. */ + if ( DescCS.Legacy.Gen.u1Long + && DescCS.Legacy.Gen.u1DefBig) + { + Log(("BranchCallGate uNewCS %04x -> both L and D are set.\n", uNewCS)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS); + } + } + + if (!DescCS.Legacy.Gate.u1Present) + { + Log(("BranchCallGate target CS is not present. uSel=%04x uNewCS=%04x -> #NP(CS)\n", uSel, uNewCS)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCS); + } + + if (enmBranch == IEMBRANCH_JUMP) + { + /** @todo This is very similar to regular far jumps; merge! */ + /* Jumps are fairly simple... */ + + /* Chop the high bits off if 16-bit gate (Intel says so). */ + if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE) + uNewRip = (uint16_t)uNewRip; + + /* Limit check for non-long segments. */ + cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy); + if (DescCS.Legacy.Gen.u1Long) + u64Base = 0; + else + { + if (uNewRip > cbLimit) + { + Log(("BranchCallGate jump %04x:%08RX64 -> out of bounds (%#x) -> #GP(0)\n", uNewCS, uNewRip, cbLimit)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0); + } + u64Base = X86DESC_BASE(&DescCS.Legacy); + } + + /* Canonical address check. */ + if (!IEM_IS_CANONICAL(uNewRip)) + { + Log(("BranchCallGate jump %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip)); + return iemRaiseNotCanonical(pVCpu); + } + + /* + * Ok, everything checked out fine. Now set the accessed bit before + * committing the result into CS, CSHID and RIP. + */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + /** @todo check what VT-x and AMD-V does. */ + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* commit */ + pVCpu->cpum.GstCtx.rip = uNewRip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCS & X86_SEL_MASK_OFF_RPL; + pVCpu->cpum.GstCtx.cs.Sel |= pVCpu->iem.s.uCpl; /** @todo is this right for conforming segs? or in general? */ + pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit; + pVCpu->cpum.GstCtx.cs.u64Base = u64Base; + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + } + else + { + Assert(enmBranch == IEMBRANCH_CALL); + /* Calls are much more complicated. */ + + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) && (DescCS.Legacy.Gen.u2Dpl < pVCpu->iem.s.uCpl)) + { + uint16_t offNewStack; /* Offset of new stack in TSS. */ + uint16_t cbNewStack; /* Number of bytes the stack information takes up in TSS. */ + uint8_t uNewCSDpl; + uint8_t cbWords; + RTSEL uNewSS; + RTSEL uOldSS; + uint64_t uOldRsp; + IEMSELDESC DescSS; + RTPTRUNION uPtrTSS; + RTGCPTR GCPtrTSS; + RTPTRUNION uPtrParmWds; + RTGCPTR GCPtrParmWds; + + /* More privilege. This is the fun part. */ + Assert(!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)); /* Filtered out above. */ + + /* + * Determine new SS:rSP from the TSS. + */ + Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType); + + /* Figure out where the new stack pointer is stored in the TSS. */ + uNewCSDpl = DescCS.Legacy.Gen.u2Dpl; + if (!IEM_IS_LONG_MODE(pVCpu)) + { + if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY) + { + offNewStack = RT_UOFFSETOF(X86TSS32, esp0) + uNewCSDpl * 8; + cbNewStack = RT_SIZEOFMEMB(X86TSS32, esp0) + RT_SIZEOFMEMB(X86TSS32, ss0); + } + else + { + Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY); + offNewStack = RT_UOFFSETOF(X86TSS16, sp0) + uNewCSDpl * 4; + cbNewStack = RT_SIZEOFMEMB(X86TSS16, sp0) + RT_SIZEOFMEMB(X86TSS16, ss0); + } + } + else + { + Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY); + offNewStack = RT_UOFFSETOF(X86TSS64, rsp0) + uNewCSDpl * RT_SIZEOFMEMB(X86TSS64, rsp0); + cbNewStack = RT_SIZEOFMEMB(X86TSS64, rsp0); + } + + /* Check against TSS limit. */ + if ((uint16_t)(offNewStack + cbNewStack - 1) > pVCpu->cpum.GstCtx.tr.u32Limit) + { + Log(("BranchCallGate inner stack past TSS limit - %u > %u -> #TS(TSS)\n", offNewStack + cbNewStack - 1, pVCpu->cpum.GstCtx.tr.u32Limit)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, pVCpu->cpum.GstCtx.tr.Sel); + } + + GCPtrTSS = pVCpu->cpum.GstCtx.tr.u64Base + offNewStack; + rcStrict = iemMemMap(pVCpu, &uPtrTSS.pv, cbNewStack, UINT8_MAX, GCPtrTSS, IEM_ACCESS_SYS_R, 0); + if (rcStrict != VINF_SUCCESS) + { + Log(("BranchCallGate: TSS mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + if (!IEM_IS_LONG_MODE(pVCpu)) + { + if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY) + { + uNewRsp = uPtrTSS.pu32[0]; + uNewSS = uPtrTSS.pu16[2]; + } + else + { + Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY); + uNewRsp = uPtrTSS.pu16[0]; + uNewSS = uPtrTSS.pu16[1]; + } + } + else + { + Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY); + /* SS will be a NULL selector, but that's valid. */ + uNewRsp = uPtrTSS.pu64[0]; + uNewSS = uNewCSDpl; + } + + /* Done with the TSS now. */ + rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrTSS.pv, IEM_ACCESS_SYS_R); + if (rcStrict != VINF_SUCCESS) + { + Log(("BranchCallGate: TSS unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Only used outside of long mode. */ + cbWords = pDesc->Legacy.Gate.u5ParmCount; + + /* If EFER.LMA is 0, there's extra work to do. */ + if (!IEM_IS_LONG_MODE(pVCpu)) + { + if ((uNewSS & X86_SEL_MASK_OFF_RPL) == 0) + { + Log(("BranchCallGate new SS NULL -> #TS(NewSS)\n")); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS); + } + + /* Grab the new SS descriptor. */ + rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Ensure that CS.DPL == SS.RPL == SS.DPL. */ + if ( (DescCS.Legacy.Gen.u2Dpl != (uNewSS & X86_SEL_RPL)) + || (DescCS.Legacy.Gen.u2Dpl != DescSS.Legacy.Gen.u2Dpl)) + { + Log(("BranchCallGate call bad RPL/DPL uNewSS=%04x SS DPL=%d CS DPL=%u -> #TS(NewSS)\n", + uNewSS, DescCS.Legacy.Gen.u2Dpl, DescCS.Legacy.Gen.u2Dpl)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS); + } + + /* Ensure new SS is a writable data segment. */ + if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE) + { + Log(("BranchCallGate call new SS -> not a writable data selector (u4Type=%#x)\n", DescSS.Legacy.Gen.u4Type)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS); + } + + if (!DescSS.Legacy.Gen.u1Present) + { + Log(("BranchCallGate New stack not present uSel=%04x -> #SS(NewSS)\n", uNewSS)); + return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS); + } + if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE) + cbNewStack = (uint16_t)sizeof(uint32_t) * (4 + cbWords); + else + cbNewStack = (uint16_t)sizeof(uint16_t) * (4 + cbWords); + } + else + { + /* Just grab the new (NULL) SS descriptor. */ + /** @todo testcase: Check whether the zero GDT entry is actually loaded here + * like we do... */ + rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + cbNewStack = sizeof(uint64_t) * 4; + } + + /** @todo According to Intel, new stack is checked for enough space first, + * then switched. According to AMD, the stack is switched first and + * then pushes might fault! + * NB: OS/2 Warp 3/4 actively relies on the fact that possible + * incoming stack \#PF happens before actual stack switch. AMD is + * either lying or implicitly assumes that new state is committed + * only if and when an instruction doesn't fault. + */ + + /** @todo According to AMD, CS is loaded first, then SS. + * According to Intel, it's the other way around!? + */ + + /** @todo Intel and AMD disagree on when exactly the CPL changes! */ + + /* Set the accessed bit before committing new SS. */ + if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* Remember the old SS:rSP and their linear address. */ + uOldSS = pVCpu->cpum.GstCtx.ss.Sel; + uOldRsp = pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig ? pVCpu->cpum.GstCtx.rsp : pVCpu->cpum.GstCtx.sp; + + GCPtrParmWds = pVCpu->cpum.GstCtx.ss.u64Base + uOldRsp; + + /* HACK ALERT! Probe if the write to the new stack will succeed. May #SS(NewSS) + or #PF, the former is not implemented in this workaround. */ + /** @todo Proper fix callgate target stack exceptions. */ + /** @todo testcase: Cover callgates with partially or fully inaccessible + * target stacks. */ + void *pvNewFrame; + RTGCPTR GCPtrNewStack = X86DESC_BASE(&DescSS.Legacy) + uNewRsp - cbNewStack; + rcStrict = iemMemMap(pVCpu, &pvNewFrame, cbNewStack, UINT8_MAX, GCPtrNewStack, IEM_ACCESS_SYS_RW, 0); + if (rcStrict != VINF_SUCCESS) + { + Log(("BranchCallGate: Incoming stack (%04x:%08RX64) not accessible, rc=%Rrc\n", uNewSS, uNewRsp, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewFrame, IEM_ACCESS_SYS_RW); + if (rcStrict != VINF_SUCCESS) + { + Log(("BranchCallGate: New stack probe unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Commit new SS:rSP. */ + pVCpu->cpum.GstCtx.ss.Sel = uNewSS; + pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy); + pVCpu->cpum.GstCtx.ss.u32Limit = X86DESC_LIMIT_G(&DescSS.Legacy); + pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy); + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.rsp = uNewRsp; + pVCpu->iem.s.uCpl = uNewCSDpl; /** @todo is the parameter words accessed using the new CPL or the old CPL? */ + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS); + + /* At this point the stack access must not fail because new state was already committed. */ + /** @todo this can still fail due to SS.LIMIT not check. */ + rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbNewStack, + IEM_IS_LONG_MODE(pVCpu) ? 7 + : pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 3 : 1, + &uPtrRet.pv, &uNewRsp); + AssertMsgReturn(rcStrict == VINF_SUCCESS, ("BranchCallGate: New stack mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)), + VERR_INTERNAL_ERROR_5); + + if (!IEM_IS_LONG_MODE(pVCpu)) + { + if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE) + { + if (cbWords) + { + /* Map the relevant chunk of the old stack. */ + rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, cbWords * 4, UINT8_MAX, GCPtrParmWds, + IEM_ACCESS_DATA_R, 0 /** @todo Can uNewCSDpl == 3? Then we need alignment mask here! */); + if (rcStrict != VINF_SUCCESS) + { + Log(("BranchCallGate: Old stack mapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Copy the parameter (d)words. */ + for (int i = 0; i < cbWords; ++i) + uPtrRet.pu32[2 + i] = uPtrParmWds.pu32[i]; + + /* Unmap the old stack. */ + rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrParmWds.pv, IEM_ACCESS_DATA_R); + if (rcStrict != VINF_SUCCESS) + { + Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } + + /* Push the old CS:rIP. */ + uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr; + uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */ + + /* Push the old SS:rSP. */ + uPtrRet.pu32[2 + cbWords + 0] = uOldRsp; + uPtrRet.pu32[2 + cbWords + 1] = uOldSS; + } + else + { + Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE); + + if (cbWords) + { + /* Map the relevant chunk of the old stack. */ + rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, cbWords * 2, UINT8_MAX, GCPtrParmWds, + IEM_ACCESS_DATA_R, 0 /** @todo Can uNewCSDpl == 3? Then we need alignment mask here! */); + if (rcStrict != VINF_SUCCESS) + { + Log(("BranchCallGate: Old stack mapping (16-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Copy the parameter words. */ + for (int i = 0; i < cbWords; ++i) + uPtrRet.pu16[2 + i] = uPtrParmWds.pu16[i]; + + /* Unmap the old stack. */ + rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrParmWds.pv, IEM_ACCESS_DATA_R); + if (rcStrict != VINF_SUCCESS) + { + Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } + + /* Push the old CS:rIP. */ + uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr; + uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel; + + /* Push the old SS:rSP. */ + uPtrRet.pu16[2 + cbWords + 0] = uOldRsp; + uPtrRet.pu16[2 + cbWords + 1] = uOldSS; + } + } + else + { + Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE); + + /* For 64-bit gates, no parameters are copied. Just push old SS:rSP and CS:rIP. */ + uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr; + uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */ + uPtrRet.pu64[2] = uOldRsp; + uPtrRet.pu64[3] = uOldSS; /** @todo Testcase: What is written to the high words when pushing SS? */ + } + + rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp); + if (rcStrict != VINF_SUCCESS) + { + Log(("BranchCallGate: New stack unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Chop the high bits off if 16-bit gate (Intel says so). */ + if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE) + uNewRip = (uint16_t)uNewRip; + + /* Limit / canonical check. */ + cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy); + if (!IEM_IS_LONG_MODE(pVCpu)) + { + if (uNewRip > cbLimit) + { + Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0); + } + u64Base = X86DESC_BASE(&DescCS.Legacy); + } + else + { + Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE); + if (!IEM_IS_CANONICAL(uNewRip)) + { + Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip)); + return iemRaiseNotCanonical(pVCpu); + } + u64Base = 0; + } + + /* + * Now set the accessed bit before + * writing the return address to the stack and committing the result into + * CS, CSHID and RIP. + */ + /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + /** @todo check what VT-x and AMD-V does. */ + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* Commit new CS:rIP. */ + pVCpu->cpum.GstCtx.rip = uNewRip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCS & X86_SEL_MASK_OFF_RPL; + pVCpu->cpum.GstCtx.cs.Sel |= pVCpu->iem.s.uCpl; + pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit; + pVCpu->cpum.GstCtx.cs.u64Base = u64Base; + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + } + else + { + /* Same privilege. */ + /** @todo This is very similar to regular far calls; merge! */ + + /* Check stack first - may #SS(0). */ + /** @todo check how gate size affects pushing of CS! Does callf 16:32 in + * 16-bit code cause a two or four byte CS to be pushed? */ + rcStrict = iemMemStackPushBeginSpecial(pVCpu, + IEM_IS_LONG_MODE(pVCpu) ? 8+8 + : pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 4+4 : 2+2, + IEM_IS_LONG_MODE(pVCpu) ? 7 + : pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 3 : 2, + &uPtrRet.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Chop the high bits off if 16-bit gate (Intel says so). */ + if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE) + uNewRip = (uint16_t)uNewRip; + + /* Limit / canonical check. */ + cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy); + if (!IEM_IS_LONG_MODE(pVCpu)) + { + if (uNewRip > cbLimit) + { + Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0); + } + u64Base = X86DESC_BASE(&DescCS.Legacy); + } + else + { + if (!IEM_IS_CANONICAL(uNewRip)) + { + Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip)); + return iemRaiseNotCanonical(pVCpu); + } + u64Base = 0; + } + + /* + * Now set the accessed bit before + * writing the return address to the stack and committing the result into + * CS, CSHID and RIP. + */ + /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + /** @todo check what VT-x and AMD-V does. */ + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* stack */ + if (!IEM_IS_LONG_MODE(pVCpu)) + { + if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE) + { + uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr; + uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */ + } + else + { + Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE); + uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr; + uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel; + } + } + else + { + Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE); + uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr; + uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */ + } + + rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* commit */ + pVCpu->cpum.GstCtx.rip = uNewRip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCS & X86_SEL_MASK_OFF_RPL; + pVCpu->cpum.GstCtx.cs.Sel |= pVCpu->iem.s.uCpl; + pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit; + pVCpu->cpum.GstCtx.cs.u64Base = u64Base; + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + } + } + pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0; +/** @todo single stepping */ + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); + return VINF_SUCCESS; +#endif /* IEM_IMPLEMENTS_CALLGATE */ +} + + +/** + * Implements far jumps and calls thru system selectors. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param cbInstr The current instruction length. + * @param uSel The selector. + * @param enmBranch The kind of branching we're performing. + * @param enmEffOpSize The effective operand size. + * @param pDesc The descriptor corresponding to @a uSel. + */ +static VBOXSTRICTRC iemCImpl_BranchSysSel(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch, + IEMMODE enmEffOpSize, PIEMSELDESC pDesc) +{ + Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL); + Assert((uSel & X86_SEL_MASK_OFF_RPL)); + IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK); + + if (IEM_IS_LONG_MODE(pVCpu)) + switch (pDesc->Legacy.Gen.u4Type) + { + case AMD64_SEL_TYPE_SYS_CALL_GATE: + return iemCImpl_BranchCallGate(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc); + + default: + case AMD64_SEL_TYPE_SYS_LDT: + case AMD64_SEL_TYPE_SYS_TSS_BUSY: + case AMD64_SEL_TYPE_SYS_TSS_AVAIL: + case AMD64_SEL_TYPE_SYS_TRAP_GATE: + case AMD64_SEL_TYPE_SYS_INT_GATE: + Log(("branch %04x -> wrong sys selector (64-bit): %d\n", uSel, pDesc->Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + + switch (pDesc->Legacy.Gen.u4Type) + { + case X86_SEL_TYPE_SYS_286_CALL_GATE: + case X86_SEL_TYPE_SYS_386_CALL_GATE: + return iemCImpl_BranchCallGate(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc); + + case X86_SEL_TYPE_SYS_TASK_GATE: + return iemCImpl_BranchTaskGate(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc); + + case X86_SEL_TYPE_SYS_286_TSS_AVAIL: + case X86_SEL_TYPE_SYS_386_TSS_AVAIL: + return iemCImpl_BranchTaskSegment(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc); + + case X86_SEL_TYPE_SYS_286_TSS_BUSY: + Log(("branch %04x -> busy 286 TSS\n", uSel)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + + case X86_SEL_TYPE_SYS_386_TSS_BUSY: + Log(("branch %04x -> busy 386 TSS\n", uSel)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + + default: + case X86_SEL_TYPE_SYS_LDT: + case X86_SEL_TYPE_SYS_286_INT_GATE: + case X86_SEL_TYPE_SYS_286_TRAP_GATE: + case X86_SEL_TYPE_SYS_386_INT_GATE: + case X86_SEL_TYPE_SYS_386_TRAP_GATE: + Log(("branch %04x -> wrong sys selector: %d\n", uSel, pDesc->Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } +} + + +/** + * Implements far jumps. + * + * @param uSel The selector. + * @param offSeg The segment offset. + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_3(iemCImpl_FarJmp, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize) +{ + NOREF(cbInstr); + Assert(offSeg <= UINT32_MAX || (!IEM_IS_GUEST_CPU_AMD(pVCpu) && pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)); + + /* + * Real mode and V8086 mode are easy. The only snag seems to be that + * CS.limit doesn't change and the limit check is done against the current + * limit. + */ + /** @todo Robert Collins claims (The Segment Descriptor Cache, DDJ August + * 1998) that up to and including the Intel 486, far control + * transfers in real mode set default CS attributes (0x93) and also + * set a 64K segment limit. Starting with the Pentium, the + * attributes and limit are left alone but the access rights are + * ignored. We only implement the Pentium+ behavior. + * */ + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + Assert(enmEffOpSize == IEMMODE_16BIT || enmEffOpSize == IEMMODE_32BIT); + if (offSeg > pVCpu->cpum.GstCtx.cs.u32Limit) + { + Log(("iemCImpl_FarJmp: 16-bit limit\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if (enmEffOpSize == IEMMODE_16BIT) /** @todo WRONG, must pass this. */ + pVCpu->cpum.GstCtx.rip = offSeg; + else + pVCpu->cpum.GstCtx.rip = offSeg & UINT16_MAX; + pVCpu->cpum.GstCtx.cs.Sel = uSel; + pVCpu->cpum.GstCtx.cs.ValidSel = uSel; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uSel << 4; + + return iemRegFinishClearingRF(pVCpu); + } + + /* + * Protected mode. Need to parse the specified descriptor... + */ + if (!(uSel & X86_SEL_MASK_OFF_RPL)) + { + Log(("jmpf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Fetch the descriptor. */ + IEMSELDESC Desc; + VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Is it there? */ + if (!Desc.Legacy.Gen.u1Present) /** @todo this is probably checked too early. Testcase! */ + { + Log(("jmpf %04x:%08RX64 -> segment not present\n", uSel, offSeg)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel); + } + + /* + * Deal with it according to its type. We do the standard code selectors + * here and dispatch the system selectors to worker functions. + */ + if (!Desc.Legacy.Gen.u1DescType) + return iemCImpl_BranchSysSel(pVCpu, cbInstr, uSel, IEMBRANCH_JUMP, enmEffOpSize, &Desc); + + /* Only code segments. */ + if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)) + { + Log(("jmpf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + + /* L vs D. */ + if ( Desc.Legacy.Gen.u1Long + && Desc.Legacy.Gen.u1DefBig + && IEM_IS_LONG_MODE(pVCpu)) + { + Log(("jmpf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + + /* DPL/RPL/CPL check, where conforming segments makes a difference. */ + if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) + { + if (pVCpu->iem.s.uCpl < Desc.Legacy.Gen.u2Dpl) + { + Log(("jmpf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n", + uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + } + else + { + if (pVCpu->iem.s.uCpl != Desc.Legacy.Gen.u2Dpl) + { + Log(("jmpf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + if ((uSel & X86_SEL_RPL) > pVCpu->iem.s.uCpl) + { + Log(("jmpf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + } + + /* Chop the high bits if 16-bit (Intel says so). */ + if (enmEffOpSize == IEMMODE_16BIT) + offSeg &= UINT16_MAX; + + /* Limit check and get the base. */ + uint64_t u64Base; + uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy); + if ( !Desc.Legacy.Gen.u1Long + || !IEM_IS_LONG_MODE(pVCpu)) + { + if (RT_LIKELY(offSeg <= cbLimit)) + u64Base = X86DESC_BASE(&Desc.Legacy); + else + { + Log(("jmpf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit)); + /** @todo Intel says this is \#GP(0)! */ + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + } + else + u64Base = 0; + + /* + * Ok, everything checked out fine. Now set the accessed bit before + * committing the result into CS, CSHID and RIP. + */ + if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + /** @todo check what VT-x and AMD-V does. */ + Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* commit */ + pVCpu->cpum.GstCtx.rip = offSeg; + pVCpu->cpum.GstCtx.cs.Sel = uSel & X86_SEL_MASK_OFF_RPL; + pVCpu->cpum.GstCtx.cs.Sel |= pVCpu->iem.s.uCpl; /** @todo is this right for conforming segs? or in general? */ + pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit; + pVCpu->cpum.GstCtx.cs.u64Base = u64Base; + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + /** @todo check if the hidden bits are loaded correctly for 64-bit + * mode. */ + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); + + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Implements far calls. + * + * This very similar to iemCImpl_FarJmp. + * + * @param uSel The selector. + * @param offSeg The segment offset. + * @param enmEffOpSize The operand size (in case we need it). + */ +IEM_CIMPL_DEF_3(iemCImpl_callf, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize) +{ + VBOXSTRICTRC rcStrict; + uint64_t uNewRsp; + RTPTRUNION uPtrRet; + + /* + * Real mode and V8086 mode are easy. The only snag seems to be that + * CS.limit doesn't change and the limit check is done against the current + * limit. + */ + /** @todo See comment for similar code in iemCImpl_FarJmp */ + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + Assert(enmEffOpSize == IEMMODE_16BIT || enmEffOpSize == IEMMODE_32BIT); + + /* Check stack first - may #SS(0). */ + rcStrict = iemMemStackPushBeginSpecial(pVCpu, enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2, + enmEffOpSize == IEMMODE_32BIT ? 3 : 1, + &uPtrRet.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Check the target address range. */ +/** @todo this must be wrong! Write unreal mode tests! */ + if (offSeg > UINT32_MAX) + return iemRaiseGeneralProtectionFault0(pVCpu); + + /* Everything is fine, push the return address. */ + if (enmEffOpSize == IEMMODE_16BIT) + { + uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr; + uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel; + } + else + { + uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr; + uPtrRet.pu16[2] = pVCpu->cpum.GstCtx.cs.Sel; + } + rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Branch. */ + pVCpu->cpum.GstCtx.rip = offSeg; + pVCpu->cpum.GstCtx.cs.Sel = uSel; + pVCpu->cpum.GstCtx.cs.ValidSel = uSel; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uSel << 4; + + return iemRegFinishClearingRF(pVCpu); + } + + /* + * Protected mode. Need to parse the specified descriptor... + */ + if (!(uSel & X86_SEL_MASK_OFF_RPL)) + { + Log(("callf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Fetch the descriptor. */ + IEMSELDESC Desc; + rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Deal with it according to its type. We do the standard code selectors + * here and dispatch the system selectors to worker functions. + */ + if (!Desc.Legacy.Gen.u1DescType) + return iemCImpl_BranchSysSel(pVCpu, cbInstr, uSel, IEMBRANCH_CALL, enmEffOpSize, &Desc); + + /* Only code segments. */ + if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)) + { + Log(("callf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + + /* L vs D. */ + if ( Desc.Legacy.Gen.u1Long + && Desc.Legacy.Gen.u1DefBig + && IEM_IS_LONG_MODE(pVCpu)) + { + Log(("callf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + + /* DPL/RPL/CPL check, where conforming segments makes a difference. */ + if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) + { + if (pVCpu->iem.s.uCpl < Desc.Legacy.Gen.u2Dpl) + { + Log(("callf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n", + uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + } + else + { + if (pVCpu->iem.s.uCpl != Desc.Legacy.Gen.u2Dpl) + { + Log(("callf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + if ((uSel & X86_SEL_RPL) > pVCpu->iem.s.uCpl) + { + Log(("callf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + } + + /* Is it there? */ + if (!Desc.Legacy.Gen.u1Present) + { + Log(("callf %04x:%08RX64 -> segment not present\n", uSel, offSeg)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel); + } + + /* Check stack first - may #SS(0). */ + /** @todo check how operand prefix affects pushing of CS! Does callf 16:32 in + * 16-bit code cause a two or four byte CS to be pushed? */ + rcStrict = iemMemStackPushBeginSpecial(pVCpu, + enmEffOpSize == IEMMODE_64BIT ? 8+8 : enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2, + enmEffOpSize == IEMMODE_64BIT ? 7 : enmEffOpSize == IEMMODE_32BIT ? 3 : 1, + &uPtrRet.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Chop the high bits if 16-bit (Intel says so). */ + if (enmEffOpSize == IEMMODE_16BIT) + offSeg &= UINT16_MAX; + + /* Limit / canonical check. */ + uint64_t u64Base; + uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy); + if ( !Desc.Legacy.Gen.u1Long + || !IEM_IS_LONG_MODE(pVCpu)) + { + if (RT_LIKELY(offSeg <= cbLimit)) + u64Base = X86DESC_BASE(&Desc.Legacy); + else + { + Log(("jmpf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit)); + /** @todo Intel says this is \#GP(0)! */ + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + } + else if (IEM_IS_CANONICAL(offSeg)) + u64Base = 0; + else + { + Log(("callf %04x:%016RX64 - not canonical -> #GP\n", uSel, offSeg)); + return iemRaiseNotCanonical(pVCpu); + } + + /* + * Now set the accessed bit before + * writing the return address to the stack and committing the result into + * CS, CSHID and RIP. + */ + /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */ + if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + /** @todo check what VT-x and AMD-V does. */ + Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* stack */ + if (enmEffOpSize == IEMMODE_16BIT) + { + uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr; + uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel; + } + else if (enmEffOpSize == IEMMODE_32BIT) + { + uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr; + uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when callf is pushing CS? */ + } + else + { + uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr; + uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when callf is pushing CS? */ + } + rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* commit */ + pVCpu->cpum.GstCtx.rip = offSeg; + pVCpu->cpum.GstCtx.cs.Sel = uSel & X86_SEL_MASK_OFF_RPL; + pVCpu->cpum.GstCtx.cs.Sel |= pVCpu->iem.s.uCpl; + pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit; + pVCpu->cpum.GstCtx.cs.u64Base = u64Base; + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + /** @todo check if the hidden bits are loaded correctly for 64-bit + * mode. */ + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); + + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Implements retf. + * + * @param enmEffOpSize The effective operand size. + * @param cbPop The amount of arguments to pop from the stack + * (bytes). + */ +IEM_CIMPL_DEF_2(iemCImpl_retf, IEMMODE, enmEffOpSize, uint16_t, cbPop) +{ + VBOXSTRICTRC rcStrict; + RTCPTRUNION uPtrFrame; + RTUINT64U NewRsp; + uint64_t uNewRip; + uint16_t uNewCs; + NOREF(cbInstr); + + /* + * Read the stack values first. + */ + uint32_t cbRetPtr = enmEffOpSize == IEMMODE_16BIT ? 2+2 + : enmEffOpSize == IEMMODE_32BIT ? 4+4 : 8+8; + rcStrict = iemMemStackPopBeginSpecial(pVCpu, cbRetPtr, + enmEffOpSize == IEMMODE_16BIT ? 1 : enmEffOpSize == IEMMODE_32BIT ? 3 : 7, + &uPtrFrame.pv, &NewRsp.u); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + if (enmEffOpSize == IEMMODE_16BIT) + { + uNewRip = uPtrFrame.pu16[0]; + uNewCs = uPtrFrame.pu16[1]; + } + else if (enmEffOpSize == IEMMODE_32BIT) + { + uNewRip = uPtrFrame.pu32[0]; + uNewCs = uPtrFrame.pu16[2]; + } + else + { + uNewRip = uPtrFrame.pu64[0]; + uNewCs = uPtrFrame.pu16[4]; + } + rcStrict = iemMemStackPopDoneSpecial(pVCpu, uPtrFrame.pv); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* extremely likely */ } + else + return rcStrict; + + /* + * Real mode and V8086 mode are easy. + */ + /** @todo See comment for similar code in iemCImpl_FarJmp */ + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT); + /** @todo check how this is supposed to work if sp=0xfffe. */ + + /* Check the limit of the new EIP. */ + /** @todo Intel pseudo code only does the limit check for 16-bit + * operands, AMD does not make any distinction. What is right? */ + if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit) + return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION); + + /* commit the operation. */ + if (cbPop) + iemRegAddToRspEx(pVCpu, &NewRsp, cbPop); + pVCpu->cpum.GstCtx.rsp = NewRsp.u; + pVCpu->cpum.GstCtx.rip = uNewRip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCs; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uNewCs << 4; + return iemRegFinishClearingRF(pVCpu); + } + + /* + * Protected mode is complicated, of course. + */ + if (!(uNewCs & X86_SEL_MASK_OFF_RPL)) + { + Log(("retf %04x:%08RX64 -> invalid selector, #GP(0)\n", uNewCs, uNewRip)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR); + + /* Fetch the descriptor. */ + IEMSELDESC DescCs; + rcStrict = iemMemFetchSelDesc(pVCpu, &DescCs, uNewCs, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Can only return to a code selector. */ + if ( !DescCs.Legacy.Gen.u1DescType + || !(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) ) + { + Log(("retf %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n", + uNewCs, uNewRip, DescCs.Legacy.Gen.u1DescType, DescCs.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + + /* L vs D. */ + if ( DescCs.Legacy.Gen.u1Long /** @todo Testcase: far return to a selector with both L and D set. */ + && DescCs.Legacy.Gen.u1DefBig + && IEM_IS_LONG_MODE(pVCpu)) + { + Log(("retf %04x:%08RX64 -> both L & D set.\n", uNewCs, uNewRip)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + + /* DPL/RPL/CPL checks. */ + if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl) + { + Log(("retf %04x:%08RX64 -> RPL < CPL(%d).\n", uNewCs, uNewRip, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + + if (DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) + { + if ((uNewCs & X86_SEL_RPL) < DescCs.Legacy.Gen.u2Dpl) + { + Log(("retf %04x:%08RX64 -> DPL violation (conforming); DPL=%u RPL=%u\n", + uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL))); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + } + else + { + if ((uNewCs & X86_SEL_RPL) != DescCs.Legacy.Gen.u2Dpl) + { + Log(("retf %04x:%08RX64 -> RPL != DPL; DPL=%u RPL=%u\n", + uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL))); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + } + + /* Is it there? */ + if (!DescCs.Legacy.Gen.u1Present) + { + Log(("retf %04x:%08RX64 -> segment not present\n", uNewCs, uNewRip)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs); + } + + /* + * Return to outer privilege? (We'll typically have entered via a call gate.) + */ + if ((uNewCs & X86_SEL_RPL) != pVCpu->iem.s.uCpl) + { + /* Read the outer stack pointer stored *after* the parameters. */ + rcStrict = iemMemStackPopContinueSpecial(pVCpu, cbPop /*off*/, cbRetPtr, &uPtrFrame.pv, NewRsp.u); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + uint16_t uNewOuterSs; + RTUINT64U NewOuterRsp; + if (enmEffOpSize == IEMMODE_16BIT) + { + NewOuterRsp.u = uPtrFrame.pu16[0]; + uNewOuterSs = uPtrFrame.pu16[1]; + } + else if (enmEffOpSize == IEMMODE_32BIT) + { + NewOuterRsp.u = uPtrFrame.pu32[0]; + uNewOuterSs = uPtrFrame.pu16[2]; + } + else + { + NewOuterRsp.u = uPtrFrame.pu64[0]; + uNewOuterSs = uPtrFrame.pu16[4]; + } + rcStrict = iemMemStackPopDoneSpecial(pVCpu, uPtrFrame.pv); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* extremely likely */ } + else + return rcStrict; + + /* Check for NULL stack selector (invalid in ring-3 and non-long mode) + and read the selector. */ + IEMSELDESC DescSs; + if (!(uNewOuterSs & X86_SEL_MASK_OFF_RPL)) + { + if ( !DescCs.Legacy.Gen.u1Long + || (uNewOuterSs & X86_SEL_RPL) == 3) + { + Log(("retf %04x:%08RX64 %04x:%08RX64 -> invalid stack selector, #GP\n", + uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + /** @todo Testcase: Return far to ring-1 or ring-2 with SS=0. */ + iemMemFakeStackSelDesc(&DescSs, (uNewOuterSs & X86_SEL_RPL)); + } + else + { + /* Fetch the descriptor for the new stack segment. */ + rcStrict = iemMemFetchSelDesc(pVCpu, &DescSs, uNewOuterSs, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + /* Check that RPL of stack and code selectors match. */ + if ((uNewCs & X86_SEL_RPL) != (uNewOuterSs & X86_SEL_RPL)) + { + Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.RPL != CS.RPL -> #GP(SS)\n", uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs); + } + + /* Must be a writable data segment. */ + if ( !DescSs.Legacy.Gen.u1DescType + || (DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) + || !(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) ) + { + Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not a writable data segment (u1DescType=%u u4Type=%#x) -> #GP(SS).\n", + uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u, DescSs.Legacy.Gen.u1DescType, DescSs.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs); + } + + /* L vs D. (Not mentioned by intel.) */ + if ( DescSs.Legacy.Gen.u1Long /** @todo Testcase: far return to a stack selector with both L and D set. */ + && DescSs.Legacy.Gen.u1DefBig + && IEM_IS_LONG_MODE(pVCpu)) + { + Log(("retf %04x:%08RX64 %04x:%08RX64 - SS has both L & D set -> #GP(SS).\n", + uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs); + } + + /* DPL/RPL/CPL checks. */ + if (DescSs.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL)) + { + Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.DPL(%u) != CS.RPL (%u) -> #GP(SS).\n", + uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u, DescSs.Legacy.Gen.u2Dpl, uNewCs & X86_SEL_RPL)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs); + } + + /* Is it there? */ + if (!DescSs.Legacy.Gen.u1Present) + { + Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not present -> #NP(SS).\n", uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs); + } + + /* Calc SS limit.*/ + uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSs.Legacy); + + /* Is RIP canonical or within CS.limit? */ + uint64_t u64Base; + uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy); + + /** @todo Testcase: Is this correct? */ + if ( DescCs.Legacy.Gen.u1Long + && IEM_IS_LONG_MODE(pVCpu) ) + { + if (!IEM_IS_CANONICAL(uNewRip)) + { + Log(("retf %04x:%08RX64 %04x:%08RX64 - not canonical -> #GP.\n", uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u)); + return iemRaiseNotCanonical(pVCpu); + } + u64Base = 0; + } + else + { + if (uNewRip > cbLimitCs) + { + Log(("retf %04x:%08RX64 %04x:%08RX64 - out of bounds (%#x)-> #GP(CS).\n", + uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u, cbLimitCs)); + /** @todo Intel says this is \#GP(0)! */ + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + u64Base = X86DESC_BASE(&DescCs.Legacy); + } + + /* + * Now set the accessed bit before + * writing the return address to the stack and committing the result into + * CS, CSHID and RIP. + */ + /** @todo Testcase: Need to check WHEN exactly the CS accessed bit is set. */ + if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + /** @todo check what VT-x and AMD-V does. */ + DescCs.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + /** @todo Testcase: Need to check WHEN exactly the SS accessed bit is set. */ + if (!(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewOuterSs); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + /** @todo check what VT-x and AMD-V does. */ + DescSs.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* commit */ + if (enmEffOpSize == IEMMODE_16BIT) + pVCpu->cpum.GstCtx.rip = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */ + else + pVCpu->cpum.GstCtx.rip = uNewRip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCs; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCs.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCs; + pVCpu->cpum.GstCtx.cs.u64Base = u64Base; + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + pVCpu->cpum.GstCtx.ss.Sel = uNewOuterSs; + pVCpu->cpum.GstCtx.ss.ValidSel = uNewOuterSs; + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSs.Legacy); + pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSs; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pVCpu->cpum.GstCtx.ss.u64Base = 0; + else + pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSs.Legacy); + if (cbPop) + iemRegAddToRspEx(pVCpu, &NewOuterRsp, cbPop); + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pVCpu->cpum.GstCtx.rsp = NewOuterRsp.u; + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + pVCpu->cpum.GstCtx.rsp = (uint32_t)NewOuterRsp.u; + else + pVCpu->cpum.GstCtx.sp = (uint16_t)NewOuterRsp.u; + + pVCpu->iem.s.uCpl = (uNewCs & X86_SEL_RPL); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs); + + /** @todo check if the hidden bits are loaded correctly for 64-bit + * mode. */ + } + /* + * Return to the same privilege level + */ + else + { + /* Limit / canonical check. */ + uint64_t u64Base; + uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy); + + /** @todo Testcase: Is this correct? */ + if ( DescCs.Legacy.Gen.u1Long + && IEM_IS_LONG_MODE(pVCpu) ) + { + if (!IEM_IS_CANONICAL(uNewRip)) + { + Log(("retf %04x:%08RX64 - not canonical -> #GP\n", uNewCs, uNewRip)); + return iemRaiseNotCanonical(pVCpu); + } + u64Base = 0; + } + else + { + if (uNewRip > cbLimitCs) + { + Log(("retf %04x:%08RX64 -> out of bounds (%#x)\n", uNewCs, uNewRip, cbLimitCs)); + /** @todo Intel says this is \#GP(0)! */ + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + u64Base = X86DESC_BASE(&DescCs.Legacy); + } + + /* + * Now set the accessed bit before + * writing the return address to the stack and committing the result into + * CS, CSHID and RIP. + */ + /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */ + if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + /** @todo check what VT-x and AMD-V does. */ + DescCs.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* commit */ + if (cbPop) + iemRegAddToRspEx(pVCpu, &NewRsp, cbPop); + if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + pVCpu->cpum.GstCtx.sp = (uint16_t)NewRsp.u; + else + pVCpu->cpum.GstCtx.rsp = NewRsp.u; + if (enmEffOpSize == IEMMODE_16BIT) + pVCpu->cpum.GstCtx.rip = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */ + else + pVCpu->cpum.GstCtx.rip = uNewRip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCs; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCs.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCs; + pVCpu->cpum.GstCtx.cs.u64Base = u64Base; + /** @todo check if the hidden bits are loaded correctly for 64-bit + * mode. */ + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + } + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo use light flush for same privlege? */ + + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Implements retn and retn imm16. + * + * We're doing this in C because of the \#GP that might be raised if the popped + * program counter is out of bounds. + * + * The hope with this forced inline worker function, is that the compiler will + * be clever enough to eliminate unused code for the constant enmEffOpSize and + * maybe cbPop parameters. + * + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param cbInstr The current instruction length. + * @param enmEffOpSize The effective operand size. This is constant. + * @param cbPop The amount of arguments to pop from the stack + * (bytes). This can be constant (zero). + */ +DECL_FORCE_INLINE(VBOXSTRICTRC) iemCImpl_ReturnNearCommon(PVMCPUCC pVCpu, uint8_t cbInstr, IEMMODE enmEffOpSize, uint16_t cbPop) +{ + /* Fetch the RSP from the stack. */ + VBOXSTRICTRC rcStrict; + RTUINT64U NewRip; + RTUINT64U NewRsp; + NewRsp.u = pVCpu->cpum.GstCtx.rsp; + + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + NewRip.u = 0; + rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRip.Words.w0, &NewRsp); + break; + case IEMMODE_32BIT: + NewRip.u = 0; + rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRip.DWords.dw0, &NewRsp); + break; + case IEMMODE_64BIT: + rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRip.u, &NewRsp); + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Check the new RSP before loading it. */ + /** @todo Should test this as the intel+amd pseudo code doesn't mention half + * of it. The canonical test is performed here and for call. */ + if (enmEffOpSize != IEMMODE_64BIT) + { + if (RT_LIKELY(NewRip.DWords.dw0 <= pVCpu->cpum.GstCtx.cs.u32Limit)) + { /* likely */ } + else + { + Log(("retn newrip=%llx - out of bounds (%x) -> #GP\n", NewRip.u, pVCpu->cpum.GstCtx.cs.u32Limit)); + return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION); + } + } + else + { + if (RT_LIKELY(IEM_IS_CANONICAL(NewRip.u))) + { /* likely */ } + else + { + Log(("retn newrip=%llx - not canonical -> #GP\n", NewRip.u)); + return iemRaiseNotCanonical(pVCpu); + } + } + + /* Apply cbPop */ + if (cbPop) + iemRegAddToRspEx(pVCpu, &NewRsp, cbPop); + + /* Commit it. */ + pVCpu->cpum.GstCtx.rip = NewRip.u; + pVCpu->cpum.GstCtx.rsp = NewRsp.u; + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo only need a light flush here, don't we? We don't really need any flushing... */ + RT_NOREF(cbInstr); + + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Implements retn imm16 with 16-bit effective operand size. + * + * @param cbPop The amount of arguments to pop from the stack (bytes). + */ +IEM_CIMPL_DEF_1(iemCImpl_retn_iw_16, uint16_t, cbPop) +{ + return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_16BIT, cbPop); +} + + +/** + * Implements retn imm16 with 32-bit effective operand size. + * + * @param cbPop The amount of arguments to pop from the stack (bytes). + */ +IEM_CIMPL_DEF_1(iemCImpl_retn_iw_32, uint16_t, cbPop) +{ + return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_32BIT, cbPop); +} + + +/** + * Implements retn imm16 with 64-bit effective operand size. + * + * @param cbPop The amount of arguments to pop from the stack (bytes). + */ +IEM_CIMPL_DEF_1(iemCImpl_retn_iw_64, uint16_t, cbPop) +{ + return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_64BIT, cbPop); +} + + +/** + * Implements retn with 16-bit effective operand size. + */ +IEM_CIMPL_DEF_0(iemCImpl_retn_16) +{ + return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_16BIT, 0); +} + + +/** + * Implements retn with 32-bit effective operand size. + */ +IEM_CIMPL_DEF_0(iemCImpl_retn_32) +{ + return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_32BIT, 0); +} + + +/** + * Implements retn with 64-bit effective operand size. + */ +IEM_CIMPL_DEF_0(iemCImpl_retn_64) +{ + return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_64BIT, 0); +} + + +/** + * Implements enter. + * + * We're doing this in C because the instruction is insane, even for the + * u8NestingLevel=0 case dealing with the stack is tedious. + * + * @param enmEffOpSize The effective operand size. + * @param cbFrame Frame size. + * @param cParameters Frame parameter count. + */ +IEM_CIMPL_DEF_3(iemCImpl_enter, IEMMODE, enmEffOpSize, uint16_t, cbFrame, uint8_t, cParameters) +{ + /* Push RBP, saving the old value in TmpRbp. */ + RTUINT64U NewRsp; NewRsp.u = pVCpu->cpum.GstCtx.rsp; + RTUINT64U TmpRbp; TmpRbp.u = pVCpu->cpum.GstCtx.rbp; + RTUINT64U NewRbp; + VBOXSTRICTRC rcStrict; + if (enmEffOpSize == IEMMODE_64BIT) + { + rcStrict = iemMemStackPushU64Ex(pVCpu, TmpRbp.u, &NewRsp); + NewRbp = NewRsp; + } + else if (enmEffOpSize == IEMMODE_32BIT) + { + rcStrict = iemMemStackPushU32Ex(pVCpu, TmpRbp.DWords.dw0, &NewRsp); + NewRbp = NewRsp; + } + else + { + rcStrict = iemMemStackPushU16Ex(pVCpu, TmpRbp.Words.w0, &NewRsp); + NewRbp = TmpRbp; + NewRbp.Words.w0 = NewRsp.Words.w0; + } + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Copy the parameters (aka nesting levels by Intel). */ + cParameters &= 0x1f; + if (cParameters > 0) + { + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + TmpRbp.DWords.dw0 -= 2; + else + TmpRbp.Words.w0 -= 2; + do + { + uint16_t u16Tmp; + rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Tmp, &TmpRbp); + if (rcStrict != VINF_SUCCESS) + break; + rcStrict = iemMemStackPushU16Ex(pVCpu, u16Tmp, &NewRsp); + } while (--cParameters > 0 && rcStrict == VINF_SUCCESS); + break; + + case IEMMODE_32BIT: + if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + TmpRbp.DWords.dw0 -= 4; + else + TmpRbp.Words.w0 -= 4; + do + { + uint32_t u32Tmp; + rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Tmp, &TmpRbp); + if (rcStrict != VINF_SUCCESS) + break; + rcStrict = iemMemStackPushU32Ex(pVCpu, u32Tmp, &NewRsp); + } while (--cParameters > 0 && rcStrict == VINF_SUCCESS); + break; + + case IEMMODE_64BIT: + TmpRbp.u -= 8; + do + { + uint64_t u64Tmp; + rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Tmp, &TmpRbp); + if (rcStrict != VINF_SUCCESS) + break; + rcStrict = iemMemStackPushU64Ex(pVCpu, u64Tmp, &NewRsp); + } while (--cParameters > 0 && rcStrict == VINF_SUCCESS); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + if (rcStrict != VINF_SUCCESS) + return VINF_SUCCESS; + + /* Push the new RBP */ + if (enmEffOpSize == IEMMODE_64BIT) + rcStrict = iemMemStackPushU64Ex(pVCpu, NewRbp.u, &NewRsp); + else if (enmEffOpSize == IEMMODE_32BIT) + rcStrict = iemMemStackPushU32Ex(pVCpu, NewRbp.DWords.dw0, &NewRsp); + else + rcStrict = iemMemStackPushU16Ex(pVCpu, NewRbp.Words.w0, &NewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + } + + /* Recalc RSP. */ + iemRegSubFromRspEx(pVCpu, &NewRsp, cbFrame); + + /** @todo Should probe write access at the new RSP according to AMD. */ + /** @todo Should handle accesses to the VMX APIC-access page. */ + + /* Commit it. */ + pVCpu->cpum.GstCtx.rbp = NewRbp.u; + pVCpu->cpum.GstCtx.rsp = NewRsp.u; + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + + +/** + * Implements leave. + * + * We're doing this in C because messing with the stack registers is annoying + * since they depends on SS attributes. + * + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_1(iemCImpl_leave, IEMMODE, enmEffOpSize) +{ + /* Calculate the intermediate RSP from RBP and the stack attributes. */ + RTUINT64U NewRsp; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + NewRsp.u = pVCpu->cpum.GstCtx.rbp; + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + NewRsp.u = pVCpu->cpum.GstCtx.ebp; + else + { + /** @todo Check that LEAVE actually preserve the high EBP bits. */ + NewRsp.u = pVCpu->cpum.GstCtx.rsp; + NewRsp.Words.w0 = pVCpu->cpum.GstCtx.bp; + } + + /* Pop RBP according to the operand size. */ + VBOXSTRICTRC rcStrict; + RTUINT64U NewRbp; + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + NewRbp.u = pVCpu->cpum.GstCtx.rbp; + rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRbp.Words.w0, &NewRsp); + break; + case IEMMODE_32BIT: + NewRbp.u = 0; + rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRbp.DWords.dw0, &NewRsp); + break; + case IEMMODE_64BIT: + rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRbp.u, &NewRsp); + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + + /* Commit it. */ + pVCpu->cpum.GstCtx.rbp = NewRbp.u; + pVCpu->cpum.GstCtx.rsp = NewRsp.u; + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements int3 and int XX. + * + * @param u8Int The interrupt vector number. + * @param enmInt The int instruction type. + */ +IEM_CIMPL_DEF_2(iemCImpl_int, uint8_t, u8Int, IEMINT, enmInt) +{ + Assert(pVCpu->iem.s.cXcptRecursions == 0); + + /* + * We must check if this INT3 might belong to DBGF before raising a #BP. + */ + if (u8Int == 3) + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if (pVM->dbgf.ro.cEnabledInt3Breakpoints == 0) + { /* likely: No vbox debugger breakpoints */ } + else + { + VBOXSTRICTRC rcStrict = DBGFTrap03Handler(pVM, pVCpu, &pVCpu->cpum.GstCtx); + Log(("iemCImpl_int: DBGFTrap03Handler -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) )); + if (rcStrict != VINF_EM_RAW_GUEST_TRAP) + return iemSetPassUpStatus(pVCpu, rcStrict); + } + } +/** @todo single stepping */ + return iemRaiseXcptOrInt(pVCpu, + cbInstr, + u8Int, + IEM_XCPT_FLAGS_T_SOFT_INT | enmInt, + 0, + 0); +} + + +/** + * Implements iret for real mode and V8086 mode. + * + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_1(iemCImpl_iret_real_v8086, IEMMODE, enmEffOpSize) +{ + X86EFLAGS Efl; + Efl.u = IEMMISC_GET_EFL(pVCpu); + NOREF(cbInstr); + + /* + * iret throws an exception if VME isn't enabled. + */ + if ( Efl.Bits.u1VM + && Efl.Bits.u2IOPL != 3 + && !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME)) + return iemRaiseGeneralProtectionFault0(pVCpu); + + /* + * Do the stack bits, but don't commit RSP before everything checks + * out right. + */ + Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT); + VBOXSTRICTRC rcStrict; + RTCPTRUNION uFrame; + uint16_t uNewCs; + uint32_t uNewEip; + uint32_t uNewFlags; + uint64_t uNewRsp; + if (enmEffOpSize == IEMMODE_32BIT) + { + rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, 1, &uFrame.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + uNewEip = uFrame.pu32[0]; + if (uNewEip > UINT16_MAX) + return iemRaiseGeneralProtectionFault0(pVCpu); + + uNewCs = (uint16_t)uFrame.pu32[1]; + uNewFlags = uFrame.pu32[2]; + uNewFlags &= X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF + | X86_EFL_TF | X86_EFL_IF | X86_EFL_DF | X86_EFL_OF | X86_EFL_IOPL | X86_EFL_NT + | X86_EFL_RF /*| X86_EFL_VM*/ | X86_EFL_AC /*|X86_EFL_VIF*/ /*|X86_EFL_VIP*/ + | X86_EFL_ID; + if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386) + uNewFlags &= ~(X86_EFL_AC | X86_EFL_ID | X86_EFL_VIF | X86_EFL_VIP); + uNewFlags |= Efl.u & (X86_EFL_VM | X86_EFL_VIF | X86_EFL_VIP | X86_EFL_1); + } + else + { + rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, 1, &uFrame.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + uNewEip = uFrame.pu16[0]; + uNewCs = uFrame.pu16[1]; + uNewFlags = uFrame.pu16[2]; + uNewFlags &= X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF + | X86_EFL_TF | X86_EFL_IF | X86_EFL_DF | X86_EFL_OF | X86_EFL_IOPL | X86_EFL_NT; + uNewFlags |= Efl.u & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF); + /** @todo The intel pseudo code does not indicate what happens to + * reserved flags. We just ignore them. */ + /* Ancient CPU adjustments: See iemCImpl_popf. */ + if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286) + uNewFlags &= ~(X86_EFL_NT | X86_EFL_IOPL); + } + rcStrict = iemMemStackPopDoneSpecial(pVCpu, uFrame.pv); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* extremely likely */ } + else + return rcStrict; + + /** @todo Check how this is supposed to work if sp=0xfffe. */ + Log7(("iemCImpl_iret_real_v8086: uNewCs=%#06x uNewRip=%#010x uNewFlags=%#x uNewRsp=%#18llx\n", + uNewCs, uNewEip, uNewFlags, uNewRsp)); + + /* + * Check the limit of the new EIP. + */ + /** @todo Only the AMD pseudo code check the limit here, what's + * right? */ + if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit) + return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION); + + /* + * V8086 checks and flag adjustments + */ + if (Efl.Bits.u1VM) + { + if (Efl.Bits.u2IOPL == 3) + { + /* Preserve IOPL and clear RF. */ + uNewFlags &= ~(X86_EFL_IOPL | X86_EFL_RF); + uNewFlags |= Efl.u & (X86_EFL_IOPL); + } + else if ( enmEffOpSize == IEMMODE_16BIT + && ( !(uNewFlags & X86_EFL_IF) + || !Efl.Bits.u1VIP ) + && !(uNewFlags & X86_EFL_TF) ) + { + /* Move IF to VIF, clear RF and preserve IF and IOPL.*/ + uNewFlags &= ~X86_EFL_VIF; + uNewFlags |= (uNewFlags & X86_EFL_IF) << (19 - 9); + uNewFlags &= ~(X86_EFL_IF | X86_EFL_IOPL | X86_EFL_RF); + uNewFlags |= Efl.u & (X86_EFL_IF | X86_EFL_IOPL); + } + else + return iemRaiseGeneralProtectionFault0(pVCpu); + Log7(("iemCImpl_iret_real_v8086: u1VM=1: adjusted uNewFlags=%#x\n", uNewFlags)); + } + + /* + * Commit the operation. + */ +#ifdef DBGFTRACE_ENABLED + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/rm %04x:%04x -> %04x:%04x %x %04llx", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewRsp); +#endif + pVCpu->cpum.GstCtx.rsp = uNewRsp; + pVCpu->cpum.GstCtx.rip = uNewEip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCs; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)uNewCs << 4; + /** @todo do we load attribs and limit as well? */ + Assert(uNewFlags & X86_EFL_1); + IEMMISC_SET_EFL(pVCpu, uNewFlags); + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo can do light flush in real mode at least */ + +/** @todo single stepping */ + return VINF_SUCCESS; +} + + +/** + * Loads a segment register when entering V8086 mode. + * + * @param pSReg The segment register. + * @param uSeg The segment to load. + */ +static void iemCImplCommonV8086LoadSeg(PCPUMSELREG pSReg, uint16_t uSeg) +{ + pSReg->Sel = uSeg; + pSReg->ValidSel = uSeg; + pSReg->fFlags = CPUMSELREG_FLAGS_VALID; + pSReg->u64Base = (uint32_t)uSeg << 4; + pSReg->u32Limit = 0xffff; + pSReg->Attr.u = X86_SEL_TYPE_RW_ACC | RT_BIT(4) /*!sys*/ | RT_BIT(7) /*P*/ | (3 /*DPL*/ << 5); /* VT-x wants 0xf3 */ + /** @todo Testcase: Check if VT-x really needs this and what it does itself when + * IRET'ing to V8086. */ +} + + +/** + * Implements iret for protected mode returning to V8086 mode. + * + * @param uNewEip The new EIP. + * @param uNewCs The new CS. + * @param uNewFlags The new EFLAGS. + * @param uNewRsp The RSP after the initial IRET frame. + * + * @note This can only be a 32-bit iret du to the X86_EFL_VM position. + */ +IEM_CIMPL_DEF_4(iemCImpl_iret_prot_v8086, uint32_t, uNewEip, uint16_t, uNewCs, uint32_t, uNewFlags, uint64_t, uNewRsp) +{ + RT_NOREF_PV(cbInstr); + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK); + + /* + * Pop the V8086 specific frame bits off the stack. + */ + VBOXSTRICTRC rcStrict; + RTCPTRUNION uFrame; + rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0 /*off*/, 24 /*cbMem*/, &uFrame.pv, uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + uint32_t uNewEsp = uFrame.pu32[0]; + uint16_t uNewSs = uFrame.pu32[1]; + uint16_t uNewEs = uFrame.pu32[2]; + uint16_t uNewDs = uFrame.pu32[3]; + uint16_t uNewFs = uFrame.pu32[4]; + uint16_t uNewGs = uFrame.pu32[5]; + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uFrame.pv, IEM_ACCESS_STACK_R); /* don't use iemMemStackPopCommitSpecial here. */ + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Commit the operation. + */ + uNewFlags &= X86_EFL_LIVE_MASK; + uNewFlags |= X86_EFL_RA1_MASK; +#ifdef DBGFTRACE_ENABLED + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/p/v %04x:%08x -> %04x:%04x %x %04x:%04x", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp); +#endif + Log7(("iemCImpl_iret_prot_v8086: %04x:%08x -> %04x:%04x %x %04x:%04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp)); + + IEMMISC_SET_EFL(pVCpu, uNewFlags); + iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.cs, uNewCs); + iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ss, uNewSs); + iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.es, uNewEs); + iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ds, uNewDs); + iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.fs, uNewFs); + iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.gs, uNewGs); + pVCpu->cpum.GstCtx.rip = (uint16_t)uNewEip; + pVCpu->cpum.GstCtx.rsp = uNewEsp; /** @todo check this out! */ + pVCpu->iem.s.uCpl = 3; + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); + +/** @todo single stepping */ + return VINF_SUCCESS; +} + + +/** + * Implements iret for protected mode returning via a nested task. + * + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_1(iemCImpl_iret_prot_NestedTask, IEMMODE, enmEffOpSize) +{ + Log7(("iemCImpl_iret_prot_NestedTask:\n")); +#ifndef IEM_IMPLEMENTS_TASKSWITCH + IEM_RETURN_ASPECT_NOT_IMPLEMENTED(); +#else + RT_NOREF_PV(enmEffOpSize); + + /* + * Read the segment selector in the link-field of the current TSS. + */ + RTSEL uSelRet; + VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &uSelRet, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Fetch the returning task's TSS descriptor from the GDT. + */ + if (uSelRet & X86_SEL_LDT) + { + Log(("iret_prot_NestedTask TSS not in LDT. uSelRet=%04x -> #TS\n", uSelRet)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet); + } + + IEMSELDESC TssDesc; + rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelRet, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + if (TssDesc.Legacy.Gate.u1DescType) + { + Log(("iret_prot_NestedTask Invalid TSS type. uSelRet=%04x -> #TS\n", uSelRet)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL); + } + + if ( TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_286_TSS_BUSY + && TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY) + { + Log(("iret_prot_NestedTask TSS is not busy. uSelRet=%04x DescType=%#x -> #TS\n", uSelRet, TssDesc.Legacy.Gate.u4Type)); + return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL); + } + + if (!TssDesc.Legacy.Gate.u1Present) + { + Log(("iret_prot_NestedTask TSS is not present. uSelRet=%04x -> #NP\n", uSelRet)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL); + } + + uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr; + return iemTaskSwitch(pVCpu, IEMTASKSWITCH_IRET, uNextEip, 0 /* fFlags */, 0 /* uErr */, + 0 /* uCr2 */, uSelRet, &TssDesc); +#endif +} + + +/** + * Implements iret for protected mode + * + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_1(iemCImpl_iret_prot, IEMMODE, enmEffOpSize) +{ + NOREF(cbInstr); + Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT); + + /* + * Nested task return. + */ + if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT) + return IEM_CIMPL_CALL_1(iemCImpl_iret_prot_NestedTask, enmEffOpSize); + + /* + * Normal return. + * + * Do the stack bits, but don't commit RSP before everything checks + * out right. + */ + Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT); + VBOXSTRICTRC rcStrict; + RTCPTRUNION uFrame; + uint16_t uNewCs; + uint32_t uNewEip; + uint32_t uNewFlags; + uint64_t uNewRsp; + if (enmEffOpSize == IEMMODE_32BIT) + { + rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, 3, &uFrame.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + uNewEip = uFrame.pu32[0]; + uNewCs = (uint16_t)uFrame.pu32[1]; + uNewFlags = uFrame.pu32[2]; + } + else + { + rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, 1, &uFrame.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + uNewEip = uFrame.pu16[0]; + uNewCs = uFrame.pu16[1]; + uNewFlags = uFrame.pu16[2]; + } + rcStrict = iemMemStackPopDoneSpecial(pVCpu, (void *)uFrame.pv); /* don't use iemMemStackPopCommitSpecial here. */ + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* extremely likely */ } + else + return rcStrict; + Log7(("iemCImpl_iret_prot: uNewCs=%#06x uNewEip=%#010x uNewFlags=%#x uNewRsp=%#18llx uCpl=%u\n", uNewCs, uNewEip, uNewFlags, uNewRsp, pVCpu->iem.s.uCpl)); + + /* + * We're hopefully not returning to V8086 mode... + */ + if ( (uNewFlags & X86_EFL_VM) + && pVCpu->iem.s.uCpl == 0) + { + Assert(enmEffOpSize == IEMMODE_32BIT); + return IEM_CIMPL_CALL_4(iemCImpl_iret_prot_v8086, uNewEip, uNewCs, uNewFlags, uNewRsp); + } + + /* + * Protected mode. + */ + /* Read the CS descriptor. */ + if (!(uNewCs & X86_SEL_MASK_OFF_RPL)) + { + Log(("iret %04x:%08x -> invalid CS selector, #GP(0)\n", uNewCs, uNewEip)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + IEMSELDESC DescCS; + rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + { + Log(("iret %04x:%08x - rcStrict=%Rrc when fetching CS\n", uNewCs, uNewEip, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Must be a code descriptor. */ + if (!DescCS.Legacy.Gen.u1DescType) + { + Log(("iret %04x:%08x - CS is system segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)) + { + Log(("iret %04x:%08x - not code segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + + /* Privilege checks. */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)) + { + if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl) + { + Log(("iret %04x:%08x - RPL != DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + } + else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl) + { + Log(("iret %04x:%08x - RPL < DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl) + { + Log(("iret %04x:%08x - RPL < CPL (%d) -> #GP\n", uNewCs, uNewEip, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + + /* Present? */ + if (!DescCS.Legacy.Gen.u1Present) + { + Log(("iret %04x:%08x - CS not present -> #NP\n", uNewCs, uNewEip)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs); + } + + uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy); + + /* + * Return to outer level? + */ + if ((uNewCs & X86_SEL_RPL) != pVCpu->iem.s.uCpl) + { + uint16_t uNewSS; + uint32_t uNewESP; + if (enmEffOpSize == IEMMODE_32BIT) + { + rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0/*off*/, 8 /*cbMem*/, &uFrame.pv, uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; +/** @todo We might be popping a 32-bit ESP from the IRET frame, but whether + * 16-bit or 32-bit are being loaded into SP depends on the D/B + * bit of the popped SS selector it turns out. */ + uNewESP = uFrame.pu32[0]; + uNewSS = (uint16_t)uFrame.pu32[1]; + } + else + { + rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0 /*off*/, 4 /*cbMem*/, &uFrame.pv, uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + uNewESP = uFrame.pu16[0]; + uNewSS = uFrame.pu16[1]; + } + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uFrame.pv, IEM_ACCESS_STACK_R); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + Log7(("iemCImpl_iret_prot: uNewSS=%#06x uNewESP=%#010x\n", uNewSS, uNewESP)); + + /* Read the SS descriptor. */ + if (!(uNewSS & X86_SEL_MASK_OFF_RPL)) + { + Log(("iret %04x:%08x/%04x:%08x -> invalid SS selector, #GP(0)\n", uNewCs, uNewEip, uNewSS, uNewESP)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + IEMSELDESC DescSS; + rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_GP); /** @todo Correct exception? */ + if (rcStrict != VINF_SUCCESS) + { + Log(("iret %04x:%08x/%04x:%08x - %Rrc when fetching SS\n", + uNewCs, uNewEip, uNewSS, uNewESP, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Privilege checks. */ + if ((uNewSS & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL)) + { + Log(("iret %04x:%08x/%04x:%08x -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewEip, uNewSS, uNewESP)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS); + } + if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL)) + { + Log(("iret %04x:%08x/%04x:%08x -> SS.DPL (%d) != CS.RPL -> #GP\n", + uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u2Dpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS); + } + + /* Must be a writeable data segment descriptor. */ + if (!DescSS.Legacy.Gen.u1DescType) + { + Log(("iret %04x:%08x/%04x:%08x -> SS is system segment (%#x) -> #GP\n", + uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS); + } + if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE) + { + Log(("iret %04x:%08x/%04x:%08x - not writable data segment (%#x) -> #GP\n", + uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS); + } + + /* Present? */ + if (!DescSS.Legacy.Gen.u1Present) + { + Log(("iret %04x:%08x/%04x:%08x -> SS not present -> #SS\n", uNewCs, uNewEip, uNewSS, uNewESP)); + return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS); + } + + uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy); + + /* Check EIP. */ + if (uNewEip > cbLimitCS) + { + Log(("iret %04x:%08x/%04x:%08x -> EIP is out of bounds (%#x) -> #GP(0)\n", + uNewCs, uNewEip, uNewSS, uNewESP, cbLimitCS)); + /** @todo Which is it, \#GP(0) or \#GP(sel)? */ + return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs); + } + + /* + * Commit the changes, marking CS and SS accessed first since + * that may fail. + */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + uint32_t fEFlagsMask = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF + | X86_EFL_TF | X86_EFL_DF | X86_EFL_OF | X86_EFL_NT; + if (enmEffOpSize != IEMMODE_16BIT) + fEFlagsMask |= X86_EFL_RF | X86_EFL_AC | X86_EFL_ID; + if (pVCpu->iem.s.uCpl == 0) + fEFlagsMask |= X86_EFL_IF | X86_EFL_IOPL | X86_EFL_VIF | X86_EFL_VIP; /* VM is 0 */ + else if (pVCpu->iem.s.uCpl <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL) + fEFlagsMask |= X86_EFL_IF; + if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386) + fEFlagsMask &= ~(X86_EFL_AC | X86_EFL_ID | X86_EFL_VIF | X86_EFL_VIP); + uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu); + fEFlagsNew &= ~fEFlagsMask; + fEFlagsNew |= uNewFlags & fEFlagsMask; +#ifdef DBGFTRACE_ENABLED + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up%u %04x:%08x -> %04x:%04x %x %04x:%04x", + pVCpu->iem.s.uCpl, uNewCs & X86_SEL_RPL, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, + uNewCs, uNewEip, uNewFlags, uNewSS, uNewESP); +#endif + + IEMMISC_SET_EFL(pVCpu, fEFlagsNew); + pVCpu->cpum.GstCtx.rip = uNewEip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCs; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS; + pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy); + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + + pVCpu->cpum.GstCtx.ss.Sel = uNewSS; + pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS; + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy); + pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSs; + pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy); + if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + pVCpu->cpum.GstCtx.sp = (uint16_t)uNewESP; + else + pVCpu->cpum.GstCtx.rsp = uNewESP; + + pVCpu->iem.s.uCpl = uNewCs & X86_SEL_RPL; + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs); + + /* Done! */ + + } + /* + * Return to the same level. + */ + else + { + /* Check EIP. */ + if (uNewEip > cbLimitCS) + { + Log(("iret %04x:%08x - EIP is out of bounds (%#x) -> #GP(0)\n", uNewCs, uNewEip, cbLimitCS)); + /** @todo Which is it, \#GP(0) or \#GP(sel)? */ + return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs); + } + + /* + * Commit the changes, marking CS first since it may fail. + */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + X86EFLAGS NewEfl; + NewEfl.u = IEMMISC_GET_EFL(pVCpu); + uint32_t fEFlagsMask = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF + | X86_EFL_TF | X86_EFL_DF | X86_EFL_OF | X86_EFL_NT; + if (enmEffOpSize != IEMMODE_16BIT) + fEFlagsMask |= X86_EFL_RF | X86_EFL_AC | X86_EFL_ID; + if (pVCpu->iem.s.uCpl == 0) + fEFlagsMask |= X86_EFL_IF | X86_EFL_IOPL | X86_EFL_VIF | X86_EFL_VIP; /* VM is 0 */ + else if (pVCpu->iem.s.uCpl <= NewEfl.Bits.u2IOPL) + fEFlagsMask |= X86_EFL_IF; + if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386) + fEFlagsMask &= ~(X86_EFL_AC | X86_EFL_ID | X86_EFL_VIF | X86_EFL_VIP); + NewEfl.u &= ~fEFlagsMask; + NewEfl.u |= fEFlagsMask & uNewFlags; +#ifdef DBGFTRACE_ENABLED + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up %04x:%08x -> %04x:%04x %x %04x:%04llx", + pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, + uNewCs, uNewEip, uNewFlags, pVCpu->cpum.GstCtx.ss.Sel, uNewRsp); +#endif + + IEMMISC_SET_EFL(pVCpu, NewEfl.u); + pVCpu->cpum.GstCtx.rip = uNewEip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCs; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS; + pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy); + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + pVCpu->cpum.GstCtx.sp = (uint16_t)uNewRsp; + else + pVCpu->cpum.GstCtx.rsp = uNewRsp; + /* Done! */ + } + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo may light flush if same ring? */ + +/** @todo single stepping */ + return VINF_SUCCESS; +} + + +/** + * Implements iret for long mode + * + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_1(iemCImpl_iret_64bit, IEMMODE, enmEffOpSize) +{ + NOREF(cbInstr); + + /* + * Nested task return is not supported in long mode. + */ + if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT) + { + Log(("iretq with NT=1 (eflags=%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.eflags.u)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Normal return. + * + * Do the stack bits, but don't commit RSP before everything checks + * out right. + */ + VBOXSTRICTRC rcStrict; + RTCPTRUNION uFrame; + uint64_t uNewRip; + uint16_t uNewCs; + uint16_t uNewSs; + uint32_t uNewFlags; + uint64_t uNewRsp; + if (enmEffOpSize == IEMMODE_64BIT) + { + rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*8, 7, &uFrame.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + uNewRip = uFrame.pu64[0]; + uNewCs = (uint16_t)uFrame.pu64[1]; + uNewFlags = (uint32_t)uFrame.pu64[2]; + uNewRsp = uFrame.pu64[3]; + uNewSs = (uint16_t)uFrame.pu64[4]; + } + else if (enmEffOpSize == IEMMODE_32BIT) + { + rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*4, 3, &uFrame.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + uNewRip = uFrame.pu32[0]; + uNewCs = (uint16_t)uFrame.pu32[1]; + uNewFlags = uFrame.pu32[2]; + uNewRsp = uFrame.pu32[3]; + uNewSs = (uint16_t)uFrame.pu32[4]; + } + else + { + Assert(enmEffOpSize == IEMMODE_16BIT); + rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*2, 1, &uFrame.pv, &uNewRsp); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + uNewRip = uFrame.pu16[0]; + uNewCs = uFrame.pu16[1]; + uNewFlags = uFrame.pu16[2]; + uNewRsp = uFrame.pu16[3]; + uNewSs = uFrame.pu16[4]; + } + rcStrict = iemMemStackPopDoneSpecial(pVCpu, (void *)uFrame.pv); /* don't use iemMemStackPopCommitSpecial here. */ + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* extremely like */ } + else + return rcStrict; + Log7(("iretq stack: cs:rip=%04x:%016RX64 rflags=%016RX64 ss:rsp=%04x:%016RX64\n", uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp)); + + /* + * Check stuff. + */ + /* Read the CS descriptor. */ + if (!(uNewCs & X86_SEL_MASK_OFF_RPL)) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 -> invalid CS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + IEMSELDESC DescCS; + rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP); + if (rcStrict != VINF_SUCCESS) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 - rcStrict=%Rrc when fetching CS\n", + uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Must be a code descriptor. */ + if ( !DescCS.Legacy.Gen.u1DescType + || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 - CS is not a code segment T=%u T=%#xu -> #GP\n", + uNewCs, uNewRip, uNewSs, uNewRsp, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + + /* Privilege checks. */ + uint8_t const uNewCpl = uNewCs & X86_SEL_RPL; + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)) + { + if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl) + { + Log(("iret %04x:%016RX64 - RPL != DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + } + else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl) + { + Log(("iret %04x:%016RX64 - RPL < DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl) + { + Log(("iret %04x:%016RX64 - RPL < CPL (%d) -> #GP\n", uNewCs, uNewRip, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs); + } + + /* Present? */ + if (!DescCS.Legacy.Gen.u1Present) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 - CS not present -> #NP\n", uNewCs, uNewRip, uNewSs, uNewRsp)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs); + } + + uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy); + + /* Read the SS descriptor. */ + IEMSELDESC DescSS; + if (!(uNewSs & X86_SEL_MASK_OFF_RPL)) + { + if ( !DescCS.Legacy.Gen.u1Long + || DescCS.Legacy.Gen.u1DefBig /** @todo exactly how does iret (and others) behave with u1Long=1 and u1DefBig=1? \#GP(sel)? */ + || uNewCpl > 2) /** @todo verify SS=0 impossible for ring-3. */ + { + Log(("iret %04x:%016RX64/%04x:%016RX64 -> invalid SS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + /* Make sure SS is sensible, marked as accessed etc. */ + iemMemFakeStackSelDesc(&DescSS, (uNewSs & X86_SEL_RPL)); + } + else + { + rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSs, X86_XCPT_GP); /** @todo Correct exception? */ + if (rcStrict != VINF_SUCCESS) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 - %Rrc when fetching SS\n", + uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } + + /* Privilege checks. */ + if ((uNewSs & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL)) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewRip, uNewSs, uNewRsp)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs); + } + + uint32_t cbLimitSs; + if (!(uNewSs & X86_SEL_MASK_OFF_RPL)) + cbLimitSs = UINT32_MAX; + else + { + if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL)) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS.DPL (%d) != CS.RPL -> #GP\n", + uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u2Dpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs); + } + + /* Must be a writeable data segment descriptor. */ + if (!DescSS.Legacy.Gen.u1DescType) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS is system segment (%#x) -> #GP\n", + uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs); + } + if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 - not writable data segment (%#x) -> #GP\n", + uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs); + } + + /* Present? */ + if (!DescSS.Legacy.Gen.u1Present) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS not present -> #SS\n", uNewCs, uNewRip, uNewSs, uNewRsp)); + return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSs); + } + cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy); + } + + /* Check EIP. */ + if (DescCS.Legacy.Gen.u1Long) + { + if (!IEM_IS_CANONICAL(uNewRip)) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 -> RIP is not canonical -> #GP(0)\n", + uNewCs, uNewRip, uNewSs, uNewRsp)); + return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs); + } + } + else + { + if (uNewRip > cbLimitCS) + { + Log(("iret %04x:%016RX64/%04x:%016RX64 -> EIP is out of bounds (%#x) -> #GP(0)\n", + uNewCs, uNewRip, uNewSs, uNewRsp, cbLimitCS)); + /** @todo Which is it, \#GP(0) or \#GP(sel)? */ + return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs); + } + } + + /* + * Commit the changes, marking CS and SS accessed first since + * that may fail. + */ + /** @todo where exactly are these actually marked accessed by a real CPU? */ + if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSs); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + uint32_t fEFlagsMask = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF + | X86_EFL_TF | X86_EFL_DF | X86_EFL_OF | X86_EFL_NT; + if (enmEffOpSize != IEMMODE_16BIT) + fEFlagsMask |= X86_EFL_RF | X86_EFL_AC | X86_EFL_ID; + if (pVCpu->iem.s.uCpl == 0) + fEFlagsMask |= X86_EFL_IF | X86_EFL_IOPL | X86_EFL_VIF | X86_EFL_VIP; /* VM is ignored */ + else if (pVCpu->iem.s.uCpl <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL) + fEFlagsMask |= X86_EFL_IF; + uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu); + fEFlagsNew &= ~fEFlagsMask; + fEFlagsNew |= uNewFlags & fEFlagsMask; +#ifdef DBGFTRACE_ENABLED + RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%ul%u %08llx -> %04x:%04llx %llx %04x:%04llx", + pVCpu->iem.s.uCpl, uNewCpl, pVCpu->cpum.GstCtx.rip, uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp); +#endif + + IEMMISC_SET_EFL(pVCpu, fEFlagsNew); + pVCpu->cpum.GstCtx.rip = uNewRip; + pVCpu->cpum.GstCtx.cs.Sel = uNewCs; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy); + pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS; + pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy); + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long || pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) + pVCpu->cpum.GstCtx.rsp = uNewRsp; + else + pVCpu->cpum.GstCtx.sp = (uint16_t)uNewRsp; + pVCpu->cpum.GstCtx.ss.Sel = uNewSs; + pVCpu->cpum.GstCtx.ss.ValidSel = uNewSs; + if (!(uNewSs & X86_SEL_MASK_OFF_RPL)) + { + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_UNUSABLE | (uNewCpl << X86DESCATTR_DPL_SHIFT); + pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX; + pVCpu->cpum.GstCtx.ss.u64Base = 0; + Log2(("iretq new SS: NULL\n")); + } + else + { + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy); + pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSs; + pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy); + Log2(("iretq new SS: base=%#RX64 lim=%#x attr=%#x\n", pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u)); + } + + if (pVCpu->iem.s.uCpl != uNewCpl) + { + pVCpu->iem.s.uCpl = uNewCpl; + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.ds); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.es); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.fs); + iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.gs); + } + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo may light flush if the ring + mode doesn't change */ + +/** @todo single stepping */ + return VINF_SUCCESS; +} + + +/** + * Implements iret. + * + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_1(iemCImpl_iret, IEMMODE, enmEffOpSize) +{ + bool fBlockingNmi = CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx); + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + /* + * Record whether NMI (or virtual-NMI) blocking is in effect during the execution + * of this IRET instruction. We need to provide this information as part of some + * VM-exits. + * + * See Intel spec. 27.2.2 "Information for VM Exits Due to Vectored Events". + */ + if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_VIRT_NMI)) + pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking; + else + pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = fBlockingNmi; + + /* + * If "NMI exiting" is set, IRET does not affect blocking of NMIs. + * See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation". + */ + if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_NMI_EXIT)) + fBlockingNmi = false; + + /* Clear virtual-NMI blocking, if any, before causing any further exceptions. */ + pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false; + } +#endif + + /* + * The SVM nested-guest intercept for IRET takes priority over all exceptions, + * The NMI is still held pending (which I assume means blocking of further NMIs + * is in effect). + * + * See AMD spec. 15.9 "Instruction Intercepts". + * See AMD spec. 15.21.9 "NMI Support". + */ + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IRET)) + { + Log(("iret: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IRET, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Clear NMI blocking, if any, before causing any further exceptions. + * See Intel spec. 6.7.1 "Handling Multiple NMIs". + */ + if (fBlockingNmi) + CPUMClearInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx); + + /* + * Call a mode specific worker. + */ + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + return IEM_CIMPL_CALL_1(iemCImpl_iret_real_v8086, enmEffOpSize); + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR); + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + return IEM_CIMPL_CALL_1(iemCImpl_iret_64bit, enmEffOpSize); + return IEM_CIMPL_CALL_1(iemCImpl_iret_prot, enmEffOpSize); +} + + +static void iemLoadallSetSelector(PVMCPUCC pVCpu, uint8_t iSegReg, uint16_t uSel) +{ + PCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iSegReg); + + pHid->Sel = uSel; + pHid->ValidSel = uSel; + pHid->fFlags = CPUMSELREG_FLAGS_VALID; +} + + +static void iemLoadall286SetDescCache(PVMCPUCC pVCpu, uint8_t iSegReg, uint8_t const *pbMem) +{ + PCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iSegReg); + + /* The base is in the first three bytes. */ + pHid->u64Base = pbMem[0] + (pbMem[1] << 8) + (pbMem[2] << 16); + /* The attributes are in the fourth byte. */ + pHid->Attr.u = pbMem[3]; + /* The limit is in the last two bytes. */ + pHid->u32Limit = pbMem[4] + (pbMem[5] << 8); +} + + +/** + * Implements 286 LOADALL (286 CPUs only). + */ +IEM_CIMPL_DEF_0(iemCImpl_loadall286) +{ + NOREF(cbInstr); + + /* Data is loaded from a buffer at 800h. No checks are done on the + * validity of loaded state. + * + * LOADALL only loads the internal CPU state, it does not access any + * GDT, LDT, or similar tables. + */ + + if (pVCpu->iem.s.uCpl != 0) + { + Log(("loadall286: CPL must be 0 not %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + uint8_t const *pbMem = NULL; + uint16_t const *pa16Mem; + uint8_t const *pa8Mem; + RTGCPHYS GCPtrStart = 0x800; /* Fixed table location. */ + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pbMem, 0x66, UINT8_MAX, GCPtrStart, IEM_ACCESS_SYS_R, 0); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* The MSW is at offset 0x06. */ + pa16Mem = (uint16_t const *)(pbMem + 0x06); + /* Even LOADALL can't clear the MSW.PE bit, though it can set it. */ + uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0 & ~(X86_CR0_MP | X86_CR0_EM | X86_CR0_TS); + uNewCr0 |= *pa16Mem & (X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS); + uint64_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0; + + CPUMSetGuestCR0(pVCpu, uNewCr0); + Assert(pVCpu->cpum.GstCtx.cr0 == uNewCr0); + + /* Inform PGM if mode changed. */ + if ((uNewCr0 & X86_CR0_PE) != (uOldCr0 & X86_CR0_PE)) + { + int rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */); + AssertRCReturn(rc, rc); + /* ignore informational status codes */ + } + rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER, + false /* fForce */); + + /* TR selector is at offset 0x16. */ + pa16Mem = (uint16_t const *)(pbMem + 0x16); + pVCpu->cpum.GstCtx.tr.Sel = pa16Mem[0]; + pVCpu->cpum.GstCtx.tr.ValidSel = pa16Mem[0]; + pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID; + + /* Followed by FLAGS... */ + pVCpu->cpum.GstCtx.eflags.u = pa16Mem[1] | X86_EFL_1; + pVCpu->cpum.GstCtx.ip = pa16Mem[2]; /* ...and IP. */ + + /* LDT is at offset 0x1C. */ + pa16Mem = (uint16_t const *)(pbMem + 0x1C); + pVCpu->cpum.GstCtx.ldtr.Sel = pa16Mem[0]; + pVCpu->cpum.GstCtx.ldtr.ValidSel = pa16Mem[0]; + pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID; + + /* Segment registers are at offset 0x1E. */ + pa16Mem = (uint16_t const *)(pbMem + 0x1E); + iemLoadallSetSelector(pVCpu, X86_SREG_DS, pa16Mem[0]); + iemLoadallSetSelector(pVCpu, X86_SREG_SS, pa16Mem[1]); + iemLoadallSetSelector(pVCpu, X86_SREG_CS, pa16Mem[2]); + iemLoadallSetSelector(pVCpu, X86_SREG_ES, pa16Mem[3]); + + /* GPRs are at offset 0x26. */ + pa16Mem = (uint16_t const *)(pbMem + 0x26); + pVCpu->cpum.GstCtx.di = pa16Mem[0]; + pVCpu->cpum.GstCtx.si = pa16Mem[1]; + pVCpu->cpum.GstCtx.bp = pa16Mem[2]; + pVCpu->cpum.GstCtx.sp = pa16Mem[3]; + pVCpu->cpum.GstCtx.bx = pa16Mem[4]; + pVCpu->cpum.GstCtx.dx = pa16Mem[5]; + pVCpu->cpum.GstCtx.cx = pa16Mem[6]; + pVCpu->cpum.GstCtx.ax = pa16Mem[7]; + + /* Descriptor caches are at offset 0x36, 6 bytes per entry. */ + iemLoadall286SetDescCache(pVCpu, X86_SREG_ES, pbMem + 0x36); + iemLoadall286SetDescCache(pVCpu, X86_SREG_CS, pbMem + 0x3C); + iemLoadall286SetDescCache(pVCpu, X86_SREG_SS, pbMem + 0x42); + iemLoadall286SetDescCache(pVCpu, X86_SREG_DS, pbMem + 0x48); + + /* GDTR contents are at offset 0x4E, 6 bytes. */ + RTGCPHYS GCPtrBase; + uint16_t cbLimit; + pa8Mem = pbMem + 0x4E; + /* NB: Fourth byte "should be zero"; we are ignoring it. */ + GCPtrBase = pa8Mem[0] + (pa8Mem[1] << 8) + (pa8Mem[2] << 16); + cbLimit = pa8Mem[4] + (pa8Mem[5] << 8); + CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit); + + /* IDTR contents are at offset 0x5A, 6 bytes. */ + pa8Mem = pbMem + 0x5A; + GCPtrBase = pa8Mem[0] + (pa8Mem[1] << 8) + (pa8Mem[2] << 16); + cbLimit = pa8Mem[4] + (pa8Mem[5] << 8); + CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit); + + Log(("LOADALL: GDTR:%08RX64/%04X, IDTR:%08RX64/%04X\n", pVCpu->cpum.GstCtx.gdtr.pGdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.idtr.pIdt, pVCpu->cpum.GstCtx.idtr.cbIdt)); + Log(("LOADALL: CS:%04X, CS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.cs.u64Base, pVCpu->cpum.GstCtx.cs.u32Limit, pVCpu->cpum.GstCtx.cs.Attr.u)); + Log(("LOADALL: DS:%04X, DS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.ds.u64Base, pVCpu->cpum.GstCtx.ds.u32Limit, pVCpu->cpum.GstCtx.ds.Attr.u)); + Log(("LOADALL: ES:%04X, ES base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.es.Sel, pVCpu->cpum.GstCtx.es.u64Base, pVCpu->cpum.GstCtx.es.u32Limit, pVCpu->cpum.GstCtx.es.Attr.u)); + Log(("LOADALL: SS:%04X, SS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u)); + Log(("LOADALL: SI:%04X, DI:%04X, AX:%04X, BX:%04X, CX:%04X, DX:%04X\n", pVCpu->cpum.GstCtx.si, pVCpu->cpum.GstCtx.di, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.cx, pVCpu->cpum.GstCtx.dx)); + + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pbMem, IEM_ACCESS_SYS_R); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* The CPL may change. It is taken from the "DPL fields of the SS and CS + * descriptor caches" but there is no word as to what happens if those are + * not identical (probably bad things). + */ + pVCpu->iem.s.uCpl = pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl; + + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS | CPUM_CHANGED_IDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_TR | CPUM_CHANGED_LDTR); + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); + +/** @todo single stepping */ + return rcStrict; +} + + +/** + * Implements SYSCALL (AMD and Intel64). + */ +IEM_CIMPL_DEF_0(iemCImpl_syscall) +{ + /** @todo hack, LOADALL should be decoded as such on a 286. */ + if (RT_UNLIKELY(pVCpu->iem.s.uTargetCpu == IEMTARGETCPU_286)) + return iemCImpl_loadall286(pVCpu, cbInstr); + + /* + * Check preconditions. + * + * Note that CPUs described in the documentation may load a few odd values + * into CS and SS than we allow here. This has yet to be checked on real + * hardware. + */ + if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE)) + { + Log(("syscall: Not enabled in EFER -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)) + { + Log(("syscall: Protected mode is required -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + if (IEM_IS_GUEST_CPU_INTEL(pVCpu) && !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu))) + { + Log(("syscall: Only available in long mode on intel -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS); + + /** @todo verify RPL ignoring and CS=0xfff8 (i.e. SS == 0). */ + /** @todo what about LDT selectors? Shouldn't matter, really. */ + uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSCALL_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL; + uint16_t uNewSs = uNewCs + 8; + if (uNewCs == 0 || uNewSs == 0) + { + /** @todo Neither Intel nor AMD document this check. */ + Log(("syscall: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Long mode and legacy mode differs. */ + if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu))) + { + uint64_t uNewRip = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.msrLSTAR : pVCpu->cpum.GstCtx. msrCSTAR; + + /* This test isn't in the docs, but I'm not trusting the guys writing + the MSRs to have validated the values as canonical like they should. */ + if (!IEM_IS_CANONICAL(uNewRip)) + { + /** @todo Intel claims this can't happen because IA32_LSTAR MSR can't be written with non-canonical address. */ + Log(("syscall: New RIP not canonical -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + + /* + * Commit it. + */ + Log(("syscall: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, uNewRip)); + pVCpu->cpum.GstCtx.rcx = pVCpu->cpum.GstCtx.rip + cbInstr; + pVCpu->cpum.GstCtx.rip = uNewRip; + + pVCpu->cpum.GstCtx.rflags.u &= ~X86_EFL_RF; + pVCpu->cpum.GstCtx.r11 = pVCpu->cpum.GstCtx.rflags.u; + pVCpu->cpum.GstCtx.rflags.u &= ~pVCpu->cpum.GstCtx.msrSFMASK; + pVCpu->cpum.GstCtx.rflags.u |= X86_EFL_1; + + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_L | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_RW_ACC; + } + else + { + /* + * Commit it. + */ + Log(("syscall: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, (uint32_t)(pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK))); + pVCpu->cpum.GstCtx.rcx = pVCpu->cpum.GstCtx.eip + cbInstr; + pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK; + pVCpu->cpum.GstCtx.rflags.u &= ~(X86_EFL_VM | X86_EFL_IF | X86_EFL_RF); + + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_RW_ACC; + } + pVCpu->cpum.GstCtx.cs.Sel = uNewCs; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs; + pVCpu->cpum.GstCtx.cs.u64Base = 0; + pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + + pVCpu->cpum.GstCtx.ss.Sel = uNewSs; + pVCpu->cpum.GstCtx.ss.ValidSel = uNewSs; + pVCpu->cpum.GstCtx.ss.u64Base = 0; + pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX; + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + + pVCpu->iem.s.uCpl = 0; + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); + +/** @todo single step */ + return VINF_SUCCESS; +} + + +/** + * Implements SYSRET (AMD and Intel64). + */ +IEM_CIMPL_DEF_0(iemCImpl_sysret) + +{ + RT_NOREF_PV(cbInstr); + + /* + * Check preconditions. + * + * Note that CPUs described in the documentation may load a few odd values + * into CS and SS than we allow here. This has yet to be checked on real + * hardware. + */ + if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE)) + { + Log(("sysret: Not enabled in EFER -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (IEM_IS_GUEST_CPU_INTEL(pVCpu) && !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu))) + { + Log(("sysret: Only available in long mode on intel -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)) + { + Log(("sysret: Protected mode is required -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + if (pVCpu->iem.s.uCpl != 0) + { + Log(("sysret: CPL must be 0 not %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS); + + /** @todo Does SYSRET verify CS != 0 and SS != 0? Neither is valid in ring-3. */ + uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSRET_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL; + uint16_t uNewSs = uNewCs + 8; + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + uNewCs += 16; + if (uNewCs == 0 || uNewSs == 0) + { + Log(("sysret: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Commit it. + */ + if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu))) + { + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64 [r11=%#llx]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.r11)); + /* Note! We disregard intel manual regarding the RCX canonical + check, ask intel+xen why AMD doesn't do it. */ + pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rcx; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_L | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC + | (3 << X86DESCATTR_DPL_SHIFT); + } + else + { + Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%08RX32 [r11=%#llx]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.r11)); + pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.ecx; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC + | (3 << X86DESCATTR_DPL_SHIFT); + } + /** @todo testcase: See what kind of flags we can make SYSRET restore and + * what it really ignores. RF and VM are hinted at being zero, by AMD. + * Intel says: RFLAGS := (R11 & 3C7FD7H) | 2; */ + pVCpu->cpum.GstCtx.rflags.u = pVCpu->cpum.GstCtx.r11 & (X86_EFL_POPF_BITS | X86_EFL_VIF | X86_EFL_VIP); + pVCpu->cpum.GstCtx.rflags.u |= X86_EFL_1; + } + else + { + Log(("sysret: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx)); + pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rcx; + pVCpu->cpum.GstCtx.rflags.u |= X86_EFL_IF; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC + | (3 << X86DESCATTR_DPL_SHIFT); + } + pVCpu->cpum.GstCtx.cs.Sel = uNewCs | 3; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs | 3; + pVCpu->cpum.GstCtx.cs.u64Base = 0; + pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + + pVCpu->cpum.GstCtx.ss.Sel = uNewSs | 3; + pVCpu->cpum.GstCtx.ss.ValidSel = uNewSs | 3; + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + /* The SS hidden bits remains unchanged says AMD. To that I say "Yeah, right!". */ + pVCpu->cpum.GstCtx.ss.Attr.u |= (3 << X86DESCATTR_DPL_SHIFT); + /** @todo Testcase: verify that SS.u1Long and SS.u1DefBig are left unchanged + * on sysret. */ + + pVCpu->iem.s.uCpl = 3; + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); + +/** @todo single step */ + return VINF_SUCCESS; +} + + +/** + * Implements SYSENTER (Intel, 32-bit AMD). + */ +IEM_CIMPL_DEF_0(iemCImpl_sysenter) +{ + RT_NOREF(cbInstr); + + /* + * Check preconditions. + * + * Note that CPUs described in the documentation may load a few odd values + * into CS and SS than we allow here. This has yet to be checked on real + * hardware. + */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSysEnter) + { + Log(("sysenter: not supported -=> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)) + { + Log(("sysenter: Protected or long mode is required -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + bool fIsLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)); + if (IEM_IS_GUEST_CPU_AMD(pVCpu) && fIsLongMode) + { + Log(("sysenter: Only available in protected mode on AMD -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS); + uint16_t uNewCs = pVCpu->cpum.GstCtx.SysEnter.cs; + if ((uNewCs & X86_SEL_MASK_OFF_RPL) == 0) + { + Log(("sysenter: SYSENTER_CS = %#x -> #GP(0)\n", uNewCs)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* This test isn't in the docs, it's just a safeguard against missing + canonical checks when writing the registers. */ + if (RT_LIKELY( !fIsLongMode + || ( IEM_IS_CANONICAL(pVCpu->cpum.GstCtx.SysEnter.eip) + && IEM_IS_CANONICAL(pVCpu->cpum.GstCtx.SysEnter.esp)))) + { /* likely */ } + else + { + Log(("sysenter: SYSENTER_EIP = %#RX64 or/and SYSENTER_ESP = %#RX64 not canonical -> #GP(0)\n", + pVCpu->cpum.GstCtx.SysEnter.eip, pVCpu->cpum.GstCtx.SysEnter.esp)); + return iemRaiseUndefinedOpcode(pVCpu); + } + +/** @todo Test: Sysenter from ring-0, ring-1 and ring-2. */ + + /* + * Update registers and commit. + */ + if (fIsLongMode) + { + Log(("sysenter: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, + pVCpu->cpum.GstCtx.rflags.u, uNewCs & X86_SEL_MASK_OFF_RPL, pVCpu->cpum.GstCtx.SysEnter.eip)); + pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.SysEnter.eip; + pVCpu->cpum.GstCtx.rsp = pVCpu->cpum.GstCtx.SysEnter.esp; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_L | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT + | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_ER_ACC; + } + else + { + Log(("sysenter: %04x:%08RX32 [efl=%#llx] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs, (uint32_t)pVCpu->cpum.GstCtx.rip, + pVCpu->cpum.GstCtx.rflags.u, uNewCs & X86_SEL_MASK_OFF_RPL, (uint32_t)pVCpu->cpum.GstCtx.SysEnter.eip)); + pVCpu->cpum.GstCtx.rip = (uint32_t)pVCpu->cpum.GstCtx.SysEnter.eip; + pVCpu->cpum.GstCtx.rsp = (uint32_t)pVCpu->cpum.GstCtx.SysEnter.esp; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_D | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT + | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_ER_ACC; + } + pVCpu->cpum.GstCtx.cs.Sel = uNewCs & X86_SEL_MASK_OFF_RPL; + pVCpu->cpum.GstCtx.cs.ValidSel = uNewCs & X86_SEL_MASK_OFF_RPL; + pVCpu->cpum.GstCtx.cs.u64Base = 0; + pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + + pVCpu->cpum.GstCtx.ss.Sel = (uNewCs & X86_SEL_MASK_OFF_RPL) + 8; + pVCpu->cpum.GstCtx.ss.ValidSel = (uNewCs & X86_SEL_MASK_OFF_RPL) + 8; + pVCpu->cpum.GstCtx.ss.u64Base = 0; + pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_D | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT + | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_RW_ACC; + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + + pVCpu->cpum.GstCtx.rflags.Bits.u1IF = 0; + pVCpu->cpum.GstCtx.rflags.Bits.u1VM = 0; + pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0; + + pVCpu->iem.s.uCpl = 0; + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); + +/** @todo single stepping */ + return VINF_SUCCESS; +} + + +/** + * Implements SYSEXIT (Intel, 32-bit AMD). + * + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_1(iemCImpl_sysexit, IEMMODE, enmEffOpSize) +{ + RT_NOREF(cbInstr); + + /* + * Check preconditions. + * + * Note that CPUs described in the documentation may load a few odd values + * into CS and SS than we allow here. This has yet to be checked on real + * hardware. + */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSysEnter) + { + Log(("sysexit: not supported -=> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)) + { + Log(("sysexit: Protected or long mode is required -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + bool fIsLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)); + if (IEM_IS_GUEST_CPU_AMD(pVCpu) && fIsLongMode) + { + Log(("sysexit: Only available in protected mode on AMD -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (pVCpu->iem.s.uCpl != 0) + { + Log(("sysexit: CPL(=%u) != 0 -> #GP(0)\n", pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS); + uint16_t uNewCs = pVCpu->cpum.GstCtx.SysEnter.cs; + if ((uNewCs & X86_SEL_MASK_OFF_RPL) == 0) + { + Log(("sysexit: SYSENTER_CS = %#x -> #GP(0)\n", uNewCs)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Update registers and commit. + */ + if (enmEffOpSize == IEMMODE_64BIT) + { + Log(("sysexit: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, + pVCpu->cpum.GstCtx.rflags.u, (uNewCs | 3) + 32, pVCpu->cpum.GstCtx.rcx)); + pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rdx; + pVCpu->cpum.GstCtx.rsp = pVCpu->cpum.GstCtx.rcx; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_L | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT + | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_ER_ACC | (3 << X86DESCATTR_DPL_SHIFT); + pVCpu->cpum.GstCtx.cs.Sel = (uNewCs | 3) + 32; + pVCpu->cpum.GstCtx.cs.ValidSel = (uNewCs | 3) + 32; + pVCpu->cpum.GstCtx.ss.Sel = (uNewCs | 3) + 40; + pVCpu->cpum.GstCtx.ss.ValidSel = (uNewCs | 3) + 40; + } + else + { + Log(("sysexit: %04x:%08RX64 [efl=%#llx] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, + pVCpu->cpum.GstCtx.rflags.u, (uNewCs | 3) + 16, (uint32_t)pVCpu->cpum.GstCtx.edx)); + pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.edx; + pVCpu->cpum.GstCtx.rsp = pVCpu->cpum.GstCtx.ecx; + pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_D | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT + | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_ER_ACC | (3 << X86DESCATTR_DPL_SHIFT); + pVCpu->cpum.GstCtx.cs.Sel = (uNewCs | 3) + 16; + pVCpu->cpum.GstCtx.cs.ValidSel = (uNewCs | 3) + 16; + pVCpu->cpum.GstCtx.ss.Sel = (uNewCs | 3) + 24; + pVCpu->cpum.GstCtx.ss.ValidSel = (uNewCs | 3) + 24; + } + pVCpu->cpum.GstCtx.cs.u64Base = 0; + pVCpu->cpum.GstCtx.cs.u32Limit = UINT32_MAX; + pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID; + + pVCpu->cpum.GstCtx.ss.u64Base = 0; + pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX; + pVCpu->cpum.GstCtx.ss.Attr.u = X86DESCATTR_D | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT + | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_RW_ACC | (3 << X86DESCATTR_DPL_SHIFT); + pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0; + + pVCpu->iem.s.uCpl = 3; +/** @todo single stepping */ + + /* Flush the prefetch buffer. */ + IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); + + return VINF_SUCCESS; +} + + +/** + * Completes a MOV SReg,XXX or POP SReg instruction. + * + * When not modifying SS or when we're already in an interrupt shadow we + * can update RIP and finish the instruction the normal way. + * + * Otherwise, the MOV/POP SS interrupt shadow that we now enable will block + * both TF and DBx events. The TF will be ignored while the DBx ones will + * be delayed till the next instruction boundrary. For more details see + * @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching Stacks}. + */ +DECLINLINE(VBOXSTRICTRC) iemCImpl_LoadSRegFinish(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iSegReg) +{ + if (iSegReg != X86_SREG_SS || CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + iemRegAddToRip(pVCpu, cbInstr); + pVCpu->cpum.GstCtx.eflags.uBoth &= ~X86_EFL_RF; /* Shadow int isn't set and DRx is delayed, so only clear RF. */ + CPUMSetInInterruptShadowSs(&pVCpu->cpum.GstCtx); + + return VINF_SUCCESS; +} + + +/** + * Common worker for 'pop SReg', 'mov SReg, GReg' and 'lXs GReg, reg/mem'. + * + * @param pVCpu The cross context virtual CPU structure of the calling + * thread. + * @param iSegReg The segment register number (valid). + * @param uSel The new selector value. + */ +static VBOXSTRICTRC iemCImpl_LoadSRegWorker(PVMCPUCC pVCpu, uint8_t iSegReg, uint16_t uSel) +{ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + uint16_t *pSel = iemSRegRef(pVCpu, iSegReg); + PCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iSegReg); + + Assert(iSegReg <= X86_SREG_GS && iSegReg != X86_SREG_CS); + + /* + * Real mode and V8086 mode are easy. + */ + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + *pSel = uSel; + pHid->u64Base = (uint32_t)uSel << 4; + pHid->ValidSel = uSel; + pHid->fFlags = CPUMSELREG_FLAGS_VALID; +#if 0 /* AMD Volume 2, chapter 4.1 - "real mode segmentation" - states that limit and attributes are untouched. */ + /** @todo Does the CPU actually load limits and attributes in the + * real/V8086 mode segment load case? It doesn't for CS in far + * jumps... Affects unreal mode. */ + pHid->u32Limit = 0xffff; + pHid->Attr.u = 0; + pHid->Attr.n.u1Present = 1; + pHid->Attr.n.u1DescType = 1; + pHid->Attr.n.u4Type = iSegReg != X86_SREG_CS + ? X86_SEL_TYPE_RW + : X86_SEL_TYPE_READ | X86_SEL_TYPE_CODE; +#endif + } + /* + * Protected mode. + * + * Check if it's a null segment selector value first, that's OK for DS, ES, + * FS and GS. If not null, then we have to load and parse the descriptor. + */ + else if (!(uSel & X86_SEL_MASK_OFF_RPL)) + { + Assert(iSegReg != X86_SREG_CS); /** @todo testcase for \#UD on MOV CS, ax! */ + if (iSegReg == X86_SREG_SS) + { + /* In 64-bit kernel mode, the stack can be 0 because of the way + interrupts are dispatched. AMD seems to have a slighly more + relaxed relationship to SS.RPL than intel does. */ + /** @todo We cannot 'mov ss, 3' in 64-bit kernel mode, can we? There is a testcase (bs-cpu-xcpt-1), but double check this! */ + if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT + || pVCpu->iem.s.uCpl > 2 + || ( uSel != pVCpu->iem.s.uCpl + && !IEM_IS_GUEST_CPU_AMD(pVCpu)) ) + { + Log(("load sreg %#x -> invalid stack selector, #GP(0)\n", uSel)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + + *pSel = uSel; /* Not RPL, remember :-) */ + iemHlpLoadNullDataSelectorProt(pVCpu, pHid, uSel); + if (iSegReg == X86_SREG_SS) + pHid->Attr.u |= pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT; + } + else + { + + /* Fetch the descriptor. */ + IEMSELDESC Desc; + VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP); /** @todo Correct exception? */ + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Check GPs first. */ + if (!Desc.Legacy.Gen.u1DescType) + { + Log(("load sreg %d (=%#x) - system selector (%#x) -> #GP\n", iSegReg, uSel, Desc.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + if (iSegReg == X86_SREG_SS) /* SS gets different treatment */ + { + if ( (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) + || !(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) ) + { + Log(("load sreg SS, %#x - code or read only (%#x) -> #GP\n", uSel, Desc.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + if ((uSel & X86_SEL_RPL) != pVCpu->iem.s.uCpl) + { + Log(("load sreg SS, %#x - RPL and CPL (%d) differs -> #GP\n", uSel, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + if (Desc.Legacy.Gen.u2Dpl != pVCpu->iem.s.uCpl) + { + Log(("load sreg SS, %#x - DPL (%d) and CPL (%d) differs -> #GP\n", uSel, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + } + else + { + if ((Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE) + { + Log(("load sreg%u, %#x - execute only segment -> #GP\n", iSegReg, uSel)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) + != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) + { +#if 0 /* this is what intel says. */ + if ( (uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl + && pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl) + { + Log(("load sreg%u, %#x - both RPL (%d) and CPL (%d) are greater than DPL (%d) -> #GP\n", + iSegReg, uSel, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl, Desc.Legacy.Gen.u2Dpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } +#else /* this is what makes more sense. */ + if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl) + { + Log(("load sreg%u, %#x - RPL (%d) is greater than DPL (%d) -> #GP\n", + iSegReg, uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } + if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl) + { + Log(("load sreg%u, %#x - CPL (%d) is greater than DPL (%d) -> #GP\n", + iSegReg, uSel, pVCpu->iem.s.uCpl, Desc.Legacy.Gen.u2Dpl)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel); + } +#endif + } + } + + /* Is it there? */ + if (!Desc.Legacy.Gen.u1Present) + { + Log(("load sreg%d,%#x - segment not present -> #NP\n", iSegReg, uSel)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel); + } + + /* The base and limit. */ + uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy); + uint64_t u64Base = X86DESC_BASE(&Desc.Legacy); + + /* + * Ok, everything checked out fine. Now set the accessed bit before + * committing the result into the registers. + */ + if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED)) + { + rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED; + } + + /* commit */ + *pSel = uSel; + pHid->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy); + pHid->u32Limit = cbLimit; + pHid->u64Base = u64Base; + pHid->ValidSel = uSel; + pHid->fFlags = CPUMSELREG_FLAGS_VALID; + + /** @todo check if the hidden bits are loaded correctly for 64-bit + * mode. */ + } + + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pHid)); + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS); + return VINF_SUCCESS; +} + + +/** + * Implements 'mov SReg, r/m'. + * + * @param iSegReg The segment register number (valid). + * @param uSel The new selector value. + */ +IEM_CIMPL_DEF_2(iemCImpl_load_SReg, uint8_t, iSegReg, uint16_t, uSel) +{ + VBOXSTRICTRC rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemCImpl_LoadSRegFinish(pVCpu, cbInstr, iSegReg); + return rcStrict; +} + + +/** + * Implements 'pop SReg'. + * + * @param iSegReg The segment register number (valid). + * @param enmEffOpSize The efficient operand size (valid). + */ +IEM_CIMPL_DEF_2(iemCImpl_pop_Sreg, uint8_t, iSegReg, IEMMODE, enmEffOpSize) +{ + VBOXSTRICTRC rcStrict; + + /* + * Read the selector off the stack and join paths with mov ss, reg. + */ + RTUINT64U TmpRsp; + TmpRsp.u = pVCpu->cpum.GstCtx.rsp; + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t uSel; + rcStrict = iemMemStackPopU16Ex(pVCpu, &uSel, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel); + break; + } + + case IEMMODE_32BIT: + { + uint32_t u32Value; + rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Value, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, (uint16_t)u32Value); + break; + } + + case IEMMODE_64BIT: + { + uint64_t u64Value; + rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Value, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, (uint16_t)u64Value); + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + /* + * If the load succeeded, commit the stack change and finish the instruction. + */ + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rsp = TmpRsp.u; + rcStrict = iemCImpl_LoadSRegFinish(pVCpu, cbInstr, iSegReg); + } + + return rcStrict; +} + + +/** + * Implements lgs, lfs, les, lds & lss. + */ +IEM_CIMPL_DEF_5(iemCImpl_load_SReg_Greg, uint16_t, uSel, uint64_t, offSeg, uint8_t, iSegReg, uint8_t, iGReg, IEMMODE, enmEffOpSize) +{ + /* + * Use iemCImpl_LoadSRegWorker to do the tricky segment register loading. + */ + /** @todo verify and test that mov, pop and lXs works the segment + * register loading in the exact same way. */ + VBOXSTRICTRC rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel); + if (rcStrict == VINF_SUCCESS) + { + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + *(uint16_t *)iemGRegRef(pVCpu, iGReg) = offSeg; + break; + case IEMMODE_32BIT: + case IEMMODE_64BIT: + *(uint64_t *)iemGRegRef(pVCpu, iGReg) = offSeg; + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + return rcStrict; +} + + +/** + * Helper for VERR, VERW, LAR, and LSL and loads the descriptor into memory. + * + * @retval VINF_SUCCESS on success. + * @retval VINF_IEM_SELECTOR_NOT_OK if the selector isn't ok. + * @retval iemMemFetchSysU64 return value. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param uSel The selector value. + * @param fAllowSysDesc Whether system descriptors are OK or not. + * @param pDesc Where to return the descriptor on success. + */ +static VBOXSTRICTRC iemCImpl_LoadDescHelper(PVMCPUCC pVCpu, uint16_t uSel, bool fAllowSysDesc, PIEMSELDESC pDesc) +{ + pDesc->Long.au64[0] = 0; + pDesc->Long.au64[1] = 0; + + if (!(uSel & X86_SEL_MASK_OFF_RPL)) /** @todo test this on 64-bit. */ + return VINF_IEM_SELECTOR_NOT_OK; + + /* Within the table limits? */ + RTGCPTR GCPtrBase; + if (uSel & X86_SEL_LDT) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR); + if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present + || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit ) + return VINF_IEM_SELECTOR_NOT_OK; + GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base; + } + else + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR); + if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt) + return VINF_IEM_SELECTOR_NOT_OK; + GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt; + } + + /* Fetch the descriptor. */ + VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK)); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + if (!pDesc->Legacy.Gen.u1DescType) + { + if (!fAllowSysDesc) + return VINF_IEM_SELECTOR_NOT_OK; + if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu))) + { + rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 8); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + } + + return VINF_SUCCESS; +} + + +/** + * Implements verr (fWrite = false) and verw (fWrite = true). + */ +IEM_CIMPL_DEF_2(iemCImpl_VerX, uint16_t, uSel, bool, fWrite) +{ + Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu)); + + /** @todo figure whether the accessed bit is set or not. */ + + bool fAccessible = true; + IEMSELDESC Desc; + VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, false /*fAllowSysDesc*/, &Desc); + if (rcStrict == VINF_SUCCESS) + { + /* Check the descriptor, order doesn't matter much here. */ + if ( !Desc.Legacy.Gen.u1DescType + || !Desc.Legacy.Gen.u1Present) + fAccessible = false; + else + { + if ( fWrite + ? (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE + : (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE) + fAccessible = false; + + /** @todo testcase for the conforming behavior. */ + if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) + != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) + { + if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl) + fAccessible = false; + else if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl) + fAccessible = false; + } + } + + } + else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK) + fAccessible = false; + else + return rcStrict; + + /* commit */ + pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fAccessible; + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements LAR and LSL with 64-bit operand size. + * + * @returns VINF_SUCCESS. + * @param pu64Dst Pointer to the destination register. + * @param uSel The selector to load details for. + * @param fIsLar true = LAR, false = LSL. + */ +IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u64, uint64_t *, pu64Dst, uint16_t, uSel, bool, fIsLar) +{ + Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu)); + + /** @todo figure whether the accessed bit is set or not. */ + + bool fDescOk = true; + IEMSELDESC Desc; + VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, true /*fAllowSysDesc*/, &Desc); + if (rcStrict == VINF_SUCCESS) + { + /* + * Check the descriptor type. + */ + if (!Desc.Legacy.Gen.u1DescType) + { + if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu))) + { + if (Desc.Long.Gen.u5Zeros) + fDescOk = false; + else + switch (Desc.Long.Gen.u4Type) + { + /** @todo Intel lists 0 as valid for LSL, verify whether that's correct */ + case AMD64_SEL_TYPE_SYS_TSS_AVAIL: + case AMD64_SEL_TYPE_SYS_TSS_BUSY: + case AMD64_SEL_TYPE_SYS_LDT: /** @todo Intel lists this as invalid for LAR, AMD and 32-bit does otherwise. */ + break; + case AMD64_SEL_TYPE_SYS_CALL_GATE: + fDescOk = fIsLar; + break; + default: + fDescOk = false; + break; + } + } + else + { + switch (Desc.Long.Gen.u4Type) + { + case X86_SEL_TYPE_SYS_286_TSS_AVAIL: + case X86_SEL_TYPE_SYS_286_TSS_BUSY: + case X86_SEL_TYPE_SYS_386_TSS_AVAIL: + case X86_SEL_TYPE_SYS_386_TSS_BUSY: + case X86_SEL_TYPE_SYS_LDT: + break; + case X86_SEL_TYPE_SYS_286_CALL_GATE: + case X86_SEL_TYPE_SYS_TASK_GATE: + case X86_SEL_TYPE_SYS_386_CALL_GATE: + fDescOk = fIsLar; + break; + default: + fDescOk = false; + break; + } + } + } + if (fDescOk) + { + /* + * Check the RPL/DPL/CPL interaction.. + */ + /** @todo testcase for the conforming behavior. */ + if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF) + || !Desc.Legacy.Gen.u1DescType) + { + if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl) + fDescOk = false; + else if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl) + fDescOk = false; + } + } + + if (fDescOk) + { + /* + * All fine, start committing the result. + */ + if (fIsLar) + *pu64Dst = Desc.Legacy.au32[1] & UINT32_C(0x00ffff00); + else + *pu64Dst = X86DESC_LIMIT_G(&Desc.Legacy); + } + + } + else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK) + fDescOk = false; + else + return rcStrict; + + /* commit flags value and advance rip. */ + pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fDescOk; + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements LAR and LSL with 16-bit operand size. + * + * @returns VINF_SUCCESS. + * @param pu16Dst Pointer to the destination register. + * @param uSel The selector to load details for. + * @param fIsLar true = LAR, false = LSL. + */ +IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u16, uint16_t *, pu16Dst, uint16_t, uSel, bool, fIsLar) +{ + uint64_t u64TmpDst = *pu16Dst; + IEM_CIMPL_CALL_3(iemCImpl_LarLsl_u64, &u64TmpDst, uSel, fIsLar); + *pu16Dst = u64TmpDst; + return VINF_SUCCESS; +} + + +/** + * Implements lgdt. + * + * @param iEffSeg The segment of the new gdtr contents + * @param GCPtrEffSrc The address of the new gdtr contents. + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_3(iemCImpl_lgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize) +{ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT)) + { + Log(("lgdt: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_LGDT, cbInstr); + } + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_WRITES)) + { + Log(("lgdt: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Fetch the limit and base address. + */ + uint16_t cbLimit; + RTGCPTR GCPtrBase; + VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize); + if (rcStrict == VINF_SUCCESS) + { + if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT + || X86_IS_CANONICAL(GCPtrBase)) + { + rcStrict = CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + else + { + Log(("iemCImpl_lgdt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + return rcStrict; +} + + +/** + * Implements sgdt. + * + * @param iEffSeg The segment where to store the gdtr content. + * @param GCPtrEffDst The address where to store the gdtr content. + */ +IEM_CIMPL_DEF_2(iemCImpl_sgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst) +{ + /* + * Join paths with sidt. + * Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if + * you really must know. + */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT)) + { + Log(("sgdt: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_SGDT, cbInstr); + } + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_READS)) + { + Log(("sgdt: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_READ, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR); + VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.gdtr.pGdt, iEffSeg, GCPtrEffDst); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return rcStrict; +} + + +/** + * Implements lidt. + * + * @param iEffSeg The segment of the new idtr contents + * @param GCPtrEffSrc The address of the new idtr contents. + * @param enmEffOpSize The effective operand size. + */ +IEM_CIMPL_DEF_3(iemCImpl_lidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize) +{ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_WRITES)) + { + Log(("lidt: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Fetch the limit and base address. + */ + uint16_t cbLimit; + RTGCPTR GCPtrBase; + VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize); + if (rcStrict == VINF_SUCCESS) + { + if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT + || X86_IS_CANONICAL(GCPtrBase)) + { + CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit); + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + else + { + Log(("iemCImpl_lidt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + return rcStrict; +} + + +/** + * Implements sidt. + * + * @param iEffSeg The segment where to store the idtr content. + * @param GCPtrEffDst The address where to store the idtr content. + */ +IEM_CIMPL_DEF_2(iemCImpl_sidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst) +{ + /* + * Join paths with sgdt. + * Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if + * you really must know. + */ + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_READS)) + { + Log(("sidt: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_READ, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_IDTR); + VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.idtr.cbIdt, pVCpu->cpum.GstCtx.idtr.pIdt, iEffSeg, GCPtrEffDst); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return rcStrict; +} + + +/** + * Implements lldt. + * + * @param uNewLdt The new LDT selector value. + */ +IEM_CIMPL_DEF_1(iemCImpl_lldt, uint16_t, uNewLdt) +{ + /* + * Check preconditions. + */ + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + Log(("lldt %04x - real or v8086 mode -> #GP(0)\n", uNewLdt)); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (pVCpu->iem.s.uCpl != 0) + { + Log(("lldt %04x - CPL is %d -> #GP(0)\n", uNewLdt, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + /* Nested-guest VMX intercept. */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT)) + { + Log(("lldt: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LLDT, cbInstr); + } + if (uNewLdt & X86_SEL_LDT) + { + Log(("lldt %04x - LDT selector -> #GP\n", uNewLdt)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewLdt); + } + + /* + * Now, loading a NULL selector is easy. + */ + if (!(uNewLdt & X86_SEL_MASK_OFF_RPL)) + { + /* Nested-guest SVM intercept. */ + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES)) + { + Log(("lldt: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + Log(("lldt %04x: Loading NULL selector.\n", uNewLdt)); + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_LDTR; + CPUMSetGuestLDTR(pVCpu, uNewLdt); + pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt; + pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID; + if (IEM_IS_GUEST_CPU_AMD(pVCpu)) + { + /* AMD-V seems to leave the base and limit alone. */ + pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE; + } + else + { + /* VT-x (Intel 3960x) seems to be doing the following. */ + pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D; + pVCpu->cpum.GstCtx.ldtr.u64Base = 0; + pVCpu->cpum.GstCtx.ldtr.u32Limit = UINT32_MAX; + } + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* + * Read the descriptor. + */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR | CPUMCTX_EXTRN_GDTR); + IEMSELDESC Desc; + VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewLdt, X86_XCPT_GP); /** @todo Correct exception? */ + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Check GPs first. */ + if (Desc.Legacy.Gen.u1DescType) + { + Log(("lldt %#x - not system selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL); + } + if (Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT) + { + Log(("lldt %#x - not LDT selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL); + } + uint64_t u64Base; + if (!IEM_IS_LONG_MODE(pVCpu)) + u64Base = X86DESC_BASE(&Desc.Legacy); + else + { + if (Desc.Long.Gen.u5Zeros) + { + Log(("lldt %#x - u5Zeros=%#x -> #GP\n", uNewLdt, Desc.Long.Gen.u5Zeros)); + return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL); + } + + u64Base = X86DESC64_BASE(&Desc.Long); + if (!IEM_IS_CANONICAL(u64Base)) + { + Log(("lldt %#x - non-canonical base address %#llx -> #GP\n", uNewLdt, u64Base)); + return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL); + } + } + + /* NP */ + if (!Desc.Legacy.Gen.u1Present) + { + Log(("lldt %#x - segment not present -> #NP\n", uNewLdt)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewLdt); + } + + /* Nested-guest SVM intercept. */ + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES)) + { + Log(("lldt: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * It checks out alright, update the registers. + */ +/** @todo check if the actual value is loaded or if the RPL is dropped */ + CPUMSetGuestLDTR(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL); + pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt & X86_SEL_MASK_OFF_RPL; + pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy); + pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy); + pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base; + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements sldt GReg + * + * @param iGReg The general register to store the CRx value in. + * @param enmEffOpSize The operand size. + */ +IEM_CIMPL_DEF_2(iemCImpl_sldt_reg, uint8_t, iGReg, uint8_t, enmEffOpSize) +{ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT)) + { + Log(("sldt: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_SLDT, cbInstr); + } + + IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR); + switch (enmEffOpSize) + { + case IEMMODE_16BIT: *(uint16_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break; + case IEMMODE_32BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break; + case IEMMODE_64BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements sldt mem. + * + * @param iEffSeg The effective segment register to use with @a GCPtrMem. + * @param GCPtrEffDst Where to store the 16-bit CR0 value. + */ +IEM_CIMPL_DEF_2(iemCImpl_sldt_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst) +{ + IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR); + VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.ldtr.Sel); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return rcStrict; +} + + +/** + * Implements ltr. + * + * @param uNewTr The new TSS selector value. + */ +IEM_CIMPL_DEF_1(iemCImpl_ltr, uint16_t, uNewTr) +{ + /* + * Check preconditions. + */ + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + Log(("ltr %04x - real or v8086 mode -> #GP(0)\n", uNewTr)); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (pVCpu->iem.s.uCpl != 0) + { + Log(("ltr %04x - CPL is %d -> #GP(0)\n", uNewTr, pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT)) + { + Log(("ltr: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LTR, cbInstr); + } + if (uNewTr & X86_SEL_LDT) + { + Log(("ltr %04x - LDT selector -> #GP\n", uNewTr)); + return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewTr); + } + if (!(uNewTr & X86_SEL_MASK_OFF_RPL)) + { + Log(("ltr %04x - NULL selector -> #GP(0)\n", uNewTr)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TR_WRITES)) + { + Log(("ltr: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Read the descriptor. + */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_TR); + IEMSELDESC Desc; + VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewTr, X86_XCPT_GP); /** @todo Correct exception? */ + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Check GPs first. */ + if (Desc.Legacy.Gen.u1DescType) + { + Log(("ltr %#x - not system selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL); + } + if ( Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL /* same as AMD64_SEL_TYPE_SYS_TSS_AVAIL */ + && ( Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL + || IEM_IS_LONG_MODE(pVCpu)) ) + { + Log(("ltr %#x - not an available TSS selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type)); + return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL); + } + uint64_t u64Base; + if (!IEM_IS_LONG_MODE(pVCpu)) + u64Base = X86DESC_BASE(&Desc.Legacy); + else + { + if (Desc.Long.Gen.u5Zeros) + { + Log(("ltr %#x - u5Zeros=%#x -> #GP\n", uNewTr, Desc.Long.Gen.u5Zeros)); + return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL); + } + + u64Base = X86DESC64_BASE(&Desc.Long); + if (!IEM_IS_CANONICAL(u64Base)) + { + Log(("ltr %#x - non-canonical base address %#llx -> #GP\n", uNewTr, u64Base)); + return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL); + } + } + + /* NP */ + if (!Desc.Legacy.Gen.u1Present) + { + Log(("ltr %#x - segment not present -> #NP\n", uNewTr)); + return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewTr); + } + + /* + * Set it busy. + * Note! Intel says this should lock down the whole descriptor, but we'll + * restrict our selves to 32-bit for now due to lack of inline + * assembly and such. + */ + void *pvDesc; + rcStrict = iemMemMap(pVCpu, &pvDesc, 8, UINT8_MAX, pVCpu->cpum.GstCtx.gdtr.pGdt + (uNewTr & X86_SEL_MASK_OFF_RPL), + IEM_ACCESS_DATA_RW, 0); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + switch ((uintptr_t)pvDesc & 3) + { + case 0: ASMAtomicBitSet(pvDesc, 40 + 1); break; + case 1: ASMAtomicBitSet((uint8_t *)pvDesc + 3, 40 + 1 - 24); break; + case 2: ASMAtomicBitSet((uint8_t *)pvDesc + 2, 40 + 1 - 16); break; + case 3: ASMAtomicBitSet((uint8_t *)pvDesc + 1, 40 + 1 - 8); break; + } + rcStrict = iemMemCommitAndUnmap(pVCpu, pvDesc, IEM_ACCESS_DATA_RW); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK; + + /* + * It checks out alright, update the registers. + */ +/** @todo check if the actual value is loaded or if the RPL is dropped */ + CPUMSetGuestTR(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL); + pVCpu->cpum.GstCtx.tr.ValidSel = uNewTr & X86_SEL_MASK_OFF_RPL; + pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy); + pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy); + pVCpu->cpum.GstCtx.tr.u64Base = u64Base; + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements str GReg + * + * @param iGReg The general register to store the CRx value in. + * @param enmEffOpSize The operand size. + */ +IEM_CIMPL_DEF_2(iemCImpl_str_reg, uint8_t, iGReg, uint8_t, enmEffOpSize) +{ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT)) + { + Log(("str_reg: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr); + } + + IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR); + switch (enmEffOpSize) + { + case IEMMODE_16BIT: *(uint16_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break; + case IEMMODE_32BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break; + case IEMMODE_64BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements str mem. + * + * @param iEffSeg The effective segment register to use with @a GCPtrMem. + * @param GCPtrEffDst Where to store the 16-bit CR0 value. + */ +IEM_CIMPL_DEF_2(iemCImpl_str_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst) +{ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT)) + { + Log(("str_mem: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr); + } + + IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR); + VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.tr.Sel); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return rcStrict; +} + + +/** + * Implements mov GReg,CRx. + * + * @param iGReg The general register to store the CRx value in. + * @param iCrReg The CRx register to read (valid). + */ +IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Cd, uint8_t, iGReg, uint8_t, iCrReg) +{ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + + if (IEM_SVM_IS_READ_CR_INTERCEPT_SET(pVCpu, iCrReg)) + { + Log(("iemCImpl_mov_Rd_Cd: Guest intercept CR%u -> #VMEXIT\n", iCrReg)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_READ_CR0 + iCrReg, IEMACCESSCRX_MOV_CRX, iGReg); + } + + /* Read it. */ + uint64_t crX; + switch (iCrReg) + { + case 0: + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); + crX = pVCpu->cpum.GstCtx.cr0; + if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386) + crX |= UINT32_C(0x7fffffe0); /* All reserved CR0 flags are set on a 386, just like MSW on 286. */ + break; + case 2: + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR2); + crX = pVCpu->cpum.GstCtx.cr2; + break; + case 3: + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3); + crX = pVCpu->cpum.GstCtx.cr3; + break; + case 4: + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4); + crX = pVCpu->cpum.GstCtx.cr4; + break; + case 8: + { + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr8(pVCpu, iGReg, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + + /* + * If the Mov-from-CR8 doesn't cause a VM-exit, bits 7:4 of the VTPR is copied + * to bits 0:3 of the destination operand. Bits 63:4 of the destination operand + * are cleared. + * + * See Intel Spec. 29.3 "Virtualizing CR8-based TPR Accesses" + */ + if (IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW)) + { + uint32_t const uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR); + crX = (uTpr >> 4) & 0xf; + break; + } + } +#endif +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))) + { + PCSVMVMCBCTRL pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl; + if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu))) + { + crX = pVmcbCtrl->IntCtrl.n.u8VTPR & 0xf; + break; + } + } +#endif + uint8_t uTpr; + int rc = APICGetTpr(pVCpu, &uTpr, NULL, NULL); + if (RT_SUCCESS(rc)) + crX = uTpr >> 4; + else + crX = 0; + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */ + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + switch (iCrReg) + { + /* CR0/CR4 reads are subject to masking when in VMX non-root mode. */ + case 0: crX = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u); break; + case 4: crX = CPUMGetGuestVmxMaskedCr4(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr4Mask.u); break; + + case 3: + { + VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr3(pVCpu, iGReg, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + break; + } + } + } +#endif + + /* Store it. */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + *(uint64_t *)iemGRegRef(pVCpu, iGReg) = crX; + else + *(uint64_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)crX; + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements smsw GReg. + * + * @param iGReg The general register to store the CRx value in. + * @param enmEffOpSize The operand size. + */ +IEM_CIMPL_DEF_2(iemCImpl_smsw_reg, uint8_t, iGReg, uint8_t, enmEffOpSize) +{ + IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + uint64_t u64MaskedCr0; + if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + u64MaskedCr0 = pVCpu->cpum.GstCtx.cr0; + else + u64MaskedCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u); + uint64_t const u64GuestCr0 = u64MaskedCr0; +#else + uint64_t const u64GuestCr0 = pVCpu->cpum.GstCtx.cr0; +#endif + + switch (enmEffOpSize) + { + case IEMMODE_16BIT: + if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386) + *(uint16_t *)iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0; + else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386) + *(uint16_t *)iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0 | 0xffe0; + else + *(uint16_t *)iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0 | 0xfff0; + break; + + case IEMMODE_32BIT: + *(uint32_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)u64GuestCr0; + break; + + case IEMMODE_64BIT: + *(uint64_t *)iemGRegRef(pVCpu, iGReg) = u64GuestCr0; + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements smsw mem. + * + * @param iEffSeg The effective segment register to use with @a GCPtrMem. + * @param GCPtrEffDst Where to store the 16-bit CR0 value. + */ +IEM_CIMPL_DEF_2(iemCImpl_smsw_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst) +{ + IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + uint64_t u64MaskedCr0; + if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + u64MaskedCr0 = pVCpu->cpum.GstCtx.cr0; + else + u64MaskedCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u); + uint64_t const u64GuestCr0 = u64MaskedCr0; +#else + uint64_t const u64GuestCr0 = pVCpu->cpum.GstCtx.cr0; +#endif + + uint16_t u16Value; + if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386) + u16Value = (uint16_t)u64GuestCr0; + else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386) + u16Value = (uint16_t)u64GuestCr0 | 0xffe0; + else + u16Value = (uint16_t)u64GuestCr0 | 0xfff0; + + VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, u16Value); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return rcStrict; +} + + +/** + * Helper for mapping CR3 and PAE PDPEs for 'mov CRx,GReg'. + */ +#define IEM_MAP_PAE_PDPES_AT_CR3_RET(a_pVCpu, a_iCrReg, a_uCr3) \ + do \ + { \ + int const rcX = PGMGstMapPaePdpesAtCr3(a_pVCpu, a_uCr3); \ + if (RT_SUCCESS(rcX)) \ + { /* likely */ } \ + else \ + { \ + /* Either invalid PDPTEs or CR3 second-level translation failed. Raise #GP(0) either way. */ \ + Log(("iemCImpl_load_Cr%#x: Trying to load invalid PAE PDPEs\n", a_iCrReg)); \ + return iemRaiseGeneralProtectionFault0(a_pVCpu); \ + } \ + } while (0) + + +/** + * Used to implemented 'mov CRx,GReg' and 'lmsw r/m16'. + * + * @param iCrReg The CRx register to write (valid). + * @param uNewCrX The new value. + * @param enmAccessCrX The instruction that caused the CrX load. + * @param iGReg The general register in case of a 'mov CRx,GReg' + * instruction. + */ +IEM_CIMPL_DEF_4(iemCImpl_load_CrX, uint8_t, iCrReg, uint64_t, uNewCrX, IEMACCESSCRX, enmAccessCrX, uint8_t, iGReg) +{ + VBOXSTRICTRC rcStrict; + int rc; +#ifndef VBOX_WITH_NESTED_HWVIRT_SVM + RT_NOREF2(iGReg, enmAccessCrX); +#endif + + /* + * Try store it. + * Unfortunately, CPUM only does a tiny bit of the work. + */ + switch (iCrReg) + { + case 0: + { + /* + * Perform checks. + */ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); + + uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr0; + uint32_t const fValid = CPUMGetGuestCR0ValidMask(); + + /* ET is hardcoded on 486 and later. */ + if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_486) + uNewCrX |= X86_CR0_ET; + /* The 386 and 486 didn't #GP(0) on attempting to set reserved CR0 bits. ET was settable on 386. */ + else if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_486) + { + uNewCrX &= fValid; + uNewCrX |= X86_CR0_ET; + } + else + uNewCrX &= X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG | X86_CR0_ET; + + /* Check for reserved bits. */ + if (uNewCrX & ~(uint64_t)fValid) + { + Log(("Trying to set reserved CR0 bits: NewCR0=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Check for invalid combinations. */ + if ( (uNewCrX & X86_CR0_PG) + && !(uNewCrX & X86_CR0_PE) ) + { + Log(("Trying to set CR0.PG without CR0.PE\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if ( !(uNewCrX & X86_CR0_CD) + && (uNewCrX & X86_CR0_NW) ) + { + Log(("Trying to clear CR0.CD while leaving CR0.NW set\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if ( !(uNewCrX & X86_CR0_PG) + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE)) + { + Log(("Trying to clear CR0.PG while leaving CR4.PCID set\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Long mode consistency checks. */ + if ( (uNewCrX & X86_CR0_PG) + && !(uOldCrX & X86_CR0_PG) + && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) ) + { + if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)) + { + Log(("Trying to enabled long mode paging without CR4.PAE set\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long) + { + Log(("Trying to enabled long mode paging with a long CS descriptor loaded.\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* Check for bits that must remain set or cleared in VMX operation, + see Intel spec. 23.8 "Restrictions on VMX operation". */ + if (IEM_VMX_IS_ROOT_MODE(pVCpu)) + { + uint64_t const uCr0Fixed0 = iemVmxGetCr0Fixed0(pVCpu, IEM_VMX_IS_NON_ROOT_MODE(pVCpu)); + if ((uNewCrX & uCr0Fixed0) != uCr0Fixed0) + { + Log(("Trying to clear reserved CR0 bits in VMX operation: NewCr0=%#llx MB1=%#llx\n", uNewCrX, uCr0Fixed0)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + uint64_t const uCr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1; + if (uNewCrX & ~uCr0Fixed1) + { + Log(("Trying to set reserved CR0 bits in VMX operation: NewCr0=%#llx MB0=%#llx\n", uNewCrX, uCr0Fixed1)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } +#endif + + /* + * SVM nested-guest CR0 write intercepts. + */ + if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, iCrReg)) + { + Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR0, enmAccessCrX, iGReg); + } + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE)) + { + /* 'lmsw' intercepts regardless of whether the TS/MP bits are actually toggled. */ + if ( enmAccessCrX == IEMACCESSCRX_LMSW + || (uNewCrX & ~(X86_CR0_TS | X86_CR0_MP)) != (uOldCrX & ~(X86_CR0_TS | X86_CR0_MP))) + { + Assert(enmAccessCrX != IEMACCESSCRX_CLTS); + Log(("iemCImpl_load_Cr%#x: lmsw or bits other than TS/MP changed: Guest intercept -> #VMEXIT\n", iCrReg)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_CR0_SEL_WRITE, enmAccessCrX, iGReg); + } + } + + /* + * Change EFER.LMA if entering or leaving long mode. + */ + uint64_t NewEFER = pVCpu->cpum.GstCtx.msrEFER; + if ( (uNewCrX & X86_CR0_PG) != (uOldCrX & X86_CR0_PG) + && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) ) + { + if (uNewCrX & X86_CR0_PG) + NewEFER |= MSR_K6_EFER_LMA; + else + NewEFER &= ~MSR_K6_EFER_LMA; + + CPUMSetGuestEFER(pVCpu, NewEFER); + Assert(pVCpu->cpum.GstCtx.msrEFER == NewEFER); + } + + /* + * Inform PGM. + */ + if ( (uNewCrX & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE | X86_CR0_CD | X86_CR0_NW)) + != (uOldCrX & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE | X86_CR0_CD | X86_CR0_NW)) ) + { + if ( enmAccessCrX != IEMACCESSCRX_MOV_CRX + || !CPUMIsPaePagingEnabled(uNewCrX, pVCpu->cpum.GstCtx.cr4, NewEFER) + || CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))) + { /* likely */ } + else + IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, pVCpu->cpum.GstCtx.cr3); + rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */); + AssertRCReturn(rc, rc); + /* ignore informational status codes */ + } + + /* + * Change CR0. + */ + CPUMSetGuestCR0(pVCpu, uNewCrX); + Assert(pVCpu->cpum.GstCtx.cr0 == uNewCrX); + + rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER, + false /* fForce */); + break; + } + + /* + * CR2 can be changed without any restrictions. + */ + case 2: + { + if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 2)) + { + Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR2, enmAccessCrX, iGReg); + } + pVCpu->cpum.GstCtx.cr2 = uNewCrX; + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_CR2; + rcStrict = VINF_SUCCESS; + break; + } + + /* + * CR3 is relatively simple, although AMD and Intel have different + * accounts of how setting reserved bits are handled. We take intel's + * word for the lower bits and AMD's for the high bits (63:52). The + * lower reserved bits are ignored and left alone; OpenBSD 5.8 relies + * on this. + */ + /** @todo Testcase: Setting reserved bits in CR3, especially before + * enabling paging. */ + case 3: + { + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3); + + /* Bit 63 being clear in the source operand with PCIDE indicates no invalidations are required. */ + if ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE) + && (uNewCrX & RT_BIT_64(63))) + { + /** @todo r=ramshankar: avoiding a TLB flush altogether here causes Windows 10 + * SMP(w/o nested-paging) to hang during bootup on Skylake systems, see + * Intel spec. 4.10.4.1 "Operations that Invalidate TLBs and + * Paging-Structure Caches". */ + uNewCrX &= ~RT_BIT_64(63); + } + + /* Check / mask the value. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + /* See Intel spec. 27.2.2 "EPT Translation Mechanism" footnote. */ + uint64_t const fInvPhysMask = !CPUMIsGuestVmxEptPagingEnabledEx(IEM_GET_CTX(pVCpu)) + ? (UINT64_MAX << IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth) + : (~X86_CR3_EPT_PAGE_MASK & X86_PAGE_4K_BASE_MASK); +#else + uint64_t const fInvPhysMask = UINT64_C(0xfff0000000000000); +#endif + if (uNewCrX & fInvPhysMask) + { + /** @todo Should we raise this only for 64-bit mode like Intel claims? AMD is + * very vague in this area. As mentioned above, need testcase on real + * hardware... Sigh. */ + Log(("Trying to load CR3 with invalid high bits set: %#llx\n", uNewCrX)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + uint64_t fValid; + if ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE) + && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME)) + { + /** @todo Redundant? This value has already been validated above. */ + fValid = UINT64_C(0x000fffffffffffff); + } + else + fValid = UINT64_C(0xffffffff); + if (uNewCrX & ~fValid) + { + Log(("Automatically clearing reserved MBZ bits in CR3 load: NewCR3=%#llx ClearedBits=%#llx\n", + uNewCrX, uNewCrX & ~fValid)); + uNewCrX &= fValid; + } + + if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 3)) + { + Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR3, enmAccessCrX, iGReg); + } + + /* Inform PGM. */ + if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) + { + if ( !CPUMIsGuestInPAEModeEx(IEM_GET_CTX(pVCpu)) + || CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))) + { /* likely */ } + else + { + Assert(enmAccessCrX == IEMACCESSCRX_MOV_CRX); + IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, uNewCrX); + } + rc = PGMFlushTLB(pVCpu, uNewCrX, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE)); + AssertRCReturn(rc, rc); + /* ignore informational status codes */ + } + + /* Make the change. */ + rc = CPUMSetGuestCR3(pVCpu, uNewCrX); + AssertRCSuccessReturn(rc, rc); + + rcStrict = VINF_SUCCESS; + break; + } + + /* + * CR4 is a bit more tedious as there are bits which cannot be cleared + * under some circumstances and such. + */ + case 4: + { + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4); + uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr4; + + /* Reserved bits. */ + uint32_t const fValid = CPUMGetGuestCR4ValidMask(pVCpu->CTX_SUFF(pVM)); + if (uNewCrX & ~(uint64_t)fValid) + { + Log(("Trying to set reserved CR4 bits: NewCR4=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + bool const fPcide = !(uOldCrX & X86_CR4_PCIDE) && (uNewCrX & X86_CR4_PCIDE); + bool const fLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)); + + /* PCIDE check. */ + if ( fPcide + && ( !fLongMode + || (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff)))) + { + Log(("Trying to set PCIDE with invalid PCID or outside long mode. Pcid=%#x\n", (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff)))); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* PAE check. */ + if ( fLongMode + && (uOldCrX & X86_CR4_PAE) + && !(uNewCrX & X86_CR4_PAE)) + { + Log(("Trying to set clear CR4.PAE while long mode is active\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 4)) + { + Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR4, enmAccessCrX, iGReg); + } + + /* Check for bits that must remain set or cleared in VMX operation, + see Intel spec. 23.8 "Restrictions on VMX operation". */ + if (IEM_VMX_IS_ROOT_MODE(pVCpu)) + { + uint64_t const uCr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0; + if ((uNewCrX & uCr4Fixed0) != uCr4Fixed0) + { + Log(("Trying to clear reserved CR4 bits in VMX operation: NewCr4=%#llx MB1=%#llx\n", uNewCrX, uCr4Fixed0)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + uint64_t const uCr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1; + if (uNewCrX & ~uCr4Fixed1) + { + Log(("Trying to set reserved CR4 bits in VMX operation: NewCr4=%#llx MB0=%#llx\n", uNewCrX, uCr4Fixed1)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + + /* + * Notify PGM. + */ + if ((uNewCrX ^ uOldCrX) & (X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_PCIDE /* | X86_CR4_SMEP */)) + { + if ( !CPUMIsPaePagingEnabled(pVCpu->cpum.GstCtx.cr0, uNewCrX, pVCpu->cpum.GstCtx.msrEFER) + || CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))) + { /* likely */ } + else + { + Assert(enmAccessCrX == IEMACCESSCRX_MOV_CRX); + IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, pVCpu->cpum.GstCtx.cr3); + } + rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */); + AssertRCReturn(rc, rc); + /* ignore informational status codes */ + } + + /* + * Change it. + */ + rc = CPUMSetGuestCR4(pVCpu, uNewCrX); + AssertRCSuccessReturn(rc, rc); + Assert(pVCpu->cpum.GstCtx.cr4 == uNewCrX); + + rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER, + false /* fForce */); + break; + } + + /* + * CR8 maps to the APIC TPR. + */ + case 8: + { + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR); + if (uNewCrX & ~(uint64_t)0xf) + { + Log(("Trying to set reserved CR8 bits (%#RX64)\n", uNewCrX)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW)) + { + /* + * If the Mov-to-CR8 doesn't cause a VM-exit, bits 0:3 of the source operand + * is copied to bits 7:4 of the VTPR. Bits 0:3 and bits 31:8 of the VTPR are + * cleared. Following this the processor performs TPR virtualization. + * + * However, we should not perform TPR virtualization immediately here but + * after this instruction has completed. + * + * See Intel spec. 29.3 "Virtualizing CR8-based TPR Accesses" + * See Intel spec. 27.1 "Architectural State Before A VM-exit" + */ + uint32_t const uTpr = (uNewCrX & 0xf) << 4; + Log(("iemCImpl_load_Cr%#x: Virtualizing TPR (%#x) write\n", iCrReg, uTpr)); + iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_TPR, uTpr); + iemVmxVirtApicSetPendingWrite(pVCpu, XAPIC_OFF_TPR); + rcStrict = VINF_SUCCESS; + break; + } +#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))) + { + if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 8)) + { + Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR8, enmAccessCrX, iGReg); + } + + pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.IntCtrl.n.u8VTPR = uNewCrX; + if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu))) + { + rcStrict = VINF_SUCCESS; + break; + } + } +#endif + uint8_t const u8Tpr = (uint8_t)uNewCrX << 4; + APICSetTpr(pVCpu, u8Tpr); + rcStrict = VINF_SUCCESS; + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */ + } + + /* + * Advance the RIP on success. + */ + if (RT_SUCCESS(rcStrict)) + { + if (rcStrict != VINF_SUCCESS) + iemSetPassUpStatus(pVCpu, rcStrict); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + return rcStrict; +} + + +/** + * Implements mov CRx,GReg. + * + * @param iCrReg The CRx register to write (valid). + * @param iGReg The general register to load the CRx value from. + */ +IEM_CIMPL_DEF_2(iemCImpl_mov_Cd_Rd, uint8_t, iCrReg, uint8_t, iGReg) +{ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + + /* + * Read the new value from the source register and call common worker. + */ + uint64_t uNewCrX; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + uNewCrX = iemGRegFetchU64(pVCpu, iGReg); + else + uNewCrX = iemGRegFetchU32(pVCpu, iGReg); + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VBOXSTRICTRC rcStrict = VINF_VMX_INTERCEPT_NOT_ACTIVE; + switch (iCrReg) + { + case 0: + case 4: rcStrict = iemVmxVmexitInstrMovToCr0Cr4(pVCpu, iCrReg, &uNewCrX, iGReg, cbInstr); break; + case 3: rcStrict = iemVmxVmexitInstrMovToCr3(pVCpu, uNewCrX, iGReg, cbInstr); break; + case 8: rcStrict = iemVmxVmexitInstrMovToCr8(pVCpu, iGReg, cbInstr); break; + } + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#endif + + return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, iCrReg, uNewCrX, IEMACCESSCRX_MOV_CRX, iGReg); +} + + +/** + * Implements 'LMSW r/m16' + * + * @param u16NewMsw The new value. + * @param GCPtrEffDst The guest-linear address of the source operand in case + * of a memory operand. For register operand, pass + * NIL_RTGCPTR. + */ +IEM_CIMPL_DEF_2(iemCImpl_lmsw, uint16_t, u16NewMsw, RTGCPTR, GCPtrEffDst) +{ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* Check nested-guest VMX intercept and get updated MSW if there's no VM-exit. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VBOXSTRICTRC rcStrict = iemVmxVmexitInstrLmsw(pVCpu, pVCpu->cpum.GstCtx.cr0, &u16NewMsw, GCPtrEffDst, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#else + RT_NOREF_PV(GCPtrEffDst); +#endif + + /* + * Compose the new CR0 value and call common worker. + */ + uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0 & ~(X86_CR0_MP | X86_CR0_EM | X86_CR0_TS); + uNewCr0 |= u16NewMsw & (X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS); + return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /*cr*/ 0, uNewCr0, IEMACCESSCRX_LMSW, UINT8_MAX /* iGReg */); +} + + +/** + * Implements 'CLTS'. + */ +IEM_CIMPL_DEF_0(iemCImpl_clts) +{ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); + uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0; + uNewCr0 &= ~X86_CR0_TS; + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VBOXSTRICTRC rcStrict = iemVmxVmexitInstrClts(pVCpu, cbInstr); + if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR) + uNewCr0 |= (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS); + else if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#endif + + return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /*cr*/ 0, uNewCr0, IEMACCESSCRX_CLTS, UINT8_MAX /* iGReg */); +} + + +/** + * Implements mov GReg,DRx. + * + * @param iGReg The general register to store the DRx value in. + * @param iDrReg The DRx register to read (0-7). + */ +IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Dd, uint8_t, iGReg, uint8_t, iDrReg) +{ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* + * Check nested-guest VMX intercept. + * Unlike most other intercepts, the Mov DRx intercept takes preceedence + * over CPL and CR4.DE and even DR4/DR5 checks. + * + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_FROM_DRX, iDrReg, iGReg, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#endif + + /* + * Check preconditions. + */ + /* Raise GPs. */ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7); + + /** @todo \#UD in outside ring-0 too? */ + if (iDrReg == 4 || iDrReg == 5) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR4); + if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE) + { + Log(("mov r%u,dr%u: CR4.DE=1 -> #GP(0)\n", iGReg, iDrReg)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + iDrReg += 2; + } + + /* Raise #DB if general access detect is enabled. */ + if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD) + { + Log(("mov r%u,dr%u: DR7.GD=1 -> #DB\n", iGReg, iDrReg)); + return iemRaiseDebugException(pVCpu); + } + + /* + * Read the debug register and store it in the specified general register. + */ + uint64_t drX; + switch (iDrReg) + { + case 0: + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + drX = pVCpu->cpum.GstCtx.dr[0]; + break; + case 1: + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + drX = pVCpu->cpum.GstCtx.dr[1]; + break; + case 2: + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + drX = pVCpu->cpum.GstCtx.dr[2]; + break; + case 3: + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + drX = pVCpu->cpum.GstCtx.dr[3]; + break; + case 6: + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6); + drX = pVCpu->cpum.GstCtx.dr[6]; + drX |= X86_DR6_RA1_MASK; + drX &= ~X86_DR6_RAZ_MASK; + break; + case 7: + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7); + drX = pVCpu->cpum.GstCtx.dr[7]; + drX |=X86_DR7_RA1_MASK; + drX &= ~X86_DR7_RAZ_MASK; + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* caller checks */ + } + + /** @todo SVM nested-guest intercept for DR8-DR15? */ + /* + * Check for any SVM nested-guest intercepts for the DRx read. + */ + if (IEM_SVM_IS_READ_DR_INTERCEPT_SET(pVCpu, iDrReg)) + { + Log(("mov r%u,dr%u: Guest intercept -> #VMEXIT\n", iGReg, iDrReg)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_READ_DR0 + (iDrReg & 0xf), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */); + } + + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + *(uint64_t *)iemGRegRef(pVCpu, iGReg) = drX; + else + *(uint64_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)drX; + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements mov DRx,GReg. + * + * @param iDrReg The DRx register to write (valid). + * @param iGReg The general register to load the DRx value from. + */ +IEM_CIMPL_DEF_2(iemCImpl_mov_Dd_Rd, uint8_t, iDrReg, uint8_t, iGReg) +{ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* + * Check nested-guest VMX intercept. + * Unlike most other intercepts, the Mov DRx intercept takes preceedence + * over CPL and CR4.DE and even DR4/DR5 checks. + * + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_TO_DRX, iDrReg, iGReg, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#endif + + /* + * Check preconditions. + */ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7); + + if (iDrReg == 4 || iDrReg == 5) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR4); + if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE) + { + Log(("mov dr%u,r%u: CR4.DE=1 -> #GP(0)\n", iDrReg, iGReg)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + iDrReg += 2; + } + + /* Raise #DB if general access detect is enabled. */ + /** @todo is \#DB/DR7.GD raised before any reserved high bits in DR7/DR6 + * \#GP? */ + if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD) + { + Log(("mov dr%u,r%u: DR7.GD=1 -> #DB\n", iDrReg, iGReg)); + return iemRaiseDebugException(pVCpu); + } + + /* + * Read the new value from the source register. + */ + uint64_t uNewDrX; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + uNewDrX = iemGRegFetchU64(pVCpu, iGReg); + else + uNewDrX = iemGRegFetchU32(pVCpu, iGReg); + + /* + * Adjust it. + */ + switch (iDrReg) + { + case 0: + case 1: + case 2: + case 3: + /* nothing to adjust */ + break; + + case 6: + if (uNewDrX & X86_DR6_MBZ_MASK) + { + Log(("mov dr%u,%#llx: DR6 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + uNewDrX |= X86_DR6_RA1_MASK; + uNewDrX &= ~X86_DR6_RAZ_MASK; + break; + + case 7: + if (uNewDrX & X86_DR7_MBZ_MASK) + { + Log(("mov dr%u,%#llx: DR7 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + uNewDrX |= X86_DR7_RA1_MASK; + uNewDrX &= ~X86_DR7_RAZ_MASK; + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + /** @todo SVM nested-guest intercept for DR8-DR15? */ + /* + * Check for any SVM nested-guest intercepts for the DRx write. + */ + if (IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(pVCpu, iDrReg)) + { + Log2(("mov dr%u,r%u: Guest intercept -> #VMEXIT\n", iDrReg, iGReg)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_DR0 + (iDrReg & 0xf), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */); + } + + /* + * Do the actual setting. + */ + if (iDrReg < 4) + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3); + else if (iDrReg == 6) + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6); + + int rc = CPUMSetGuestDRx(pVCpu, iDrReg, uNewDrX); + AssertRCSuccessReturn(rc, RT_SUCCESS_NP(rc) ? VERR_IEM_IPE_1 : rc); + + /* + * Re-init hardware breakpoint summary if it was DR7 that got changed. + */ + if (iDrReg == 7) + { + pVCpu->iem.s.fPendingInstructionBreakpoints = false; + pVCpu->iem.s.fPendingDataBreakpoints = false; + pVCpu->iem.s.fPendingIoBreakpoints = false; + iemInitPendingBreakpointsSlow(pVCpu); + } + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements mov GReg,TRx. + * + * @param iGReg The general register to store the + * TRx value in. + * @param iTrReg The TRx register to read (6/7). + */ +IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Td, uint8_t, iGReg, uint8_t, iTrReg) +{ + /* + * Check preconditions. NB: This instruction is 386/486 only. + */ + + /* Raise GPs. */ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + + if (iTrReg < 6 || iTrReg > 7) + { + /** @todo Do Intel CPUs reject this or are the TRs aliased? */ + Log(("mov r%u,tr%u: invalid register -> #GP(0)\n", iGReg, iTrReg)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Read the test register and store it in the specified general register. + * This is currently a dummy implementation that only exists to satisfy + * old debuggers like WDEB386 or OS/2 KDB which unconditionally read the + * TR6/TR7 registers. Software which actually depends on the TR values + * (different on 386/486) is exceedingly rare. + */ + uint64_t trX; + switch (iTrReg) + { + case 6: + trX = 0; /* Currently a dummy. */ + break; + case 7: + trX = 0; /* Currently a dummy. */ + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */ + } + + *(uint64_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)trX; + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements mov TRx,GReg. + * + * @param iTrReg The TRx register to write (valid). + * @param iGReg The general register to load the TRx + * value from. + */ +IEM_CIMPL_DEF_2(iemCImpl_mov_Td_Rd, uint8_t, iTrReg, uint8_t, iGReg) +{ + /* + * Check preconditions. NB: This instruction is 386/486 only. + */ + + /* Raise GPs. */ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + + if (iTrReg < 6 || iTrReg > 7) + { + /** @todo Do Intel CPUs reject this or are the TRs aliased? */ + Log(("mov r%u,tr%u: invalid register -> #GP(0)\n", iGReg, iTrReg)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Read the new value from the source register. + */ + uint64_t uNewTrX; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + uNewTrX = iemGRegFetchU64(pVCpu, iGReg); + else + uNewTrX = iemGRegFetchU32(pVCpu, iGReg); + + /* + * Here we would do the actual setting if this weren't a dummy implementation. + * This is currently a dummy implementation that only exists to prevent + * old debuggers like WDEB386 or OS/2 KDB from crashing. + */ + RT_NOREF(uNewTrX); + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'INVLPG m'. + * + * @param GCPtrPage The effective address of the page to invalidate. + * @remarks Updates the RIP. + */ +IEM_CIMPL_DEF_1(iemCImpl_invlpg, RTGCPTR, GCPtrPage) +{ + /* ring-0 only. */ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER); + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT)) + { + Log(("invlpg: Guest intercept (%RGp) -> VM-exit\n", GCPtrPage)); + return iemVmxVmexitInstrInvlpg(pVCpu, GCPtrPage, cbInstr); + } +#endif + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INVLPG)) + { + Log(("invlpg: Guest intercept (%RGp) -> #VMEXIT\n", GCPtrPage)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_INVLPG, + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? GCPtrPage : 0, 0 /* uExitInfo2 */); + } + + int rc = PGMInvalidatePage(pVCpu, GCPtrPage); + if (rc == VINF_SUCCESS) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + if (rc == VINF_PGM_SYNC_CR3) + { + iemSetPassUpStatus(pVCpu, rc); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc)); + Log(("PGMInvalidatePage(%RGv) -> %Rrc\n", GCPtrPage, rc)); + return rc; +} + + +/** + * Implements INVPCID. + * + * @param iEffSeg The segment of the invpcid descriptor. + * @param GCPtrInvpcidDesc The address of invpcid descriptor. + * @param uInvpcidType The invalidation type. + * @remarks Updates the RIP. + */ +IEM_CIMPL_DEF_3(iemCImpl_invpcid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvpcidDesc, uint64_t, uInvpcidType) +{ + /* + * Check preconditions. + */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fInvpcid) + return iemRaiseUndefinedOpcode(pVCpu); + + /* When in VMX non-root mode and INVPCID is not enabled, it results in #UD. */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_INVPCID)) + { + Log(("invpcid: Not enabled for nested-guest execution -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + + if (pVCpu->iem.s.uCpl != 0) + { + Log(("invpcid: CPL != 0 -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if (IEM_IS_V86_MODE(pVCpu)) + { + Log(("invpcid: v8086 mode -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Check nested-guest intercept. + * + * INVPCID causes a VM-exit if "enable INVPCID" and "INVLPG exiting" are + * both set. We have already checked the former earlier in this function. + * + * CPL and virtual-8086 mode checks take priority over this VM-exit. + * See Intel spec. "25.1.1 Relative Priority of Faults and VM Exits". + */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT)) + { + Log(("invpcid: Guest intercept -> #VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_INVPCID, VMXINSTRID_NONE, cbInstr); + } + + if (uInvpcidType > X86_INVPCID_TYPE_MAX_VALID) + { + Log(("invpcid: invalid/unrecognized invpcid type %#RX64 -> #GP(0)\n", uInvpcidType)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER); + + /* + * Fetch the invpcid descriptor from guest memory. + */ + RTUINT128U uDesc; + VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInvpcidDesc); + if (rcStrict == VINF_SUCCESS) + { + /* + * Validate the descriptor. + */ + if (uDesc.s.Lo > 0xfff) + { + Log(("invpcid: reserved bits set in invpcid descriptor %#RX64 -> #GP(0)\n", uDesc.s.Lo)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + RTGCUINTPTR64 const GCPtrInvAddr = uDesc.s.Hi; + uint8_t const uPcid = uDesc.s.Lo & UINT64_C(0xfff); + uint32_t const uCr4 = pVCpu->cpum.GstCtx.cr4; + uint64_t const uCr3 = pVCpu->cpum.GstCtx.cr3; + switch (uInvpcidType) + { + case X86_INVPCID_TYPE_INDV_ADDR: + { + if (!IEM_IS_CANONICAL(GCPtrInvAddr)) + { + Log(("invpcid: invalidation address %#RGP is not canonical -> #GP(0)\n", GCPtrInvAddr)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + if ( !(uCr4 & X86_CR4_PCIDE) + && uPcid != 0) + { + Log(("invpcid: invalid pcid %#x\n", uPcid)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Invalidate mappings for the linear address tagged with PCID except global translations. */ + PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */); + break; + } + + case X86_INVPCID_TYPE_SINGLE_CONTEXT: + { + if ( !(uCr4 & X86_CR4_PCIDE) + && uPcid != 0) + { + Log(("invpcid: invalid pcid %#x\n", uPcid)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + /* Invalidate all mappings associated with PCID except global translations. */ + PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */); + break; + } + + case X86_INVPCID_TYPE_ALL_CONTEXT_INCL_GLOBAL: + { + PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */); + break; + } + + case X86_INVPCID_TYPE_ALL_CONTEXT_EXCL_GLOBAL: + { + PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */); + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + return rcStrict; +} + + +/** + * Implements INVD. + */ +IEM_CIMPL_DEF_0(iemCImpl_invd) +{ + if (pVCpu->iem.s.uCpl != 0) + { + Log(("invd: CPL != 0 -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_INVD, cbInstr); + + IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_INVD, SVM_EXIT_INVD, 0, 0); + + /* We currently take no action here. */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements WBINVD. + */ +IEM_CIMPL_DEF_0(iemCImpl_wbinvd) +{ + if (pVCpu->iem.s.uCpl != 0) + { + Log(("wbinvd: CPL != 0 -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WBINVD, cbInstr); + + IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_WBINVD, SVM_EXIT_WBINVD, 0, 0); + + /* We currently take no action here. */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** Opcode 0x0f 0xaa. */ +IEM_CIMPL_DEF_0(iemCImpl_rsm) +{ + IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_RSM, SVM_EXIT_RSM, 0, 0); + NOREF(cbInstr); + return iemRaiseUndefinedOpcode(pVCpu); +} + + +/** + * Implements RDTSC. + */ +IEM_CIMPL_DEF_0(iemCImpl_rdtsc) +{ + /* + * Check preconditions. + */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fTsc) + return iemRaiseUndefinedOpcode(pVCpu); + + if (pVCpu->iem.s.uCpl != 0) + { + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4); + if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD) + { + Log(("rdtsc: CR4.TSD and CPL=%u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT)) + { + Log(("rdtsc: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSC, cbInstr); + } + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSC)) + { + Log(("rdtsc: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSC, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Do the job. + */ + uint64_t uTicks = TMCpuTickGet(pVCpu); +#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) + uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks); +#endif + pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks); + pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks); + pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX); /* For IEMExecDecodedRdtsc. */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements RDTSC. + */ +IEM_CIMPL_DEF_0(iemCImpl_rdtscp) +{ + /* + * Check preconditions. + */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fRdTscP) + return iemRaiseUndefinedOpcode(pVCpu); + + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_RDTSCP)) + { + Log(("rdtscp: Not enabled for VMX non-root mode -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + + if (pVCpu->iem.s.uCpl != 0) + { + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4); + if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD) + { + Log(("rdtscp: CR4.TSD and CPL=%u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT)) + { + Log(("rdtscp: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSCP, cbInstr); + } + else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSCP)) + { + Log(("rdtscp: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSCP, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Do the job. + * Query the MSR first in case of trips to ring-3. + */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TSC_AUX); + VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pVCpu->cpum.GstCtx.rcx); + if (rcStrict == VINF_SUCCESS) + { + /* Low dword of the TSC_AUX msr only. */ + pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff); + + uint64_t uTicks = TMCpuTickGet(pVCpu); +#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) + uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks); +#endif + pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks); + pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks); + pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX); /* For IEMExecDecodedRdtscp. */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + return rcStrict; +} + + +/** + * Implements RDPMC. + */ +IEM_CIMPL_DEF_0(iemCImpl_rdpmc) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4); + + if ( pVCpu->iem.s.uCpl != 0 + && !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCE)) + return iemRaiseGeneralProtectionFault0(pVCpu); + + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDPMC_EXIT)) + { + Log(("rdpmc: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDPMC, cbInstr); + } + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDPMC)) + { + Log(("rdpmc: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDPMC, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /** @todo Emulate performance counters, for now just return 0. */ + pVCpu->cpum.GstCtx.rax = 0; + pVCpu->cpum.GstCtx.rdx = 0; + pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX); + /** @todo We should trigger a \#GP here if the CPU doesn't support the index in + * ecx but see @bugref{3472}! */ + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements RDMSR. + */ +IEM_CIMPL_DEF_0(iemCImpl_rdmsr) +{ + /* + * Check preconditions. + */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr) + return iemRaiseUndefinedOpcode(pVCpu); + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + + /* + * Check nested-guest intercepts. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_RDMSR, pVCpu->cpum.GstCtx.ecx)) + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDMSR, cbInstr); + } +#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT)) + { + VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, pVCpu->cpum.GstCtx.ecx, false /* fWrite */); + if (rcStrict == VINF_SVM_VMEXIT) + return VINF_SUCCESS; + if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE) + { + Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", pVCpu->cpum.GstCtx.ecx, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } +#endif + + /* + * Do the job. + */ + RTUINT64U uValue; + /** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS); + + VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pVCpu->cpum.GstCtx.ecx, &uValue.u); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rax = uValue.s.Lo; + pVCpu->cpum.GstCtx.rdx = uValue.s.Hi; + pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX); + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + +#ifndef IN_RING3 + /* Deferred to ring-3. */ + if (rcStrict == VINF_CPUM_R3_MSR_READ) + { + Log(("IEM: rdmsr(%#x) -> ring-3\n", pVCpu->cpum.GstCtx.ecx)); + return rcStrict; + } +#endif + + /* Often a unimplemented MSR or MSR bit, so worth logging. */ + if (pVCpu->iem.s.cLogRelRdMsr < 32) + { + pVCpu->iem.s.cLogRelRdMsr++; + LogRel(("IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx)); + } + else + Log(( "IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx)); + AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS); + return iemRaiseGeneralProtectionFault0(pVCpu); +} + + +/** + * Implements WRMSR. + */ +IEM_CIMPL_DEF_0(iemCImpl_wrmsr) +{ + /* + * Check preconditions. + */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr) + return iemRaiseUndefinedOpcode(pVCpu); + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + + RTUINT64U uValue; + uValue.s.Lo = pVCpu->cpum.GstCtx.eax; + uValue.s.Hi = pVCpu->cpum.GstCtx.edx; + + uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; + + /** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS); + + /* + * Check nested-guest intercepts. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_WRMSR, idMsr)) + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WRMSR, cbInstr); + } +#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT)) + { + VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, idMsr, true /* fWrite */); + if (rcStrict == VINF_SVM_VMEXIT) + return VINF_SUCCESS; + if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE) + { + Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", idMsr, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } +#endif + + /* + * Do the job. + */ + VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, idMsr, uValue.u); + if (rcStrict == VINF_SUCCESS) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + +#ifndef IN_RING3 + /* Deferred to ring-3. */ + if (rcStrict == VINF_CPUM_R3_MSR_WRITE) + { + Log(("IEM: wrmsr(%#x) -> ring-3\n", idMsr)); + return rcStrict; + } +#endif + + /* Often a unimplemented MSR or MSR bit, so worth logging. */ + if (pVCpu->iem.s.cLogRelWrMsr < 32) + { + pVCpu->iem.s.cLogRelWrMsr++; + LogRel(("IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo)); + } + else + Log(( "IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo)); + AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS); + return iemRaiseGeneralProtectionFault0(pVCpu); +} + + +/** + * Implements 'IN eAX, port'. + * + * @param u16Port The source port. + * @param fImm Whether the port was specified through an immediate operand + * or the implicit DX register. + * @param cbReg The register size. + */ +IEM_CIMPL_DEF_3(iemCImpl_in, uint16_t, u16Port, bool, fImm, uint8_t, cbReg) +{ + /* + * CPL check + */ + VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Check VMX nested-guest IO intercept. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_IN, u16Port, fImm, cbReg, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#else + RT_NOREF(fImm); +#endif + + /* + * Check SVM nested-guest IO intercept. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT)) + { + uint8_t cAddrSizeBits; + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: cAddrSizeBits = 16; break; + case IEMMODE_32BIT: cAddrSizeBits = 32; break; + case IEMMODE_64BIT: cAddrSizeBits = 64; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_IN, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */, + false /* fRep */, false /* fStrIo */, cbInstr); + if (rcStrict == VINF_SVM_VMEXIT) + return VINF_SUCCESS; + if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE) + { + Log(("iemCImpl_in: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } +#endif + + /* + * Perform the I/O. + */ + PVMCC const pVM = pVCpu->CTX_SUFF(pVM); + uint32_t u32Value = 0; + rcStrict = IOMIOPortRead(pVM, pVCpu, u16Port, &u32Value, cbReg); + if (IOM_SUCCESS(rcStrict)) + { + switch (cbReg) + { + case 1: pVCpu->cpum.GstCtx.al = (uint8_t)u32Value; break; + case 2: pVCpu->cpum.GstCtx.ax = (uint16_t)u32Value; break; + case 4: pVCpu->cpum.GstCtx.rax = u32Value; break; + default: AssertFailedReturn(VERR_IEM_IPE_3); + } + + pVCpu->iem.s.cPotentialExits++; + if (rcStrict != VINF_SUCCESS) + iemSetPassUpStatus(pVCpu, rcStrict); + + /* + * Check for I/O breakpoints before we complete the instruction. + */ + uint32_t const fDr7 = pVCpu->cpum.GstCtx.dr[7]; + if (RT_UNLIKELY( ( ( (fDr7 & X86_DR7_ENABLED_MASK) + && X86_DR7_ANY_RW_IO(fDr7) + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE)) + || pVM->dbgf.ro.cEnabledHwIoBreakpoints > 0) + && rcStrict == VINF_SUCCESS)) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR6); + pVCpu->cpum.GstCtx.eflags.uBoth |= DBGFBpCheckIo2(pVM, pVCpu, u16Port, cbReg); + } + + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + return rcStrict; +} + + +/** + * Implements 'IN eAX, DX'. + * + * @param cbReg The register size. + */ +IEM_CIMPL_DEF_1(iemCImpl_in_eAX_DX, uint8_t, cbReg) +{ + return IEM_CIMPL_CALL_3(iemCImpl_in, pVCpu->cpum.GstCtx.dx, false /* fImm */, cbReg); +} + + +/** + * Implements 'OUT port, eAX'. + * + * @param u16Port The destination port. + * @param fImm Whether the port was specified through an immediate operand + * or the implicit DX register. + * @param cbReg The register size. + */ +IEM_CIMPL_DEF_3(iemCImpl_out, uint16_t, u16Port, bool, fImm, uint8_t, cbReg) +{ + /* + * CPL check + */ + VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Check VMX nested-guest I/O intercept. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_OUT, u16Port, fImm, cbReg, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#else + RT_NOREF(fImm); +#endif + + /* + * Check SVM nested-guest I/O intercept. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT)) + { + uint8_t cAddrSizeBits; + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: cAddrSizeBits = 16; break; + case IEMMODE_32BIT: cAddrSizeBits = 32; break; + case IEMMODE_64BIT: cAddrSizeBits = 64; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_OUT, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */, + false /* fRep */, false /* fStrIo */, cbInstr); + if (rcStrict == VINF_SVM_VMEXIT) + return VINF_SUCCESS; + if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE) + { + Log(("iemCImpl_out: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } +#endif + + /* + * Perform the I/O. + */ + PVMCC const pVM = pVCpu->CTX_SUFF(pVM); + uint32_t u32Value; + switch (cbReg) + { + case 1: u32Value = pVCpu->cpum.GstCtx.al; break; + case 2: u32Value = pVCpu->cpum.GstCtx.ax; break; + case 4: u32Value = pVCpu->cpum.GstCtx.eax; break; + default: AssertFailedReturn(VERR_IEM_IPE_4); + } + rcStrict = IOMIOPortWrite(pVM, pVCpu, u16Port, u32Value, cbReg); + if (IOM_SUCCESS(rcStrict)) + { + pVCpu->iem.s.cPotentialExits++; + if (rcStrict != VINF_SUCCESS) + iemSetPassUpStatus(pVCpu, rcStrict); + + /* + * Check for I/O breakpoints before we complete the instruction. + */ + uint32_t const fDr7 = pVCpu->cpum.GstCtx.dr[7]; + if (RT_UNLIKELY( ( ( (fDr7 & X86_DR7_ENABLED_MASK) + && X86_DR7_ANY_RW_IO(fDr7) + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE)) + || pVM->dbgf.ro.cEnabledHwIoBreakpoints > 0) + && rcStrict == VINF_SUCCESS)) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR6); + pVCpu->cpum.GstCtx.eflags.uBoth |= DBGFBpCheckIo2(pVM, pVCpu, u16Port, cbReg); + } + + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + return rcStrict; +} + + +/** + * Implements 'OUT DX, eAX'. + * + * @param cbReg The register size. + */ +IEM_CIMPL_DEF_1(iemCImpl_out_DX_eAX, uint8_t, cbReg) +{ + return IEM_CIMPL_CALL_3(iemCImpl_out, pVCpu->cpum.GstCtx.dx, false /* fImm */, cbReg); +} + + +/** + * Implements 'CLI'. + */ +IEM_CIMPL_DEF_0(iemCImpl_cli) +{ + uint32_t fEfl = IEMMISC_GET_EFL(pVCpu); +#ifdef LOG_ENABLED + uint32_t const fEflOld = fEfl; +#endif + + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4); + if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) + { + uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl); + if (!(fEfl & X86_EFL_VM)) + { + if (pVCpu->iem.s.uCpl <= uIopl) + fEfl &= ~X86_EFL_IF; + else if ( pVCpu->iem.s.uCpl == 3 + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI) ) + fEfl &= ~X86_EFL_VIF; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + } + /* V8086 */ + else if (uIopl == 3) + fEfl &= ~X86_EFL_IF; + else if ( uIopl < 3 + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) ) + fEfl &= ~X86_EFL_VIF; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + } + /* real mode */ + else + fEfl &= ~X86_EFL_IF; + + /* Commit. */ + IEMMISC_SET_EFL(pVCpu, fEfl); + VBOXSTRICTRC const rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + Log2(("CLI: %#x -> %#x\n", fEflOld, fEfl)); + return rcStrict; +} + + +/** + * Implements 'STI'. + */ +IEM_CIMPL_DEF_0(iemCImpl_sti) +{ + uint32_t fEfl = IEMMISC_GET_EFL(pVCpu); + uint32_t const fEflOld = fEfl; + + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4); + if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) + { + uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl); + if (!(fEfl & X86_EFL_VM)) + { + if (pVCpu->iem.s.uCpl <= uIopl) + fEfl |= X86_EFL_IF; + else if ( pVCpu->iem.s.uCpl == 3 + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI) + && !(fEfl & X86_EFL_VIP) ) + fEfl |= X86_EFL_VIF; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + } + /* V8086 */ + else if (uIopl == 3) + fEfl |= X86_EFL_IF; + else if ( uIopl < 3 + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) + && !(fEfl & X86_EFL_VIP) ) + fEfl |= X86_EFL_VIF; + else + return iemRaiseGeneralProtectionFault0(pVCpu); + } + /* real mode */ + else + fEfl |= X86_EFL_IF; + + /* + * Commit. + * + * Note! Setting the shadow interrupt flag must be done after RIP updating. + */ + IEMMISC_SET_EFL(pVCpu, fEfl); + VBOXSTRICTRC const rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + if (!(fEflOld & X86_EFL_IF) && (fEfl & X86_EFL_IF)) + { + /** @todo only set it the shadow flag if it was clear before? */ + CPUMSetInInterruptShadowSti(&pVCpu->cpum.GstCtx); + } + Log2(("STI: %#x -> %#x\n", fEflOld, fEfl)); + return rcStrict; +} + + +/** + * Implements 'HLT'. + */ +IEM_CIMPL_DEF_0(iemCImpl_hlt) +{ + if (pVCpu->iem.s.uCpl != 0) + return iemRaiseGeneralProtectionFault0(pVCpu); + + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_HLT_EXIT)) + { + Log2(("hlt: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_HLT, cbInstr); + } + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_HLT)) + { + Log2(("hlt: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_HLT, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /** @todo finish: This ASSUMES that iemRegAddToRipAndFinishingClearingRF won't + * be returning any status codes relating to non-guest events being raised, as + * we'll mess up the guest HALT otherwise. */ + VBOXSTRICTRC rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + if (rcStrict == VINF_SUCCESS) + rcStrict = VINF_EM_HALT; + return rcStrict; +} + + +/** + * Implements 'MONITOR'. + */ +IEM_CIMPL_DEF_1(iemCImpl_monitor, uint8_t, iEffSeg) +{ + /* + * Permission checks. + */ + if (pVCpu->iem.s.uCpl != 0) + { + Log2(("monitor: CPL != 0\n")); + return iemRaiseUndefinedOpcode(pVCpu); /** @todo MSR[0xC0010015].MonMwaitUserEn if we care. */ + } + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait) + { + Log2(("monitor: Not in CPUID\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + + /* + * Check VMX guest-intercept. + * This should be considered a fault-like VM-exit. + * See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits". + */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MONITOR_EXIT)) + { + Log2(("monitor: Guest intercept -> #VMEXIT\n")); + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_MONITOR, cbInstr); + } + + /* + * Gather the operands and validate them. + */ + RTGCPTR GCPtrMem = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax; + uint32_t uEcx = pVCpu->cpum.GstCtx.ecx; + uint32_t uEdx = pVCpu->cpum.GstCtx.edx; +/** @todo Test whether EAX or ECX is processed first, i.e. do we get \#PF or + * \#GP first. */ + if (uEcx != 0) + { + Log2(("monitor rax=%RX64, ecx=%RX32, edx=%RX32; ECX != 0 -> #GP(0)\n", GCPtrMem, uEcx, uEdx)); NOREF(uEdx); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + RTGCPHYS GCPhysMem; + /** @todo access size */ + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, 1, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, &GCPhysMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS)) + { + /* + * MONITOR does not access the memory, just monitors the address. However, + * if the address falls in the APIC-access page, the address monitored must + * instead be the corresponding address in the virtual-APIC page. + * + * See Intel spec. 29.4.4 "Instruction-Specific Considerations". + */ + rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem, 1, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA); + if ( rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE + && rcStrict != VINF_VMX_MODIFIES_BEHAVIOR) + return rcStrict; + } +#endif + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MONITOR)) + { + Log2(("monitor: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MONITOR, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Call EM to prepare the monitor/wait. + */ + rcStrict = EMMonitorWaitPrepare(pVCpu, pVCpu->cpum.GstCtx.rax, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.rdx, GCPhysMem); + Assert(rcStrict == VINF_SUCCESS); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return rcStrict; +} + + +/** + * Implements 'MWAIT'. + */ +IEM_CIMPL_DEF_0(iemCImpl_mwait) +{ + /* + * Permission checks. + */ + if (pVCpu->iem.s.uCpl != 0) + { + Log2(("mwait: CPL != 0\n")); + /** @todo MSR[0xC0010015].MonMwaitUserEn if we care. (Remember to check + * EFLAGS.VM then.) */ + return iemRaiseUndefinedOpcode(pVCpu); + } + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait) + { + Log2(("mwait: Not in CPUID\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + + /* Check VMX nested-guest intercept. */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MWAIT_EXIT)) + IEM_VMX_VMEXIT_MWAIT_RET(pVCpu, EMMonitorIsArmed(pVCpu), cbInstr); + + /* + * Gather the operands and validate them. + */ + uint32_t const uEax = pVCpu->cpum.GstCtx.eax; + uint32_t const uEcx = pVCpu->cpum.GstCtx.ecx; + if (uEcx != 0) + { + /* Only supported extension is break on IRQ when IF=0. */ + if (uEcx > 1) + { + Log2(("mwait eax=%RX32, ecx=%RX32; ECX > 1 -> #GP(0)\n", uEax, uEcx)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + uint32_t fMWaitFeatures = 0; + uint32_t uIgnore = 0; + CPUMGetGuestCpuId(pVCpu, 5, 0, -1 /*f64BitMode*/, &uIgnore, &uIgnore, &fMWaitFeatures, &uIgnore); + if ( (fMWaitFeatures & (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0)) + != (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0)) + { + Log2(("mwait eax=%RX32, ecx=%RX32; break-on-IRQ-IF=0 extension not enabled -> #GP(0)\n", uEax, uEcx)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* + * If the interrupt-window exiting control is set or a virtual-interrupt is pending + * for delivery; and interrupts are disabled the processor does not enter its + * mwait state but rather passes control to the next instruction. + * + * See Intel spec. 25.3 "Changes to Instruction Behavior In VMX Non-root Operation". + */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && !pVCpu->cpum.GstCtx.eflags.Bits.u1IF) + { + if ( IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INT_WINDOW_EXIT) + || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST)) + /** @todo finish: check up this out after we move int window stuff out of the + * run loop and into the instruction finishing logic here. */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } +#endif + } + + /* + * Check SVM nested-guest mwait intercepts. + */ + if ( IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT_ARMED) + && EMMonitorIsArmed(pVCpu)) + { + Log2(("mwait: Guest intercept (monitor hardware armed) -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT_ARMED, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT)) + { + Log2(("mwait: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Call EM to prepare the monitor/wait. + * + * This will return VINF_EM_HALT. If there the trap flag is set, we may + * override it when executing iemRegAddToRipAndFinishingClearingRF ASSUMING + * that will only return guest related events. + */ + VBOXSTRICTRC rcStrict = EMMonitorWaitPerform(pVCpu, uEax, uEcx); + + /** @todo finish: This needs more thinking as we should suppress internal + * debugger events here, or we'll bugger up the guest state even more than we + * alread do around VINF_EM_HALT. */ + VBOXSTRICTRC rcStrict2 = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + if (rcStrict2 != VINF_SUCCESS) + { + Log2(("mwait: %Rrc (perform) -> %Rrc (finish)!\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2) )); + rcStrict = rcStrict2; + } + + return rcStrict; +} + + +/** + * Implements 'SWAPGS'. + */ +IEM_CIMPL_DEF_0(iemCImpl_swapgs) +{ + Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT); /* Caller checks this. */ + + /* + * Permission checks. + */ + if (pVCpu->iem.s.uCpl != 0) + { + Log2(("swapgs: CPL != 0\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + + /* + * Do the job. + */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_GS); + uint64_t uOtherGsBase = pVCpu->cpum.GstCtx.msrKERNELGSBASE; + pVCpu->cpum.GstCtx.msrKERNELGSBASE = pVCpu->cpum.GstCtx.gs.u64Base; + pVCpu->cpum.GstCtx.gs.u64Base = uOtherGsBase; + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +#ifndef VBOX_WITHOUT_CPUID_HOST_CALL +/** + * Handles a CPUID call. + */ +static VBOXSTRICTRC iemCpuIdVBoxCall(PVMCPUCC pVCpu, uint32_t iFunction, + uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx) +{ + switch (iFunction) + { + case VBOX_CPUID_FN_ID: + LogFlow(("iemCpuIdVBoxCall: VBOX_CPUID_FN_ID\n")); + *pEax = VBOX_CPUID_RESP_ID_EAX; + *pEbx = VBOX_CPUID_RESP_ID_EBX; + *pEcx = VBOX_CPUID_RESP_ID_ECX; + *pEdx = VBOX_CPUID_RESP_ID_EDX; + break; + + case VBOX_CPUID_FN_LOG: + { + CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX | CPUMCTX_EXTRN_RSI + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK); + + /* Validate input. */ + uint32_t cchToLog = *pEdx; + if (cchToLog <= _2M) + { + uint32_t const uLogPicker = *pEbx; + if (uLogPicker <= 1) + { + /* Resolve the logger. */ + PRTLOGGER const pLogger = !uLogPicker + ? RTLogDefaultInstanceEx(UINT32_MAX) : RTLogRelGetDefaultInstanceEx(UINT32_MAX); + if (pLogger) + { + /* Copy over the data: */ + RTGCPTR GCPtrSrc = pVCpu->cpum.GstCtx.rsi; + while (cchToLog > 0) + { + uint32_t cbToMap = GUEST_PAGE_SIZE - (GCPtrSrc & GUEST_PAGE_OFFSET_MASK); + if (cbToMap > cchToLog) + cbToMap = cchToLog; + /** @todo Extend iemMemMap to allowing page size accessing and avoid 7 + * unnecessary calls & iterations per pages. */ + if (cbToMap > 512) + cbToMap = 512; + void *pvSrc = NULL; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvSrc, cbToMap, UINT8_MAX, GCPtrSrc, IEM_ACCESS_DATA_R, 0); + if (rcStrict == VINF_SUCCESS) + { + RTLogBulkNestedWrite(pLogger, (const char *)pvSrc, cbToMap, "Gst:"); + rcStrict = iemMemCommitAndUnmap(pVCpu, pvSrc, IEM_ACCESS_DATA_R); + AssertRCSuccessReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict); + } + else + { + Log(("iemCpuIdVBoxCall: %Rrc at %RGp LB %#x\n", VBOXSTRICTRC_VAL(rcStrict), GCPtrSrc, cbToMap)); + return rcStrict; + } + + /* Advance. */ + pVCpu->cpum.GstCtx.rsi = GCPtrSrc += cbToMap; + *pEdx = cchToLog -= cbToMap; + } + *pEax = VINF_SUCCESS; + } + else + *pEax = (uint32_t)VERR_NOT_FOUND; + } + else + *pEax = (uint32_t)VERR_NOT_FOUND; + } + else + *pEax = (uint32_t)VERR_TOO_MUCH_DATA; + *pEdx = VBOX_CPUID_RESP_GEN_EDX; + *pEcx = VBOX_CPUID_RESP_GEN_ECX; + *pEbx = VBOX_CPUID_RESP_GEN_EBX; + break; + } + + default: + LogFlow(("iemCpuIdVBoxCall: Invalid function %#x (%#x, %#x)\n", iFunction, *pEbx, *pEdx)); + *pEax = (uint32_t)VERR_INVALID_FUNCTION; + *pEbx = (uint32_t)VERR_INVALID_FUNCTION; + *pEcx = (uint32_t)VERR_INVALID_FUNCTION; + *pEdx = (uint32_t)VERR_INVALID_FUNCTION; + break; + } + return VINF_SUCCESS; +} +#endif /* VBOX_WITHOUT_CPUID_HOST_CALL */ + +/** + * Implements 'CPUID'. + */ +IEM_CIMPL_DEF_0(iemCImpl_cpuid) +{ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + Log2(("cpuid: Guest intercept -> VM-exit\n")); + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_CPUID, cbInstr); + } + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CPUID)) + { + Log2(("cpuid: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_CPUID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + + uint32_t const uEax = pVCpu->cpum.GstCtx.eax; + uint32_t const uEcx = pVCpu->cpum.GstCtx.ecx; + +#ifndef VBOX_WITHOUT_CPUID_HOST_CALL + /* + * CPUID host call backdoor. + */ + if ( uEax == VBOX_CPUID_REQ_EAX_FIXED + && (uEcx & VBOX_CPUID_REQ_ECX_FIXED_MASK) == VBOX_CPUID_REQ_ECX_FIXED + && pVCpu->CTX_SUFF(pVM)->iem.s.fCpuIdHostCall) + { + VBOXSTRICTRC rcStrict = iemCpuIdVBoxCall(pVCpu, uEcx & VBOX_CPUID_REQ_ECX_FN_MASK, + &pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, + &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + /* + * Regular CPUID. + */ + else +#endif + CPUMGetGuestCpuId(pVCpu, uEax, uEcx, pVCpu->cpum.GstCtx.cs.Attr.n.u1Long, + &pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx); + + pVCpu->cpum.GstCtx.rax &= UINT32_C(0xffffffff); + pVCpu->cpum.GstCtx.rbx &= UINT32_C(0xffffffff); + pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff); + pVCpu->cpum.GstCtx.rdx &= UINT32_C(0xffffffff); + pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX); + + pVCpu->iem.s.cPotentialExits++; + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'AAD'. + * + * @param bImm The immediate operand. + */ +IEM_CIMPL_DEF_1(iemCImpl_aad, uint8_t, bImm) +{ + uint16_t const ax = pVCpu->cpum.GstCtx.ax; + uint8_t const al = (uint8_t)ax + (uint8_t)(ax >> 8) * bImm; + pVCpu->cpum.GstCtx.ax = al; + iemHlpUpdateArithEFlagsU8(pVCpu, al, + X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, + X86_EFL_OF | X86_EFL_AF | X86_EFL_CF); + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'AAM'. + * + * @param bImm The immediate operand. Cannot be 0. + */ +IEM_CIMPL_DEF_1(iemCImpl_aam, uint8_t, bImm) +{ + Assert(bImm != 0); /* #DE on 0 is handled in the decoder. */ + + uint16_t const ax = pVCpu->cpum.GstCtx.ax; + uint8_t const al = (uint8_t)ax % bImm; + uint8_t const ah = (uint8_t)ax / bImm; + pVCpu->cpum.GstCtx.ax = (ah << 8) + al; + iemHlpUpdateArithEFlagsU8(pVCpu, al, + X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, + X86_EFL_OF | X86_EFL_AF | X86_EFL_CF); + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'DAA'. + */ +IEM_CIMPL_DEF_0(iemCImpl_daa) +{ + uint8_t const al = pVCpu->cpum.GstCtx.al; + bool const fCarry = pVCpu->cpum.GstCtx.eflags.Bits.u1CF; + + if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF + || (al & 0xf) >= 10) + { + pVCpu->cpum.GstCtx.al = al + 6; + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1; + } + else + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0; + + if (al >= 0x9a || fCarry) + { + pVCpu->cpum.GstCtx.al += 0x60; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1; + } + else + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0; + + iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'DAS'. + */ +IEM_CIMPL_DEF_0(iemCImpl_das) +{ + uint8_t const uInputAL = pVCpu->cpum.GstCtx.al; + bool const fCarry = pVCpu->cpum.GstCtx.eflags.Bits.u1CF; + + if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF + || (uInputAL & 0xf) >= 10) + { + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1; + if (uInputAL < 6) + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1; + pVCpu->cpum.GstCtx.al = uInputAL - 6; + } + else + { + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0; + } + + if (uInputAL >= 0x9a || fCarry) + { + pVCpu->cpum.GstCtx.al -= 0x60; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1; + } + + iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'AAA'. + */ +IEM_CIMPL_DEF_0(iemCImpl_aaa) +{ + if (IEM_IS_GUEST_CPU_AMD(pVCpu)) + { + if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF + || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10) + { + iemAImpl_add_u16(&pVCpu->cpum.GstCtx.ax, 0x106, &pVCpu->cpum.GstCtx.eflags.uBoth); + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1; + } + else + { + iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF); + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0; + } + pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f); + } + else + { + if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF + || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10) + { + pVCpu->cpum.GstCtx.ax += UINT16_C(0x106); + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1; + } + else + { + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0; + } + pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f); + iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF); + } + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'AAS'. + */ +IEM_CIMPL_DEF_0(iemCImpl_aas) +{ + if (IEM_IS_GUEST_CPU_AMD(pVCpu)) + { + if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF + || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10) + { + iemAImpl_sub_u16(&pVCpu->cpum.GstCtx.ax, 0x106, &pVCpu->cpum.GstCtx.eflags.uBoth); + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1; + } + else + { + iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF); + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0; + } + pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f); + } + else + { + if ( pVCpu->cpum.GstCtx.eflags.Bits.u1AF + || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10) + { + pVCpu->cpum.GstCtx.ax -= UINT16_C(0x106); + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1; + } + else + { + pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0; + pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0; + } + pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f); + iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF); + } + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements the 16-bit version of 'BOUND'. + * + * @note We have separate 16-bit and 32-bit variants of this function due to + * the decoder using unsigned parameters, whereas we want signed one to + * do the job. This is significant for a recompiler. + */ +IEM_CIMPL_DEF_3(iemCImpl_bound_16, int16_t, idxArray, int16_t, idxLowerBound, int16_t, idxUpperBound) +{ + /* + * Check if the index is inside the bounds, otherwise raise #BR. + */ + if ( idxArray >= idxLowerBound + && idxArray <= idxUpperBound) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return iemRaiseBoundRangeExceeded(pVCpu); +} + + +/** + * Implements the 32-bit version of 'BOUND'. + */ +IEM_CIMPL_DEF_3(iemCImpl_bound_32, int32_t, idxArray, int32_t, idxLowerBound, int32_t, idxUpperBound) +{ + /* + * Check if the index is inside the bounds, otherwise raise #BR. + */ + if ( idxArray >= idxLowerBound + && idxArray <= idxUpperBound) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return iemRaiseBoundRangeExceeded(pVCpu); +} + + + +/* + * Instantiate the various string operation combinations. + */ +#define OP_SIZE 8 +#define ADDR_SIZE 16 +#include "IEMAllCImplStrInstr.cpp.h" +#define OP_SIZE 8 +#define ADDR_SIZE 32 +#include "IEMAllCImplStrInstr.cpp.h" +#define OP_SIZE 8 +#define ADDR_SIZE 64 +#include "IEMAllCImplStrInstr.cpp.h" + +#define OP_SIZE 16 +#define ADDR_SIZE 16 +#include "IEMAllCImplStrInstr.cpp.h" +#define OP_SIZE 16 +#define ADDR_SIZE 32 +#include "IEMAllCImplStrInstr.cpp.h" +#define OP_SIZE 16 +#define ADDR_SIZE 64 +#include "IEMAllCImplStrInstr.cpp.h" + +#define OP_SIZE 32 +#define ADDR_SIZE 16 +#include "IEMAllCImplStrInstr.cpp.h" +#define OP_SIZE 32 +#define ADDR_SIZE 32 +#include "IEMAllCImplStrInstr.cpp.h" +#define OP_SIZE 32 +#define ADDR_SIZE 64 +#include "IEMAllCImplStrInstr.cpp.h" + +#define OP_SIZE 64 +#define ADDR_SIZE 32 +#include "IEMAllCImplStrInstr.cpp.h" +#define OP_SIZE 64 +#define ADDR_SIZE 64 +#include "IEMAllCImplStrInstr.cpp.h" + + +/** + * Implements 'XGETBV'. + */ +IEM_CIMPL_DEF_0(iemCImpl_xgetbv) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4); + if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) + { + uint32_t uEcx = pVCpu->cpum.GstCtx.ecx; + switch (uEcx) + { + case 0: + break; + + case 1: /** @todo Implement XCR1 support. */ + default: + Log(("xgetbv ecx=%RX32 -> #GP(0)\n", uEcx)); + return iemRaiseGeneralProtectionFault0(pVCpu); + + } + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx); + pVCpu->cpum.GstCtx.rax = RT_LO_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]); + pVCpu->cpum.GstCtx.rdx = RT_HI_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]); + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + Log(("xgetbv CR4.OSXSAVE=0 -> UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); +} + + +/** + * Implements 'XSETBV'. + */ +IEM_CIMPL_DEF_0(iemCImpl_xsetbv) +{ + if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) + { + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_XSETBV)) + { + Log2(("xsetbv: Guest intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XSETBV, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + if (pVCpu->iem.s.uCpl == 0) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx); + + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_XSETBV, cbInstr); + + uint32_t uEcx = pVCpu->cpum.GstCtx.ecx; + uint64_t uNewValue = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx); + switch (uEcx) + { + case 0: + { + int rc = CPUMSetGuestXcr0(pVCpu, uNewValue); + if (rc == VINF_SUCCESS) + break; + Assert(rc == VERR_CPUM_RAISE_GP_0); + Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + case 1: /** @todo Implement XCR1 support. */ + default: + Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue)); + return iemRaiseGeneralProtectionFault0(pVCpu); + + } + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + Log(("xsetbv cpl=%u -> GP(0)\n", pVCpu->iem.s.uCpl)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + Log(("xsetbv CR4.OSXSAVE=0 -> UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); +} + +#ifndef RT_ARCH_ARM64 +# ifdef IN_RING3 + +/** Argument package for iemCImpl_cmpxchg16b_fallback_rendezvous_callback. */ +struct IEMCIMPLCX16ARGS +{ + PRTUINT128U pu128Dst; + PRTUINT128U pu128RaxRdx; + PRTUINT128U pu128RbxRcx; + uint32_t *pEFlags; +# ifdef VBOX_STRICT + uint32_t cCalls; +# endif +}; + +/** + * @callback_method_impl{FNVMMEMTRENDEZVOUS, + * Worker for iemCImpl_cmpxchg16b_fallback_rendezvous} + */ +static DECLCALLBACK(VBOXSTRICTRC) iemCImpl_cmpxchg16b_fallback_rendezvous_callback(PVM pVM, PVMCPUCC pVCpu, void *pvUser) +{ + RT_NOREF(pVM, pVCpu); + struct IEMCIMPLCX16ARGS *pArgs = (struct IEMCIMPLCX16ARGS *)pvUser; +# ifdef VBOX_STRICT + Assert(pArgs->cCalls == 0); + pArgs->cCalls++; +# endif + + iemAImpl_cmpxchg16b_fallback(pArgs->pu128Dst, pArgs->pu128RaxRdx, pArgs->pu128RbxRcx, pArgs->pEFlags); + return VINF_SUCCESS; +} + +# endif /* IN_RING3 */ + +/** + * Implements 'CMPXCHG16B' fallback using rendezvous. + */ +IEM_CIMPL_DEF_4(iemCImpl_cmpxchg16b_fallback_rendezvous, PRTUINT128U, pu128Dst, PRTUINT128U, pu128RaxRdx, + PRTUINT128U, pu128RbxRcx, uint32_t *, pEFlags) +{ +# ifdef IN_RING3 + struct IEMCIMPLCX16ARGS Args; + Args.pu128Dst = pu128Dst; + Args.pu128RaxRdx = pu128RaxRdx; + Args.pu128RbxRcx = pu128RbxRcx; + Args.pEFlags = pEFlags; +# ifdef VBOX_STRICT + Args.cCalls = 0; +# endif + VBOXSTRICTRC rcStrict = VMMR3EmtRendezvous(pVCpu->CTX_SUFF(pVM), VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, + iemCImpl_cmpxchg16b_fallback_rendezvous_callback, &Args); + Assert(Args.cCalls == 1); + if (rcStrict == VINF_SUCCESS) + { + /* Duplicated tail code. */ + rcStrict = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_RW); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.eflags.u = *pEFlags; /* IEM_MC_COMMIT_EFLAGS */ + if (!(*pEFlags & X86_EFL_ZF)) + { + pVCpu->cpum.GstCtx.rax = pu128RaxRdx->s.Lo; + pVCpu->cpum.GstCtx.rdx = pu128RaxRdx->s.Hi; + } + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + return rcStrict; +# else + RT_NOREF(pVCpu, cbInstr, pu128Dst, pu128RaxRdx, pu128RbxRcx, pEFlags); + return VERR_IEM_ASPECT_NOT_IMPLEMENTED; /* This should get us to ring-3 for now. Should perhaps be replaced later. */ +# endif +} + +#endif /* RT_ARCH_ARM64 */ + +/** + * Implements 'CLFLUSH' and 'CLFLUSHOPT'. + * + * This is implemented in C because it triggers a load like behaviour without + * actually reading anything. Since that's not so common, it's implemented + * here. + * + * @param iEffSeg The effective segment. + * @param GCPtrEff The address of the image. + */ +IEM_CIMPL_DEF_2(iemCImpl_clflush_clflushopt, uint8_t, iEffSeg, RTGCPTR, GCPtrEff) +{ + /* + * Pretend to do a load w/o reading (see also iemCImpl_monitor and iemMemMap). + */ + VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrEff); + if (rcStrict == VINF_SUCCESS) + { + RTGCPHYS GCPhysMem; + /** @todo access size. */ + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrEff, 1, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, &GCPhysMem); + if (rcStrict == VINF_SUCCESS) + { +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS)) + { + /* + * CLFLUSH/CLFLUSHOPT does not access the memory, but flushes the cache-line + * that contains the address. However, if the address falls in the APIC-access + * page, the address flushed must instead be the corresponding address in the + * virtual-APIC page. + * + * See Intel spec. 29.4.4 "Instruction-Specific Considerations". + */ + rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem, 1, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA); + if ( rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE + && rcStrict != VINF_VMX_MODIFIES_BEHAVIOR) + return rcStrict; + } +#endif + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + + return rcStrict; +} + + +/** + * Implements 'FINIT' and 'FNINIT'. + * + * @param fCheckXcpts Whether to check for umasked pending exceptions or + * not. + */ +IEM_CIMPL_DEF_1(iemCImpl_finit, bool, fCheckXcpts) +{ + /* + * Exceptions. + */ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); + if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) + return iemRaiseDeviceNotAvailable(pVCpu); + + iemFpuActualizeStateForChange(pVCpu); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_X87); + + /* FINIT: Raise #MF on pending exception(s): */ + if (fCheckXcpts && (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)) + return iemRaiseMathFault(pVCpu); + + /* + * Reset the state. + */ + PX86XSAVEAREA pXState = &pVCpu->cpum.GstCtx.XState; + + /* Rotate the stack to account for changed TOS. */ + iemFpuRotateStackSetTop(&pXState->x87, 0); + + pXState->x87.FCW = 0x37f; + pXState->x87.FSW = 0; + pXState->x87.FTW = 0x00; /* 0 - empty. */ + /** @todo Intel says the instruction and data pointers are not cleared on + * 387, presume that 8087 and 287 doesn't do so either. */ + /** @todo test this stuff. */ + if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386) + { + pXState->x87.FPUDP = 0; + pXState->x87.DS = 0; //?? + pXState->x87.Rsrvd2 = 0; + pXState->x87.FPUIP = 0; + pXState->x87.CS = 0; //?? + pXState->x87.Rsrvd1 = 0; + } + pXState->x87.FOP = 0; + + iemHlpUsedFpu(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'FXSAVE'. + * + * @param iEffSeg The effective segment. + * @param GCPtrEff The address of the image. + * @param enmEffOpSize The operand size (only REX.W really matters). + */ +IEM_CIMPL_DEF_3(iemCImpl_fxsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX); + + /* + * Raise exceptions. + */ + if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS | X86_CR0_EM)) + return iemRaiseDeviceNotAvailable(pVCpu); + + /* + * Access the memory. + */ + void *pvMem512; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE, + 15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_GP_OR_AC); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + PX86FXSTATE pDst = (PX86FXSTATE)pvMem512; + PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.XState.x87; + + /* + * Store the registers. + */ + /** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's + * implementation specific whether MXCSR and XMM0-XMM7 are saved. */ + + /* common for all formats */ + pDst->FCW = pSrc->FCW; + pDst->FSW = pSrc->FSW; + pDst->FTW = pSrc->FTW & UINT16_C(0xff); + pDst->FOP = pSrc->FOP; + pDst->MXCSR = pSrc->MXCSR; + pDst->MXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM)); + for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++) + { + /** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing + * them for now... */ + pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0]; + pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1]; + pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff); + pDst->aRegs[i].au32[3] = 0; + } + + /* FPU IP, CS, DP and DS. */ + pDst->FPUIP = pSrc->FPUIP; + pDst->CS = pSrc->CS; + pDst->FPUDP = pSrc->FPUDP; + pDst->DS = pSrc->DS; + if (enmEffOpSize == IEMMODE_64BIT) + { + /* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */ + pDst->Rsrvd1 = pSrc->Rsrvd1; + pDst->Rsrvd2 = pSrc->Rsrvd2; + } + else + { + pDst->Rsrvd1 = 0; + pDst->Rsrvd2 = 0; + } + + /* XMM registers. Skipped in 64-bit CPL0 if EFER.FFXSR (AMD only) is set. */ + if ( !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR) + || pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT + || pVCpu->iem.s.uCpl != 0) + { + uint32_t cXmmRegs = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? 16 : 8; + for (uint32_t i = 0; i < cXmmRegs; i++) + pDst->aXMM[i] = pSrc->aXMM[i]; + /** @todo Testcase: What happens to the reserved XMM registers? Untouched, + * right? */ + } + + /* + * Commit the memory. + */ + rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'FXRSTOR'. + * + * @param iEffSeg The effective segment register for @a GCPtrEff. + * @param GCPtrEff The address of the image. + * @param enmEffOpSize The operand size (only REX.W really matters). + */ +IEM_CIMPL_DEF_3(iemCImpl_fxrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX); + + /* + * Raise exceptions. + */ + if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS | X86_CR0_EM)) + return iemRaiseDeviceNotAvailable(pVCpu); + + /* + * Access the memory. + */ + void *pvMem512; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R, + 15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_GP_OR_AC); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512; + PX86FXSTATE pDst = &pVCpu->cpum.GstCtx.XState.x87; + + /* + * Check the state for stuff which will #GP(0). + */ + uint32_t const fMXCSR = pSrc->MXCSR; + uint32_t const fMXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM)); + if (fMXCSR & ~fMXCSR_MASK) + { + Log(("fxrstor: MXCSR=%#x (MXCSR_MASK=%#x) -> #GP(0)\n", fMXCSR, fMXCSR_MASK)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Load the registers. + */ + /** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's + * implementation specific whether MXCSR and XMM0-XMM7 are + * restored according to Intel. + * AMD says MXCSR and XMM registers are never loaded if + * CR4.OSFXSR=0. + */ + + /* common for all formats */ + pDst->FCW = pSrc->FCW; + pDst->FSW = pSrc->FSW; + pDst->FTW = pSrc->FTW & UINT16_C(0xff); + pDst->FOP = pSrc->FOP; + pDst->MXCSR = fMXCSR; + /* (MXCSR_MASK is read-only) */ + for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++) + { + pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0]; + pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1]; + pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff); + pDst->aRegs[i].au32[3] = 0; + } + + /* FPU IP, CS, DP and DS. */ + /** @todo AMD says this is only done if FSW.ES is set after loading. */ + if (enmEffOpSize == IEMMODE_64BIT) + { + pDst->FPUIP = pSrc->FPUIP; + pDst->CS = pSrc->CS; + pDst->Rsrvd1 = pSrc->Rsrvd1; + pDst->FPUDP = pSrc->FPUDP; + pDst->DS = pSrc->DS; + pDst->Rsrvd2 = pSrc->Rsrvd2; + } + else + { + pDst->FPUIP = pSrc->FPUIP; + pDst->CS = pSrc->CS; + pDst->Rsrvd1 = 0; + pDst->FPUDP = pSrc->FPUDP; + pDst->DS = pSrc->DS; + pDst->Rsrvd2 = 0; + } + + /* XMM registers. Skipped in 64-bit CPL0 if EFER.FFXSR (AMD only) is set. + * Does not affect MXCSR, only registers. + */ + if ( !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR) + || pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT + || pVCpu->iem.s.uCpl != 0) + { + uint32_t cXmmRegs = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? 16 : 8; + for (uint32_t i = 0; i < cXmmRegs; i++) + pDst->aXMM[i] = pSrc->aXMM[i]; + } + + pDst->FCW &= ~X86_FCW_ZERO_MASK | X86_FCW_IC_MASK; /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */ + iemFpuRecalcExceptionStatus(pDst); + + if (pDst->FSW & X86_FSW_ES) + Log11(("fxrstor: %04x:%08RX64: loading state with pending FPU exception (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pSrc->FSW)); + + /* + * Unmap the memory. + */ + rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_R); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + iemHlpUsedFpu(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'XSAVE'. + * + * @param iEffSeg The effective segment. + * @param GCPtrEff The address of the image. + * @param enmEffOpSize The operand size (only REX.W really matters). + */ +IEM_CIMPL_DEF_3(iemCImpl_xsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); + + /* + * Raise exceptions. + */ + if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) + return iemRaiseUndefinedOpcode(pVCpu); + /* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS)) + { + Log(("xrstor: Not enabled for nested-guest execution -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) + return iemRaiseDeviceNotAvailable(pVCpu); + + /* + * Calc the requested mask. + */ + uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0]; + AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 | XSAVE_C_SSE | XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED); + uint64_t const fXInUse = pVCpu->cpum.GstCtx.aXcr[0]; + +/** @todo figure out the exact protocol for the memory access. Currently we + * just need this crap to work halfways to make it possible to test + * AVX instructions. */ +/** @todo figure out the XINUSE and XMODIFIED */ + + /* + * Access the x87 memory state. + */ + /* The x87+SSE state. */ + void *pvMem512; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE, + 63 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_GP_OR_AC); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + PX86FXSTATE pDst = (PX86FXSTATE)pvMem512; + PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.XState.x87; + + /* The header. */ + PX86XSAVEHDR pHdr; + rcStrict = iemMemMap(pVCpu, (void **)&pHdr, sizeof(&pHdr), iEffSeg, GCPtrEff + 512, IEM_ACCESS_DATA_RW, 0 /* checked above */); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Store the X87 state. + */ + if (fReqComponents & XSAVE_C_X87) + { + /* common for all formats */ + pDst->FCW = pSrc->FCW; + pDst->FSW = pSrc->FSW; + pDst->FTW = pSrc->FTW & UINT16_C(0xff); + pDst->FOP = pSrc->FOP; + pDst->FPUIP = pSrc->FPUIP; + pDst->CS = pSrc->CS; + pDst->FPUDP = pSrc->FPUDP; + pDst->DS = pSrc->DS; + if (enmEffOpSize == IEMMODE_64BIT) + { + /* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */ + pDst->Rsrvd1 = pSrc->Rsrvd1; + pDst->Rsrvd2 = pSrc->Rsrvd2; + } + else + { + pDst->Rsrvd1 = 0; + pDst->Rsrvd2 = 0; + } + for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++) + { + /** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing + * them for now... */ + pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0]; + pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1]; + pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff); + pDst->aRegs[i].au32[3] = 0; + } + + } + + if (fReqComponents & (XSAVE_C_SSE | XSAVE_C_YMM)) + { + pDst->MXCSR = pSrc->MXCSR; + pDst->MXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM)); + } + + if (fReqComponents & XSAVE_C_SSE) + { + /* XMM registers. */ + uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8; + for (uint32_t i = 0; i < cXmmRegs; i++) + pDst->aXMM[i] = pSrc->aXMM[i]; + /** @todo Testcase: What happens to the reserved XMM registers? Untouched, + * right? */ + } + + /* Commit the x87 state bits. (probably wrong) */ + rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Store AVX state. + */ + if (fReqComponents & XSAVE_C_YMM) + { + /** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */ + AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9); + PCX86XSAVEYMMHI pCompSrc = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PCX86XSAVEYMMHI); + PX86XSAVEYMMHI pCompDst; + rcStrict = iemMemMap(pVCpu, (void **)&pCompDst, sizeof(*pCompDst), iEffSeg, GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT], + IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE, 0 /* checked above */); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8; + for (uint32_t i = 0; i < cXmmRegs; i++) + pCompDst->aYmmHi[i] = pCompSrc->aYmmHi[i]; + + rcStrict = iemMemCommitAndUnmap(pVCpu, pCompDst, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + /* + * Update the header. + */ + pHdr->bmXState = (pHdr->bmXState & ~fReqComponents) + | (fReqComponents & fXInUse); + + rcStrict = iemMemCommitAndUnmap(pVCpu, pHdr, IEM_ACCESS_DATA_RW); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'XRSTOR'. + * + * @param iEffSeg The effective segment. + * @param GCPtrEff The address of the image. + * @param enmEffOpSize The operand size (only REX.W really matters). + */ +IEM_CIMPL_DEF_3(iemCImpl_xrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); + + /* + * Raise exceptions. + */ + if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) + return iemRaiseUndefinedOpcode(pVCpu); + /* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS)) + { + Log(("xrstor: Not enabled for nested-guest execution -> #UD\n")); + return iemRaiseUndefinedOpcode(pVCpu); + } + if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) + return iemRaiseDeviceNotAvailable(pVCpu); + if (GCPtrEff & 63) + { + /** @todo CPU/VM detection possible! \#AC might not be signal for + * all/any misalignment sizes, intel says its an implementation detail. */ + if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM) + && pVCpu->cpum.GstCtx.eflags.Bits.u1AC + && pVCpu->iem.s.uCpl == 3) + return iemRaiseAlignmentCheckException(pVCpu); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + +/** @todo figure out the exact protocol for the memory access. Currently we + * just need this crap to work halfways to make it possible to test + * AVX instructions. */ +/** @todo figure out the XINUSE and XMODIFIED */ + + /* + * Access the x87 memory state. + */ + /* The x87+SSE state. */ + void *pvMem512; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R, + 63 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_GP_OR_AC); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512; + PX86FXSTATE pDst = &pVCpu->cpum.GstCtx.XState.x87; + + /* + * Calc the requested mask + */ + PX86XSAVEHDR pHdrDst = &pVCpu->cpum.GstCtx.XState.Hdr; + PCX86XSAVEHDR pHdrSrc; + rcStrict = iemMemMap(pVCpu, (void **)&pHdrSrc, sizeof(&pHdrSrc), iEffSeg, GCPtrEff + 512, + IEM_ACCESS_DATA_R, 0 /* checked above */); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0]; + AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 | XSAVE_C_SSE | XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED); + //uint64_t const fXInUse = pVCpu->cpum.GstCtx.aXcr[0]; + uint64_t const fRstorMask = pHdrSrc->bmXState; + uint64_t const fCompMask = pHdrSrc->bmXComp; + + AssertLogRelReturn(!(fCompMask & XSAVE_C_X), VERR_IEM_ASPECT_NOT_IMPLEMENTED); + + uint32_t const cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8; + + /* We won't need this any longer. */ + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pHdrSrc, IEM_ACCESS_DATA_R); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Load the X87 state. + */ + if (fReqComponents & XSAVE_C_X87) + { + if (fRstorMask & XSAVE_C_X87) + { + pDst->FCW = pSrc->FCW; + pDst->FSW = pSrc->FSW; + pDst->FTW = pSrc->FTW & UINT16_C(0xff); + pDst->FOP = pSrc->FOP; + pDst->FPUIP = pSrc->FPUIP; + pDst->CS = pSrc->CS; + pDst->FPUDP = pSrc->FPUDP; + pDst->DS = pSrc->DS; + if (enmEffOpSize == IEMMODE_64BIT) + { + /* Load upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */ + pDst->Rsrvd1 = pSrc->Rsrvd1; + pDst->Rsrvd2 = pSrc->Rsrvd2; + } + else + { + pDst->Rsrvd1 = 0; + pDst->Rsrvd2 = 0; + } + for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++) + { + pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0]; + pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1]; + pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff); + pDst->aRegs[i].au32[3] = 0; + } + + pDst->FCW &= ~X86_FCW_ZERO_MASK | X86_FCW_IC_MASK; /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */ + iemFpuRecalcExceptionStatus(pDst); + + if (pDst->FSW & X86_FSW_ES) + Log11(("xrstor: %04x:%08RX64: loading state with pending FPU exception (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pSrc->FSW)); + } + else + { + pDst->FCW = 0x37f; + pDst->FSW = 0; + pDst->FTW = 0x00; /* 0 - empty. */ + pDst->FPUDP = 0; + pDst->DS = 0; //?? + pDst->Rsrvd2= 0; + pDst->FPUIP = 0; + pDst->CS = 0; //?? + pDst->Rsrvd1= 0; + pDst->FOP = 0; + for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++) + { + pDst->aRegs[i].au32[0] = 0; + pDst->aRegs[i].au32[1] = 0; + pDst->aRegs[i].au32[2] = 0; + pDst->aRegs[i].au32[3] = 0; + } + } + pHdrDst->bmXState |= XSAVE_C_X87; /* playing safe for now */ + } + + /* MXCSR */ + if (fReqComponents & (XSAVE_C_SSE | XSAVE_C_YMM)) + { + if (fRstorMask & (XSAVE_C_SSE | XSAVE_C_YMM)) + pDst->MXCSR = pSrc->MXCSR; + else + pDst->MXCSR = 0x1f80; + } + + /* XMM registers. */ + if (fReqComponents & XSAVE_C_SSE) + { + if (fRstorMask & XSAVE_C_SSE) + { + for (uint32_t i = 0; i < cXmmRegs; i++) + pDst->aXMM[i] = pSrc->aXMM[i]; + /** @todo Testcase: What happens to the reserved XMM registers? Untouched, + * right? */ + } + else + { + for (uint32_t i = 0; i < cXmmRegs; i++) + { + pDst->aXMM[i].au64[0] = 0; + pDst->aXMM[i].au64[1] = 0; + } + } + pHdrDst->bmXState |= XSAVE_C_SSE; /* playing safe for now */ + } + + /* Unmap the x87 state bits (so we've don't run out of mapping). */ + rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_R); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Restore AVX state. + */ + if (fReqComponents & XSAVE_C_YMM) + { + AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9); + PX86XSAVEYMMHI pCompDst = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PX86XSAVEYMMHI); + + if (fRstorMask & XSAVE_C_YMM) + { + /** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */ + PCX86XSAVEYMMHI pCompSrc; + rcStrict = iemMemMap(pVCpu, (void **)&pCompSrc, sizeof(*pCompDst), + iEffSeg, GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT], + IEM_ACCESS_DATA_R, 0 /* checked above */); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + for (uint32_t i = 0; i < cXmmRegs; i++) + { + pCompDst->aYmmHi[i].au64[0] = pCompSrc->aYmmHi[i].au64[0]; + pCompDst->aYmmHi[i].au64[1] = pCompSrc->aYmmHi[i].au64[1]; + } + + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pCompSrc, IEM_ACCESS_DATA_R); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + else + { + for (uint32_t i = 0; i < cXmmRegs; i++) + { + pCompDst->aYmmHi[i].au64[0] = 0; + pCompDst->aYmmHi[i].au64[1] = 0; + } + } + pHdrDst->bmXState |= XSAVE_C_YMM; /* playing safe for now */ + } + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + + + +/** + * Implements 'STMXCSR'. + * + * @param iEffSeg The effective segment register for @a GCPtrEff. + * @param GCPtrEff The address of the image. + */ +IEM_CIMPL_DEF_2(iemCImpl_stmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX); + + /* + * Raise exceptions. + */ + if ( !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) + { + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) + { + /* + * Do the job. + */ + VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.XState.x87.MXCSR); + if (rcStrict == VINF_SUCCESS) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return rcStrict; + } + return iemRaiseDeviceNotAvailable(pVCpu); + } + return iemRaiseUndefinedOpcode(pVCpu); +} + + +/** + * Implements 'VSTMXCSR'. + * + * @param iEffSeg The effective segment register for @a GCPtrEff. + * @param GCPtrEff The address of the image. + */ +IEM_CIMPL_DEF_2(iemCImpl_vstmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_XCRx); + + /* + * Raise exceptions. + */ + if ( ( !IEM_IS_GUEST_CPU_AMD(pVCpu) + ? (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_SSE | XSAVE_C_YMM)) == (XSAVE_C_SSE | XSAVE_C_YMM) + : !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)) /* AMD Jaguar CPU (f0x16,m0,s1) behaviour */ + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) + { + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) + { + /* + * Do the job. + */ + VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.XState.x87.MXCSR); + if (rcStrict == VINF_SUCCESS) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return rcStrict; + } + return iemRaiseDeviceNotAvailable(pVCpu); + } + return iemRaiseUndefinedOpcode(pVCpu); +} + + +/** + * Implements 'LDMXCSR'. + * + * @param iEffSeg The effective segment register for @a GCPtrEff. + * @param GCPtrEff The address of the image. + */ +IEM_CIMPL_DEF_2(iemCImpl_ldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX); + + /* + * Raise exceptions. + */ + /** @todo testcase - order of LDMXCSR faults. Does \#PF, \#GP and \#SS + * happen after or before \#UD and \#EM? */ + if ( !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) + && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) + { + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) + { + /* + * Do the job. + */ + uint32_t fNewMxCsr; + VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, &fNewMxCsr, iEffSeg, GCPtrEff); + if (rcStrict == VINF_SUCCESS) + { + uint32_t const fMxCsrMask = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM)); + if (!(fNewMxCsr & ~fMxCsrMask)) + { + pVCpu->cpum.GstCtx.XState.x87.MXCSR = fNewMxCsr; + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + Log(("ldmxcsr: New MXCSR=%#RX32 & ~MASK=%#RX32 = %#RX32 -> #GP(0)\n", + fNewMxCsr, fMxCsrMask, fNewMxCsr & ~fMxCsrMask)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + return rcStrict; + } + return iemRaiseDeviceNotAvailable(pVCpu); + } + return iemRaiseUndefinedOpcode(pVCpu); +} + + +/** + * Commmon routine for fnstenv and fnsave. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param enmEffOpSize The effective operand size. + * @param uPtr Where to store the state. + */ +static void iemCImplCommonFpuStoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTPTRUNION uPtr) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87); + PCX86FXSTATE pSrcX87 = &pVCpu->cpum.GstCtx.XState.x87; + if (enmEffOpSize == IEMMODE_16BIT) + { + uPtr.pu16[0] = pSrcX87->FCW; + uPtr.pu16[1] = pSrcX87->FSW; + uPtr.pu16[2] = iemFpuCalcFullFtw(pSrcX87); + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + /** @todo Testcase: How does this work when the FPUIP/CS was saved in + * protected mode or long mode and we save it in real mode? And vice + * versa? And with 32-bit operand size? I think CPU is storing the + * effective address ((CS << 4) + IP) in the offset register and not + * doing any address calculations here. */ + uPtr.pu16[3] = (uint16_t)pSrcX87->FPUIP; + uPtr.pu16[4] = ((pSrcX87->FPUIP >> 4) & UINT16_C(0xf000)) | pSrcX87->FOP; + uPtr.pu16[5] = (uint16_t)pSrcX87->FPUDP; + uPtr.pu16[6] = (pSrcX87->FPUDP >> 4) & UINT16_C(0xf000); + } + else + { + uPtr.pu16[3] = pSrcX87->FPUIP; + uPtr.pu16[4] = pSrcX87->CS; + uPtr.pu16[5] = pSrcX87->FPUDP; + uPtr.pu16[6] = pSrcX87->DS; + } + } + else + { + /** @todo Testcase: what is stored in the "gray" areas? (figure 8-9 and 8-10) */ + uPtr.pu16[0*2] = pSrcX87->FCW; + uPtr.pu16[0*2+1] = 0xffff; /* (0xffff observed on intel skylake.) */ + uPtr.pu16[1*2] = pSrcX87->FSW; + uPtr.pu16[1*2+1] = 0xffff; + uPtr.pu16[2*2] = iemFpuCalcFullFtw(pSrcX87); + uPtr.pu16[2*2+1] = 0xffff; + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + uPtr.pu16[3*2] = (uint16_t)pSrcX87->FPUIP; + uPtr.pu32[4] = ((pSrcX87->FPUIP & UINT32_C(0xffff0000)) >> 4) | pSrcX87->FOP; + uPtr.pu16[5*2] = (uint16_t)pSrcX87->FPUDP; + uPtr.pu32[6] = (pSrcX87->FPUDP & UINT32_C(0xffff0000)) >> 4; + } + else + { + uPtr.pu32[3] = pSrcX87->FPUIP; + uPtr.pu16[4*2] = pSrcX87->CS; + uPtr.pu16[4*2+1] = pSrcX87->FOP; + uPtr.pu32[5] = pSrcX87->FPUDP; + uPtr.pu16[6*2] = pSrcX87->DS; + uPtr.pu16[6*2+1] = 0xffff; + } + } +} + + +/** + * Commmon routine for fldenv and frstor + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param enmEffOpSize The effective operand size. + * @param uPtr Where to store the state. + */ +static void iemCImplCommonFpuRestoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTCPTRUNION uPtr) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87); + PX86FXSTATE pDstX87 = &pVCpu->cpum.GstCtx.XState.x87; + if (enmEffOpSize == IEMMODE_16BIT) + { + pDstX87->FCW = uPtr.pu16[0]; + pDstX87->FSW = uPtr.pu16[1]; + pDstX87->FTW = uPtr.pu16[2]; + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + pDstX87->FPUIP = uPtr.pu16[3] | ((uint32_t)(uPtr.pu16[4] & UINT16_C(0xf000)) << 4); + pDstX87->FPUDP = uPtr.pu16[5] | ((uint32_t)(uPtr.pu16[6] & UINT16_C(0xf000)) << 4); + pDstX87->FOP = uPtr.pu16[4] & UINT16_C(0x07ff); + pDstX87->CS = 0; + pDstX87->Rsrvd1= 0; + pDstX87->DS = 0; + pDstX87->Rsrvd2= 0; + } + else + { + pDstX87->FPUIP = uPtr.pu16[3]; + pDstX87->CS = uPtr.pu16[4]; + pDstX87->Rsrvd1= 0; + pDstX87->FPUDP = uPtr.pu16[5]; + pDstX87->DS = uPtr.pu16[6]; + pDstX87->Rsrvd2= 0; + /** @todo Testcase: Is FOP cleared when doing 16-bit protected mode fldenv? */ + } + } + else + { + pDstX87->FCW = uPtr.pu16[0*2]; + pDstX87->FSW = uPtr.pu16[1*2]; + pDstX87->FTW = uPtr.pu16[2*2]; + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + pDstX87->FPUIP = uPtr.pu16[3*2] | ((uPtr.pu32[4] & UINT32_C(0x0ffff000)) << 4); + pDstX87->FOP = uPtr.pu32[4] & UINT16_C(0x07ff); + pDstX87->FPUDP = uPtr.pu16[5*2] | ((uPtr.pu32[6] & UINT32_C(0x0ffff000)) << 4); + pDstX87->CS = 0; + pDstX87->Rsrvd1= 0; + pDstX87->DS = 0; + pDstX87->Rsrvd2= 0; + } + else + { + pDstX87->FPUIP = uPtr.pu32[3]; + pDstX87->CS = uPtr.pu16[4*2]; + pDstX87->Rsrvd1= 0; + pDstX87->FOP = uPtr.pu16[4*2+1]; + pDstX87->FPUDP = uPtr.pu32[5]; + pDstX87->DS = uPtr.pu16[6*2]; + pDstX87->Rsrvd2= 0; + } + } + + /* Make adjustments. */ + pDstX87->FTW = iemFpuCompressFtw(pDstX87->FTW); +#ifdef LOG_ENABLED + uint16_t const fOldFsw = pDstX87->FSW; +#endif + pDstX87->FCW &= ~X86_FCW_ZERO_MASK | X86_FCW_IC_MASK; /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */ + iemFpuRecalcExceptionStatus(pDstX87); +#ifdef LOG_ENABLED + if ((pDstX87->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES)) + Log11(("iemCImplCommonFpuRestoreEnv: %04x:%08RX64: %s FPU exception (FCW=%#x FSW=%#x -> %#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fOldFsw & X86_FSW_ES ? "Supressed" : "Raised", + pDstX87->FCW, fOldFsw, pDstX87->FSW)); +#endif + + /** @todo Testcase: Check if ES and/or B are automatically cleared if no + * exceptions are pending after loading the saved state? */ +} + + +/** + * Implements 'FNSTENV'. + * + * @param enmEffOpSize The operand size (only REX.W really matters). + * @param iEffSeg The effective segment register for @a GCPtrEffDst. + * @param GCPtrEffDst The address of the image. + */ +IEM_CIMPL_DEF_3(iemCImpl_fnstenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst) +{ + RTPTRUNION uPtr; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 14 : 28, + iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE, + enmEffOpSize == IEMMODE_16BIT ? 1 : 3 /** @todo ? */); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr); + + rcStrict = iemMemCommitAndUnmap(pVCpu, uPtr.pv, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Mask all math exceptions. Any possibly pending exceptions will be cleared. */ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + pFpuCtx->FCW |= X86_FCW_XCPT_MASK; +#ifdef LOG_ENABLED + uint16_t fOldFsw = pFpuCtx->FSW; +#endif + iemFpuRecalcExceptionStatus(pFpuCtx); +#ifdef LOG_ENABLED + if ((pFpuCtx->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES)) + Log11(("fnstenv: %04x:%08RX64: %s FPU exception (FCW=%#x, FSW %#x -> %#x)\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + fOldFsw & X86_FSW_ES ? "Supressed" : "Raised", pFpuCtx->FCW, fOldFsw, pFpuCtx->FSW)); +#endif + + iemHlpUsedFpu(pVCpu); + + /* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'FNSAVE'. + * + * @param enmEffOpSize The operand size. + * @param iEffSeg The effective segment register for @a GCPtrEffDst. + * @param GCPtrEffDst The address of the image. + */ +IEM_CIMPL_DEF_3(iemCImpl_fnsave, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87); + + RTPTRUNION uPtr; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 94 : 108, + iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE, 3 /** @todo ? */); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr); + PRTFLOAT80U paRegs = (PRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28)); + for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++) + { + paRegs[i].au32[0] = pFpuCtx->aRegs[i].au32[0]; + paRegs[i].au32[1] = pFpuCtx->aRegs[i].au32[1]; + paRegs[i].au16[4] = pFpuCtx->aRegs[i].au16[4]; + } + + rcStrict = iemMemCommitAndUnmap(pVCpu, uPtr.pv, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* Rotate the stack to account for changed TOS. */ + iemFpuRotateStackSetTop(pFpuCtx, 0); + + /* + * Re-initialize the FPU context. + */ + pFpuCtx->FCW = 0x37f; + pFpuCtx->FSW = 0; + pFpuCtx->FTW = 0x00; /* 0 - empty */ + pFpuCtx->FPUDP = 0; + pFpuCtx->DS = 0; + pFpuCtx->Rsrvd2= 0; + pFpuCtx->FPUIP = 0; + pFpuCtx->CS = 0; + pFpuCtx->Rsrvd1= 0; + pFpuCtx->FOP = 0; + + iemHlpUsedFpu(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + + +/** + * Implements 'FLDENV'. + * + * @param enmEffOpSize The operand size (only REX.W really matters). + * @param iEffSeg The effective segment register for @a GCPtrEffSrc. + * @param GCPtrEffSrc The address of the image. + */ +IEM_CIMPL_DEF_3(iemCImpl_fldenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc) +{ + RTCPTRUNION uPtr; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 14 : 28, + iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R, + enmEffOpSize == IEMMODE_16BIT ? 1 : 3 /** @todo ?*/); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr); + + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uPtr.pv, IEM_ACCESS_DATA_R); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + iemHlpUsedFpu(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'FRSTOR'. + * + * @param enmEffOpSize The operand size. + * @param iEffSeg The effective segment register for @a GCPtrEffSrc. + * @param GCPtrEffSrc The address of the image. + */ +IEM_CIMPL_DEF_3(iemCImpl_frstor, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc) +{ + RTCPTRUNION uPtr; + VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 94 : 108, + iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R, 3 /** @todo ?*/ ); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr); + PCRTFLOAT80U paRegs = (PCRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28)); + for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++) + { + pFpuCtx->aRegs[i].au32[0] = paRegs[i].au32[0]; + pFpuCtx->aRegs[i].au32[1] = paRegs[i].au32[1]; + pFpuCtx->aRegs[i].au32[2] = paRegs[i].au16[4]; + pFpuCtx->aRegs[i].au32[3] = 0; + } + + rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uPtr.pv, IEM_ACCESS_DATA_R); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + iemHlpUsedFpu(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'FLDCW'. + * + * @param u16Fcw The new FCW. + */ +IEM_CIMPL_DEF_1(iemCImpl_fldcw, uint16_t, u16Fcw) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87); + + /** @todo Testcase: Check what happens when trying to load X86_FCW_PC_RSVD. */ + /** @todo Testcase: Try see what happens when trying to set undefined bits + * (other than 6 and 7). Currently ignoring them. */ + /** @todo Testcase: Test that it raises and loweres the FPU exception bits + * according to FSW. (This is what is currently implemented.) */ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + pFpuCtx->FCW = u16Fcw & (~X86_FCW_ZERO_MASK | X86_FCW_IC_MASK); /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */ +#ifdef LOG_ENABLED + uint16_t fOldFsw = pFpuCtx->FSW; +#endif + iemFpuRecalcExceptionStatus(pFpuCtx); +#ifdef LOG_ENABLED + if ((pFpuCtx->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES)) + Log11(("fldcw: %04x:%08RX64: %s FPU exception (FCW=%#x, FSW %#x -> %#x)\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + fOldFsw & X86_FSW_ES ? "Supressed" : "Raised", pFpuCtx->FCW, fOldFsw, pFpuCtx->FSW)); +#endif + + /* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */ + iemHlpUsedFpu(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + + +/** + * Implements the underflow case of fxch. + * + * @param iStReg The other stack register. + */ +IEM_CIMPL_DEF_1(iemCImpl_fxch_underflow, uint8_t, iStReg) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87); + + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + unsigned const iReg1 = X86_FSW_TOP_GET(pFpuCtx->FSW); + unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK; + Assert(!(RT_BIT(iReg1) & pFpuCtx->FTW) || !(RT_BIT(iReg2) & pFpuCtx->FTW)); + + /** @todo Testcase: fxch underflow. Making assumptions that underflowed + * registers are read as QNaN and then exchanged. This could be + * wrong... */ + if (pFpuCtx->FCW & X86_FCW_IM) + { + if (RT_BIT(iReg1) & pFpuCtx->FTW) + { + if (RT_BIT(iReg2) & pFpuCtx->FTW) + iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80); + else + pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[iStReg].r80; + iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80); + } + else + { + pFpuCtx->aRegs[iStReg].r80 = pFpuCtx->aRegs[0].r80; + iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80); + } + pFpuCtx->FSW &= ~X86_FSW_C_MASK; + pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF; + } + else + { + /* raise underflow exception, don't change anything. */ + pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_XCPT_MASK); + pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B; + Log11(("fxch: %04x:%08RX64: Underflow exception (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW)); + } + + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemHlpUsedFpu(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'FCOMI', 'FCOMIP', 'FUCOMI', and 'FUCOMIP'. + * + * @param iStReg The other stack register. + * @param pfnAImpl The assembly comparison implementation. + * @param fPop Whether we should pop the stack when done or not. + */ +IEM_CIMPL_DEF_3(iemCImpl_fcomi_fucomi, uint8_t, iStReg, PFNIEMAIMPLFPUR80EFL, pfnAImpl, bool, fPop) +{ + Assert(iStReg < 8); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87); + + /* + * Raise exceptions. + */ + if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) + return iemRaiseDeviceNotAvailable(pVCpu); + + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + uint16_t u16Fsw = pFpuCtx->FSW; + if (u16Fsw & X86_FSW_ES) + return iemRaiseMathFault(pVCpu); + + /* + * Check if any of the register accesses causes #SF + #IA. + */ + unsigned const iReg1 = X86_FSW_TOP_GET(u16Fsw); + unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK; + if ((pFpuCtx->FTW & (RT_BIT(iReg1) | RT_BIT(iReg2))) == (RT_BIT(iReg1) | RT_BIT(iReg2))) + { + uint32_t u32Eflags = pfnAImpl(pFpuCtx, &u16Fsw, &pFpuCtx->aRegs[0].r80, &pFpuCtx->aRegs[iStReg].r80); + + pFpuCtx->FSW &= ~X86_FSW_C1; + pFpuCtx->FSW |= u16Fsw & ~X86_FSW_TOP_MASK; + if ( !(u16Fsw & X86_FSW_IE) + || (pFpuCtx->FCW & X86_FCW_IM) ) + { + pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF); + pVCpu->cpum.GstCtx.eflags.u |= u32Eflags & (X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF); + } + } + else if (pFpuCtx->FCW & X86_FCW_IM) + { + /* Masked underflow. */ + pFpuCtx->FSW &= ~X86_FSW_C1; + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF; + pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF); + pVCpu->cpum.GstCtx.eflags.u |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; + } + else + { + /* Raise underflow - don't touch EFLAGS or TOP. */ + pFpuCtx->FSW &= ~X86_FSW_C1; + pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B; + Log11(("fxch: %04x:%08RX64: Raising IE+SF exception (FSW=%#x)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW)); + fPop = false; + } + + /* + * Pop if necessary. + */ + if (fPop) + { + pFpuCtx->FTW &= ~RT_BIT(iReg1); + iemFpuStackIncTop(pVCpu); + } + + iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx); + iemHlpUsedFpu(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + +/** @} */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h b/src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h new file mode 100644 index 00000000..4bef3cfb --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h @@ -0,0 +1,1773 @@ +/* $Id: IEMAllCImplStrInstr.cpp.h $ */ +/** @file + * IEM - String Instruction Implementation Code Template. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/******************************************************************************* +* Defined Constants And Macros * +*******************************************************************************/ +#if OP_SIZE == 8 +# define OP_rAX al +#elif OP_SIZE == 16 +# define OP_rAX ax +#elif OP_SIZE == 32 +# define OP_rAX eax +#elif OP_SIZE == 64 +# define OP_rAX rax +#else +# error "Bad OP_SIZE." +#endif +#define OP_TYPE RT_CONCAT3(uint,OP_SIZE,_t) + +#if ADDR_SIZE == 16 +# define ADDR_rDI di +# define ADDR_rSI si +# define ADDR_rCX cx +# define ADDR2_TYPE uint32_t +# define ADDR_VMXSTRIO 0 +#elif ADDR_SIZE == 32 +# define ADDR_rDI edi +# define ADDR_rSI esi +# define ADDR_rCX ecx +# define ADDR2_TYPE uint32_t +# define ADDR_VMXSTRIO 1 +#elif ADDR_SIZE == 64 +# define ADDR_rDI rdi +# define ADDR_rSI rsi +# define ADDR_rCX rcx +# define ADDR2_TYPE uint64_t +# define ADDR_VMXSTRIO 2 +# define IS_64_BIT_CODE(a_pVCpu) (true) +#else +# error "Bad ADDR_SIZE." +#endif +#define ADDR_TYPE RT_CONCAT3(uint,ADDR_SIZE,_t) + +#if ADDR_SIZE == 64 || OP_SIZE == 64 +# define IS_64_BIT_CODE(a_pVCpu) (true) +#elif ADDR_SIZE == 32 +# define IS_64_BIT_CODE(a_pVCpu) ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT) +#else +# define IS_64_BIT_CODE(a_pVCpu) (false) +#endif + +/** @def IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN + * Used in the outer (page-by-page) loop to check for reasons for returnning + * before completing the instruction. In raw-mode we temporarily enable + * interrupts to let the host interrupt us. We cannot let big string operations + * hog the CPU, especially not in raw-mode. + */ +#define IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(a_pVM, a_pVCpu, a_fEflags) \ + do { \ + if (RT_LIKELY( !VMCPU_FF_IS_ANY_SET(a_pVCpu, (a_fEflags) & X86_EFL_IF ? VMCPU_FF_YIELD_REPSTR_MASK \ + : VMCPU_FF_YIELD_REPSTR_NOINT_MASK) \ + && !VM_FF_IS_ANY_SET(a_pVM, VM_FF_YIELD_REPSTR_MASK) \ + )) \ + { /* probable */ } \ + else \ + { \ + LogFlow(("%s: Leaving early (outer)! ffcpu=%#RX64 ffvm=%#x\n", \ + __FUNCTION__, (uint64_t)(a_pVCpu)->fLocalForcedActions, (a_pVM)->fGlobalForcedActions)); \ + return VINF_SUCCESS; \ + } \ + } while (0) + +/** @def IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN + * This is used in some of the inner loops to make sure we respond immediately + * to VMCPU_FF_IOM as well as outside requests. Use this for expensive + * instructions. Use IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN for + * ones that are typically cheap. */ +#define IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(a_pVM, a_pVCpu, a_fExitExpr) \ + do { \ + if (RT_LIKELY( ( !VMCPU_FF_IS_ANY_SET(a_pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_REPSTR_MASK) \ + && !VM_FF_IS_ANY_SET(a_pVM, VM_FF_HIGH_PRIORITY_POST_REPSTR_MASK)) \ + || (a_fExitExpr) )) \ + { /* very likely */ } \ + else \ + { \ + LogFlow(("%s: Leaving early (inner)! ffcpu=%#RX64 ffvm=%#x\n", \ + __FUNCTION__, (uint64_t)(a_pVCpu)->fLocalForcedActions, (a_pVM)->fGlobalForcedActions)); \ + return VINF_SUCCESS; \ + } \ + } while (0) + + +/** @def IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN + * This is used in the inner loops where + * IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN isn't used. It only + * checks the CPU FFs so that we respond immediately to the pending IOM FF + * (status code is hidden in IEMCPU::rcPassUp by IEM memory commit code). + */ +#define IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(a_pVM, a_pVCpu, a_fExitExpr) \ + do { \ + if (RT_LIKELY( !VMCPU_FF_IS_ANY_SET(a_pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_REPSTR_MASK) \ + || (a_fExitExpr) )) \ + { /* very likely */ } \ + else \ + { \ + LogFlow(("%s: Leaving early (inner)! ffcpu=%#RX64 (ffvm=%#x)\n", \ + __FUNCTION__, (uint64_t)(a_pVCpu)->fLocalForcedActions, (a_pVM)->fGlobalForcedActions)); \ + return VINF_SUCCESS; \ + } \ + } while (0) + + +/** + * Implements 'REPE CMPS'. + */ +IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_repe_cmps_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Setup. + */ + ADDR_TYPE uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX; + if (uCounterReg == 0) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg) | CPUMCTX_EXTRN_ES); + + PCCPUMSELREGHID pSrc1Hid = iemSRegGetHid(pVCpu, iEffSeg); + uint64_t uSrc1Base = 0; /* gcc may not be used uninitialized */ + VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrc1Hid, iEffSeg, &uSrc1Base); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + uint64_t uSrc2Base = 0; /* gcc may not be used uninitialized */ + rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uSrc2Base); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + int8_t const cbIncr = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8); + ADDR_TYPE uSrc1AddrReg = pVCpu->cpum.GstCtx.ADDR_rSI; + ADDR_TYPE uSrc2AddrReg = pVCpu->cpum.GstCtx.ADDR_rDI; + uint32_t uEFlags = pVCpu->cpum.GstCtx.eflags.u; + + /* + * The loop. + */ + for (;;) + { + /* + * Do segmentation and virtual page stuff. + */ + ADDR2_TYPE uVirtSrc1Addr = uSrc1AddrReg + (ADDR2_TYPE)uSrc1Base; + ADDR2_TYPE uVirtSrc2Addr = uSrc2AddrReg + (ADDR2_TYPE)uSrc2Base; + uint32_t cLeftSrc1Page = (GUEST_PAGE_SIZE - (uVirtSrc1Addr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + if (cLeftSrc1Page > uCounterReg) + cLeftSrc1Page = uCounterReg; + uint32_t cLeftSrc2Page = (GUEST_PAGE_SIZE - (uVirtSrc2Addr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + uint32_t cLeftPage = RT_MIN(cLeftSrc1Page, cLeftSrc2Page); + + if ( cLeftPage > 0 /* can be null if unaligned, do one fallback round. */ + && cbIncr > 0 /** @todo Optimize reverse direction string ops. */ + && ( IS_64_BIT_CODE(pVCpu) + || ( uSrc1AddrReg < pSrc1Hid->u32Limit + && uSrc1AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrc1Hid->u32Limit + && uSrc2AddrReg < pVCpu->cpum.GstCtx.es.u32Limit + && uSrc2AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit) + ) + ) + { + RTGCPHYS GCPhysSrc1Mem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc1Addr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrc1Mem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + RTGCPHYS GCPhysSrc2Mem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc2Addr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrc2Mem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * If we can map the page without trouble, do a block processing + * until the end of the current page. + */ + PGMPAGEMAPLOCK PgLockSrc2Mem; + OP_TYPE const *puSrc2Mem; + rcStrict = iemMemPageMap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, (void **)&puSrc2Mem, &PgLockSrc2Mem); + if (rcStrict == VINF_SUCCESS) + { + PGMPAGEMAPLOCK PgLockSrc1Mem; + OP_TYPE const *puSrc1Mem; + rcStrict = iemMemPageMap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, (void **)&puSrc1Mem, &PgLockSrc1Mem); + if (rcStrict == VINF_SUCCESS) + { + if (!memcmp(puSrc2Mem, puSrc1Mem, cLeftPage * (OP_SIZE / 8))) + { + /* All matches, only compare the last itme to get the right eflags. */ + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[cLeftPage-1], puSrc2Mem[cLeftPage-1], &uEFlags); + uSrc1AddrReg += cLeftPage * cbIncr; + uSrc2AddrReg += cLeftPage * cbIncr; + uCounterReg -= cLeftPage; + } + else + { + /* Some mismatch, compare each item (and keep volatile + memory in mind). */ + uint32_t off = 0; + do + { + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[off], puSrc2Mem[off], &uEFlags); + off++; + } while ( off < cLeftPage + && (uEFlags & X86_EFL_ZF)); + uSrc1AddrReg += cbIncr * off; + uSrc2AddrReg += cbIncr * off; + uCounterReg -= off; + } + + /* Update the registers before looping. */ + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg; + pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg; + pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg; + pVCpu->cpum.GstCtx.eflags.u = uEFlags; + + iemMemPageUnmap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, puSrc1Mem, &PgLockSrc1Mem); + iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem); + if ( uCounterReg == 0 + || !(uEFlags & X86_EFL_ZF)) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags); + continue; + } + iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem); + } + } + + /* + * Fallback - slow processing till the end of the current page. + * In the cross page boundrary case we will end up here with cLeftPage + * as 0, we execute one loop then. + */ + do + { + OP_TYPE uValue1; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue1, iEffSeg, uSrc1AddrReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + OP_TYPE uValue2; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue2, X86_SREG_ES, uSrc2AddrReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)(&uValue1, uValue2, &uEFlags); + + pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg; + pVCpu->cpum.GstCtx.eflags.u = uEFlags; + cLeftPage--; + IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || !(uEFlags & X86_EFL_ZF)); + } while ( (int32_t)cLeftPage > 0 + && (uEFlags & X86_EFL_ZF)); + + /* + * Next page? Must check for interrupts and stuff here. + */ + if ( uCounterReg == 0 + || !(uEFlags & X86_EFL_ZF)) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags); + } + + /* + * Done. + */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'REPNE CMPS'. + */ +IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_repne_cmps_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Setup. + */ + ADDR_TYPE uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX; + if (uCounterReg == 0) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg) | CPUMCTX_EXTRN_ES); + + PCCPUMSELREGHID pSrc1Hid = iemSRegGetHid(pVCpu, iEffSeg); + uint64_t uSrc1Base = 0; /* gcc may not be used uninitialized */; + VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrc1Hid, iEffSeg, &uSrc1Base); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + uint64_t uSrc2Base = 0; /* gcc may not be used uninitialized */ + rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uSrc2Base); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + int8_t const cbIncr = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8); + ADDR_TYPE uSrc1AddrReg = pVCpu->cpum.GstCtx.ADDR_rSI; + ADDR_TYPE uSrc2AddrReg = pVCpu->cpum.GstCtx.ADDR_rDI; + uint32_t uEFlags = pVCpu->cpum.GstCtx.eflags.u; + + /* + * The loop. + */ + for (;;) + { + /* + * Do segmentation and virtual page stuff. + */ + ADDR2_TYPE uVirtSrc1Addr = uSrc1AddrReg + (ADDR2_TYPE)uSrc1Base; + ADDR2_TYPE uVirtSrc2Addr = uSrc2AddrReg + (ADDR2_TYPE)uSrc2Base; + uint32_t cLeftSrc1Page = (GUEST_PAGE_SIZE - (uVirtSrc1Addr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + if (cLeftSrc1Page > uCounterReg) + cLeftSrc1Page = uCounterReg; + uint32_t cLeftSrc2Page = (GUEST_PAGE_SIZE - (uVirtSrc2Addr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + uint32_t cLeftPage = RT_MIN(cLeftSrc1Page, cLeftSrc2Page); + + if ( cLeftPage > 0 /* can be null if unaligned, do one fallback round. */ + && cbIncr > 0 /** @todo Optimize reverse direction string ops. */ + && ( IS_64_BIT_CODE(pVCpu) + || ( uSrc1AddrReg < pSrc1Hid->u32Limit + && uSrc1AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrc1Hid->u32Limit + && uSrc2AddrReg < pVCpu->cpum.GstCtx.es.u32Limit + && uSrc2AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit) + ) + ) + { + RTGCPHYS GCPhysSrc1Mem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc1Addr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrc1Mem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + RTGCPHYS GCPhysSrc2Mem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc2Addr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrc2Mem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * If we can map the page without trouble, do a block processing + * until the end of the current page. + */ + OP_TYPE const *puSrc2Mem; + PGMPAGEMAPLOCK PgLockSrc2Mem; + rcStrict = iemMemPageMap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, (void **)&puSrc2Mem, &PgLockSrc2Mem); + if (rcStrict == VINF_SUCCESS) + { + OP_TYPE const *puSrc1Mem; + PGMPAGEMAPLOCK PgLockSrc1Mem; + rcStrict = iemMemPageMap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, (void **)&puSrc1Mem, &PgLockSrc1Mem); + if (rcStrict == VINF_SUCCESS) + { + if (memcmp(puSrc2Mem, puSrc1Mem, cLeftPage * (OP_SIZE / 8))) + { + /* All matches, only compare the last item to get the right eflags. */ + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[cLeftPage-1], puSrc2Mem[cLeftPage-1], &uEFlags); + uSrc1AddrReg += cLeftPage * cbIncr; + uSrc2AddrReg += cLeftPage * cbIncr; + uCounterReg -= cLeftPage; + } + else + { + /* Some mismatch, compare each item (and keep volatile + memory in mind). */ + uint32_t off = 0; + do + { + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[off], puSrc2Mem[off], &uEFlags); + off++; + } while ( off < cLeftPage + && !(uEFlags & X86_EFL_ZF)); + uSrc1AddrReg += cbIncr * off; + uSrc2AddrReg += cbIncr * off; + uCounterReg -= off; + } + + /* Update the registers before looping. */ + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg; + pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg; + pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg; + pVCpu->cpum.GstCtx.eflags.u = uEFlags; + + iemMemPageUnmap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, puSrc1Mem, &PgLockSrc1Mem); + iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem); + if ( uCounterReg == 0 + || (uEFlags & X86_EFL_ZF)) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags); + continue; + } + iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem); + } + } + + /* + * Fallback - slow processing till the end of the current page. + * In the cross page boundrary case we will end up here with cLeftPage + * as 0, we execute one loop then. + */ + do + { + OP_TYPE uValue1; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue1, iEffSeg, uSrc1AddrReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + OP_TYPE uValue2; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue2, X86_SREG_ES, uSrc2AddrReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)(&uValue1, uValue2, &uEFlags); + + pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg; + pVCpu->cpum.GstCtx.eflags.u = uEFlags; + cLeftPage--; + IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || (uEFlags & X86_EFL_ZF)); + } while ( (int32_t)cLeftPage > 0 + && !(uEFlags & X86_EFL_ZF)); + + /* + * Next page? Must check for interrupts and stuff here. + */ + if ( uCounterReg == 0 + || (uEFlags & X86_EFL_ZF)) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags); + } + + /* + * Done. + */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'REPE SCAS'. + */ +IEM_CIMPL_DEF_0(RT_CONCAT4(iemCImpl_repe_scas_,OP_rAX,_m,ADDR_SIZE)) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Setup. + */ + ADDR_TYPE uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX; + if (uCounterReg == 0) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES); + uint64_t uBaseAddr = 0; /* gcc may not be used uninitialized */ + VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + int8_t const cbIncr = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8); + OP_TYPE const uValueReg = pVCpu->cpum.GstCtx.OP_rAX; + ADDR_TYPE uAddrReg = pVCpu->cpum.GstCtx.ADDR_rDI; + uint32_t uEFlags = pVCpu->cpum.GstCtx.eflags.u; + + /* + * The loop. + */ + for (;;) + { + /* + * Do segmentation and virtual page stuff. + */ + ADDR2_TYPE uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr; + uint32_t cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + if (cLeftPage > uCounterReg) + cLeftPage = uCounterReg; + if ( cLeftPage > 0 /* can be null if unaligned, do one fallback round. */ + && cbIncr > 0 /** @todo Implement reverse direction string ops. */ + && ( IS_64_BIT_CODE(pVCpu) + || ( uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit + && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit) + ) + ) + { + RTGCPHYS GCPhysMem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * If we can map the page without trouble, do a block processing + * until the end of the current page. + */ + PGMPAGEMAPLOCK PgLockMem; + OP_TYPE const *puMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem); + if (rcStrict == VINF_SUCCESS) + { + /* Search till we find a mismatching item. */ + OP_TYPE uTmpValue; + bool fQuit; + uint32_t i = 0; + do + { + uTmpValue = puMem[i++]; + fQuit = uTmpValue != uValueReg; + } while (i < cLeftPage && !fQuit); + + /* Update the regs. */ + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags); + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= i; + pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += i * cbIncr; + pVCpu->cpum.GstCtx.eflags.u = uEFlags; + Assert(!(uEFlags & X86_EFL_ZF) == fQuit); + iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem); + if ( fQuit + || uCounterReg == 0) + break; + + /* If unaligned, we drop thru and do the page crossing access + below. Otherwise, do the next page. */ + if (!(uVirtAddr & (OP_SIZE / 8 - 1))) + { + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags); + continue; + } + cLeftPage = 0; + } + } + + /* + * Fallback - slow processing till the end of the current page. + * In the cross page boundrary case we will end up here with cLeftPage + * as 0, we execute one loop then. + */ + do + { + OP_TYPE uTmpValue; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uTmpValue, X86_SREG_ES, uAddrReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags); + + pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg; + pVCpu->cpum.GstCtx.eflags.u = uEFlags; + cLeftPage--; + IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || !(uEFlags & X86_EFL_ZF)); + } while ( (int32_t)cLeftPage > 0 + && (uEFlags & X86_EFL_ZF)); + + /* + * Next page? Must check for interrupts and stuff here. + */ + if ( uCounterReg == 0 + || !(uEFlags & X86_EFL_ZF)) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags); + } + + /* + * Done. + */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'REPNE SCAS'. + */ +IEM_CIMPL_DEF_0(RT_CONCAT4(iemCImpl_repne_scas_,OP_rAX,_m,ADDR_SIZE)) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Setup. + */ + ADDR_TYPE uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX; + if (uCounterReg == 0) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES); + uint64_t uBaseAddr = 0; /* gcc may not be used uninitialized */ + VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + int8_t const cbIncr = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8); + OP_TYPE const uValueReg = pVCpu->cpum.GstCtx.OP_rAX; + ADDR_TYPE uAddrReg = pVCpu->cpum.GstCtx.ADDR_rDI; + uint32_t uEFlags = pVCpu->cpum.GstCtx.eflags.u; + + /* + * The loop. + */ + for (;;) + { + /* + * Do segmentation and virtual page stuff. + */ + ADDR2_TYPE uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr; + uint32_t cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + if (cLeftPage > uCounterReg) + cLeftPage = uCounterReg; + if ( cLeftPage > 0 /* can be null if unaligned, do one fallback round. */ + && cbIncr > 0 /** @todo Implement reverse direction string ops. */ + && ( IS_64_BIT_CODE(pVCpu) + || ( uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit + && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit) + ) + ) + { + RTGCPHYS GCPhysMem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * If we can map the page without trouble, do a block processing + * until the end of the current page. + */ + PGMPAGEMAPLOCK PgLockMem; + OP_TYPE const *puMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem); + if (rcStrict == VINF_SUCCESS) + { + /* Search till we find a mismatching item. */ + OP_TYPE uTmpValue; + bool fQuit; + uint32_t i = 0; + do + { + uTmpValue = puMem[i++]; + fQuit = uTmpValue == uValueReg; + } while (i < cLeftPage && !fQuit); + + /* Update the regs. */ + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags); + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= i; + pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += i * cbIncr; + pVCpu->cpum.GstCtx.eflags.u = uEFlags; + Assert(!!(uEFlags & X86_EFL_ZF) == fQuit); + iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem); + if ( fQuit + || uCounterReg == 0) + break; + + /* If unaligned, we drop thru and do the page crossing access + below. Otherwise, do the next page. */ + if (!(uVirtAddr & (OP_SIZE / 8 - 1))) + { + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags); + continue; + } + cLeftPage = 0; + } + } + + /* + * Fallback - slow processing till the end of the current page. + * In the cross page boundrary case we will end up here with cLeftPage + * as 0, we execute one loop then. + */ + do + { + OP_TYPE uTmpValue; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uTmpValue, X86_SREG_ES, uAddrReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags); + pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg; + pVCpu->cpum.GstCtx.eflags.u = uEFlags; + cLeftPage--; + IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || (uEFlags & X86_EFL_ZF)); + } while ( (int32_t)cLeftPage > 0 + && !(uEFlags & X86_EFL_ZF)); + + /* + * Next page? Must check for interrupts and stuff here. + */ + if ( uCounterReg == 0 + || (uEFlags & X86_EFL_ZF)) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags); + } + + /* + * Done. + */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + + + +/** + * Implements 'REP MOVS'. + */ +IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_rep_movs_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Setup. + */ + ADDR_TYPE uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX; + if (uCounterReg == 0) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg) | CPUMCTX_EXTRN_ES); + + PCCPUMSELREGHID pSrcHid = iemSRegGetHid(pVCpu, iEffSeg); + uint64_t uSrcBase = 0; /* gcc may not be used uninitialized */ + VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrcHid, iEffSeg, &uSrcBase); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + uint64_t uDstBase = 0; /* gcc may not be used uninitialized */ + rcStrict = iemMemSegCheckWriteAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uDstBase); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + int8_t const cbIncr = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8); + ADDR_TYPE uSrcAddrReg = pVCpu->cpum.GstCtx.ADDR_rSI; + ADDR_TYPE uDstAddrReg = pVCpu->cpum.GstCtx.ADDR_rDI; + + /* + * Be careful with handle bypassing. + */ + if (pVCpu->iem.s.fBypassHandlers) + { + Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__)); + return VERR_IEM_ASPECT_NOT_IMPLEMENTED; + } + + /* + * The loop. + */ + for (;;) + { + /* + * Do segmentation and virtual page stuff. + */ + ADDR2_TYPE uVirtSrcAddr = uSrcAddrReg + (ADDR2_TYPE)uSrcBase; + ADDR2_TYPE uVirtDstAddr = uDstAddrReg + (ADDR2_TYPE)uDstBase; + uint32_t cLeftSrcPage = (GUEST_PAGE_SIZE - (uVirtSrcAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + if (cLeftSrcPage > uCounterReg) + cLeftSrcPage = uCounterReg; + uint32_t cLeftDstPage = (GUEST_PAGE_SIZE - (uVirtDstAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + uint32_t cLeftPage = RT_MIN(cLeftSrcPage, cLeftDstPage); + + if ( cLeftPage > 0 /* can be null if unaligned, do one fallback round. */ + && cbIncr > 0 /** @todo Implement reverse direction string ops. */ + && ( IS_64_BIT_CODE(pVCpu) + || ( uSrcAddrReg < pSrcHid->u32Limit + && uSrcAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrcHid->u32Limit + && uDstAddrReg < pVCpu->cpum.GstCtx.es.u32Limit + && uDstAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit) + ) + ) + { + RTGCPHYS GCPhysSrcMem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrcAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrcMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + RTGCPHYS GCPhysDstMem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtDstAddr, OP_SIZE / 8, IEM_ACCESS_DATA_W, &GCPhysDstMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * If we can map the page without trouble, do a block processing + * until the end of the current page. + */ + PGMPAGEMAPLOCK PgLockDstMem; + OP_TYPE *puDstMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysDstMem, IEM_ACCESS_DATA_W, (void **)&puDstMem, &PgLockDstMem); + if (rcStrict == VINF_SUCCESS) + { + PGMPAGEMAPLOCK PgLockSrcMem; + OP_TYPE const *puSrcMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysSrcMem, IEM_ACCESS_DATA_R, (void **)&puSrcMem, &PgLockSrcMem); + if (rcStrict == VINF_SUCCESS) + { + Assert( (GCPhysSrcMem >> GUEST_PAGE_SHIFT) != (GCPhysDstMem >> GUEST_PAGE_SHIFT) + || ((uintptr_t)puSrcMem >> GUEST_PAGE_SHIFT) == ((uintptr_t)puDstMem >> GUEST_PAGE_SHIFT)); + + /* Perform the operation exactly (don't use memcpy to avoid + having to consider how its implementation would affect + any overlapping source and destination area). */ + OP_TYPE const *puSrcCur = puSrcMem; + OP_TYPE *puDstCur = puDstMem; + uint32_t cTodo = cLeftPage; + while (cTodo-- > 0) + *puDstCur++ = *puSrcCur++; + + /* Update the registers. */ + pVCpu->cpum.GstCtx.ADDR_rSI = uSrcAddrReg += cLeftPage * cbIncr; + pVCpu->cpum.GstCtx.ADDR_rDI = uDstAddrReg += cLeftPage * cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage; + + iemMemPageUnmap(pVCpu, GCPhysSrcMem, IEM_ACCESS_DATA_R, puSrcMem, &PgLockSrcMem); + iemMemPageUnmap(pVCpu, GCPhysDstMem, IEM_ACCESS_DATA_W, puDstMem, &PgLockDstMem); + + if (uCounterReg == 0) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + continue; + } + iemMemPageUnmap(pVCpu, GCPhysDstMem, IEM_ACCESS_DATA_W, puDstMem, &PgLockDstMem); + } + } + + /* + * Fallback - slow processing till the end of the current page. + * In the cross page boundrary case we will end up here with cLeftPage + * as 0, we execute one loop then. + */ + do + { + OP_TYPE uValue; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue, iEffSeg, uSrcAddrReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + rcStrict = RT_CONCAT(iemMemStoreDataU,OP_SIZE)(pVCpu, X86_SREG_ES, uDstAddrReg, uValue); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + pVCpu->cpum.GstCtx.ADDR_rSI = uSrcAddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rDI = uDstAddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg; + cLeftPage--; + IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0); + } while ((int32_t)cLeftPage > 0); + + /* + * Next page. Must check for interrupts and stuff here. + */ + if (uCounterReg == 0) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + } + + /* + * Done. + */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'REP STOS'. + */ +IEM_CIMPL_DEF_0(RT_CONCAT4(iemCImpl_stos_,OP_rAX,_m,ADDR_SIZE)) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Setup. + */ + ADDR_TYPE uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX; + if (uCounterReg == 0) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES); + + uint64_t uBaseAddr = 0; /* gcc may not be used uninitialized */ + VBOXSTRICTRC rcStrict = iemMemSegCheckWriteAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + int8_t const cbIncr = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8); + OP_TYPE const uValue = pVCpu->cpum.GstCtx.OP_rAX; + ADDR_TYPE uAddrReg = pVCpu->cpum.GstCtx.ADDR_rDI; + + /* + * Be careful with handle bypassing. + */ + /** @todo Permit doing a page if correctly aligned. */ + if (pVCpu->iem.s.fBypassHandlers) + { + Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__)); + return VERR_IEM_ASPECT_NOT_IMPLEMENTED; + } + + /* + * The loop. + */ + for (;;) + { + /* + * Do segmentation and virtual page stuff. + */ + ADDR2_TYPE uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr; + uint32_t cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + if (cLeftPage > uCounterReg) + cLeftPage = uCounterReg; + if ( cLeftPage > 0 /* can be null if unaligned, do one fallback round. */ + && cbIncr > 0 /** @todo Implement reverse direction string ops. */ + && ( IS_64_BIT_CODE(pVCpu) + || ( uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit + && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit) + ) + ) + { + RTGCPHYS GCPhysMem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_W, &GCPhysMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * If we can map the page without trouble, do a block processing + * until the end of the current page. + */ + PGMPAGEMAPLOCK PgLockMem; + OP_TYPE *puMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, (void **)&puMem, &PgLockMem); + if (rcStrict == VINF_SUCCESS) + { + /* Update the regs first so we can loop on cLeftPage. */ + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage; + pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cLeftPage * cbIncr; + + /* Do the memsetting. */ +#if OP_SIZE == 8 + memset(puMem, uValue, cLeftPage); +/*#elif OP_SIZE == 32 + ASMMemFill32(puMem, cLeftPage * (OP_SIZE / 8), uValue);*/ +#else + while (cLeftPage-- > 0) + *puMem++ = uValue; +#endif + + iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, puMem, &PgLockMem); + + if (uCounterReg == 0) + break; + + /* If unaligned, we drop thru and do the page crossing access + below. Otherwise, do the next page. */ + if (!(uVirtAddr & (OP_SIZE / 8 - 1))) + { + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + continue; + } + cLeftPage = 0; + } + /* If we got an invalid physical address in the page table, just skip + ahead to the next page or the counter reaches zero. This crazy + optimization is for a buggy EFI firmware that's driving me nuts. */ + else if (rcStrict == VERR_PGM_PHYS_TLB_UNASSIGNED) + { + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage; + pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cLeftPage * cbIncr; + if (uCounterReg == 0) + break; + if (!(uVirtAddr & (OP_SIZE / 8 - 1))) + { + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + continue; + } + } + } + + /* + * Fallback - slow processing till the end of the current page. + * In the cross page boundrary case we will end up here with cLeftPage + * as 0, we execute one loop then. + */ + do + { + rcStrict = RT_CONCAT(iemMemStoreDataU,OP_SIZE)(pVCpu, X86_SREG_ES, uAddrReg, uValue); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg; + cLeftPage--; + IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0); + } while ((int32_t)cLeftPage > 0); + + /* + * Next page. Must check for interrupts and stuff here. + */ + if (uCounterReg == 0) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + } + + /* + * Done. + */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'REP LODS'. + */ +IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_lods_,OP_rAX,_m,ADDR_SIZE), int8_t, iEffSeg) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Setup. + */ + ADDR_TYPE uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX; + if (uCounterReg == 0) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg)); + PCCPUMSELREGHID pSrcHid = iemSRegGetHid(pVCpu, iEffSeg); + uint64_t uBaseAddr = 0; /* gcc may not be used uninitialized */ + VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrcHid, iEffSeg, &uBaseAddr); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + int8_t const cbIncr = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8); + ADDR_TYPE uAddrReg = pVCpu->cpum.GstCtx.ADDR_rSI; + + /* + * The loop. + */ + for (;;) + { + /* + * Do segmentation and virtual page stuff. + */ + ADDR2_TYPE uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr; + uint32_t cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + if (cLeftPage > uCounterReg) + cLeftPage = uCounterReg; + if ( cLeftPage > 0 /* can be null if unaligned, do one fallback round. */ + && cbIncr > 0 /** @todo Implement reverse direction string ops. */ + && ( IS_64_BIT_CODE(pVCpu) + || ( uAddrReg < pSrcHid->u32Limit + && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrcHid->u32Limit) + ) + ) + { + RTGCPHYS GCPhysMem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * If we can map the page without trouble, we can get away with + * just reading the last value on the page. + */ + PGMPAGEMAPLOCK PgLockMem; + OP_TYPE const *puMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem); + if (rcStrict == VINF_SUCCESS) + { + /* Only get the last byte, the rest doesn't matter in direct access mode. */ +#if OP_SIZE == 32 + pVCpu->cpum.GstCtx.rax = puMem[cLeftPage - 1]; +#else + pVCpu->cpum.GstCtx.OP_rAX = puMem[cLeftPage - 1]; +#endif + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage; + pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg += cLeftPage * cbIncr; + iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem); + + if (uCounterReg == 0) + break; + + /* If unaligned, we drop thru and do the page crossing access + below. Otherwise, do the next page. */ + if (!(uVirtAddr & (OP_SIZE / 8 - 1))) + { + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + continue; + } + cLeftPage = 0; + } + } + + /* + * Fallback - slow processing till the end of the current page. + * In the cross page boundrary case we will end up here with cLeftPage + * as 0, we execute one loop then. + */ + do + { + OP_TYPE uTmpValue; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uTmpValue, iEffSeg, uAddrReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; +#if OP_SIZE == 32 + pVCpu->cpum.GstCtx.rax = uTmpValue; +#else + pVCpu->cpum.GstCtx.OP_rAX = uTmpValue; +#endif + pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg; + cLeftPage--; + IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0); + } while ((int32_t)cLeftPage > 0); + + if (rcStrict != VINF_SUCCESS) + break; + + /* + * Next page. Must check for interrupts and stuff here. + */ + if (uCounterReg == 0) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + } + + /* + * Done. + */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +#if OP_SIZE != 64 + +/** + * Implements 'INS' (no rep) + */ +IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_ins_op,OP_SIZE,_addr,ADDR_SIZE), bool, fIoChecked) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VBOXSTRICTRC rcStrict; + + /* + * Be careful with handle bypassing. + */ + if (pVCpu->iem.s.fBypassHandlers) + { + Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__)); + return VERR_IEM_ASPECT_NOT_IMPLEMENTED; + } + + /* + * ASSUMES the #GP for I/O permission is taken first, then any #GP for + * segmentation and finally any #PF due to virtual address translation. + * ASSUMES nothing is read from the I/O port before traps are taken. + */ + if (!fIoChecked) + { + rcStrict = iemHlpCheckPortIOPermission(pVCpu, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + /* + * Check nested-guest I/O intercepts. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VMXEXITINSTRINFO ExitInstrInfo; + ExitInstrInfo.u = 0; + ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO; + ExitInstrInfo.StrIo.iSegReg = X86_SREG_ES; + rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_INS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, false /* fRep */, + ExitInstrInfo, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT)) + { + rcStrict = iemSvmHandleIOIntercept(pVCpu, pVCpu->cpum.GstCtx.dx, SVMIOIOTYPE_IN, OP_SIZE / 8, ADDR_SIZE, X86_SREG_ES, + false /* fRep */, true /* fStrIo */, cbInstr); + if (rcStrict == VINF_SVM_VMEXIT) + return VINF_SUCCESS; + if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE) + { + Log(("iemCImpl_ins_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", pVCpu->cpum.GstCtx.dx, + OP_SIZE / 8, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } +#endif + + OP_TYPE *puMem; + rcStrict = iemMemMap(pVCpu, (void **)&puMem, OP_SIZE / 8, X86_SREG_ES, pVCpu->cpum.GstCtx.ADDR_rDI, + IEM_ACCESS_DATA_W, OP_SIZE / 8 - 1); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + uint32_t u32Value = 0; + rcStrict = IOMIOPortRead(pVM, pVCpu, pVCpu->cpum.GstCtx.dx, &u32Value, OP_SIZE / 8); + if (IOM_SUCCESS(rcStrict)) + { + /** + * @todo I/O breakpoint support for INS + */ + *puMem = (OP_TYPE)u32Value; +# ifdef IN_RING3 + VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmap(pVCpu, puMem, IEM_ACCESS_DATA_W); +# else + VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmapPostponeTroubleToR3(pVCpu, puMem, IEM_ACCESS_DATA_W); +# endif + if (RT_LIKELY(rcStrict2 == VINF_SUCCESS)) + { + if (!pVCpu->cpum.GstCtx.eflags.Bits.u1DF) + pVCpu->cpum.GstCtx.ADDR_rDI += OP_SIZE / 8; + else + pVCpu->cpum.GstCtx.ADDR_rDI -= OP_SIZE / 8; + + /** @todo finish: work out how this should work wrt status codes. Not sure we + * can use iemSetPassUpStatus here, but it depends on what + * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */ + rcStrict2 = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + if (rcStrict2 != VINF_SUCCESS) + { + iemSetPassUpStatus(pVCpu, rcStrict); + rcStrict = rcStrict2; + } + pVCpu->iem.s.cPotentialExits++; + } + else + AssertLogRelMsgFailedReturn(("rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)), RT_FAILURE_NP(rcStrict2) ? rcStrict2 : VERR_IEM_IPE_1); + } + return rcStrict; +} + + +/** + * Implements 'REP INS'. + */ +IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_rep_ins_op,OP_SIZE,_addr,ADDR_SIZE), bool, fIoChecked) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_TR); + + /* + * Setup. + */ + uint16_t const u16Port = pVCpu->cpum.GstCtx.dx; + VBOXSTRICTRC rcStrict; + if (!fIoChecked) + { +/** @todo check if this is too early for ecx=0. */ + rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, OP_SIZE / 8); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + /* + * Check nested-guest I/O intercepts. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VMXEXITINSTRINFO ExitInstrInfo; + ExitInstrInfo.u = 0; + ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO; + ExitInstrInfo.StrIo.iSegReg = X86_SREG_ES; + rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_INS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, true /* fRep */, + ExitInstrInfo, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT)) + { + rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_IN, OP_SIZE / 8, ADDR_SIZE, X86_SREG_ES, true /* fRep */, + true /* fStrIo */, cbInstr); + if (rcStrict == VINF_SVM_VMEXIT) + return VINF_SUCCESS; + if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE) + { + Log(("iemCImpl_rep_ins_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, OP_SIZE / 8, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } +#endif + + ADDR_TYPE uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX; + if (uCounterReg == 0) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + uint64_t uBaseAddr = 0; /* gcc may not be used uninitialized */ + rcStrict = iemMemSegCheckWriteAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + int8_t const cbIncr = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8); + ADDR_TYPE uAddrReg = pVCpu->cpum.GstCtx.ADDR_rDI; + + /* + * Be careful with handle bypassing. + */ + if (pVCpu->iem.s.fBypassHandlers) + { + Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__)); + return VERR_IEM_ASPECT_NOT_IMPLEMENTED; + } + + /* + * The loop. + */ + for (;;) + { + /* + * Do segmentation and virtual page stuff. + */ + ADDR2_TYPE uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr; + uint32_t cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + if (cLeftPage > uCounterReg) + cLeftPage = uCounterReg; + if ( cLeftPage > 0 /* can be null if unaligned, do one fallback round. */ + && cbIncr > 0 /** @todo Implement reverse direction string ops. */ + && ( IS_64_BIT_CODE(pVCpu) + || ( uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit + && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit) + ) + ) + { + RTGCPHYS GCPhysMem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_W, &GCPhysMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * If we can map the page without trouble, use the IOM + * string I/O interface to do the work. + */ + PGMPAGEMAPLOCK PgLockMem; + OP_TYPE *puMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, (void **)&puMem, &PgLockMem); + if (rcStrict == VINF_SUCCESS) + { + uint32_t cTransfers = cLeftPage; + rcStrict = IOMIOPortReadString(pVM, pVCpu, u16Port, puMem, &cTransfers, OP_SIZE / 8); + + uint32_t cActualTransfers = cLeftPage - cTransfers; + Assert(cActualTransfers <= cLeftPage); + pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr * cActualTransfers; + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cActualTransfers; + puMem += cActualTransfers; + + iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, puMem, &PgLockMem); + + if (rcStrict != VINF_SUCCESS) + { + if (IOM_SUCCESS(rcStrict)) + { + /** @todo finish: work out how this should work wrt status codes. Not sure we + * can use iemSetPassUpStatus here, but it depends on what + * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */ + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + if (uCounterReg == 0) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + pVCpu->iem.s.cPotentialExits++; + } + return rcStrict; + } + + /* If unaligned, we drop thru and do the page crossing access + below. Otherwise, do the next page. */ + if (uCounterReg == 0) + break; + if (!(uVirtAddr & (OP_SIZE / 8 - 1))) + { + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + continue; + } + cLeftPage = 0; + } + } + + /* + * Fallback - slow processing till the end of the current page. + * In the cross page boundrary case we will end up here with cLeftPage + * as 0, we execute one loop then. + * + * Note! We ASSUME the CPU will raise #PF or #GP before access the + * I/O port, otherwise it wouldn't really be restartable. + */ + /** @todo investigate what the CPU actually does with \#PF/\#GP + * during INS. */ + do + { + OP_TYPE *puMem; + rcStrict = iemMemMap(pVCpu, (void **)&puMem, OP_SIZE / 8, X86_SREG_ES, uAddrReg, + IEM_ACCESS_DATA_W, OP_SIZE / 8 - 1); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + uint32_t u32Value = 0; + rcStrict = IOMIOPortRead(pVM, pVCpu, u16Port, &u32Value, OP_SIZE / 8); + if (!IOM_SUCCESS(rcStrict)) + { + iemMemRollback(pVCpu); + return rcStrict; + } + + *puMem = (OP_TYPE)u32Value; +# ifdef IN_RING3 + VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmap(pVCpu, puMem, IEM_ACCESS_DATA_W); +# else + VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmapPostponeTroubleToR3(pVCpu, puMem, IEM_ACCESS_DATA_W); +# endif + if (rcStrict2 == VINF_SUCCESS) + { /* likely */ } + else + AssertLogRelMsgFailedReturn(("rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)), + RT_FAILURE(rcStrict2) ? rcStrict2 : VERR_IEM_IPE_1); + + pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg; + + cLeftPage--; + if (rcStrict != VINF_SUCCESS) + { + /** @todo finish: work out how this should work wrt status codes. Not sure we + * can use iemSetPassUpStatus here, but it depends on what + * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */ + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + if (uCounterReg == 0) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + pVCpu->iem.s.cPotentialExits++; + return rcStrict; + } + + IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0); + } while ((int32_t)cLeftPage > 0); + + + /* + * Next page. Must check for interrupts and stuff here. + */ + if (uCounterReg == 0) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + } + + /* + * Done. + */ + pVCpu->iem.s.cPotentialExits++; + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Implements 'OUTS' (no rep) + */ +IEM_CIMPL_DEF_2(RT_CONCAT4(iemCImpl_outs_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg, bool, fIoChecked) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VBOXSTRICTRC rcStrict; + + /* + * ASSUMES the #GP for I/O permission is taken first, then any #GP for + * segmentation and finally any #PF due to virtual address translation. + * ASSUMES nothing is read from the I/O port before traps are taken. + */ + if (!fIoChecked) + { + rcStrict = iemHlpCheckPortIOPermission(pVCpu, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + /* + * Check nested-guest I/O intercepts. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VMXEXITINSTRINFO ExitInstrInfo; + ExitInstrInfo.u = 0; + ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO; + ExitInstrInfo.StrIo.iSegReg = iEffSeg; + rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_OUTS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, false /* fRep */, + ExitInstrInfo, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT)) + { + rcStrict = iemSvmHandleIOIntercept(pVCpu, pVCpu->cpum.GstCtx.dx, SVMIOIOTYPE_OUT, OP_SIZE / 8, ADDR_SIZE, iEffSeg, + false /* fRep */, true /* fStrIo */, cbInstr); + if (rcStrict == VINF_SVM_VMEXIT) + return VINF_SUCCESS; + if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE) + { + Log(("iemCImpl_outs_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", pVCpu->cpum.GstCtx.dx, + OP_SIZE / 8, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } +#endif + + OP_TYPE uValue; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue, iEffSeg, pVCpu->cpum.GstCtx.ADDR_rSI); + if (rcStrict == VINF_SUCCESS) + { + rcStrict = IOMIOPortWrite(pVM, pVCpu, pVCpu->cpum.GstCtx.dx, uValue, OP_SIZE / 8); + if (IOM_SUCCESS(rcStrict)) + { + if (!pVCpu->cpum.GstCtx.eflags.Bits.u1DF) + pVCpu->cpum.GstCtx.ADDR_rSI += OP_SIZE / 8; + else + pVCpu->cpum.GstCtx.ADDR_rSI -= OP_SIZE / 8; + /** @todo finish: work out how this should work wrt status codes. Not sure we + * can use iemSetPassUpStatus here, but it depends on what + * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */ + if (rcStrict != VINF_SUCCESS) + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + pVCpu->iem.s.cPotentialExits++; + } + } + return rcStrict; +} + + +/** + * Implements 'REP OUTS'. + */ +IEM_CIMPL_DEF_2(RT_CONCAT4(iemCImpl_rep_outs_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg, bool, fIoChecked) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Setup. + */ + uint16_t const u16Port = pVCpu->cpum.GstCtx.dx; + VBOXSTRICTRC rcStrict; + if (!fIoChecked) + { +/** @todo check if this is too early for ecx=0. */ + rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, OP_SIZE / 8); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + /* + * Check nested-guest I/O intercepts. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VMXEXITINSTRINFO ExitInstrInfo; + ExitInstrInfo.u = 0; + ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO; + ExitInstrInfo.StrIo.iSegReg = iEffSeg; + rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_OUTS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, true /* fRep */, + ExitInstrInfo, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT)) + { + rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_OUT, OP_SIZE / 8, ADDR_SIZE, iEffSeg, true /* fRep */, + true /* fStrIo */, cbInstr); + if (rcStrict == VINF_SVM_VMEXIT) + return VINF_SUCCESS; + if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE) + { + Log(("iemCImpl_rep_outs_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, OP_SIZE / 8, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } +#endif + + ADDR_TYPE uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX; + if (uCounterReg == 0) + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + + PCCPUMSELREGHID pHid = iemSRegGetHid(pVCpu, iEffSeg); + uint64_t uBaseAddr = 0; /* gcc may not be used uninitialized */ + rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pHid, iEffSeg, &uBaseAddr); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + int8_t const cbIncr = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8); + ADDR_TYPE uAddrReg = pVCpu->cpum.GstCtx.ADDR_rSI; + + /* + * The loop. + */ + for (;;) + { + /* + * Do segmentation and virtual page stuff. + */ + ADDR2_TYPE uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr; + uint32_t cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8); + if (cLeftPage > uCounterReg) + cLeftPage = uCounterReg; + if ( cLeftPage > 0 /* can be null if unaligned, do one fallback round. */ + && cbIncr > 0 /** @todo Implement reverse direction string ops. */ + && ( IS_64_BIT_CODE(pVCpu) + || ( uAddrReg < pHid->u32Limit + && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pHid->u32Limit) + ) + ) + { + RTGCPHYS GCPhysMem; + rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysMem); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * If we can map the page without trouble, we use the IOM + * string I/O interface to do the job. + */ + PGMPAGEMAPLOCK PgLockMem; + OP_TYPE const *puMem; + rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem); + if (rcStrict == VINF_SUCCESS) + { + uint32_t cTransfers = cLeftPage; + rcStrict = IOMIOPortWriteString(pVM, pVCpu, u16Port, puMem, &cTransfers, OP_SIZE / 8); + + uint32_t cActualTransfers = cLeftPage - cTransfers; + Assert(cActualTransfers <= cLeftPage); + pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg += cbIncr * cActualTransfers; + pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cActualTransfers; + puMem += cActualTransfers; + + iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem); + + if (rcStrict != VINF_SUCCESS) + { + if (IOM_SUCCESS(rcStrict)) + { + /** @todo finish: work out how this should work wrt status codes. Not sure we + * can use iemSetPassUpStatus here, but it depends on what + * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */ + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + if (uCounterReg == 0) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + pVCpu->iem.s.cPotentialExits++; + } + return rcStrict; + } + + if (uCounterReg == 0) + break; + + /* If unaligned, we drop thru and do the page crossing access + below. Otherwise, do the next page. */ + if (!(uVirtAddr & (OP_SIZE / 8 - 1))) + { + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + continue; + } + cLeftPage = 0; + } + } + + /* + * Fallback - slow processing till the end of the current page. + * In the cross page boundrary case we will end up here with cLeftPage + * as 0, we execute one loop then. + * + * Note! We ASSUME the CPU will raise #PF or #GP before access the + * I/O port, otherwise it wouldn't really be restartable. + */ + /** @todo investigate what the CPU actually does with \#PF/\#GP + * during INS. */ + do + { + OP_TYPE uValue; + rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue, iEffSeg, uAddrReg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + rcStrict = IOMIOPortWrite(pVM, pVCpu, u16Port, uValue, OP_SIZE / 8); + if (IOM_SUCCESS(rcStrict)) + { + pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg += cbIncr; + pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg; + cLeftPage--; + } + if (rcStrict != VINF_SUCCESS) + { + if (IOM_SUCCESS(rcStrict)) + { + /** @todo finish: work out how this should work wrt status codes. Not sure we + * can use iemSetPassUpStatus here, but it depends on what + * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */ + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + if (uCounterReg == 0) + iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + pVCpu->iem.s.cPotentialExits++; + } + return rcStrict; + } + IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0); + } while ((int32_t)cLeftPage > 0); + + + /* + * Next page. Must check for interrupts and stuff here. + */ + if (uCounterReg == 0) + break; + IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u); + } + + /* + * Done. + */ + pVCpu->iem.s.cPotentialExits++; + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + +#endif /* OP_SIZE != 64-bit */ + + +#undef OP_rAX +#undef OP_SIZE +#undef ADDR_SIZE +#undef ADDR_rDI +#undef ADDR_rSI +#undef ADDR_rCX +#undef ADDR_rIP +#undef ADDR2_TYPE +#undef ADDR_TYPE +#undef ADDR2_TYPE +#undef ADDR_VMXSTRIO +#undef IS_64_BIT_CODE +#undef IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN +#undef IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN +#undef IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN + diff --git a/src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp b/src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp new file mode 100644 index 00000000..85e47c08 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp @@ -0,0 +1,1622 @@ +/* $Id: IEMAllCImplSvmInstr.cpp $ */ +/** @file + * IEM - AMD-V (Secure Virtual Machine) instruction implementation. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_IEM_SVM +#define VMCPU_INCL_CPUM_GST_CTX +#include +#include +#include +#include +#include +#include +#include +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +# include +#endif +#include +#include +#include "IEMInternal.h" +#include +#include +#include /* for OP_VMMCALL */ +#include +#include +#include +#include +#include + +#include "IEMInline.h" + +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM /* Almost the whole file. */ + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** + * Check the common SVM instruction preconditions. + */ +# define IEM_SVM_INSTR_COMMON_CHECKS(a_pVCpu, a_Instr) \ + do { \ + if (!CPUMIsGuestSvmEnabled(IEM_GET_CTX(a_pVCpu))) \ + { \ + Log((RT_STR(a_Instr) ": EFER.SVME not enabled -> #UD\n")); \ + return iemRaiseUndefinedOpcode(a_pVCpu); \ + } \ + if (IEM_IS_REAL_OR_V86_MODE(a_pVCpu)) \ + { \ + Log((RT_STR(a_Instr) ": Real or v8086 mode -> #UD\n")); \ + return iemRaiseUndefinedOpcode(a_pVCpu); \ + } \ + if ((a_pVCpu)->iem.s.uCpl != 0) \ + { \ + Log((RT_STR(a_Instr) ": CPL != 0 -> #GP(0)\n")); \ + return iemRaiseGeneralProtectionFault0(a_pVCpu); \ + } \ + } while (0) + + +/** + * Converts an IEM exception event type to an SVM event type. + * + * @returns The SVM event type. + * @retval UINT8_MAX if the specified type of event isn't among the set + * of recognized IEM event types. + * + * @param uVector The vector of the event. + * @param fIemXcptFlags The IEM exception / interrupt flags. + */ +IEM_STATIC uint8_t iemGetSvmEventType(uint32_t uVector, uint32_t fIemXcptFlags) +{ + if (fIemXcptFlags & IEM_XCPT_FLAGS_T_CPU_XCPT) + { + if (uVector != X86_XCPT_NMI) + return SVM_EVENT_EXCEPTION; + return SVM_EVENT_NMI; + } + + /* See AMD spec. Table 15-1. "Guest Exception or Interrupt Types". */ + if (fIemXcptFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_ICEBP_INSTR | IEM_XCPT_FLAGS_OF_INSTR)) + return SVM_EVENT_EXCEPTION; + + if (fIemXcptFlags & IEM_XCPT_FLAGS_T_EXT_INT) + return SVM_EVENT_EXTERNAL_IRQ; + + if (fIemXcptFlags & IEM_XCPT_FLAGS_T_SOFT_INT) + return SVM_EVENT_SOFTWARE_INT; + + AssertMsgFailed(("iemGetSvmEventType: Invalid IEM xcpt/int. type %#x, uVector=%#x\n", fIemXcptFlags, uVector)); + return UINT8_MAX; +} + + +/** + * Performs an SVM world-switch (VMRUN, \#VMEXIT) updating PGM and IEM internals. + * + * @returns Strict VBox status code from PGMChangeMode. + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(VBOXSTRICTRC) iemSvmWorldSwitch(PVMCPUCC pVCpu) +{ + /* + * Inform PGM about paging mode changes. + * We include X86_CR0_PE because PGM doesn't handle paged-real mode yet, + * see comment in iemMemPageTranslateAndCheckAccess(). + */ + int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0 | X86_CR0_PE, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER, + true /* fForce */); + AssertRCReturn(rc, rc); + + /* Invalidate IEM TLBs now that we've forced a PGM mode change. */ + IEMTlbInvalidateAll(pVCpu); + + /* Inform CPUM (recompiler), can later be removed. */ + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL); + + /* Re-initialize IEM cache/state after the drastic mode switch. */ + iemReInitExec(pVCpu); + return rc; +} + + +/** + * SVM \#VMEXIT handler. + * + * @returns Strict VBox status code. + * @retval VINF_SVM_VMEXIT when the \#VMEXIT is successful. + * @retval VERR_SVM_VMEXIT_FAILED when the \#VMEXIT failed restoring the guest's + * "host state" and a shutdown is required. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uExitCode The exit code. + * @param uExitInfo1 The exit info. 1 field. + * @param uExitInfo2 The exit info. 2 field. + */ +VBOXSTRICTRC iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict; + if ( CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)) + || uExitCode == SVM_EXIT_INVALID) + { + Log2(("iemSvmVmexit: CS:RIP=%04x:%08RX64 uExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, uExitInfo1, uExitInfo2)); + + /* + * Disable the global-interrupt flag to prevent interrupts during the 'atomic' world switch. + */ + CPUMSetGuestGif(&pVCpu->cpum.GstCtx, false); + + /* + * Map the nested-guest VMCB from its location in guest memory. + * Write exactly what the CPU does on #VMEXIT thereby preserving most other bits in the + * guest's VMCB in memory, see @bugref{7243#c113} and related comment on iemSvmVmrun(). + */ + PSVMVMCB pVmcbMem; + PGMPAGEMAPLOCK PgLockMem; + PSVMVMCBCTRL pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl; + rcStrict = iemMemPageMap(pVCpu, pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb, IEM_ACCESS_DATA_RW, (void **)&pVmcbMem, + &PgLockMem); + if (rcStrict == VINF_SUCCESS) + { + /* + * Notify HM in case the nested-guest was executed using hardware-assisted SVM (which + * would have modified some VMCB state) that might need to be restored on #VMEXIT before + * writing the VMCB back to guest memory. + */ + HMNotifySvmNstGstVmexit(pVCpu, IEM_GET_CTX(pVCpu)); + + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds)); + + /* + * Save the nested-guest state into the VMCB state-save area. + */ + PSVMVMCBSTATESAVE pVmcbMemState = &pVmcbMem->guest; + HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, ES, es); + HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, CS, cs); + HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, SS, ss); + HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, DS, ds); + pVmcbMemState->GDTR.u32Limit = pVCpu->cpum.GstCtx.gdtr.cbGdt; + pVmcbMemState->GDTR.u64Base = pVCpu->cpum.GstCtx.gdtr.pGdt; + pVmcbMemState->IDTR.u32Limit = pVCpu->cpum.GstCtx.idtr.cbIdt; + pVmcbMemState->IDTR.u64Base = pVCpu->cpum.GstCtx.idtr.pIdt; + pVmcbMemState->u64EFER = pVCpu->cpum.GstCtx.msrEFER; + pVmcbMemState->u64CR4 = pVCpu->cpum.GstCtx.cr4; + pVmcbMemState->u64CR3 = pVCpu->cpum.GstCtx.cr3; + pVmcbMemState->u64CR2 = pVCpu->cpum.GstCtx.cr2; + pVmcbMemState->u64CR0 = pVCpu->cpum.GstCtx.cr0; + /** @todo Nested paging. */ + pVmcbMemState->u64RFlags = pVCpu->cpum.GstCtx.rflags.u; + pVmcbMemState->u64RIP = pVCpu->cpum.GstCtx.rip; + pVmcbMemState->u64RSP = pVCpu->cpum.GstCtx.rsp; + pVmcbMemState->u64RAX = pVCpu->cpum.GstCtx.rax; + pVmcbMemState->u64DR7 = pVCpu->cpum.GstCtx.dr[7]; + pVmcbMemState->u64DR6 = pVCpu->cpum.GstCtx.dr[6]; + pVmcbMemState->u8CPL = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl; /* See comment in CPUMGetGuestCPL(). */ + Assert(CPUMGetGuestCPL(pVCpu) == pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl); + if (CPUMIsGuestSvmNestedPagingEnabled(pVCpu, IEM_GET_CTX(pVCpu))) + pVmcbMemState->u64PAT = pVCpu->cpum.GstCtx.msrPAT; + + /* + * Save additional state and intercept information. + * + * - V_IRQ: Tracked using VMCPU_FF_INTERRUPT_NESTED_GUEST force-flag and updated below. + * - V_TPR: Updated by iemCImpl_load_CrX or by the physical CPU for hardware-assisted + * SVM execution. + * - Interrupt shadow: Tracked using VMCPU_FF_INHIBIT_INTERRUPTS and RIP. + */ + PSVMVMCBCTRL pVmcbMemCtrl = &pVmcbMem->ctrl; + if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST)) /* V_IRQ. */ + pVmcbMemCtrl->IntCtrl.n.u1VIrqPending = 0; + else + { + Assert(pVmcbCtrl->IntCtrl.n.u1VIrqPending); + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST); + } + + pVmcbMemCtrl->IntCtrl.n.u8VTPR = pVmcbCtrl->IntCtrl.n.u8VTPR; /* V_TPR. */ + + if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx)) /* Interrupt shadow. */ + pVmcbMemCtrl->IntShadow.n.u1IntShadow = 0; + else + { + pVmcbMemCtrl->IntShadow.n.u1IntShadow = 1; + LogFlow(("iemSvmVmexit: Interrupt shadow till %#RX64\n", pVCpu->cpum.GstCtx.rip)); + CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); + } + + /* + * Save nRIP, instruction length and byte fields. + */ + pVmcbMemCtrl->u64NextRIP = pVmcbCtrl->u64NextRIP; + pVmcbMemCtrl->cbInstrFetched = pVmcbCtrl->cbInstrFetched; + memcpy(&pVmcbMemCtrl->abInstr[0], &pVmcbCtrl->abInstr[0], sizeof(pVmcbMemCtrl->abInstr)); + + /* + * Save exit information. + */ + pVmcbMemCtrl->u64ExitCode = uExitCode; + pVmcbMemCtrl->u64ExitInfo1 = uExitInfo1; + pVmcbMemCtrl->u64ExitInfo2 = uExitInfo2; + + /* + * Update the exit interrupt-information field if this #VMEXIT happened as a result + * of delivering an event through IEM. + * + * Don't update the exit interrupt-information field if the event wasn't being injected + * through IEM, as it would have been updated by real hardware if the nested-guest was + * executed using hardware-assisted SVM. + */ + { + uint8_t uExitIntVector; + uint32_t uExitIntErr; + uint32_t fExitIntFlags; + bool const fRaisingEvent = IEMGetCurrentXcpt(pVCpu, &uExitIntVector, &fExitIntFlags, &uExitIntErr, + NULL /* uExitIntCr2 */); + if (fRaisingEvent) + { + pVmcbCtrl->ExitIntInfo.n.u1Valid = 1; + pVmcbCtrl->ExitIntInfo.n.u8Vector = uExitIntVector; + pVmcbCtrl->ExitIntInfo.n.u3Type = iemGetSvmEventType(uExitIntVector, fExitIntFlags); + if (fExitIntFlags & IEM_XCPT_FLAGS_ERR) + { + pVmcbCtrl->ExitIntInfo.n.u1ErrorCodeValid = true; + pVmcbCtrl->ExitIntInfo.n.u32ErrorCode = uExitIntErr; + } + } + } + + /* + * Save the exit interrupt-information field. + * + * We write the whole field including overwriting reserved bits as it was observed on an + * AMD Ryzen 5 Pro 1500 that the CPU does not preserve reserved bits in EXITINTINFO. + */ + pVmcbMemCtrl->ExitIntInfo = pVmcbCtrl->ExitIntInfo; + + /* + * Clear event injection. + */ + pVmcbMemCtrl->EventInject.n.u1Valid = 0; + + iemMemPageUnmap(pVCpu, pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb, IEM_ACCESS_DATA_RW, pVmcbMem, &PgLockMem); + } + + /* + * Prepare for guest's "host mode" by clearing internal processor state bits. + * + * We don't need to zero out the state-save area, just the controls should be + * sufficient because it has the critical bit of indicating whether we're inside + * the nested-guest or not. + */ + memset(pVmcbCtrl, 0, sizeof(*pVmcbCtrl)); + Assert(!CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))); + + /* + * Restore the subset of the inhibit flags that were preserved. + */ + pVCpu->cpum.GstCtx.eflags.uBoth |= pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit; + + if (rcStrict == VINF_SUCCESS) + { + /** @todo Nested paging. */ + /** @todo ASID. */ + + /* + * If we are switching to PAE mode host, validate the PDPEs first. + * Any invalid PDPEs here causes a VCPU shutdown. + */ + PCSVMHOSTSTATE pHostState = &pVCpu->cpum.GstCtx.hwvirt.svm.HostState; + bool const fHostInPaeMode = CPUMIsPaePagingEnabled(pHostState->uCr0, pHostState->uCr4, pHostState->uEferMsr); + if (fHostInPaeMode) + rcStrict = PGMGstMapPaePdpesAtCr3(pVCpu, pHostState->uCr3); + if (RT_SUCCESS(rcStrict)) + { + /* + * Reload the host state. + */ + CPUMSvmVmExitRestoreHostState(pVCpu, IEM_GET_CTX(pVCpu)); + + /* + * Update PGM, IEM and others of a world-switch. + */ + rcStrict = iemSvmWorldSwitch(pVCpu); + if (rcStrict == VINF_SUCCESS) + rcStrict = VINF_SVM_VMEXIT; + else if (RT_SUCCESS(rcStrict)) + { + LogFlow(("iemSvmVmexit: Setting passup status from iemSvmWorldSwitch %Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + iemSetPassUpStatus(pVCpu, rcStrict); + rcStrict = VINF_SVM_VMEXIT; + } + else + LogFlow(("iemSvmVmexit: iemSvmWorldSwitch unexpected failure. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + } + else + { + Log(("iemSvmVmexit: PAE PDPEs invalid while restoring host state. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + rcStrict = VINF_EM_TRIPLE_FAULT; + } + } + else + { + AssertMsgFailed(("iemSvmVmexit: Mapping VMCB at %#RGp failed. rc=%Rrc\n", pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb, VBOXSTRICTRC_VAL(rcStrict))); + rcStrict = VINF_EM_TRIPLE_FAULT; + } + } + else + { + AssertMsgFailed(("iemSvmVmexit: Not in SVM guest mode! uExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitCode, uExitInfo1, uExitInfo2)); + rcStrict = VERR_SVM_IPE_3; + } + +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3) + /* CLGI/STGI may not have been intercepted and thus not executed in IEM. */ + if ( HMIsEnabled(pVCpu->CTX_SUFF(pVM)) + && HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))) + return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, false); +# endif + return rcStrict; +} + + +/** + * Interface for HM and EM to emulate \#VMEXIT. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uExitCode The exit code. + * @param uExitInfo1 The exit info. 1 field. + * @param uExitInfo2 The exit info. 2 field. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2) +{ + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK); + VBOXSTRICTRC rcStrict = iemSvmVmexit(pVCpu, uExitCode, uExitInfo1, uExitInfo2); + if (pVCpu->iem.s.cActiveMappings) + iemMemRollback(pVCpu); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * Performs the operations necessary that are part of the vmrun instruction + * execution in the guest. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @retval VINF_SUCCESS successfully executed VMRUN and entered nested-guest + * code execution. + * @retval VINF_SVM_VMEXIT when executing VMRUN causes a \#VMEXIT + * (SVM_EXIT_INVALID most likely). + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The length of the VMRUN instruction. + * @param GCPhysVmcb Guest physical address of the VMCB to run. + */ +static VBOXSTRICTRC iemSvmVmrun(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPHYS GCPhysVmcb) RT_NOEXCEPT +{ + LogFlow(("iemSvmVmrun\n")); + + /* + * Cache the physical address of the VMCB for #VMEXIT exceptions. + */ + pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb = GCPhysVmcb; + + /* + * Save the host state. + */ + CPUMSvmVmRunSaveHostState(IEM_GET_CTX(pVCpu), cbInstr); + + /* + * Read the guest VMCB. + */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + int rc = PGMPhysSimpleReadGCPhys(pVM, &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb, GCPhysVmcb, sizeof(SVMVMCB)); + if (RT_SUCCESS(rc)) + { + /* + * AMD-V seems to preserve reserved fields and only writes back selected, recognized + * fields on #VMEXIT. However, not all reserved bits are preserved (e.g, EXITINTINFO) + * but in our implementation we try to preserve as much as we possibly can. + * + * We could read the entire page here and only write back the relevant fields on + * #VMEXIT but since our internal VMCB is also being used by HM during hardware-assisted + * SVM execution, it creates a potential for a nested-hypervisor to set bits that are + * currently reserved but may be recognized as features bits in future CPUs causing + * unexpected & undesired results. Hence, we zero out unrecognized fields here as we + * typically enter hardware-assisted SVM soon anyway, see @bugref{7243#c113}. + */ + PSVMVMCBCTRL pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl; + PSVMVMCBSTATESAVE pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.guest; + + RT_ZERO(pVmcbCtrl->u8Reserved0); + RT_ZERO(pVmcbCtrl->u8Reserved1); + RT_ZERO(pVmcbCtrl->u8Reserved2); + RT_ZERO(pVmcbNstGst->u8Reserved0); + RT_ZERO(pVmcbNstGst->u8Reserved1); + RT_ZERO(pVmcbNstGst->u8Reserved2); + RT_ZERO(pVmcbNstGst->u8Reserved3); + RT_ZERO(pVmcbNstGst->u8Reserved4); + RT_ZERO(pVmcbNstGst->u8Reserved5); + pVmcbCtrl->u32Reserved0 = 0; + pVmcbCtrl->TLBCtrl.n.u24Reserved = 0; + pVmcbCtrl->IntCtrl.n.u6Reserved = 0; + pVmcbCtrl->IntCtrl.n.u3Reserved = 0; + pVmcbCtrl->IntCtrl.n.u5Reserved = 0; + pVmcbCtrl->IntCtrl.n.u24Reserved = 0; + pVmcbCtrl->IntShadow.n.u30Reserved = 0; + pVmcbCtrl->ExitIntInfo.n.u19Reserved = 0; + pVmcbCtrl->NestedPagingCtrl.n.u29Reserved = 0; + pVmcbCtrl->EventInject.n.u19Reserved = 0; + pVmcbCtrl->LbrVirt.n.u30Reserved = 0; + + /* + * Validate guest-state and controls. + */ + /* VMRUN must always be intercepted. */ + if (!CPUMIsGuestSvmCtrlInterceptSet(pVCpu, IEM_GET_CTX(pVCpu), SVM_CTRL_INTERCEPT_VMRUN)) + { + Log(("iemSvmVmrun: VMRUN instruction not intercepted -> #VMEXIT\n")); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* Nested paging. */ + if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging + && !pVM->cpum.ro.GuestFeatures.fSvmNestedPaging) + { + Log(("iemSvmVmrun: Nested paging not supported -> Disabling\n")); + pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging = 0; + } + + /* AVIC. */ + if ( pVmcbCtrl->IntCtrl.n.u1AvicEnable + && !pVM->cpum.ro.GuestFeatures.fSvmAvic) + { + Log(("iemSvmVmrun: AVIC not supported -> Disabling\n")); + pVmcbCtrl->IntCtrl.n.u1AvicEnable = 0; + } + + /* Last branch record (LBR) virtualization. */ + if ( pVmcbCtrl->LbrVirt.n.u1LbrVirt + && !pVM->cpum.ro.GuestFeatures.fSvmLbrVirt) + { + Log(("iemSvmVmrun: LBR virtualization not supported -> Disabling\n")); + pVmcbCtrl->LbrVirt.n.u1LbrVirt = 0; + } + + /* Virtualized VMSAVE/VMLOAD. */ + if ( pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload + && !pVM->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload) + { + Log(("iemSvmVmrun: Virtualized VMSAVE/VMLOAD not supported -> Disabling\n")); + pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload = 0; + } + + /* Virtual GIF. */ + if ( pVmcbCtrl->IntCtrl.n.u1VGifEnable + && !pVM->cpum.ro.GuestFeatures.fSvmVGif) + { + Log(("iemSvmVmrun: Virtual GIF not supported -> Disabling\n")); + pVmcbCtrl->IntCtrl.n.u1VGifEnable = 0; + } + + /* Guest ASID. */ + if (!pVmcbCtrl->TLBCtrl.n.u32ASID) + { + Log(("iemSvmVmrun: Guest ASID is invalid -> #VMEXIT\n")); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* Guest AVIC. */ + if ( pVmcbCtrl->IntCtrl.n.u1AvicEnable + && !pVM->cpum.ro.GuestFeatures.fSvmAvic) + { + Log(("iemSvmVmrun: AVIC not supported -> Disabling\n")); + pVmcbCtrl->IntCtrl.n.u1AvicEnable = 0; + } + + /* Guest Secure Encrypted Virtualization. */ + if ( ( pVmcbCtrl->NestedPagingCtrl.n.u1Sev + || pVmcbCtrl->NestedPagingCtrl.n.u1SevEs) + && !pVM->cpum.ro.GuestFeatures.fSvmAvic) + { + Log(("iemSvmVmrun: SEV not supported -> Disabling\n")); + pVmcbCtrl->NestedPagingCtrl.n.u1Sev = 0; + pVmcbCtrl->NestedPagingCtrl.n.u1SevEs = 0; + } + + /* Flush by ASID. */ + if ( !pVM->cpum.ro.GuestFeatures.fSvmFlusbByAsid + && pVmcbCtrl->TLBCtrl.n.u8TLBFlush != SVM_TLB_FLUSH_NOTHING + && pVmcbCtrl->TLBCtrl.n.u8TLBFlush != SVM_TLB_FLUSH_ENTIRE) + { + Log(("iemSvmVmrun: Flush-by-ASID not supported -> #VMEXIT\n")); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* IO permission bitmap. */ + RTGCPHYS const GCPhysIOBitmap = pVmcbCtrl->u64IOPMPhysAddr; + if ( (GCPhysIOBitmap & X86_PAGE_4K_OFFSET_MASK) + || !PGMPhysIsGCPhysNormal(pVM, GCPhysIOBitmap) + || !PGMPhysIsGCPhysNormal(pVM, GCPhysIOBitmap + X86_PAGE_4K_SIZE) + || !PGMPhysIsGCPhysNormal(pVM, GCPhysIOBitmap + (X86_PAGE_4K_SIZE << 1))) + { + Log(("iemSvmVmrun: IO bitmap physaddr invalid. GCPhysIOBitmap=%#RX64 -> #VMEXIT\n", GCPhysIOBitmap)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* MSR permission bitmap. */ + RTGCPHYS const GCPhysMsrBitmap = pVmcbCtrl->u64MSRPMPhysAddr; + if ( (GCPhysMsrBitmap & X86_PAGE_4K_OFFSET_MASK) + || !PGMPhysIsGCPhysNormal(pVM, GCPhysMsrBitmap) + || !PGMPhysIsGCPhysNormal(pVM, GCPhysMsrBitmap + X86_PAGE_4K_SIZE)) + { + Log(("iemSvmVmrun: MSR bitmap physaddr invalid. GCPhysMsrBitmap=%#RX64 -> #VMEXIT\n", GCPhysMsrBitmap)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* CR0. */ + if ( !(pVmcbNstGst->u64CR0 & X86_CR0_CD) + && (pVmcbNstGst->u64CR0 & X86_CR0_NW)) + { + Log(("iemSvmVmrun: CR0 no-write through with cache disabled. CR0=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64CR0)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + if (pVmcbNstGst->u64CR0 >> 32) + { + Log(("iemSvmVmrun: CR0 reserved bits set. CR0=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64CR0)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + /** @todo Implement all reserved bits/illegal combinations for CR3, CR4. */ + + /* DR6 and DR7. */ + if ( pVmcbNstGst->u64DR6 >> 32 + || pVmcbNstGst->u64DR7 >> 32) + { + Log(("iemSvmVmrun: DR6 and/or DR7 reserved bits set. DR6=%#RX64 DR7=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64DR6, + pVmcbNstGst->u64DR6)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * PAT (Page Attribute Table) MSR. + * + * The CPU only validates and loads it when nested-paging is enabled. + * See AMD spec. "15.25.4 Nested Paging and VMRUN/#VMEXIT". + */ + if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging + && !CPUMIsPatMsrValid(pVmcbNstGst->u64PAT)) + { + Log(("iemSvmVmrun: PAT invalid. u64PAT=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64PAT)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Copy the IO permission bitmap into the cache. + */ + AssertCompile(sizeof(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap) == SVM_IOPM_PAGES * X86_PAGE_4K_SIZE); + rc = PGMPhysSimpleReadGCPhys(pVM, pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, GCPhysIOBitmap, + sizeof(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap)); + if (RT_FAILURE(rc)) + { + Log(("iemSvmVmrun: Failed reading the IO permission bitmap at %#RGp. rc=%Rrc\n", GCPhysIOBitmap, rc)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Copy the MSR permission bitmap into the cache. + */ + AssertCompile(sizeof(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap) == SVM_MSRPM_PAGES * X86_PAGE_4K_SIZE); + rc = PGMPhysSimpleReadGCPhys(pVM, pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap, GCPhysMsrBitmap, + sizeof(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap)); + if (RT_FAILURE(rc)) + { + Log(("iemSvmVmrun: Failed reading the MSR permission bitmap at %#RGp. rc=%Rrc\n", GCPhysMsrBitmap, rc)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Copy segments from nested-guest VMCB state to the guest-CPU state. + * + * We do this here as we need to use the CS attributes and it's easier this way + * then using the VMCB format selectors. It doesn't really matter where we copy + * the state, we restore the guest-CPU context state on the \#VMEXIT anyway. + */ + HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, ES, es); + HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, CS, cs); + HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, SS, ss); + HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, DS, ds); + + /** @todo Segment attribute overrides by VMRUN. */ + + /* + * CPL adjustments and overrides. + * + * SS.DPL is apparently the CPU's CPL, see comment in CPUMGetGuestCPL(). + * We shall thus adjust both CS.DPL and SS.DPL here. + */ + pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl = pVmcbNstGst->u8CPL; + if (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(pVCpu))) + pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl = 3; + if (CPUMIsGuestInRealModeEx(IEM_GET_CTX(pVCpu))) + pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl = 0; + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + + /* + * Continue validating guest-state and controls. + * + * We pass CR0 as 0 to CPUMIsGuestEferMsrWriteValid() below to skip the illegal + * EFER.LME bit transition check. We pass the nested-guest's EFER as both the + * old and new EFER value to not have any guest EFER bits influence the new + * nested-guest EFER. + */ + uint64_t uValidEfer; + rc = CPUMIsGuestEferMsrWriteValid(pVM, 0 /* CR0 */, pVmcbNstGst->u64EFER, pVmcbNstGst->u64EFER, &uValidEfer); + if (RT_FAILURE(rc)) + { + Log(("iemSvmVmrun: EFER invalid uOldEfer=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64EFER)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* Validate paging and CPU mode bits. */ + bool const fSvm = RT_BOOL(uValidEfer & MSR_K6_EFER_SVME); + bool const fLongModeSupported = RT_BOOL(pVM->cpum.ro.GuestFeatures.fLongMode); + bool const fLongModeEnabled = RT_BOOL(uValidEfer & MSR_K6_EFER_LME); + bool const fPaging = RT_BOOL(pVmcbNstGst->u64CR0 & X86_CR0_PG); + bool const fPae = RT_BOOL(pVmcbNstGst->u64CR4 & X86_CR4_PAE); + bool const fProtMode = RT_BOOL(pVmcbNstGst->u64CR0 & X86_CR0_PE); + bool const fLongModeWithPaging = fLongModeEnabled && fPaging; + bool const fLongModeConformCS = pVCpu->cpum.GstCtx.cs.Attr.n.u1Long && pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig; + /* Adjust EFER.LMA (this is normally done by the CPU when system software writes CR0). */ + if (fLongModeWithPaging) + uValidEfer |= MSR_K6_EFER_LMA; + bool const fLongModeActiveOrEnabled = RT_BOOL(uValidEfer & (MSR_K6_EFER_LME | MSR_K6_EFER_LMA)); + if ( !fSvm + || (!fLongModeSupported && fLongModeActiveOrEnabled) + || (fLongModeWithPaging && !fPae) + || (fLongModeWithPaging && !fProtMode) + || ( fLongModeEnabled + && fPaging + && fPae + && fLongModeConformCS)) + { + Log(("iemSvmVmrun: EFER invalid. uValidEfer=%#RX64 -> #VMEXIT\n", uValidEfer)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* + * Preserve the required force-flags. + * + * We only preserve the force-flags that would affect the execution of the + * nested-guest (or the guest). + * + * - VMCPU_FF_BLOCK_NMIS needs to be preserved as it blocks NMI until the + * execution of a subsequent IRET instruction in the guest. + * + * The remaining FFs (e.g. timers) can stay in place so that we will be able to + * generate interrupts that should cause #VMEXITs for the nested-guest. + * + * VMRUN has implicit GIF (Global Interrupt Flag) handling, we don't need to + * preserve VMCPU_FF_INHIBIT_INTERRUPTS. + */ + pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_INHIBIT_NMI; + pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_INHIBIT_NMI; + + /* + * Pause filter. + */ + if (pVM->cpum.ro.GuestFeatures.fSvmPauseFilter) + { + pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter = pVmcbCtrl->u16PauseFilterCount; + if (pVM->cpum.ro.GuestFeatures.fSvmPauseFilterThreshold) + pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilterThreshold = pVmcbCtrl->u16PauseFilterCount; + } + + /* + * Interrupt shadow. + */ + if (pVmcbCtrl->IntShadow.n.u1IntShadow) + { + LogFlow(("iemSvmVmrun: setting interrupt shadow. inhibit PC=%#RX64\n", pVmcbNstGst->u64RIP)); + /** @todo will this cause trouble if the nested-guest is 64-bit but the guest is 32-bit? */ + CPUMSetInInterruptShadowEx(&pVCpu->cpum.GstCtx, pVmcbNstGst->u64RIP); + } + + /* + * TLB flush control. + * Currently disabled since it's redundant as we unconditionally flush the TLB + * in iemSvmWorldSwitch() below. + */ +# if 0 + /** @todo @bugref{7243}: ASID based PGM TLB flushes. */ + if ( pVmcbCtrl->TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE + || pVmcbCtrl->TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT + || pVmcbCtrl->TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS) + PGMFlushTLB(pVCpu, pVmcbNstGst->u64CR3, true /* fGlobal */); +# endif + + /* + * Validate and map PAE PDPEs if the guest will be using PAE paging. + * Invalid PAE PDPEs here causes a #VMEXIT. + */ + if ( !pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging + && CPUMIsPaePagingEnabled(pVmcbNstGst->u64CR0, pVmcbNstGst->u64CR4, uValidEfer)) + { + rc = PGMGstMapPaePdpesAtCr3(pVCpu, pVmcbNstGst->u64CR3); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + Log(("iemSvmVmrun: PAE PDPEs invalid -> #VMEXIT\n")); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + } + + /* + * Copy the remaining guest state from the VMCB to the guest-CPU context. + */ + pVCpu->cpum.GstCtx.gdtr.cbGdt = pVmcbNstGst->GDTR.u32Limit; + pVCpu->cpum.GstCtx.gdtr.pGdt = pVmcbNstGst->GDTR.u64Base; + pVCpu->cpum.GstCtx.idtr.cbIdt = pVmcbNstGst->IDTR.u32Limit; + pVCpu->cpum.GstCtx.idtr.pIdt = pVmcbNstGst->IDTR.u64Base; + CPUMSetGuestCR0(pVCpu, pVmcbNstGst->u64CR0); + CPUMSetGuestCR4(pVCpu, pVmcbNstGst->u64CR4); + pVCpu->cpum.GstCtx.cr3 = pVmcbNstGst->u64CR3; + pVCpu->cpum.GstCtx.cr2 = pVmcbNstGst->u64CR2; + pVCpu->cpum.GstCtx.dr[6] = pVmcbNstGst->u64DR6; + pVCpu->cpum.GstCtx.dr[7] = pVmcbNstGst->u64DR7; + pVCpu->cpum.GstCtx.rflags.u = pVmcbNstGst->u64RFlags; + pVCpu->cpum.GstCtx.rax = pVmcbNstGst->u64RAX; + pVCpu->cpum.GstCtx.rsp = pVmcbNstGst->u64RSP; + pVCpu->cpum.GstCtx.rip = pVmcbNstGst->u64RIP; + CPUMSetGuestEferMsrNoChecks(pVCpu, pVCpu->cpum.GstCtx.msrEFER, uValidEfer); + if (pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging) + pVCpu->cpum.GstCtx.msrPAT = pVmcbNstGst->u64PAT; + + /* Mask DR6, DR7 bits mandatory set/clear bits. */ + pVCpu->cpum.GstCtx.dr[6] &= ~(X86_DR6_RAZ_MASK | X86_DR6_MBZ_MASK); + pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_RA1_MASK; + pVCpu->cpum.GstCtx.dr[7] &= ~(X86_DR7_RAZ_MASK | X86_DR7_MBZ_MASK); + pVCpu->cpum.GstCtx.dr[7] |= X86_DR7_RA1_MASK; + + /* + * Check for pending virtual interrupts. + */ + if (pVmcbCtrl->IntCtrl.n.u1VIrqPending) + VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST); + else + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST)); + + /* + * Update PGM, IEM and others of a world-switch. + */ + VBOXSTRICTRC rcStrict = iemSvmWorldSwitch(pVCpu); + if (rcStrict == VINF_SUCCESS) + { /* likely */ } + else if (RT_SUCCESS(rcStrict)) + { + LogFlow(("iemSvmVmrun: iemSvmWorldSwitch returned %Rrc, setting passup status\n", VBOXSTRICTRC_VAL(rcStrict))); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } + else + { + LogFlow(("iemSvmVmrun: iemSvmWorldSwitch unexpected failure. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* + * Set the global-interrupt flag to allow interrupts in the guest. + */ + CPUMSetGuestGif(&pVCpu->cpum.GstCtx, true); + + /* + * Event injection. + */ + PCSVMEVENT pEventInject = &pVmcbCtrl->EventInject; + pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = !pEventInject->n.u1Valid; + if (pEventInject->n.u1Valid) + { + uint8_t const uVector = pEventInject->n.u8Vector; + TRPMEVENT const enmType = HMSvmEventToTrpmEventType(pEventInject, uVector); + uint16_t const uErrorCode = pEventInject->n.u1ErrorCodeValid ? pEventInject->n.u32ErrorCode : 0; + + /* Validate vectors for hardware exceptions, see AMD spec. 15.20 "Event Injection". */ + if (RT_UNLIKELY(enmType == TRPM_32BIT_HACK)) + { + Log(("iemSvmVmrun: Invalid event type =%#x -> #VMEXIT\n", (uint8_t)pEventInject->n.u3Type)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + if (pEventInject->n.u3Type == SVM_EVENT_EXCEPTION) + { + if ( uVector == X86_XCPT_NMI + || uVector > X86_XCPT_LAST) + { + Log(("iemSvmVmrun: Invalid vector for hardware exception. uVector=%#x -> #VMEXIT\n", uVector)); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + if ( uVector == X86_XCPT_BR + && CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu))) + { + Log(("iemSvmVmrun: Cannot inject #BR when not in long mode -> #VMEXIT\n")); + return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + /** @todo any others? */ + } + + /* + * Invalidate the exit interrupt-information field here. This field is fully updated + * on #VMEXIT as events other than the one below can also cause intercepts during + * their injection (e.g. exceptions). + */ + pVmcbCtrl->ExitIntInfo.n.u1Valid = 0; + + /* + * Clear the event injection valid bit here. While the AMD spec. mentions that the CPU + * clears this bit from the VMCB unconditionally on #VMEXIT, internally the CPU could be + * clearing it at any time, most likely before/after injecting the event. Since VirtualBox + * doesn't have any virtual-CPU internal representation of this bit, we clear/update the + * VMCB here. This also has the added benefit that we avoid the risk of injecting the event + * twice if we fallback to executing the nested-guest using hardware-assisted SVM after + * injecting the event through IEM here. + */ + pVmcbCtrl->EventInject.n.u1Valid = 0; + + /** @todo NRIP: Software interrupts can only be pushed properly if we support + * NRIP for the nested-guest to calculate the instruction length + * below. */ + LogFlow(("iemSvmVmrun: Injecting event: %04x:%08RX64 vec=%#x type=%d uErr=%u cr2=%#RX64 cr3=%#RX64 efer=%#RX64\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uVector, enmType, uErrorCode, pVCpu->cpum.GstCtx.cr2, + pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.msrEFER)); + + /* + * We shall not inject the event here right away. There may be paging mode related updates + * as a result of the world-switch above that are yet to be honored. Instead flag the event + * as pending for injection. + */ + TRPMAssertTrap(pVCpu, uVector, enmType); + if (pEventInject->n.u1ErrorCodeValid) + TRPMSetErrorCode(pVCpu, uErrorCode); + if ( enmType == TRPM_TRAP + && uVector == X86_XCPT_PF) + TRPMSetFaultAddress(pVCpu, pVCpu->cpum.GstCtx.cr2); + } + else + LogFlow(("iemSvmVmrun: Entering nested-guest: %04x:%08RX64 cr0=%#RX64 cr3=%#RX64 cr4=%#RX64 efer=%#RX64 efl=%#x\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, + pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER, pVCpu->cpum.GstCtx.eflags.u)); + + LogFlow(("iemSvmVmrun: returns %d\n", VBOXSTRICTRC_VAL(rcStrict))); + +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3) + /* If CLGI/STGI isn't intercepted we force IEM-only nested-guest execution here. */ + if ( HMIsEnabled(pVM) + && HMIsSvmVGifActive(pVM)) + return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, true); +# endif + + return rcStrict; + } + + /* Shouldn't really happen as the caller should've validated the physical address already. */ + Log(("iemSvmVmrun: Failed to read nested-guest VMCB at %#RGp (rc=%Rrc) -> #VMEXIT\n", GCPhysVmcb, rc)); + return rc; +} + + +/** + * Checks if the event intercepts and performs the \#VMEXIT if the corresponding + * intercept is active. + * + * @returns Strict VBox status code. + * @retval VINF_HM_INTERCEPT_NOT_ACTIVE if the intercept is not active or + * we're not executing a nested-guest. + * @retval VINF_SVM_VMEXIT if the intercept is active and the \#VMEXIT occurred + * successfully. + * @retval VERR_SVM_VMEXIT_FAILED if the intercept is active and the \#VMEXIT + * failed and a shutdown needs to be initiated for the guest. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u8Vector The interrupt or exception vector. + * @param fFlags The exception flags (see IEM_XCPT_FLAGS_XXX). + * @param uErr The error-code associated with the exception. + * @param uCr2 The CR2 value in case of a \#PF exception. + */ +VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2) RT_NOEXCEPT +{ + Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))); + + /* + * Handle SVM exception and software interrupt intercepts, see AMD spec. 15.12 "Exception Intercepts". + * + * - NMI intercepts have their own exit code and do not cause SVM_EXIT_XCPT_2 #VMEXITs. + * - External interrupts and software interrupts (INTn instruction) do not check the exception intercepts + * even when they use a vector in the range 0 to 31. + * - ICEBP should not trigger #DB intercept, but its own intercept. + * - For #PF exceptions, its intercept is checked before CR2 is written by the exception. + */ + /* Check NMI intercept */ + if ( u8Vector == X86_XCPT_NMI + && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT) + && IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_NMI)) + { + Log2(("iemHandleSvmNstGstEventIntercept: NMI intercept -> #VMEXIT\n")); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_NMI, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* Check ICEBP intercept. */ + if ( (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR) + && IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_ICEBP)) + { + Log2(("iemHandleSvmNstGstEventIntercept: ICEBP intercept -> #VMEXIT\n")); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_ICEBP, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* Check CPU exception intercepts. */ + if ( (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT) + && IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, u8Vector)) + { + Assert(u8Vector <= X86_XCPT_LAST); + uint64_t const uExitInfo1 = fFlags & IEM_XCPT_FLAGS_ERR ? uErr : 0; + uint64_t const uExitInfo2 = fFlags & IEM_XCPT_FLAGS_CR2 ? uCr2 : 0; + if ( IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists + && u8Vector == X86_XCPT_PF + && !(uErr & X86_TRAP_PF_ID)) + { + PSVMVMCBCTRL pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl; +# ifdef IEM_WITH_CODE_TLB + uint8_t const *pbInstrBuf = pVCpu->iem.s.pbInstrBuf; + uint8_t const cbInstrBuf = pVCpu->iem.s.cbInstrBuf; + pVmcbCtrl->cbInstrFetched = RT_MIN(cbInstrBuf, SVM_CTRL_GUEST_INSTR_BYTES_MAX); + if ( pbInstrBuf + && cbInstrBuf > 0) + memcpy(&pVmcbCtrl->abInstr[0], pbInstrBuf, pVmcbCtrl->cbInstrFetched); +# else + uint8_t const cbOpcode = pVCpu->iem.s.cbOpcode; + pVmcbCtrl->cbInstrFetched = RT_MIN(cbOpcode, SVM_CTRL_GUEST_INSTR_BYTES_MAX); + if (cbOpcode > 0) + memcpy(&pVmcbCtrl->abInstr[0], &pVCpu->iem.s.abOpcode[0], pVmcbCtrl->cbInstrFetched); +# endif + } + if (u8Vector == X86_XCPT_BR) + IEM_SVM_UPDATE_NRIP(pVCpu); + Log2(("iemHandleSvmNstGstEventIntercept: Xcpt intercept u32InterceptXcpt=%#RX32 u8Vector=%#x " + "uExitInfo1=%#RX64 uExitInfo2=%#RX64 -> #VMEXIT\n", pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u32InterceptXcpt, + u8Vector, uExitInfo1, uExitInfo2)); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_0 + u8Vector, uExitInfo1, uExitInfo2); + } + + /* Check software interrupt (INTn) intercepts. */ + if ( (fFlags & ( IEM_XCPT_FLAGS_T_SOFT_INT + | IEM_XCPT_FLAGS_BP_INSTR + | IEM_XCPT_FLAGS_ICEBP_INSTR + | IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT + && IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INTN)) + { + uint64_t const uExitInfo1 = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? u8Vector : 0; + Log2(("iemHandleSvmNstGstEventIntercept: Software INT intercept (u8Vector=%#x) -> #VMEXIT\n", u8Vector)); + IEM_SVM_UPDATE_NRIP(pVCpu); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SWINT, uExitInfo1, 0 /* uExitInfo2 */); + } + + return VINF_SVM_INTERCEPT_NOT_ACTIVE; +} + + +/** + * Checks the SVM IO permission bitmap and performs the \#VMEXIT if the + * corresponding intercept is active. + * + * @returns Strict VBox status code. + * @retval VINF_HM_INTERCEPT_NOT_ACTIVE if the intercept is not active or + * we're not executing a nested-guest. + * @retval VINF_SVM_VMEXIT if the intercept is active and the \#VMEXIT occurred + * successfully. + * @retval VERR_SVM_VMEXIT_FAILED if the intercept is active and the \#VMEXIT + * failed and a shutdown needs to be initiated for the guest. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param u16Port The IO port being accessed. + * @param enmIoType The type of IO access. + * @param cbReg The IO operand size in bytes. + * @param cAddrSizeBits The address size bits (for 16, 32 or 64). + * @param iEffSeg The effective segment number. + * @param fRep Whether this is a repeating IO instruction (REP prefix). + * @param fStrIo Whether this is a string IO instruction. + * @param cbInstr The length of the IO instruction in bytes. + */ +VBOXSTRICTRC iemSvmHandleIOIntercept(PVMCPUCC pVCpu, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg, + uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT)); + Assert(cAddrSizeBits == 16 || cAddrSizeBits == 32 || cAddrSizeBits == 64); + Assert(cbReg == 1 || cbReg == 2 || cbReg == 4 || cbReg == 8); + + Log3(("iemSvmHandleIOIntercept: u16Port=%#x (%u)\n", u16Port, u16Port)); + + SVMIOIOEXITINFO IoExitInfo; + bool const fIntercept = CPUMIsSvmIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap, u16Port, enmIoType, cbReg, + cAddrSizeBits, iEffSeg, fRep, fStrIo, &IoExitInfo); + if (fIntercept) + { + Log3(("iemSvmHandleIOIntercept: u16Port=%#x (%u) -> #VMEXIT\n", u16Port, u16Port)); + IEM_SVM_UPDATE_NRIP(pVCpu); + return iemSvmVmexit(pVCpu, SVM_EXIT_IOIO, IoExitInfo.u, pVCpu->cpum.GstCtx.rip + cbInstr); + } + + /** @todo remove later (for debugging as VirtualBox always traps all IO + * intercepts). */ + AssertMsgFailed(("iemSvmHandleIOIntercept: We expect an IO intercept here!\n")); + return VINF_SVM_INTERCEPT_NOT_ACTIVE; +} + + +/** + * Checks the SVM MSR permission bitmap and performs the \#VMEXIT if the + * corresponding intercept is active. + * + * @returns Strict VBox status code. + * @retval VINF_HM_INTERCEPT_NOT_ACTIVE if the MSR permission bitmap does not + * specify interception of the accessed MSR @a idMsr. + * @retval VINF_SVM_VMEXIT if the intercept is active and the \#VMEXIT occurred + * successfully. + * @retval VERR_SVM_VMEXIT_FAILED if the intercept is active and the \#VMEXIT + * failed and a shutdown needs to be initiated for the guest. + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The MSR being accessed in the nested-guest. + * @param fWrite Whether this is an MSR write access, @c false implies an + * MSR read. + * @param cbInstr The length of the MSR read/write instruction in bytes. + */ +VBOXSTRICTRC iemSvmHandleMsrIntercept(PVMCPUCC pVCpu, uint32_t idMsr, bool fWrite) RT_NOEXCEPT +{ + /* + * Check if any MSRs are being intercepted. + */ + Assert(CPUMIsGuestSvmCtrlInterceptSet(pVCpu, IEM_GET_CTX(pVCpu), SVM_CTRL_INTERCEPT_MSR_PROT)); + Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))); + + uint64_t const uExitInfo1 = fWrite ? SVM_EXIT1_MSR_WRITE : SVM_EXIT1_MSR_READ; + + /* + * Get the byte and bit offset of the permission bits corresponding to the MSR. + */ + uint16_t offMsrpm; + uint8_t uMsrpmBit; + int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit); + if (RT_SUCCESS(rc)) + { + Assert(uMsrpmBit == 0 || uMsrpmBit == 2 || uMsrpmBit == 4 || uMsrpmBit == 6); + Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT); + if (fWrite) + ++uMsrpmBit; + + /* + * Check if the bit is set, if so, trigger a #VMEXIT. + */ + if (pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)) + { + IEM_SVM_UPDATE_NRIP(pVCpu); + return iemSvmVmexit(pVCpu, SVM_EXIT_MSR, uExitInfo1, 0 /* uExitInfo2 */); + } + } + else + { + /* + * This shouldn't happen, but if it does, cause a #VMEXIT and let the "host" (nested hypervisor) deal with it. + */ + Log(("iemSvmHandleMsrIntercept: Invalid/out-of-range MSR %#RX32 fWrite=%RTbool -> #VMEXIT\n", idMsr, fWrite)); + return iemSvmVmexit(pVCpu, SVM_EXIT_MSR, uExitInfo1, 0 /* uExitInfo2 */); + } + return VINF_SVM_INTERCEPT_NOT_ACTIVE; +} + + + +/** + * Implements 'VMRUN'. + */ +IEM_CIMPL_DEF_0(iemCImpl_vmrun) +{ +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3) + RT_NOREF2(pVCpu, cbInstr); + return VINF_EM_RAW_EMULATE_INSTR; +# else + LogFlow(("iemCImpl_vmrun\n")); + IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, vmrun); + + /** @todo Check effective address size using address size prefix. */ + RTGCPHYS const GCPhysVmcb = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax; + if ( (GCPhysVmcb & X86_PAGE_4K_OFFSET_MASK) + || !PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcb)) + { + Log(("vmrun: VMCB physaddr (%#RGp) not valid -> #GP(0)\n", GCPhysVmcb)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMRUN)) + { + Log(("vmrun: Guest intercept -> #VMEXIT\n")); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMRUN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + VBOXSTRICTRC rcStrict = iemSvmVmrun(pVCpu, cbInstr, GCPhysVmcb); + if (rcStrict == VERR_SVM_VMEXIT_FAILED) + { + Assert(!CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))); + rcStrict = VINF_EM_TRIPLE_FAULT; + } + return rcStrict; +# endif +} + + +/** + * Interface for HM and EM to emulate the VMRUN instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cbInstr The instruction length in bytes. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmrun(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMRUN_MASK); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmrun); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Implements 'VMLOAD'. + */ +IEM_CIMPL_DEF_0(iemCImpl_vmload) +{ +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3) + RT_NOREF2(pVCpu, cbInstr); + return VINF_EM_RAW_EMULATE_INSTR; +# else + LogFlow(("iemCImpl_vmload\n")); + IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, vmload); + + /** @todo Check effective address size using address size prefix. */ + RTGCPHYS const GCPhysVmcb = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax; + if ( (GCPhysVmcb & X86_PAGE_4K_OFFSET_MASK) + || !PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcb)) + { + Log(("vmload: VMCB physaddr (%#RGp) not valid -> #GP(0)\n", GCPhysVmcb)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMLOAD)) + { + Log(("vmload: Guest intercept -> #VMEXIT\n")); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMLOAD, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + SVMVMCBSTATESAVE VmcbNstGst; + VBOXSTRICTRC rcStrict = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcbNstGst, GCPhysVmcb + RT_UOFFSETOF(SVMVMCB, guest), + sizeof(SVMVMCBSTATESAVE)); + if (rcStrict == VINF_SUCCESS) + { + LogFlow(("vmload: Loading VMCB at %#RGp enmEffAddrMode=%d\n", GCPhysVmcb, pVCpu->iem.s.enmEffAddrMode)); + HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, FS, fs); + HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, GS, gs); + HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, TR, tr); + HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, LDTR, ldtr); + + pVCpu->cpum.GstCtx.msrKERNELGSBASE = VmcbNstGst.u64KernelGSBase; + pVCpu->cpum.GstCtx.msrSTAR = VmcbNstGst.u64STAR; + pVCpu->cpum.GstCtx.msrLSTAR = VmcbNstGst.u64LSTAR; + pVCpu->cpum.GstCtx.msrCSTAR = VmcbNstGst.u64CSTAR; + pVCpu->cpum.GstCtx.msrSFMASK = VmcbNstGst.u64SFMASK; + + pVCpu->cpum.GstCtx.SysEnter.cs = VmcbNstGst.u64SysEnterCS; + pVCpu->cpum.GstCtx.SysEnter.esp = VmcbNstGst.u64SysEnterESP; + pVCpu->cpum.GstCtx.SysEnter.eip = VmcbNstGst.u64SysEnterEIP; + + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + return rcStrict; +# endif +} + + +/** + * Interface for HM and EM to emulate the VMLOAD instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cbInstr The instruction length in bytes. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmload(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmload); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Implements 'VMSAVE'. + */ +IEM_CIMPL_DEF_0(iemCImpl_vmsave) +{ +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3) + RT_NOREF2(pVCpu, cbInstr); + return VINF_EM_RAW_EMULATE_INSTR; +# else + LogFlow(("iemCImpl_vmsave\n")); + IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, vmsave); + + /** @todo Check effective address size using address size prefix. */ + RTGCPHYS const GCPhysVmcb = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax; + if ( (GCPhysVmcb & X86_PAGE_4K_OFFSET_MASK) + || !PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcb)) + { + Log(("vmsave: VMCB physaddr (%#RGp) not valid -> #GP(0)\n", GCPhysVmcb)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMSAVE)) + { + Log(("vmsave: Guest intercept -> #VMEXIT\n")); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMSAVE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + SVMVMCBSTATESAVE VmcbNstGst; + VBOXSTRICTRC rcStrict = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcbNstGst, GCPhysVmcb + RT_UOFFSETOF(SVMVMCB, guest), + sizeof(SVMVMCBSTATESAVE)); + if (rcStrict == VINF_SUCCESS) + { + LogFlow(("vmsave: Saving VMCB at %#RGp enmEffAddrMode=%d\n", GCPhysVmcb, pVCpu->iem.s.enmEffAddrMode)); + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_FS | CPUMCTX_EXTRN_GS | CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_LDTR + | CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_SYSENTER_MSRS); + + HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, FS, fs); + HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, GS, gs); + HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, TR, tr); + HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, LDTR, ldtr); + + VmcbNstGst.u64KernelGSBase = pVCpu->cpum.GstCtx.msrKERNELGSBASE; + VmcbNstGst.u64STAR = pVCpu->cpum.GstCtx.msrSTAR; + VmcbNstGst.u64LSTAR = pVCpu->cpum.GstCtx.msrLSTAR; + VmcbNstGst.u64CSTAR = pVCpu->cpum.GstCtx.msrCSTAR; + VmcbNstGst.u64SFMASK = pVCpu->cpum.GstCtx.msrSFMASK; + + VmcbNstGst.u64SysEnterCS = pVCpu->cpum.GstCtx.SysEnter.cs; + VmcbNstGst.u64SysEnterESP = pVCpu->cpum.GstCtx.SysEnter.esp; + VmcbNstGst.u64SysEnterEIP = pVCpu->cpum.GstCtx.SysEnter.eip; + + rcStrict = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmcb + RT_UOFFSETOF(SVMVMCB, guest), &VmcbNstGst, + sizeof(SVMVMCBSTATESAVE)); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + return rcStrict; +# endif +} + + +/** + * Interface for HM and EM to emulate the VMSAVE instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cbInstr The instruction length in bytes. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmsave(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmsave); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Implements 'CLGI'. + */ +IEM_CIMPL_DEF_0(iemCImpl_clgi) +{ +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3) + RT_NOREF2(pVCpu, cbInstr); + return VINF_EM_RAW_EMULATE_INSTR; +# else + LogFlow(("iemCImpl_clgi\n")); + IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, clgi); + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CLGI)) + { + Log(("clgi: Guest intercept -> #VMEXIT\n")); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_CLGI, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + CPUMSetGuestGif(&pVCpu->cpum.GstCtx, false); + +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3) + iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, true); +# else + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +# endif +# endif +} + + +/** + * Interface for HM and EM to emulate the CLGI instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cbInstr The instruction length in bytes. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClgi(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clgi); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Implements 'STGI'. + */ +IEM_CIMPL_DEF_0(iemCImpl_stgi) +{ +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3) + RT_NOREF2(pVCpu, cbInstr); + return VINF_EM_RAW_EMULATE_INSTR; +# else + LogFlow(("iemCImpl_stgi\n")); + IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, stgi); + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_STGI)) + { + Log2(("stgi: Guest intercept -> #VMEXIT\n")); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_STGI, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + CPUMSetGuestGif(&pVCpu->cpum.GstCtx, true); + +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3) + iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, false); +# else + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +# endif +# endif +} + + +/** + * Interface for HM and EM to emulate the STGI instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cbInstr The instruction length in bytes. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedStgi(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_stgi); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Implements 'INVLPGA'. + */ +IEM_CIMPL_DEF_0(iemCImpl_invlpga) +{ + /** @todo Check effective address size using address size prefix. */ + RTGCPTR const GCPtrPage = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax; + /** @todo PGM needs virtual ASID support. */ +# if 0 + uint32_t const uAsid = pVCpu->cpum.GstCtx.ecx; +# endif + + IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, invlpga); + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INVLPGA)) + { + Log2(("invlpga: Guest intercept (%RGp) -> #VMEXIT\n", GCPtrPage)); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_INVLPGA, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + PGMInvalidatePage(pVCpu, GCPtrPage); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Interface for HM and EM to emulate the INVLPGA instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cbInstr The instruction length in bytes. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpga(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invlpga); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Implements 'SKINIT'. + */ +IEM_CIMPL_DEF_0(iemCImpl_skinit) +{ + IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, invlpga); + + uint32_t uIgnore; + uint32_t fFeaturesECX; + CPUMGetGuestCpuId(pVCpu, 0x80000001, 0 /* iSubLeaf */, -1 /*f64BitMode*/, &uIgnore, &uIgnore, &fFeaturesECX, &uIgnore); + if (!(fFeaturesECX & X86_CPUID_AMD_FEATURE_ECX_SKINIT)) + return iemRaiseUndefinedOpcode(pVCpu); + + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SKINIT)) + { + Log2(("skinit: Guest intercept -> #VMEXIT\n")); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SKINIT, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + RT_NOREF(cbInstr); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +} + + +/** + * Implements SVM's implementation of PAUSE. + */ +IEM_CIMPL_DEF_0(iemCImpl_svm_pause) +{ + bool fCheckIntercept = true; + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmPauseFilter) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT); + + /* TSC based pause-filter thresholding. */ + if ( IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmPauseFilterThreshold + && pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilterThreshold > 0) + { + uint64_t const uTick = TMCpuTickGet(pVCpu); + if (uTick - pVCpu->cpum.GstCtx.hwvirt.svm.uPrevPauseTick > pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilterThreshold) + pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter = CPUMGetGuestSvmPauseFilterCount(pVCpu, IEM_GET_CTX(pVCpu)); + pVCpu->cpum.GstCtx.hwvirt.svm.uPrevPauseTick = uTick; + } + + /* Simple pause-filter counter. */ + if (pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter > 0) + { + --pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter; + fCheckIntercept = false; + } + } + + if (fCheckIntercept) + IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_PAUSE, SVM_EXIT_PAUSE, 0, 0); + + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + +#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */ + +/** + * Common code for iemCImpl_vmmcall and iemCImpl_vmcall (latter in IEMAllCImplVmxInstr.cpp.h). + */ +IEM_CIMPL_DEF_1(iemCImpl_Hypercall, uint16_t, uDisOpcode) +{ + if (EMAreHypercallInstructionsEnabled(pVCpu)) + { + NOREF(uDisOpcode); + VBOXSTRICTRC rcStrict = GIMHypercallEx(pVCpu, IEM_GET_CTX(pVCpu), uDisOpcode, cbInstr); + if (RT_SUCCESS(rcStrict)) + { + /** @todo finish: Sort out assertion here when iemRegAddToRipAndFinishingClearingRF + * starts returning non-VINF_SUCCESS statuses. */ + if (rcStrict == VINF_SUCCESS) + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + if ( rcStrict == VINF_SUCCESS + || rcStrict == VINF_GIM_HYPERCALL_CONTINUING) + return VINF_SUCCESS; + AssertMsgReturn(rcStrict == VINF_GIM_R3_HYPERCALL, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IEM_IPE_4); + return rcStrict; + } + AssertMsgReturn( rcStrict == VERR_GIM_HYPERCALL_ACCESS_DENIED + || rcStrict == VERR_GIM_HYPERCALLS_NOT_AVAILABLE + || rcStrict == VERR_GIM_NOT_ENABLED + || rcStrict == VERR_GIM_HYPERCALL_MEMORY_READ_FAILED + || rcStrict == VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED, + ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IEM_IPE_4); + + /* Raise #UD on all failures. */ + } + return iemRaiseUndefinedOpcode(pVCpu); +} + + +/** + * Implements 'VMMCALL'. + */ +IEM_CIMPL_DEF_0(iemCImpl_vmmcall) +{ + if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMMCALL)) + { + Log(("vmmcall: Guest intercept -> #VMEXIT\n")); + IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMMCALL, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */); + } + + /* This is a little bit more complicated than the VT-x version because HM/SVM may + patch MOV CR8 instructions to speed up APIC.TPR access for 32-bit windows guests. */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if (VM_IS_HM_ENABLED(pVM)) + { + int rc = HMHCMaybeMovTprSvmHypercall(pVM, pVCpu); + if (RT_SUCCESS(rc)) + { + Log(("vmmcall: MovTpr\n")); + return VINF_SUCCESS; + } + } + + /* Join forces with vmcall. */ + return IEM_CIMPL_CALL_1(iemCImpl_Hypercall, OP_VMMCALL); +} + diff --git a/src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp b/src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp new file mode 100644 index 00000000..8ffcab9e --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp @@ -0,0 +1,10000 @@ +/* $Id: IEMAllCImplVmxInstr.cpp $ */ +/** @file + * IEM - VT-x instruction implementation. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_IEM_VMX +#define VMCPU_INCL_CPUM_GST_CTX +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +# include +#endif +#include +#include "IEMInternal.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "IEMInline.h" + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** + * Gets the ModR/M, SIB and displacement byte(s) from decoded opcodes given their + * relative offsets. + */ +# ifdef IEM_WITH_CODE_TLB /** @todo IEM TLB */ +# define IEM_MODRM_GET_U8(a_pVCpu, a_bModRm, a_offModRm) do { a_bModRm = 0; RT_NOREF(a_offModRm); } while (0) +# define IEM_SIB_GET_U8(a_pVCpu, a_bSib, a_offSib) do { a_bSib = 0; RT_NOREF(a_offSib); } while (0) +# define IEM_DISP_GET_U16(a_pVCpu, a_u16Disp, a_offDisp) do { a_u16Disp = 0; RT_NOREF(a_offDisp); } while (0) +# define IEM_DISP_GET_S8_SX_U16(a_pVCpu, a_u16Disp, a_offDisp) do { a_u16Disp = 0; RT_NOREF(a_offDisp); } while (0) +# define IEM_DISP_GET_U32(a_pVCpu, a_u32Disp, a_offDisp) do { a_u32Disp = 0; RT_NOREF(a_offDisp); } while (0) +# define IEM_DISP_GET_S8_SX_U32(a_pVCpu, a_u32Disp, a_offDisp) do { a_u32Disp = 0; RT_NOREF(a_offDisp); } while (0) +# define IEM_DISP_GET_S32_SX_U64(a_pVCpu, a_u64Disp, a_offDisp) do { a_u64Disp = 0; RT_NOREF(a_offDisp); } while (0) +# define IEM_DISP_GET_S8_SX_U64(a_pVCpu, a_u64Disp, a_offDisp) do { a_u64Disp = 0; RT_NOREF(a_offDisp); } while (0) +# if 0 +# error "Implement me: Getting ModR/M, SIB, displacement needs to work even when instruction crosses a page boundary." +# endif +# else /* !IEM_WITH_CODE_TLB */ +# define IEM_MODRM_GET_U8(a_pVCpu, a_bModRm, a_offModRm) \ + do \ + { \ + Assert((a_offModRm) < (a_pVCpu)->iem.s.cbOpcode); \ + (a_bModRm) = (a_pVCpu)->iem.s.abOpcode[(a_offModRm)]; \ + } while (0) + +# define IEM_SIB_GET_U8(a_pVCpu, a_bSib, a_offSib) IEM_MODRM_GET_U8(a_pVCpu, a_bSib, a_offSib) + +# define IEM_DISP_GET_U16(a_pVCpu, a_u16Disp, a_offDisp) \ + do \ + { \ + Assert((a_offDisp) + 1 < (a_pVCpu)->iem.s.cbOpcode); \ + uint8_t const bTmpLo = (a_pVCpu)->iem.s.abOpcode[(a_offDisp)]; \ + uint8_t const bTmpHi = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 1]; \ + (a_u16Disp) = RT_MAKE_U16(bTmpLo, bTmpHi); \ + } while (0) + +# define IEM_DISP_GET_S8_SX_U16(a_pVCpu, a_u16Disp, a_offDisp) \ + do \ + { \ + Assert((a_offDisp) < (a_pVCpu)->iem.s.cbOpcode); \ + (a_u16Disp) = (int8_t)((a_pVCpu)->iem.s.abOpcode[(a_offDisp)]); \ + } while (0) + +# define IEM_DISP_GET_U32(a_pVCpu, a_u32Disp, a_offDisp) \ + do \ + { \ + Assert((a_offDisp) + 3 < (a_pVCpu)->iem.s.cbOpcode); \ + uint8_t const bTmp0 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp)]; \ + uint8_t const bTmp1 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 1]; \ + uint8_t const bTmp2 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 2]; \ + uint8_t const bTmp3 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 3]; \ + (a_u32Disp) = RT_MAKE_U32_FROM_U8(bTmp0, bTmp1, bTmp2, bTmp3); \ + } while (0) + +# define IEM_DISP_GET_S8_SX_U32(a_pVCpu, a_u32Disp, a_offDisp) \ + do \ + { \ + Assert((a_offDisp) + 1 < (a_pVCpu)->iem.s.cbOpcode); \ + (a_u32Disp) = (int8_t)((a_pVCpu)->iem.s.abOpcode[(a_offDisp)]); \ + } while (0) + +# define IEM_DISP_GET_S8_SX_U64(a_pVCpu, a_u64Disp, a_offDisp) \ + do \ + { \ + Assert((a_offDisp) + 1 < (a_pVCpu)->iem.s.cbOpcode); \ + (a_u64Disp) = (int8_t)((a_pVCpu)->iem.s.abOpcode[(a_offDisp)]); \ + } while (0) + +# define IEM_DISP_GET_S32_SX_U64(a_pVCpu, a_u64Disp, a_offDisp) \ + do \ + { \ + Assert((a_offDisp) + 3 < (a_pVCpu)->iem.s.cbOpcode); \ + uint8_t const bTmp0 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp)]; \ + uint8_t const bTmp1 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 1]; \ + uint8_t const bTmp2 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 2]; \ + uint8_t const bTmp3 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 3]; \ + (a_u64Disp) = (int32_t)RT_MAKE_U32_FROM_U8(bTmp0, bTmp1, bTmp2, bTmp3); \ + } while (0) +# endif /* !IEM_WITH_CODE_TLB */ + +/** Check for VMX instructions requiring to be in VMX operation. + * @note Any changes here, check if IEMOP_HLP_IN_VMX_OPERATION needs updating. */ +# define IEM_VMX_IN_VMX_OPERATION(a_pVCpu, a_szInstr, a_InsDiagPrefix) \ + do \ + { \ + if (IEM_VMX_IS_ROOT_MODE(a_pVCpu)) \ + { /* likely */ } \ + else \ + { \ + Log((a_szInstr ": Not in VMX operation (root mode) -> #UD\n")); \ + (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_VmxRoot; \ + return iemRaiseUndefinedOpcode(a_pVCpu); \ + } \ + } while (0) + +/** Marks a VM-entry failure with a diagnostic reason, logs and returns. */ +# define IEM_VMX_VMENTRY_FAILED_RET(a_pVCpu, a_pszInstr, a_pszFailure, a_VmxDiag) \ + do \ + { \ + LogRel(("%s: VM-entry failed! enmDiag=%u (%s) -> %s\n", (a_pszInstr), (a_VmxDiag), \ + HMGetVmxDiagDesc(a_VmxDiag), (a_pszFailure))); \ + (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmDiag = (a_VmxDiag); \ + return VERR_VMX_VMENTRY_FAILED; \ + } while (0) + +/** Marks a VM-exit failure with a diagnostic reason and logs. */ +# define IEM_VMX_VMEXIT_FAILED(a_pVCpu, a_uExitReason, a_pszFailure, a_VmxDiag) \ + do \ + { \ + LogRel(("VM-exit failed! uExitReason=%u enmDiag=%u (%s) -> %s\n", (a_uExitReason), (a_VmxDiag), \ + HMGetVmxDiagDesc(a_VmxDiag), (a_pszFailure))); \ + (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmDiag = (a_VmxDiag); \ + } while (0) + +/** Marks a VM-exit failure with a diagnostic reason, logs and returns. */ +# define IEM_VMX_VMEXIT_FAILED_RET(a_pVCpu, a_uExitReason, a_pszFailure, a_VmxDiag) \ + do \ + { \ + IEM_VMX_VMEXIT_FAILED(a_pVCpu, a_uExitReason, a_pszFailure, a_VmxDiag); \ + return VERR_VMX_VMEXIT_FAILED; \ + } while (0) + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +/** @todo NSTVMX: The following VM-exit intercepts are pending: + * VMX_EXIT_IO_SMI + * VMX_EXIT_SMI + * VMX_EXIT_GETSEC + * VMX_EXIT_RSM + * VMX_EXIT_MONITOR (APIC access VM-exit caused by MONITOR pending) + * VMX_EXIT_ERR_MACHINE_CHECK (we never need to raise this?) + * VMX_EXIT_RDRAND + * VMX_EXIT_VMFUNC + * VMX_EXIT_ENCLS + * VMX_EXIT_RDSEED + * VMX_EXIT_PML_FULL + * VMX_EXIT_XSAVES + * VMX_EXIT_XRSTORS + */ +/** + * Map of VMCS field encodings to their virtual-VMCS structure offsets. + * + * The first array dimension is VMCS field encoding of Width OR'ed with Type and the + * second dimension is the Index, see VMXVMCSFIELD. + */ +uint16_t const g_aoffVmcsMap[16][VMX_V_VMCS_MAX_INDEX + 1] = +{ + /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_CONTROL: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u16Vpid), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u16PostIntNotifyVector), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u16EptpIndex), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u16HlatPrefixSize), + /* 4-11 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 12-19 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 20-27 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 28-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */ + { + /* 0-7 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 8-15 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 16-23 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 24-31 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 32-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_GUEST_STATE: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, GuestEs), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, GuestCs), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, GuestSs), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, GuestDs), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, GuestFs), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, GuestGs), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, GuestLdtr), + /* 7 */ RT_UOFFSETOF(VMXVVMCS, GuestTr), + /* 8 */ RT_UOFFSETOF(VMXVVMCS, u16GuestIntStatus), + /* 9 */ RT_UOFFSETOF(VMXVVMCS, u16PmlIndex), + /* 10-17 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 18-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 26-33 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 34 */ UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_HOST_STATE: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, HostEs), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, HostCs), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, HostSs), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, HostDs), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, HostFs), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, HostGs), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, HostTr), + /* 7-14 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 15-22 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 23-30 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 31-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_CONTROL: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u64AddrIoBitmapA), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u64AddrIoBitmapB), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u64AddrMsrBitmap), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u64AddrExitMsrStore), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, u64AddrExitMsrLoad), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, u64AddrEntryMsrLoad), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, u64ExecVmcsPtr), + /* 7 */ RT_UOFFSETOF(VMXVVMCS, u64AddrPml), + /* 8 */ RT_UOFFSETOF(VMXVVMCS, u64TscOffset), + /* 9 */ RT_UOFFSETOF(VMXVVMCS, u64AddrVirtApic), + /* 10 */ RT_UOFFSETOF(VMXVVMCS, u64AddrApicAccess), + /* 11 */ RT_UOFFSETOF(VMXVVMCS, u64AddrPostedIntDesc), + /* 12 */ RT_UOFFSETOF(VMXVVMCS, u64VmFuncCtls), + /* 13 */ RT_UOFFSETOF(VMXVVMCS, u64EptPtr), + /* 14 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap0), + /* 15 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap1), + /* 16 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap2), + /* 17 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap3), + /* 18 */ RT_UOFFSETOF(VMXVVMCS, u64AddrEptpList), + /* 19 */ RT_UOFFSETOF(VMXVVMCS, u64AddrVmreadBitmap), + /* 20 */ RT_UOFFSETOF(VMXVVMCS, u64AddrVmwriteBitmap), + /* 21 */ RT_UOFFSETOF(VMXVVMCS, u64AddrXcptVeInfo), + /* 22 */ RT_UOFFSETOF(VMXVVMCS, u64XssExitBitmap), + /* 23 */ RT_UOFFSETOF(VMXVVMCS, u64EnclsExitBitmap), + /* 24 */ RT_UOFFSETOF(VMXVVMCS, u64SppTablePtr), + /* 25 */ RT_UOFFSETOF(VMXVVMCS, u64TscMultiplier), + /* 26 */ RT_UOFFSETOF(VMXVVMCS, u64ProcCtls3), + /* 27 */ RT_UOFFSETOF(VMXVVMCS, u64EnclvExitBitmap), + /* 28 */ UINT16_MAX, + /* 29 */ UINT16_MAX, + /* 30 */ UINT16_MAX, + /* 31 */ RT_UOFFSETOF(VMXVVMCS, u64PconfigExitBitmap), + /* 32 */ RT_UOFFSETOF(VMXVVMCS, u64HlatPtr), + /* 33 */ UINT16_MAX, + /* 34 */ RT_UOFFSETOF(VMXVVMCS, u64ExitCtls2) + }, + /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u64RoGuestPhysAddr), + /* 1-8 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 9-16 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 17-24 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 25-32 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 33-34*/ UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_GUEST_STATE: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u64VmcsLinkPtr), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u64GuestDebugCtlMsr), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPatMsr), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u64GuestEferMsr), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPerfGlobalCtlMsr), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte0), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte1), + /* 7 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte2), + /* 8 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte3), + /* 9 */ RT_UOFFSETOF(VMXVVMCS, u64GuestBndcfgsMsr), + /* 10 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRtitCtlMsr), + /* 11 */ UINT16_MAX, + /* 12 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPkrsMsr), + /* 13-20 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 21-28 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 29-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_HOST_STATE: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u64HostPatMsr), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u64HostEferMsr), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u64HostPerfGlobalCtlMsr), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u64HostPkrsMsr), + /* 4-11 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 12-19 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 20-27 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 28-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_CONTROL: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u32PinCtls), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u32ProcCtls), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u32XcptBitmap), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u32XcptPFMask), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, u32XcptPFMatch), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, u32Cr3TargetCount), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, u32ExitCtls), + /* 7 */ RT_UOFFSETOF(VMXVVMCS, u32ExitMsrStoreCount), + /* 8 */ RT_UOFFSETOF(VMXVVMCS, u32ExitMsrLoadCount), + /* 9 */ RT_UOFFSETOF(VMXVVMCS, u32EntryCtls), + /* 10 */ RT_UOFFSETOF(VMXVVMCS, u32EntryMsrLoadCount), + /* 11 */ RT_UOFFSETOF(VMXVVMCS, u32EntryIntInfo), + /* 12 */ RT_UOFFSETOF(VMXVVMCS, u32EntryXcptErrCode), + /* 13 */ RT_UOFFSETOF(VMXVVMCS, u32EntryInstrLen), + /* 14 */ RT_UOFFSETOF(VMXVVMCS, u32TprThreshold), + /* 15 */ RT_UOFFSETOF(VMXVVMCS, u32ProcCtls2), + /* 16 */ RT_UOFFSETOF(VMXVVMCS, u32PleGap), + /* 17 */ RT_UOFFSETOF(VMXVVMCS, u32PleWindow), + /* 18-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 26-33 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 34 */ UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u32RoVmInstrError), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitReason), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitIntInfo), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitIntErrCode), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, u32RoIdtVectoringInfo), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, u32RoIdtVectoringErrCode), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitInstrLen), + /* 7 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitInstrInfo), + /* 8-15 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 16-23 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 24-31 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 32-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_GUEST_STATE: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u32GuestEsLimit), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u32GuestCsLimit), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSsLimit), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u32GuestDsLimit), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, u32GuestFsLimit), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, u32GuestGsLimit), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, u32GuestLdtrLimit), + /* 7 */ RT_UOFFSETOF(VMXVVMCS, u32GuestTrLimit), + /* 8 */ RT_UOFFSETOF(VMXVVMCS, u32GuestGdtrLimit), + /* 9 */ RT_UOFFSETOF(VMXVVMCS, u32GuestIdtrLimit), + /* 10 */ RT_UOFFSETOF(VMXVVMCS, u32GuestEsAttr), + /* 11 */ RT_UOFFSETOF(VMXVVMCS, u32GuestCsAttr), + /* 12 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSsAttr), + /* 13 */ RT_UOFFSETOF(VMXVVMCS, u32GuestDsAttr), + /* 14 */ RT_UOFFSETOF(VMXVVMCS, u32GuestFsAttr), + /* 15 */ RT_UOFFSETOF(VMXVVMCS, u32GuestGsAttr), + /* 16 */ RT_UOFFSETOF(VMXVVMCS, u32GuestLdtrAttr), + /* 17 */ RT_UOFFSETOF(VMXVVMCS, u32GuestTrAttr), + /* 18 */ RT_UOFFSETOF(VMXVVMCS, u32GuestIntrState), + /* 19 */ RT_UOFFSETOF(VMXVVMCS, u32GuestActivityState), + /* 20 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSmBase), + /* 21 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSysenterCS), + /* 22 */ UINT16_MAX, + /* 23 */ RT_UOFFSETOF(VMXVVMCS, u32PreemptTimer), + /* 24-31 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 32-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_HOST_STATE: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u32HostSysenterCs), + /* 1-8 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 9-16 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 17-24 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 25-32 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 33-34 */ UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_CONTROL: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u64Cr0Mask), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u64Cr4Mask), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u64Cr0ReadShadow), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u64Cr4ReadShadow), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target0), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target1), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target2), + /* 7 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target3), + /* 8-15 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 16-23 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 24-27 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 32-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u64RoExitQual), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRcx), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRsi), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRdi), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRip), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, u64RoGuestLinearAddr), + /* 6-13 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 14-21 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 22-29 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 30-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_GUEST_STATE: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCr0), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCr3), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCr4), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u64GuestEsBase), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCsBase), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSsBase), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, u64GuestDsBase), + /* 7 */ RT_UOFFSETOF(VMXVVMCS, u64GuestFsBase), + /* 8 */ RT_UOFFSETOF(VMXVVMCS, u64GuestGsBase), + /* 9 */ RT_UOFFSETOF(VMXVVMCS, u64GuestLdtrBase), + /* 10 */ RT_UOFFSETOF(VMXVVMCS, u64GuestTrBase), + /* 11 */ RT_UOFFSETOF(VMXVVMCS, u64GuestGdtrBase), + /* 12 */ RT_UOFFSETOF(VMXVVMCS, u64GuestIdtrBase), + /* 13 */ RT_UOFFSETOF(VMXVVMCS, u64GuestDr7), + /* 14 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRsp), + /* 15 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRip), + /* 16 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRFlags), + /* 17 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPendingDbgXcpts), + /* 18 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSysenterEsp), + /* 19 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSysenterEip), + /* 20 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSCetMsr), + /* 21 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSsp), + /* 22 */ RT_UOFFSETOF(VMXVVMCS, u64GuestIntrSspTableAddrMsr), + /* 23-27 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 31-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX + }, + /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_HOST_STATE: */ + { + /* 0 */ RT_UOFFSETOF(VMXVVMCS, u64HostCr0), + /* 1 */ RT_UOFFSETOF(VMXVVMCS, u64HostCr3), + /* 2 */ RT_UOFFSETOF(VMXVVMCS, u64HostCr4), + /* 3 */ RT_UOFFSETOF(VMXVVMCS, u64HostFsBase), + /* 4 */ RT_UOFFSETOF(VMXVVMCS, u64HostGsBase), + /* 5 */ RT_UOFFSETOF(VMXVVMCS, u64HostTrBase), + /* 6 */ RT_UOFFSETOF(VMXVVMCS, u64HostGdtrBase), + /* 7 */ RT_UOFFSETOF(VMXVVMCS, u64HostIdtrBase), + /* 8 */ RT_UOFFSETOF(VMXVVMCS, u64HostSysenterEsp), + /* 9 */ RT_UOFFSETOF(VMXVVMCS, u64HostSysenterEip), + /* 10 */ RT_UOFFSETOF(VMXVVMCS, u64HostRsp), + /* 11 */ RT_UOFFSETOF(VMXVVMCS, u64HostRip), + /* 12 */ RT_UOFFSETOF(VMXVVMCS, u64HostSCetMsr), + /* 13 */ RT_UOFFSETOF(VMXVVMCS, u64HostSsp), + /* 14 */ RT_UOFFSETOF(VMXVVMCS, u64HostIntrSspTableAddrMsr), + /* 15-22 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 23-30 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, + /* 31-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX + } +}; + + +/** + * Gets a host selector from the VMCS. + * + * @param pVmcs Pointer to the virtual VMCS. + * @param iSelReg The index of the segment register (X86_SREG_XXX). + */ +DECLINLINE(RTSEL) iemVmxVmcsGetHostSelReg(PCVMXVVMCS pVmcs, uint8_t iSegReg) +{ + Assert(iSegReg < X86_SREG_COUNT); + RTSEL HostSel; + uint8_t const uWidth = VMX_VMCSFIELD_WIDTH_16BIT; + uint8_t const uType = VMX_VMCSFIELD_TYPE_HOST_STATE; + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = iSegReg + RT_BF_GET(VMX_VMCS16_HOST_ES_SEL, VMX_BF_VMCSFIELD_INDEX); + Assert(uIndex <= VMX_V_VMCS_MAX_INDEX); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + uint8_t const *pbVmcs = (uint8_t *)pVmcs; + uint8_t const *pbField = pbVmcs + offField; + HostSel = *(uint16_t *)pbField; + return HostSel; +} + + +/** + * Sets a guest segment register in the VMCS. + * + * @param pVmcs Pointer to the virtual VMCS. + * @param iSegReg The index of the segment register (X86_SREG_XXX). + * @param pSelReg Pointer to the segment register. + */ +static void iemVmxVmcsSetGuestSegReg(PCVMXVVMCS pVmcs, uint8_t iSegReg, PCCPUMSELREG pSelReg) RT_NOEXCEPT +{ + Assert(pSelReg); + Assert(iSegReg < X86_SREG_COUNT); + + /* Selector. */ + { + uint8_t const uWidth = VMX_VMCSFIELD_WIDTH_16BIT; + uint8_t const uType = VMX_VMCSFIELD_TYPE_GUEST_STATE; + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = iSegReg + RT_BF_GET(VMX_VMCS16_GUEST_ES_SEL, VMX_BF_VMCSFIELD_INDEX); + Assert(uIndex <= VMX_V_VMCS_MAX_INDEX); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + uint8_t *pbVmcs = (uint8_t *)pVmcs; + uint8_t *pbField = pbVmcs + offField; + *(uint16_t *)pbField = pSelReg->Sel; + } + + /* Limit. */ + { + uint8_t const uWidth = VMX_VMCSFIELD_WIDTH_32BIT; + uint8_t const uType = VMX_VMCSFIELD_TYPE_GUEST_STATE; + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_LIMIT, VMX_BF_VMCSFIELD_INDEX); + Assert(uIndex <= VMX_V_VMCS_MAX_INDEX); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + uint8_t *pbVmcs = (uint8_t *)pVmcs; + uint8_t *pbField = pbVmcs + offField; + *(uint32_t *)pbField = pSelReg->u32Limit; + } + + /* Base. */ + { + uint8_t const uWidth = VMX_VMCSFIELD_WIDTH_NATURAL; + uint8_t const uType = VMX_VMCSFIELD_TYPE_GUEST_STATE; + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = iSegReg + RT_BF_GET(VMX_VMCS_GUEST_ES_BASE, VMX_BF_VMCSFIELD_INDEX); + Assert(uIndex <= VMX_V_VMCS_MAX_INDEX); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + uint8_t const *pbVmcs = (uint8_t *)pVmcs; + uint8_t const *pbField = pbVmcs + offField; + *(uint64_t *)pbField = pSelReg->u64Base; + } + + /* Attributes. */ + { + uint32_t const fValidAttrMask = X86DESCATTR_TYPE | X86DESCATTR_DT | X86DESCATTR_DPL | X86DESCATTR_P + | X86DESCATTR_AVL | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G + | X86DESCATTR_UNUSABLE; + uint8_t const uWidth = VMX_VMCSFIELD_WIDTH_32BIT; + uint8_t const uType = VMX_VMCSFIELD_TYPE_GUEST_STATE; + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, VMX_BF_VMCSFIELD_INDEX); + Assert(uIndex <= VMX_V_VMCS_MAX_INDEX); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + uint8_t *pbVmcs = (uint8_t *)pVmcs; + uint8_t *pbField = pbVmcs + offField; + *(uint32_t *)pbField = pSelReg->Attr.u & fValidAttrMask; + } +} + + +/** + * Gets a guest segment register from the VMCS. + * + * @returns VBox status code. + * @param pVmcs Pointer to the virtual VMCS. + * @param iSegReg The index of the segment register (X86_SREG_XXX). + * @param pSelReg Where to store the segment register (only updated when + * VINF_SUCCESS is returned). + * + * @remarks Warning! This does not validate the contents of the retrieved segment + * register. + */ +static int iemVmxVmcsGetGuestSegReg(PCVMXVVMCS pVmcs, uint8_t iSegReg, PCPUMSELREG pSelReg) RT_NOEXCEPT +{ + Assert(pSelReg); + Assert(iSegReg < X86_SREG_COUNT); + + /* Selector. */ + uint16_t u16Sel; + { + uint8_t const uWidth = VMX_VMCSFIELD_WIDTH_16BIT; + uint8_t const uType = VMX_VMCSFIELD_TYPE_GUEST_STATE; + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = iSegReg + RT_BF_GET(VMX_VMCS16_GUEST_ES_SEL, VMX_BF_VMCSFIELD_INDEX); + AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + uint8_t const *pbVmcs = (uint8_t *)pVmcs; + uint8_t const *pbField = pbVmcs + offField; + u16Sel = *(uint16_t *)pbField; + } + + /* Limit. */ + uint32_t u32Limit; + { + uint8_t const uWidth = VMX_VMCSFIELD_WIDTH_32BIT; + uint8_t const uType = VMX_VMCSFIELD_TYPE_GUEST_STATE; + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_LIMIT, VMX_BF_VMCSFIELD_INDEX); + AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + uint8_t const *pbVmcs = (uint8_t *)pVmcs; + uint8_t const *pbField = pbVmcs + offField; + u32Limit = *(uint32_t *)pbField; + } + + /* Base. */ + uint64_t u64Base; + { + uint8_t const uWidth = VMX_VMCSFIELD_WIDTH_NATURAL; + uint8_t const uType = VMX_VMCSFIELD_TYPE_GUEST_STATE; + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = iSegReg + RT_BF_GET(VMX_VMCS_GUEST_ES_BASE, VMX_BF_VMCSFIELD_INDEX); + AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + uint8_t const *pbVmcs = (uint8_t *)pVmcs; + uint8_t const *pbField = pbVmcs + offField; + u64Base = *(uint64_t *)pbField; + /** @todo NSTVMX: Should we zero out high bits here for 32-bit virtual CPUs? */ + } + + /* Attributes. */ + uint32_t u32Attr; + { + uint8_t const uWidth = VMX_VMCSFIELD_WIDTH_32BIT; + uint8_t const uType = VMX_VMCSFIELD_TYPE_GUEST_STATE; + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, VMX_BF_VMCSFIELD_INDEX); + AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + uint8_t const *pbVmcs = (uint8_t *)pVmcs; + uint8_t const *pbField = pbVmcs + offField; + u32Attr = *(uint32_t *)pbField; + } + + pSelReg->Sel = u16Sel; + pSelReg->ValidSel = u16Sel; + pSelReg->fFlags = CPUMSELREG_FLAGS_VALID; + pSelReg->u32Limit = u32Limit; + pSelReg->u64Base = u64Base; + pSelReg->Attr.u = u32Attr; + return VINF_SUCCESS; +} + + +/** + * Converts an IEM exception event type to a VMX event type. + * + * @returns The VMX event type. + * @param uVector The interrupt / exception vector. + * @param fFlags The IEM event flag (see IEM_XCPT_FLAGS_XXX). + */ +DECLINLINE(uint8_t) iemVmxGetEventType(uint32_t uVector, uint32_t fFlags) +{ + /* Paranoia (callers may use these interchangeably). */ + AssertCompile(VMX_EXIT_INT_INFO_TYPE_NMI == VMX_IDT_VECTORING_INFO_TYPE_NMI); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_HW_XCPT == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_EXT_INT == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_XCPT == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_INT == VMX_IDT_VECTORING_INFO_TYPE_SW_INT); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_NMI == VMX_ENTRY_INT_INFO_TYPE_NMI); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_HW_XCPT == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_EXT_INT == VMX_ENTRY_INT_INFO_TYPE_EXT_INT); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_XCPT == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_INT == VMX_ENTRY_INT_INFO_TYPE_SW_INT); + AssertCompile(VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT); + + if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT) + { + if (uVector == X86_XCPT_NMI) + return VMX_EXIT_INT_INFO_TYPE_NMI; + return VMX_EXIT_INT_INFO_TYPE_HW_XCPT; + } + + if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) + { + if (fFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_OF_INSTR)) + return VMX_EXIT_INT_INFO_TYPE_SW_XCPT; + if (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR) + return VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT; + return VMX_EXIT_INT_INFO_TYPE_SW_INT; + } + + Assert(fFlags & IEM_XCPT_FLAGS_T_EXT_INT); + return VMX_EXIT_INT_INFO_TYPE_EXT_INT; +} + + +/** + * Determines whether the guest is using PAE paging given the VMCS. + * + * @returns @c true if PAE paging mode is used, @c false otherwise. + * @param pVmcs Pointer to the virtual VMCS. + * + * @warning Only use this prior to switching the guest-CPU state with the + * nested-guest CPU state! + */ +DECL_FORCE_INLINE(bool) iemVmxVmcsIsGuestPaePagingEnabled(PCVMXVVMCS pVmcs) +{ + return ( !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST) + && (pVmcs->u64GuestCr4.u & X86_CR4_PAE) + && (pVmcs->u64GuestCr0.u & X86_CR0_PG)); +} + + +/** + * Sets the Exit qualification VMCS field. + * + * @param pVCpu The cross context virtual CPU structure. + * @param u64ExitQual The Exit qualification. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetExitQual(PVMCPUCC pVCpu, uint64_t u64ExitQual) +{ + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64RoExitQual.u = u64ExitQual; +} + + +/** + * Sets the VM-exit interruption information field. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uExitIntInfo The VM-exit interruption information. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetExitIntInfo(PVMCPUCC pVCpu, uint32_t uExitIntInfo) +{ + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoExitIntInfo = uExitIntInfo; +} + + +/** + * Sets the VM-exit interruption error code. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uErrCode The error code. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetExitIntErrCode(PVMCPUCC pVCpu, uint32_t uErrCode) +{ + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoExitIntErrCode = uErrCode; +} + + +/** + * Sets the IDT-vectoring information field. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uIdtVectorInfo The IDT-vectoring information. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetIdtVectoringInfo(PVMCPUCC pVCpu, uint32_t uIdtVectorInfo) +{ + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoIdtVectoringInfo = uIdtVectorInfo; +} + + +/** + * Sets the IDT-vectoring error code field. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uErrCode The error code. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetIdtVectoringErrCode(PVMCPUCC pVCpu, uint32_t uErrCode) +{ + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoIdtVectoringErrCode = uErrCode; +} + + +/** + * Sets the VM-exit guest-linear address VMCS field. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uGuestLinearAddr The VM-exit guest-linear address. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetExitGuestLinearAddr(PVMCPUCC pVCpu, uint64_t uGuestLinearAddr) +{ + /* Bits 63:32 of guest-linear address MBZ if the guest isn't in long mode prior to the VM-exit. */ + Assert(CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)) || !(uGuestLinearAddr & UINT64_C(0xffffffff00000000))); + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64RoGuestLinearAddr.u = uGuestLinearAddr; +} + + +/** + * Sets the VM-exit guest-physical address VMCS field. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uGuestPhysAddr The VM-exit guest-physical address. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetExitGuestPhysAddr(PVMCPUCC pVCpu, uint64_t uGuestPhysAddr) +{ + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64RoGuestPhysAddr.u = uGuestPhysAddr; +} + + +/** + * Sets the VM-exit instruction length VMCS field. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The VM-exit instruction length in bytes. + * + * @remarks Callers may clear this field to 0. Hence, this function does not check + * the validity of the instruction length. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetExitInstrLen(PVMCPUCC pVCpu, uint32_t cbInstr) +{ + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoExitInstrLen = cbInstr; +} + + +/** + * Sets the VM-exit instruction info. VMCS field. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uExitInstrInfo The VM-exit instruction information. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetExitInstrInfo(PVMCPUCC pVCpu, uint32_t uExitInstrInfo) +{ + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoExitInstrInfo = uExitInstrInfo; +} + + +/** + * Sets the guest pending-debug exceptions field. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uGuestPendingDbgXcpts The guest pending-debug exceptions. + */ +DECL_FORCE_INLINE(void) iemVmxVmcsSetGuestPendingDbgXcpts(PVMCPUCC pVCpu, uint64_t uGuestPendingDbgXcpts) +{ + Assert(!(uGuestPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_VALID_MASK)); + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64GuestPendingDbgXcpts.u = uGuestPendingDbgXcpts; +} + + +/** + * Implements VMSucceed for VMX instruction success. + * + * @param pVCpu The cross context virtual CPU structure. + */ +DECL_FORCE_INLINE(void) iemVmxVmSucceed(PVMCPUCC pVCpu) +{ + return CPUMSetGuestVmxVmSucceed(&pVCpu->cpum.GstCtx); +} + + +/** + * Implements VMFailInvalid for VMX instruction failure. + * + * @param pVCpu The cross context virtual CPU structure. + */ +DECL_FORCE_INLINE(void) iemVmxVmFailInvalid(PVMCPUCC pVCpu) +{ + return CPUMSetGuestVmxVmFailInvalid(&pVCpu->cpum.GstCtx); +} + + +/** + * Implements VMFail for VMX instruction failure. + * + * @param pVCpu The cross context virtual CPU structure. + * @param enmInsErr The VM instruction error. + */ +DECL_FORCE_INLINE(void) iemVmxVmFail(PVMCPUCC pVCpu, VMXINSTRERR enmInsErr) +{ + return CPUMSetGuestVmxVmFail(&pVCpu->cpum.GstCtx, enmInsErr); +} + + +/** + * Checks if the given auto-load/store MSR area count is valid for the + * implementation. + * + * @returns @c true if it's within the valid limit, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param uMsrCount The MSR area count to check. + */ +DECL_FORCE_INLINE(bool) iemVmxIsAutoMsrCountValid(PCVMCPU pVCpu, uint32_t uMsrCount) +{ + uint64_t const u64VmxMiscMsr = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Misc; + uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(u64VmxMiscMsr); + Assert(cMaxSupportedMsrs <= VMX_V_AUTOMSR_AREA_SIZE / sizeof(VMXAUTOMSR)); + if (uMsrCount <= cMaxSupportedMsrs) + return true; + return false; +} + + +/** + * Flushes the current VMCS contents back to guest memory. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + */ +DECL_FORCE_INLINE(int) iemVmxWriteCurrentVmcsToGstMem(PVMCPUCC pVCpu) +{ + Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu)); + int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), IEM_VMX_GET_CURRENT_VMCS(pVCpu), + &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs)); + return rc; +} + + +/** + * Populates the current VMCS contents from guest memory. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + */ +DECL_FORCE_INLINE(int) iemVmxReadCurrentVmcsFromGstMem(PVMCPUCC pVCpu) +{ + Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu)); + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs, + IEM_VMX_GET_CURRENT_VMCS(pVCpu), sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs)); + return rc; +} + + +/** + * Gets the instruction diagnostic for segment base checks during VM-entry of a + * nested-guest. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegBase(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegBaseCs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegBaseDs; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegBaseEs; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegBaseFs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegBaseGs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegBaseSs; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_1); + } +} + + +/** + * Gets the instruction diagnostic for segment base checks during VM-entry of a + * nested-guest that is in Virtual-8086 mode. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegBaseV86(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegBaseV86Cs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegBaseV86Ds; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegBaseV86Es; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegBaseV86Fs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegBaseV86Gs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegBaseV86Ss; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_2); + } +} + + +/** + * Gets the instruction diagnostic for segment limit checks during VM-entry of a + * nested-guest that is in Virtual-8086 mode. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegLimitV86(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegLimitV86Cs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegLimitV86Ds; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegLimitV86Es; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegLimitV86Fs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegLimitV86Gs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegLimitV86Ss; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_3); + } +} + + +/** + * Gets the instruction diagnostic for segment attribute checks during VM-entry of a + * nested-guest that is in Virtual-8086 mode. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegAttrV86(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrV86Cs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrV86Ds; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrV86Es; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrV86Fs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrV86Gs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrV86Ss; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_4); + } +} + + +/** + * Gets the instruction diagnostic for segment attributes reserved bits failure + * during VM-entry of a nested-guest. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegAttrRsvd(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdCs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdDs; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrRsvdEs; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdFs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdGs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdSs; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_5); + } +} + + +/** + * Gets the instruction diagnostic for segment attributes descriptor-type + * (code/segment or system) failure during VM-entry of a nested-guest. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegAttrDescType(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeCs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeDs; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeEs; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeFs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeGs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeSs; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_6); + } +} + + +/** + * Gets the instruction diagnostic for segment attributes descriptor-type + * (code/segment or system) failure during VM-entry of a nested-guest. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegAttrPresent(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrPresentCs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrPresentDs; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrPresentEs; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrPresentFs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrPresentGs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrPresentSs; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_7); + } +} + + +/** + * Gets the instruction diagnostic for segment attribute granularity failure during + * VM-entry of a nested-guest. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegAttrGran(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrGranCs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrGranDs; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrGranEs; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrGranFs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrGranGs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrGranSs; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_8); + } +} + +/** + * Gets the instruction diagnostic for segment attribute DPL/RPL failure during + * VM-entry of a nested-guest. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegAttrDplRpl(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplCs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplDs; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrDplRplEs; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplFs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplGs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplSs; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_9); + } +} + + +/** + * Gets the instruction diagnostic for segment attribute type accessed failure + * during VM-entry of a nested-guest. + * + * @param iSegReg The segment index (X86_SREG_XXX). + */ +static VMXVDIAG iemVmxGetDiagVmentrySegAttrTypeAcc(unsigned iSegReg) RT_NOEXCEPT +{ + switch (iSegReg) + { + case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccCs; + case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccDs; + case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccEs; + case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccFs; + case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccGs; + case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccSs; + IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_10); + } +} + + +/** + * Saves the guest control registers, debug registers and some MSRs are part of + * VM-exit. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxVmexitSaveGuestControlRegsMsrs(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Saves the guest control registers, debug registers and some MSRs. + * See Intel spec. 27.3.1 "Saving Control Registers, Debug Registers and MSRs". + */ + PVMXVVMCS pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* Save control registers. */ + pVmcs->u64GuestCr0.u = pVCpu->cpum.GstCtx.cr0; + pVmcs->u64GuestCr3.u = pVCpu->cpum.GstCtx.cr3; + pVmcs->u64GuestCr4.u = pVCpu->cpum.GstCtx.cr4; + + /* Save SYSENTER CS, ESP, EIP. */ + pVmcs->u32GuestSysenterCS = pVCpu->cpum.GstCtx.SysEnter.cs; + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + pVmcs->u64GuestSysenterEsp.u = pVCpu->cpum.GstCtx.SysEnter.esp; + pVmcs->u64GuestSysenterEip.u = pVCpu->cpum.GstCtx.SysEnter.eip; + } + else + { + pVmcs->u64GuestSysenterEsp.s.Lo = pVCpu->cpum.GstCtx.SysEnter.esp; + pVmcs->u64GuestSysenterEip.s.Lo = pVCpu->cpum.GstCtx.SysEnter.eip; + } + + /* Save debug registers (DR7 and IA32_DEBUGCTL MSR). */ + if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_DEBUG) + { + pVmcs->u64GuestDr7.u = pVCpu->cpum.GstCtx.dr[7]; + /** @todo NSTVMX: Support IA32_DEBUGCTL MSR */ + } + + /* Save PAT MSR. */ + if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_PAT_MSR) + pVmcs->u64GuestPatMsr.u = pVCpu->cpum.GstCtx.msrPAT; + + /* Save EFER MSR. */ + if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_EFER_MSR) + pVmcs->u64GuestEferMsr.u = pVCpu->cpum.GstCtx.msrEFER; + + /* We don't support clearing IA32_BNDCFGS MSR yet. */ + Assert(!(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_CLEAR_BNDCFGS_MSR)); + + /* Nothing to do for SMBASE register - We don't support SMM yet. */ +} + + +/** + * Saves the guest force-flags in preparation of entering the nested-guest. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxVmentrySaveNmiBlockingFF(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* We shouldn't be called multiple times during VM-entry. */ + Assert(pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit == 0); + + /* MTF should not be set outside VMX non-root mode. */ + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF)); + + /* + * Preserve the required force-flags. + * + * We cache and clear force-flags that would affect the execution of the + * nested-guest. Cached flags are then restored while returning to the guest + * if necessary. + * + * - VMCPU_FF_INHIBIT_INTERRUPTS need not be cached as it only affects + * interrupts until the completion of the current VMLAUNCH/VMRESUME + * instruction. Interrupt inhibition for any nested-guest instruction + * is supplied by the guest-interruptibility state VMCS field and will + * be set up as part of loading the guest state. Technically + * blocking-by-STI is possible with VMLAUNCH/VMRESUME but we currently + * disallow it since we can't distinguish it from blocking-by-MovSS + * and no nested-hypervisor we care about uses STI immediately + * followed by VMLAUNCH/VMRESUME. + * + * - VMCPU_FF_BLOCK_NMIS needs to be cached as VM-exits caused before + * successful VM-entry (due to invalid guest-state) need to continue + * blocking NMIs if it was in effect before VM-entry. + * + * - MTF need not be preserved as it's used only in VMX non-root mode and + * is supplied through the VM-execution controls. + * + * The remaining FFs (e.g. timers, APIC updates) can stay in place so that + * we will be able to generate interrupts that may cause VM-exits for + * the nested-guest. + */ + pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_INHIBIT_NMI; +} + + +/** + * Restores the guest force-flags in preparation of exiting the nested-guest. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxVmexitRestoreNmiBlockingFF(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /** @todo r=bird: why aren't we clearing the nested guest flags first here? + * If there is some other code doing that already, it would be great + * to point to it here... */ + pVCpu->cpum.GstCtx.eflags.uBoth |= pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit; + pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit = 0; +} + + +/** + * Performs the VMX transition to/from VMX non-root mode. + * + * @param pVCpu The cross context virtual CPU structure. +*/ +static int iemVmxTransition(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Inform PGM about paging mode changes. + * We include X86_CR0_PE because PGM doesn't handle paged-real mode yet, + * see comment in iemMemPageTranslateAndCheckAccess(). + */ + int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0 | X86_CR0_PE, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER, + true /* fForce */); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + return rc; + + /* Invalidate IEM TLBs now that we've forced a PGM mode change. */ + IEMTlbInvalidateAll(pVCpu); + + /* Inform CPUM (recompiler), can later be removed. */ + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL); + + /* Re-initialize IEM cache/state after the drastic mode switch. */ + iemReInitExec(pVCpu); + return rc; +} + + +/** + * Calculates the current VMX-preemption timer value. + * + * @returns The current VMX-preemption timer value. + * @param pVCpu The cross context virtual CPU structure. + */ +static uint32_t iemVmxCalcPreemptTimer(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Assume the following: + * PreemptTimerShift = 5 + * VmcsPreemptTimer = 2 (i.e. need to decrement by 1 every 2 * RT_BIT(5) = 20000 TSC ticks) + * EntryTick = 50000 (TSC at time of VM-entry) + * + * CurTick Delta PreemptTimerVal + * ---------------------------------- + * 60000 10000 2 + * 80000 30000 1 + * 90000 40000 0 -> VM-exit. + * + * If Delta >= VmcsPreemptTimer * RT_BIT(PreemptTimerShift) cause a VMX-preemption timer VM-exit. + * The saved VMX-preemption timer value is calculated as follows: + * PreemptTimerVal = VmcsPreemptTimer - (Delta / (VmcsPreemptTimer * RT_BIT(PreemptTimerShift))) + * E.g.: + * Delta = 10000 + * Tmp = 10000 / (2 * 10000) = 0.5 + * NewPt = 2 - 0.5 = 2 + * Delta = 30000 + * Tmp = 30000 / (2 * 10000) = 1.5 + * NewPt = 2 - 1.5 = 1 + * Delta = 40000 + * Tmp = 40000 / 20000 = 2 + * NewPt = 2 - 2 = 0 + */ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT); + uint32_t const uVmcsPreemptVal = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PreemptTimer; + if (uVmcsPreemptVal > 0) + { + uint64_t const uCurTick = TMCpuTickGetNoCheck(pVCpu); + uint64_t const uEntryTick = pVCpu->cpum.GstCtx.hwvirt.vmx.uEntryTick; + uint64_t const uDelta = uCurTick - uEntryTick; + uint32_t const uPreemptTimer = uVmcsPreemptVal + - ASMDivU64ByU32RetU32(uDelta, uVmcsPreemptVal * RT_BIT(VMX_V_PREEMPT_TIMER_SHIFT)); + return uPreemptTimer; + } + return 0; +} + + +/** + * Saves guest segment registers, GDTR, IDTR, LDTR, TR as part of VM-exit. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxVmexitSaveGuestSegRegs(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Save guest segment registers, GDTR, IDTR, LDTR, TR. + * See Intel spec 27.3.2 "Saving Segment Registers and Descriptor-Table Registers". + */ + /* CS, SS, ES, DS, FS, GS. */ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++) + { + PCCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg]; + if (!pSelReg->Attr.n.u1Unusable) + iemVmxVmcsSetGuestSegReg(pVmcs, iSegReg, pSelReg); + else + { + /* + * For unusable segments the attributes are undefined except for CS and SS. + * For the rest we don't bother preserving anything but the unusable bit. + */ + switch (iSegReg) + { + case X86_SREG_CS: + pVmcs->GuestCs = pSelReg->Sel; + pVmcs->u64GuestCsBase.u = pSelReg->u64Base; + pVmcs->u32GuestCsLimit = pSelReg->u32Limit; + pVmcs->u32GuestCsAttr = pSelReg->Attr.u & ( X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G + | X86DESCATTR_UNUSABLE); + break; + + case X86_SREG_SS: + pVmcs->GuestSs = pSelReg->Sel; + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + pVmcs->u64GuestSsBase.u &= UINT32_C(0xffffffff); + pVmcs->u32GuestSsAttr = pSelReg->Attr.u & (X86DESCATTR_DPL | X86DESCATTR_UNUSABLE); + break; + + case X86_SREG_DS: + pVmcs->GuestDs = pSelReg->Sel; + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + pVmcs->u64GuestDsBase.u &= UINT32_C(0xffffffff); + pVmcs->u32GuestDsAttr = X86DESCATTR_UNUSABLE; + break; + + case X86_SREG_ES: + pVmcs->GuestEs = pSelReg->Sel; + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + pVmcs->u64GuestEsBase.u &= UINT32_C(0xffffffff); + pVmcs->u32GuestEsAttr = X86DESCATTR_UNUSABLE; + break; + + case X86_SREG_FS: + pVmcs->GuestFs = pSelReg->Sel; + pVmcs->u64GuestFsBase.u = pSelReg->u64Base; + pVmcs->u32GuestFsAttr = X86DESCATTR_UNUSABLE; + break; + + case X86_SREG_GS: + pVmcs->GuestGs = pSelReg->Sel; + pVmcs->u64GuestGsBase.u = pSelReg->u64Base; + pVmcs->u32GuestGsAttr = X86DESCATTR_UNUSABLE; + break; + } + } + } + + /* Segment attribute bits 31:17 and 11:8 MBZ. */ + uint32_t const fValidAttrMask = X86DESCATTR_TYPE | X86DESCATTR_DT | X86DESCATTR_DPL | X86DESCATTR_P + | X86DESCATTR_AVL | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G + | X86DESCATTR_UNUSABLE; + /* LDTR. */ + { + PCCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.ldtr; + pVmcs->GuestLdtr = pSelReg->Sel; + pVmcs->u64GuestLdtrBase.u = pSelReg->u64Base; + Assert(X86_IS_CANONICAL(pSelReg->u64Base)); + pVmcs->u32GuestLdtrLimit = pSelReg->u32Limit; + pVmcs->u32GuestLdtrAttr = pSelReg->Attr.u & fValidAttrMask; + } + + /* TR. */ + { + PCCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.tr; + pVmcs->GuestTr = pSelReg->Sel; + pVmcs->u64GuestTrBase.u = pSelReg->u64Base; + pVmcs->u32GuestTrLimit = pSelReg->u32Limit; + pVmcs->u32GuestTrAttr = pSelReg->Attr.u & fValidAttrMask; + } + + /* GDTR. */ + pVmcs->u64GuestGdtrBase.u = pVCpu->cpum.GstCtx.gdtr.pGdt; + pVmcs->u32GuestGdtrLimit = pVCpu->cpum.GstCtx.gdtr.cbGdt; + + /* IDTR. */ + pVmcs->u64GuestIdtrBase.u = pVCpu->cpum.GstCtx.idtr.pIdt; + pVmcs->u32GuestIdtrLimit = pVCpu->cpum.GstCtx.idtr.cbIdt; +} + + +/** + * Saves guest non-register state as part of VM-exit. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason. + */ +static void iemVmxVmexitSaveGuestNonRegState(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT +{ + /* + * Save guest non-register state. + * See Intel spec. 27.3.4 "Saving Non-Register State". + */ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* + * Activity state. + * Most VM-exits will occur in the active state. However, if the first instruction + * following the VM-entry is a HLT instruction, and the MTF VM-execution control is set, + * the VM-exit will be from the HLT activity state. + * + * See Intel spec. 25.5.2 "Monitor Trap Flag". + */ + /** @todo NSTVMX: Does triple-fault VM-exit reflect a shutdown activity state or + * not? */ + EMSTATE const enmActivityState = EMGetState(pVCpu); + switch (enmActivityState) + { + case EMSTATE_HALTED: pVmcs->u32GuestActivityState = VMX_VMCS_GUEST_ACTIVITY_HLT; break; + default: pVmcs->u32GuestActivityState = VMX_VMCS_GUEST_ACTIVITY_ACTIVE; break; + } + + /* + * Interruptibility-state. + */ + /* NMI. */ + pVmcs->u32GuestIntrState = 0; + if (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI) + { + if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking) + pVmcs->u32GuestIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI; + } + else + { + if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx)) + pVmcs->u32GuestIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI; + } + + /* Blocking-by-STI. */ + if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx)) + { /* probable */} + else + { + /** @todo NSTVMX: We can't distinguish between blocking-by-MovSS and blocking-by-STI + * currently. */ + if (pVCpu->cpum.GstCtx.rip == pVCpu->cpum.GstCtx.uRipInhibitInt) + pVmcs->u32GuestIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_STI; /** @todo r=bird: Why the STI one? MOVSS seems to block more and the one to use. */ + + /* Clear inhibition unconditionally since we've ensured it isn't set prior to executing VMLAUNCH/VMRESUME. */ + CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); + } + /* Nothing to do for SMI/enclave. We don't support enclaves or SMM yet. */ + + /* + * Pending debug exceptions. + * + * For VM-exits where it is not applicable, we can safely zero out the field. + * For VM-exits where it is applicable, it's expected to be updated by the caller already. + */ + if ( uExitReason != VMX_EXIT_INIT_SIGNAL + && uExitReason != VMX_EXIT_SMI + && uExitReason != VMX_EXIT_ERR_MACHINE_CHECK + && !VMXIsVmexitTrapLike(uExitReason)) + { + /** @todo NSTVMX: also must exclude VM-exits caused by debug exceptions when + * block-by-MovSS is in effect. */ + pVmcs->u64GuestPendingDbgXcpts.u = 0; + } + + /* + * Save the VMX-preemption timer value back into the VMCS if the feature is enabled. + * + * For VMX-preemption timer VM-exits, we should have already written back 0 if the + * feature is supported back into the VMCS, and thus there is nothing further to do here. + */ + if ( uExitReason != VMX_EXIT_PREEMPT_TIMER + && (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER)) + pVmcs->u32PreemptTimer = iemVmxCalcPreemptTimer(pVCpu); + + /* + * Save the guest PAE PDPTEs. + */ + if ( !CPUMIsGuestInPAEModeEx(&pVCpu->cpum.GstCtx) + || !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT)) + { + /* + * Without EPT or when the nested-guest is not using PAE paging, the values saved + * in the VMCS during VM-exit are undefined. We zero them here for consistency. + */ + pVmcs->u64GuestPdpte0.u = 0; + pVmcs->u64GuestPdpte1.u = 0; + pVmcs->u64GuestPdpte2.u = 0; + pVmcs->u64GuestPdpte3.u = 0; + } + else + { + /* + * With EPT and when the nested-guest is using PAE paging, we update the PDPTEs from + * the nested-guest CPU context. Both IEM (Mov CRx) and hardware-assisted execution + * of the nested-guest is expected to have updated them. + */ + pVmcs->u64GuestPdpte0.u = pVCpu->cpum.GstCtx.aPaePdpes[0].u; + pVmcs->u64GuestPdpte1.u = pVCpu->cpum.GstCtx.aPaePdpes[1].u; + pVmcs->u64GuestPdpte2.u = pVCpu->cpum.GstCtx.aPaePdpes[2].u; + pVmcs->u64GuestPdpte3.u = pVCpu->cpum.GstCtx.aPaePdpes[3].u; + } + + /* Clear PGM's copy of the EPT pointer for added safety. */ + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT) + PGMSetGuestEptPtr(pVCpu, 0 /* uEptPtr */); +} + + +/** + * Saves the guest-state as part of VM-exit. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason. + */ +static void iemVmxVmexitSaveGuestState(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT +{ + iemVmxVmexitSaveGuestControlRegsMsrs(pVCpu); + iemVmxVmexitSaveGuestSegRegs(pVCpu); + + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64GuestRip.u = pVCpu->cpum.GstCtx.rip; + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64GuestRsp.u = pVCpu->cpum.GstCtx.rsp; + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64GuestRFlags.u = pVCpu->cpum.GstCtx.rflags.u; /** @todo NSTVMX: Check RFLAGS.RF handling. */ + + iemVmxVmexitSaveGuestNonRegState(pVCpu, uExitReason); +} + + +/** + * Saves the guest MSRs into the VM-exit MSR-store area as part of VM-exit. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason (for diagnostic purposes). + */ +static int iemVmxVmexitSaveGuestAutoMsrs(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT +{ + /* + * Save guest MSRs. + * See Intel spec. 27.4 "Saving MSRs". + */ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char * const pszFailure = "VMX-abort"; + + /* + * The VM-exit MSR-store area address need not be a valid guest-physical address if the + * VM-exit MSR-store count is 0. If this is the case, bail early without reading it. + * See Intel spec. 24.7.2 "VM-Exit Controls for MSRs". + */ + uint32_t const cMsrs = RT_MIN(pVmcs->u32ExitMsrStoreCount, RT_ELEMENTS(pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrStoreArea)); + if (!cMsrs) + return VINF_SUCCESS; + + /* + * Verify the MSR auto-store count. Physical CPUs can behave unpredictably if the count + * is exceeded including possibly raising #MC exceptions during VMX transition. Our + * implementation causes a VMX-abort followed by a triple-fault. + */ + bool const fIsMsrCountValid = iemVmxIsAutoMsrCountValid(pVCpu, cMsrs); + if (fIsMsrCountValid) + { /* likely */ } + else + IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStoreCount); + + /* + * Optimization if the nested hypervisor is using the same guest-physical page for both + * the VM-entry MSR-load area as well as the VM-exit MSR store area. + */ + PVMXAUTOMSR pMsrArea; + RTGCPHYS const GCPhysVmEntryMsrLoadArea = pVmcs->u64AddrEntryMsrLoad.u; + RTGCPHYS const GCPhysVmExitMsrStoreArea = pVmcs->u64AddrExitMsrStore.u; + if (GCPhysVmEntryMsrLoadArea == GCPhysVmExitMsrStoreArea) + pMsrArea = pVCpu->cpum.GstCtx.hwvirt.vmx.aEntryMsrLoadArea; + else + { + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrStoreArea[0], + GCPhysVmExitMsrStoreArea, cMsrs * sizeof(VMXAUTOMSR)); + if (RT_SUCCESS(rc)) + pMsrArea = pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrStoreArea; + else + { + AssertMsgFailed(("VM-exit: Failed to read MSR auto-store area at %#RGp, rc=%Rrc\n", GCPhysVmExitMsrStoreArea, rc)); + IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStorePtrReadPhys); + } + } + + /* + * Update VM-exit MSR store area. + */ + PVMXAUTOMSR pMsr = pMsrArea; + for (uint32_t idxMsr = 0; idxMsr < cMsrs; idxMsr++, pMsr++) + { + if ( !pMsr->u32Reserved + && pMsr->u32Msr != MSR_IA32_SMBASE + && pMsr->u32Msr >> 8 != MSR_IA32_X2APIC_START >> 8) + { + VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pMsr->u32Msr, &pMsr->u64Value); + if (rcStrict == VINF_SUCCESS) + continue; + + /* + * If we're in ring-0, we cannot handle returns to ring-3 at this point and continue VM-exit. + * If any nested hypervisor loads MSRs that require ring-3 handling, we cause a VMX-abort + * recording the MSR index in the auxiliary info. field and indicated further by our + * own, specific diagnostic code. Later, we can try implement handling of the MSR in ring-0 + * if possible, or come up with a better, generic solution. + */ + pVCpu->cpum.GstCtx.hwvirt.vmx.uAbortAux = pMsr->u32Msr; + VMXVDIAG const enmDiag = rcStrict == VINF_CPUM_R3_MSR_READ + ? kVmxVDiag_Vmexit_MsrStoreRing3 + : kVmxVDiag_Vmexit_MsrStore; + IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, enmDiag); + } + else + { + pVCpu->cpum.GstCtx.hwvirt.vmx.uAbortAux = pMsr->u32Msr; + IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStoreRsvd); + } + } + + /* + * Commit the VM-exit MSR store are to guest memory. + */ + int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmExitMsrStoreArea, pMsrArea, cMsrs * sizeof(VMXAUTOMSR)); + if (RT_SUCCESS(rc)) + return VINF_SUCCESS; + + NOREF(uExitReason); + NOREF(pszFailure); + + AssertMsgFailed(("VM-exit: Failed to write MSR auto-store area at %#RGp, rc=%Rrc\n", GCPhysVmExitMsrStoreArea, rc)); + IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStorePtrWritePhys); +} + + +/** + * Performs a VMX abort (due to an fatal error during VM-exit). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param enmAbort The VMX abort reason. + */ +static VBOXSTRICTRC iemVmxAbort(PVMCPUCC pVCpu, VMXABORT enmAbort) RT_NOEXCEPT +{ + /* + * Perform the VMX abort. + * See Intel spec. 27.7 "VMX Aborts". + */ + LogFunc(("enmAbort=%u (%s) -> RESET\n", enmAbort, VMXGetAbortDesc(enmAbort))); + + /* We don't support SMX yet. */ + pVCpu->cpum.GstCtx.hwvirt.vmx.enmAbort = enmAbort; + if (IEM_VMX_HAS_CURRENT_VMCS(pVCpu)) + { + RTGCPHYS const GCPhysVmcs = IEM_VMX_GET_CURRENT_VMCS(pVCpu); + uint32_t const offVmxAbort = RT_UOFFSETOF(VMXVVMCS, enmVmxAbort); + PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmcs + offVmxAbort, &enmAbort, sizeof(enmAbort)); + } + + return VINF_EM_TRIPLE_FAULT; +} + + +/** + * Loads host control registers, debug registers and MSRs as part of VM-exit. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxVmexitLoadHostControlRegsMsrs(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Load host control registers, debug registers and MSRs. + * See Intel spec. 27.5.1 "Loading Host Control Registers, Debug Registers, MSRs". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + bool const fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE); + + /* CR0. */ + { + /* Bits 63:32, 28:19, 17, 15:6, ET, CD, NW and CR0 fixed bits are not modified. */ + uint64_t const fCr0IgnMask = VMX_EXIT_HOST_CR0_IGNORE_MASK; + uint64_t const uHostCr0 = pVmcs->u64HostCr0.u; + uint64_t const uGuestCr0 = pVCpu->cpum.GstCtx.cr0; + uint64_t const uValidHostCr0 = (uHostCr0 & ~fCr0IgnMask) | (uGuestCr0 & fCr0IgnMask); + + /* Verify we have not modified CR0 fixed bits in VMX operation. */ +#ifdef VBOX_STRICT + uint64_t const uCr0Mb1 = iemVmxGetCr0Fixed0(pVCpu, true /* fVmxNonRootMode */); + bool const fUx = RT_BOOL(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST); + AssertMsg( (uValidHostCr0 & uCr0Mb1) == uCr0Mb1 + && (uValidHostCr0 & ~VMX_V_CR0_FIXED1) == 0, + ("host=%#RX64 guest=%#RX64 mb1=%#RX64 valid_host_cr0=%#RX64 fUx=%RTbool\n", + uHostCr0, uGuestCr0, uCr0Mb1, uValidHostCr0, fUx)); +#endif + Assert(!(uValidHostCr0 >> 32)); + CPUMSetGuestCR0(pVCpu, uValidHostCr0); + } + + /* CR4. */ + { + /* CR4 fixed bits are not modified. */ + uint64_t const uCr4Mb1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0; + uint64_t const uCr4Mb0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1; + uint64_t const uHostCr4 = pVmcs->u64HostCr4.u; + uint64_t uValidHostCr4 = (uHostCr4 & uCr4Mb0) | uCr4Mb1; + if (fHostInLongMode) + uValidHostCr4 |= X86_CR4_PAE; + else + uValidHostCr4 &= ~(uint64_t)X86_CR4_PCIDE; + + /* Verify we have not modified CR4 fixed bits in VMX non-root operation. */ + AssertMsg( (uValidHostCr4 & uCr4Mb1) == uCr4Mb1 + && (uValidHostCr4 & ~uCr4Mb0) == 0, + ("host=%#RX64 guest=%#RX64, uCr4Mb1=%#RX64 uCr4Mb0=%#RX64 valid_host_cr4=%#RX64\n", + uHostCr4, pVCpu->cpum.GstCtx.cr4, uCr4Mb1, uCr4Mb0, uValidHostCr4)); + CPUMSetGuestCR4(pVCpu, uValidHostCr4); + } + + /* CR3 (host value validated while checking host-state during VM-entry). */ + pVCpu->cpum.GstCtx.cr3 = pVmcs->u64HostCr3.u; + + /* DR7. */ + pVCpu->cpum.GstCtx.dr[7] = X86_DR7_INIT_VAL; + + /** @todo NSTVMX: Support IA32_DEBUGCTL MSR */ + + /* Save SYSENTER CS, ESP, EIP (host value validated while checking host-state during VM-entry). */ + pVCpu->cpum.GstCtx.SysEnter.eip = pVmcs->u64HostSysenterEip.u; + pVCpu->cpum.GstCtx.SysEnter.esp = pVmcs->u64HostSysenterEsp.u; + pVCpu->cpum.GstCtx.SysEnter.cs = pVmcs->u32HostSysenterCs; + + /* FS, GS bases are loaded later while we load host segment registers. */ + + /* EFER MSR (host value validated while checking host-state during VM-entry). */ + if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR) + pVCpu->cpum.GstCtx.msrEFER = pVmcs->u64HostEferMsr.u; + else if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + if (fHostInLongMode) + pVCpu->cpum.GstCtx.msrEFER |= (MSR_K6_EFER_LMA | MSR_K6_EFER_LME); + else + pVCpu->cpum.GstCtx.msrEFER &= ~(MSR_K6_EFER_LMA | MSR_K6_EFER_LME); + } + + /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */ + + /* PAT MSR (host value is validated while checking host-state during VM-entry). */ + if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_PAT_MSR) + pVCpu->cpum.GstCtx.msrPAT = pVmcs->u64HostPatMsr.u; + + /* We don't support IA32_BNDCFGS MSR yet. */ +} + + +/** + * Loads host segment registers, GDTR, IDTR, LDTR and TR as part of VM-exit. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxVmexitLoadHostSegRegs(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Load host segment registers, GDTR, IDTR, LDTR and TR. + * See Intel spec. 27.5.2 "Loading Host Segment and Descriptor-Table Registers". + * + * Warning! Be careful to not touch fields that are reserved by VT-x, + * e.g. segment limit high bits stored in segment attributes (in bits 11:8). + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + bool const fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE); + + /* CS, SS, ES, DS, FS, GS. */ + for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++) + { + RTSEL const HostSel = iemVmxVmcsGetHostSelReg(pVmcs, iSegReg); + bool const fUnusable = RT_BOOL(HostSel == 0); + PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg]; + + /* Selector. */ + pSelReg->Sel = HostSel; + pSelReg->ValidSel = HostSel; + pSelReg->fFlags = CPUMSELREG_FLAGS_VALID; + + /* Limit. */ + pSelReg->u32Limit = 0xffffffff; + + /* Base. */ + pSelReg->u64Base = 0; + + /* Attributes. */ + if (iSegReg == X86_SREG_CS) + { + pSelReg->Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_ACCESSED; + pSelReg->Attr.n.u1DescType = 1; + pSelReg->Attr.n.u2Dpl = 0; + pSelReg->Attr.n.u1Present = 1; + pSelReg->Attr.n.u1Long = fHostInLongMode; + pSelReg->Attr.n.u1DefBig = !fHostInLongMode; + pSelReg->Attr.n.u1Granularity = 1; + Assert(!pSelReg->Attr.n.u1Unusable); + Assert(!fUnusable); + } + else + { + pSelReg->Attr.n.u4Type = X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED; + pSelReg->Attr.n.u1DescType = 1; + pSelReg->Attr.n.u2Dpl = 0; + pSelReg->Attr.n.u1Present = 1; + pSelReg->Attr.n.u1DefBig = 1; + pSelReg->Attr.n.u1Granularity = 1; + pSelReg->Attr.n.u1Unusable = fUnusable; + } + } + + /* FS base. */ + if ( !pVCpu->cpum.GstCtx.fs.Attr.n.u1Unusable + || fHostInLongMode) + { + Assert(X86_IS_CANONICAL(pVmcs->u64HostFsBase.u)); + pVCpu->cpum.GstCtx.fs.u64Base = pVmcs->u64HostFsBase.u; + } + + /* GS base. */ + if ( !pVCpu->cpum.GstCtx.gs.Attr.n.u1Unusable + || fHostInLongMode) + { + Assert(X86_IS_CANONICAL(pVmcs->u64HostGsBase.u)); + pVCpu->cpum.GstCtx.gs.u64Base = pVmcs->u64HostGsBase.u; + } + + /* TR. */ + Assert(X86_IS_CANONICAL(pVmcs->u64HostTrBase.u)); + Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1Unusable); + pVCpu->cpum.GstCtx.tr.Sel = pVmcs->HostTr; + pVCpu->cpum.GstCtx.tr.ValidSel = pVmcs->HostTr; + pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.tr.u32Limit = X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN; + pVCpu->cpum.GstCtx.tr.u64Base = pVmcs->u64HostTrBase.u; + pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY; + pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType = 0; + pVCpu->cpum.GstCtx.tr.Attr.n.u2Dpl = 0; + pVCpu->cpum.GstCtx.tr.Attr.n.u1Present = 1; + pVCpu->cpum.GstCtx.tr.Attr.n.u1DefBig = 0; + pVCpu->cpum.GstCtx.tr.Attr.n.u1Granularity = 0; + + /* LDTR (Warning! do not touch the base and limits here). */ + pVCpu->cpum.GstCtx.ldtr.Sel = 0; + pVCpu->cpum.GstCtx.ldtr.ValidSel = 0; + pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE; + + /* GDTR. */ + Assert(X86_IS_CANONICAL(pVmcs->u64HostGdtrBase.u)); + pVCpu->cpum.GstCtx.gdtr.pGdt = pVmcs->u64HostGdtrBase.u; + pVCpu->cpum.GstCtx.gdtr.cbGdt = 0xffff; + + /* IDTR.*/ + Assert(X86_IS_CANONICAL(pVmcs->u64HostIdtrBase.u)); + pVCpu->cpum.GstCtx.idtr.pIdt = pVmcs->u64HostIdtrBase.u; + pVCpu->cpum.GstCtx.idtr.cbIdt = 0xffff; +} + + +/** + * Loads the host MSRs from the VM-exit MSR-load area as part of VM-exit. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VMX instruction name (for logging purposes). + */ +static int iemVmxVmexitLoadHostAutoMsrs(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT +{ + /* + * Load host MSRs. + * See Intel spec. 27.6 "Loading MSRs". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char * const pszFailure = "VMX-abort"; + + /* + * The VM-exit MSR-load area address need not be a valid guest-physical address if the + * VM-exit MSR load count is 0. If this is the case, bail early without reading it. + * See Intel spec. 24.7.2 "VM-Exit Controls for MSRs". + */ + uint32_t const cMsrs = RT_MIN(pVmcs->u32ExitMsrLoadCount, RT_ELEMENTS(pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrLoadArea)); + if (!cMsrs) + return VINF_SUCCESS; + + /* + * Verify the MSR auto-load count. Physical CPUs can behave unpredictably if the count + * is exceeded including possibly raising #MC exceptions during VMX transition. Our + * implementation causes a VMX-abort followed by a triple-fault. + */ + bool const fIsMsrCountValid = iemVmxIsAutoMsrCountValid(pVCpu, cMsrs); + if (fIsMsrCountValid) + { /* likely */ } + else + IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrLoadCount); + + RTGCPHYS const GCPhysVmExitMsrLoadArea = pVmcs->u64AddrExitMsrLoad.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrLoadArea[0], + GCPhysVmExitMsrLoadArea, cMsrs * sizeof(VMXAUTOMSR)); + if (RT_SUCCESS(rc)) + { + PCVMXAUTOMSR pMsr = pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrLoadArea; + for (uint32_t idxMsr = 0; idxMsr < cMsrs; idxMsr++, pMsr++) + { + if ( !pMsr->u32Reserved + && pMsr->u32Msr != MSR_K8_FS_BASE + && pMsr->u32Msr != MSR_K8_GS_BASE + && pMsr->u32Msr != MSR_K6_EFER + && pMsr->u32Msr != MSR_IA32_SMM_MONITOR_CTL + && pMsr->u32Msr >> 8 != MSR_IA32_X2APIC_START >> 8) + { + VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pMsr->u32Msr, pMsr->u64Value); + if (rcStrict == VINF_SUCCESS) + continue; + + /* + * If we're in ring-0, we cannot handle returns to ring-3 at this point and continue VM-exit. + * If any nested hypervisor loads MSRs that require ring-3 handling, we cause a VMX-abort + * recording the MSR index in the auxiliary info. field and indicated further by our + * own, specific diagnostic code. Later, we can try implement handling of the MSR in ring-0 + * if possible, or come up with a better, generic solution. + */ + pVCpu->cpum.GstCtx.hwvirt.vmx.uAbortAux = pMsr->u32Msr; + VMXVDIAG const enmDiag = rcStrict == VINF_CPUM_R3_MSR_WRITE + ? kVmxVDiag_Vmexit_MsrLoadRing3 + : kVmxVDiag_Vmexit_MsrLoad; + IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, enmDiag); + } + else + IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrLoadRsvd); + } + } + else + { + AssertMsgFailed(("VM-exit: Failed to read MSR auto-load area at %#RGp, rc=%Rrc\n", GCPhysVmExitMsrLoadArea, rc)); + IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrLoadPtrReadPhys); + } + + NOREF(uExitReason); + NOREF(pszFailure); + return VINF_SUCCESS; +} + + +/** + * Loads the host state as part of VM-exit. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason (for logging purposes). + */ +static VBOXSTRICTRC iemVmxVmexitLoadHostState(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT +{ + /* + * Load host state. + * See Intel spec. 27.5 "Loading Host State". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + bool const fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE); + + /* We cannot return from a long-mode guest to a host that is not in long mode. */ + if ( CPUMIsGuestInLongMode(pVCpu) + && !fHostInLongMode) + { + Log(("VM-exit from long-mode guest to host not in long-mode -> VMX-Abort\n")); + return iemVmxAbort(pVCpu, VMXABORT_HOST_NOT_IN_LONG_MODE); + } + + /* + * Check host PAE PDPTEs prior to loading the host state. + * See Intel spec. 26.5.4 "Checking and Loading Host Page-Directory-Pointer-Table Entries". + */ + if ( (pVmcs->u64HostCr4.u & X86_CR4_PAE) + && !fHostInLongMode + && ( !CPUMIsGuestInPAEModeEx(&pVCpu->cpum.GstCtx) + || pVmcs->u64HostCr3.u != pVCpu->cpum.GstCtx.cr3)) + { + int const rc = PGMGstMapPaePdpesAtCr3(pVCpu, pVmcs->u64HostCr3.u); + if (RT_SUCCESS(rc)) + { /* likely*/ } + else + { + IEM_VMX_VMEXIT_FAILED(pVCpu, uExitReason, "VMX-abort", kVmxVDiag_Vmexit_HostPdpte); + return iemVmxAbort(pVCpu, VMXBOART_HOST_PDPTE); + } + } + + iemVmxVmexitLoadHostControlRegsMsrs(pVCpu); + iemVmxVmexitLoadHostSegRegs(pVCpu); + + /* + * Load host RIP, RSP and RFLAGS. + * See Intel spec. 27.5.3 "Loading Host RIP, RSP and RFLAGS" + */ + pVCpu->cpum.GstCtx.rip = pVmcs->u64HostRip.u; + pVCpu->cpum.GstCtx.rsp = pVmcs->u64HostRsp.u; + pVCpu->cpum.GstCtx.rflags.u = X86_EFL_1; + + /* Clear address range monitoring. */ + EMMonitorWaitClear(pVCpu); + + /* Perform the VMX transition (PGM updates). */ + VBOXSTRICTRC rcStrict = iemVmxTransition(pVCpu); + if (rcStrict == VINF_SUCCESS) + { /* likely */ } + else if (RT_SUCCESS(rcStrict)) + { + Log3(("VM-exit: iemVmxTransition returns %Rrc (uExitReason=%u) -> Setting passup status\n", VBOXSTRICTRC_VAL(rcStrict), + uExitReason)); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } + else + { + Log3(("VM-exit: iemVmxTransition failed! rc=%Rrc (uExitReason=%u)\n", VBOXSTRICTRC_VAL(rcStrict), uExitReason)); + return VBOXSTRICTRC_VAL(rcStrict); + } + + Assert(rcStrict == VINF_SUCCESS); + + /* Load MSRs from the VM-exit auto-load MSR area. */ + int rc = iemVmxVmexitLoadHostAutoMsrs(pVCpu, uExitReason); + if (RT_FAILURE(rc)) + { + Log(("VM-exit failed while loading host MSRs -> VMX-Abort\n")); + return iemVmxAbort(pVCpu, VMXABORT_LOAD_HOST_MSR); + } + return VINF_SUCCESS; +} + + +/** + * Gets VM-exit instruction information along with any displacement for an + * instruction VM-exit. + * + * @returns The VM-exit instruction information. + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason. + * @param uInstrId The VM-exit instruction identity (VMXINSTRID_XXX). + * @param pGCPtrDisp Where to store the displacement field. Optional, can be + * NULL. + */ +static uint32_t iemVmxGetExitInstrInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, PRTGCPTR pGCPtrDisp) RT_NOEXCEPT +{ + RTGCPTR GCPtrDisp; + VMXEXITINSTRINFO ExitInstrInfo; + ExitInstrInfo.u = 0; + + /* + * Get and parse the ModR/M byte from our decoded opcodes. + */ + uint8_t bRm; + uint8_t const offModRm = pVCpu->iem.s.offModRm; + IEM_MODRM_GET_U8(pVCpu, bRm, offModRm); + if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT)) + { + /* + * ModR/M indicates register addressing. + * + * The primary/secondary register operands are reported in the iReg1 or iReg2 + * fields depending on whether it is a read/write form. + */ + uint8_t idxReg1; + uint8_t idxReg2; + if (!VMXINSTRID_IS_MODRM_PRIMARY_OP_W(uInstrId)) + { + idxReg1 = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg; + idxReg2 = (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB; + } + else + { + idxReg1 = (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB; + idxReg2 = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg; + } + ExitInstrInfo.All.u2Scaling = 0; + ExitInstrInfo.All.iReg1 = idxReg1; + ExitInstrInfo.All.u3AddrSize = pVCpu->iem.s.enmEffAddrMode; + ExitInstrInfo.All.fIsRegOperand = 1; + ExitInstrInfo.All.uOperandSize = pVCpu->iem.s.enmEffOpSize; + ExitInstrInfo.All.iSegReg = 0; + ExitInstrInfo.All.iIdxReg = 0; + ExitInstrInfo.All.fIdxRegInvalid = 1; + ExitInstrInfo.All.iBaseReg = 0; + ExitInstrInfo.All.fBaseRegInvalid = 1; + ExitInstrInfo.All.iReg2 = idxReg2; + + /* Displacement not applicable for register addressing. */ + GCPtrDisp = 0; + } + else + { + /* + * ModR/M indicates memory addressing. + */ + uint8_t uScale = 0; + bool fBaseRegValid = false; + bool fIdxRegValid = false; + uint8_t iBaseReg = 0; + uint8_t iIdxReg = 0; + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT) + { + /* + * Parse the ModR/M, displacement for 16-bit addressing mode. + * See Intel instruction spec. Table 2-1. "16-Bit Addressing Forms with the ModR/M Byte". + */ + uint16_t u16Disp = 0; + uint8_t const offDisp = offModRm + sizeof(bRm); + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6) + { + /* Displacement without any registers. */ + IEM_DISP_GET_U16(pVCpu, u16Disp, offDisp); + } + else + { + /* Register (index and base). */ + switch (bRm & X86_MODRM_RM_MASK) + { + case 0: fBaseRegValid = true; iBaseReg = X86_GREG_xBX; fIdxRegValid = true; iIdxReg = X86_GREG_xSI; break; + case 1: fBaseRegValid = true; iBaseReg = X86_GREG_xBX; fIdxRegValid = true; iIdxReg = X86_GREG_xDI; break; + case 2: fBaseRegValid = true; iBaseReg = X86_GREG_xBP; fIdxRegValid = true; iIdxReg = X86_GREG_xSI; break; + case 3: fBaseRegValid = true; iBaseReg = X86_GREG_xBP; fIdxRegValid = true; iIdxReg = X86_GREG_xDI; break; + case 4: fIdxRegValid = true; iIdxReg = X86_GREG_xSI; break; + case 5: fIdxRegValid = true; iIdxReg = X86_GREG_xDI; break; + case 6: fBaseRegValid = true; iBaseReg = X86_GREG_xBP; break; + case 7: fBaseRegValid = true; iBaseReg = X86_GREG_xBX; break; + } + + /* Register + displacement. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: break; + case 1: IEM_DISP_GET_S8_SX_U16(pVCpu, u16Disp, offDisp); break; + case 2: IEM_DISP_GET_U16(pVCpu, u16Disp, offDisp); break; + default: + { + /* Register addressing, handled at the beginning. */ + AssertMsgFailed(("ModR/M %#x implies register addressing, memory addressing expected!", bRm)); + break; + } + } + } + + Assert(!uScale); /* There's no scaling/SIB byte for 16-bit addressing. */ + GCPtrDisp = (int16_t)u16Disp; /* Sign-extend the displacement. */ + } + else if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT) + { + /* + * Parse the ModR/M, SIB, displacement for 32-bit addressing mode. + * See Intel instruction spec. Table 2-2. "32-Bit Addressing Forms with the ModR/M Byte". + */ + uint32_t u32Disp = 0; + if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5) + { + /* Displacement without any registers. */ + uint8_t const offDisp = offModRm + sizeof(bRm); + IEM_DISP_GET_U32(pVCpu, u32Disp, offDisp); + } + else + { + /* Register (and perhaps scale, index and base). */ + uint8_t offDisp = offModRm + sizeof(bRm); + iBaseReg = (bRm & X86_MODRM_RM_MASK); + if (iBaseReg == 4) + { + /* An SIB byte follows the ModR/M byte, parse it. */ + uint8_t bSib; + uint8_t const offSib = offModRm + sizeof(bRm); + IEM_SIB_GET_U8(pVCpu, bSib, offSib); + + /* A displacement may follow SIB, update its offset. */ + offDisp += sizeof(bSib); + + /* Get the scale. */ + uScale = (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK; + + /* Get the index register. */ + iIdxReg = (bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK; + fIdxRegValid = RT_BOOL(iIdxReg != 4); + + /* Get the base register. */ + iBaseReg = bSib & X86_SIB_BASE_MASK; + fBaseRegValid = true; + if (iBaseReg == 5) + { + if ((bRm & X86_MODRM_MOD_MASK) == 0) + { + /* Mod is 0 implies a 32-bit displacement with no base. */ + fBaseRegValid = false; + IEM_DISP_GET_U32(pVCpu, u32Disp, offDisp); + } + else + { + /* Mod is not 0 implies an 8-bit/32-bit displacement (handled below) with an EBP base. */ + iBaseReg = X86_GREG_xBP; + } + } + } + + /* Register + displacement. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: /* Handled above */ break; + case 1: IEM_DISP_GET_S8_SX_U32(pVCpu, u32Disp, offDisp); break; + case 2: IEM_DISP_GET_U32(pVCpu, u32Disp, offDisp); break; + default: + { + /* Register addressing, handled at the beginning. */ + AssertMsgFailed(("ModR/M %#x implies register addressing, memory addressing expected!", bRm)); + break; + } + } + } + + GCPtrDisp = (int32_t)u32Disp; /* Sign-extend the displacement. */ + } + else + { + Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT); + + /* + * Parse the ModR/M, SIB, displacement for 64-bit addressing mode. + * See Intel instruction spec. 2.2 "IA-32e Mode". + */ + uint64_t u64Disp = 0; + bool const fRipRelativeAddr = RT_BOOL((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5); + if (fRipRelativeAddr) + { + /* + * RIP-relative addressing mode. + * + * The displacement is 32-bit signed implying an offset range of +/-2G. + * See Intel instruction spec. 2.2.1.6 "RIP-Relative Addressing". + */ + uint8_t const offDisp = offModRm + sizeof(bRm); + IEM_DISP_GET_S32_SX_U64(pVCpu, u64Disp, offDisp); + } + else + { + uint8_t offDisp = offModRm + sizeof(bRm); + + /* + * Register (and perhaps scale, index and base). + * + * REX.B extends the most-significant bit of the base register. However, REX.B + * is ignored while determining whether an SIB follows the opcode. Hence, we + * shall OR any REX.B bit -after- inspecting for an SIB byte below. + * + * See Intel instruction spec. Table 2-5. "Special Cases of REX Encodings". + */ + iBaseReg = (bRm & X86_MODRM_RM_MASK); + if (iBaseReg == 4) + { + /* An SIB byte follows the ModR/M byte, parse it. Displacement (if any) follows SIB. */ + uint8_t bSib; + uint8_t const offSib = offModRm + sizeof(bRm); + IEM_SIB_GET_U8(pVCpu, bSib, offSib); + + /* Displacement may follow SIB, update its offset. */ + offDisp += sizeof(bSib); + + /* Get the scale. */ + uScale = (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK; + + /* Get the index. */ + iIdxReg = ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex; + fIdxRegValid = RT_BOOL(iIdxReg != 4); /* R12 -can- be used as an index register. */ + + /* Get the base. */ + iBaseReg = (bSib & X86_SIB_BASE_MASK); + fBaseRegValid = true; + if (iBaseReg == 5) + { + if ((bRm & X86_MODRM_MOD_MASK) == 0) + { + /* Mod is 0 implies a signed 32-bit displacement with no base. */ + IEM_DISP_GET_S32_SX_U64(pVCpu, u64Disp, offDisp); + } + else + { + /* Mod is non-zero implies an 8-bit/32-bit displacement (handled below) with RBP or R13 as base. */ + iBaseReg = pVCpu->iem.s.uRexB ? X86_GREG_x13 : X86_GREG_xBP; + } + } + } + iBaseReg |= pVCpu->iem.s.uRexB; + + /* Register + displacement. */ + switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK) + { + case 0: /* Handled above */ break; + case 1: IEM_DISP_GET_S8_SX_U64(pVCpu, u64Disp, offDisp); break; + case 2: IEM_DISP_GET_S32_SX_U64(pVCpu, u64Disp, offDisp); break; + default: + { + /* Register addressing, handled at the beginning. */ + AssertMsgFailed(("ModR/M %#x implies register addressing, memory addressing expected!", bRm)); + break; + } + } + } + + GCPtrDisp = fRipRelativeAddr ? pVCpu->cpum.GstCtx.rip + u64Disp : u64Disp; + } + + /* + * The primary or secondary register operand is reported in iReg2 depending + * on whether the primary operand is in read/write form. + */ + uint8_t idxReg2; + if (!VMXINSTRID_IS_MODRM_PRIMARY_OP_W(uInstrId)) + { + idxReg2 = bRm & X86_MODRM_RM_MASK; + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT) + idxReg2 |= pVCpu->iem.s.uRexB; + } + else + { + idxReg2 = (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK; + if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT) + idxReg2 |= pVCpu->iem.s.uRexReg; + } + ExitInstrInfo.All.u2Scaling = uScale; + ExitInstrInfo.All.iReg1 = 0; /* Not applicable for memory addressing. */ + ExitInstrInfo.All.u3AddrSize = pVCpu->iem.s.enmEffAddrMode; + ExitInstrInfo.All.fIsRegOperand = 0; + ExitInstrInfo.All.uOperandSize = pVCpu->iem.s.enmEffOpSize; + ExitInstrInfo.All.iSegReg = pVCpu->iem.s.iEffSeg; + ExitInstrInfo.All.iIdxReg = iIdxReg; + ExitInstrInfo.All.fIdxRegInvalid = !fIdxRegValid; + ExitInstrInfo.All.iBaseReg = iBaseReg; + ExitInstrInfo.All.iIdxReg = !fBaseRegValid; + ExitInstrInfo.All.iReg2 = idxReg2; + } + + /* + * Handle exceptions to the norm for certain instructions. + * (e.g. some instructions convey an instruction identity in place of iReg2). + */ + switch (uExitReason) + { + case VMX_EXIT_GDTR_IDTR_ACCESS: + { + Assert(VMXINSTRID_IS_VALID(uInstrId)); + Assert(VMXINSTRID_GET_ID(uInstrId) == (uInstrId & 0x3)); + ExitInstrInfo.GdtIdt.u2InstrId = VMXINSTRID_GET_ID(uInstrId); + ExitInstrInfo.GdtIdt.u2Undef0 = 0; + break; + } + + case VMX_EXIT_LDTR_TR_ACCESS: + { + Assert(VMXINSTRID_IS_VALID(uInstrId)); + Assert(VMXINSTRID_GET_ID(uInstrId) == (uInstrId & 0x3)); + ExitInstrInfo.LdtTr.u2InstrId = VMXINSTRID_GET_ID(uInstrId); + ExitInstrInfo.LdtTr.u2Undef0 = 0; + break; + } + + case VMX_EXIT_RDRAND: + case VMX_EXIT_RDSEED: + { + Assert(ExitInstrInfo.RdrandRdseed.u2OperandSize != 3); + break; + } + } + + /* Update displacement and return the constructed VM-exit instruction information field. */ + if (pGCPtrDisp) + *pGCPtrDisp = GCPtrDisp; + + return ExitInstrInfo.u; +} + + +/** + * VMX VM-exit handler. + * + * @returns Strict VBox status code. + * @retval VINF_VMX_VMEXIT when the VM-exit is successful. + * @retval VINF_EM_TRIPLE_FAULT when VM-exit is unsuccessful and leads to a + * triple-fault. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason. + * @param u64ExitQual The Exit qualification. + * + * @remarks We need not necessarily have completed VM-entry before a VM-exit is + * called. Failures during VM-entry can cause VM-exits as well, so we + * -cannot- assert we're in VMX non-root mode here. + */ +VBOXSTRICTRC iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual) RT_NOEXCEPT +{ +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3) + RT_NOREF3(pVCpu, uExitReason, u64ExitQual); + AssertMsgFailed(("VM-exit should only be invoked from ring-3 when nested-guest executes only in ring-3!\n")); + return VERR_IEM_IPE_7; +# else + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* Just count this as an exit and be done with that. */ + pVCpu->iem.s.cPotentialExits++; + + /* + * Import all the guest-CPU state. + * + * HM on returning to guest execution would have to reset up a whole lot of state + * anyway, (e.g., VM-entry/VM-exit controls) and we do not ever import a part of + * the state and flag reloading the entire state on re-entry. So import the entire + * state here, see HMNotifyVmxNstGstVmexit() for more comments. + */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL); + + /* + * Ensure VM-entry interruption information valid bit is cleared. + * + * We do it here on every VM-exit so that even premature VM-exits (e.g. those caused + * by invalid-guest state or machine-check exceptions) also clear this bit. + * + * See Intel spec. 27.2 "Recording VM-exit Information And Updating VM-entry control fields". + */ + if (VMX_ENTRY_INT_INFO_IS_VALID(pVmcs->u32EntryIntInfo)) + pVmcs->u32EntryIntInfo &= ~VMX_ENTRY_INT_INFO_VALID; + + /* + * Update the VM-exit reason and Exit qualification. + * Other VMCS read-only data fields are expected to be updated by the caller already. + */ + pVmcs->u32RoExitReason = uExitReason; + pVmcs->u64RoExitQual.u = u64ExitQual; + + Log2(("vmexit: reason=%u qual=%#RX64 cs:rip=%04x:%08RX64 cr0=%#RX64 cr3=%#RX64 cr4=%#RX64 eflags=%#RX32\n", uExitReason, + pVmcs->u64RoExitQual.u, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.cr0, + pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.eflags.u)); + + /* + * Update the IDT-vectoring information fields if the VM-exit is triggered during delivery of an event. + * See Intel spec. 27.2.4 "Information for VM Exits During Event Delivery". + */ + { + uint8_t uVector; + uint32_t fFlags; + uint32_t uErrCode; + bool const fInEventDelivery = IEMGetCurrentXcpt(pVCpu, &uVector, &fFlags, &uErrCode, NULL /* puCr2 */); + if (fInEventDelivery) + { + /* + * A VM-exit is not considered to occur during event delivery when the VM-exit is + * caused by a triple-fault or the original event results in a double-fault that + * causes the VM exit directly (exception bitmap). Therefore, we must not set the + * original event information into the IDT-vectoring information fields. + * + * See Intel spec. 27.2.4 "Information for VM Exits During Event Delivery". + */ + if ( uExitReason != VMX_EXIT_TRIPLE_FAULT + && ( uExitReason != VMX_EXIT_XCPT_OR_NMI + || !VMX_EXIT_INT_INFO_IS_XCPT_DF(pVmcs->u32RoExitIntInfo))) + { + uint8_t const uIdtVectoringType = iemVmxGetEventType(uVector, fFlags); + uint8_t const fErrCodeValid = RT_BOOL(fFlags & IEM_XCPT_FLAGS_ERR); + uint32_t const uIdtVectoringInfo = RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_VECTOR, uVector) + | RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_TYPE, uIdtVectoringType) + | RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_ERR_CODE_VALID, fErrCodeValid) + | RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_VALID, 1); + iemVmxVmcsSetIdtVectoringInfo(pVCpu, uIdtVectoringInfo); + iemVmxVmcsSetIdtVectoringErrCode(pVCpu, uErrCode); + Log2(("vmexit: idt_info=%#RX32 idt_err_code=%#RX32 cr2=%#RX64\n", uIdtVectoringInfo, uErrCode, + pVCpu->cpum.GstCtx.cr2)); + } + } + } + + /* The following VMCS fields should always be zero since we don't support injecting SMIs into a guest. */ + Assert(pVmcs->u64RoIoRcx.u == 0); + Assert(pVmcs->u64RoIoRsi.u == 0); + Assert(pVmcs->u64RoIoRdi.u == 0); + Assert(pVmcs->u64RoIoRip.u == 0); + + /* + * Save the guest state back into the VMCS. + * We only need to save the state when the VM-entry was successful. + */ + bool const fVmentryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason); + if (!fVmentryFailed) + { + /* We should not cause an NMI-window/interrupt-window VM-exit when injecting events as part of VM-entry. */ + if (!CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx)) + { + Assert(uExitReason != VMX_EXIT_NMI_WINDOW); + Assert(uExitReason != VMX_EXIT_INT_WINDOW); + } + + /* For exception or NMI VM-exits the VM-exit interruption info. field must be valid. */ + Assert(uExitReason != VMX_EXIT_XCPT_OR_NMI || VMX_EXIT_INT_INFO_IS_VALID(pVmcs->u32RoExitIntInfo)); + + /* + * If we support storing EFER.LMA into IA32e-mode guest field on VM-exit, we need to do that now. + * See Intel spec. 27.2 "Recording VM-exit Information And Updating VM-entry Control". + * + * It is not clear from the Intel spec. if this is done only when VM-entry succeeds. + * If a VM-exit happens before loading guest EFER, we risk restoring the host EFER.LMA + * as guest-CPU state would not been modified. Hence for now, we do this only when + * the VM-entry succeeded. + */ + /** @todo r=ramshankar: Figure out if this bit gets set to host EFER.LMA on real + * hardware when VM-exit fails during VM-entry (e.g. VERR_VMX_INVALID_GUEST_STATE). */ + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxExitSaveEferLma) + { + if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA) + pVmcs->u32EntryCtls |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST; + else + pVmcs->u32EntryCtls &= ~VMX_ENTRY_CTLS_IA32E_MODE_GUEST; + } + + /* + * The rest of the high bits of the VM-exit reason are only relevant when the VM-exit + * occurs in enclave mode/SMM which we don't support yet. + * + * If we ever add support for it, we can pass just the lower bits to the functions + * below, till then an assert should suffice. + */ + Assert(!RT_HI_U16(uExitReason)); + + /* Save the guest state into the VMCS and restore guest MSRs from the auto-store guest MSR area. */ + iemVmxVmexitSaveGuestState(pVCpu, uExitReason); + int rc = iemVmxVmexitSaveGuestAutoMsrs(pVCpu, uExitReason); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + return iemVmxAbort(pVCpu, VMXABORT_SAVE_GUEST_MSRS); + + /* Clear any saved NMI-blocking state so we don't assert on next VM-entry (if it was in effect on the previous one). */ + pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit &= ~CPUMCTX_INHIBIT_NMI; + } + else + { + /* Restore the NMI-blocking state if VM-entry failed due to invalid guest state or while loading MSRs. */ + uint32_t const uExitReasonBasic = VMX_EXIT_REASON_BASIC(uExitReason); + if ( uExitReasonBasic == VMX_EXIT_ERR_INVALID_GUEST_STATE + || uExitReasonBasic == VMX_EXIT_ERR_MSR_LOAD) + iemVmxVmexitRestoreNmiBlockingFF(pVCpu); + } + + /* + * Stop any running VMX-preemption timer if necessary. + */ + if (pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER) + CPUMStopGuestVmxPremptTimer(pVCpu); + + /* + * Clear any pending VMX nested-guest force-flags. + * These force-flags have no effect on (outer) guest execution and will + * be re-evaluated and setup on the next nested-guest VM-entry. + */ + VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_VMX_ALL_MASK); + + /* + * We're no longer in nested-guest execution mode. + * + * It is important to do this prior to loading the host state because + * PGM looks at fInVmxNonRootMode to determine if it needs to perform + * second-level address translation while switching to host CR3. + */ + pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode = false; + + /* Restore the host (outer guest) state. */ + VBOXSTRICTRC rcStrict = iemVmxVmexitLoadHostState(pVCpu, uExitReason); + if (RT_SUCCESS(rcStrict)) + { + Assert(rcStrict == VINF_SUCCESS); + rcStrict = VINF_VMX_VMEXIT; + } + else + Log(("vmexit: Loading host-state failed. uExitReason=%u rc=%Rrc\n", uExitReason, VBOXSTRICTRC_VAL(rcStrict))); + + if (VM_IS_HM_ENABLED(pVCpu->CTX_SUFF(pVM))) + { + /* Notify HM that the current VMCS fields have been modified. */ + HMNotifyVmxNstGstCurrentVmcsChanged(pVCpu); + + /* Notify HM that we've completed the VM-exit. */ + HMNotifyVmxNstGstVmexit(pVCpu); + } + +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3) + /* Revert any IEM-only nested-guest execution policy, otherwise return rcStrict. */ + Log(("vmexit: Disabling IEM-only EM execution policy!\n")); + int rcSched = EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, false); + if (rcSched != VINF_SUCCESS) + iemSetPassUpStatus(pVCpu, rcSched); +# endif + return rcStrict; +# endif +} + + +/** + * VMX VM-exit handler for VM-exits due to instruction execution. + * + * This is intended for instructions where the caller provides all the relevant + * VM-exit information. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pExitInfo Pointer to the VM-exit information. + */ +static VBOXSTRICTRC iemVmxVmexitInstrWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + /* + * For instructions where any of the following fields are not applicable: + * - Exit qualification must be cleared. + * - VM-exit instruction info. is undefined. + * - Guest-linear address is undefined. + * - Guest-physical address is undefined. + * + * The VM-exit instruction length is mandatory for all VM-exits that are caused by + * instruction execution. For VM-exits that are not due to instruction execution this + * field is undefined. + * + * In our implementation in IEM, all undefined fields are generally cleared. However, + * if the caller supplies information (from say the physical CPU directly) it is + * then possible that the undefined fields are not cleared. + * + * See Intel spec. 27.2.1 "Basic VM-Exit Information". + * See Intel spec. 27.2.4 "Information for VM Exits Due to Instruction Execution". + */ + Assert(pExitInfo); + AssertMsg(pExitInfo->uReason <= VMX_EXIT_MAX, ("uReason=%u\n", pExitInfo->uReason)); + AssertMsg(pExitInfo->cbInstr >= 1 && pExitInfo->cbInstr <= 15, + ("uReason=%u cbInstr=%u\n", pExitInfo->uReason, pExitInfo->cbInstr)); + + /* Update all the relevant fields from the VM-exit instruction information struct. */ + iemVmxVmcsSetExitInstrInfo(pVCpu, pExitInfo->InstrInfo.u); + iemVmxVmcsSetExitGuestLinearAddr(pVCpu, pExitInfo->u64GuestLinearAddr); + iemVmxVmcsSetExitGuestPhysAddr(pVCpu, pExitInfo->u64GuestPhysAddr); + iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr); + + /* Perform the VM-exit. */ + return iemVmxVmexit(pVCpu, pExitInfo->uReason, pExitInfo->u64Qual); +} + + +/** + * VMX VM-exit handler for VM-exits due to instruction execution. + * + * This is intended for instructions that only provide the VM-exit instruction + * length. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr) RT_NOEXCEPT +{ +#ifdef VBOX_STRICT + /* + * To prevent us from shooting ourselves in the foot. + * The follow instructions should convey more than just the instruction length. + */ + switch (uExitReason) + { + case VMX_EXIT_INVEPT: + case VMX_EXIT_INVPCID: + case VMX_EXIT_INVVPID: + case VMX_EXIT_LDTR_TR_ACCESS: + case VMX_EXIT_GDTR_IDTR_ACCESS: + case VMX_EXIT_VMCLEAR: + case VMX_EXIT_VMPTRLD: + case VMX_EXIT_VMPTRST: + case VMX_EXIT_VMREAD: + case VMX_EXIT_VMWRITE: + case VMX_EXIT_VMXON: + case VMX_EXIT_XRSTORS: + case VMX_EXIT_XSAVES: + case VMX_EXIT_RDRAND: + case VMX_EXIT_RDSEED: + case VMX_EXIT_IO_INSTR: + AssertMsgFailedReturn(("Use iemVmxVmexitInstrNeedsInfo for uExitReason=%u\n", uExitReason), VERR_IEM_IPE_5); + break; + } +#endif + + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_INSTR_LEN(uExitReason, cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); +} + + +/** + * Interface for HM and EM to emulate VM-exit due to a triple-fault. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTripleFault(PVMCPUCC pVCpu) +{ + VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate VM-exit due to startup-IPI (SIPI). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uVector The SIPI vector. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitStartupIpi(PVMCPUCC pVCpu, uint8_t uVector) +{ + VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_SIPI, uVector); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate a VM-exit. + * + * If a specialized version of a VM-exit handler exists, that must be used instead. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uExitReason The VM-exit reason. + * @param u64ExitQual The Exit qualification. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual) +{ + VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, uExitReason, u64ExitQual); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate a VM-exit due to an instruction. + * + * This is meant to be used for those instructions that VMX provides additional + * decoding information beyond just the instruction length! + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstrWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + VBOXSTRICTRC rcStrict = iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate a VM-exit due to an instruction. + * + * This is meant to be used for those instructions that VMX provides only the + * instruction length. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @param cbInstr The instruction length in bytes. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr) +{ + VBOXSTRICTRC rcStrict = iemVmxVmexitInstr(pVCpu, uExitReason, cbInstr); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * VMX VM-exit handler for VM-exits due to instruction execution. + * + * This is intended for instructions that have a ModR/M byte and update the VM-exit + * instruction information and Exit qualification fields. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason. + * @param uInstrid The instruction identity (VMXINSTRID_XXX). + * @param cbInstr The instruction length in bytes. + * + * @remarks Do not use this for INS/OUTS instruction. + */ +VBOXSTRICTRC iemVmxVmexitInstrNeedsInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, uint8_t cbInstr) RT_NOEXCEPT +{ +#ifdef VBOX_STRICT + /* + * To prevent us from shooting ourselves in the foot. + * The follow instructions convey specific info that require using their respective handlers. + */ + switch (uExitReason) + { + case VMX_EXIT_INVEPT: + case VMX_EXIT_INVPCID: + case VMX_EXIT_INVVPID: + case VMX_EXIT_LDTR_TR_ACCESS: + case VMX_EXIT_GDTR_IDTR_ACCESS: + case VMX_EXIT_VMCLEAR: + case VMX_EXIT_VMPTRLD: + case VMX_EXIT_VMPTRST: + case VMX_EXIT_VMREAD: + case VMX_EXIT_VMWRITE: + case VMX_EXIT_VMXON: + case VMX_EXIT_XRSTORS: + case VMX_EXIT_XSAVES: + case VMX_EXIT_RDRAND: + case VMX_EXIT_RDSEED: + break; + default: + AssertMsgFailedReturn(("Use instruction-specific handler\n"), VERR_IEM_IPE_5); + break; + } +#endif + + /* + * Update the Exit qualification field with displacement bytes. + * See Intel spec. 27.2.1 "Basic VM-Exit Information". + */ + /* Construct the VM-exit instruction information. */ + RTGCPTR GCPtrDisp; + uint32_t const uInstrInfo = iemVmxGetExitInstrInfo(pVCpu, uExitReason, uInstrId, &GCPtrDisp); + + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO(uExitReason, GCPtrDisp, uInstrInfo, cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); +} + + +/** + * VMX VM-exit handler for VM-exits due to INVLPG. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtrPage The guest-linear address of the page being invalidated. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrInvlpg(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, uint8_t cbInstr) RT_NOEXCEPT +{ + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_INVLPG, GCPtrPage, cbInstr); + Assert(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode || !RT_HI_U32(ExitInfo.u64Qual)); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); +} + + +/** + * VMX VM-exit handler for VM-exits due to LMSW. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uGuestCr0 The current guest CR0. + * @param pu16NewMsw The machine-status word specified in LMSW's source + * operand. This will be updated depending on the VMX + * guest/host CR0 mask if LMSW is not intercepted. + * @param GCPtrEffDst The guest-linear address of the source operand in case + * of a memory operand. For register operand, pass + * NIL_RTGCPTR. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrLmsw(PVMCPUCC pVCpu, uint32_t uGuestCr0, uint16_t *pu16NewMsw, + RTGCPTR GCPtrEffDst, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(pu16NewMsw); + + uint16_t const uNewMsw = *pu16NewMsw; + if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw)) + { + Log2(("lmsw: Guest intercept -> VM-exit\n")); + bool const fMemOperand = RT_BOOL(GCPtrEffDst != NIL_RTGCPTR); + VMXVEXITINFO ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 0) /* CR0 */ + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_LMSW_OP, fMemOperand) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_LMSW_DATA, uNewMsw) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, VMX_EXIT_QUAL_CRX_ACCESS_LMSW), + cbInstr); + if (fMemOperand) + { + Assert(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode || !RT_HI_U32(GCPtrEffDst)); + ExitInfo.u64GuestLinearAddr = GCPtrEffDst; + } + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + + /* + * If LMSW did not cause a VM-exit, any CR0 bits in the range 0:3 that is set in the + * CR0 guest/host mask must be left unmodified. + * + * See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation". + */ + uint32_t const fGstHostMask = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u; + uint32_t const fGstHostLmswMask = fGstHostMask & (X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS); + *pu16NewMsw = (uGuestCr0 & fGstHostLmswMask) | (uNewMsw & ~fGstHostLmswMask); + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to CLTS. + * + * @returns Strict VBox status code. + * @retval VINF_VMX_MODIFIES_BEHAVIOR if the CLTS instruction did not cause a + * VM-exit but must not modify the guest CR0.TS bit. + * @retval VINF_VMX_INTERCEPT_NOT_ACTIVE if the CLTS instruction did not cause a + * VM-exit and modification to the guest CR0.TS bit is allowed (subject to + * CR0 fixed bits in VMX operation). + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrClts(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT +{ + /* + * If CR0.TS is owned by the host: + * - If CR0.TS is set in the read-shadow, we must cause a VM-exit. + * - If CR0.TS is cleared in the read-shadow, no VM-exit is caused and the + * CLTS instruction completes without clearing CR0.TS. + * + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + uint32_t const fGstHostMask = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u; + if (fGstHostMask & X86_CR0_TS) + { + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0ReadShadow.u & X86_CR0_TS) + { + Log2(("clts: Guest intercept -> VM-exit\n")); + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 0) /* CR0 */ + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, + VMX_EXIT_QUAL_CRX_ACCESS_CLTS), + cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return VINF_VMX_MODIFIES_BEHAVIOR; + } + + /* + * If CR0.TS is not owned by the host, the CLTS instructions operates normally + * and may modify CR0.TS (subject to CR0 fixed bits in VMX operation). + */ + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to 'Mov CR0,GReg' and 'Mov CR4,GReg' + * (CR0/CR4 write). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param iCrReg The control register (either CR0 or CR4). + * @param uGuestCrX The current guest CR0/CR4. + * @param puNewCrX Pointer to the new CR0/CR4 value. Will be updated if no + * VM-exit is caused. + * @param iGReg The general register from which the CR0/CR4 value is being + * loaded. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrMovToCr0Cr4(PVMCPUCC pVCpu, uint8_t iCrReg, uint64_t *puNewCrX, + uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(puNewCrX); + Assert(iCrReg == 0 || iCrReg == 4); + Assert(iGReg < X86_GREG_COUNT); + + uint64_t const uNewCrX = *puNewCrX; + if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX)) + { + Log2(("mov_Cr_Rd: (CR%u) Guest intercept -> VM-exit\n", iCrReg)); + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, iCrReg) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG, iGReg) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, VMX_EXIT_QUAL_CRX_ACCESS_WRITE), + cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + + /* + * If the Mov-to-CR0/CR4 did not cause a VM-exit, any bits owned by the host + * must not be modified the instruction. + * + * See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation". + */ + uint64_t uGuestCrX; + uint64_t fGstHostMask; + if (iCrReg == 0) + { + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); + uGuestCrX = pVCpu->cpum.GstCtx.cr0; + fGstHostMask = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u; + } + else + { + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4); + uGuestCrX = pVCpu->cpum.GstCtx.cr4; + fGstHostMask = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr4Mask.u; + } + + *puNewCrX = (uGuestCrX & fGstHostMask) | (*puNewCrX & ~fGstHostMask); + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to 'Mov GReg,CR3' (CR3 read). + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param iGReg The general register to which the CR3 value is being stored. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrMovFromCr3(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(iGReg < X86_GREG_COUNT); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3); + + /* + * If the CR3-store exiting control is set, we must cause a VM-exit. + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_CR3_STORE_EXIT) + { + Log2(("mov_Rd_Cr: (CR3) Guest intercept -> VM-exit\n")); + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 3) /* CR3 */ + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG, iGReg) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, VMX_EXIT_QUAL_CRX_ACCESS_READ), + cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to 'Mov CR3,GReg' (CR3 write). + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uNewCr3 The new CR3 value. + * @param iGReg The general register from which the CR3 value is being + * loaded. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrMovToCr3(PVMCPUCC pVCpu, uint64_t uNewCr3, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(iGReg < X86_GREG_COUNT); + + /* + * If the CR3-load exiting control is set and the new CR3 value does not + * match any of the CR3-target values in the VMCS, we must cause a VM-exit. + * + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + if (CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCr3)) + { + Log2(("mov_Cr_Rd: (CR3) Guest intercept -> VM-exit\n")); + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 3) /* CR3 */ + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG, iGReg) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, + VMX_EXIT_QUAL_CRX_ACCESS_WRITE), + cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to 'Mov GReg,CR8' (CR8 read). + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param iGReg The general register to which the CR8 value is being stored. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrMovFromCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(iGReg < X86_GREG_COUNT); + + /* + * If the CR8-store exiting control is set, we must cause a VM-exit. + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_CR8_STORE_EXIT) + { + Log2(("mov_Rd_Cr: (CR8) Guest intercept -> VM-exit\n")); + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 8) /* CR8 */ + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG, iGReg) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, VMX_EXIT_QUAL_CRX_ACCESS_READ), + cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to 'Mov CR8,GReg' (CR8 write). + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param iGReg The general register from which the CR8 value is being + * loaded. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrMovToCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(iGReg < X86_GREG_COUNT); + + /* + * If the CR8-load exiting control is set, we must cause a VM-exit. + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_CR8_LOAD_EXIT) + { + Log2(("mov_Cr_Rd: (CR8) Guest intercept -> VM-exit\n")); + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 8) /* CR8 */ + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG, iGReg) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, VMX_EXIT_QUAL_CRX_ACCESS_WRITE), + cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to 'Mov DRx,GReg' (DRx write) and 'Mov + * GReg,DRx' (DRx read). + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uInstrid The instruction identity (VMXINSTRID_MOV_TO_DRX or + * VMXINSTRID_MOV_FROM_DRX). + * @param iDrReg The debug register being accessed. + * @param iGReg The general register to/from which the DRx value is being + * store/loaded. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrMovDrX(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint8_t iDrReg, + uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(iDrReg <= 7); + Assert(uInstrId == VMXINSTRID_MOV_TO_DRX || uInstrId == VMXINSTRID_MOV_FROM_DRX); + Assert(iGReg < X86_GREG_COUNT); + + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT) + { + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_DRX, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_DRX_REGISTER, iDrReg) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_DRX_GENREG, iGReg) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_DRX_DIRECTION, + uInstrId == VMXINSTRID_MOV_TO_DRX + ? VMX_EXIT_QUAL_DRX_DIRECTION_WRITE + : VMX_EXIT_QUAL_DRX_DIRECTION_READ), + cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to I/O instructions (IN and OUT). + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uInstrId The VM-exit instruction identity (VMXINSTRID_IO_IN or + * VMXINSTRID_IO_OUT). + * @param u16Port The I/O port being accessed. + * @param fImm Whether the I/O port was encoded using an immediate operand + * or the implicit DX register. + * @param cbAccess The size of the I/O access in bytes (1, 2 or 4 bytes). + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port, + bool fImm, uint8_t cbAccess, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(uInstrId == VMXINSTRID_IO_IN || uInstrId == VMXINSTRID_IO_OUT); + Assert(cbAccess == 1 || cbAccess == 2 || cbAccess == 4); + + if (CPUMIsGuestVmxIoInterceptSet(pVCpu, u16Port, cbAccess)) + { + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_IO_INSTR, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_WIDTH, cbAccess - 1) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_ENCODING, fImm) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_PORT, u16Port) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_DIRECTION, + uInstrId == VMXINSTRID_IO_IN + ? VMX_EXIT_QUAL_IO_DIRECTION_IN + : VMX_EXIT_QUAL_IO_DIRECTION_OUT), + cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to string I/O instructions (INS and OUTS). + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uInstrId The VM-exit instruction identity (VMXINSTRID_IO_INS or + * VMXINSTRID_IO_OUTS). + * @param u16Port The I/O port being accessed. + * @param cbAccess The size of the I/O access in bytes (1, 2 or 4 bytes). + * @param fRep Whether the instruction has a REP prefix or not. + * @param ExitInstrInfo The VM-exit instruction info. field. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrStrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port, uint8_t cbAccess, + bool fRep, VMXEXITINSTRINFO ExitInstrInfo, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(uInstrId == VMXINSTRID_IO_INS || uInstrId == VMXINSTRID_IO_OUTS); + Assert(cbAccess == 1 || cbAccess == 2 || cbAccess == 4); + Assert(ExitInstrInfo.StrIo.iSegReg < X86_SREG_COUNT); + Assert(ExitInstrInfo.StrIo.u3AddrSize == 0 || ExitInstrInfo.StrIo.u3AddrSize == 1 || ExitInstrInfo.StrIo.u3AddrSize == 2); + Assert(uInstrId != VMXINSTRID_IO_INS || ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES); + + if (CPUMIsGuestVmxIoInterceptSet(pVCpu, u16Port, cbAccess)) + { + /* + * Figure out the guest-linear address and the direction bit (INS/OUTS). + */ + /** @todo r=ramshankar: Is there something in IEM that already does this? */ + static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) }; + uint8_t const iSegReg = ExitInstrInfo.StrIo.iSegReg; + uint8_t const uAddrSize = ExitInstrInfo.StrIo.u3AddrSize; + uint64_t const uAddrSizeMask = s_auAddrSizeMasks[uAddrSize]; + + uint32_t uDirection; + uint64_t uGuestLinearAddr; + if (uInstrId == VMXINSTRID_IO_INS) + { + uDirection = VMX_EXIT_QUAL_IO_DIRECTION_IN; + uGuestLinearAddr = pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base + (pVCpu->cpum.GstCtx.rdi & uAddrSizeMask); + } + else + { + uDirection = VMX_EXIT_QUAL_IO_DIRECTION_OUT; + uGuestLinearAddr = pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base + (pVCpu->cpum.GstCtx.rsi & uAddrSizeMask); + } + + /* + * If the segment is unusable, the guest-linear address in undefined. + * We shall clear it for consistency. + * + * See Intel spec. 27.2.1 "Basic VM-Exit Information". + */ + if (pVCpu->cpum.GstCtx.aSRegs[iSegReg].Attr.n.u1Unusable) + uGuestLinearAddr = 0; + + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_AND_LIN_ADDR(VMX_EXIT_IO_INSTR, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_WIDTH, cbAccess - 1) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_DIRECTION, uDirection) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_IS_STRING, 1) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_IS_REP, fRep) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_ENCODING, + VMX_EXIT_QUAL_IO_ENCODING_DX) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_PORT, u16Port), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxInsOutInfo + ? ExitInstrInfo.u : 0, + cbInstr, + uGuestLinearAddr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to MWAIT. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param fMonitorHwArmed Whether the address-range monitor hardware is armed. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitInstrMwait(PVMCPUCC pVCpu, bool fMonitorHwArmed, uint8_t cbInstr) RT_NOEXCEPT +{ + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MWAIT, fMonitorHwArmed, cbInstr); + return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); +} + + +/** + * VMX VM-exit handler for VM-exits due to PAUSE. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + */ +static VBOXSTRICTRC iemVmxVmexitInstrPause(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT +{ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* + * The PAUSE VM-exit is controlled by the "PAUSE exiting" control and the + * "PAUSE-loop exiting" control. + * + * The PLE-Gap is the maximum number of TSC ticks between two successive executions of + * the PAUSE instruction before we cause a VM-exit. The PLE-Window is the maximum amount + * of TSC ticks the guest is allowed to execute in a pause loop before we must cause + * a VM-exit. + * + * See Intel spec. 24.6.13 "Controls for PAUSE-Loop Exiting". + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + bool fIntercept = false; + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT) + fIntercept = true; + else if ( (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT) + && pVCpu->iem.s.uCpl == 0) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT); + + /* + * A previous-PAUSE-tick value of 0 is used to identify the first time + * execution of a PAUSE instruction after VM-entry at CPL 0. We must + * consider this to be the first execution of PAUSE in a loop according + * to the Intel. + * + * All subsequent records for the previous-PAUSE-tick we ensure that it + * cannot be zero by OR'ing 1 to rule out the TSC wrap-around cases at 0. + */ + uint64_t *puFirstPauseLoopTick = &pVCpu->cpum.GstCtx.hwvirt.vmx.uFirstPauseLoopTick; + uint64_t *puPrevPauseTick = &pVCpu->cpum.GstCtx.hwvirt.vmx.uPrevPauseTick; + uint64_t const uTick = TMCpuTickGet(pVCpu); + uint32_t const uPleGap = pVmcs->u32PleGap; + uint32_t const uPleWindow = pVmcs->u32PleWindow; + if ( *puPrevPauseTick == 0 + || uTick - *puPrevPauseTick > uPleGap) + *puFirstPauseLoopTick = uTick; + else if (uTick - *puFirstPauseLoopTick > uPleWindow) + fIntercept = true; + + *puPrevPauseTick = uTick | 1; + } + + if (fIntercept) + return iemVmxVmexitInstr(pVCpu, VMX_EXIT_PAUSE, cbInstr); + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exits due to task switches. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param enmTaskSwitch The cause of the task switch. + * @param SelNewTss The selector of the new TSS. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr) RT_NOEXCEPT +{ + /* + * Task-switch VM-exits are unconditional and provide the Exit qualification. + * + * If the cause of the task switch is due to execution of CALL, IRET or the JMP + * instruction or delivery of the exception generated by one of these instructions + * lead to a task switch through a task gate in the IDT, we need to provide the + * VM-exit instruction length. Any other means of invoking a task switch VM-exit + * leaves the VM-exit instruction length field undefined. + * + * See Intel spec. 25.2 "Other Causes Of VM Exits". + * See Intel spec. 27.2.4 "Information for VM Exits Due to Instruction Execution". + */ + Assert(cbInstr <= 15); + + uint8_t uType; + switch (enmTaskSwitch) + { + case IEMTASKSWITCH_CALL: uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_CALL; break; + case IEMTASKSWITCH_IRET: uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IRET; break; + case IEMTASKSWITCH_JUMP: uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_JMP; break; + case IEMTASKSWITCH_INT_XCPT: uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT; break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + uint64_t const u64ExitQual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_TASK_SWITCH_NEW_TSS, SelNewTss) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_TASK_SWITCH_SOURCE, uType); + iemVmxVmcsSetExitInstrLen(pVCpu, cbInstr); + return iemVmxVmexit(pVCpu, VMX_EXIT_TASK_SWITCH, u64ExitQual); +} + + +/** + * VMX VM-exit handler for trap-like VM-exits. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pExitInfo Pointer to the VM-exit information. + * @param pExitEventInfo Pointer to the VM-exit event information. + */ +static VBOXSTRICTRC iemVmxVmexitTrapLikeWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + Assert(VMXIsVmexitTrapLike(pExitInfo->uReason)); + iemVmxVmcsSetGuestPendingDbgXcpts(pVCpu, pExitInfo->u64GuestPendingDbgXcpts); + return iemVmxVmexit(pVCpu, pExitInfo->uReason, pExitInfo->u64Qual); +} + + +/** + * Interface for HM and EM to emulate a trap-like VM-exit (MTF, APIC-write, + * Virtualized-EOI, TPR-below threshold). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTrapLike(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + Assert(pExitInfo); + VBOXSTRICTRC rcStrict = iemVmxVmexitTrapLikeWithInfo(pVCpu, pExitInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * VMX VM-exit handler for VM-exits due to task switches. + * + * This is intended for task switches where the caller provides all the relevant + * VM-exit information. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pExitInfo Pointer to the VM-exit information. + * @param pExitEventInfo Pointer to the VM-exit event information. + */ +static VBOXSTRICTRC iemVmxVmexitTaskSwitchWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, + PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT +{ + Assert(pExitInfo->uReason == VMX_EXIT_TASK_SWITCH); + iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr); + iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo); + iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode); + return iemVmxVmexit(pVCpu, VMX_EXIT_TASK_SWITCH, pExitInfo->u64Qual); +} + + +/** + * Interface for HM and EM to emulate a VM-exit due to a task switch. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @param pExitEventInfo Pointer to the VM-exit event information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTaskSwitch(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo) +{ + Assert(pExitInfo); + Assert(pExitEventInfo); + Assert(pExitInfo->uReason == VMX_EXIT_TASK_SWITCH); + VBOXSTRICTRC rcStrict = iemVmxVmexitTaskSwitchWithInfo(pVCpu, pExitInfo, pExitEventInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * VMX VM-exit handler for VM-exits due to expiring of the preemption timer. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + */ +VBOXSTRICTRC iemVmxVmexitPreemptTimer(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER)); + Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER); + + /* Import the hardware virtualization state (for nested-guest VM-entry TSC-tick). */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT); + + /* Save the VMX-preemption timer value (of 0) back in to the VMCS if the CPU supports this feature. */ + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ExitCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER) + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PreemptTimer = 0; + + /* Cause the VMX-preemption timer VM-exit. The Exit qualification MBZ. */ + return iemVmxVmexit(pVCpu, VMX_EXIT_PREEMPT_TIMER, 0 /* u64ExitQual */); +} + + +/** + * Interface for HM and EM to emulate VM-exit due to expiry of the preemption timer. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitPreemptTimer(PVMCPUCC pVCpu) +{ + VBOXSTRICTRC rcStrict = iemVmxVmexitPreemptTimer(pVCpu); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * VMX VM-exit handler for VM-exits due to external interrupts. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uVector The external interrupt vector (pass 0 if the interrupt + * is still pending since we typically won't know the + * vector). + * @param fIntPending Whether the external interrupt is pending or + * acknowledged in the interrupt controller. + */ +static VBOXSTRICTRC iemVmxVmexitExtInt(PVMCPUCC pVCpu, uint8_t uVector, bool fIntPending) RT_NOEXCEPT +{ + Assert(!fIntPending || uVector == 0); + + /* The VM-exit is subject to "External interrupt exiting" being set. */ + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PinCtls & VMX_PIN_CTLS_EXT_INT_EXIT) + { + if (fIntPending) + { + /* + * If the interrupt is pending and we don't need to acknowledge the + * interrupt on VM-exit, cause the VM-exit immediately. + * + * See Intel spec 25.2 "Other Causes Of VM Exits". + */ + if (!(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)) + return iemVmxVmexit(pVCpu, VMX_EXIT_EXT_INT, 0 /* u64ExitQual */); + + /* + * If the interrupt is pending and we -do- need to acknowledge the interrupt + * on VM-exit, postpone VM-exit till after the interrupt controller has been + * acknowledged that the interrupt has been consumed. Callers would have to call + * us again after getting the vector (and ofc, with fIntPending with false). + */ + return VINF_VMX_INTERCEPT_NOT_ACTIVE; + } + + /* + * If the interrupt is no longer pending (i.e. it has been acknowledged) and the + * "External interrupt exiting" and "Acknowledge interrupt on VM-exit" controls are + * all set, we need to record the vector of the external interrupt in the + * VM-exit interruption information field. Otherwise, mark this field as invalid. + * + * See Intel spec. 27.2.2 "Information for VM Exits Due to Vectored Events". + */ + uint32_t uExitIntInfo; + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT) + { + bool const fNmiUnblocking = pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret; + uExitIntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, uVector) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_EXT_INT) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_NMI_UNBLOCK_IRET, fNmiUnblocking) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID, 1); + } + else + uExitIntInfo = 0; + iemVmxVmcsSetExitIntInfo(pVCpu, uExitIntInfo); + + /* + * Cause the VM-exit whether or not the vector has been stored + * in the VM-exit interruption-information field. + */ + return iemVmxVmexit(pVCpu, VMX_EXIT_EXT_INT, 0 /* u64ExitQual */); + } + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * Interface for HM and EM to emulate VM-exit due to external interrupts. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uVector The external interrupt vector (pass 0 if the external + * interrupt is still pending). + * @param fIntPending Whether the external interrupt is pending or + * acknowdledged in the interrupt controller. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitExtInt(PVMCPUCC pVCpu, uint8_t uVector, bool fIntPending) +{ + VBOXSTRICTRC rcStrict = iemVmxVmexitExtInt(pVCpu, uVector, fIntPending); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * VMX VM-exit handler for VM-exits due to a double fault caused during delivery of + * an event. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + */ +VBOXSTRICTRC iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + uint32_t const fXcptBitmap = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap; + if (fXcptBitmap & RT_BIT(X86_XCPT_DF)) + { + /* + * The NMI-unblocking due to IRET field need not be set for double faults. + * See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception". + */ + uint32_t const uExitIntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, X86_XCPT_DF) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_ERR_CODE_VALID, 1) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_NMI_UNBLOCK_IRET, 0) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID, 1); + iemVmxVmcsSetExitIntInfo(pVCpu, uExitIntInfo); + return iemVmxVmexit(pVCpu, VMX_EXIT_XCPT_OR_NMI, 0 /* u64ExitQual */); + } + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for VM-exit due to delivery of an events. + * + * This is intended for VM-exit due to exceptions or NMIs where the caller provides + * all the relevant VM-exit information. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pExitInfo Pointer to the VM-exit information. + * @param pExitEventInfo Pointer to the VM-exit event information. + */ +static VBOXSTRICTRC iemVmxVmexitEventWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT +{ + Assert(pExitInfo); + Assert(pExitEventInfo); + Assert(pExitInfo->uReason == VMX_EXIT_XCPT_OR_NMI); + Assert(VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo)); + + iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr); + iemVmxVmcsSetExitIntInfo(pVCpu, pExitEventInfo->uExitIntInfo); + iemVmxVmcsSetExitIntErrCode(pVCpu, pExitEventInfo->uExitIntErrCode); + iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo); + iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode); + return iemVmxVmexit(pVCpu, VMX_EXIT_XCPT_OR_NMI, pExitInfo->u64Qual); +} + + +/** + * Interface for HM and EM to emulate VM-exit due to NMIs. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcptNmi(PVMCPUCC pVCpu) +{ + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_ONLY_REASON(VMX_EXIT_XCPT_OR_NMI); + VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_INT( RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID, 1) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE, + VMX_EXIT_INT_INFO_TYPE_NMI) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, + X86_XCPT_NMI), + 0); + VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, &ExitInfo, &ExitEventInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * Interface for HM and EM to emulate VM-exit due to exceptions. + * + * Exception includes NMIs, software exceptions (those generated by INT3 or + * INTO) and privileged software exceptions (those generated by INT1/ICEBP). + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @param pExitEventInfo Pointer to the VM-exit event information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcpt(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo) +{ + Assert(pExitInfo); + Assert(pExitEventInfo); + VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, pExitInfo, pExitEventInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * VMX VM-exit handler for VM-exits due to delivery of an event. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uVector The interrupt / exception vector. + * @param fFlags The flags (see IEM_XCPT_FLAGS_XXX). + * @param uErrCode The error code associated with the event. + * @param uCr2 The CR2 value in case of a \#PF exception. + * @param cbInstr The instruction length in bytes. + */ +VBOXSTRICTRC iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode, + uint64_t uCr2, uint8_t cbInstr) RT_NOEXCEPT +{ + /* + * If the event is being injected as part of VM-entry, it is -not- subject to event + * intercepts in the nested-guest. However, secondary exceptions that occur during + * injection of any event -are- subject to event interception. + * + * See Intel spec. 26.5.1.2 "VM Exits During Event Injection". + */ + if (!CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx)) + { + /* + * If the event is a virtual-NMI (which is an NMI being inject during VM-entry) + * virtual-NMI blocking must be set in effect rather than physical NMI blocking. + * + * See Intel spec. 24.6.1 "Pin-Based VM-Execution Controls". + */ + if ( uVector == X86_XCPT_NMI + && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT) + && (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)) + pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = true; + else + Assert(!pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking); + + CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true); + return VINF_VMX_INTERCEPT_NOT_ACTIVE; + } + + /* + * We are injecting an external interrupt, check if we need to cause a VM-exit now. + * If not, the caller will continue delivery of the external interrupt as it would + * normally. The interrupt is no longer pending in the interrupt controller at this + * point. + */ + if (fFlags & IEM_XCPT_FLAGS_T_EXT_INT) + { + Assert(!VMX_IDT_VECTORING_INFO_IS_VALID(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoIdtVectoringInfo)); + return iemVmxVmexitExtInt(pVCpu, uVector, false /* fIntPending */); + } + + /* + * Evaluate intercepts for hardware exceptions, software exceptions (#BP, #OF), + * and privileged software exceptions (#DB generated by INT1/ICEBP) and software + * interrupts. + */ + Assert(fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_SOFT_INT)); + bool fIntercept; + if ( !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) + || (fFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_OF_INSTR | IEM_XCPT_FLAGS_ICEBP_INSTR))) + fIntercept = CPUMIsGuestVmxXcptInterceptSet(&pVCpu->cpum.GstCtx, uVector, uErrCode); + else + { + /* Software interrupts cannot be intercepted and therefore do not cause a VM-exit. */ + fIntercept = false; + } + + /* + * Now that we've determined whether the event causes a VM-exit, we need to construct the + * relevant VM-exit information and cause the VM-exit. + */ + if (fIntercept) + { + Assert(!(fFlags & IEM_XCPT_FLAGS_T_EXT_INT)); + + /* Construct the rest of the event related information fields and cause the VM-exit. */ + uint64_t u64ExitQual; + if (uVector == X86_XCPT_PF) + { + Assert(fFlags & IEM_XCPT_FLAGS_CR2); + u64ExitQual = uCr2; + } + else if (uVector == X86_XCPT_DB) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6); + u64ExitQual = pVCpu->cpum.GstCtx.dr[6] & VMX_VMCS_EXIT_QUAL_VALID_MASK; + } + else + u64ExitQual = 0; + + uint8_t const fNmiUnblocking = pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret; + bool const fErrCodeValid = RT_BOOL(fFlags & IEM_XCPT_FLAGS_ERR); + uint8_t const uIntInfoType = iemVmxGetEventType(uVector, fFlags); + uint32_t const uExitIntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, uVector) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE, uIntInfoType) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_ERR_CODE_VALID, fErrCodeValid) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_NMI_UNBLOCK_IRET, fNmiUnblocking) + | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID, 1); + iemVmxVmcsSetExitIntInfo(pVCpu, uExitIntInfo); + iemVmxVmcsSetExitIntErrCode(pVCpu, uErrCode); + + /* + * For VM-exits due to software exceptions (those generated by INT3 or INTO) or privileged + * software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction + * length. + */ + if ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) + || (fFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_OF_INSTR | IEM_XCPT_FLAGS_ICEBP_INSTR))) + iemVmxVmcsSetExitInstrLen(pVCpu, cbInstr); + else + iemVmxVmcsSetExitInstrLen(pVCpu, 0); + + return iemVmxVmexit(pVCpu, VMX_EXIT_XCPT_OR_NMI, u64ExitQual); + } + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * VMX VM-exit handler for EPT misconfiguration. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhysAddr The physical address causing the EPT misconfiguration. + * This need not be page aligned (e.g. nested-guest in real + * mode). + */ +static VBOXSTRICTRC iemVmxVmexitEptMisconfig(PVMCPUCC pVCpu, RTGCPHYS GCPhysAddr) RT_NOEXCEPT +{ + iemVmxVmcsSetExitGuestPhysAddr(pVCpu, GCPhysAddr); + return iemVmxVmexit(pVCpu, VMX_EXIT_EPT_MISCONFIG, 0 /* u64ExitQual */); +} + + +/** + * VMX VM-exit handler for EPT misconfiguration. + * + * This is intended for EPT misconfigurations where the caller provides all the + * relevant VM-exit information. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhysAddr The physical address causing the EPT misconfiguration. + * This need not be page aligned (e.g. nested-guest in real + * mode). + * @param pExitEventInfo Pointer to the VM-exit event information. + */ +static VBOXSTRICTRC iemVmxVmexitEptMisconfigWithInfo(PVMCPUCC pVCpu, RTGCPHYS GCPhysAddr, PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT +{ + Assert(pExitEventInfo); + Assert(!VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo)); + iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo); + iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode); + iemVmxVmcsSetExitGuestPhysAddr(pVCpu, GCPhysAddr); + return iemVmxVmexit(pVCpu, VMX_EXIT_EPT_MISCONFIG, 0 /* u64ExitQual */); +} + + +/** + * Interface for HM and EM to emulate a VM-exit due to an EPT misconfiguration. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPhysAddr The nested-guest physical address causing the EPT + * misconfiguration. + * @param pExitEventInfo Pointer to the VM-exit event information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitEptMisconfig(PVMCPUCC pVCpu, RTGCPHYS GCPhysAddr, PCVMXVEXITEVENTINFO pExitEventInfo) +{ + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = iemVmxVmexitEptMisconfigWithInfo(pVCpu, GCPhysAddr, pExitEventInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * VMX VM-exit handler for EPT violation. + * + * @param pVCpu The cross context virtual CPU structure. + * @param fAccess The access causing the EPT violation, IEM_ACCESS_XXX. + * @param fSlatFail The SLAT failure info, IEM_SLAT_FAIL_XXX. + * @param fEptAccess The EPT paging structure bits. + * @param GCPhysAddr The physical address causing the EPT violation. This + * need not be page aligned (e.g. nested-guest in real + * mode). + * @param fIsLinearAddrValid Whether translation of a linear address caused this + * EPT violation. If @c false, GCPtrAddr must be 0. + * @param GCPtrAddr The linear address causing the EPT violation. + * @param cbInstr The VM-exit instruction length. + */ +static VBOXSTRICTRC iemVmxVmexitEptViolation(PVMCPUCC pVCpu, uint32_t fAccess, uint32_t fSlatFail, + uint64_t fEptAccess, RTGCPHYS GCPhysAddr, bool fIsLinearAddrValid, + uint64_t GCPtrAddr, uint8_t cbInstr) RT_NOEXCEPT +{ + /* + * If the linear address isn't valid (can happen when loading PDPTEs + * as part of MOV CR execution) the linear address field is undefined. + * While we can leave it this way, it's preferrable to zero it for consistency. + */ + Assert(fIsLinearAddrValid || GCPtrAddr == 0); + + uint64_t const fCaps = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64EptVpidCaps; + bool const fSupportsAccessDirty = RT_BOOL(fCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY); + + uint32_t const fDataRdMask = IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_READ; + uint32_t const fDataWrMask = IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE; + uint32_t const fInstrMask = IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_EXEC; + bool const fDataRead = ((fAccess & fDataRdMask) == IEM_ACCESS_DATA_R) | fSupportsAccessDirty; + bool const fDataWrite = ((fAccess & fDataWrMask) == IEM_ACCESS_DATA_W) | fSupportsAccessDirty; + bool const fInstrFetch = ((fAccess & fInstrMask) == IEM_ACCESS_INSTRUCTION); + bool const fEptRead = RT_BOOL(fEptAccess & EPT_E_READ); + bool const fEptWrite = RT_BOOL(fEptAccess & EPT_E_WRITE); + bool const fEptExec = RT_BOOL(fEptAccess & EPT_E_EXECUTE); + bool const fNmiUnblocking = pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret; + bool const fIsLinearToPhysAddr = fIsLinearAddrValid & RT_BOOL(fSlatFail & IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR); + + uint64_t const u64ExitQual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ACCESS_READ, fDataRead) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ACCESS_WRITE, fDataWrite) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH, fInstrFetch) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ENTRY_READ, fEptRead) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ENTRY_WRITE, fEptWrite) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ENTRY_EXECUTE, fEptExec) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_LINEAR_ADDR_VALID, fIsLinearAddrValid) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_LINEAR_TO_PHYS_ADDR, fIsLinearToPhysAddr) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_NMI_UNBLOCK_IRET, fNmiUnblocking); + +#ifdef VBOX_STRICT + uint64_t const fMiscCaps = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Misc; + uint32_t const fProcCtls2 = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2; + Assert(!(fCaps & MSR_IA32_VMX_EPT_VPID_CAP_ADVEXITINFO_EPT_VIOLATION)); /* Advanced VM-exit info. not supported */ + Assert(!(fCaps & MSR_IA32_VMX_EPT_VPID_CAP_SUPER_SHW_STACK)); /* Supervisor shadow stack control not supported. */ + Assert(!(RT_BF_GET(fMiscCaps, VMX_BF_MISC_INTEL_PT))); /* Intel PT not supported. */ + Assert(!(fProcCtls2 & VMX_PROC_CTLS2_MODE_BASED_EPT_PERM)); /* Mode-based execute control not supported. */ +#endif + + iemVmxVmcsSetExitGuestPhysAddr(pVCpu, GCPhysAddr); + iemVmxVmcsSetExitGuestLinearAddr(pVCpu, GCPtrAddr); + iemVmxVmcsSetExitInstrLen(pVCpu, cbInstr); + + return iemVmxVmexit(pVCpu, VMX_EXIT_EPT_VIOLATION, u64ExitQual); +} + + +/** + * VMX VM-exit handler for EPT violation. + * + * This is intended for EPT violations where the caller provides all the + * relevant VM-exit information. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pExitInfo Pointer to the VM-exit information. + * @param pExitEventInfo Pointer to the VM-exit event information. + */ +static VBOXSTRICTRC iemVmxVmexitEptViolationWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, + PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT +{ + Assert(pExitInfo); + Assert(pExitEventInfo); + Assert(pExitInfo->uReason == VMX_EXIT_EPT_VIOLATION); + Assert(!VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo)); + + iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo); + iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode); + + iemVmxVmcsSetExitGuestPhysAddr(pVCpu, pExitInfo->u64GuestPhysAddr); + if (pExitInfo->u64Qual & VMX_BF_EXIT_QUAL_EPT_LINEAR_ADDR_VALID_MASK) + iemVmxVmcsSetExitGuestLinearAddr(pVCpu, pExitInfo->u64GuestLinearAddr); + else + iemVmxVmcsSetExitGuestLinearAddr(pVCpu, 0); + iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr); + return iemVmxVmexit(pVCpu, VMX_EXIT_EPT_VIOLATION, pExitInfo->u64Qual); +} + + +/** + * Interface for HM and EM to emulate a VM-exit due to an EPT violation. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @param pExitEventInfo Pointer to the VM-exit event information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitEptViolation(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, + PCVMXVEXITEVENTINFO pExitEventInfo) +{ + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = iemVmxVmexitEptViolationWithInfo(pVCpu, pExitInfo, pExitEventInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * VMX VM-exit handler for EPT-induced VM-exits. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pWalk The page walk info. + * @param fAccess The access causing the EPT event, IEM_ACCESS_XXX. + * @param fSlatFail Additional SLAT info, IEM_SLAT_FAIL_XXX. + * @param cbInstr The VM-exit instruction length if applicable. Pass 0 if not + * applicable. + */ +VBOXSTRICTRC iemVmxVmexitEpt(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint32_t fAccess, uint32_t fSlatFail, uint8_t cbInstr) RT_NOEXCEPT +{ + Assert(pWalk->fIsSlat); + Assert(pWalk->fFailed & PGM_WALKFAIL_EPT); + Assert(!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxEptXcptVe); /* #VE exceptions not supported. */ + Assert(!(pWalk->fFailed & PGM_WALKFAIL_EPT_VIOLATION_CONVERTIBLE)); /* Without #VE, convertible violations not possible. */ + + if (pWalk->fFailed & PGM_WALKFAIL_EPT_VIOLATION) + { + LogFlow(("EptViolation: cs:rip=%04x:%08RX64 fAccess=%#RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fAccess)); + uint64_t const fEptAccess = (pWalk->fEffective & PGM_PTATTRS_EPT_MASK) >> PGM_PTATTRS_EPT_SHIFT; + return iemVmxVmexitEptViolation(pVCpu, fAccess, fSlatFail, fEptAccess, pWalk->GCPhysNested, pWalk->fIsLinearAddrValid, + pWalk->GCPtr, cbInstr); + } + + LogFlow(("EptMisconfig: cs:rip=%04x:%08RX64 fAccess=%#RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fAccess)); + Assert(pWalk->fFailed & PGM_WALKFAIL_EPT_MISCONFIG); + return iemVmxVmexitEptMisconfig(pVCpu, pWalk->GCPhysNested); +} + + +/** + * VMX VM-exit handler for APIC accesses. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offAccess The offset of the register being accessed. + * @param fAccess The type of access, see IEM_ACCESS_XXX. + */ +static VBOXSTRICTRC iemVmxVmexitApicAccess(PVMCPUCC pVCpu, uint16_t offAccess, uint32_t fAccess) RT_NOEXCEPT +{ + VMXAPICACCESS enmAccess; + bool const fInEventDelivery = IEMGetCurrentXcpt(pVCpu, NULL, NULL, NULL, NULL); + if (fInEventDelivery) + enmAccess = VMXAPICACCESS_LINEAR_EVENT_DELIVERY; + else if ((fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == IEM_ACCESS_INSTRUCTION) + enmAccess = VMXAPICACCESS_LINEAR_INSTR_FETCH; + else if (fAccess & IEM_ACCESS_TYPE_WRITE) + enmAccess = VMXAPICACCESS_LINEAR_WRITE; + else + enmAccess = VMXAPICACCESS_LINEAR_READ; + + uint64_t const u64ExitQual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_OFFSET, offAccess) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_TYPE, enmAccess); + return iemVmxVmexit(pVCpu, VMX_EXIT_APIC_ACCESS, u64ExitQual); +} + + +/** + * VMX VM-exit handler for APIC accesses. + * + * This is intended for APIC accesses where the caller provides all the + * relevant VM-exit information. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pExitInfo Pointer to the VM-exit information. + * @param pExitEventInfo Pointer to the VM-exit event information. + */ +static VBOXSTRICTRC iemVmxVmexitApicAccessWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, + PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT +{ + /* VM-exit interruption information should not be valid for APIC-access VM-exits. */ + Assert(!VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo)); + Assert(pExitInfo->uReason == VMX_EXIT_APIC_ACCESS); + iemVmxVmcsSetExitIntInfo(pVCpu, 0); + iemVmxVmcsSetExitIntErrCode(pVCpu, 0); + iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr); + iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo); + iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode); + return iemVmxVmexit(pVCpu, VMX_EXIT_APIC_ACCESS, pExitInfo->u64Qual); +} + + +/** + * Interface for HM and EM to virtualize memory-mapped APIC accesses. + * + * @returns Strict VBox status code. + * @retval VINF_VMX_MODIFIES_BEHAVIOR if the memory access was virtualized. + * @retval VINF_VMX_VMEXIT if the access causes a VM-exit. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @param pExitEventInfo Pointer to the VM-exit event information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicAccess(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo) +{ + Assert(pExitInfo); + Assert(pExitEventInfo); + VBOXSTRICTRC rcStrict = iemVmxVmexitApicAccessWithInfo(pVCpu, pExitInfo, pExitEventInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * VMX VM-exit handler for APIC-write VM-exits. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offApic The write to the virtual-APIC page offset that caused this + * VM-exit. + */ +static VBOXSTRICTRC iemVmxVmexitApicWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT +{ + Assert(offApic < XAPIC_OFF_END + 4); + /* Write only bits 11:0 of the APIC offset into the Exit qualification field. */ + offApic &= UINT16_C(0xfff); + return iemVmxVmexit(pVCpu, VMX_EXIT_APIC_WRITE, offApic); +} + + +/** + * Clears any pending virtual-APIC write emulation. + * + * @returns The virtual-APIC offset that was written before clearing it. + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(uint16_t) iemVmxVirtApicClearPendingWrite(PVMCPUCC pVCpu) +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT); + uint8_t const offVirtApicWrite = pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite; + pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = 0; + Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE)); + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_VMX_APIC_WRITE); + return offVirtApicWrite; +} + + +/** + * Reads a 32-bit register from the virtual-APIC page at the given offset. + * + * @returns The register from the virtual-APIC page. + * @param pVCpu The cross context virtual CPU structure. + * @param offReg The offset of the register being read. + */ +uint32_t iemVmxVirtApicReadRaw32(PVMCPUCC pVCpu, uint16_t offReg) RT_NOEXCEPT +{ + Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint32_t)); + + uint32_t uReg = 0; + RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg, sizeof(uReg)); + AssertMsgStmt(RT_SUCCESS(rc), + ("Failed to read %u bytes at offset %#x of the virtual-APIC page at %#RGp: %Rrc\n", + sizeof(uReg), offReg, GCPhysVirtApic, rc), + uReg = 0); + return uReg; +} + + +/** + * Reads a 64-bit register from the virtual-APIC page at the given offset. + * + * @returns The register from the virtual-APIC page. + * @param pVCpu The cross context virtual CPU structure. + * @param offReg The offset of the register being read. + */ +static uint64_t iemVmxVirtApicReadRaw64(PVMCPUCC pVCpu, uint16_t offReg) RT_NOEXCEPT +{ + Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint64_t)); + + uint64_t uReg = 0; + RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg, sizeof(uReg)); + AssertMsgStmt(RT_SUCCESS(rc), + ("Failed to read %u bytes at offset %#x of the virtual-APIC page at %#RGp: %Rrc\n", + sizeof(uReg), offReg, GCPhysVirtApic, rc), + uReg = 0); + return uReg; +} + + +/** + * Writes a 32-bit register to the virtual-APIC page at the given offset. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offReg The offset of the register being written. + * @param uReg The register value to write. + */ +void iemVmxVirtApicWriteRaw32(PVMCPUCC pVCpu, uint16_t offReg, uint32_t uReg) RT_NOEXCEPT +{ + Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint32_t)); + + RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u; + int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg, &uReg, sizeof(uReg)); + AssertMsgRC(rc, ("Failed to write %u bytes at offset %#x of the virtual-APIC page at %#RGp: %Rrc\n", + sizeof(uReg), offReg, GCPhysVirtApic, rc)); +} + + +/** + * Writes a 64-bit register to the virtual-APIC page at the given offset. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offReg The offset of the register being written. + * @param uReg The register value to write. + */ +static void iemVmxVirtApicWriteRaw64(PVMCPUCC pVCpu, uint16_t offReg, uint64_t uReg) RT_NOEXCEPT +{ + Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint64_t)); + + RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u; + int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg, &uReg, sizeof(uReg)); + AssertMsgRC(rc, ("Failed to write %u bytes at offset %#x of the virtual-APIC page at %#RGp: %Rrc\n", + sizeof(uReg), offReg, GCPhysVirtApic, rc)); +} + + +/** + * Sets the vector in a virtual-APIC 256-bit sparse register. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offReg The offset of the 256-bit spare register. + * @param uVector The vector to set. + * + * @remarks This is based on our APIC device code. + */ +static void iemVmxVirtApicSetVectorInReg(PVMCPUCC pVCpu, uint16_t offReg, uint8_t uVector) RT_NOEXCEPT +{ + /* Determine the vector offset within the chunk. */ + uint16_t const offVector = (uVector & UINT32_C(0xe0)) >> 1; + + /* Read the chunk at the offset. */ + uint32_t uReg; + RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg + offVector, sizeof(uReg)); + if (RT_SUCCESS(rc)) + { + /* Modify the chunk. */ + uint16_t const idxVectorBit = uVector & UINT32_C(0x1f); + uReg |= RT_BIT(idxVectorBit); + + /* Write the chunk. */ + rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg + offVector, &uReg, sizeof(uReg)); + AssertMsgRC(rc, ("Failed to set vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp: %Rrc\n", + uVector, offReg, GCPhysVirtApic, rc)); + } + else + AssertMsgFailed(("Failed to get vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp: %Rrc\n", + uVector, offReg, GCPhysVirtApic, rc)); +} + + +/** + * Clears the vector in a virtual-APIC 256-bit sparse register. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offReg The offset of the 256-bit spare register. + * @param uVector The vector to clear. + * + * @remarks This is based on our APIC device code. + */ +static void iemVmxVirtApicClearVectorInReg(PVMCPUCC pVCpu, uint16_t offReg, uint8_t uVector) RT_NOEXCEPT +{ + /* Determine the vector offset within the chunk. */ + uint16_t const offVector = (uVector & UINT32_C(0xe0)) >> 1; + + /* Read the chunk at the offset. */ + uint32_t uReg; + RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg + offVector, sizeof(uReg)); + if (RT_SUCCESS(rc)) + { + /* Modify the chunk. */ + uint16_t const idxVectorBit = uVector & UINT32_C(0x1f); + uReg &= ~RT_BIT(idxVectorBit); + + /* Write the chunk. */ + rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg + offVector, &uReg, sizeof(uReg)); + AssertMsgRC(rc, ("Failed to clear vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp: %Rrc\n", + uVector, offReg, GCPhysVirtApic, rc)); + } + else + AssertMsgFailed(("Failed to get vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp: %Rrc\n", + uVector, offReg, GCPhysVirtApic, rc)); +} + + +/** + * Checks if a memory access to the APIC-access page must causes an APIC-access + * VM-exit. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offAccess The offset of the register being accessed. + * @param cbAccess The size of the access in bytes. + * @param fAccess The type of access, see IEM_ACCESS_XXX. + * + * @remarks This must not be used for MSR-based APIC-access page accesses! + * @sa iemVmxVirtApicAccessMsrWrite, iemVmxVirtApicAccessMsrRead. + */ +static bool iemVmxVirtApicIsMemAccessIntercepted(PVMCPUCC pVCpu, uint16_t offAccess, size_t cbAccess, uint32_t fAccess) RT_NOEXCEPT +{ + Assert(cbAccess > 0); + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* + * We must cause a VM-exit if any of the following are true: + * - TPR shadowing isn't active. + * - The access size exceeds 32-bits. + * - The access is not contained within low 4 bytes of a 16-byte aligned offset. + * + * See Intel spec. 29.4.2 "Virtualizing Reads from the APIC-Access Page". + * See Intel spec. 29.4.3.1 "Determining Whether a Write Access is Virtualized". + */ + if ( !(pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW) + || cbAccess > sizeof(uint32_t) + || ((offAccess + cbAccess - 1) & 0xc) + || offAccess >= XAPIC_OFF_END + 4) + return true; + + /* + * If the access is part of an operation where we have already + * virtualized a virtual-APIC write, we must cause a VM-exit. + */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE)) + return true; + + /* + * Check write accesses to the APIC-access page that cause VM-exits. + */ + if (fAccess & IEM_ACCESS_TYPE_WRITE) + { + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT) + { + /* + * With APIC-register virtualization, a write access to any of the + * following registers are virtualized. Accessing any other register + * causes a VM-exit. + */ + uint16_t const offAlignedAccess = offAccess & 0xfffc; + switch (offAlignedAccess) + { + case XAPIC_OFF_ID: + case XAPIC_OFF_TPR: + case XAPIC_OFF_EOI: + case XAPIC_OFF_LDR: + case XAPIC_OFF_DFR: + case XAPIC_OFF_SVR: + case XAPIC_OFF_ESR: + case XAPIC_OFF_ICR_LO: + case XAPIC_OFF_ICR_HI: + case XAPIC_OFF_LVT_TIMER: + case XAPIC_OFF_LVT_THERMAL: + case XAPIC_OFF_LVT_PERF: + case XAPIC_OFF_LVT_LINT0: + case XAPIC_OFF_LVT_LINT1: + case XAPIC_OFF_LVT_ERROR: + case XAPIC_OFF_TIMER_ICR: + case XAPIC_OFF_TIMER_DCR: + break; + default: + return true; + } + } + else if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY) + { + /* + * With virtual-interrupt delivery, a write access to any of the + * following registers are virtualized. Accessing any other register + * causes a VM-exit. + * + * Note! The specification does not allow writing to offsets in-between + * these registers (e.g. TPR + 1 byte) unlike read accesses. + */ + switch (offAccess) + { + case XAPIC_OFF_TPR: + case XAPIC_OFF_EOI: + case XAPIC_OFF_ICR_LO: + break; + default: + return true; + } + } + else + { + /* + * Without APIC-register virtualization or virtual-interrupt delivery, + * only TPR accesses are virtualized. + */ + if (offAccess == XAPIC_OFF_TPR) + { /* likely */ } + else + return true; + } + } + else + { + /* + * Check read accesses to the APIC-access page that cause VM-exits. + */ + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT) + { + /* + * With APIC-register virtualization, a read access to any of the + * following registers are virtualized. Accessing any other register + * causes a VM-exit. + */ + uint16_t const offAlignedAccess = offAccess & 0xfffc; + switch (offAlignedAccess) + { + /** @todo r=ramshankar: What about XAPIC_OFF_LVT_CMCI? */ + case XAPIC_OFF_ID: + case XAPIC_OFF_VERSION: + case XAPIC_OFF_TPR: + case XAPIC_OFF_EOI: + case XAPIC_OFF_LDR: + case XAPIC_OFF_DFR: + case XAPIC_OFF_SVR: + case XAPIC_OFF_ISR0: case XAPIC_OFF_ISR1: case XAPIC_OFF_ISR2: case XAPIC_OFF_ISR3: + case XAPIC_OFF_ISR4: case XAPIC_OFF_ISR5: case XAPIC_OFF_ISR6: case XAPIC_OFF_ISR7: + case XAPIC_OFF_TMR0: case XAPIC_OFF_TMR1: case XAPIC_OFF_TMR2: case XAPIC_OFF_TMR3: + case XAPIC_OFF_TMR4: case XAPIC_OFF_TMR5: case XAPIC_OFF_TMR6: case XAPIC_OFF_TMR7: + case XAPIC_OFF_IRR0: case XAPIC_OFF_IRR1: case XAPIC_OFF_IRR2: case XAPIC_OFF_IRR3: + case XAPIC_OFF_IRR4: case XAPIC_OFF_IRR5: case XAPIC_OFF_IRR6: case XAPIC_OFF_IRR7: + case XAPIC_OFF_ESR: + case XAPIC_OFF_ICR_LO: + case XAPIC_OFF_ICR_HI: + case XAPIC_OFF_LVT_TIMER: + case XAPIC_OFF_LVT_THERMAL: + case XAPIC_OFF_LVT_PERF: + case XAPIC_OFF_LVT_LINT0: + case XAPIC_OFF_LVT_LINT1: + case XAPIC_OFF_LVT_ERROR: + case XAPIC_OFF_TIMER_ICR: + case XAPIC_OFF_TIMER_DCR: + break; + default: + return true; + } + } + else + { + /* Without APIC-register virtualization, only TPR accesses are virtualized. */ + if (offAccess == XAPIC_OFF_TPR) + { /* likely */ } + else + return true; + } + } + + /* The APIC access is virtualized, does not cause a VM-exit. */ + return false; +} + + +/** + * Virtualizes a memory-based APIC access by certain instructions even though they + * do not use the address to access memory. + * + * This is for instructions like MONITOR, CLFLUSH, CLFLUSHOPT, ENTER which may cause + * page-faults but do not use the address to access memory. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pGCPhysAccess Pointer to the guest-physical address accessed. + * @param cbAccess The size of the access in bytes. + * @param fAccess The type of access, see IEM_ACCESS_XXX. + */ +VBOXSTRICTRC iemVmxVirtApicAccessUnused(PVMCPUCC pVCpu, PRTGCPHYS pGCPhysAccess, size_t cbAccess, uint32_t fAccess) RT_NOEXCEPT +{ + Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS); + Assert(pGCPhysAccess); + + RTGCPHYS const GCPhysAccess = *pGCPhysAccess & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + RTGCPHYS const GCPhysApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrApicAccess.u; + Assert(!(GCPhysApic & GUEST_PAGE_OFFSET_MASK)); + + if (GCPhysAccess == GCPhysApic) + { + uint16_t const offAccess = *pGCPhysAccess & GUEST_PAGE_OFFSET_MASK; + bool const fIntercept = iemVmxVirtApicIsMemAccessIntercepted(pVCpu, offAccess, cbAccess, fAccess); + if (fIntercept) + return iemVmxVmexitApicAccess(pVCpu, offAccess, fAccess); + + *pGCPhysAccess = GCPhysApic | offAccess; + return VINF_VMX_MODIFIES_BEHAVIOR; + } + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * Virtualizes a memory-based APIC access. + * + * @returns VBox strict status code. + * @retval VINF_VMX_MODIFIES_BEHAVIOR if the access was virtualized. + * @retval VINF_VMX_VMEXIT if the access causes a VM-exit. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offAccess The offset of the register being accessed (within the + * APIC-access page). + * @param cbAccess The size of the access in bytes. + * @param pvData Pointer to the data being written or where to store the data + * being read. + * @param fAccess The type of access, see IEM_ACCESS_XXX. + */ +static VBOXSTRICTRC iemVmxVirtApicAccessMem(PVMCPUCC pVCpu, uint16_t offAccess, size_t cbAccess, + void *pvData, uint32_t fAccess) RT_NOEXCEPT +{ + Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS); + Assert(pvData); + + bool const fIntercept = iemVmxVirtApicIsMemAccessIntercepted(pVCpu, offAccess, cbAccess, fAccess); + if (fIntercept) + return iemVmxVmexitApicAccess(pVCpu, offAccess, fAccess); + + if (fAccess & IEM_ACCESS_TYPE_WRITE) + { + /* + * A write access to the APIC-access page that is virtualized (rather than + * causing a VM-exit) writes data to the virtual-APIC page. + */ + uint32_t const u32Data = *(uint32_t *)pvData; + iemVmxVirtApicWriteRaw32(pVCpu, offAccess, u32Data); + + /* + * Record the currently updated APIC offset, as we need this later for figuring + * out whether to perform TPR, EOI or self-IPI virtualization as well as well + * as for supplying the exit qualification when causing an APIC-write VM-exit. + * + * After completion of the current operation, we need to perform TPR virtualization, + * EOI virtualization or APIC-write VM-exit depending on which register was written. + * + * The current operation may be a REP-prefixed string instruction, execution of any + * other instruction, or delivery of an event through the IDT. + * + * Thus things like clearing bytes 3:1 of the VTPR, clearing VEOI are not to be + * performed now but later after completion of the current operation. + * + * See Intel spec. 29.4.3.2 "APIC-Write Emulation". + */ + iemVmxVirtApicSetPendingWrite(pVCpu, offAccess); + + LogFlowFunc(("Write access at offset %#x not intercepted -> Wrote %#RX32\n", offAccess, u32Data)); + } + else + { + /* + * A read access from the APIC-access page that is virtualized (rather than + * causing a VM-exit) returns data from the virtual-APIC page. + * + * See Intel spec. 29.4.2 "Virtualizing Reads from the APIC-Access Page". + */ + Assert(fAccess & IEM_ACCESS_TYPE_READ); + + Assert(cbAccess <= 4); + Assert(offAccess < XAPIC_OFF_END + 4); + static uint32_t const s_auAccessSizeMasks[] = { 0, 0xff, 0xffff, 0xffffff, 0xffffffff }; + + uint32_t u32Data = iemVmxVirtApicReadRaw32(pVCpu, offAccess); + u32Data &= s_auAccessSizeMasks[cbAccess]; + *(uint32_t *)pvData = u32Data; + + LogFlowFunc(("Read access at offset %#x not intercepted -> Read %#RX32\n", offAccess, u32Data)); + } + + return VINF_VMX_MODIFIES_BEHAVIOR; +} + + +/** + * Virtualizes an MSR-based APIC read access. + * + * @returns VBox strict status code. + * @retval VINF_VMX_MODIFIES_BEHAVIOR if the MSR read was virtualized. + * @retval VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR read access must be + * handled by the x2APIC device. + * @retval VERR_OUT_RANGE if the MSR read was supposed to be virtualized but was + * not within the range of valid MSRs, caller must raise \#GP(0). + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The x2APIC MSR being read. + * @param pu64Value Where to store the read x2APIC MSR value (only valid when + * VINF_VMX_MODIFIES_BEHAVIOR is returned). + */ +static VBOXSTRICTRC iemVmxVirtApicAccessMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value) RT_NOEXCEPT +{ + Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE); + Assert(pu64Value); + + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT) + { + if ( idMsr >= MSR_IA32_X2APIC_START + && idMsr <= MSR_IA32_X2APIC_END) + { + uint16_t const offReg = (idMsr & 0xff) << 4; + uint64_t const u64Value = iemVmxVirtApicReadRaw64(pVCpu, offReg); + *pu64Value = u64Value; + return VINF_VMX_MODIFIES_BEHAVIOR; + } + return VERR_OUT_OF_RANGE; + } + + if (idMsr == MSR_IA32_X2APIC_TPR) + { + uint16_t const offReg = (idMsr & 0xff) << 4; + uint64_t const u64Value = iemVmxVirtApicReadRaw64(pVCpu, offReg); + *pu64Value = u64Value; + return VINF_VMX_MODIFIES_BEHAVIOR; + } + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * Virtualizes an MSR-based APIC write access. + * + * @returns VBox strict status code. + * @retval VINF_VMX_MODIFIES_BEHAVIOR if the MSR write was virtualized. + * @retval VERR_OUT_RANGE if the MSR read was supposed to be virtualized but was + * not within the range of valid MSRs, caller must raise \#GP(0). + * @retval VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR must be written normally. + * + * @param pVCpu The cross context virtual CPU structure. + * @param idMsr The x2APIC MSR being written. + * @param u64Value The value of the x2APIC MSR being written. + */ +static VBOXSTRICTRC iemVmxVirtApicAccessMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Value) RT_NOEXCEPT +{ + /* + * Check if the access is to be virtualized. + * See Intel spec. 29.5 "Virtualizing MSR-based APIC Accesses". + */ + if ( idMsr == MSR_IA32_X2APIC_TPR + || ( (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY) + && ( idMsr == MSR_IA32_X2APIC_EOI + || idMsr == MSR_IA32_X2APIC_SELF_IPI))) + { + /* Validate the MSR write depending on the register. */ + switch (idMsr) + { + case MSR_IA32_X2APIC_TPR: + case MSR_IA32_X2APIC_SELF_IPI: + { + if (u64Value & UINT64_C(0xffffffffffffff00)) + return VERR_OUT_OF_RANGE; + break; + } + case MSR_IA32_X2APIC_EOI: + { + if (u64Value != 0) + return VERR_OUT_OF_RANGE; + break; + } + } + + /* Write the MSR to the virtual-APIC page. */ + uint16_t const offReg = (idMsr & 0xff) << 4; + iemVmxVirtApicWriteRaw64(pVCpu, offReg, u64Value); + + /* + * Record the currently updated APIC offset, as we need this later for figuring + * out whether to perform TPR, EOI or self-IPI virtualization as well as well + * as for supplying the exit qualification when causing an APIC-write VM-exit. + */ + iemVmxVirtApicSetPendingWrite(pVCpu, offReg); + + return VINF_VMX_MODIFIES_BEHAVIOR; + } + + return VINF_VMX_INTERCEPT_NOT_ACTIVE; +} + + +/** + * Interface for HM and EM to virtualize x2APIC MSR accesses. + * + * @returns Strict VBox status code. + * @retval VINF_VMX_MODIFIES_BEHAVIOR if the MSR access was virtualized. + * @retval VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR access must be handled by + * the x2APIC device. + * @retval VERR_OUT_RANGE if the caller must raise \#GP(0). + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param idMsr The MSR being read. + * @param pu64Value Pointer to the value being written or where to store the + * value being read. + * @param fWrite Whether this is an MSR write or read access. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVirtApicAccessMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value, bool fWrite) +{ + Assert(pu64Value); + + VBOXSTRICTRC rcStrict; + if (fWrite) + rcStrict = iemVmxVirtApicAccessMsrWrite(pVCpu, idMsr, *pu64Value); + else + rcStrict = iemVmxVirtApicAccessMsrRead(pVCpu, idMsr, pu64Value); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); + +} + + +/** + * Finds the most significant set bit in a virtual-APIC 256-bit sparse register. + * + * @returns VBox status code. + * @retval VINF_SUCCESS when the highest set bit is found. + * @retval VERR_NOT_FOUND when no bit is set. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offReg The offset of the APIC 256-bit sparse register. + * @param pidxHighestBit Where to store the highest bit (most significant bit) + * set in the register. Only valid when VINF_SUCCESS is + * returned. + * + * @remarks The format of the 256-bit sparse register here mirrors that found in + * real APIC hardware. + */ +static int iemVmxVirtApicGetHighestSetBitInReg(PVMCPUCC pVCpu, uint16_t offReg, uint8_t *pidxHighestBit) +{ + Assert(offReg < XAPIC_OFF_END + 4); + Assert(pidxHighestBit); + + /* + * There are 8 contiguous fragments (of 16-bytes each) in the sparse register. + * However, in each fragment only the first 4 bytes are used. + */ + uint8_t const cFrags = 8; + for (int8_t iFrag = cFrags; iFrag >= 0; iFrag--) + { + uint16_t const offFrag = iFrag * 16; + uint32_t const u32Frag = iemVmxVirtApicReadRaw32(pVCpu, offReg + offFrag); + if (!u32Frag) + continue; + + unsigned idxHighestBit = ASMBitLastSetU32(u32Frag); + Assert(idxHighestBit > 0); + --idxHighestBit; + Assert(idxHighestBit <= UINT8_MAX); + *pidxHighestBit = idxHighestBit; + return VINF_SUCCESS; + } + return VERR_NOT_FOUND; +} + + +/** + * Evaluates pending virtual interrupts. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxEvalPendingVirtIntrs(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY); + + if (!(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)) + { + uint8_t const uRvi = RT_LO_U8(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u16GuestIntStatus); + uint8_t const uPpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_PPR); + + if ((uRvi >> 4) > (uPpr >> 4)) + { + Log2(("eval_virt_intrs: uRvi=%#x uPpr=%#x - Signalling pending interrupt\n", uRvi, uPpr)); + VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST); + } + else + Log2(("eval_virt_intrs: uRvi=%#x uPpr=%#x - Nothing to do\n", uRvi, uPpr)); + } +} + + +/** + * Performs PPR virtualization. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxPprVirtualization(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); + Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY); + + /* + * PPR virtualization is caused in response to a VM-entry, TPR-virtualization, + * or EOI-virtualization. + * + * See Intel spec. 29.1.3 "PPR Virtualization". + */ + uint8_t const uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR); + uint8_t const uSvi = RT_HI_U8(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u16GuestIntStatus) & 0xf0; + + uint32_t uPpr; + if ((uTpr & 0xf0) >= uSvi) + uPpr = uTpr; + else + uPpr = uSvi; + + Log2(("ppr_virt: uTpr=%#x uSvi=%#x uPpr=%#x\n", uTpr, uSvi, uPpr)); + iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_PPR, uPpr); +} + + +/** + * Performs VMX TPR virtualization. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + */ +static VBOXSTRICTRC iemVmxTprVirtualization(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); + + /* + * We should have already performed the virtual-APIC write to the TPR offset + * in the virtual-APIC page. We now perform TPR virtualization. + * + * See Intel spec. 29.1.2 "TPR Virtualization". + */ + if (!(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)) + { + uint32_t const uTprThreshold = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32TprThreshold; + uint32_t const uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR); + + /* + * If the VTPR falls below the TPR threshold, we must cause a VM-exit. + * See Intel spec. 29.1.2 "TPR Virtualization". + */ + if (((uTpr >> 4) & 0xf) < uTprThreshold) + { + Log2(("tpr_virt: uTpr=%u uTprThreshold=%u -> VM-exit\n", uTpr, uTprThreshold)); + return iemVmxVmexit(pVCpu, VMX_EXIT_TPR_BELOW_THRESHOLD, 0 /* u64ExitQual */); + } + } + else + { + iemVmxPprVirtualization(pVCpu); + iemVmxEvalPendingVirtIntrs(pVCpu); + } + + return VINF_SUCCESS; +} + + +/** + * Checks whether an EOI write for the given interrupt vector causes a VM-exit or + * not. + * + * @returns @c true if the EOI write is intercepted, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param uVector The interrupt that was acknowledged using an EOI. + */ +static bool iemVmxIsEoiInterceptSet(PCVMCPU pVCpu, uint8_t uVector) RT_NOEXCEPT +{ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY); + + if (uVector < 64) + return RT_BOOL(pVmcs->u64EoiExitBitmap0.u & RT_BIT_64(uVector)); + if (uVector < 128) + return RT_BOOL(pVmcs->u64EoiExitBitmap1.u & RT_BIT_64(uVector)); + if (uVector < 192) + return RT_BOOL(pVmcs->u64EoiExitBitmap2.u & RT_BIT_64(uVector)); + return RT_BOOL(pVmcs->u64EoiExitBitmap3.u & RT_BIT_64(uVector)); +} + + +/** + * Performs EOI virtualization. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + */ +static VBOXSTRICTRC iemVmxEoiVirtualization(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY); + + /* + * Clear the interrupt guest-interrupt as no longer in-service (ISR) + * and get the next guest-interrupt that's in-service (if any). + * + * See Intel spec. 29.1.4 "EOI Virtualization". + */ + uint8_t const uRvi = RT_LO_U8(pVmcs->u16GuestIntStatus); + uint8_t const uSvi = RT_HI_U8(pVmcs->u16GuestIntStatus); + Log2(("eoi_virt: uRvi=%#x uSvi=%#x\n", uRvi, uSvi)); + + uint8_t uVector = uSvi; + iemVmxVirtApicClearVectorInReg(pVCpu, XAPIC_OFF_ISR0, uVector); + + uVector = 0; + iemVmxVirtApicGetHighestSetBitInReg(pVCpu, XAPIC_OFF_ISR0, &uVector); + + if (uVector) + Log2(("eoi_virt: next interrupt %#x\n", uVector)); + else + Log2(("eoi_virt: no interrupt pending in ISR\n")); + + /* Update guest-interrupt status SVI (leave RVI portion as it is) in the VMCS. */ + pVmcs->u16GuestIntStatus = RT_MAKE_U16(uRvi, uVector); + + iemVmxPprVirtualization(pVCpu); + if (iemVmxIsEoiInterceptSet(pVCpu, uVector)) + return iemVmxVmexit(pVCpu, VMX_EXIT_VIRTUALIZED_EOI, uVector); + iemVmxEvalPendingVirtIntrs(pVCpu); + return VINF_SUCCESS; +} + + +/** + * Performs self-IPI virtualization. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + */ +static VBOXSTRICTRC iemVmxSelfIpiVirtualization(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); + + /* + * We should have already performed the virtual-APIC write to the self-IPI offset + * in the virtual-APIC page. We now perform self-IPI virtualization. + * + * See Intel spec. 29.1.5 "Self-IPI Virtualization". + */ + uint8_t const uVector = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_ICR_LO); + Log2(("self_ipi_virt: uVector=%#x\n", uVector)); + iemVmxVirtApicSetVectorInReg(pVCpu, XAPIC_OFF_IRR0, uVector); + uint8_t const uRvi = RT_LO_U8(pVmcs->u16GuestIntStatus); + uint8_t const uSvi = RT_HI_U8(pVmcs->u16GuestIntStatus); + if (uVector > uRvi) + pVmcs->u16GuestIntStatus = RT_MAKE_U16(uVector, uSvi); + iemVmxEvalPendingVirtIntrs(pVCpu); + return VINF_SUCCESS; +} + + +/** + * Performs VMX APIC-write emulation. + * + * @returns VBox strict status code. + * @param pVCpu The cross context virtual CPU structure. + */ +VBOXSTRICTRC iemVmxApicWriteEmulation(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* Import the virtual-APIC write offset (part of the hardware-virtualization state). */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT); + + /* + * Perform APIC-write emulation based on the virtual-APIC register written. + * See Intel spec. 29.4.3.2 "APIC-Write Emulation". + */ + uint16_t const offApicWrite = iemVmxVirtApicClearPendingWrite(pVCpu); + VBOXSTRICTRC rcStrict; + switch (offApicWrite) + { + case XAPIC_OFF_TPR: + { + /* Clear bytes 3:1 of the VTPR and perform TPR virtualization. */ + uint32_t uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR); + uTpr &= UINT32_C(0x000000ff); + iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_TPR, uTpr); + Log2(("iemVmxApicWriteEmulation: TPR write %#x\n", uTpr)); + rcStrict = iemVmxTprVirtualization(pVCpu); + break; + } + + case XAPIC_OFF_EOI: + { + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY) + { + /* Clear VEOI and perform EOI virtualization. */ + iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_EOI, 0); + Log2(("iemVmxApicWriteEmulation: EOI write\n")); + rcStrict = iemVmxEoiVirtualization(pVCpu); + } + else + rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite); + break; + } + + case XAPIC_OFF_ICR_LO: + { + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY) + { + /* If the ICR_LO is valid, write it and perform self-IPI virtualization. */ + uint32_t const uIcrLo = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR); + uint32_t const fIcrLoMb0 = UINT32_C(0xfffbb700); + uint32_t const fIcrLoMb1 = UINT32_C(0x000000f0); + if ( !(uIcrLo & fIcrLoMb0) + && (uIcrLo & fIcrLoMb1)) + { + Log2(("iemVmxApicWriteEmulation: Self-IPI virtualization with vector %#x\n", (uIcrLo & 0xff))); + rcStrict = iemVmxSelfIpiVirtualization(pVCpu); + } + else + rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite); + } + else + rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite); + break; + } + + case XAPIC_OFF_ICR_HI: + { + /* Clear bytes 2:0 of VICR_HI. No other virtualization or VM-exit must occur. */ + uint32_t uIcrHi = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_ICR_HI); + uIcrHi &= UINT32_C(0xff000000); + iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_ICR_HI, uIcrHi); + rcStrict = VINF_SUCCESS; + break; + } + + default: + { + /* Writes to any other virtual-APIC register causes an APIC-write VM-exit. */ + rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite); + break; + } + } + + return rcStrict; +} + + +/** + * Interface for HM and EM to perform an APIC-write emulation which may cause a + * VM-exit. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicWrite(PVMCPUCC pVCpu) +{ + VBOXSTRICTRC rcStrict = iemVmxApicWriteEmulation(pVCpu); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * Checks guest control registers, debug registers and MSRs as part of VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +DECLINLINE(int) iemVmxVmentryCheckGuestControlRegsMsrs(PVMCPUCC pVCpu, const char *pszInstr) +{ + /* + * Guest Control Registers, Debug Registers, and MSRs. + * See Intel spec. 26.3.1.1 "Checks on Guest Control Registers, Debug Registers, and MSRs". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char * const pszFailure = "VM-exit"; + bool const fUnrestrictedGuest = RT_BOOL(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST); + + /* CR0 reserved bits. */ + { + /* CR0 MB1 bits. */ + uint64_t const u64Cr0Fixed0 = iemVmxGetCr0Fixed0(pVCpu, true /* fVmxNonRootMode */); + if ((pVmcs->u64GuestCr0.u & u64Cr0Fixed0) == u64Cr0Fixed0) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr0Fixed0); + + /* CR0 MBZ bits. */ + uint64_t const u64Cr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1; + if (!(pVmcs->u64GuestCr0.u & ~u64Cr0Fixed1)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr0Fixed1); + + /* Without unrestricted guest support, VT-x supports does not support unpaged protected mode. */ + if ( !fUnrestrictedGuest + && (pVmcs->u64GuestCr0.u & X86_CR0_PG) + && !(pVmcs->u64GuestCr0.u & X86_CR0_PE)) + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr0PgPe); + } + + /* CR4 reserved bits. */ + { + /* CR4 MB1 bits. */ + uint64_t const u64Cr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0; + if ((pVmcs->u64GuestCr4.u & u64Cr4Fixed0) == u64Cr4Fixed0) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr4Fixed0); + + /* CR4 MBZ bits. */ + uint64_t const u64Cr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1; + if (!(pVmcs->u64GuestCr4.u & ~u64Cr4Fixed1)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr4Fixed1); + } + + /* DEBUGCTL MSR. */ + if ( !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG) + || !(pVmcs->u64GuestDebugCtlMsr.u & ~MSR_IA32_DEBUGCTL_VALID_MASK_INTEL)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestDebugCtl); + + /* 64-bit CPU checks. */ + bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST); + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + if (fGstInLongMode) + { + /* PAE must be set. */ + if ( (pVmcs->u64GuestCr0.u & X86_CR0_PG) + && (pVmcs->u64GuestCr0.u & X86_CR4_PAE)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPae); + } + else + { + /* PCIDE should not be set. */ + if (!(pVmcs->u64GuestCr4.u & X86_CR4_PCIDE)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPcide); + } + + /* CR3. */ + if (!(pVmcs->u64GuestCr3.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr3); + + /* DR7. */ + if ( !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG) + || !(pVmcs->u64GuestDr7.u & X86_DR7_MBZ_MASK)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestDr7); + + /* SYSENTER ESP and SYSENTER EIP. */ + if ( X86_IS_CANONICAL(pVmcs->u64GuestSysenterEsp.u) + && X86_IS_CANONICAL(pVmcs->u64GuestSysenterEip.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSysenterEspEip); + } + + /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */ + Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)); + + /* PAT MSR. */ + if ( !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR) + || CPUMIsPatMsrValid(pVmcs->u64GuestPatMsr.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPatMsr); + + /* EFER MSR. */ + if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR) + { + uint64_t const uValidEferMask = CPUMGetGuestEferMsrValidMask(pVCpu->CTX_SUFF(pVM)); + if (!(pVmcs->u64GuestEferMsr.u & ~uValidEferMask)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestEferMsrRsvd); + + bool const fGstLma = RT_BOOL(pVmcs->u64GuestEferMsr.u & MSR_K6_EFER_LMA); + bool const fGstLme = RT_BOOL(pVmcs->u64GuestEferMsr.u & MSR_K6_EFER_LME); + if ( fGstLma == fGstInLongMode + && ( !(pVmcs->u64GuestCr0.u & X86_CR0_PG) + || fGstLma == fGstLme)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestEferMsr); + } + + /* We don't support IA32_BNDCFGS MSR yet. */ + Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_BNDCFGS_MSR)); + + NOREF(pszInstr); + NOREF(pszFailure); + return VINF_SUCCESS; +} + + +/** + * Checks guest segment registers, LDTR and TR as part of VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +DECLINLINE(int) iemVmxVmentryCheckGuestSegRegs(PVMCPUCC pVCpu, const char *pszInstr) +{ + /* + * Segment registers. + * See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char * const pszFailure = "VM-exit"; + bool const fGstInV86Mode = RT_BOOL(pVmcs->u64GuestRFlags.u & X86_EFL_VM); + bool const fUnrestrictedGuest = RT_BOOL(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST); + bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST); + + /* Selectors. */ + if ( !fGstInV86Mode + && !fUnrestrictedGuest + && (pVmcs->GuestSs & X86_SEL_RPL) != (pVmcs->GuestCs & X86_SEL_RPL)) + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegSelCsSsRpl); + + for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++) + { + CPUMSELREG SelReg; + int rc = iemVmxVmcsGetGuestSegReg(pVmcs, iSegReg, &SelReg); + if (RT_LIKELY(rc == VINF_SUCCESS)) + { /* likely */ } + else + return rc; + + /* + * Virtual-8086 mode checks. + */ + if (fGstInV86Mode) + { + /* Base address. */ + if (SelReg.u64Base == (uint64_t)SelReg.Sel << 4) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegBaseV86(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + + /* Limit. */ + if (SelReg.u32Limit == 0xffff) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegLimitV86(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + + /* Attribute. */ + if (SelReg.Attr.u == 0xf3) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrV86(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + + /* We're done; move to checking the next segment. */ + continue; + } + + /* Checks done by 64-bit CPUs. */ + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + /* Base address. */ + if ( iSegReg == X86_SREG_FS + || iSegReg == X86_SREG_GS) + { + if (X86_IS_CANONICAL(SelReg.u64Base)) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegBase(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + } + else if (iSegReg == X86_SREG_CS) + { + if (!RT_HI_U32(SelReg.u64Base)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegBaseCs); + } + else + { + if ( SelReg.Attr.n.u1Unusable + || !RT_HI_U32(SelReg.u64Base)) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegBase(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + } + } + + /* + * Checks outside Virtual-8086 mode. + */ + uint8_t const uSegType = SelReg.Attr.n.u4Type; + uint8_t const fCodeDataSeg = SelReg.Attr.n.u1DescType; + uint8_t const fUsable = !SelReg.Attr.n.u1Unusable; + uint8_t const uDpl = SelReg.Attr.n.u2Dpl; + uint8_t const fPresent = SelReg.Attr.n.u1Present; + uint8_t const uGranularity = SelReg.Attr.n.u1Granularity; + uint8_t const uDefBig = SelReg.Attr.n.u1DefBig; + uint8_t const fSegLong = SelReg.Attr.n.u1Long; + + /* Code or usable segment. */ + if ( iSegReg == X86_SREG_CS + || fUsable) + { + /* Reserved bits (bits 31:17 and bits 11:8). */ + if (!(SelReg.Attr.u & 0xfffe0f00)) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrRsvd(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + + /* Descriptor type. */ + if (fCodeDataSeg) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrDescType(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + + /* Present. */ + if (fPresent) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrPresent(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + + /* Granularity. */ + if ( ((SelReg.u32Limit & 0x00000fff) == 0x00000fff || !uGranularity) + && ((SelReg.u32Limit & 0xfff00000) == 0x00000000 || uGranularity)) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrGran(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + } + + if (iSegReg == X86_SREG_CS) + { + /* Segment Type and DPL. */ + if ( uSegType == (X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED) + && fUnrestrictedGuest) + { + if (uDpl == 0) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDplZero); + } + else if ( uSegType == (X86_SEL_TYPE_CODE | X86_SEL_TYPE_ACCESSED) + || uSegType == (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_ACCESSED)) + { + X86DESCATTR AttrSs; AttrSs.u = pVmcs->u32GuestSsAttr; + if (uDpl == AttrSs.n.u2Dpl) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDplEqSs); + } + else if ((uSegType & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF | X86_SEL_TYPE_ACCESSED)) + == (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF | X86_SEL_TYPE_ACCESSED)) + { + X86DESCATTR AttrSs; AttrSs.u = pVmcs->u32GuestSsAttr; + if (uDpl <= AttrSs.n.u2Dpl) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDplLtSs); + } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsType); + + /* Def/Big. */ + if ( fGstInLongMode + && fSegLong) + { + if (uDefBig == 0) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDefBig); + } + } + else if (iSegReg == X86_SREG_SS) + { + /* Segment Type. */ + if ( !fUsable + || uSegType == (X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED) + || uSegType == (X86_SEL_TYPE_DOWN | X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrSsType); + + /* DPL. */ + if (!fUnrestrictedGuest) + { + if (uDpl == (SelReg.Sel & X86_SEL_RPL)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrSsDplEqRpl); + } + X86DESCATTR AttrCs; AttrCs.u = pVmcs->u32GuestCsAttr; + if ( AttrCs.n.u4Type == (X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED) + || !(pVmcs->u64GuestCr0.u & X86_CR0_PE)) + { + if (uDpl == 0) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrSsDplZero); + } + } + else + { + /* DS, ES, FS, GS. */ + if (fUsable) + { + /* Segment type. */ + if (uSegType & X86_SEL_TYPE_ACCESSED) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrTypeAcc(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + + if ( !(uSegType & X86_SEL_TYPE_CODE) + || (uSegType & X86_SEL_TYPE_READ)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsTypeRead); + + /* DPL. */ + if ( !fUnrestrictedGuest + && uSegType <= (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_ACCESSED)) + { + if (uDpl >= (SelReg.Sel & X86_SEL_RPL)) + { /* likely */ } + else + { + VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrDplRpl(iSegReg); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + } + } + } + } + + /* + * LDTR. + */ + { + CPUMSELREG Ldtr; + Ldtr.Sel = pVmcs->GuestLdtr; + Ldtr.u32Limit = pVmcs->u32GuestLdtrLimit; + Ldtr.u64Base = pVmcs->u64GuestLdtrBase.u; + Ldtr.Attr.u = pVmcs->u32GuestLdtrAttr; + + if (!Ldtr.Attr.n.u1Unusable) + { + /* Selector. */ + if (!(Ldtr.Sel & X86_SEL_LDT)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegSelLdtr); + + /* Base. */ + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + if (X86_IS_CANONICAL(Ldtr.u64Base)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegBaseLdtr); + } + + /* Attributes. */ + /* Reserved bits (bits 31:17 and bits 11:8). */ + if (!(Ldtr.Attr.u & 0xfffe0f00)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrRsvd); + + if (Ldtr.Attr.n.u4Type == X86_SEL_TYPE_SYS_LDT) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrType); + + if (!Ldtr.Attr.n.u1DescType) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrDescType); + + if (Ldtr.Attr.n.u1Present) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrPresent); + + if ( ((Ldtr.u32Limit & 0x00000fff) == 0x00000fff || !Ldtr.Attr.n.u1Granularity) + && ((Ldtr.u32Limit & 0xfff00000) == 0x00000000 || Ldtr.Attr.n.u1Granularity)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrGran); + } + } + + /* + * TR. + */ + { + CPUMSELREG Tr; + Tr.Sel = pVmcs->GuestTr; + Tr.u32Limit = pVmcs->u32GuestTrLimit; + Tr.u64Base = pVmcs->u64GuestTrBase.u; + Tr.Attr.u = pVmcs->u32GuestTrAttr; + + /* Selector. */ + if (!(Tr.Sel & X86_SEL_LDT)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegSelTr); + + /* Base. */ + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + if (X86_IS_CANONICAL(Tr.u64Base)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegBaseTr); + } + + /* Attributes. */ + /* Reserved bits (bits 31:17 and bits 11:8). */ + if (!(Tr.Attr.u & 0xfffe0f00)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrRsvd); + + if (!Tr.Attr.n.u1Unusable) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrUnusable); + + if ( Tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY + || ( !fGstInLongMode + && Tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrType); + + if (!Tr.Attr.n.u1DescType) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrDescType); + + if (Tr.Attr.n.u1Present) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrPresent); + + if ( ((Tr.u32Limit & 0x00000fff) == 0x00000fff || !Tr.Attr.n.u1Granularity) + && ((Tr.u32Limit & 0xfff00000) == 0x00000000 || Tr.Attr.n.u1Granularity)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrGran); + } + + NOREF(pszInstr); + NOREF(pszFailure); + return VINF_SUCCESS; +} + + +/** + * Checks guest GDTR and IDTR as part of VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +DECLINLINE(int) iemVmxVmentryCheckGuestGdtrIdtr(PVMCPUCC pVCpu, const char *pszInstr) +{ + /* + * GDTR and IDTR. + * See Intel spec. 26.3.1.3 "Checks on Guest Descriptor-Table Registers". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char *const pszFailure = "VM-exit"; + + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + /* Base. */ + if (X86_IS_CANONICAL(pVmcs->u64GuestGdtrBase.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestGdtrBase); + + if (X86_IS_CANONICAL(pVmcs->u64GuestIdtrBase.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIdtrBase); + } + + /* Limit. */ + if (!RT_HI_U16(pVmcs->u32GuestGdtrLimit)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestGdtrLimit); + + if (!RT_HI_U16(pVmcs->u32GuestIdtrLimit)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIdtrLimit); + + NOREF(pszInstr); + NOREF(pszFailure); + return VINF_SUCCESS; +} + + +/** + * Checks guest RIP and RFLAGS as part of VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +DECLINLINE(int) iemVmxVmentryCheckGuestRipRFlags(PVMCPUCC pVCpu, const char *pszInstr) +{ + /* + * RIP and RFLAGS. + * See Intel spec. 26.3.1.4 "Checks on Guest RIP and RFLAGS". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char *const pszFailure = "VM-exit"; + bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST); + + /* RIP. */ + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + X86DESCATTR AttrCs; + AttrCs.u = pVmcs->u32GuestCsAttr; + if ( !fGstInLongMode + || !AttrCs.n.u1Long) + { + if (!RT_HI_U32(pVmcs->u64GuestRip.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRipRsvd); + } + + if ( fGstInLongMode + && AttrCs.n.u1Long) + { + Assert(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxLinearAddrWidth == 48); /* Canonical. */ + if ( IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxLinearAddrWidth < 64 + && X86_IS_CANONICAL(pVmcs->u64GuestRip.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRip); + } + } + + /* RFLAGS (bits 63:22 (or 31:22), bits 15, 5, 3 are reserved, bit 1 MB1). */ + uint64_t const uGuestRFlags = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode ? pVmcs->u64GuestRFlags.u + : pVmcs->u64GuestRFlags.s.Lo; + if ( !(uGuestRFlags & ~(X86_EFL_LIVE_MASK | X86_EFL_RA1_MASK)) + && (uGuestRFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRFlagsRsvd); + + if (!(uGuestRFlags & X86_EFL_VM)) + { /* likely */ } + else + { + if ( fGstInLongMode + || !(pVmcs->u64GuestCr0.u & X86_CR0_PE)) + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRFlagsVm); + } + + if (VMX_ENTRY_INT_INFO_IS_EXT_INT(pVmcs->u32EntryIntInfo)) + { + if (uGuestRFlags & X86_EFL_IF) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRFlagsIf); + } + + NOREF(pszInstr); + NOREF(pszFailure); + return VINF_SUCCESS; +} + + +/** + * Checks guest non-register state as part of VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +DECLINLINE(int) iemVmxVmentryCheckGuestNonRegState(PVMCPUCC pVCpu, const char *pszInstr) +{ + /* + * Guest non-register state. + * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State". + */ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char *const pszFailure = "VM-exit"; + + /* + * Activity state. + */ + uint64_t const u64GuestVmxMiscMsr = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Misc; + uint32_t const fActivityStateMask = RT_BF_GET(u64GuestVmxMiscMsr, VMX_BF_MISC_ACTIVITY_STATES); + if (!(pVmcs->u32GuestActivityState & fActivityStateMask)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateRsvd); + + X86DESCATTR AttrSs; AttrSs.u = pVmcs->u32GuestSsAttr; + if ( !AttrSs.n.u2Dpl + || pVmcs->u32GuestActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateSsDpl); + + if ( pVmcs->u32GuestIntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI + || pVmcs->u32GuestIntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS) + { + if (pVmcs->u32GuestActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateStiMovSs); + } + + if (VMX_ENTRY_INT_INFO_IS_VALID(pVmcs->u32EntryIntInfo)) + { + uint8_t const uType = VMX_ENTRY_INT_INFO_TYPE(pVmcs->u32EntryIntInfo); + uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(pVmcs->u32EntryIntInfo); + AssertCompile(VMX_V_GUEST_ACTIVITY_STATE_MASK == (VMX_VMCS_GUEST_ACTIVITY_HLT | VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN)); + switch (pVmcs->u32GuestActivityState) + { + case VMX_VMCS_GUEST_ACTIVITY_HLT: + { + if ( uType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT + || uType == VMX_ENTRY_INT_INFO_TYPE_NMI + || ( uType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT + && ( uVector == X86_XCPT_DB + || uVector == X86_XCPT_MC)) + || ( uType == VMX_ENTRY_INT_INFO_TYPE_OTHER_EVENT + && uVector == VMX_ENTRY_INT_INFO_VECTOR_MTF)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateHlt); + break; + } + + case VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN: + { + if ( uType == VMX_ENTRY_INT_INFO_TYPE_NMI + || ( uType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT + && uVector == X86_XCPT_MC)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateShutdown); + break; + } + + case VMX_VMCS_GUEST_ACTIVITY_ACTIVE: + default: + break; + } + } + + /* + * Interruptibility state. + */ + if (!(pVmcs->u32GuestIntrState & ~VMX_VMCS_GUEST_INT_STATE_MASK)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateRsvd); + + if ((pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)) + != (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateStiMovSs); + + if ( (pVmcs->u64GuestRFlags.u & X86_EFL_IF) + || !(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateRFlagsSti); + + if (VMX_ENTRY_INT_INFO_IS_VALID(pVmcs->u32EntryIntInfo)) + { + uint8_t const uType = VMX_ENTRY_INT_INFO_TYPE(pVmcs->u32EntryIntInfo); + if (uType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT) + { + if (!(pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateExtInt); + } + else if (uType == VMX_ENTRY_INT_INFO_TYPE_NMI) + { + if (!(pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))) + { /* likely */ } + else + { + /* + * We don't support injecting NMIs when blocking-by-STI would be in effect. + * We update the Exit qualification only when blocking-by-STI is set + * without blocking-by-MovSS being set. Although in practise it does not + * make much difference since the order of checks are implementation defined. + */ + if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)) + iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_NMI_INJECT); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateNmi); + } + + if ( !(pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI) + || !(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateVirtNmi); + } + } + + /* We don't support SMM yet. So blocking-by-SMIs must not be set. */ + if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateSmi); + + /* We don't support SGX yet. So enclave-interruption must not be set. */ + if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_ENCLAVE)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateEnclave); + + /* + * Pending debug exceptions. + */ + uint64_t const uPendingDbgXcpts = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode + ? pVmcs->u64GuestPendingDbgXcpts.u + : pVmcs->u64GuestPendingDbgXcpts.s.Lo; + if (!(uPendingDbgXcpts & ~VMX_VMCS_GUEST_PENDING_DEBUG_VALID_MASK)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptRsvd); + + if ( (pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)) + || pVmcs->u32GuestActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT) + { + if ( (pVmcs->u64GuestRFlags.u & X86_EFL_TF) + && !(pVmcs->u64GuestDebugCtlMsr.u & MSR_IA32_DEBUGCTL_BTF) + && !(uPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS)) + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptBsTf); + + if ( ( !(pVmcs->u64GuestRFlags.u & X86_EFL_TF) + || (pVmcs->u64GuestDebugCtlMsr.u & MSR_IA32_DEBUGCTL_BTF)) + && (uPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS)) + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptBsNoTf); + } + + /* We don't support RTM (Real-time Transactional Memory) yet. */ + if (!(uPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_RTM)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptRtm); + + /* + * VMCS link pointer. + */ + if (pVmcs->u64VmcsLinkPtr.u != UINT64_C(0xffffffffffffffff)) + { + RTGCPHYS const GCPhysShadowVmcs = pVmcs->u64VmcsLinkPtr.u; + /* We don't support SMM yet (so VMCS link pointer cannot be the current VMCS). */ + if (GCPhysShadowVmcs != IEM_VMX_GET_CURRENT_VMCS(pVCpu)) + { /* likely */ } + else + { + iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrCurVmcs); + } + + /* Validate the address. */ + if ( !(GCPhysShadowVmcs & X86_PAGE_4K_OFFSET_MASK) + && !(GCPhysShadowVmcs >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysShadowVmcs)) + { /* likely */ } + else + { + iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVmcsLinkPtr); + } + } + + NOREF(pszInstr); + NOREF(pszFailure); + return VINF_SUCCESS; +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** + * Checks guest PDPTEs as part of VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static int iemVmxVmentryCheckGuestPdptes(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + /* + * Guest PDPTEs. + * See Intel spec. 26.3.1.5 "Checks on Guest Page-Directory-Pointer-Table Entries". + */ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char * const pszFailure = "VM-exit"; + + /* + * When EPT is used, we need to validate the PAE PDPTEs provided in the VMCS. + * Otherwise, we load any PAE PDPTEs referenced by CR3 at a later point. + */ + if ( iemVmxVmcsIsGuestPaePagingEnabled(pVmcs) + && (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT)) + { + /* Get PDPTEs from the VMCS. */ + X86PDPE aPaePdptes[X86_PG_PAE_PDPE_ENTRIES]; + aPaePdptes[0].u = pVmcs->u64GuestPdpte0.u; + aPaePdptes[1].u = pVmcs->u64GuestPdpte1.u; + aPaePdptes[2].u = pVmcs->u64GuestPdpte2.u; + aPaePdptes[3].u = pVmcs->u64GuestPdpte3.u; + + /* Check validity of the PDPTEs. */ + if (PGMGstArePaePdpesValid(pVCpu, &aPaePdptes[0])) + { /* likely */ } + else + { + iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_PDPTE); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPdpte); + } + } + + NOREF(pszFailure); + NOREF(pszInstr); + return VINF_SUCCESS; +} +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ + + +/** + * Checks guest-state as part of VM-entry. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static int iemVmxVmentryCheckGuestState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + int rc = iemVmxVmentryCheckGuestControlRegsMsrs(pVCpu, pszInstr); + if (RT_SUCCESS(rc)) + { + rc = iemVmxVmentryCheckGuestSegRegs(pVCpu, pszInstr); + if (RT_SUCCESS(rc)) + { + rc = iemVmxVmentryCheckGuestGdtrIdtr(pVCpu, pszInstr); + if (RT_SUCCESS(rc)) + { + rc = iemVmxVmentryCheckGuestRipRFlags(pVCpu, pszInstr); + if (RT_SUCCESS(rc)) + { + rc = iemVmxVmentryCheckGuestNonRegState(pVCpu, pszInstr); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (RT_SUCCESS(rc)) + rc = iemVmxVmentryCheckGuestPdptes(pVCpu, pszInstr); +#endif + } + } + } + } + return rc; +} + + +/** + * Checks host-state as part of VM-entry. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static int iemVmxVmentryCheckHostState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + /* + * Host Control Registers and MSRs. + * See Intel spec. 26.2.2 "Checks on Host Control Registers and MSRs". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char * const pszFailure = "VMFail"; + + /* CR0 reserved bits. */ + { + /* CR0 MB1 bits. */ + uint64_t const u64Cr0Fixed0 = iemVmxGetCr0Fixed0(pVCpu, true /* fVmxNonRootMode */); + if ((pVmcs->u64HostCr0.u & u64Cr0Fixed0) == u64Cr0Fixed0) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr0Fixed0); + + /* CR0 MBZ bits. */ + uint64_t const u64Cr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1; + if (!(pVmcs->u64HostCr0.u & ~u64Cr0Fixed1)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr0Fixed1); + } + + /* CR4 reserved bits. */ + { + /* CR4 MB1 bits. */ + uint64_t const u64Cr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0; + if ((pVmcs->u64HostCr4.u & u64Cr4Fixed0) == u64Cr4Fixed0) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Fixed0); + + /* CR4 MBZ bits. */ + uint64_t const u64Cr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1; + if (!(pVmcs->u64HostCr4.u & ~u64Cr4Fixed1)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Fixed1); + } + + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + /* CR3 reserved bits. */ + if (!(pVmcs->u64HostCr3.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr3); + + /* SYSENTER ESP and SYSENTER EIP. */ + if ( X86_IS_CANONICAL(pVmcs->u64HostSysenterEsp.u) + && X86_IS_CANONICAL(pVmcs->u64HostSysenterEip.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSysenterEspEip); + } + + /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */ + Assert(!(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_PERF_MSR)); + + /* PAT MSR. */ + if ( !(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_PAT_MSR) + || CPUMIsPatMsrValid(pVmcs->u64HostPatMsr.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostPatMsr); + + /* EFER MSR. */ + bool const fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE); + uint64_t const uValidEferMask = CPUMGetGuestEferMsrValidMask(pVCpu->CTX_SUFF(pVM)); + if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR) + { + if (!(pVmcs->u64HostEferMsr.u & ~uValidEferMask)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostEferMsrRsvd); + + bool const fHostLma = RT_BOOL(pVmcs->u64HostEferMsr.u & MSR_K6_EFER_LMA); + bool const fHostLme = RT_BOOL(pVmcs->u64HostEferMsr.u & MSR_K6_EFER_LME); + if ( fHostInLongMode == fHostLma + && fHostInLongMode == fHostLme) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostEferMsr); + } + + /* + * Host Segment and Descriptor-Table Registers. + * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers". + */ + /* Selector RPL and TI. */ + if ( !(pVmcs->HostCs & (X86_SEL_RPL | X86_SEL_LDT)) + && !(pVmcs->HostSs & (X86_SEL_RPL | X86_SEL_LDT)) + && !(pVmcs->HostDs & (X86_SEL_RPL | X86_SEL_LDT)) + && !(pVmcs->HostEs & (X86_SEL_RPL | X86_SEL_LDT)) + && !(pVmcs->HostFs & (X86_SEL_RPL | X86_SEL_LDT)) + && !(pVmcs->HostGs & (X86_SEL_RPL | X86_SEL_LDT)) + && !(pVmcs->HostTr & (X86_SEL_RPL | X86_SEL_LDT))) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSel); + + /* CS and TR selectors cannot be 0. */ + if ( pVmcs->HostCs + && pVmcs->HostTr) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCsTr); + + /* SS cannot be 0 if 32-bit host. */ + if ( fHostInLongMode + || pVmcs->HostSs) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSs); + + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + /* FS, GS, GDTR, IDTR, TR base address. */ + if ( X86_IS_CANONICAL(pVmcs->u64HostFsBase.u) + && X86_IS_CANONICAL(pVmcs->u64HostFsBase.u) + && X86_IS_CANONICAL(pVmcs->u64HostGdtrBase.u) + && X86_IS_CANONICAL(pVmcs->u64HostIdtrBase.u) + && X86_IS_CANONICAL(pVmcs->u64HostTrBase.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSegBase); + } + + /* + * Host address-space size for 64-bit CPUs. + * See Intel spec. 26.2.4 "Checks Related to Address-Space Size". + */ + bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST); + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + bool const fCpuInLongMode = CPUMIsGuestInLongMode(pVCpu); + + /* Logical processor in IA-32e mode. */ + if (fCpuInLongMode) + { + if (fHostInLongMode) + { + /* PAE must be set. */ + if (pVmcs->u64HostCr4.u & X86_CR4_PAE) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Pae); + + /* RIP must be canonical. */ + if (X86_IS_CANONICAL(pVmcs->u64HostRip.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostRip); + } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostLongMode); + } + else + { + /* Logical processor is outside IA-32e mode. */ + if ( !fGstInLongMode + && !fHostInLongMode) + { + /* PCIDE should not be set. */ + if (!(pVmcs->u64HostCr4.u & X86_CR4_PCIDE)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Pcide); + + /* The high 32-bits of RIP MBZ. */ + if (!pVmcs->u64HostRip.s.Hi) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostRipRsvd); + } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostGuestLongMode); + } + } + else + { + /* Host address-space size for 32-bit CPUs. */ + if ( !fGstInLongMode + && !fHostInLongMode) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostGuestLongModeNoCpu); + } + + NOREF(pszInstr); + NOREF(pszFailure); + return VINF_SUCCESS; +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** + * Checks the EPT pointer VMCS field as part of VM-entry. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uEptPtr The EPT pointer to check. + * @param penmVmxDiag Where to store the diagnostic reason on failure (not + * updated on success). Optional, can be NULL. + */ +static int iemVmxVmentryCheckEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr, VMXVDIAG *penmVmxDiag) RT_NOEXCEPT +{ + VMXVDIAG enmVmxDiag; + + /* Reserved bits. */ + uint8_t const cMaxPhysAddrWidth = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth; + uint64_t const fValidMask = VMX_EPTP_VALID_MASK & ~(UINT64_MAX << cMaxPhysAddrWidth); + if (uEptPtr & fValidMask) + { + /* Memory Type. */ + uint64_t const fCaps = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64EptVpidCaps; + uint8_t const fMemType = RT_BF_GET(uEptPtr, VMX_BF_EPTP_MEMTYPE); + if ( ( fMemType == VMX_EPTP_MEMTYPE_WB + && RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_MEMTYPE_WB)) + || ( fMemType == VMX_EPTP_MEMTYPE_UC + && RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_MEMTYPE_UC))) + { + /* + * Page walk length (PML4). + * Intel used to specify bit 7 of IA32_VMX_EPT_VPID_CAP as page walk length + * of 5 but that seems to be removed from the latest specs. leaving only PML4 + * as the maximum supported page-walk level hence we hardcode it as 3 (1 less than 4) + */ + Assert(RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)); + if (RT_BF_GET(uEptPtr, VMX_BF_EPTP_PAGE_WALK_LENGTH) == 3) + { + /* Access and dirty bits support in EPT structures. */ + if ( !RT_BF_GET(uEptPtr, VMX_BF_EPTP_ACCESS_DIRTY) + || RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY)) + return VINF_SUCCESS; + + enmVmxDiag = kVmxVDiag_Vmentry_EptpAccessDirty; + } + else + enmVmxDiag = kVmxVDiag_Vmentry_EptpPageWalkLength; + } + else + enmVmxDiag = kVmxVDiag_Vmentry_EptpMemType; + } + else + enmVmxDiag = kVmxVDiag_Vmentry_EptpRsvd; + + if (penmVmxDiag) + *penmVmxDiag = enmVmxDiag; + return VERR_VMX_VMENTRY_FAILED; +} +#endif + + +/** + * Checks VMCS controls fields as part of VM-entry. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + * + * @remarks This may update secondary-processor based VM-execution control fields + * in the current VMCS if necessary. + */ +static int iemVmxVmentryCheckCtls(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char * const pszFailure = "VMFail"; + bool const fVmxTrueMsrs = RT_BOOL(pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Basic & VMX_BF_BASIC_TRUE_CTLS_MASK); + + /* + * VM-execution controls. + * See Intel spec. 26.2.1.1 "VM-Execution Control Fields". + */ + { + /* Pin-based VM-execution controls. */ + { + VMXCTLSMSR const PinCtls = fVmxTrueMsrs ? pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.TruePinCtls + : pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.PinCtls; + if (!(~pVmcs->u32PinCtls & PinCtls.n.allowed0)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_PinCtlsDisallowed0); + + if (!(pVmcs->u32PinCtls & ~PinCtls.n.allowed1)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_PinCtlsAllowed1); + } + + /* Processor-based VM-execution controls. */ + { + VMXCTLSMSR const ProcCtls = fVmxTrueMsrs ? pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.TrueProcCtls + : pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.ProcCtls; + if (!(~pVmcs->u32ProcCtls & ProcCtls.n.allowed0)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtlsDisallowed0); + + if (!(pVmcs->u32ProcCtls & ~ProcCtls.n.allowed1)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtlsAllowed1); + } + + /* Secondary processor-based VM-execution controls. */ + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS) + { + VMXCTLSMSR const ProcCtls2 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.ProcCtls2; + if (!(~pVmcs->u32ProcCtls2 & ProcCtls2.n.allowed0)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtls2Disallowed0); + + if (!(pVmcs->u32ProcCtls2 & ~ProcCtls2.n.allowed1)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtls2Allowed1); + } + else + Assert(!pVmcs->u32ProcCtls2); + + /* CR3-target count. */ + if (pVmcs->u32Cr3TargetCount <= VMX_V_CR3_TARGET_COUNT) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_Cr3TargetCount); + + /* I/O bitmaps physical addresses. */ + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS) + { + RTGCPHYS const GCPhysIoBitmapA = pVmcs->u64AddrIoBitmapA.u; + if ( !(GCPhysIoBitmapA & X86_PAGE_4K_OFFSET_MASK) + && !(GCPhysIoBitmapA >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysIoBitmapA)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrIoBitmapA); + + RTGCPHYS const GCPhysIoBitmapB = pVmcs->u64AddrIoBitmapB.u; + if ( !(GCPhysIoBitmapB & X86_PAGE_4K_OFFSET_MASK) + && !(GCPhysIoBitmapB >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysIoBitmapB)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrIoBitmapB); + } + + /* MSR bitmap physical address. */ + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS) + { + RTGCPHYS const GCPhysMsrBitmap = pVmcs->u64AddrMsrBitmap.u; + if ( !(GCPhysMsrBitmap & X86_PAGE_4K_OFFSET_MASK) + && !(GCPhysMsrBitmap >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysMsrBitmap)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrMsrBitmap); + } + + /* TPR shadow related controls. */ + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW) + { + /* Virtual-APIC page physical address. */ + RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u; + if ( !(GCPhysVirtApic & X86_PAGE_4K_OFFSET_MASK) + && !(GCPhysVirtApic >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVirtApicPage); + + /* TPR threshold bits 31:4 MBZ without virtual-interrupt delivery. */ + if ( !(pVmcs->u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK) + || (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_TprThresholdRsvd); + + /* The rest done XXX document */ + } + else + { + if ( !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE) + && !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT) + && !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)) + { /* likely */ } + else + { + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE) + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtX2ApicTprShadow); + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT) + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ApicRegVirt); + Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtIntDelivery); + } + } + + /* NMI exiting and virtual-NMIs. */ + if ( (pVmcs->u32PinCtls & VMX_PIN_CTLS_NMI_EXIT) + || !(pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtNmi); + + /* Virtual-NMIs and NMI-window exiting. */ + if ( (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI) + || !(pVmcs->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_NmiWindowExit); + + /* Virtualize APIC accesses. */ + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS) + { + /* APIC-access physical address. */ + RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u; + if ( !(GCPhysApicAccess & X86_PAGE_4K_OFFSET_MASK) + && !(GCPhysApicAccess >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrApicAccess); + + /* + * Disallow APIC-access page and virtual-APIC page from being the same address. + * Note! This is not an Intel requirement, but one imposed by our implementation. + */ + /** @todo r=ramshankar: This is done primarily to simplify recursion scenarios while + * redirecting accesses between the APIC-access page and the virtual-APIC + * page. If any nested hypervisor requires this, we can implement it later. */ + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW) + { + RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u; + if (GCPhysVirtApic != GCPhysApicAccess) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrApicAccessEqVirtApic); + } + } + + /* Virtualize-x2APIC mode is mutually exclusive with virtualize-APIC accesses. */ + if ( !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE) + || !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtX2ApicVirtApic); + + /* Virtual-interrupt delivery requires external interrupt exiting. */ + if ( !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY) + || (pVmcs->u32PinCtls & VMX_PIN_CTLS_EXT_INT_EXIT)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtX2ApicVirtApic); + + /* VPID. */ + if ( !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VPID) + || pVmcs->u16Vpid != 0) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_Vpid); + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + /* Extended-Page-Table Pointer (EPTP). */ + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT) + { + VMXVDIAG enmVmxDiag; + int const rc = iemVmxVmentryCheckEptPtr(pVCpu, pVmcs->u64EptPtr.u, &enmVmxDiag); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmVmxDiag); + } +#else + Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT)); + Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST)); +#endif + Assert(!(pVmcs->u32PinCtls & VMX_PIN_CTLS_POSTED_INT)); /* We don't support posted interrupts yet. */ + Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_PML)); /* We don't support PML yet. */ + Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMFUNC)); /* We don't support VM functions yet. */ + Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT_XCPT_VE)); /* We don't support EPT-violation #VE yet. */ + Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_TSC_SCALING)); /* We don't support TSC-scaling yet. */ + + /* VMCS shadowing. */ + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING) + { + /* VMREAD-bitmap physical address. */ + RTGCPHYS const GCPhysVmreadBitmap = pVmcs->u64AddrVmreadBitmap.u; + if ( !(GCPhysVmreadBitmap & X86_PAGE_4K_OFFSET_MASK) + && !(GCPhysVmreadBitmap >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmreadBitmap)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVmreadBitmap); + + /* VMWRITE-bitmap physical address. */ + RTGCPHYS const GCPhysVmwriteBitmap = pVmcs->u64AddrVmreadBitmap.u; + if ( !(GCPhysVmwriteBitmap & X86_PAGE_4K_OFFSET_MASK) + && !(GCPhysVmwriteBitmap >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmwriteBitmap)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVmwriteBitmap); + } + } + + /* + * VM-exit controls. + * See Intel spec. 26.2.1.2 "VM-Exit Control Fields". + */ + { + VMXCTLSMSR const ExitCtls = fVmxTrueMsrs ? pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.TrueExitCtls + : pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.ExitCtls; + if (!(~pVmcs->u32ExitCtls & ExitCtls.n.allowed0)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ExitCtlsDisallowed0); + + if (!(pVmcs->u32ExitCtls & ~ExitCtls.n.allowed1)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ExitCtlsAllowed1); + + /* Save preemption timer without activating it. */ + if ( (pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER) + || !(pVmcs->u32ProcCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_SavePreemptTimer); + + /* VM-exit MSR-store count and VM-exit MSR-store area address. */ + if (pVmcs->u32ExitMsrStoreCount) + { + if ( !(pVmcs->u64AddrExitMsrStore.u & VMX_AUTOMSR_OFFSET_MASK) + && !(pVmcs->u64AddrExitMsrStore.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), pVmcs->u64AddrExitMsrStore.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrExitMsrStore); + } + + /* VM-exit MSR-load count and VM-exit MSR-load area address. */ + if (pVmcs->u32ExitMsrLoadCount) + { + if ( !(pVmcs->u64AddrExitMsrLoad.u & VMX_AUTOMSR_OFFSET_MASK) + && !(pVmcs->u64AddrExitMsrLoad.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), pVmcs->u64AddrExitMsrLoad.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrExitMsrLoad); + } + } + + /* + * VM-entry controls. + * See Intel spec. 26.2.1.3 "VM-Entry Control Fields". + */ + { + VMXCTLSMSR const EntryCtls = fVmxTrueMsrs ? pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.TrueEntryCtls + : pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.EntryCtls; + if (!(~pVmcs->u32EntryCtls & EntryCtls.n.allowed0)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryCtlsDisallowed0); + + if (!(pVmcs->u32EntryCtls & ~EntryCtls.n.allowed1)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryCtlsAllowed1); + + /* Event injection. */ + uint32_t const uIntInfo = pVmcs->u32EntryIntInfo; + if (RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_VALID)) + { + /* Type and vector. */ + uint8_t const uType = RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_TYPE); + uint8_t const uVector = RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_VECTOR); + uint8_t const uRsvd = RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_RSVD_12_30); + if ( !uRsvd + && VMXIsEntryIntInfoTypeValid(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxMonitorTrapFlag, uType) + && VMXIsEntryIntInfoVectorValid(uVector, uType)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryIntInfoTypeVecRsvd); + + /* Exception error code. */ + if (RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID)) + { + /* Delivery possible only in Unrestricted-guest mode when CR0.PE is set. */ + if ( !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST) + || (pVmcs->u64GuestCr0.s.Lo & X86_CR0_PE)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryIntInfoErrCodePe); + + /* Exceptions that provide an error code. */ + if ( uType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT + && ( uVector == X86_XCPT_DF + || uVector == X86_XCPT_TS + || uVector == X86_XCPT_NP + || uVector == X86_XCPT_SS + || uVector == X86_XCPT_GP + || uVector == X86_XCPT_PF + || uVector == X86_XCPT_AC)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryIntInfoErrCodeVec); + + /* Exception error-code reserved bits. */ + if (!(pVmcs->u32EntryXcptErrCode & ~VMX_ENTRY_INT_XCPT_ERR_CODE_VALID_MASK)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryXcptErrCodeRsvd); + + /* Injecting a software interrupt, software exception or privileged software exception. */ + if ( uType == VMX_ENTRY_INT_INFO_TYPE_SW_INT + || uType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT + || uType == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT) + { + /* Instruction length must be in the range 0-15. */ + if (pVmcs->u32EntryInstrLen <= VMX_ENTRY_INSTR_LEN_MAX) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryInstrLen); + + /* However, instruction length of 0 is allowed only when its CPU feature is present. */ + if ( pVmcs->u32EntryInstrLen != 0 + || IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxEntryInjectSoftInt) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryInstrLenZero); + } + } + } + + /* VM-entry MSR-load count and VM-entry MSR-load area address. */ + if (pVmcs->u32EntryMsrLoadCount) + { + if ( !(pVmcs->u64AddrEntryMsrLoad.u & VMX_AUTOMSR_OFFSET_MASK) + && !(pVmcs->u64AddrEntryMsrLoad.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth) + && PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), pVmcs->u64AddrEntryMsrLoad.u)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrEntryMsrLoad); + } + + Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)); /* We don't support SMM yet. */ + Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON)); /* We don't support dual-monitor treatment yet. */ + } + + NOREF(pszInstr); + NOREF(pszFailure); + return VINF_SUCCESS; +} + + +/** + * Loads the guest control registers, debug register and some MSRs as part of + * VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxVmentryLoadGuestControlRegsMsrs(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Load guest control registers, debug registers and MSRs. + * See Intel spec. 26.3.2.1 "Loading Guest Control Registers, Debug Registers and MSRs". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); + uint64_t const uGstCr0 = (pVmcs->u64GuestCr0.u & ~VMX_ENTRY_GUEST_CR0_IGNORE_MASK) + | (pVCpu->cpum.GstCtx.cr0 & VMX_ENTRY_GUEST_CR0_IGNORE_MASK); + pVCpu->cpum.GstCtx.cr0 = uGstCr0; + pVCpu->cpum.GstCtx.cr4 = pVmcs->u64GuestCr4.u; + pVCpu->cpum.GstCtx.cr3 = pVmcs->u64GuestCr3.u; + + if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG) + pVCpu->cpum.GstCtx.dr[7] = (pVmcs->u64GuestDr7.u & ~VMX_ENTRY_GUEST_DR7_MBZ_MASK) | VMX_ENTRY_GUEST_DR7_MB1_MASK; + + pVCpu->cpum.GstCtx.SysEnter.eip = pVmcs->u64GuestSysenterEip.s.Lo; + pVCpu->cpum.GstCtx.SysEnter.esp = pVmcs->u64GuestSysenterEsp.s.Lo; + pVCpu->cpum.GstCtx.SysEnter.cs = pVmcs->u32GuestSysenterCS; + + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode) + { + /* FS base and GS base are loaded while loading the rest of the guest segment registers. */ + + /* EFER MSR. */ + if (!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)) + { + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER); + uint64_t const uHostEfer = pVCpu->cpum.GstCtx.msrEFER; + bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST); + bool const fGstPaging = RT_BOOL(uGstCr0 & X86_CR0_PG); + if (fGstInLongMode) + { + /* If the nested-guest is in long mode, LMA and LME are both set. */ + Assert(fGstPaging); + pVCpu->cpum.GstCtx.msrEFER = uHostEfer | (MSR_K6_EFER_LMA | MSR_K6_EFER_LME); + } + else + { + /* + * If the nested-guest is outside long mode: + * - With paging: LMA is cleared, LME is cleared. + * - Without paging: LMA is cleared, LME is left unmodified. + */ + uint64_t const fLmaLmeMask = MSR_K6_EFER_LMA | (fGstPaging ? MSR_K6_EFER_LME : 0); + pVCpu->cpum.GstCtx.msrEFER = uHostEfer & ~fLmaLmeMask; + } + } + /* else: see below. */ + } + + /* PAT MSR. */ + if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR) + pVCpu->cpum.GstCtx.msrPAT = pVmcs->u64GuestPatMsr.u; + + /* EFER MSR. */ + if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR) + pVCpu->cpum.GstCtx.msrEFER = pVmcs->u64GuestEferMsr.u; + + /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */ + Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)); + + /* We don't support IA32_BNDCFGS MSR yet. */ + Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_BNDCFGS_MSR)); + + /* Nothing to do for SMBASE register - We don't support SMM yet. */ +} + + +/** + * Loads the guest segment registers, GDTR, IDTR, LDTR and TR as part of VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxVmentryLoadGuestSegRegs(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Load guest segment registers, GDTR, IDTR, LDTR and TR. + * See Intel spec. 26.3.2.2 "Loading Guest Segment Registers and Descriptor-Table Registers". + */ + /* CS, SS, ES, DS, FS, GS. */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++) + { + PCPUMSELREG pGstSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg]; + CPUMSELREG VmcsSelReg; + int rc = iemVmxVmcsGetGuestSegReg(pVmcs, iSegReg, &VmcsSelReg); + AssertRC(rc); NOREF(rc); + if (!(VmcsSelReg.Attr.u & X86DESCATTR_UNUSABLE)) + { + pGstSelReg->Sel = VmcsSelReg.Sel; + pGstSelReg->ValidSel = VmcsSelReg.Sel; + pGstSelReg->fFlags = CPUMSELREG_FLAGS_VALID; + pGstSelReg->u64Base = VmcsSelReg.u64Base; + pGstSelReg->u32Limit = VmcsSelReg.u32Limit; + pGstSelReg->Attr.u = VmcsSelReg.Attr.u; + } + else + { + pGstSelReg->Sel = VmcsSelReg.Sel; + pGstSelReg->ValidSel = VmcsSelReg.Sel; + pGstSelReg->fFlags = CPUMSELREG_FLAGS_VALID; + switch (iSegReg) + { + case X86_SREG_CS: + pGstSelReg->u64Base = VmcsSelReg.u64Base; + pGstSelReg->u32Limit = VmcsSelReg.u32Limit; + pGstSelReg->Attr.u = VmcsSelReg.Attr.u; + break; + + case X86_SREG_SS: + pGstSelReg->u64Base = VmcsSelReg.u64Base & UINT32_C(0xfffffff0); + pGstSelReg->u32Limit = 0; + pGstSelReg->Attr.u = (VmcsSelReg.Attr.u & X86DESCATTR_DPL) | X86DESCATTR_D | X86DESCATTR_UNUSABLE; + break; + + case X86_SREG_ES: + case X86_SREG_DS: + pGstSelReg->u64Base = 0; + pGstSelReg->u32Limit = 0; + pGstSelReg->Attr.u = X86DESCATTR_UNUSABLE; + break; + + case X86_SREG_FS: + case X86_SREG_GS: + pGstSelReg->u64Base = VmcsSelReg.u64Base; + pGstSelReg->u32Limit = 0; + pGstSelReg->Attr.u = X86DESCATTR_UNUSABLE; + break; + } + Assert(pGstSelReg->Attr.n.u1Unusable); + } + } + + /* LDTR. */ + pVCpu->cpum.GstCtx.ldtr.Sel = pVmcs->GuestLdtr; + pVCpu->cpum.GstCtx.ldtr.ValidSel = pVmcs->GuestLdtr; + pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID; + if (!(pVmcs->u32GuestLdtrAttr & X86DESCATTR_UNUSABLE)) + { + pVCpu->cpum.GstCtx.ldtr.u64Base = pVmcs->u64GuestLdtrBase.u; + pVCpu->cpum.GstCtx.ldtr.u32Limit = pVmcs->u32GuestLdtrLimit; + pVCpu->cpum.GstCtx.ldtr.Attr.u = pVmcs->u32GuestLdtrAttr; + } + else + { + pVCpu->cpum.GstCtx.ldtr.u64Base = 0; + pVCpu->cpum.GstCtx.ldtr.u32Limit = 0; + pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE; + } + + /* TR. */ + Assert(!(pVmcs->u32GuestTrAttr & X86DESCATTR_UNUSABLE)); + pVCpu->cpum.GstCtx.tr.Sel = pVmcs->GuestTr; + pVCpu->cpum.GstCtx.tr.ValidSel = pVmcs->GuestTr; + pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.tr.u64Base = pVmcs->u64GuestTrBase.u; + pVCpu->cpum.GstCtx.tr.u32Limit = pVmcs->u32GuestTrLimit; + pVCpu->cpum.GstCtx.tr.Attr.u = pVmcs->u32GuestTrAttr; + + /* GDTR. */ + pVCpu->cpum.GstCtx.gdtr.cbGdt = pVmcs->u32GuestGdtrLimit; + pVCpu->cpum.GstCtx.gdtr.pGdt = pVmcs->u64GuestGdtrBase.u; + + /* IDTR. */ + pVCpu->cpum.GstCtx.idtr.cbIdt = pVmcs->u32GuestIdtrLimit; + pVCpu->cpum.GstCtx.idtr.pIdt = pVmcs->u64GuestIdtrBase.u; +} + + +/** + * Loads the guest MSRs from the VM-entry MSR-load area as part of VM-entry. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static int iemVmxVmentryLoadGuestAutoMsrs(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + /* + * Load guest MSRs. + * See Intel spec. 26.4 "Loading MSRs". + */ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + const char *const pszFailure = "VM-exit"; + + /* + * The VM-entry MSR-load area address need not be a valid guest-physical address if the + * VM-entry MSR load count is 0. If this is the case, bail early without reading it. + * See Intel spec. 24.8.2 "VM-Entry Controls for MSRs". + */ + uint32_t const cMsrs = RT_MIN(pVmcs->u32EntryMsrLoadCount, RT_ELEMENTS(pVCpu->cpum.GstCtx.hwvirt.vmx.aEntryMsrLoadArea)); + if (!cMsrs) + return VINF_SUCCESS; + + /* + * Verify the MSR auto-load count. Physical CPUs can behave unpredictably if the count is + * exceeded including possibly raising #MC exceptions during VMX transition. Our + * implementation shall fail VM-entry with an VMX_EXIT_ERR_MSR_LOAD VM-exit. + */ + bool const fIsMsrCountValid = iemVmxIsAutoMsrCountValid(pVCpu, cMsrs); + if (fIsMsrCountValid) + { /* likely */ } + else + { + iemVmxVmcsSetExitQual(pVCpu, VMX_V_AUTOMSR_AREA_SIZE / sizeof(VMXAUTOMSR)); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrLoadCount); + } + + RTGCPHYS const GCPhysVmEntryMsrLoadArea = pVmcs->u64AddrEntryMsrLoad.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.aEntryMsrLoadArea[0], + GCPhysVmEntryMsrLoadArea, cMsrs * sizeof(VMXAUTOMSR)); + if (RT_SUCCESS(rc)) + { + PCVMXAUTOMSR pMsr = &pVCpu->cpum.GstCtx.hwvirt.vmx.aEntryMsrLoadArea[0]; + for (uint32_t idxMsr = 0; idxMsr < cMsrs; idxMsr++, pMsr++) + { + if ( !pMsr->u32Reserved + && pMsr->u32Msr != MSR_K8_FS_BASE + && pMsr->u32Msr != MSR_K8_GS_BASE + && pMsr->u32Msr != MSR_K6_EFER + && pMsr->u32Msr != MSR_IA32_SMM_MONITOR_CTL + && pMsr->u32Msr >> 8 != MSR_IA32_X2APIC_START >> 8) + { + VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pMsr->u32Msr, pMsr->u64Value); + if (rcStrict == VINF_SUCCESS) + continue; + + /* + * If we're in ring-0, we cannot handle returns to ring-3 at this point and continue VM-entry. + * If any nested hypervisor loads MSRs that require ring-3 handling, we cause a VM-entry failure + * recording the MSR index in the Exit qualification (as per the Intel spec.) and indicated + * further by our own, specific diagnostic code. Later, we can try implement handling of the + * MSR in ring-0 if possible, or come up with a better, generic solution. + */ + iemVmxVmcsSetExitQual(pVCpu, idxMsr); + VMXVDIAG const enmDiag = rcStrict == VINF_CPUM_R3_MSR_WRITE + ? kVmxVDiag_Vmentry_MsrLoadRing3 + : kVmxVDiag_Vmentry_MsrLoad; + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag); + } + else + { + iemVmxVmcsSetExitQual(pVCpu, idxMsr); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrLoadRsvd); + } + } + } + else + { + AssertMsgFailed(("%s: Failed to read MSR auto-load area at %#RGp, rc=%Rrc\n", pszInstr, GCPhysVmEntryMsrLoadArea, rc)); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrLoadPtrReadPhys); + } + + NOREF(pszInstr); + NOREF(pszFailure); + return VINF_SUCCESS; +} + + +/** + * Loads the guest-state non-register state as part of VM-entry. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + * + * @remarks This must be called only after loading the nested-guest register state + * (especially nested-guest RIP). + */ +static int iemVmxVmentryLoadGuestNonRegState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + /* + * Load guest non-register state. + * See Intel spec. 26.6 "Special Features of VM Entry" + */ + const char *const pszFailure = "VM-exit"; + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* + * If VM-entry is not vectoring, block-by-STI and block-by-MovSS state must be loaded. + * If VM-entry is vectoring, there is no block-by-STI or block-by-MovSS. + * + * See Intel spec. 26.6.1 "Interruptibility State". + */ + bool const fEntryVectoring = VMXIsVmentryVectoring(pVmcs->u32EntryIntInfo, NULL /* puEntryIntInfoType */); + if ( !fEntryVectoring + && (pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))) + CPUMSetInInterruptShadowEx(&pVCpu->cpum.GstCtx, pVmcs->u64GuestRip.u); + else + Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)); + + /* NMI blocking. */ + if (pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI) + { + if (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI) + pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = true; + else + { + pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false; + CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx); + } + } + else + pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false; + + /* SMI blocking is irrelevant. We don't support SMIs yet. */ + + /* + * Set PGM's copy of the EPT pointer. + * The EPTP has already been validated while checking guest state. + * + * It is important to do this prior to mapping PAE PDPTEs (below). + */ + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT) + PGMSetGuestEptPtr(pVCpu, pVmcs->u64EptPtr.u); + + /* + * Load the guest's PAE PDPTEs. + */ + if (!iemVmxVmcsIsGuestPaePagingEnabled(pVmcs)) + { + /* + * When PAE paging is not used we clear the PAE PDPTEs for safety + * in case we might be switching from a PAE host to a non-PAE guest. + */ + pVCpu->cpum.GstCtx.aPaePdpes[0].u = 0; + pVCpu->cpum.GstCtx.aPaePdpes[1].u = 0; + pVCpu->cpum.GstCtx.aPaePdpes[2].u = 0; + pVCpu->cpum.GstCtx.aPaePdpes[3].u = 0; + } + else if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT) + { + /* + * With EPT and the nested-guest using PAE paging, we've already validated the PAE PDPTEs + * while checking the guest state. We can load them into the nested-guest CPU state now. + * They'll later be used while mapping CR3 and the PAE PDPTEs. + */ + pVCpu->cpum.GstCtx.aPaePdpes[0].u = pVmcs->u64GuestPdpte0.u; + pVCpu->cpum.GstCtx.aPaePdpes[1].u = pVmcs->u64GuestPdpte1.u; + pVCpu->cpum.GstCtx.aPaePdpes[2].u = pVmcs->u64GuestPdpte2.u; + pVCpu->cpum.GstCtx.aPaePdpes[3].u = pVmcs->u64GuestPdpte3.u; + } + else + { + /* + * Without EPT and the nested-guest using PAE paging, we must load the PAE PDPTEs + * referenced by CR3. This involves loading (and mapping) CR3 and validating them now. + */ + int const rc = PGMGstMapPaePdpesAtCr3(pVCpu, pVmcs->u64GuestCr3.u); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_PDPTE); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPdpte); + } + } + + /* VPID is irrelevant. We don't support VPID yet. */ + + /* Clear address-range monitoring. */ + EMMonitorWaitClear(pVCpu); + + return VINF_SUCCESS; +} + + +/** + * Loads the guest VMCS referenced state (such as MSR bitmaps, I/O bitmaps etc). + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + * + * @remarks This assumes various VMCS related data structure pointers have already + * been verified prior to calling this function. + */ +static int iemVmxVmentryLoadGuestVmcsRefState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + const char *const pszFailure = "VM-exit"; + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* + * Virtualize APIC accesses. + */ + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS) + { + /* APIC-access physical address. */ + RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u; + + /* + * Register the handler for the APIC-access page. + * + * We don't deregister the APIC-access page handler during the VM-exit as a different + * nested-VCPU might be using the same guest-physical address for its APIC-access page. + * + * We leave the page registered until the first access that happens outside VMX non-root + * mode. Guest software is allowed to access structures such as the APIC-access page + * only when no logical processor with a current VMCS references it in VMX non-root mode, + * otherwise it can lead to unpredictable behavior including guest triple-faults. + * + * See Intel spec. 24.11.4 "Software Access to Related Structures". + */ + if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess)) + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + int rc = PGMHandlerPhysicalRegister(pVM, GCPhysApicAccess, GCPhysApicAccess | X86_PAGE_4K_OFFSET_MASK, + pVM->iem.s.hVmxApicAccessPage, 0 /*uUser*/, NULL /*pszDesc*/); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrApicAccessHandlerReg); + } + } + + /* + * VMCS shadowing. + */ + if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING) + { + /* Read the VMREAD-bitmap. */ + RTGCPHYS const GCPhysVmreadBitmap = pVmcs->u64AddrVmreadBitmap.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abVmreadBitmap[0], + GCPhysVmreadBitmap, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.abVmreadBitmap)); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmreadBitmapPtrReadPhys); + + /* Read the VMWRITE-bitmap. */ + RTGCPHYS const GCPhysVmwriteBitmap = pVmcs->u64AddrVmwriteBitmap.u; + rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abVmwriteBitmap[0], + GCPhysVmwriteBitmap, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.abVmwriteBitmap)); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmwriteBitmapPtrReadPhys); + } + + /* + * I/O bitmaps. + */ + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS) + { + /* Read the IO bitmap A. */ + RTGCPHYS const GCPhysIoBitmapA = pVmcs->u64AddrIoBitmapA.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abIoBitmap[0], + GCPhysIoBitmapA, VMX_V_IO_BITMAP_A_SIZE); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_IoBitmapAPtrReadPhys); + + /* Read the IO bitmap B. */ + RTGCPHYS const GCPhysIoBitmapB = pVmcs->u64AddrIoBitmapB.u; + rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abIoBitmap[VMX_V_IO_BITMAP_A_SIZE], + GCPhysIoBitmapB, VMX_V_IO_BITMAP_B_SIZE); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_IoBitmapBPtrReadPhys); + } + + /* + * TPR shadow and Virtual-APIC page. + */ + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW) + { + /* Verify TPR threshold and VTPR when both virtualize-APIC accesses and virtual-interrupt delivery aren't used. */ + if ( !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS) + && !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)) + { + /* Read the VTPR from the virtual-APIC page. */ + RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u; + uint8_t u8VTpr; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &u8VTpr, GCPhysVirtApic + XAPIC_OFF_TPR, sizeof(u8VTpr)); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtApicPagePtrReadPhys); + + /* Bits 3:0 of the TPR-threshold must not be greater than bits 7:4 of VTPR. */ + if ((uint8_t)RT_BF_GET(pVmcs->u32TprThreshold, VMX_BF_TPR_THRESHOLD_TPR) <= (u8VTpr & 0xf0)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_TprThresholdVTpr); + } + } + + /* + * VMCS link pointer. + */ + if (pVmcs->u64VmcsLinkPtr.u != UINT64_C(0xffffffffffffffff)) + { + /* Read the VMCS-link pointer from guest memory. */ + RTGCPHYS const GCPhysShadowVmcs = pVmcs->u64VmcsLinkPtr.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs, + GCPhysShadowVmcs, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs)); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrReadPhys); + } + + /* Verify the VMCS revision specified by the guest matches what we reported to the guest. */ + if (pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs.u32VmcsRevId.n.u31RevisionId == VMX_V_VMCS_REVISION_ID) + { /* likely */ } + else + { + iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrRevId); + } + + /* Verify the shadow bit is set if VMCS shadowing is enabled . */ + if ( !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING) + || pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs.u32VmcsRevId.n.fIsShadowVmcs) + { /* likely */ } + else + { + iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR); + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrShadow); + } + + /* Update our cache of the guest physical address of the shadow VMCS. */ + pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysShadowVmcs = GCPhysShadowVmcs; + } + + /* + * MSR bitmap. + */ + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS) + { + /* Read the MSR bitmap. */ + RTGCPHYS const GCPhysMsrBitmap = pVmcs->u64AddrMsrBitmap.u; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap[0], + GCPhysMsrBitmap, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap)); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrBitmapPtrReadPhys); + } + + NOREF(pszFailure); + NOREF(pszInstr); + return VINF_SUCCESS; +} + + +/** + * Loads the guest-state as part of VM-entry. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + * + * @remarks This must be done after all the necessary steps prior to loading of + * guest-state (e.g. checking various VMCS state). + */ +static int iemVmxVmentryLoadGuestState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + /* Load guest control registers, MSRs (that are directly part of the VMCS). */ + iemVmxVmentryLoadGuestControlRegsMsrs(pVCpu); + + /* Load guest segment registers. */ + iemVmxVmentryLoadGuestSegRegs(pVCpu); + + /* + * Load guest RIP, RSP and RFLAGS. + * See Intel spec. 26.3.2.3 "Loading Guest RIP, RSP and RFLAGS". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + pVCpu->cpum.GstCtx.rsp = pVmcs->u64GuestRsp.u; + pVCpu->cpum.GstCtx.rip = pVmcs->u64GuestRip.u; + pVCpu->cpum.GstCtx.rflags.u = pVmcs->u64GuestRFlags.u; + + /* Initialize the PAUSE-loop controls as part of VM-entry. */ + pVCpu->cpum.GstCtx.hwvirt.vmx.uFirstPauseLoopTick = 0; + pVCpu->cpum.GstCtx.hwvirt.vmx.uPrevPauseTick = 0; + + /* Load guest non-register state (such as interrupt shadows, NMI blocking etc). */ + int rc = iemVmxVmentryLoadGuestNonRegState(pVCpu, pszInstr); + if (rc == VINF_SUCCESS) + { /* likely */ } + else + return rc; + + /* Load VMX related structures and state referenced by the VMCS. */ + rc = iemVmxVmentryLoadGuestVmcsRefState(pVCpu, pszInstr); + if (rc == VINF_SUCCESS) + { /* likely */ } + else + return rc; + + NOREF(pszInstr); + return VINF_SUCCESS; +} + + +/** + * Returns whether there are is a pending debug exception on VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static bool iemVmxVmentryIsPendingDebugXcpt(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + /* + * Pending debug exceptions. + * See Intel spec. 26.6.3 "Delivery of Pending Debug Exceptions after VM Entry". + */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs); + + bool fPendingDbgXcpt = RT_BOOL(pVmcs->u64GuestPendingDbgXcpts.u & ( VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS + | VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_EN_BP)); + if (fPendingDbgXcpt) + { + uint8_t uEntryIntInfoType; + bool const fEntryVectoring = VMXIsVmentryVectoring(pVmcs->u32EntryIntInfo, &uEntryIntInfoType); + if (fEntryVectoring) + { + switch (uEntryIntInfoType) + { + case VMX_ENTRY_INT_INFO_TYPE_EXT_INT: + case VMX_ENTRY_INT_INFO_TYPE_NMI: + case VMX_ENTRY_INT_INFO_TYPE_HW_XCPT: + case VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT: + fPendingDbgXcpt = false; + break; + + case VMX_ENTRY_INT_INFO_TYPE_SW_XCPT: + { + /* + * Whether the pending debug exception for software exceptions other than + * #BP and #OF is delivered after injecting the exception or is discard + * is CPU implementation specific. We will discard them (easier). + */ + uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(pVmcs->u32EntryIntInfo); + if ( uVector != X86_XCPT_BP + && uVector != X86_XCPT_OF) + fPendingDbgXcpt = false; + RT_FALL_THRU(); + } + case VMX_ENTRY_INT_INFO_TYPE_SW_INT: + { + if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)) + fPendingDbgXcpt = false; + break; + } + } + } + else + { + /* + * When the VM-entry is not vectoring but there is blocking-by-MovSS, whether the + * pending debug exception is held pending or is discarded is CPU implementation + * specific. We will discard them (easier). + */ + if (pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS) + fPendingDbgXcpt = false; + + /* There's no pending debug exception in the shutdown or wait-for-SIPI state. */ + if (pVmcs->u32GuestActivityState & (VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN | VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT)) + fPendingDbgXcpt = false; + } + } + + NOREF(pszInstr); + return fPendingDbgXcpt; +} + + +/** + * Set up the monitor-trap flag (MTF). + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static void iemVmxVmentrySetupMtf(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs); + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG) + { + VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_MTF); + Log(("%s: Monitor-trap flag set on VM-entry\n", pszInstr)); + } + else + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF)); + NOREF(pszInstr); +} + + +/** + * Sets up NMI-window exiting. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static void iemVmxVmentrySetupNmiWindow(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs); + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT) + { + Assert(pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI); + VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW); + Log(("%s: NMI-window set on VM-entry\n", pszInstr)); + } + else + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)); + NOREF(pszInstr); +} + + +/** + * Sets up interrupt-window exiting. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static void iemVmxVmentrySetupIntWindow(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs); + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT) + { + VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW); + Log(("%s: Interrupt-window set on VM-entry\n", pszInstr)); + } + else + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)); + NOREF(pszInstr); +} + + +/** + * Set up the VMX-preemption timer. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static void iemVmxVmentrySetupPreemptTimer(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs); + if (pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER) + { + /* + * If the timer is 0, we must cause a VM-exit before executing the first + * nested-guest instruction. So we can flag as though the timer has already + * expired and we will check and cause a VM-exit at the right priority elsewhere + * in the code. + */ + uint64_t uEntryTick; + uint32_t const uPreemptTimer = pVmcs->u32PreemptTimer; + if (uPreemptTimer) + { + int rc = CPUMStartGuestVmxPremptTimer(pVCpu, uPreemptTimer, VMX_V_PREEMPT_TIMER_SHIFT, &uEntryTick); + AssertRC(rc); + Log(("%s: VM-entry set up VMX-preemption timer at %#RX64\n", pszInstr, uEntryTick)); + } + else + { + uEntryTick = TMCpuTickGetNoCheck(pVCpu); + VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER); + Log(("%s: VM-entry set up VMX-preemption timer at %#RX64 to expire immediately!\n", pszInstr, uEntryTick)); + } + + pVCpu->cpum.GstCtx.hwvirt.vmx.uEntryTick = uEntryTick; + } + else + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER)); + + NOREF(pszInstr); +} + + +/** + * Injects an event using TRPM given a VM-entry interruption info and related + * fields. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + * @param uEntryIntInfo The VM-entry interruption info. + * @param uErrCode The error code associated with the event if any. + * @param cbInstr The VM-entry instruction length (for software + * interrupts and software exceptions). Pass 0 + * otherwise. + * @param GCPtrFaultAddress The guest CR2 if this is a \#PF event. + */ +static void iemVmxVmentryInjectTrpmEvent(PVMCPUCC pVCpu, const char *pszInstr, uint32_t uEntryIntInfo, uint32_t uErrCode, + uint32_t cbInstr, RTGCUINTPTR GCPtrFaultAddress) RT_NOEXCEPT +{ + Assert(VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo)); + + uint8_t const uType = VMX_ENTRY_INT_INFO_TYPE(uEntryIntInfo); + uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(uEntryIntInfo); + TRPMEVENT const enmTrpmEvent = HMVmxEventTypeToTrpmEventType(uEntryIntInfo); + + Assert(uType != VMX_ENTRY_INT_INFO_TYPE_OTHER_EVENT); + + int rc = TRPMAssertTrap(pVCpu, uVector, enmTrpmEvent); + AssertRC(rc); + Log(("%s: Injecting: vector=%#x type=%#x (%s)\n", pszInstr, uVector, uType, VMXGetEntryIntInfoTypeDesc(uType))); + + if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(uEntryIntInfo)) + { + TRPMSetErrorCode(pVCpu, uErrCode); + Log(("%s: Injecting: err_code=%#x\n", pszInstr, uErrCode)); + } + + if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(uEntryIntInfo)) + { + TRPMSetFaultAddress(pVCpu, GCPtrFaultAddress); + Log(("%s: Injecting: fault_addr=%RGp\n", pszInstr, GCPtrFaultAddress)); + } + else if ( uType == VMX_ENTRY_INT_INFO_TYPE_SW_INT + || uType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT + || uType == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT) + { + TRPMSetInstrLength(pVCpu, cbInstr); + Log(("%s: Injecting: instr_len=%u\n", pszInstr, cbInstr)); + } + + if (VMX_ENTRY_INT_INFO_TYPE(uEntryIntInfo) == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT) + { + TRPMSetTrapDueToIcebp(pVCpu); + Log(("%s: Injecting: icebp\n", pszInstr)); + } + + NOREF(pszInstr); +} + + +/** + * Performs event injection (if any) as part of VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pszInstr The VMX instruction name (for logging purposes). + */ +static void iemVmxVmentryInjectEvent(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT +{ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* + * Inject events. + * The event that is going to be made pending for injection is not subject to VMX intercepts, + * thus we flag ignoring of intercepts. However, recursive exceptions if any during delivery + * of the current event -are- subject to intercepts, hence this flag will be flipped during + * the actually delivery of this event. + * + * See Intel spec. 26.5 "Event Injection". + */ + uint32_t const uEntryIntInfo = pVmcs->u32EntryIntInfo; + bool const fEntryIntInfoValid = VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo); + + CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, !fEntryIntInfoValid); + if (fEntryIntInfoValid) + { + if (VMX_ENTRY_INT_INFO_TYPE(uEntryIntInfo) == VMX_ENTRY_INT_INFO_TYPE_OTHER_EVENT) + { + Assert(VMX_ENTRY_INT_INFO_VECTOR(uEntryIntInfo) == VMX_ENTRY_INT_INFO_VECTOR_MTF); + VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_MTF); + } + else + iemVmxVmentryInjectTrpmEvent(pVCpu, pszInstr, uEntryIntInfo, pVmcs->u32EntryXcptErrCode, pVmcs->u32EntryInstrLen, + pVCpu->cpum.GstCtx.cr2); + + /* + * We need to clear the VM-entry interruption information field's valid bit on VM-exit. + * + * However, we do it here on VM-entry as well because while it isn't visible to guest + * software until VM-exit, when and if HM looks at the VMCS to continue nested-guest + * execution using hardware-assisted VMX, it will not try to inject the event again. + * + * See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection". + */ + pVmcs->u32EntryIntInfo &= ~VMX_ENTRY_INT_INFO_VALID; + } + else + { + /* + * Inject any pending guest debug exception. + * Unlike injecting events, this #DB injection on VM-entry is subject to #DB VMX intercept. + * See Intel spec. 26.6.3 "Delivery of Pending Debug Exceptions after VM Entry". + */ + bool const fPendingDbgXcpt = iemVmxVmentryIsPendingDebugXcpt(pVCpu, pszInstr); + if (fPendingDbgXcpt) + { + uint32_t const uDbgXcptInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1); + iemVmxVmentryInjectTrpmEvent(pVCpu, pszInstr, uDbgXcptInfo, 0 /* uErrCode */, pVmcs->u32EntryInstrLen, + 0 /* GCPtrFaultAddress */); + } + } + + NOREF(pszInstr); +} + + +/** + * Initializes all read-only VMCS fields as part of VM-entry. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void iemVmxVmentryInitReadOnlyFields(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Any VMCS field which we do not establish on every VM-exit but may potentially + * be used on the VM-exit path of a nested hypervisor -and- is not explicitly + * specified to be undefined, needs to be initialized here. + * + * Thus, it is especially important to clear the Exit qualification field + * since it must be zero for VM-exits where it is not used. Similarly, the + * VM-exit interruption information field's valid bit needs to be cleared for + * the same reasons. + */ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs); + + /* 16-bit (none currently). */ + /* 32-bit. */ + pVmcs->u32RoVmInstrError = 0; + pVmcs->u32RoExitReason = 0; + pVmcs->u32RoExitIntInfo = 0; + pVmcs->u32RoExitIntErrCode = 0; + pVmcs->u32RoIdtVectoringInfo = 0; + pVmcs->u32RoIdtVectoringErrCode = 0; + pVmcs->u32RoExitInstrLen = 0; + pVmcs->u32RoExitInstrInfo = 0; + + /* 64-bit. */ + pVmcs->u64RoGuestPhysAddr.u = 0; + + /* Natural-width. */ + pVmcs->u64RoExitQual.u = 0; + pVmcs->u64RoIoRcx.u = 0; + pVmcs->u64RoIoRsi.u = 0; + pVmcs->u64RoIoRdi.u = 0; + pVmcs->u64RoIoRip.u = 0; + pVmcs->u64RoGuestLinearAddr.u = 0; +} + + +/** + * VMLAUNCH/VMRESUME instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param uInstrId The instruction identity (VMXINSTRID_VMLAUNCH or + * VMXINSTRID_VMRESUME). + * + * @remarks Common VMX instruction checks are already expected to by the caller, + * i.e. CR4.VMXE, Real/V86 mode, EFER/CS.L checks. + */ +static VBOXSTRICTRC iemVmxVmlaunchVmresume(PVMCPUCC pVCpu, uint8_t cbInstr, VMXINSTRID uInstrId) RT_NOEXCEPT +{ +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3) + RT_NOREF3(pVCpu, cbInstr, uInstrId); + return VINF_EM_RAW_EMULATE_INSTR; +# else + Assert( uInstrId == VMXINSTRID_VMLAUNCH + || uInstrId == VMXINSTRID_VMRESUME); + const char * const pszInstr = uInstrId == VMXINSTRID_VMRESUME ? "vmresume" : "vmlaunch"; + + /* Nested-guest intercept. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + return iemVmxVmexitInstr(pVCpu, uInstrId == VMXINSTRID_VMRESUME ? VMX_EXIT_VMRESUME : VMX_EXIT_VMLAUNCH, cbInstr); + + Assert(IEM_VMX_IS_ROOT_MODE(pVCpu)); + + /* + * Basic VM-entry checks. + * The order of the CPL, current and shadow VMCS and block-by-MovSS are important. + * The checks following that do not have to follow a specific order. + * + * See Intel spec. 26.1 "Basic VM-entry Checks". + */ + + /* CPL. */ + if (pVCpu->iem.s.uCpl == 0) + { /* likely */ } + else + { + Log(("%s: CPL %u -> #GP(0)\n", pszInstr, pVCpu->iem.s.uCpl)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_Cpl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Current VMCS valid. */ + if (IEM_VMX_HAS_CURRENT_VMCS(pVCpu)) + { /* likely */ } + else + { + Log(("%s: VMCS pointer %#RGp invalid -> VMFailInvalid\n", pszInstr, IEM_VMX_GET_CURRENT_VMCS(pVCpu))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_PtrInvalid; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Current VMCS is not a shadow VMCS. */ + if (!pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32VmcsRevId.n.fIsShadowVmcs) + { /* likely */ } + else + { + Log(("%s: VMCS pointer %#RGp is a shadow VMCS -> VMFailInvalid\n", pszInstr, IEM_VMX_GET_CURRENT_VMCS(pVCpu))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_PtrShadowVmcs; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /** @todo Distinguish block-by-MovSS from block-by-STI. Currently we + * use block-by-STI here which is not quite correct. */ + if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx)) + { /* likely */ } + else + { + Log(("%s: VM entry with events blocked by MOV SS -> VMFail\n", pszInstr)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_BlocKMovSS; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMENTRY_BLOCK_MOVSS); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + if (uInstrId == VMXINSTRID_VMLAUNCH) + { + /* VMLAUNCH with non-clear VMCS. */ + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.fVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR) + { /* likely */ } + else + { + Log(("vmlaunch: VMLAUNCH with non-clear VMCS -> VMFail\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_VmcsClear; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMLAUNCH_NON_CLEAR_VMCS); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + else + { + /* VMRESUME with non-launched VMCS. */ + if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.fVmcsState == VMX_V_VMCS_LAUNCH_STATE_LAUNCHED) + { /* likely */ } + else + { + Log(("vmresume: VMRESUME with non-launched VMCS -> VMFail\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_VmcsLaunch; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMRESUME_NON_LAUNCHED_VMCS); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + + /* + * We are allowed to cache VMCS related data structures (such as I/O bitmaps, MSR bitmaps) + * while entering VMX non-root mode. We do some of this while checking VM-execution + * controls. The nested hypervisor should not make assumptions and cannot expect + * predictable behavior if changes to these structures are made in guest memory while + * executing in VMX non-root mode. As far as VirtualBox is concerned, the guest cannot + * modify them anyway as we cache them in host memory. + * + * See Intel spec. 24.11.4 "Software Access to Related Structures". + */ + PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs); + Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu)); + + int rc = iemVmxVmentryCheckCtls(pVCpu, pszInstr); + if (RT_SUCCESS(rc)) + { + rc = iemVmxVmentryCheckHostState(pVCpu, pszInstr); + if (RT_SUCCESS(rc)) + { + /* + * Initialize read-only VMCS fields before VM-entry since we don't update all of them + * for every VM-exit. This needs to be done before invoking a VM-exit (even those + * ones that may occur during VM-entry below). + */ + iemVmxVmentryInitReadOnlyFields(pVCpu); + + /* + * Blocking of NMIs need to be restored if VM-entry fails due to invalid-guest state. + * So we save the VMCPU_FF_BLOCK_NMI force-flag here so we can restore it on + * VM-exit when required. + * See Intel spec. 26.7 "VM-entry Failures During or After Loading Guest State" + */ + iemVmxVmentrySaveNmiBlockingFF(pVCpu); + + rc = iemVmxVmentryCheckGuestState(pVCpu, pszInstr); + if (RT_SUCCESS(rc)) + { + /* + * We've now entered nested-guest execution. + * + * It is important do this prior to loading the guest state because + * as part of loading the guest state, PGM (and perhaps other components + * in the future) relies on detecting whether VMX non-root mode has been + * entered. + */ + pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode = true; + + rc = iemVmxVmentryLoadGuestState(pVCpu, pszInstr); + if (RT_SUCCESS(rc)) + { + rc = iemVmxVmentryLoadGuestAutoMsrs(pVCpu, pszInstr); + if (RT_SUCCESS(rc)) + { + Assert(rc != VINF_CPUM_R3_MSR_WRITE); + + /* VMLAUNCH instruction must update the VMCS launch state. */ + if (uInstrId == VMXINSTRID_VMLAUNCH) + pVmcs->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; + + /* Perform the VMX transition (PGM updates). */ + VBOXSTRICTRC rcStrict = iemVmxTransition(pVCpu); + if (rcStrict == VINF_SUCCESS) + { /* likely */ } + else if (RT_SUCCESS(rcStrict)) + { + Log3(("%s: iemVmxTransition returns %Rrc -> Setting passup status\n", pszInstr, + VBOXSTRICTRC_VAL(rcStrict))); + rcStrict = iemSetPassUpStatus(pVCpu, rcStrict); + } + else + { + Log3(("%s: iemVmxTransition failed! rc=%Rrc\n", pszInstr, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + + /* Paranoia. */ + Assert(rcStrict == VINF_SUCCESS); + + /* + * The priority of potential VM-exits during VM-entry is important. + * The priorities of VM-exits and events are listed from highest + * to lowest as follows: + * + * 1. Event injection. + * 2. Trap on task-switch (T flag set in TSS). + * 3. TPR below threshold / APIC-write. + * 4. SMI, INIT. + * 5. MTF exit. + * 6. Debug-trap exceptions (EFLAGS.TF), pending debug exceptions. + * 7. VMX-preemption timer. + * 9. NMI-window exit. + * 10. NMI injection. + * 11. Interrupt-window exit. + * 12. Virtual-interrupt injection. + * 13. Interrupt injection. + * 14. Process next instruction (fetch, decode, execute). + */ + + /* Setup VMX-preemption timer. */ + iemVmxVmentrySetupPreemptTimer(pVCpu, pszInstr); + + /* Setup monitor-trap flag. */ + iemVmxVmentrySetupMtf(pVCpu, pszInstr); + + /* Setup NMI-window exiting. */ + iemVmxVmentrySetupNmiWindow(pVCpu, pszInstr); + + /* Setup interrupt-window exiting. */ + iemVmxVmentrySetupIntWindow(pVCpu, pszInstr); + + /* + * Inject any event that the nested hypervisor wants to inject. + * Note! We cannot immediately perform the event injection here as we may have + * pending PGM operations to perform due to switching page tables and/or + * mode. + */ + iemVmxVmentryInjectEvent(pVCpu, pszInstr); + +# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3) + /* Reschedule to IEM-only execution of the nested-guest. */ + LogFlow(("%s: Enabling IEM-only EM execution policy!\n", pszInstr)); + int rcSched = EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, true); + if (rcSched != VINF_SUCCESS) + iemSetPassUpStatus(pVCpu, rcSched); +# endif + + /* Finally, done. */ + Log2(("vmentry: %s: cs:rip=%04x:%08RX64 cr0=%#RX64 (%#RX64) cr4=%#RX64 (%#RX64) efer=%#RX64 (%#RX64)\n", + pszInstr, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.cr0, + pVmcs->u64Cr0ReadShadow.u, pVCpu->cpum.GstCtx.cr4, pVmcs->u64Cr4ReadShadow.u, + pVCpu->cpum.GstCtx.msrEFER, pVmcs->u64GuestEferMsr.u)); + return VINF_SUCCESS; + } + return iemVmxVmexit(pVCpu, VMX_EXIT_ERR_MSR_LOAD | VMX_EXIT_REASON_ENTRY_FAILED, pVmcs->u64RoExitQual.u); + } + } + return iemVmxVmexit(pVCpu, VMX_EXIT_ERR_INVALID_GUEST_STATE | VMX_EXIT_REASON_ENTRY_FAILED, pVmcs->u64RoExitQual.u); + } + + iemVmxVmFail(pVCpu, VMXINSTRERR_VMENTRY_INVALID_HOST_STATE); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + iemVmxVmFail(pVCpu, VMXINSTRERR_VMENTRY_INVALID_CTLS); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +# endif +} + + +/** + * Interface for HM and EM to emulate the VMLAUNCH/VMRESUME instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cbInstr The instruction length in bytes. + * @param uInstrId The instruction ID (VMXINSTRID_VMLAUNCH or + * VMXINSTRID_VMRESUME). + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmlaunchVmresume(PVMCPUCC pVCpu, uint8_t cbInstr, VMXINSTRID uInstrId) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = iemVmxVmlaunchVmresume(pVCpu, cbInstr, uInstrId); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Checks whether an RDMSR or WRMSR instruction for the given MSR is intercepted + * (causes a VM-exit) or not. + * + * @returns @c true if the instruction is intercepted, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason (VMX_EXIT_RDMSR or + * VMX_EXIT_WRMSR). + * @param idMsr The MSR. + */ +bool iemVmxIsRdmsrWrmsrInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint32_t idMsr) RT_NOEXCEPT +{ + Assert(IEM_VMX_IS_NON_ROOT_MODE(pVCpu)); + Assert( uExitReason == VMX_EXIT_RDMSR + || uExitReason == VMX_EXIT_WRMSR); + + /* Consult the MSR bitmap if the feature is supported. */ + PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + Assert(pVmcs); + if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS) + { + uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, idMsr); + if (uExitReason == VMX_EXIT_RDMSR) + return RT_BOOL(fMsrpm & VMXMSRPM_EXIT_RD); + return RT_BOOL(fMsrpm & VMXMSRPM_EXIT_WR); + } + + /* Without MSR bitmaps, all MSR accesses are intercepted. */ + return true; +} + + +/** + * VMREAD instruction execution worker that does not perform any validation checks. + * + * Callers are expected to have performed the necessary checks and to ensure the + * VMREAD will succeed. + * + * @param pVmcs Pointer to the virtual VMCS. + * @param pu64Dst Where to write the VMCS value. + * @param u64VmcsField The VMCS field. + * + * @remarks May be called with interrupts disabled. + */ +static void iemVmxVmreadNoCheck(PCVMXVVMCS pVmcs, uint64_t *pu64Dst, uint64_t u64VmcsField) RT_NOEXCEPT +{ + VMXVMCSFIELD VmcsField; + VmcsField.u = u64VmcsField; + uint8_t const uWidth = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_WIDTH); + uint8_t const uType = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_TYPE); + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_INDEX); + Assert(uIndex <= VMX_V_VMCS_MAX_INDEX); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + AssertMsg(offField < VMX_V_VMCS_SIZE, ("off=%u field=%#RX64 width=%#x type=%#x index=%#x (%u)\n", offField, u64VmcsField, + uWidth, uType, uIndex, uIndex)); + AssertCompile(VMX_V_SHADOW_VMCS_SIZE == VMX_V_VMCS_SIZE); + + /* + * Read the VMCS component based on the field's effective width. + * + * The effective width is 64-bit fields adjusted to 32-bits if the access-type + * indicates high bits (little endian). + * + * Note! The caller is responsible to trim the result and update registers + * or memory locations are required. Here we just zero-extend to the largest + * type (i.e. 64-bits). + */ + uint8_t const *pbVmcs = (uint8_t const *)pVmcs; + uint8_t const *pbField = pbVmcs + offField; + uint8_t const uEffWidth = VMXGetVmcsFieldWidthEff(VmcsField.u); + switch (uEffWidth) + { + case VMX_VMCSFIELD_WIDTH_64BIT: + case VMX_VMCSFIELD_WIDTH_NATURAL: *pu64Dst = *(uint64_t const *)pbField; break; + case VMX_VMCSFIELD_WIDTH_32BIT: *pu64Dst = *(uint32_t const *)pbField; break; + case VMX_VMCSFIELD_WIDTH_16BIT: *pu64Dst = *(uint16_t const *)pbField; break; + } +} + + +/** + * Interface for HM and EM to read a VMCS field from the nested-guest VMCS. + * + * It is ASSUMED the caller knows what they're doing. No VMREAD instruction checks + * are performed. Bounds checks are strict builds only. + * + * @param pVmcs Pointer to the virtual VMCS. + * @param u64VmcsField The VMCS field. + * @param pu64Dst Where to store the VMCS value. + * + * @remarks May be called with interrupts disabled. + * @todo This should probably be moved to CPUM someday. + */ +VMM_INT_DECL(void) IEMReadVmxVmcsField(PCVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t *pu64Dst) +{ + AssertPtr(pVmcs); + AssertPtr(pu64Dst); + iemVmxVmreadNoCheck(pVmcs, pu64Dst, u64VmcsField); +} + + +/** + * VMREAD common (memory/register) instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param pu64Dst Where to write the VMCS value (only updated when + * VINF_SUCCESS is returned). + * @param u64VmcsField The VMCS field. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be + * NULL. + */ +static VBOXSTRICTRC iemVmxVmreadCommon(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t *pu64Dst, + uint64_t u64VmcsField, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + /* Nested-guest intercept. */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, VMX_EXIT_VMREAD, u64VmcsField)) + { + if (pExitInfo) + return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo); + return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMREAD, VMXINSTRID_VMREAD, cbInstr); + } + + /* CPL. */ + if (pVCpu->iem.s.uCpl == 0) + { /* likely */ } + else + { + Log(("vmread: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_Cpl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + pVCpu->iem.s.cPotentialExits++; + + /* VMCS pointer in root mode. */ + if ( !IEM_VMX_IS_ROOT_MODE(pVCpu) + || IEM_VMX_HAS_CURRENT_VMCS(pVCpu)) + { /* likely */ } + else + { + Log(("vmread: VMCS pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_CURRENT_VMCS(pVCpu))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_PtrInvalid; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* VMCS-link pointer in non-root mode. */ + if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + || IEM_VMX_HAS_SHADOW_VMCS(pVCpu)) + { /* likely */ } + else + { + Log(("vmread: VMCS-link pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_SHADOW_VMCS(pVCpu))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_LinkPtrInvalid; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Supported VMCS field. */ + if (CPUMIsGuestVmxVmcsFieldValid(pVCpu->CTX_SUFF(pVM), u64VmcsField)) + { /* likely */ } + else + { + Log(("vmread: VMCS field %#RX64 invalid -> VMFail\n", u64VmcsField)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_FieldInvalid; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64VmcsField; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMREAD_INVALID_COMPONENT); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* + * Reading from the current or shadow VMCS. + */ + PCVMXVVMCS pVmcs = !IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + ? &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs + : &pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs; + iemVmxVmreadNoCheck(pVmcs, pu64Dst, u64VmcsField); + Log4(("vmread %#RX64 => %#RX64\n", u64VmcsField, *pu64Dst)); + return VINF_SUCCESS; +} + + +/** + * VMREAD (64-bit register) instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param pu64Dst Where to store the VMCS field's value. + * @param u64VmcsField The VMCS field. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be + * NULL. + */ +static VBOXSTRICTRC iemVmxVmreadReg64(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t *pu64Dst, + uint64_t u64VmcsField, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + VBOXSTRICTRC rcStrict = iemVmxVmreadCommon(pVCpu, cbInstr, pu64Dst, u64VmcsField, pExitInfo); + if (rcStrict == VINF_SUCCESS) + { + iemVmxVmSucceed(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + Log(("vmread/reg: iemVmxVmreadCommon failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +} + + +/** + * VMREAD (32-bit register) instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param pu32Dst Where to store the VMCS field's value. + * @param u32VmcsField The VMCS field. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be + * NULL. + */ +static VBOXSTRICTRC iemVmxVmreadReg32(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t *pu32Dst, + uint64_t u32VmcsField, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + uint64_t u64Dst; + VBOXSTRICTRC rcStrict = iemVmxVmreadCommon(pVCpu, cbInstr, &u64Dst, u32VmcsField, pExitInfo); + if (rcStrict == VINF_SUCCESS) + { + *pu32Dst = u64Dst; + iemVmxVmSucceed(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + Log(("vmread/reg: iemVmxVmreadCommon failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +} + + +/** + * VMREAD (memory) instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iEffSeg The effective segment register to use with @a u64Val. + * Pass UINT8_MAX if it is a register access. + * @param GCPtrDst The guest linear address to store the VMCS field's + * value. + * @param u64VmcsField The VMCS field. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be + * NULL. + */ +static VBOXSTRICTRC iemVmxVmreadMem(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDst, + uint64_t u64VmcsField, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + uint64_t u64Dst; + VBOXSTRICTRC rcStrict = iemVmxVmreadCommon(pVCpu, cbInstr, &u64Dst, u64VmcsField, pExitInfo); + if (rcStrict == VINF_SUCCESS) + { + /* + * Write the VMCS field's value to the location specified in guest-memory. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + rcStrict = iemMemStoreDataU64(pVCpu, iEffSeg, GCPtrDst, u64Dst); + else + rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrDst, u64Dst); + if (rcStrict == VINF_SUCCESS) + { + iemVmxVmSucceed(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + Log(("vmread/mem: Failed to write to memory operand at %#RGv, rc=%Rrc\n", GCPtrDst, VBOXSTRICTRC_VAL(rcStrict))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_PtrMap; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrDst; + return rcStrict; + } + + Log(("vmread/mem: iemVmxVmreadCommon failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +} + + +/** + * Interface for HM and EM to emulate the VMREAD instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmread(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + Assert(pExitInfo); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + VBOXSTRICTRC rcStrict; + uint8_t const cbInstr = pExitInfo->cbInstr; + bool const fIs64BitMode = RT_BOOL(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT); + uint64_t const u64FieldEnc = fIs64BitMode + ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2) + : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2); + if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand) + { + if (fIs64BitMode) + { + uint64_t *pu64Dst = iemGRegRefU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1); + rcStrict = iemVmxVmreadReg64(pVCpu, cbInstr, pu64Dst, u64FieldEnc, pExitInfo); + } + else + { + uint32_t *pu32Dst = iemGRegRefU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1); + rcStrict = iemVmxVmreadReg32(pVCpu, cbInstr, pu32Dst, u64FieldEnc, pExitInfo); + } + } + else + { + RTGCPTR const GCPtrDst = pExitInfo->GCPtrEffAddr; + uint8_t const iEffSeg = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg; + rcStrict = iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u64FieldEnc, pExitInfo); + } + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * VMWRITE instruction execution worker that does not perform any validation + * checks. + * + * Callers are expected to have performed the necessary checks and to ensure the + * VMWRITE will succeed. + * + * @param pVmcs Pointer to the virtual VMCS. + * @param u64Val The value to write. + * @param u64VmcsField The VMCS field. + * + * @remarks May be called with interrupts disabled. + */ +static void iemVmxVmwriteNoCheck(PVMXVVMCS pVmcs, uint64_t u64Val, uint64_t u64VmcsField) RT_NOEXCEPT +{ + VMXVMCSFIELD VmcsField; + VmcsField.u = u64VmcsField; + uint8_t const uWidth = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_WIDTH); + uint8_t const uType = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_TYPE); + uint8_t const uWidthType = (uWidth << 2) | uType; + uint8_t const uIndex = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_INDEX); + Assert(uIndex <= VMX_V_VMCS_MAX_INDEX); + uint16_t const offField = g_aoffVmcsMap[uWidthType][uIndex]; + Assert(offField < VMX_V_VMCS_SIZE); + AssertCompile(VMX_V_SHADOW_VMCS_SIZE == VMX_V_VMCS_SIZE); + + /* + * Write the VMCS component based on the field's effective width. + * + * The effective width is 64-bit fields adjusted to 32-bits if the access-type + * indicates high bits (little endian). + */ + uint8_t *pbVmcs = (uint8_t *)pVmcs; + uint8_t *pbField = pbVmcs + offField; + uint8_t const uEffWidth = VMXGetVmcsFieldWidthEff(VmcsField.u); + switch (uEffWidth) + { + case VMX_VMCSFIELD_WIDTH_64BIT: + case VMX_VMCSFIELD_WIDTH_NATURAL: *(uint64_t *)pbField = u64Val; break; + case VMX_VMCSFIELD_WIDTH_32BIT: *(uint32_t *)pbField = u64Val; break; + case VMX_VMCSFIELD_WIDTH_16BIT: *(uint16_t *)pbField = u64Val; break; + } +} + + +/** + * Interface for HM and EM to write a VMCS field in the nested-guest VMCS. + * + * It is ASSUMED the caller knows what they're doing. No VMWRITE instruction checks + * are performed. Bounds checks are strict builds only. + * + * @param pVmcs Pointer to the virtual VMCS. + * @param u64VmcsField The VMCS field. + * @param u64Val The value to write. + * + * @remarks May be called with interrupts disabled. + * @todo This should probably be moved to CPUM someday. + */ +VMM_INT_DECL(void) IEMWriteVmxVmcsField(PVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t u64Val) +{ + AssertPtr(pVmcs); + iemVmxVmwriteNoCheck(pVmcs, u64Val, u64VmcsField); +} + + +/** + * VMWRITE instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iEffSeg The effective segment register to use with @a u64Val. + * Pass UINT8_MAX if it is a register access. + * @param u64Val The value to write (or guest linear address to the + * value), @a iEffSeg will indicate if it's a memory + * operand. + * @param u64VmcsField The VMCS field. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be + * NULL. + */ +static VBOXSTRICTRC iemVmxVmwrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, uint64_t u64Val, + uint64_t u64VmcsField, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + /* Nested-guest intercept. */ + if ( IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + && CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, VMX_EXIT_VMWRITE, u64VmcsField)) + { + if (pExitInfo) + return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo); + return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMWRITE, VMXINSTRID_VMWRITE, cbInstr); + } + + /* CPL. */ + if (pVCpu->iem.s.uCpl == 0) + { /* likely */ } + else + { + Log(("vmwrite: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_Cpl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + pVCpu->iem.s.cPotentialExits++; + + /* VMCS pointer in root mode. */ + if ( !IEM_VMX_IS_ROOT_MODE(pVCpu) + || IEM_VMX_HAS_CURRENT_VMCS(pVCpu)) + { /* likely */ } + else + { + Log(("vmwrite: VMCS pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_CURRENT_VMCS(pVCpu))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_PtrInvalid; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* VMCS-link pointer in non-root mode. */ + if ( !IEM_VMX_IS_NON_ROOT_MODE(pVCpu) + || IEM_VMX_HAS_SHADOW_VMCS(pVCpu)) + { /* likely */ } + else + { + Log(("vmwrite: VMCS-link pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_SHADOW_VMCS(pVCpu))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_LinkPtrInvalid; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* If the VMWRITE instruction references memory, access the specified memory operand. */ + bool const fIsRegOperand = iEffSeg == UINT8_MAX; + if (!fIsRegOperand) + { + /* Read the value from the specified guest memory location. */ + VBOXSTRICTRC rcStrict; + RTGCPTR const GCPtrVal = u64Val; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + rcStrict = iemMemFetchDataU64(pVCpu, &u64Val, iEffSeg, GCPtrVal); + else + rcStrict = iemMemFetchDataU32_ZX_U64(pVCpu, &u64Val, iEffSeg, GCPtrVal); + if (RT_UNLIKELY(rcStrict != VINF_SUCCESS)) + { + Log(("vmwrite: Failed to read value from memory operand at %#RGv, rc=%Rrc\n", GCPtrVal, VBOXSTRICTRC_VAL(rcStrict))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_PtrMap; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVal; + return rcStrict; + } + } + else + Assert(!pExitInfo || pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand); + + /* Supported VMCS field. */ + if (CPUMIsGuestVmxVmcsFieldValid(pVCpu->CTX_SUFF(pVM), u64VmcsField)) + { /* likely */ } + else + { + Log(("vmwrite: VMCS field %#RX64 invalid -> VMFail\n", u64VmcsField)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_FieldInvalid; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64VmcsField; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMWRITE_INVALID_COMPONENT); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Read-only VMCS field. */ + bool const fIsFieldReadOnly = VMXIsVmcsFieldReadOnly(u64VmcsField); + if ( !fIsFieldReadOnly + || IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxVmwriteAll) + { /* likely */ } + else + { + Log(("vmwrite: Write to read-only VMCS component %#RX64 -> VMFail\n", u64VmcsField)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_FieldRo; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64VmcsField; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMWRITE_RO_COMPONENT); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* + * Write to the current or shadow VMCS. + */ + bool const fInVmxNonRootMode = IEM_VMX_IS_NON_ROOT_MODE(pVCpu); + PVMXVVMCS pVmcs = !fInVmxNonRootMode + ? &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs + : &pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs; + iemVmxVmwriteNoCheck(pVmcs, u64Val, u64VmcsField); + Log4(("vmwrite %#RX64 <= %#RX64\n", u64VmcsField, u64Val)); + + if ( !fInVmxNonRootMode + && VM_IS_HM_ENABLED(pVCpu->CTX_SUFF(pVM))) + { + /* Notify HM that the VMCS content might have changed. */ + HMNotifyVmxNstGstCurrentVmcsChanged(pVCpu); + } + + iemVmxVmSucceed(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Interface for HM and EM to emulate the VMWRITE instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmwrite(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + Assert(pExitInfo); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + uint64_t u64Val; + uint8_t iEffSeg; + if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand) + { + u64Val = iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1); + iEffSeg = UINT8_MAX; + } + else + { + u64Val = pExitInfo->GCPtrEffAddr; + iEffSeg = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg; + } + uint8_t const cbInstr = pExitInfo->cbInstr; + uint64_t const u64FieldEnc = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT + ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2) + : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2); + VBOXSTRICTRC rcStrict = iemVmxVmwrite(pVCpu, cbInstr, iEffSeg, u64Val, u64FieldEnc, pExitInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * VMCLEAR instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iEffSeg The effective segment register to use with @a GCPtrVmcs. + * @param GCPtrVmcs The linear address of the VMCS pointer. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be NULL. + * + * @remarks Common VMX instruction checks are already expected to by the caller, + * i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks. + */ +static VBOXSTRICTRC iemVmxVmclear(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, + RTGCPHYS GCPtrVmcs, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + /* Nested-guest intercept. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + if (pExitInfo) + return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo); + return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMCLEAR, VMXINSTRID_NONE, cbInstr); + } + + Assert(IEM_VMX_IS_ROOT_MODE(pVCpu)); + + /* CPL. */ + if (pVCpu->iem.s.uCpl == 0) + { /* likely */ } + else + { + Log(("vmclear: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmclear_Cpl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Get the VMCS pointer from the location specified by the source memory operand. */ + RTGCPHYS GCPhysVmcs; + VBOXSTRICTRC rcStrict = iemMemFetchDataU64(pVCpu, &GCPhysVmcs, iEffSeg, GCPtrVmcs); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else + { + Log(("vmclear: Failed to read VMCS physaddr from %#RGv, rc=%Rrc\n", GCPtrVmcs, VBOXSTRICTRC_VAL(rcStrict))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmclear_PtrMap; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmcs; + return rcStrict; + } + + /* VMCS pointer alignment. */ + if (!(GCPhysVmcs & X86_PAGE_4K_OFFSET_MASK)) + { /* likely */ } + else + { + Log(("vmclear: VMCS pointer not page-aligned -> VMFail()\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmclear_PtrAlign; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_INVALID_PHYSADDR); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* VMCS physical-address width limits. */ + if (!(GCPhysVmcs >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)) + { /* likely */ } + else + { + Log(("vmclear: VMCS pointer extends beyond physical-address width -> VMFail()\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmclear_PtrWidth; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_INVALID_PHYSADDR); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* VMCS is not the VMXON region. */ + if (GCPhysVmcs != pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon) + { /* likely */ } + else + { + Log(("vmclear: VMCS pointer cannot be identical to VMXON region pointer -> VMFail()\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmclear_PtrVmxon; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_VMXON_PTR); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Ensure VMCS is not MMIO, ROM etc. This is not an Intel requirement but a + restriction imposed by our implementation. */ + if (PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcs)) + { /* likely */ } + else + { + Log(("vmclear: VMCS not normal memory -> VMFail()\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmclear_PtrAbnormal; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_INVALID_PHYSADDR); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* + * VMCLEAR allows committing and clearing any valid VMCS pointer. + * + * If the current VMCS is the one being cleared, set its state to 'clear' and commit + * to guest memory. Otherwise, set the state of the VMCS referenced in guest memory + * to 'clear'. + */ + uint8_t const fVmcsLaunchStateClear = VMX_V_VMCS_LAUNCH_STATE_CLEAR; + if ( IEM_VMX_HAS_CURRENT_VMCS(pVCpu) + && IEM_VMX_GET_CURRENT_VMCS(pVCpu) == GCPhysVmcs) + { + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.fVmcsState = fVmcsLaunchStateClear; + iemVmxWriteCurrentVmcsToGstMem(pVCpu); + IEM_VMX_CLEAR_CURRENT_VMCS(pVCpu); + } + else + { + AssertCompileMemberSize(VMXVVMCS, fVmcsState, sizeof(fVmcsLaunchStateClear)); + rcStrict = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmcs + RT_UOFFSETOF(VMXVVMCS, fVmcsState), + (const void *)&fVmcsLaunchStateClear, sizeof(fVmcsLaunchStateClear)); + if (RT_FAILURE(rcStrict)) + return rcStrict; + } + + iemVmxVmSucceed(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Interface for HM and EM to emulate the VMCLEAR instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmclear(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + Assert(pExitInfo); + IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg; + uint8_t const cbInstr = pExitInfo->cbInstr; + RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr; + VBOXSTRICTRC rcStrict = iemVmxVmclear(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * VMPTRST instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iEffSeg The effective segment register to use with @a GCPtrVmcs. + * @param GCPtrVmcs The linear address of where to store the current VMCS + * pointer. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be NULL. + * + * @remarks Common VMX instruction checks are already expected to by the caller, + * i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks. + */ +static VBOXSTRICTRC iemVmxVmptrst(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, + RTGCPHYS GCPtrVmcs, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + /* Nested-guest intercept. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + if (pExitInfo) + return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo); + return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMPTRST, VMXINSTRID_NONE, cbInstr); + } + + Assert(IEM_VMX_IS_ROOT_MODE(pVCpu)); + + /* CPL. */ + if (pVCpu->iem.s.uCpl == 0) + { /* likely */ } + else + { + Log(("vmptrst: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrst_Cpl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Set the VMCS pointer to the location specified by the destination memory operand. */ + AssertCompile(NIL_RTGCPHYS == ~(RTGCPHYS)0U); + VBOXSTRICTRC rcStrict = iemMemStoreDataU64(pVCpu, iEffSeg, GCPtrVmcs, IEM_VMX_GET_CURRENT_VMCS(pVCpu)); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + iemVmxVmSucceed(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + Log(("vmptrst: Failed to store VMCS pointer to memory at destination operand %#Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrst_PtrMap; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmcs; + return rcStrict; +} + + +/** + * Interface for HM and EM to emulate the VMPTRST instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrst(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + Assert(pExitInfo); + IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg; + uint8_t const cbInstr = pExitInfo->cbInstr; + RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr; + VBOXSTRICTRC rcStrict = iemVmxVmptrst(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * VMPTRLD instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param GCPtrVmcs The linear address of the current VMCS pointer. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be NULL. + * + * @remarks Common VMX instruction checks are already expected to by the caller, + * i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks. + */ +static VBOXSTRICTRC iemVmxVmptrld(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, + RTGCPHYS GCPtrVmcs, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + /* Nested-guest intercept. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + if (pExitInfo) + return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo); + return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMPTRLD, VMXINSTRID_NONE, cbInstr); + } + + Assert(IEM_VMX_IS_ROOT_MODE(pVCpu)); + + /* CPL. */ + if (pVCpu->iem.s.uCpl == 0) + { /* likely */ } + else + { + Log(("vmptrld: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_Cpl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Get the VMCS pointer from the location specified by the source memory operand. */ + RTGCPHYS GCPhysVmcs; + VBOXSTRICTRC rcStrict = iemMemFetchDataU64(pVCpu, &GCPhysVmcs, iEffSeg, GCPtrVmcs); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else + { + Log(("vmptrld: Failed to read VMCS physaddr from %#RGv, rc=%Rrc\n", GCPtrVmcs, VBOXSTRICTRC_VAL(rcStrict))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_PtrMap; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmcs; + return rcStrict; + } + + /* VMCS pointer alignment. */ + if (!(GCPhysVmcs & X86_PAGE_4K_OFFSET_MASK)) + { /* likely */ } + else + { + Log(("vmptrld: VMCS pointer not page-aligned -> VMFail()\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_PtrAlign; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INVALID_PHYSADDR); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* VMCS physical-address width limits. */ + if (!(GCPhysVmcs >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)) + { /* likely */ } + else + { + Log(("vmptrld: VMCS pointer extends beyond physical-address width -> VMFail()\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_PtrWidth; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INVALID_PHYSADDR); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* VMCS is not the VMXON region. */ + if (GCPhysVmcs != pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon) + { /* likely */ } + else + { + Log(("vmptrld: VMCS pointer cannot be identical to VMXON region pointer -> VMFail()\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_PtrVmxon; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_VMXON_PTR); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Ensure VMCS is not MMIO, ROM etc. This is not an Intel requirement but a + restriction imposed by our implementation. */ + if (PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcs)) + { /* likely */ } + else + { + Log(("vmptrld: VMCS not normal memory -> VMFail()\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_PtrAbnormal; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INVALID_PHYSADDR); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Read just the VMCS revision from the VMCS. */ + VMXVMCSREVID VmcsRevId; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcsRevId, GCPhysVmcs, sizeof(VmcsRevId)); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + Log(("vmptrld: Failed to read revision identifier from VMCS at %#RGp, rc=%Rrc\n", GCPhysVmcs, rc)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_RevPtrReadPhys; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + return rc; + } + + /* + * Verify the VMCS revision specified by the guest matches what we reported to the guest. + * Verify the VMCS is not a shadow VMCS, if the VMCS shadowing feature is supported. + */ + if ( VmcsRevId.n.u31RevisionId == VMX_V_VMCS_REVISION_ID + && ( !VmcsRevId.n.fIsShadowVmcs + || IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxVmcsShadowing)) + { /* likely */ } + else + { + if (VmcsRevId.n.u31RevisionId != VMX_V_VMCS_REVISION_ID) + { + Log(("vmptrld: VMCS revision mismatch, expected %#RX32 got %#RX32, GCPtrVmcs=%#RGv GCPhysVmcs=%#RGp -> VMFail()\n", + VMX_V_VMCS_REVISION_ID, VmcsRevId.n.u31RevisionId, GCPtrVmcs, GCPhysVmcs)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_VmcsRevId; + } + else + { + Log(("vmptrld: Shadow VMCS -> VMFail()\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_ShadowVmcs; + } + iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INCORRECT_VMCS_REV); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* + * We cache only the current VMCS in CPUMCTX. Therefore, VMPTRLD should always flush + * the cache of an existing, current VMCS back to guest memory before loading a new, + * different current VMCS. + */ + if (IEM_VMX_GET_CURRENT_VMCS(pVCpu) != GCPhysVmcs) + { + if (IEM_VMX_HAS_CURRENT_VMCS(pVCpu)) + { + iemVmxWriteCurrentVmcsToGstMem(pVCpu); + IEM_VMX_CLEAR_CURRENT_VMCS(pVCpu); + } + + /* Set the new VMCS as the current VMCS and read it from guest memory. */ + IEM_VMX_SET_CURRENT_VMCS(pVCpu, GCPhysVmcs); + rc = iemVmxReadCurrentVmcsFromGstMem(pVCpu); + if (RT_SUCCESS(rc)) + { + /* Notify HM that a new, current VMCS is loaded. */ + if (VM_IS_HM_ENABLED(pVCpu->CTX_SUFF(pVM))) + HMNotifyVmxNstGstCurrentVmcsChanged(pVCpu); + } + else + { + Log(("vmptrld: Failed to read VMCS at %#RGp, rc=%Rrc\n", GCPhysVmcs, rc)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_PtrReadPhys; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs; + return rc; + } + } + + Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu)); + iemVmxVmSucceed(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Interface for HM and EM to emulate the VMPTRLD instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrld(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + Assert(pExitInfo); + IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg; + uint8_t const cbInstr = pExitInfo->cbInstr; + RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr; + VBOXSTRICTRC rcStrict = iemVmxVmptrld(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * INVVPID instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iEffSeg The segment of the invvpid descriptor. + * @param GCPtrInvvpidDesc The address of invvpid descriptor. + * @param u64InvvpidType The invalidation type. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be + * NULL. + * + * @remarks Common VMX instruction checks are already expected to by the caller, + * i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks. + */ +VBOXSTRICTRC iemVmxInvvpid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrInvvpidDesc, + uint64_t u64InvvpidType, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + /* Check if INVVPID instruction is supported, otherwise raise #UD. */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxVpid) + return iemRaiseUndefinedOpcode(pVCpu); + + /* Nested-guest intercept. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + if (pExitInfo) + return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo); + return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_INVVPID, VMXINSTRID_NONE, cbInstr); + } + + /* CPL. */ + if (pVCpu->iem.s.uCpl != 0) + { + Log(("invvpid: CPL != 0 -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Validate INVVPID invalidation type. + * + * The instruction specifies exactly ONE of the supported invalidation types. + * + * Each of the types has a bit in IA32_VMX_EPT_VPID_CAP MSR specifying if it is + * supported. In theory, it's possible for a CPU to not support flushing individual + * addresses but all the other types or any other combination. We do not take any + * shortcuts here by assuming the types we currently expose to the guest. + */ + uint64_t const fCaps = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64EptVpidCaps; + bool const fInvvpidSupported = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID); + bool const fTypeIndivAddr = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_INDIV_ADDR); + bool const fTypeSingleCtx = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_SINGLE_CTX); + bool const fTypeAllCtx = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_ALL_CTX); + bool const fTypeSingleCtxRetainGlobals = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_SINGLE_CTX_RETAIN_GLOBALS); + + bool afSupportedTypes[4]; + afSupportedTypes[0] = fTypeIndivAddr; + afSupportedTypes[1] = fTypeSingleCtx; + afSupportedTypes[2] = fTypeAllCtx; + afSupportedTypes[3] = fTypeSingleCtxRetainGlobals; + + if ( fInvvpidSupported + && !(u64InvvpidType & ~(uint64_t)VMX_INVVPID_VALID_MASK) + && afSupportedTypes[u64InvvpidType & 3]) + { /* likely */ } + else + { + Log(("invvpid: invalid/unsupported invvpid type %#x -> VMFail\n", u64InvvpidType)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Invvpid_TypeInvalid; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InvvpidType; + iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* + * Fetch the invvpid descriptor from guest memory. + */ + RTUINT128U uDesc; + VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInvvpidDesc); + if (rcStrict == VINF_SUCCESS) + { + /* + * Validate the descriptor. + */ + if (uDesc.s.Lo <= 0xffff) + { /* likely */ } + else + { + Log(("invvpid: reserved bits set in invvpid descriptor %#RX64 -> #GP(0)\n", uDesc.s.Lo)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Invvpid_DescRsvd; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = uDesc.s.Lo; + iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3); + RTGCUINTPTR64 const GCPtrInvAddr = uDesc.s.Hi; + uint16_t const uVpid = uDesc.Words.w0; + uint64_t const uCr3 = pVCpu->cpum.GstCtx.cr3; + switch (u64InvvpidType) + { + case VMXTLBFLUSHVPID_INDIV_ADDR: + { + if (uVpid != 0) + { + if (IEM_IS_CANONICAL(GCPtrInvAddr)) + { + /* Invalidate mappings for the linear address tagged with VPID. */ + /** @todo PGM support for VPID? Currently just flush everything. */ + PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */); + iemVmxVmSucceed(pVCpu); + } + else + { + Log(("invvpid: invalidation address %#RGP is not canonical -> VMFail\n", GCPtrInvAddr)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Invvpid_Type0InvalidAddr; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrInvAddr; + iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND); + } + } + else + { + Log(("invvpid: invalid VPID %#x for invalidation type %u -> VMFail\n", uVpid, u64InvvpidType)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Invvpid_Type0InvalidVpid; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InvvpidType; + iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND); + } + break; + } + + case VMXTLBFLUSHVPID_SINGLE_CONTEXT: + { + if (uVpid != 0) + { + /* Invalidate all mappings with VPID. */ + /** @todo PGM support for VPID? Currently just flush everything. */ + PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */); + iemVmxVmSucceed(pVCpu); + } + else + { + Log(("invvpid: invalid VPID %#x for invalidation type %u -> VMFail\n", uVpid, u64InvvpidType)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Invvpid_Type1InvalidVpid; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InvvpidType; + iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND); + } + break; + } + + case VMXTLBFLUSHVPID_ALL_CONTEXTS: + { + /* Invalidate all mappings with non-zero VPIDs. */ + /** @todo PGM support for VPID? Currently just flush everything. */ + PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */); + iemVmxVmSucceed(pVCpu); + break; + } + + case VMXTLBFLUSHVPID_SINGLE_CONTEXT_RETAIN_GLOBALS: + { + if (uVpid != 0) + { + /* Invalidate all mappings with VPID except global translations. */ + /** @todo PGM support for VPID? Currently just flush everything. */ + PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */); + iemVmxVmSucceed(pVCpu); + } + else + { + Log(("invvpid: invalid VPID %#x for invalidation type %u -> VMFail\n", uVpid, u64InvvpidType)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Invvpid_Type3InvalidVpid; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = uVpid; + iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND); + } + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + return rcStrict; +} + + +/** + * Interface for HM and EM to emulate the INVVPID instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvvpid(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 4); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + Assert(pExitInfo); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + uint8_t const iEffSeg = pExitInfo->InstrInfo.Inv.iSegReg; + uint8_t const cbInstr = pExitInfo->cbInstr; + RTGCPTR const GCPtrInvvpidDesc = pExitInfo->GCPtrEffAddr; + uint64_t const u64InvvpidType = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT + ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.Inv.iReg2) + : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.Inv.iReg2); + VBOXSTRICTRC rcStrict = iemVmxInvvpid(pVCpu, cbInstr, iEffSeg, GCPtrInvvpidDesc, u64InvvpidType, pExitInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + +/** + * INVEPT instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iEffSeg The segment of the invept descriptor. + * @param GCPtrInveptDesc The address of invept descriptor. + * @param u64InveptType The invalidation type. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be + * NULL. + * + * @remarks Common VMX instruction checks are already expected to by the caller, + * i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks. + */ +static VBOXSTRICTRC iemVmxInvept(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrInveptDesc, + uint64_t u64InveptType, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + /* Check if EPT is supported, otherwise raise #UD. */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxEpt) + return iemRaiseUndefinedOpcode(pVCpu); + + /* Nested-guest intercept. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + if (pExitInfo) + return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo); + return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_INVEPT, VMXINSTRID_NONE, cbInstr); + } + + /* CPL. */ + if (pVCpu->iem.s.uCpl != 0) + { + Log(("invept: CPL != 0 -> #GP(0)\n")); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* + * Validate INVEPT invalidation type. + * + * The instruction specifies exactly ONE of the supported invalidation types. + * + * Each of the types has a bit in IA32_VMX_EPT_VPID_CAP MSR specifying if it is + * supported. In theory, it's possible for a CPU to not support flushing individual + * addresses but all the other types or any other combination. We do not take any + * shortcuts here by assuming the types we currently expose to the guest. + */ + uint64_t const fCaps = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64EptVpidCaps; + bool const fInveptSupported = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVEPT); + bool const fTypeSingleCtx = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVEPT_SINGLE_CTX); + bool const fTypeAllCtx = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVEPT_ALL_CTX); + + bool afSupportedTypes[4]; + afSupportedTypes[0] = false; + afSupportedTypes[1] = fTypeSingleCtx; + afSupportedTypes[2] = fTypeAllCtx; + afSupportedTypes[3] = false; + + if ( fInveptSupported + && !(u64InveptType & ~(uint64_t)VMX_INVEPT_VALID_MASK) + && afSupportedTypes[u64InveptType & 3]) + { /* likely */ } + else + { + Log(("invept: invalid/unsupported invvpid type %#x -> VMFail\n", u64InveptType)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Invept_TypeInvalid; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InveptType; + iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* + * Fetch the invept descriptor from guest memory. + */ + RTUINT128U uDesc; + VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInveptDesc); + if (rcStrict == VINF_SUCCESS) + { + /* + * Validate the descriptor. + * + * The Intel spec. does not explicit say the INVEPT instruction fails when reserved + * bits in the descriptor are set, but it -does- for INVVPID. Until we test on real + * hardware, it's assumed INVEPT behaves the same as INVVPID in this regard. It's + * better to be strict in our emulation until proven otherwise. + */ + if (uDesc.s.Hi) + { + Log(("invept: reserved bits set in invept descriptor %#RX64 -> VMFail\n", uDesc.s.Hi)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Invept_DescRsvd; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = uDesc.s.Hi; + iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* + * Flush TLB mappings based on the EPT type. + */ + if (u64InveptType == VMXTLBFLUSHEPT_SINGLE_CONTEXT) + { + uint64_t const GCPhysEptPtr = uDesc.s.Lo; + int const rc = iemVmxVmentryCheckEptPtr(pVCpu, GCPhysEptPtr, NULL /* enmDiag */); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + Log(("invept: EPTP invalid %#RX64 -> VMFail\n", GCPhysEptPtr)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Invept_EptpInvalid; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysEptPtr; + iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + } + + /** @todo PGM support for EPT tags? Currently just flush everything. */ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3); + uint64_t const uCr3 = pVCpu->cpum.GstCtx.cr3; + PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */); + + iemVmxVmSucceed(pVCpu); + rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + return rcStrict; +} + + +/** + * Interface for HM and EM to emulate the INVEPT instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvept(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 4); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + Assert(pExitInfo); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + uint8_t const iEffSeg = pExitInfo->InstrInfo.Inv.iSegReg; + uint8_t const cbInstr = pExitInfo->cbInstr; + RTGCPTR const GCPtrInveptDesc = pExitInfo->GCPtrEffAddr; + uint64_t const u64InveptType = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT + ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.Inv.iReg2) + : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.Inv.iReg2); + VBOXSTRICTRC rcStrict = iemVmxInvept(pVCpu, cbInstr, iEffSeg, GCPtrInveptDesc, u64InveptType, pExitInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ + +/** + * VMXON instruction execution worker. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length in bytes. + * @param iEffSeg The effective segment register to use with @a + * GCPtrVmxon. + * @param GCPtrVmxon The linear address of the VMXON pointer. + * @param pExitInfo Pointer to the VM-exit information. Optional, can be NULL. + * + * @remarks Common VMX instruction checks are already expected to by the caller, + * i.e. CR4.VMXE, Real/V86 mode, EFER/CS.L checks. + */ +static VBOXSTRICTRC iemVmxVmxon(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, + RTGCPHYS GCPtrVmxon, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT +{ + if (!IEM_VMX_IS_ROOT_MODE(pVCpu)) + { + /* CPL. */ + if (pVCpu->iem.s.uCpl == 0) + { /* likely */ } + else + { + Log(("vmxon: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cpl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* A20M (A20 Masked) mode. */ + if (PGMPhysIsA20Enabled(pVCpu)) + { /* likely */ } + else + { + Log(("vmxon: A20M mode -> #GP(0)\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_A20M; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* CR0. */ + { + /* + * CR0 MB1 bits. + * + * We use VMX_V_CR0_FIXED0 below to ensure CR0.PE and CR0.PG are always set + * while executing VMXON. CR0.PE and CR0.PG are only allowed to be clear + * when the guest running in VMX non-root mode with unrestricted-guest control + * enabled in the VMCS. + */ + uint64_t const uCr0Fixed0 = VMX_V_CR0_FIXED0; + if ((pVCpu->cpum.GstCtx.cr0 & uCr0Fixed0) == uCr0Fixed0) + { /* likely */ } + else + { + Log(("vmxon: CR0 fixed0 bits cleared -> #GP(0)\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr0Fixed0; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* CR0 MBZ bits. */ + uint64_t const uCr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1; + if (!(pVCpu->cpum.GstCtx.cr0 & ~uCr0Fixed1)) + { /* likely */ } + else + { + Log(("vmxon: CR0 fixed1 bits set -> #GP(0)\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr0Fixed1; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + + /* CR4. */ + { + /* CR4 MB1 bits. */ + uint64_t const uCr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0; + if ((pVCpu->cpum.GstCtx.cr4 & uCr4Fixed0) == uCr4Fixed0) + { /* likely */ } + else + { + Log(("vmxon: CR4 fixed0 bits cleared -> #GP(0)\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr4Fixed0; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* CR4 MBZ bits. */ + uint64_t const uCr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1; + if (!(pVCpu->cpum.GstCtx.cr4 & ~uCr4Fixed1)) + { /* likely */ } + else + { + Log(("vmxon: CR4 fixed1 bits set -> #GP(0)\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr4Fixed1; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + } + + /* Feature control MSR's LOCK and VMXON bits. */ + uint64_t const uMsrFeatCtl = CPUMGetGuestIa32FeatCtrl(pVCpu); + if ((uMsrFeatCtl & (MSR_IA32_FEATURE_CONTROL_LOCK | MSR_IA32_FEATURE_CONTROL_VMXON)) + == (MSR_IA32_FEATURE_CONTROL_LOCK | MSR_IA32_FEATURE_CONTROL_VMXON)) + { /* likely */ } + else + { + Log(("vmxon: Feature control lock bit or VMXON bit cleared -> #GP(0)\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_MsrFeatCtl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Get the VMXON pointer from the location specified by the source memory operand. */ + RTGCPHYS GCPhysVmxon; + VBOXSTRICTRC rcStrict = iemMemFetchDataU64(pVCpu, &GCPhysVmxon, iEffSeg, GCPtrVmxon); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else + { + Log(("vmxon: Failed to read VMXON region physaddr from %#RGv, rc=%Rrc\n", GCPtrVmxon, VBOXSTRICTRC_VAL(rcStrict))); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_PtrMap; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmxon; + return rcStrict; + } + + /* VMXON region pointer alignment. */ + if (!(GCPhysVmxon & X86_PAGE_4K_OFFSET_MASK)) + { /* likely */ } + else + { + Log(("vmxon: VMXON region pointer not page-aligned -> VMFailInvalid\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_PtrAlign; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmxon; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* VMXON physical-address width limits. */ + if (!(GCPhysVmxon >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)) + { /* likely */ } + else + { + Log(("vmxon: VMXON region pointer extends beyond physical-address width -> VMFailInvalid\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_PtrWidth; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmxon; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Ensure VMXON region is not MMIO, ROM etc. This is not an Intel requirement but a + restriction imposed by our implementation. */ + if (PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmxon)) + { /* likely */ } + else + { + Log(("vmxon: VMXON region not normal memory -> VMFailInvalid\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_PtrAbnormal; + pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmxon; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Read the VMCS revision ID from the VMXON region. */ + VMXVMCSREVID VmcsRevId; + int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcsRevId, GCPhysVmxon, sizeof(VmcsRevId)); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + Log(("vmxon: Failed to read VMXON region at %#RGp, rc=%Rrc\n", GCPhysVmxon, rc)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_PtrReadPhys; + return rc; + } + + /* Verify the VMCS revision specified by the guest matches what we reported to the guest. */ + if (RT_LIKELY(VmcsRevId.u == VMX_V_VMCS_REVISION_ID)) + { /* likely */ } + else + { + /* Revision ID mismatch. */ + if (!VmcsRevId.n.fIsShadowVmcs) + { + Log(("vmxon: VMCS revision mismatch, expected %#RX32 got %#RX32 -> VMFailInvalid\n", VMX_V_VMCS_REVISION_ID, + VmcsRevId.n.u31RevisionId)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_VmcsRevId; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Shadow VMCS disallowed. */ + Log(("vmxon: Shadow VMCS -> VMFailInvalid\n")); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_ShadowVmcs; + iemVmxVmFailInvalid(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* + * Record that we're in VMX operation, block INIT, block and disable A20M. + */ + pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon = GCPhysVmxon; + IEM_VMX_CLEAR_CURRENT_VMCS(pVCpu); + pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxRootMode = true; + + /* Clear address-range monitoring. */ + EMMonitorWaitClear(pVCpu); + /** @todo NSTVMX: Intel PT. */ + + iemVmxVmSucceed(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + /* Nested-guest intercept. */ + if (pExitInfo) + return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo); + return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMXON, VMXINSTRID_NONE, cbInstr); + } + + Assert(IEM_VMX_IS_ROOT_MODE(pVCpu)); + + /* CPL. */ + if (pVCpu->iem.s.uCpl > 0) + { + Log(("vmxon: In VMX root mode: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_VmxRootCpl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* VMXON when already in VMX root mode. */ + iemVmxVmFail(pVCpu, VMXINSTRERR_VMXON_IN_VMXROOTMODE); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_VmxAlreadyRoot; + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Interface for HM and EM to emulate the VMXON instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pExitInfo Pointer to the VM-exit information. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxon(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) +{ + Assert(pExitInfo); + IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + + uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg; + uint8_t const cbInstr = pExitInfo->cbInstr; + RTGCPTR const GCPtrVmxon = pExitInfo->GCPtrEffAddr; + VBOXSTRICTRC rcStrict = iemVmxVmxon(pVCpu, cbInstr, iEffSeg, GCPtrVmxon, pExitInfo); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Implements 'VMXOFF'. + * + * @remarks Common VMX instruction checks are already expected to by the caller, + * i.e. CR4.VMXE, Real/V86 mode, EFER/CS.L checks. + */ +IEM_CIMPL_DEF_0(iemCImpl_vmxoff) +{ + /* Nested-guest intercept. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + return iemVmxVmexitInstr(pVCpu, VMX_EXIT_VMXOFF, cbInstr); + + /* CPL. */ + if (pVCpu->iem.s.uCpl == 0) + { /* likely */ } + else + { + Log(("vmxoff: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl)); + pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxoff_Cpl; + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + /* Dual monitor treatment of SMIs and SMM. */ + uint64_t const fSmmMonitorCtl = CPUMGetGuestIa32SmmMonitorCtl(pVCpu); + if (!(fSmmMonitorCtl & MSR_IA32_SMM_MONITOR_VALID)) + { /* likely */ } + else + { + iemVmxVmFail(pVCpu, VMXINSTRERR_VMXOFF_DUAL_MON); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); + } + + /* Record that we're no longer in VMX root operation, block INIT, block and disable A20M. */ + pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxRootMode = false; + Assert(!pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode); + + if (fSmmMonitorCtl & MSR_IA32_SMM_MONITOR_VMXOFF_UNBLOCK_SMI) + { /** @todo NSTVMX: Unblock SMI. */ } + + EMMonitorWaitClear(pVCpu); + /** @todo NSTVMX: Unblock and enable A20M. */ + + iemVmxVmSucceed(pVCpu); + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + + +/** + * Interface for HM and EM to emulate the VMXOFF instruction. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cbInstr The instruction length in bytes. + * @thread EMT(pVCpu) + */ +VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxoff(PVMCPUCC pVCpu, uint8_t cbInstr) +{ + IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3); + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + + iemInitExec(pVCpu, false /*fBypassHandlers*/); + VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmxoff); + Assert(!pVCpu->iem.s.cActiveMappings); + return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict); +} + + +/** + * Implements 'VMXON'. + */ +IEM_CIMPL_DEF_2(iemCImpl_vmxon, uint8_t, iEffSeg, RTGCPTR, GCPtrVmxon) +{ + return iemVmxVmxon(pVCpu, cbInstr, iEffSeg, GCPtrVmxon, NULL /* pExitInfo */); +} + + +/** + * Implements 'VMLAUNCH'. + */ +IEM_CIMPL_DEF_0(iemCImpl_vmlaunch) +{ + return iemVmxVmlaunchVmresume(pVCpu, cbInstr, VMXINSTRID_VMLAUNCH); +} + + +/** + * Implements 'VMRESUME'. + */ +IEM_CIMPL_DEF_0(iemCImpl_vmresume) +{ + return iemVmxVmlaunchVmresume(pVCpu, cbInstr, VMXINSTRID_VMRESUME); +} + + +/** + * Implements 'VMPTRLD'. + */ +IEM_CIMPL_DEF_2(iemCImpl_vmptrld, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs) +{ + return iemVmxVmptrld(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, NULL /* pExitInfo */); +} + + +/** + * Implements 'VMPTRST'. + */ +IEM_CIMPL_DEF_2(iemCImpl_vmptrst, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs) +{ + return iemVmxVmptrst(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, NULL /* pExitInfo */); +} + + +/** + * Implements 'VMCLEAR'. + */ +IEM_CIMPL_DEF_2(iemCImpl_vmclear, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs) +{ + return iemVmxVmclear(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, NULL /* pExitInfo */); +} + + +/** + * Implements 'VMWRITE' register. + */ +IEM_CIMPL_DEF_2(iemCImpl_vmwrite_reg, uint64_t, u64Val, uint64_t, u64VmcsField) +{ + return iemVmxVmwrite(pVCpu, cbInstr, UINT8_MAX /* iEffSeg */, u64Val, u64VmcsField, NULL /* pExitInfo */); +} + + +/** + * Implements 'VMWRITE' memory. + */ +IEM_CIMPL_DEF_3(iemCImpl_vmwrite_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrVal, uint32_t, u64VmcsField) +{ + return iemVmxVmwrite(pVCpu, cbInstr, iEffSeg, GCPtrVal, u64VmcsField, NULL /* pExitInfo */); +} + + +/** + * Implements 'VMREAD' register (64-bit). + */ +IEM_CIMPL_DEF_2(iemCImpl_vmread_reg64, uint64_t *, pu64Dst, uint64_t, u64VmcsField) +{ + return iemVmxVmreadReg64(pVCpu, cbInstr, pu64Dst, u64VmcsField, NULL /* pExitInfo */); +} + + +/** + * Implements 'VMREAD' register (32-bit). + */ +IEM_CIMPL_DEF_2(iemCImpl_vmread_reg32, uint32_t *, pu32Dst, uint32_t, u32VmcsField) +{ + return iemVmxVmreadReg32(pVCpu, cbInstr, pu32Dst, u32VmcsField, NULL /* pExitInfo */); +} + + +/** + * Implements 'VMREAD' memory, 64-bit register. + */ +IEM_CIMPL_DEF_3(iemCImpl_vmread_mem_reg64, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u64VmcsField) +{ + return iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u64VmcsField, NULL /* pExitInfo */); +} + + +/** + * Implements 'VMREAD' memory, 32-bit register. + */ +IEM_CIMPL_DEF_3(iemCImpl_vmread_mem_reg32, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u32VmcsField) +{ + return iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u32VmcsField, NULL /* pExitInfo */); +} + + +/** + * Implements 'INVVPID'. + */ +IEM_CIMPL_DEF_3(iemCImpl_invvpid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvvpidDesc, uint64_t, uInvvpidType) +{ + return iemVmxInvvpid(pVCpu, cbInstr, iEffSeg, GCPtrInvvpidDesc, uInvvpidType, NULL /* pExitInfo */); +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** + * Implements 'INVEPT'. + */ +IEM_CIMPL_DEF_3(iemCImpl_invept, uint8_t, iEffSeg, RTGCPTR, GCPtrInveptDesc, uint64_t, uInveptType) +{ + return iemVmxInvept(pVCpu, cbInstr, iEffSeg, GCPtrInveptDesc, uInveptType, NULL /* pExitInfo */); +} +#endif + + +/** + * Implements VMX's implementation of PAUSE. + */ +IEM_CIMPL_DEF_0(iemCImpl_vmx_pause) +{ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + { + VBOXSTRICTRC rcStrict = iemVmxVmexitInstrPause(pVCpu, cbInstr); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } + + /* + * Outside VMX non-root operation or if the PAUSE instruction does not cause + * a VM-exit, the instruction operates normally. + */ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr); +} + +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + + +/** + * Implements 'VMCALL'. + */ +IEM_CIMPL_DEF_0(iemCImpl_vmcall) +{ + pVCpu->iem.s.cPotentialExits++; + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* Nested-guest intercept. */ + if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu)) + return iemVmxVmexitInstr(pVCpu, VMX_EXIT_VMCALL, cbInstr); +#endif + + /* Join forces with vmmcall. */ + return IEM_CIMPL_CALL_1(iemCImpl_Hypercall, OP_VMCALL); +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + +/** + * @callback_method_impl{FNPGMPHYSHANDLER, VMX APIC-access page accesses} + * + * @remarks The @a uUser argument is currently unused. + */ +DECLCALLBACK(VBOXSTRICTRC) iemVmxApicAccessPageHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys, + void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, + PGMACCESSORIGIN enmOrigin, uint64_t uUser) +{ + RT_NOREF3(pvPhys, enmOrigin, uUser); + + RTGCPHYS const GCPhysAccessBase = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu))) + { + Assert(CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(pVCpu), VMX_PROC_CTLS2_VIRT_APIC_ACCESS)); + Assert(CPUMGetGuestVmxApicAccessPageAddrEx(IEM_GET_CTX(pVCpu)) == GCPhysAccessBase); + + uint32_t const fAccess = enmAccessType == PGMACCESSTYPE_WRITE ? IEM_ACCESS_DATA_W : IEM_ACCESS_DATA_R; + uint16_t const offAccess = GCPhysFault & GUEST_PAGE_OFFSET_MASK; + + LogFlowFunc(("Fault at %#RGp (cbBuf=%u fAccess=%#x)\n", GCPhysFault, cbBuf, fAccess)); + VBOXSTRICTRC rcStrict = iemVmxVirtApicAccessMem(pVCpu, offAccess, cbBuf, pvBuf, fAccess); + if (RT_FAILURE(rcStrict)) + return rcStrict; + + /* Any access on this APIC-access page has been handled, caller should not carry out the access. */ + return VINF_SUCCESS; + } + + LogFunc(("Accessed outside VMX non-root mode, deregistering page handler for %#RGp\n", GCPhysAccessBase)); + int rc = PGMHandlerPhysicalDeregister(pVM, GCPhysAccessBase); + if (RT_FAILURE(rc)) + return rc; + + /* Instruct the caller of this handler to perform the read/write as normal memory. */ + return VINF_PGM_HANDLER_DO_DEFAULT; +} + + +# ifndef IN_RING3 +/** + * @callback_method_impl{FNPGMRZPHYSPFHANDLER, + * \#PF access handler callback for guest VMX APIC-access page.} + */ +DECLCALLBACK(VBOXSTRICTRC) iemVmxApicAccessPagePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, + RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser) + +{ + RT_NOREF3(pVM, pCtx, uUser); + + /* + * Handle the VMX APIC-access page only when the guest is in VMX non-root mode. + * Otherwise we must deregister the page and allow regular RAM access. + * Failing to do so lands us with endless EPT VM-exits. + */ + RTGCPHYS const GCPhysPage = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu))) + { + Assert(CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(pVCpu), VMX_PROC_CTLS2_VIRT_APIC_ACCESS)); + Assert(CPUMGetGuestVmxApicAccessPageAddrEx(IEM_GET_CTX(pVCpu)) == GCPhysPage); + + /* + * Check if the access causes an APIC-access VM-exit. + */ + uint32_t fAccess; + if (uErr & X86_TRAP_PF_ID) + fAccess = IEM_ACCESS_INSTRUCTION; + else if (uErr & X86_TRAP_PF_RW) + fAccess = IEM_ACCESS_DATA_W; + else + fAccess = IEM_ACCESS_DATA_R; + + RTGCPHYS const GCPhysNestedFault = (RTGCPHYS)pvFault; + uint16_t const offAccess = GCPhysNestedFault & GUEST_PAGE_OFFSET_MASK; + bool const fIntercept = iemVmxVirtApicIsMemAccessIntercepted(pVCpu, offAccess, 1 /* cbAccess */, fAccess); + LogFlowFunc(("#PF at %#RGp (GCPhysNestedFault=%#RGp offAccess=%#x)\n", GCPhysFault, GCPhysNestedFault, offAccess)); + if (fIntercept) + { + /* + * Query the source VM-exit (from the execution engine) that caused this access + * within the APIC-access page. Currently only HM is supported. + */ + AssertMsg(VM_IS_HM_ENABLED(pVM), + ("VM-exit auxiliary info. fetching not supported for execution engine %d\n", pVM->bMainExecutionEngine)); + + HMEXITAUX HmExitAux; + RT_ZERO(HmExitAux); + int const rc = HMR0GetExitAuxInfo(pVCpu, &HmExitAux, HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE); + AssertRC(rc); + + /* + * Verify the VM-exit reason must be an EPT violation. + * Other accesses should go through the other handler (iemVmxApicAccessPageHandler). + * Refer to @bugref{10092#c33s} for a more detailed explanation. + */ + AssertMsgReturn(HmExitAux.Vmx.uReason == VMX_EXIT_EPT_VIOLATION, + ("Unexpected call to APIC-access page #PF handler for %#RGp offAcesss=%u uErr=%#RGx uReason=%u\n", + GCPhysPage, offAccess, uErr, HmExitAux.Vmx.uReason), VERR_IEM_IPE_7); + + /* + * Construct the virtual APIC-access VM-exit. + */ + VMXAPICACCESS enmAccess; + if (HmExitAux.Vmx.u64Qual & VMX_EXIT_QUAL_EPT_LINEAR_ADDR_VALID) + { + if (VMX_IDT_VECTORING_INFO_IS_VALID(HmExitAux.Vmx.uIdtVectoringInfo)) + enmAccess = VMXAPICACCESS_LINEAR_EVENT_DELIVERY; + else if (fAccess == IEM_ACCESS_INSTRUCTION) + enmAccess = VMXAPICACCESS_LINEAR_INSTR_FETCH; + else if (fAccess & IEM_ACCESS_TYPE_WRITE) + enmAccess = VMXAPICACCESS_LINEAR_WRITE; + else + enmAccess = VMXAPICACCESS_LINEAR_READ; + + /* For linear-address accesss the instruction length must be valid. */ + AssertMsg(HmExitAux.Vmx.cbInstr > 0, + ("Invalid APIC-access VM-exit instruction length. cbInstr=%u\n", HmExitAux.Vmx.cbInstr)); + } + else + { + if (VMX_IDT_VECTORING_INFO_IS_VALID(HmExitAux.Vmx.uIdtVectoringInfo)) + enmAccess = VMXAPICACCESS_PHYSICAL_EVENT_DELIVERY; + else + { + /** @todo How to distinguish between monitoring/trace vs other instructions + * here? */ + enmAccess = VMXAPICACCESS_PHYSICAL_INSTR; + } + + /* For physical accesses the instruction length is undefined, we zero it for safety and consistency. */ + HmExitAux.Vmx.cbInstr = 0; + } + + /* + * Raise the APIC-access VM-exit. + */ + LogFlowFunc(("Raising APIC-access VM-exit from #PF handler at offset %#x\n", offAccess)); + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_APIC_ACCESS, + RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_OFFSET, offAccess) + | RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_TYPE, enmAccess), + HmExitAux.Vmx.cbInstr); + VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(HmExitAux.Vmx.uIdtVectoringInfo, + HmExitAux.Vmx.uIdtVectoringErrCode); + VBOXSTRICTRC const rcStrict = iemVmxVmexitApicAccessWithInfo(pVCpu, &ExitInfo, &ExitEventInfo); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); + } + + /* + * The access isn't intercepted, which means it needs to be virtualized. + * + * This requires emulating the instruction because we need the bytes being + * read/written by the instruction not just the offset being accessed within + * the APIC-access page (which we derive from the faulting address). + */ + LogFlowFunc(("Access at offset %#x not intercepted -> VINF_EM_RAW_EMULATE_INSTR\n", offAccess)); + return VINF_EM_RAW_EMULATE_INSTR; + } + + /** @todo This isn't ideal but works for now as nested-hypervisors generally play + * nice because the spec states that this page should be modified only when + * no CPU refers to it VMX non-root mode. Nonetheless, we could use an atomic + * reference counter to ensure the aforementioned condition before + * de-registering the page. */ + LogFunc(("Accessed outside VMX non-root mode, deregistering page handler for %#RGp\n", GCPhysPage)); + int const rc = PGMHandlerPhysicalDeregister(pVM, GCPhysPage); + if (RT_FAILURE(rc)) + return rc; + + return VINF_SUCCESS; +} +# endif /* !IN_RING3 */ + +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h new file mode 100644 index 00000000..bf186b94 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h @@ -0,0 +1,143 @@ +/* $Id: IEMAllInstructions3DNow.cpp.h $ */ +/** @file + * IEM - Instruction Decoding and Emulation, 3DNow!. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/** @name 3DNow! instructions (0x0f 0x0f) + * + * @{ + */ + +/** Opcode 0x0f 0x0f 0x0c. */ +FNIEMOP_STUB(iemOp_3Dnow_pi2fw_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x0d. */ +FNIEMOP_STUB(iemOp_3Dnow_pi2fd_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x1c. */ +FNIEMOP_STUB(iemOp_3Dnow_pf2fw_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x1d. */ +FNIEMOP_STUB(iemOp_3Dnow_pf2fd_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x8a. */ +FNIEMOP_STUB(iemOp_3Dnow_pfnacc_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x8e. */ +FNIEMOP_STUB(iemOp_3Dnow_pfpnacc_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x90. */ +FNIEMOP_STUB(iemOp_3Dnow_pfcmpge_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x94. */ +FNIEMOP_STUB(iemOp_3Dnow_pfmin_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x96. */ +FNIEMOP_STUB(iemOp_3Dnow_pfrcp_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x97. */ +FNIEMOP_STUB(iemOp_3Dnow_pfrsqrt_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x9a. */ +FNIEMOP_STUB(iemOp_3Dnow_pfsub_Pq_Qq); + +/** Opcode 0x0f 0x0f 0x9e. */ +FNIEMOP_STUB(iemOp_3Dnow_pfadd_PQ_Qq); + +/** Opcode 0x0f 0x0f 0xa0. */ +FNIEMOP_STUB(iemOp_3Dnow_pfcmpgt_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xa4. */ +FNIEMOP_STUB(iemOp_3Dnow_pfmax_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xa6. */ +FNIEMOP_STUB(iemOp_3Dnow_pfrcpit1_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xa7. */ +FNIEMOP_STUB(iemOp_3Dnow_pfrsqit1_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xaa. */ +FNIEMOP_STUB(iemOp_3Dnow_pfsubr_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xae. */ +FNIEMOP_STUB(iemOp_3Dnow_pfacc_PQ_Qq); + +/** Opcode 0x0f 0x0f 0xb0. */ +FNIEMOP_STUB(iemOp_3Dnow_pfcmpeq_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xb4. */ +FNIEMOP_STUB(iemOp_3Dnow_pfmul_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xb6. */ +FNIEMOP_STUB(iemOp_3Dnow_pfrcpit2_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xb7. */ +FNIEMOP_STUB(iemOp_3Dnow_pmulhrw_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xbb. */ +FNIEMOP_STUB(iemOp_3Dnow_pswapd_Pq_Qq); + +/** Opcode 0x0f 0x0f 0xbf. */ +FNIEMOP_STUB(iemOp_3Dnow_pavgusb_PQ_Qq); + + +/** Opcode 0x0f 0x0f. */ +FNIEMOP_DEF_1(iemOp_3DNowDispatcher, uint8_t, b) +{ + /* This is pretty sparse, use switch instead of table. */ + switch (b) + { + case 0x0c: return FNIEMOP_CALL(iemOp_3Dnow_pi2fw_Pq_Qq); + case 0x0d: return FNIEMOP_CALL(iemOp_3Dnow_pi2fd_Pq_Qq); + case 0x1c: return FNIEMOP_CALL(iemOp_3Dnow_pf2fw_Pq_Qq); + case 0x1d: return FNIEMOP_CALL(iemOp_3Dnow_pf2fd_Pq_Qq); + case 0x8a: return FNIEMOP_CALL(iemOp_3Dnow_pfnacc_Pq_Qq); + case 0x8e: return FNIEMOP_CALL(iemOp_3Dnow_pfpnacc_Pq_Qq); + case 0x90: return FNIEMOP_CALL(iemOp_3Dnow_pfcmpge_Pq_Qq); + case 0x94: return FNIEMOP_CALL(iemOp_3Dnow_pfmin_Pq_Qq); + case 0x96: return FNIEMOP_CALL(iemOp_3Dnow_pfrcp_Pq_Qq); + case 0x97: return FNIEMOP_CALL(iemOp_3Dnow_pfrsqrt_Pq_Qq); + case 0x9a: return FNIEMOP_CALL(iemOp_3Dnow_pfsub_Pq_Qq); + case 0x9e: return FNIEMOP_CALL(iemOp_3Dnow_pfadd_PQ_Qq); + case 0xa0: return FNIEMOP_CALL(iemOp_3Dnow_pfcmpgt_Pq_Qq); + case 0xa4: return FNIEMOP_CALL(iemOp_3Dnow_pfmax_Pq_Qq); + case 0xa6: return FNIEMOP_CALL(iemOp_3Dnow_pfrcpit1_Pq_Qq); + case 0xa7: return FNIEMOP_CALL(iemOp_3Dnow_pfrsqit1_Pq_Qq); + case 0xaa: return FNIEMOP_CALL(iemOp_3Dnow_pfsubr_Pq_Qq); + case 0xae: return FNIEMOP_CALL(iemOp_3Dnow_pfacc_PQ_Qq); + case 0xb0: return FNIEMOP_CALL(iemOp_3Dnow_pfcmpeq_Pq_Qq); + case 0xb4: return FNIEMOP_CALL(iemOp_3Dnow_pfmul_Pq_Qq); + case 0xb6: return FNIEMOP_CALL(iemOp_3Dnow_pfrcpit2_Pq_Qq); + case 0xb7: return FNIEMOP_CALL(iemOp_3Dnow_pmulhrw_Pq_Qq); + case 0xbb: return FNIEMOP_CALL(iemOp_3Dnow_pswapd_Pq_Qq); + case 0xbf: return FNIEMOP_CALL(iemOp_3Dnow_pavgusb_PQ_Qq); + default: + return IEMOP_RAISE_INVALID_OPCODE(); + } +} + +/** @} */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsInterpretOnly.cpp b/src/VBox/VMM/VMMAll/IEMAllInstructionsInterpretOnly.cpp new file mode 100644 index 00000000..06a2ded9 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsInterpretOnly.cpp @@ -0,0 +1,1678 @@ +/* $Id: IEMAllInstructionsInterpretOnly.cpp $ */ +/** @file + * IEM - Instruction Decoding and Emulation. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#ifndef LOG_GROUP /* defined when included by tstIEMCheckMc.cpp */ +# define LOG_GROUP LOG_GROUP_IEM +#endif +#define VMCPU_INCL_CPUM_GST_CTX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +# include +# include +#endif +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +# include +#endif +#include +#include +#include +#ifndef TST_IEM_CHECK_MC +# include "IEMInternal.h" +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef TST_IEM_CHECK_MC +# include "IEMInline.h" +# include "IEMOpHlp.h" +# include "IEMMc.h" +#endif + + +#ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable: 4702) /* Unreachable code like return in iemOp_Grp6_lldt. */ +#endif + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +#ifndef TST_IEM_CHECK_MC +/** Function table for the ADD instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_add = +{ + iemAImpl_add_u8, iemAImpl_add_u8_locked, + iemAImpl_add_u16, iemAImpl_add_u16_locked, + iemAImpl_add_u32, iemAImpl_add_u32_locked, + iemAImpl_add_u64, iemAImpl_add_u64_locked +}; + +/** Function table for the ADC instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_adc = +{ + iemAImpl_adc_u8, iemAImpl_adc_u8_locked, + iemAImpl_adc_u16, iemAImpl_adc_u16_locked, + iemAImpl_adc_u32, iemAImpl_adc_u32_locked, + iemAImpl_adc_u64, iemAImpl_adc_u64_locked +}; + +/** Function table for the SUB instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sub = +{ + iemAImpl_sub_u8, iemAImpl_sub_u8_locked, + iemAImpl_sub_u16, iemAImpl_sub_u16_locked, + iemAImpl_sub_u32, iemAImpl_sub_u32_locked, + iemAImpl_sub_u64, iemAImpl_sub_u64_locked +}; + +/** Function table for the SBB instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sbb = +{ + iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked, + iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked, + iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked, + iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked +}; + +/** Function table for the OR instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_or = +{ + iemAImpl_or_u8, iemAImpl_or_u8_locked, + iemAImpl_or_u16, iemAImpl_or_u16_locked, + iemAImpl_or_u32, iemAImpl_or_u32_locked, + iemAImpl_or_u64, iemAImpl_or_u64_locked +}; + +/** Function table for the XOR instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_xor = +{ + iemAImpl_xor_u8, iemAImpl_xor_u8_locked, + iemAImpl_xor_u16, iemAImpl_xor_u16_locked, + iemAImpl_xor_u32, iemAImpl_xor_u32_locked, + iemAImpl_xor_u64, iemAImpl_xor_u64_locked +}; + +/** Function table for the AND instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_and = +{ + iemAImpl_and_u8, iemAImpl_and_u8_locked, + iemAImpl_and_u16, iemAImpl_and_u16_locked, + iemAImpl_and_u32, iemAImpl_and_u32_locked, + iemAImpl_and_u64, iemAImpl_and_u64_locked +}; + +/** Function table for the CMP instruction. + * @remarks Making operand order ASSUMPTIONS. + */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_cmp = +{ + iemAImpl_cmp_u8, NULL, + iemAImpl_cmp_u16, NULL, + iemAImpl_cmp_u32, NULL, + iemAImpl_cmp_u64, NULL +}; + +/** Function table for the TEST instruction. + * @remarks Making operand order ASSUMPTIONS. + */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_test = +{ + iemAImpl_test_u8, NULL, + iemAImpl_test_u16, NULL, + iemAImpl_test_u32, NULL, + iemAImpl_test_u64, NULL +}; + + +/** Function table for the BT instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bt = +{ + NULL, NULL, + iemAImpl_bt_u16, NULL, + iemAImpl_bt_u32, NULL, + iemAImpl_bt_u64, NULL +}; + +/** Function table for the BTC instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btc = +{ + NULL, NULL, + iemAImpl_btc_u16, iemAImpl_btc_u16_locked, + iemAImpl_btc_u32, iemAImpl_btc_u32_locked, + iemAImpl_btc_u64, iemAImpl_btc_u64_locked +}; + +/** Function table for the BTR instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btr = +{ + NULL, NULL, + iemAImpl_btr_u16, iemAImpl_btr_u16_locked, + iemAImpl_btr_u32, iemAImpl_btr_u32_locked, + iemAImpl_btr_u64, iemAImpl_btr_u64_locked +}; + +/** Function table for the BTS instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bts = +{ + NULL, NULL, + iemAImpl_bts_u16, iemAImpl_bts_u16_locked, + iemAImpl_bts_u32, iemAImpl_bts_u32_locked, + iemAImpl_bts_u64, iemAImpl_bts_u64_locked +}; + +/** Function table for the BSF instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf = +{ + NULL, NULL, + iemAImpl_bsf_u16, NULL, + iemAImpl_bsf_u32, NULL, + iemAImpl_bsf_u64, NULL +}; + +/** Function table for the BSF instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf_amd = +{ + NULL, NULL, + iemAImpl_bsf_u16_amd, NULL, + iemAImpl_bsf_u32_amd, NULL, + iemAImpl_bsf_u64_amd, NULL +}; + +/** Function table for the BSF instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf_intel = +{ + NULL, NULL, + iemAImpl_bsf_u16_intel, NULL, + iemAImpl_bsf_u32_intel, NULL, + iemAImpl_bsf_u64_intel, NULL +}; + +/** EFLAGS variation selection table for the BSF instruction. */ +IEM_STATIC const IEMOPBINSIZES * const g_iemAImpl_bsf_eflags[] = +{ + &g_iemAImpl_bsf, + &g_iemAImpl_bsf_intel, + &g_iemAImpl_bsf_amd, + &g_iemAImpl_bsf, +}; + +/** Function table for the BSR instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr = +{ + NULL, NULL, + iemAImpl_bsr_u16, NULL, + iemAImpl_bsr_u32, NULL, + iemAImpl_bsr_u64, NULL +}; + +/** Function table for the BSR instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr_amd = +{ + NULL, NULL, + iemAImpl_bsr_u16_amd, NULL, + iemAImpl_bsr_u32_amd, NULL, + iemAImpl_bsr_u64_amd, NULL +}; + +/** Function table for the BSR instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr_intel = +{ + NULL, NULL, + iemAImpl_bsr_u16_intel, NULL, + iemAImpl_bsr_u32_intel, NULL, + iemAImpl_bsr_u64_intel, NULL +}; + +/** EFLAGS variation selection table for the BSR instruction. */ +IEM_STATIC const IEMOPBINSIZES * const g_iemAImpl_bsr_eflags[] = +{ + &g_iemAImpl_bsr, + &g_iemAImpl_bsr_intel, + &g_iemAImpl_bsr_amd, + &g_iemAImpl_bsr, +}; + +/** Function table for the IMUL instruction. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two = +{ + NULL, NULL, + iemAImpl_imul_two_u16, NULL, + iemAImpl_imul_two_u32, NULL, + iemAImpl_imul_two_u64, NULL +}; + +/** Function table for the IMUL instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two_amd = +{ + NULL, NULL, + iemAImpl_imul_two_u16_amd, NULL, + iemAImpl_imul_two_u32_amd, NULL, + iemAImpl_imul_two_u64_amd, NULL +}; + +/** Function table for the IMUL instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two_intel = +{ + NULL, NULL, + iemAImpl_imul_two_u16_intel, NULL, + iemAImpl_imul_two_u32_intel, NULL, + iemAImpl_imul_two_u64_intel, NULL +}; + +/** EFLAGS variation selection table for the IMUL instruction. */ +IEM_STATIC const IEMOPBINSIZES * const g_iemAImpl_imul_two_eflags[] = +{ + &g_iemAImpl_imul_two, + &g_iemAImpl_imul_two_intel, + &g_iemAImpl_imul_two_amd, + &g_iemAImpl_imul_two, +}; + +/** EFLAGS variation selection table for the 16-bit IMUL instruction. */ +IEM_STATIC PFNIEMAIMPLBINU16 const g_iemAImpl_imul_two_u16_eflags[] = +{ + iemAImpl_imul_two_u16, + iemAImpl_imul_two_u16_intel, + iemAImpl_imul_two_u16_amd, + iemAImpl_imul_two_u16, +}; + +/** EFLAGS variation selection table for the 32-bit IMUL instruction. */ +IEM_STATIC PFNIEMAIMPLBINU32 const g_iemAImpl_imul_two_u32_eflags[] = +{ + iemAImpl_imul_two_u32, + iemAImpl_imul_two_u32_intel, + iemAImpl_imul_two_u32_amd, + iemAImpl_imul_two_u32, +}; + +/** EFLAGS variation selection table for the 64-bit IMUL instruction. */ +IEM_STATIC PFNIEMAIMPLBINU64 const g_iemAImpl_imul_two_u64_eflags[] = +{ + iemAImpl_imul_two_u64, + iemAImpl_imul_two_u64_intel, + iemAImpl_imul_two_u64_amd, + iemAImpl_imul_two_u64, +}; + +/** Group 1 /r lookup table. */ +IEM_STATIC const PCIEMOPBINSIZES g_apIemImplGrp1[8] = +{ + &g_iemAImpl_add, + &g_iemAImpl_or, + &g_iemAImpl_adc, + &g_iemAImpl_sbb, + &g_iemAImpl_and, + &g_iemAImpl_sub, + &g_iemAImpl_xor, + &g_iemAImpl_cmp +}; + +/** Function table for the INC instruction. */ +IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_inc = +{ + iemAImpl_inc_u8, iemAImpl_inc_u8_locked, + iemAImpl_inc_u16, iemAImpl_inc_u16_locked, + iemAImpl_inc_u32, iemAImpl_inc_u32_locked, + iemAImpl_inc_u64, iemAImpl_inc_u64_locked +}; + +/** Function table for the DEC instruction. */ +IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_dec = +{ + iemAImpl_dec_u8, iemAImpl_dec_u8_locked, + iemAImpl_dec_u16, iemAImpl_dec_u16_locked, + iemAImpl_dec_u32, iemAImpl_dec_u32_locked, + iemAImpl_dec_u64, iemAImpl_dec_u64_locked +}; + +/** Function table for the NEG instruction. */ +IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_neg = +{ + iemAImpl_neg_u8, iemAImpl_neg_u8_locked, + iemAImpl_neg_u16, iemAImpl_neg_u16_locked, + iemAImpl_neg_u32, iemAImpl_neg_u32_locked, + iemAImpl_neg_u64, iemAImpl_neg_u64_locked +}; + +/** Function table for the NOT instruction. */ +IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_not = +{ + iemAImpl_not_u8, iemAImpl_not_u8_locked, + iemAImpl_not_u16, iemAImpl_not_u16_locked, + iemAImpl_not_u32, iemAImpl_not_u32_locked, + iemAImpl_not_u64, iemAImpl_not_u64_locked +}; + + +/** Function table for the ROL instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol = +{ + iemAImpl_rol_u8, + iemAImpl_rol_u16, + iemAImpl_rol_u32, + iemAImpl_rol_u64 +}; + +/** Function table for the ROL instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol_amd = +{ + iemAImpl_rol_u8_amd, + iemAImpl_rol_u16_amd, + iemAImpl_rol_u32_amd, + iemAImpl_rol_u64_amd +}; + +/** Function table for the ROL instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol_intel = +{ + iemAImpl_rol_u8_intel, + iemAImpl_rol_u16_intel, + iemAImpl_rol_u32_intel, + iemAImpl_rol_u64_intel +}; + +/** EFLAGS variation selection table for the ROL instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_rol_eflags[] = +{ + &g_iemAImpl_rol, + &g_iemAImpl_rol_intel, + &g_iemAImpl_rol_amd, + &g_iemAImpl_rol, +}; + + +/** Function table for the ROR instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror = +{ + iemAImpl_ror_u8, + iemAImpl_ror_u16, + iemAImpl_ror_u32, + iemAImpl_ror_u64 +}; + +/** Function table for the ROR instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror_amd = +{ + iemAImpl_ror_u8_amd, + iemAImpl_ror_u16_amd, + iemAImpl_ror_u32_amd, + iemAImpl_ror_u64_amd +}; + +/** Function table for the ROR instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror_intel = +{ + iemAImpl_ror_u8_intel, + iemAImpl_ror_u16_intel, + iemAImpl_ror_u32_intel, + iemAImpl_ror_u64_intel +}; + +/** EFLAGS variation selection table for the ROR instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_ror_eflags[] = +{ + &g_iemAImpl_ror, + &g_iemAImpl_ror_intel, + &g_iemAImpl_ror_amd, + &g_iemAImpl_ror, +}; + + +/** Function table for the RCL instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl = +{ + iemAImpl_rcl_u8, + iemAImpl_rcl_u16, + iemAImpl_rcl_u32, + iemAImpl_rcl_u64 +}; + +/** Function table for the RCL instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl_amd = +{ + iemAImpl_rcl_u8_amd, + iemAImpl_rcl_u16_amd, + iemAImpl_rcl_u32_amd, + iemAImpl_rcl_u64_amd +}; + +/** Function table for the RCL instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl_intel = +{ + iemAImpl_rcl_u8_intel, + iemAImpl_rcl_u16_intel, + iemAImpl_rcl_u32_intel, + iemAImpl_rcl_u64_intel +}; + +/** EFLAGS variation selection table for the RCL instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_rcl_eflags[] = +{ + &g_iemAImpl_rcl, + &g_iemAImpl_rcl_intel, + &g_iemAImpl_rcl_amd, + &g_iemAImpl_rcl, +}; + + +/** Function table for the RCR instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr = +{ + iemAImpl_rcr_u8, + iemAImpl_rcr_u16, + iemAImpl_rcr_u32, + iemAImpl_rcr_u64 +}; + +/** Function table for the RCR instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr_amd = +{ + iemAImpl_rcr_u8_amd, + iemAImpl_rcr_u16_amd, + iemAImpl_rcr_u32_amd, + iemAImpl_rcr_u64_amd +}; + +/** Function table for the RCR instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr_intel = +{ + iemAImpl_rcr_u8_intel, + iemAImpl_rcr_u16_intel, + iemAImpl_rcr_u32_intel, + iemAImpl_rcr_u64_intel +}; + +/** EFLAGS variation selection table for the RCR instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_rcr_eflags[] = +{ + &g_iemAImpl_rcr, + &g_iemAImpl_rcr_intel, + &g_iemAImpl_rcr_amd, + &g_iemAImpl_rcr, +}; + + +/** Function table for the SHL instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl = +{ + iemAImpl_shl_u8, + iemAImpl_shl_u16, + iemAImpl_shl_u32, + iemAImpl_shl_u64 +}; + +/** Function table for the SHL instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl_amd = +{ + iemAImpl_shl_u8_amd, + iemAImpl_shl_u16_amd, + iemAImpl_shl_u32_amd, + iemAImpl_shl_u64_amd +}; + +/** Function table for the SHL instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl_intel = +{ + iemAImpl_shl_u8_intel, + iemAImpl_shl_u16_intel, + iemAImpl_shl_u32_intel, + iemAImpl_shl_u64_intel +}; + +/** EFLAGS variation selection table for the SHL instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_shl_eflags[] = +{ + &g_iemAImpl_shl, + &g_iemAImpl_shl_intel, + &g_iemAImpl_shl_amd, + &g_iemAImpl_shl, +}; + + +/** Function table for the SHR instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr = +{ + iemAImpl_shr_u8, + iemAImpl_shr_u16, + iemAImpl_shr_u32, + iemAImpl_shr_u64 +}; + +/** Function table for the SHR instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr_amd = +{ + iemAImpl_shr_u8_amd, + iemAImpl_shr_u16_amd, + iemAImpl_shr_u32_amd, + iemAImpl_shr_u64_amd +}; + +/** Function table for the SHR instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr_intel = +{ + iemAImpl_shr_u8_intel, + iemAImpl_shr_u16_intel, + iemAImpl_shr_u32_intel, + iemAImpl_shr_u64_intel +}; + +/** EFLAGS variation selection table for the SHR instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_shr_eflags[] = +{ + &g_iemAImpl_shr, + &g_iemAImpl_shr_intel, + &g_iemAImpl_shr_amd, + &g_iemAImpl_shr, +}; + + +/** Function table for the SAR instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar = +{ + iemAImpl_sar_u8, + iemAImpl_sar_u16, + iemAImpl_sar_u32, + iemAImpl_sar_u64 +}; + +/** Function table for the SAR instruction, AMD EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar_amd = +{ + iemAImpl_sar_u8_amd, + iemAImpl_sar_u16_amd, + iemAImpl_sar_u32_amd, + iemAImpl_sar_u64_amd +}; + +/** Function table for the SAR instruction, Intel EFLAGS variant. */ +IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar_intel = +{ + iemAImpl_sar_u8_intel, + iemAImpl_sar_u16_intel, + iemAImpl_sar_u32_intel, + iemAImpl_sar_u64_intel +}; + +/** EFLAGS variation selection table for the SAR instruction. */ +IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_sar_eflags[] = +{ + &g_iemAImpl_sar, + &g_iemAImpl_sar_intel, + &g_iemAImpl_sar_amd, + &g_iemAImpl_sar, +}; + + +/** Function table for the MUL instruction. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul = +{ + iemAImpl_mul_u8, + iemAImpl_mul_u16, + iemAImpl_mul_u32, + iemAImpl_mul_u64 +}; + +/** Function table for the MUL instruction, AMD EFLAGS variation. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul_amd = +{ + iemAImpl_mul_u8_amd, + iemAImpl_mul_u16_amd, + iemAImpl_mul_u32_amd, + iemAImpl_mul_u64_amd +}; + +/** Function table for the MUL instruction, Intel EFLAGS variation. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul_intel = +{ + iemAImpl_mul_u8_intel, + iemAImpl_mul_u16_intel, + iemAImpl_mul_u32_intel, + iemAImpl_mul_u64_intel +}; + +/** EFLAGS variation selection table for the MUL instruction. */ +IEM_STATIC const IEMOPMULDIVSIZES * const g_iemAImpl_mul_eflags[] = +{ + &g_iemAImpl_mul, + &g_iemAImpl_mul_intel, + &g_iemAImpl_mul_amd, + &g_iemAImpl_mul, +}; + +/** EFLAGS variation selection table for the 8-bit MUL instruction. */ +IEM_STATIC PFNIEMAIMPLMULDIVU8 const g_iemAImpl_mul_u8_eflags[] = +{ + iemAImpl_mul_u8, + iemAImpl_mul_u8_intel, + iemAImpl_mul_u8_amd, + iemAImpl_mul_u8 +}; + + +/** Function table for the IMUL instruction working implicitly on rAX. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul = +{ + iemAImpl_imul_u8, + iemAImpl_imul_u16, + iemAImpl_imul_u32, + iemAImpl_imul_u64 +}; + +/** Function table for the IMUL instruction working implicitly on rAX, AMD EFLAGS variation. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul_amd = +{ + iemAImpl_imul_u8_amd, + iemAImpl_imul_u16_amd, + iemAImpl_imul_u32_amd, + iemAImpl_imul_u64_amd +}; + +/** Function table for the IMUL instruction working implicitly on rAX, Intel EFLAGS variation. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul_intel = +{ + iemAImpl_imul_u8_intel, + iemAImpl_imul_u16_intel, + iemAImpl_imul_u32_intel, + iemAImpl_imul_u64_intel +}; + +/** EFLAGS variation selection table for the IMUL instruction. */ +IEM_STATIC const IEMOPMULDIVSIZES * const g_iemAImpl_imul_eflags[] = +{ + &g_iemAImpl_imul, + &g_iemAImpl_imul_intel, + &g_iemAImpl_imul_amd, + &g_iemAImpl_imul, +}; + +/** EFLAGS variation selection table for the 8-bit IMUL instruction. */ +IEM_STATIC PFNIEMAIMPLMULDIVU8 const g_iemAImpl_imul_u8_eflags[] = +{ + iemAImpl_imul_u8, + iemAImpl_imul_u8_intel, + iemAImpl_imul_u8_amd, + iemAImpl_imul_u8 +}; + + +/** Function table for the DIV instruction. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div = +{ + iemAImpl_div_u8, + iemAImpl_div_u16, + iemAImpl_div_u32, + iemAImpl_div_u64 +}; + +/** Function table for the DIV instruction, AMD EFLAGS variation. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div_amd = +{ + iemAImpl_div_u8_amd, + iemAImpl_div_u16_amd, + iemAImpl_div_u32_amd, + iemAImpl_div_u64_amd +}; + +/** Function table for the DIV instruction, Intel EFLAGS variation. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div_intel = +{ + iemAImpl_div_u8_intel, + iemAImpl_div_u16_intel, + iemAImpl_div_u32_intel, + iemAImpl_div_u64_intel +}; + +/** EFLAGS variation selection table for the DIV instruction. */ +IEM_STATIC const IEMOPMULDIVSIZES * const g_iemAImpl_div_eflags[] = +{ + &g_iemAImpl_div, + &g_iemAImpl_div_intel, + &g_iemAImpl_div_amd, + &g_iemAImpl_div, +}; + +/** EFLAGS variation selection table for the 8-bit DIV instruction. */ +IEM_STATIC PFNIEMAIMPLMULDIVU8 const g_iemAImpl_div_u8_eflags[] = +{ + iemAImpl_div_u8, + iemAImpl_div_u8_intel, + iemAImpl_div_u8_amd, + iemAImpl_div_u8 +}; + + +/** Function table for the IDIV instruction. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv = +{ + iemAImpl_idiv_u8, + iemAImpl_idiv_u16, + iemAImpl_idiv_u32, + iemAImpl_idiv_u64 +}; + +/** Function table for the IDIV instruction, AMD EFLAGS variation. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv_amd = +{ + iemAImpl_idiv_u8_amd, + iemAImpl_idiv_u16_amd, + iemAImpl_idiv_u32_amd, + iemAImpl_idiv_u64_amd +}; + +/** Function table for the IDIV instruction, Intel EFLAGS variation. */ +IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv_intel = +{ + iemAImpl_idiv_u8_intel, + iemAImpl_idiv_u16_intel, + iemAImpl_idiv_u32_intel, + iemAImpl_idiv_u64_intel +}; + +/** EFLAGS variation selection table for the IDIV instruction. */ +IEM_STATIC const IEMOPMULDIVSIZES * const g_iemAImpl_idiv_eflags[] = +{ + &g_iemAImpl_idiv, + &g_iemAImpl_idiv_intel, + &g_iemAImpl_idiv_amd, + &g_iemAImpl_idiv, +}; + +/** EFLAGS variation selection table for the 8-bit IDIV instruction. */ +IEM_STATIC PFNIEMAIMPLMULDIVU8 const g_iemAImpl_idiv_u8_eflags[] = +{ + iemAImpl_idiv_u8, + iemAImpl_idiv_u8_intel, + iemAImpl_idiv_u8_amd, + iemAImpl_idiv_u8 +}; + + +/** Function table for the SHLD instruction. */ +IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld = +{ + iemAImpl_shld_u16, + iemAImpl_shld_u32, + iemAImpl_shld_u64, +}; + +/** Function table for the SHLD instruction, AMD EFLAGS variation. */ +IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld_amd = +{ + iemAImpl_shld_u16_amd, + iemAImpl_shld_u32_amd, + iemAImpl_shld_u64_amd +}; + +/** Function table for the SHLD instruction, Intel EFLAGS variation. */ +IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld_intel = +{ + iemAImpl_shld_u16_intel, + iemAImpl_shld_u32_intel, + iemAImpl_shld_u64_intel +}; + +/** EFLAGS variation selection table for the SHLD instruction. */ +IEM_STATIC const IEMOPSHIFTDBLSIZES * const g_iemAImpl_shld_eflags[] = +{ + &g_iemAImpl_shld, + &g_iemAImpl_shld_intel, + &g_iemAImpl_shld_amd, + &g_iemAImpl_shld +}; + +/** Function table for the SHRD instruction. */ +IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd = +{ + iemAImpl_shrd_u16, + iemAImpl_shrd_u32, + iemAImpl_shrd_u64 +}; + +/** Function table for the SHRD instruction, AMD EFLAGS variation. */ +IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd_amd = +{ + iemAImpl_shrd_u16_amd, + iemAImpl_shrd_u32_amd, + iemAImpl_shrd_u64_amd +}; + +/** Function table for the SHRD instruction, Intel EFLAGS variation. */ +IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd_intel = +{ + iemAImpl_shrd_u16_intel, + iemAImpl_shrd_u32_intel, + iemAImpl_shrd_u64_intel +}; + +/** EFLAGS variation selection table for the SHRD instruction. */ +IEM_STATIC const IEMOPSHIFTDBLSIZES * const g_iemAImpl_shrd_eflags[] = +{ + &g_iemAImpl_shrd, + &g_iemAImpl_shrd_intel, + &g_iemAImpl_shrd_amd, + &g_iemAImpl_shrd +}; + + +# ifndef IEM_WITHOUT_ASSEMBLY +/** Function table for the VPXOR instruction */ +IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpand = { iemAImpl_vpand_u128, iemAImpl_vpand_u256 }; +/** Function table for the VPXORN instruction */ +IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpandn = { iemAImpl_vpandn_u128, iemAImpl_vpandn_u256 }; +/** Function table for the VPOR instruction */ +IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpor = { iemAImpl_vpor_u128, iemAImpl_vpor_u256 }; +/** Function table for the VPXOR instruction */ +IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpxor = { iemAImpl_vpxor_u128, iemAImpl_vpxor_u256 }; +# endif + +/** Function table for the VPAND instruction, software fallback. */ +IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpand_fallback = { iemAImpl_vpand_u128_fallback, iemAImpl_vpand_u256_fallback }; +/** Function table for the VPANDN instruction, software fallback. */ +IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpandn_fallback= { iemAImpl_vpandn_u128_fallback, iemAImpl_vpandn_u256_fallback }; +/** Function table for the VPOR instruction, software fallback. */ +IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpor_fallback = { iemAImpl_vpor_u128_fallback, iemAImpl_vpor_u256_fallback }; +/** Function table for the VPXOR instruction, software fallback. */ +IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpxor_fallback = { iemAImpl_vpxor_u128_fallback, iemAImpl_vpxor_u256_fallback }; + +#endif /* !TST_IEM_CHECK_MC */ + + +/** + * Common worker for instructions like ADD, AND, OR, ++ with a byte + * memory/register as the destination. + * + * @param pImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rm_r8, PCIEMOPBINSIZES, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t, u8Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * We're accessing memory. + * Note! We're putting the eflags on the stack here so we can commit them + * after the memory. + */ + uint32_t const fAccess = pImpl->pfnLockedU8 ? IEM_ACCESS_DATA_RW : IEM_ACCESS_DATA_R; /* CMP,TEST */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t, u8Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + if (!pImpl->pfnLockedU8) + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu8Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU8, pu8Dst, u8Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for word/dword/qword instructions like ADD, AND, OR, ++ with + * memory/register as the destination. + * + * @param pImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rm_rv, PCIEMOPBINSIZES, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + + if ((pImpl != &g_iemAImpl_test) && (pImpl != &g_iemAImpl_cmp)) + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're accessing memory. + * Note! We're putting the eflags on the stack here so we can commit them + * after the memory. + */ + uint32_t const fAccess = pImpl->pfnLockedU8 ? IEM_ACCESS_DATA_RW : IEM_ACCESS_DATA_R /* CMP,TEST */; + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + if (!pImpl->pfnLockedU16) + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + if (!pImpl->pfnLockedU32) + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + if (!pImpl->pfnLockedU64) + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * Common worker for byte instructions like ADD, AND, OR, ++ with a register as + * the destination. + * + * @param pImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpBinaryOperator_r8_rm, PCIEMOPBINSIZES, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t, u8Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * We're accessing memory. + */ + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t, u8Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8(u8Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for word/dword/qword instructions like ADD, AND, OR, ++ with a + * register as the destination. + * + * @param pImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rv_rm, PCIEMOPBINSIZES, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + + if (pImpl != &g_iemAImpl_cmp) /* Not used with TEST. */ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're accessing memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + + if (pImpl != &g_iemAImpl_cmp) + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * Common worker for instructions like ADD, AND, OR, ++ with working on AL with + * a byte immediate. + * + * @param pImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpBinaryOperator_AL_Ib, PCIEMOPBINSIZES, pImpl) +{ + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG_CONST(uint8_t, u8Src,/*=*/ u8Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U8(pu8Dst, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * Common worker for instructions like ADD, AND, OR, ++ with working on + * AX/EAX/RAX with a word/dword immediate. + * + * @param pImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rAX_Iz, PCIEMOPBINSIZES, pImpl) +{ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint16_t, u16Src,/*=*/ u16Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U16(pu16Dst, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + } + + case IEMMODE_32BIT: + { + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint32_t, u32Src,/*=*/ u32Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U32(pu32Dst, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + + if ((pImpl != &g_iemAImpl_test) && (pImpl != &g_iemAImpl_cmp)) + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + } + + case IEMMODE_64BIT: + { + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint64_t, u64Src,/*=*/ u64Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U64(pu64Dst, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** Opcodes 0xf1, 0xd6. */ +FNIEMOP_DEF(iemOp_Invalid) +{ + IEMOP_MNEMONIC(Invalid, "Invalid"); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid with RM byte . */ +FNIEMOPRM_DEF(iemOp_InvalidWithRM) +{ + RT_NOREF_PV(bRm); + IEMOP_MNEMONIC(InvalidWithRm, "InvalidWithRM"); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid with RM byte where intel decodes any additional address encoding + * bytes. */ +FNIEMOPRM_DEF(iemOp_InvalidWithRMNeedDecode) +{ + IEMOP_MNEMONIC(InvalidWithRMNeedDecode, "InvalidWithRMNeedDecode"); + if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL) + { +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + } + IEMOP_HLP_DONE_DECODING(); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid with RM byte where both AMD and Intel decodes any additional + * address encoding bytes. */ +FNIEMOPRM_DEF(iemOp_InvalidWithRMAllNeeded) +{ + IEMOP_MNEMONIC(InvalidWithRMAllNeeded, "InvalidWithRMAllNeeded"); +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + IEMOP_HLP_DONE_DECODING(); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid with RM byte where intel requires 8-byte immediate. + * Intel will also need SIB and displacement if bRm indicates memory. */ +FNIEMOPRM_DEF(iemOp_InvalidWithRMNeedImm8) +{ + IEMOP_MNEMONIC(InvalidWithRMNeedImm8, "InvalidWithRMNeedImm8"); + if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL) + { +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + uint8_t bImm8; IEM_OPCODE_GET_NEXT_U8(&bImm8); RT_NOREF(bRm); + } + IEMOP_HLP_DONE_DECODING(); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid with RM byte where intel requires 8-byte immediate. + * Both AMD and Intel also needs SIB and displacement according to bRm. */ +FNIEMOPRM_DEF(iemOp_InvalidWithRMAllNeedImm8) +{ + IEMOP_MNEMONIC(InvalidWithRMAllNeedImm8, "InvalidWithRMAllNeedImm8"); +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + uint8_t bImm8; IEM_OPCODE_GET_NEXT_U8(&bImm8); RT_NOREF(bRm); + IEMOP_HLP_DONE_DECODING(); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid opcode where intel requires Mod R/M sequence. */ +FNIEMOP_DEF(iemOp_InvalidNeedRM) +{ + IEMOP_MNEMONIC(InvalidNeedRM, "InvalidNeedRM"); + if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL) + { + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm); +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + } + IEMOP_HLP_DONE_DECODING(); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid opcode where both AMD and Intel requires Mod R/M sequence. */ +FNIEMOP_DEF(iemOp_InvalidAllNeedRM) +{ + IEMOP_MNEMONIC(InvalidAllNeedRM, "InvalidAllNeedRM"); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm); +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + IEMOP_HLP_DONE_DECODING(); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid opcode where intel requires Mod R/M sequence and 8-byte + * immediate. */ +FNIEMOP_DEF(iemOp_InvalidNeedRMImm8) +{ + IEMOP_MNEMONIC(InvalidNeedRMImm8, "InvalidNeedRMImm8"); + if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL) + { + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm); +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); RT_NOREF(bImm); + } + IEMOP_HLP_DONE_DECODING(); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid opcode where intel requires a 3rd escape byte and a Mod R/M + * sequence. */ +FNIEMOP_DEF(iemOp_InvalidNeed3ByteEscRM) +{ + IEMOP_MNEMONIC(InvalidNeed3ByteEscRM, "InvalidNeed3ByteEscRM"); + if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL) + { + uint8_t b3rd; IEM_OPCODE_GET_NEXT_U8(&b3rd); RT_NOREF(b3rd); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm); +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + } + IEMOP_HLP_DONE_DECODING(); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Invalid opcode where intel requires a 3rd escape byte, Mod R/M sequence, and + * a 8-byte immediate. */ +FNIEMOP_DEF(iemOp_InvalidNeed3ByteEscRMImm8) +{ + IEMOP_MNEMONIC(InvalidNeed3ByteEscRMImm8, "InvalidNeed3ByteEscRMImm8"); + if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL) + { + uint8_t b3rd; IEM_OPCODE_GET_NEXT_U8(&b3rd); RT_NOREF(b3rd); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm); +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 1, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); RT_NOREF(bImm); + IEMOP_HLP_DONE_DECODING(); + } + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Repeats a_fn four times. For decoding tables. */ +#define IEMOP_X4(a_fn) a_fn, a_fn, a_fn, a_fn + +/* + * Include the tables. + */ +#ifdef IEM_WITH_3DNOW +# include "IEMAllInstructions3DNow.cpp.h" +#endif +#ifdef IEM_WITH_THREE_0F_38 +# include "IEMAllInstructionsThree0f38.cpp.h" +#endif +#ifdef IEM_WITH_THREE_0F_3A +# include "IEMAllInstructionsThree0f3a.cpp.h" +#endif +#include "IEMAllInstructionsTwoByte0f.cpp.h" +#ifdef IEM_WITH_VEX +# include "IEMAllInstructionsVexMap1.cpp.h" +# include "IEMAllInstructionsVexMap2.cpp.h" +# include "IEMAllInstructionsVexMap3.cpp.h" +#endif +#include "IEMAllInstructionsOneByte.cpp.h" + + +#ifdef _MSC_VER +# pragma warning(pop) +#endif + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h new file mode 100644 index 00000000..8c181e54 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h @@ -0,0 +1,11860 @@ +/* $Id: IEMAllInstructionsOneByte.cpp.h $ */ +/** @file + * IEM - Instruction Decoding and Emulation. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/******************************************************************************* +* Global Variables * +*******************************************************************************/ +extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */ + +/* Instruction group definitions: */ + +/** @defgroup og_gen General + * @{ */ + /** @defgroup og_gen_arith Arithmetic + * @{ */ + /** @defgroup og_gen_arith_bin Binary numbers */ + /** @defgroup og_gen_arith_dec Decimal numbers */ + /** @} */ +/** @} */ + +/** @defgroup og_stack Stack + * @{ */ + /** @defgroup og_stack_sreg Segment registers */ +/** @} */ + +/** @defgroup og_prefix Prefixes */ +/** @defgroup og_escapes Escape bytes */ + + + +/** @name One byte opcodes. + * @{ + */ + +/* Instruction specification format - work in progress: */ + +/** + * @opcode 0x00 + * @opmnemonic add + * @op1 rm:Eb + * @op2 reg:Gb + * @opmaps one + * @openc ModR/M + * @opflmodify cf,pf,af,zf,sf,of + * @ophints harmless ignores_op_sizes + * @opstats add_Eb_Gb + * @opgroup og_gen_arith_bin + * @optest op1=1 op2=1 -> op1=2 efl&|=nc,pe,na,nz,pl,nv + * @optest efl|=cf op1=1 op2=2 -> op1=3 efl&|=nc,po,na,nz,pl,nv + * @optest op1=254 op2=1 -> op1=255 efl&|=nc,po,na,nz,ng,nv + * @optest op1=128 op2=128 -> op1=0 efl&|=ov,pl,zf,na,po,cf + */ +FNIEMOP_DEF(iemOp_add_Eb_Gb) +{ + IEMOP_MNEMONIC2(MR, ADD, add, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_add); +} + + +/** + * @opcode 0x01 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @optest op1=1 op2=1 -> op1=2 efl&|=nc,pe,na,nz,pl,nv + * @optest efl|=cf op1=2 op2=2 -> op1=4 efl&|=nc,pe,na,nz,pl,nv + * @optest efl&~=cf op1=-1 op2=1 -> op1=0 efl&|=cf,po,af,zf,pl,nv + * @optest op1=-1 op2=-1 -> op1=-2 efl&|=cf,pe,af,nz,ng,nv + */ +FNIEMOP_DEF(iemOp_add_Ev_Gv) +{ + IEMOP_MNEMONIC2(MR, ADD, add, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_add); +} + + +/** + * @opcode 0x02 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opcopytests iemOp_add_Eb_Gb + */ +FNIEMOP_DEF(iemOp_add_Gb_Eb) +{ + IEMOP_MNEMONIC2(RM, ADD, add, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_add); +} + + +/** + * @opcode 0x03 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opcopytests iemOp_add_Ev_Gv + */ +FNIEMOP_DEF(iemOp_add_Gv_Ev) +{ + IEMOP_MNEMONIC2(RM, ADD, add, Gv, Ev, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_add); +} + + +/** + * @opcode 0x04 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opcopytests iemOp_add_Eb_Gb + */ +FNIEMOP_DEF(iemOp_add_Al_Ib) +{ + IEMOP_MNEMONIC2(FIXED, ADD, add, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_add); +} + + +/** + * @opcode 0x05 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @optest op1=1 op2=1 -> op1=2 efl&|=nv,pl,nz,na,pe + * @optest efl|=cf op1=2 op2=2 -> op1=4 efl&|=nc,pe,na,nz,pl,nv + * @optest efl&~=cf op1=-1 op2=1 -> op1=0 efl&|=cf,po,af,zf,pl,nv + * @optest op1=-1 op2=-1 -> op1=-2 efl&|=cf,pe,af,nz,ng,nv + */ +FNIEMOP_DEF(iemOp_add_eAX_Iz) +{ + IEMOP_MNEMONIC2(FIXED, ADD, add, rAX, Iz, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_add); +} + + +/** + * @opcode 0x06 + * @opgroup og_stack_sreg + */ +FNIEMOP_DEF(iemOp_push_ES) +{ + IEMOP_MNEMONIC1(FIXED, PUSH, push, ES, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); + IEMOP_HLP_NO_64BIT(); + return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_ES); +} + + +/** + * @opcode 0x07 + * @opgroup og_stack_sreg + */ +FNIEMOP_DEF(iemOp_pop_ES) +{ + IEMOP_MNEMONIC1(FIXED, POP, pop, ES, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); + IEMOP_HLP_NO_64BIT(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_ES, pVCpu->iem.s.enmEffOpSize); +} + + +/** + * @opcode 0x08 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + * @optest op1=7 op2=12 -> op1=15 efl&|=nc,po,na,nz,pl,nv + * @optest efl|=of,cf op1=0 op2=0 -> op1=0 efl&|=nc,po,na,zf,pl,nv + * @optest op1=0xee op2=0x11 -> op1=0xff efl&|=nc,po,na,nz,ng,nv + * @optest op1=0xff op2=0xff -> op1=0xff efl&|=nc,po,na,nz,ng,nv + */ +FNIEMOP_DEF(iemOp_or_Eb_Gb) +{ + IEMOP_MNEMONIC2(MR, OR, or, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_or); +} + + +/* + * @opcode 0x09 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + * @optest efl|=of,cf op1=12 op2=7 -> op1=15 efl&|=nc,po,na,nz,pl,nv + * @optest efl|=of,cf op1=0 op2=0 -> op1=0 efl&|=nc,po,na,zf,pl,nv + * @optest op1=-2 op2=1 -> op1=-1 efl&|=nc,po,na,nz,ng,nv + * @optest o16 / op1=0x5a5a op2=0xa5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv + * @optest o32 / op1=0x5a5a5a5a op2=0xa5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv + * @optest o64 / op1=0x5a5a5a5a5a5a5a5a op2=0xa5a5a5a5a5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv + */ +FNIEMOP_DEF(iemOp_or_Ev_Gv) +{ + IEMOP_MNEMONIC2(MR, OR, or, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_or); +} + + +/** + * @opcode 0x0a + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + * @opcopytests iemOp_or_Eb_Gb + */ +FNIEMOP_DEF(iemOp_or_Gb_Eb) +{ + IEMOP_MNEMONIC2(RM, OR, or, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_or); +} + + +/** + * @opcode 0x0b + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + * @opcopytests iemOp_or_Ev_Gv + */ +FNIEMOP_DEF(iemOp_or_Gv_Ev) +{ + IEMOP_MNEMONIC2(RM, OR, or, Gv, Ev, DISOPTYPE_HARMLESS, 0); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_or); +} + + +/** + * @opcode 0x0c + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + * @opcopytests iemOp_or_Eb_Gb + */ +FNIEMOP_DEF(iemOp_or_Al_Ib) +{ + IEMOP_MNEMONIC2(FIXED, OR, or, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_or); +} + + +/** + * @opcode 0x0d + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + * @optest efl|=of,cf op1=12 op2=7 -> op1=15 efl&|=nc,po,na,nz,pl,nv + * @optest efl|=of,cf op1=0 op2=0 -> op1=0 efl&|=nc,po,na,zf,pl,nv + * @optest op1=-2 op2=1 -> op1=-1 efl&|=nc,po,na,nz,ng,nv + * @optest o16 / op1=0x5a5a op2=0xa5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv + * @optest o32 / op1=0x5a5a5a5a op2=0xa5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv + * @optest o64 / op1=0x5a5a5a5a5a5a5a5a op2=0xa5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv + * @optest o64 / op1=0x5a5a5a5aa5a5a5a5 op2=0x5a5a5a5a -> op1=0x5a5a5a5affffffff efl&|=nc,po,na,nz,pl,nv + */ +FNIEMOP_DEF(iemOp_or_eAX_Iz) +{ + IEMOP_MNEMONIC2(FIXED, OR, or, rAX, Iz, DISOPTYPE_HARMLESS, 0); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_or); +} + + +/** + * @opcode 0x0e + * @opgroup og_stack_sreg + */ +FNIEMOP_DEF(iemOp_push_CS) +{ + IEMOP_MNEMONIC1(FIXED, PUSH, push, CS, DISOPTYPE_HARMLESS | DISOPTYPE_POTENTIALLY_DANGEROUS | DISOPTYPE_INVALID_64, 0); + IEMOP_HLP_NO_64BIT(); + return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_CS); +} + + +/** + * @opcode 0x0f + * @opmnemonic EscTwo0f + * @openc two0f + * @opdisenum OP_2B_ESC + * @ophints harmless + * @opgroup og_escapes + */ +FNIEMOP_DEF(iemOp_2byteEscape) +{ +#ifdef VBOX_STRICT + /* Sanity check the table the first time around. */ + static bool s_fTested = false; + if (RT_LIKELY(s_fTested)) { /* likely */ } + else + { + s_fTested = true; + Assert(g_apfnTwoByteMap[0xbc * 4 + 0] == iemOp_bsf_Gv_Ev); + Assert(g_apfnTwoByteMap[0xbc * 4 + 1] == iemOp_bsf_Gv_Ev); + Assert(g_apfnTwoByteMap[0xbc * 4 + 2] == iemOp_tzcnt_Gv_Ev); + Assert(g_apfnTwoByteMap[0xbc * 4 + 3] == iemOp_bsf_Gv_Ev); + } +#endif + + if (RT_LIKELY(IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_286)) + { + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + IEMOP_HLP_MIN_286(); + return FNIEMOP_CALL(g_apfnTwoByteMap[(uintptr_t)b * 4 + pVCpu->iem.s.idxPrefix]); + } + /* @opdone */ + + /* + * On the 8086 this is a POP CS instruction. + * For the time being we don't specify this this. + */ + IEMOP_MNEMONIC1(FIXED, POP, pop, CS, DISOPTYPE_HARMLESS | DISOPTYPE_POTENTIALLY_DANGEROUS | DISOPTYPE_INVALID_64, IEMOPHINT_SKIP_PYTHON); + IEMOP_HLP_NO_64BIT(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_ES, pVCpu->iem.s.enmEffOpSize); +} + +/** + * @opcode 0x10 + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + * @optest op1=1 op2=1 efl&~=cf -> op1=2 efl&|=nc,pe,na,nz,pl,nv + * @optest op1=1 op2=1 efl|=cf -> op1=3 efl&|=nc,po,na,nz,pl,nv + * @optest op1=0xff op2=0 efl|=cf -> op1=0 efl&|=cf,po,af,zf,pl,nv + * @optest op1=0 op2=0 efl|=cf -> op1=1 efl&|=nc,pe,na,nz,pl,nv + * @optest op1=0 op2=0 efl&~=cf -> op1=0 efl&|=nc,po,na,zf,pl,nv + */ +FNIEMOP_DEF(iemOp_adc_Eb_Gb) +{ + IEMOP_MNEMONIC2(MR, ADC, adc, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_adc); +} + + +/** + * @opcode 0x11 + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + * @optest op1=1 op2=1 efl&~=cf -> op1=2 efl&|=nc,pe,na,nz,pl,nv + * @optest op1=1 op2=1 efl|=cf -> op1=3 efl&|=nc,po,na,nz,pl,nv + * @optest op1=-1 op2=0 efl|=cf -> op1=0 efl&|=cf,po,af,zf,pl,nv + * @optest op1=0 op2=0 efl|=cf -> op1=1 efl&|=nc,pe,na,nz,pl,nv + * @optest op1=0 op2=0 efl&~=cf -> op1=0 efl&|=nc,po,na,zf,pl,nv + */ +FNIEMOP_DEF(iemOp_adc_Ev_Gv) +{ + IEMOP_MNEMONIC2(MR, ADC, adc, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_adc); +} + + +/** + * @opcode 0x12 + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + * @opcopytests iemOp_adc_Eb_Gb + */ +FNIEMOP_DEF(iemOp_adc_Gb_Eb) +{ + IEMOP_MNEMONIC2(RM, ADC, adc, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_adc); +} + + +/** + * @opcode 0x13 + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + * @opcopytests iemOp_adc_Ev_Gv + */ +FNIEMOP_DEF(iemOp_adc_Gv_Ev) +{ + IEMOP_MNEMONIC2(RM, ADC, adc, Gv, Ev, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_adc); +} + + +/** + * @opcode 0x14 + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + * @opcopytests iemOp_adc_Eb_Gb + */ +FNIEMOP_DEF(iemOp_adc_Al_Ib) +{ + IEMOP_MNEMONIC2(FIXED, ADC, adc, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_adc); +} + + +/** + * @opcode 0x15 + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + * @opcopytests iemOp_adc_Ev_Gv + */ +FNIEMOP_DEF(iemOp_adc_eAX_Iz) +{ + IEMOP_MNEMONIC2(FIXED, ADC, adc, rAX, Iz, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_adc); +} + + +/** + * @opcode 0x16 + */ +FNIEMOP_DEF(iemOp_push_SS) +{ + IEMOP_MNEMONIC1(FIXED, PUSH, push, SS, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64 | DISOPTYPE_RRM_DANGEROUS, 0); + IEMOP_HLP_NO_64BIT(); + return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_SS); +} + + +/** + * @opcode 0x17 + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_pop_SS) +{ + IEMOP_MNEMONIC1(FIXED, POP, pop, SS, DISOPTYPE_HARMLESS | DISOPTYPE_INHIBIT_IRQS | DISOPTYPE_INVALID_64 | DISOPTYPE_RRM_DANGEROUS , 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_NO_64BIT(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_SS, pVCpu->iem.s.enmEffOpSize); +} + + +/** + * @opcode 0x18 + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sbb_Eb_Gb) +{ + IEMOP_MNEMONIC2(MR, SBB, sbb, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_sbb); +} + + +/** + * @opcode 0x19 + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sbb_Ev_Gv) +{ + IEMOP_MNEMONIC2(MR, SBB, sbb, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_sbb); +} + + +/** + * @opcode 0x1a + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sbb_Gb_Eb) +{ + IEMOP_MNEMONIC2(RM, SBB, sbb, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_sbb); +} + + +/** + * @opcode 0x1b + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sbb_Gv_Ev) +{ + IEMOP_MNEMONIC2(RM, SBB, sbb, Gv, Ev, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_sbb); +} + + +/** + * @opcode 0x1c + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sbb_Al_Ib) +{ + IEMOP_MNEMONIC2(FIXED, SBB, sbb, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_sbb); +} + + +/** + * @opcode 0x1d + * @opgroup og_gen_arith_bin + * @opfltest cf + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sbb_eAX_Iz) +{ + IEMOP_MNEMONIC2(FIXED, SBB, sbb, rAX, Iz, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_sbb); +} + + +/** + * @opcode 0x1e + * @opgroup og_stack_sreg + */ +FNIEMOP_DEF(iemOp_push_DS) +{ + IEMOP_MNEMONIC1(FIXED, PUSH, push, DS, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); + IEMOP_HLP_NO_64BIT(); + return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_DS); +} + + +/** + * @opcode 0x1f + * @opgroup og_stack_sreg + */ +FNIEMOP_DEF(iemOp_pop_DS) +{ + IEMOP_MNEMONIC1(FIXED, POP, pop, DS, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64 | DISOPTYPE_RRM_DANGEROUS, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_NO_64BIT(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_DS, pVCpu->iem.s.enmEffOpSize); +} + + +/** + * @opcode 0x20 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_and_Eb_Gb) +{ + IEMOP_MNEMONIC2(MR, AND, and, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_and); +} + + +/** + * @opcode 0x21 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_and_Ev_Gv) +{ + IEMOP_MNEMONIC2(MR, AND, and, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_and); +} + + +/** + * @opcode 0x22 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_and_Gb_Eb) +{ + IEMOP_MNEMONIC2(RM, AND, and, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_and); +} + + +/** + * @opcode 0x23 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_and_Gv_Ev) +{ + IEMOP_MNEMONIC2(RM, AND, and, Gv, Ev, DISOPTYPE_HARMLESS, 0); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_and); +} + + +/** + * @opcode 0x24 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_and_Al_Ib) +{ + IEMOP_MNEMONIC2(FIXED, AND, and, AL, Ib, DISOPTYPE_HARMLESS, 0); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_and); +} + + +/** + * @opcode 0x25 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_and_eAX_Iz) +{ + IEMOP_MNEMONIC2(FIXED, AND, and, rAX, Iz, DISOPTYPE_HARMLESS, 0); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_and); +} + + +/** + * @opcode 0x26 + * @opmnemonic SEG + * @op1 ES + * @opgroup og_prefix + * @openc prefix + * @opdisenum OP_SEG + * @ophints harmless + */ +FNIEMOP_DEF(iemOp_seg_ES) +{ + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg es"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_ES; + pVCpu->iem.s.iEffSeg = X86_SREG_ES; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0x27 + * @opfltest af,cf + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef of + */ +FNIEMOP_DEF(iemOp_daa) +{ + IEMOP_MNEMONIC0(FIXED, DAA, daa, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL register use */ + IEMOP_HLP_NO_64BIT(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_daa); +} + + +/** + * @opcode 0x28 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sub_Eb_Gb) +{ + IEMOP_MNEMONIC2(MR, SUB, sub, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_sub); +} + + +/** + * @opcode 0x29 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sub_Ev_Gv) +{ + IEMOP_MNEMONIC2(MR, SUB, sub, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_sub); +} + + +/** + * @opcode 0x2a + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sub_Gb_Eb) +{ + IEMOP_MNEMONIC2(RM, SUB, sub, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_sub); +} + + +/** + * @opcode 0x2b + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sub_Gv_Ev) +{ + IEMOP_MNEMONIC2(RM, SUB, sub, Gv, Ev, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_sub); +} + + +/** + * @opcode 0x2c + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sub_Al_Ib) +{ + IEMOP_MNEMONIC2(FIXED, SUB, sub, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_sub); +} + + +/** + * @opcode 0x2d + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + */ +FNIEMOP_DEF(iemOp_sub_eAX_Iz) +{ + IEMOP_MNEMONIC2(FIXED, SUB, sub, rAX, Iz, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_sub); +} + + +/** + * @opcode 0x2e + * @opmnemonic SEG + * @op1 CS + * @opgroup og_prefix + * @openc prefix + * @opdisenum OP_SEG + * @ophints harmless + */ +FNIEMOP_DEF(iemOp_seg_CS) +{ + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg cs"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_CS; + pVCpu->iem.s.iEffSeg = X86_SREG_CS; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0x2f + * @opfltest af,cf + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef of + */ +FNIEMOP_DEF(iemOp_das) +{ + IEMOP_MNEMONIC0(FIXED, DAS, das, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL register use */ + IEMOP_HLP_NO_64BIT(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_das); +} + + +/** + * @opcode 0x30 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_xor_Eb_Gb) +{ + IEMOP_MNEMONIC2(MR, XOR, xor, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_xor); +} + + +/** + * @opcode 0x31 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_xor_Ev_Gv) +{ + IEMOP_MNEMONIC2(MR, XOR, xor, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_xor); +} + + +/** + * @opcode 0x32 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_xor_Gb_Eb) +{ + IEMOP_MNEMONIC2(RM, XOR, xor, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_xor); +} + + +/** + * @opcode 0x33 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_xor_Gv_Ev) +{ + IEMOP_MNEMONIC2(RM, XOR, xor, Gv, Ev, DISOPTYPE_HARMLESS, 0); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_xor); +} + + +/** + * @opcode 0x34 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_xor_Al_Ib) +{ + IEMOP_MNEMONIC2(FIXED, XOR, xor, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_xor); +} + + +/** + * @opcode 0x35 + * @opgroup og_gen_arith_bin + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef af + * @opflclear of,cf + */ +FNIEMOP_DEF(iemOp_xor_eAX_Iz) +{ + IEMOP_MNEMONIC2(FIXED, XOR, xor, rAX, Iz, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_xor); +} + + +/** + * @opcode 0x36 + * @opmnemonic SEG + * @op1 SS + * @opgroup og_prefix + * @openc prefix + * @opdisenum OP_SEG + * @ophints harmless + */ +FNIEMOP_DEF(iemOp_seg_SS) +{ + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg ss"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_SS; + pVCpu->iem.s.iEffSeg = X86_SREG_SS; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0x37 + * @opfltest af,cf + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef pf,zf,sf,of + * @opgroup og_gen_arith_dec + * @optest efl&~=af ax=9 -> efl&|=nc,po,na,nz,pl,nv + * @optest efl&~=af ax=0 -> efl&|=nc,po,na,zf,pl,nv + * @optest intel / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,zf,pl,nv + * @optest amd / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,nz,pl,nv + * @optest efl&~=af ax=0x00f9 -> ax=0x0009 efl&|=nc,po,na,nz,pl,nv + * @optest efl|=af ax=0 -> ax=0x0106 efl&|=cf,po,af,nz,pl,nv + * @optest efl|=af ax=0x0100 -> ax=0x0206 efl&|=cf,po,af,nz,pl,nv + * @optest intel / efl|=af ax=0x000a -> ax=0x0100 efl&|=cf,po,af,zf,pl,nv + * @optest amd / efl|=af ax=0x000a -> ax=0x0100 efl&|=cf,pe,af,nz,pl,nv + * @optest intel / efl|=af ax=0x010a -> ax=0x0200 efl&|=cf,po,af,zf,pl,nv + * @optest amd / efl|=af ax=0x010a -> ax=0x0200 efl&|=cf,pe,af,nz,pl,nv + * @optest intel / efl|=af ax=0x0f0a -> ax=0x1000 efl&|=cf,po,af,zf,pl,nv + * @optest amd / efl|=af ax=0x0f0a -> ax=0x1000 efl&|=cf,pe,af,nz,pl,nv + * @optest intel / efl|=af ax=0x7f0a -> ax=0x8000 efl&|=cf,po,af,zf,pl,nv + * @optest amd / efl|=af ax=0x7f0a -> ax=0x8000 efl&|=cf,pe,af,nz,ng,ov + * @optest intel / efl|=af ax=0xff0a -> ax=0x0000 efl&|=cf,po,af,zf,pl,nv + * @optest amd / efl|=af ax=0xff0a -> ax=0x0000 efl&|=cf,pe,af,nz,pl,nv + * @optest intel / efl&~=af ax=0xff0a -> ax=0x0000 efl&|=cf,po,af,zf,pl,nv + * @optest amd / efl&~=af ax=0xff0a -> ax=0x0000 efl&|=cf,pe,af,nz,pl,nv + * @optest intel / efl&~=af ax=0x000b -> ax=0x0101 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000b -> ax=0x0101 efl&|=cf,po,af,nz,pl,nv + * @optest intel / efl&~=af ax=0x000c -> ax=0x0102 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000c -> ax=0x0102 efl&|=cf,po,af,nz,pl,nv + * @optest intel / efl&~=af ax=0x000d -> ax=0x0103 efl&|=cf,po,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000d -> ax=0x0103 efl&|=cf,pe,af,nz,pl,nv + * @optest intel / efl&~=af ax=0x000e -> ax=0x0104 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000e -> ax=0x0104 efl&|=cf,po,af,nz,pl,nv + * @optest intel / efl&~=af ax=0x000f -> ax=0x0105 efl&|=cf,po,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000f -> ax=0x0105 efl&|=cf,pe,af,nz,pl,nv + * @optest intel / efl&~=af ax=0x020f -> ax=0x0305 efl&|=cf,po,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x020f -> ax=0x0305 efl&|=cf,pe,af,nz,pl,nv + */ +FNIEMOP_DEF(iemOp_aaa) +{ + IEMOP_MNEMONIC0(FIXED, AAA, aaa, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL/AX register use */ + IEMOP_HLP_NO_64BIT(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF); + + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_aaa); +} + + +/** + * @opcode 0x38 + */ +FNIEMOP_DEF(iemOp_cmp_Eb_Gb) +{ + IEMOP_MNEMONIC(cmp_Eb_Gb, "cmp Eb,Gb"); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_cmp); +} + + +/** + * @opcode 0x39 + */ +FNIEMOP_DEF(iemOp_cmp_Ev_Gv) +{ + IEMOP_MNEMONIC(cmp_Ev_Gv, "cmp Ev,Gv"); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_cmp); +} + + +/** + * @opcode 0x3a + */ +FNIEMOP_DEF(iemOp_cmp_Gb_Eb) +{ + IEMOP_MNEMONIC(cmp_Gb_Eb, "cmp Gb,Eb"); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_cmp); +} + + +/** + * @opcode 0x3b + */ +FNIEMOP_DEF(iemOp_cmp_Gv_Ev) +{ + IEMOP_MNEMONIC(cmp_Gv_Ev, "cmp Gv,Ev"); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_cmp); +} + + +/** + * @opcode 0x3c + */ +FNIEMOP_DEF(iemOp_cmp_Al_Ib) +{ + IEMOP_MNEMONIC(cmp_al_Ib, "cmp al,Ib"); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_cmp); +} + + +/** + * @opcode 0x3d + */ +FNIEMOP_DEF(iemOp_cmp_eAX_Iz) +{ + IEMOP_MNEMONIC(cmp_rAX_Iz, "cmp rAX,Iz"); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_cmp); +} + + +/** + * @opcode 0x3e + */ +FNIEMOP_DEF(iemOp_seg_DS) +{ + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg ds"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_DS; + pVCpu->iem.s.iEffSeg = X86_SREG_DS; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0x3f + * @opfltest af,cf + * @opflmodify cf,pf,af,zf,sf,of + * @opflundef pf,zf,sf,of + * @opgroup og_gen_arith_dec + * @optest / efl&~=af ax=0x0009 -> efl&|=nc,po,na,nz,pl,nv + * @optest / efl&~=af ax=0x0000 -> efl&|=nc,po,na,zf,pl,nv + * @optest intel / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,zf,pl,nv + * @optest amd / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,nz,pl,nv + * @optest / efl&~=af ax=0x00f9 -> ax=0x0009 efl&|=nc,po,na,nz,pl,nv + * @optest intel / efl|=af ax=0x0000 -> ax=0xfe0a efl&|=cf,po,af,nz,pl,nv + * @optest amd / efl|=af ax=0x0000 -> ax=0xfe0a efl&|=cf,po,af,nz,ng,nv + * @optest intel / efl|=af ax=0x0100 -> ax=0xff0a efl&|=cf,po,af,nz,pl,nv + * @optest amd / efl|=af ax=0x0100 -> ax=0xff0a efl&|=cf,po,af,nz,ng,nv + * @optest intel / efl|=af ax=0x000a -> ax=0xff04 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl|=af ax=0x000a -> ax=0xff04 efl&|=cf,pe,af,nz,ng,nv + * @optest / efl|=af ax=0x010a -> ax=0x0004 efl&|=cf,pe,af,nz,pl,nv + * @optest / efl|=af ax=0x020a -> ax=0x0104 efl&|=cf,pe,af,nz,pl,nv + * @optest / efl|=af ax=0x0f0a -> ax=0x0e04 efl&|=cf,pe,af,nz,pl,nv + * @optest / efl|=af ax=0x7f0a -> ax=0x7e04 efl&|=cf,pe,af,nz,pl,nv + * @optest intel / efl|=af ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl|=af ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,ng,nv + * @optest intel / efl&~=af ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl&~=af ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,ng,nv + * @optest intel / efl&~=af ax=0xff09 -> ax=0xff09 efl&|=nc,po,na,nz,pl,nv + * @optest amd / efl&~=af ax=0xff09 -> ax=0xff09 efl&|=nc,po,na,nz,ng,nv + * @optest intel / efl&~=af ax=0x000b -> ax=0xff05 efl&|=cf,po,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000b -> ax=0xff05 efl&|=cf,po,af,nz,ng,nv + * @optest intel / efl&~=af ax=0x000c -> ax=0xff06 efl&|=cf,po,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000c -> ax=0xff06 efl&|=cf,po,af,nz,ng,nv + * @optest intel / efl&~=af ax=0x000d -> ax=0xff07 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000d -> ax=0xff07 efl&|=cf,pe,af,nz,ng,nv + * @optest intel / efl&~=af ax=0x000e -> ax=0xff08 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000e -> ax=0xff08 efl&|=cf,pe,af,nz,ng,nv + * @optest intel / efl&~=af ax=0x000f -> ax=0xff09 efl&|=cf,po,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x000f -> ax=0xff09 efl&|=cf,po,af,nz,ng,nv + * @optest intel / efl&~=af ax=0x00fa -> ax=0xff04 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl&~=af ax=0x00fa -> ax=0xff04 efl&|=cf,pe,af,nz,ng,nv + * @optest intel / efl&~=af ax=0xfffa -> ax=0xfe04 efl&|=cf,pe,af,nz,pl,nv + * @optest amd / efl&~=af ax=0xfffa -> ax=0xfe04 efl&|=cf,pe,af,nz,ng,nv + */ +FNIEMOP_DEF(iemOp_aas) +{ + IEMOP_MNEMONIC0(FIXED, AAS, aas, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL/AX register use */ + IEMOP_HLP_NO_64BIT(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_OF); + + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_aas); +} + + +/** + * Common 'inc/dec/not/neg register' helper. + */ +FNIEMOP_DEF_2(iemOpCommonUnaryGReg, PCIEMOPUNARYSIZES, pImpl, uint8_t, iReg) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + IEM_MC_REF_GREG_U16(pu16Dst, iReg); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU16, pu16Dst, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + IEM_MC_REF_GREG_U32(pu32Dst, iReg); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU32, pu32Dst, pEFlags); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + IEM_MC_REF_GREG_U64(pu64Dst, iReg); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU64, pu64Dst, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x40 + */ +FNIEMOP_DEF(iemOp_inc_eAX) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(inc_eAX, "inc eAX"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xAX); +} + + +/** + * @opcode 0x41 + */ +FNIEMOP_DEF(iemOp_inc_eCX) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.b"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B; + pVCpu->iem.s.uRexB = 1 << 3; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(inc_eCX, "inc eCX"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xCX); +} + + +/** + * @opcode 0x42 + */ +FNIEMOP_DEF(iemOp_inc_eDX) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.x"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_X; + pVCpu->iem.s.uRexIndex = 1 << 3; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(inc_eDX, "inc eDX"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xDX); +} + + + +/** + * @opcode 0x43 + */ +FNIEMOP_DEF(iemOp_inc_eBX) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.bx"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X; + pVCpu->iem.s.uRexB = 1 << 3; + pVCpu->iem.s.uRexIndex = 1 << 3; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(inc_eBX, "inc eBX"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xBX); +} + + +/** + * @opcode 0x44 + */ +FNIEMOP_DEF(iemOp_inc_eSP) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.r"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R; + pVCpu->iem.s.uRexReg = 1 << 3; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(inc_eSP, "inc eSP"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xSP); +} + + +/** + * @opcode 0x45 + */ +FNIEMOP_DEF(iemOp_inc_eBP) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rb"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B; + pVCpu->iem.s.uRexReg = 1 << 3; + pVCpu->iem.s.uRexB = 1 << 3; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(inc_eBP, "inc eBP"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xBP); +} + + +/** + * @opcode 0x46 + */ +FNIEMOP_DEF(iemOp_inc_eSI) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rx"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_X; + pVCpu->iem.s.uRexReg = 1 << 3; + pVCpu->iem.s.uRexIndex = 1 << 3; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(inc_eSI, "inc eSI"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xSI); +} + + +/** + * @opcode 0x47 + */ +FNIEMOP_DEF(iemOp_inc_eDI) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rbx"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X; + pVCpu->iem.s.uRexReg = 1 << 3; + pVCpu->iem.s.uRexB = 1 << 3; + pVCpu->iem.s.uRexIndex = 1 << 3; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(inc_eDI, "inc eDI"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xDI); +} + + +/** + * @opcode 0x48 + */ +FNIEMOP_DEF(iemOp_dec_eAX) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.w"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_SIZE_REX_W; + iemRecalEffOpSize(pVCpu); + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(dec_eAX, "dec eAX"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xAX); +} + + +/** + * @opcode 0x49 + */ +FNIEMOP_DEF(iemOp_dec_eCX) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.bw"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B | IEM_OP_PRF_SIZE_REX_W; + pVCpu->iem.s.uRexB = 1 << 3; + iemRecalEffOpSize(pVCpu); + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(dec_eCX, "dec eCX"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xCX); +} + + +/** + * @opcode 0x4a + */ +FNIEMOP_DEF(iemOp_dec_eDX) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.xw"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W; + pVCpu->iem.s.uRexIndex = 1 << 3; + iemRecalEffOpSize(pVCpu); + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(dec_eDX, "dec eDX"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xDX); +} + + +/** + * @opcode 0x4b + */ +FNIEMOP_DEF(iemOp_dec_eBX) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.bxw"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W; + pVCpu->iem.s.uRexB = 1 << 3; + pVCpu->iem.s.uRexIndex = 1 << 3; + iemRecalEffOpSize(pVCpu); + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(dec_eBX, "dec eBX"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xBX); +} + + +/** + * @opcode 0x4c + */ +FNIEMOP_DEF(iemOp_dec_eSP) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rw"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_SIZE_REX_W; + pVCpu->iem.s.uRexReg = 1 << 3; + iemRecalEffOpSize(pVCpu); + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(dec_eSP, "dec eSP"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xSP); +} + + +/** + * @opcode 0x4d + */ +FNIEMOP_DEF(iemOp_dec_eBP) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rbw"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_SIZE_REX_W; + pVCpu->iem.s.uRexReg = 1 << 3; + pVCpu->iem.s.uRexB = 1 << 3; + iemRecalEffOpSize(pVCpu); + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(dec_eBP, "dec eBP"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xBP); +} + + +/** + * @opcode 0x4e + */ +FNIEMOP_DEF(iemOp_dec_eSI) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rxw"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W; + pVCpu->iem.s.uRexReg = 1 << 3; + pVCpu->iem.s.uRexIndex = 1 << 3; + iemRecalEffOpSize(pVCpu); + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(dec_eSI, "dec eSI"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xSI); +} + + +/** + * @opcode 0x4f + */ +FNIEMOP_DEF(iemOp_dec_eDI) +{ + /* + * This is a REX prefix in 64-bit mode. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rbxw"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W; + pVCpu->iem.s.uRexReg = 1 << 3; + pVCpu->iem.s.uRexB = 1 << 3; + pVCpu->iem.s.uRexIndex = 1 << 3; + iemRecalEffOpSize(pVCpu); + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); + } + + IEMOP_MNEMONIC(dec_eDI, "dec eDI"); + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xDI); +} + + +/** + * Common 'push register' helper. + */ +FNIEMOP_DEF_1(iemOpCommonPushGReg, uint8_t, iReg) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + iReg |= pVCpu->iem.s.uRexB; + pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; + pVCpu->iem.s.enmEffOpSize = !(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP) ? IEMMODE_64BIT : IEMMODE_16BIT; + } + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_FETCH_GREG_U16(u16Value, iReg); + IEM_MC_PUSH_U16(u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_FETCH_GREG_U32(u32Value, iReg); + IEM_MC_PUSH_U32(u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_GREG_U64(u64Value, iReg); + IEM_MC_PUSH_U64(u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x50 + */ +FNIEMOP_DEF(iemOp_push_eAX) +{ + IEMOP_MNEMONIC(push_rAX, "push rAX"); + return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xAX); +} + + +/** + * @opcode 0x51 + */ +FNIEMOP_DEF(iemOp_push_eCX) +{ + IEMOP_MNEMONIC(push_rCX, "push rCX"); + return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xCX); +} + + +/** + * @opcode 0x52 + */ +FNIEMOP_DEF(iemOp_push_eDX) +{ + IEMOP_MNEMONIC(push_rDX, "push rDX"); + return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xDX); +} + + +/** + * @opcode 0x53 + */ +FNIEMOP_DEF(iemOp_push_eBX) +{ + IEMOP_MNEMONIC(push_rBX, "push rBX"); + return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xBX); +} + + +/** + * @opcode 0x54 + */ +FNIEMOP_DEF(iemOp_push_eSP) +{ + IEMOP_MNEMONIC(push_rSP, "push rSP"); + if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_8086) + { + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_FETCH_GREG_U16(u16Value, X86_GREG_xSP); + IEM_MC_SUB_LOCAL_U16(u16Value, 2); + IEM_MC_PUSH_U16(u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xSP); +} + + +/** + * @opcode 0x55 + */ +FNIEMOP_DEF(iemOp_push_eBP) +{ + IEMOP_MNEMONIC(push_rBP, "push rBP"); + return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xBP); +} + + +/** + * @opcode 0x56 + */ +FNIEMOP_DEF(iemOp_push_eSI) +{ + IEMOP_MNEMONIC(push_rSI, "push rSI"); + return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xSI); +} + + +/** + * @opcode 0x57 + */ +FNIEMOP_DEF(iemOp_push_eDI) +{ + IEMOP_MNEMONIC(push_rDI, "push rDI"); + return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xDI); +} + + +/** + * Common 'pop register' helper. + */ +FNIEMOP_DEF_1(iemOpCommonPopGReg, uint8_t, iReg) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + iReg |= pVCpu->iem.s.uRexB; + pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; + pVCpu->iem.s.enmEffOpSize = !(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP) ? IEMMODE_64BIT : IEMMODE_16BIT; + } + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t *, pu16Dst); + IEM_MC_REF_GREG_U16(pu16Dst, iReg); + IEM_MC_POP_U16(pu16Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t *, pu32Dst); + IEM_MC_REF_GREG_U32(pu32Dst, iReg); + IEM_MC_POP_U32(pu32Dst); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); /** @todo testcase*/ + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t *, pu64Dst); + IEM_MC_REF_GREG_U64(pu64Dst, iReg); + IEM_MC_POP_U64(pu64Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x58 + */ +FNIEMOP_DEF(iemOp_pop_eAX) +{ + IEMOP_MNEMONIC(pop_rAX, "pop rAX"); + return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xAX); +} + + +/** + * @opcode 0x59 + */ +FNIEMOP_DEF(iemOp_pop_eCX) +{ + IEMOP_MNEMONIC(pop_rCX, "pop rCX"); + return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xCX); +} + + +/** + * @opcode 0x5a + */ +FNIEMOP_DEF(iemOp_pop_eDX) +{ + IEMOP_MNEMONIC(pop_rDX, "pop rDX"); + return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xDX); +} + + +/** + * @opcode 0x5b + */ +FNIEMOP_DEF(iemOp_pop_eBX) +{ + IEMOP_MNEMONIC(pop_rBX, "pop rBX"); + return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xBX); +} + + +/** + * @opcode 0x5c + */ +FNIEMOP_DEF(iemOp_pop_eSP) +{ + IEMOP_MNEMONIC(pop_rSP, "pop rSP"); + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + if (pVCpu->iem.s.uRexB) + return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xSP); + pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; + pVCpu->iem.s.enmEffOpSize = !(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP) ? IEMMODE_64BIT : IEMMODE_16BIT; + } + + IEMOP_HLP_DECODED_NL_1(OP_POP, IEMOPFORM_FIXED, OP_PARM_REG_ESP, + DISOPTYPE_HARMLESS | DISOPTYPE_DEFAULT_64_OP_SIZE | DISOPTYPE_REXB_EXTENDS_OPREG); + /** @todo add testcase for this instruction. */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Dst); + IEM_MC_POP_U16(&u16Dst); /** @todo not correct MC, fix later. */ + IEM_MC_STORE_GREG_U16(X86_GREG_xSP, u16Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Dst); + IEM_MC_POP_U32(&u32Dst); + IEM_MC_STORE_GREG_U32(X86_GREG_xSP, u32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Dst); + IEM_MC_POP_U64(&u64Dst); + IEM_MC_STORE_GREG_U64(X86_GREG_xSP, u64Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x5d + */ +FNIEMOP_DEF(iemOp_pop_eBP) +{ + IEMOP_MNEMONIC(pop_rBP, "pop rBP"); + return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xBP); +} + + +/** + * @opcode 0x5e + */ +FNIEMOP_DEF(iemOp_pop_eSI) +{ + IEMOP_MNEMONIC(pop_rSI, "pop rSI"); + return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xSI); +} + + +/** + * @opcode 0x5f + */ +FNIEMOP_DEF(iemOp_pop_eDI) +{ + IEMOP_MNEMONIC(pop_rDI, "pop rDI"); + return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xDI); +} + + +/** + * @opcode 0x60 + */ +FNIEMOP_DEF(iemOp_pusha) +{ + IEMOP_MNEMONIC(pusha, "pusha"); + IEMOP_HLP_MIN_186(); + IEMOP_HLP_NO_64BIT(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_pusha_16); + Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_pusha_32); +} + + +/** + * @opcode 0x61 + */ +FNIEMOP_DEF(iemOp_popa__mvex) +{ + if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) + { + IEMOP_MNEMONIC(popa, "popa"); + IEMOP_HLP_MIN_186(); + IEMOP_HLP_NO_64BIT(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_popa_16); + Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_popa_32); + } + IEMOP_MNEMONIC(mvex, "mvex"); + Log(("mvex prefix is not supported!\n")); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opcode 0x62 + * @opmnemonic bound + * @op1 Gv_RO + * @op2 Ma + * @opmincpu 80186 + * @ophints harmless invalid_64 + * @optest op1=0 op2=0 -> + * @optest op1=1 op2=0 -> value.xcpt=5 + * @optest o16 / op1=0xffff op2=0x0000fffe -> + * @optest o16 / op1=0xfffe op2=0x0000fffe -> + * @optest o16 / op1=0x7fff op2=0x0000fffe -> value.xcpt=5 + * @optest o16 / op1=0x7fff op2=0x7ffffffe -> + * @optest o16 / op1=0x7fff op2=0xfffe8000 -> value.xcpt=5 + * @optest o16 / op1=0x8000 op2=0xfffe8000 -> + * @optest o16 / op1=0xffff op2=0xfffe8000 -> value.xcpt=5 + * @optest o16 / op1=0xfffe op2=0xfffe8000 -> + * @optest o16 / op1=0xfffe op2=0x8000fffe -> value.xcpt=5 + * @optest o16 / op1=0x8000 op2=0x8000fffe -> value.xcpt=5 + * @optest o16 / op1=0x0000 op2=0x8000fffe -> value.xcpt=5 + * @optest o16 / op1=0x0001 op2=0x8000fffe -> value.xcpt=5 + * @optest o16 / op1=0xffff op2=0x0001000f -> value.xcpt=5 + * @optest o16 / op1=0x0000 op2=0x0001000f -> value.xcpt=5 + * @optest o16 / op1=0x0001 op2=0x0001000f -> value.xcpt=5 + * @optest o16 / op1=0x0002 op2=0x0001000f -> value.xcpt=5 + * @optest o16 / op1=0x0003 op2=0x0001000f -> value.xcpt=5 + * @optest o16 / op1=0x0004 op2=0x0001000f -> value.xcpt=5 + * @optest o16 / op1=0x000e op2=0x0001000f -> value.xcpt=5 + * @optest o16 / op1=0x000f op2=0x0001000f -> value.xcpt=5 + * @optest o16 / op1=0x0010 op2=0x0001000f -> value.xcpt=5 + * @optest o16 / op1=0x0011 op2=0x0001000f -> value.xcpt=5 + * @optest o32 / op1=0xffffffff op2=0x00000000fffffffe -> + * @optest o32 / op1=0xfffffffe op2=0x00000000fffffffe -> + * @optest o32 / op1=0x7fffffff op2=0x00000000fffffffe -> value.xcpt=5 + * @optest o32 / op1=0x7fffffff op2=0x7ffffffffffffffe -> + * @optest o32 / op1=0x7fffffff op2=0xfffffffe80000000 -> value.xcpt=5 + * @optest o32 / op1=0x80000000 op2=0xfffffffe80000000 -> + * @optest o32 / op1=0xffffffff op2=0xfffffffe80000000 -> value.xcpt=5 + * @optest o32 / op1=0xfffffffe op2=0xfffffffe80000000 -> + * @optest o32 / op1=0xfffffffe op2=0x80000000fffffffe -> value.xcpt=5 + * @optest o32 / op1=0x80000000 op2=0x80000000fffffffe -> value.xcpt=5 + * @optest o32 / op1=0x00000000 op2=0x80000000fffffffe -> value.xcpt=5 + * @optest o32 / op1=0x00000002 op2=0x80000000fffffffe -> value.xcpt=5 + * @optest o32 / op1=0x00000001 op2=0x0000000100000003 -> value.xcpt=5 + * @optest o32 / op1=0x00000002 op2=0x0000000100000003 -> value.xcpt=5 + * @optest o32 / op1=0x00000003 op2=0x0000000100000003 -> value.xcpt=5 + * @optest o32 / op1=0x00000004 op2=0x0000000100000003 -> value.xcpt=5 + * @optest o32 / op1=0x00000005 op2=0x0000000100000003 -> value.xcpt=5 + * @optest o32 / op1=0x0000000e op2=0x0000000100000003 -> value.xcpt=5 + * @optest o32 / op1=0x0000000f op2=0x0000000100000003 -> value.xcpt=5 + * @optest o32 / op1=0x00000010 op2=0x0000000100000003 -> value.xcpt=5 + */ +FNIEMOP_DEF(iemOp_bound_Gv_Ma__evex) +{ + /* The BOUND instruction is invalid 64-bit mode. In legacy and + compatability mode it is invalid with MOD=3. + + In 32-bit mode, the EVEX prefix works by having the top two bits (MOD) + both be set. In the Intel EVEX documentation (sdm vol 2) these are simply + given as R and X without an exact description, so we assume it builds on + the VEX one and means they are inverted wrt REX.R and REX.X. Thus, just + like with the 3-byte VEX, 32-bit code is restrict wrt addressable registers. */ + uint8_t bRm; + if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) + { + IEMOP_MNEMONIC2(RM_MEM, BOUND, bound, Gv_RO, Ma, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_HLP_MIN_186(); + IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /** @todo testcase: check that there are two memory accesses involved. Check + * whether they're both read before the \#BR triggers. */ + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t, u16Index, 0); /* Note! All operands are actually signed. Lazy unsigned bird. */ + IEM_MC_ARG(uint16_t, u16LowerBounds, 1); + IEM_MC_ARG(uint16_t, u16UpperBounds, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_FETCH_GREG_U16(u16Index, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_FETCH_MEM_U16(u16LowerBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_MEM_U16_DISP(u16UpperBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 2); + + IEM_MC_CALL_CIMPL_3(iemCImpl_bound_16, u16Index, u16LowerBounds, u16UpperBounds); /* returns */ + IEM_MC_END(); + } + else /* 32-bit operands */ + { + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t, u32Index, 0); /* Note! All operands are actually signed. Lazy unsigned bird. */ + IEM_MC_ARG(uint32_t, u32LowerBounds, 1); + IEM_MC_ARG(uint32_t, u32UpperBounds, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_FETCH_GREG_U32(u32Index, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_FETCH_MEM_U32(u32LowerBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_MEM_U32_DISP(u32UpperBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 4); + + IEM_MC_CALL_CIMPL_3(iemCImpl_bound_32, u32Index, u32LowerBounds, u32UpperBounds); /* returns */ + IEM_MC_END(); + } + } + + /* + * @opdone + */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx512Foundation) + { + /* Note that there is no need for the CPU to fetch further bytes + here because MODRM.MOD == 3. */ + Log(("evex not supported by the guest CPU!\n")); + return IEMOP_RAISE_INVALID_OPCODE(); + } + } + else + { + /** @todo check how this is decoded in 64-bit mode w/o EVEX. Intel probably + * does modr/m read, whereas AMD probably doesn't... */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx512Foundation) + { + Log(("evex not supported by the guest CPU!\n")); + return FNIEMOP_CALL(iemOp_InvalidAllNeedRM); + } + IEM_OPCODE_GET_NEXT_U8(&bRm); + } + + IEMOP_MNEMONIC(evex, "evex"); + uint8_t bP2; IEM_OPCODE_GET_NEXT_U8(&bP2); + uint8_t bP3; IEM_OPCODE_GET_NEXT_U8(&bP3); + Log(("evex prefix is not implemented!\n")); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +} + + +/** Opcode 0x63 - non-64-bit modes. */ +FNIEMOP_DEF(iemOp_arpl_Ew_Gw) +{ + IEMOP_MNEMONIC(arpl_Ew_Gw, "arpl Ew,Gw"); + IEMOP_HLP_MIN_286(); + IEMOP_HLP_NO_REAL_OR_V86_MODE(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* Register */ + IEMOP_HLP_DECODED_NL_2(OP_ARPL, IEMOPFORM_MR_REG, OP_PARM_Ew, OP_PARM_Gw, DISOPTYPE_HARMLESS); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_arpl, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* Memory */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DECODED_NL_2(OP_ARPL, IEMOPFORM_MR_REG, OP_PARM_Ew, OP_PARM_Gw, DISOPTYPE_HARMLESS); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_arpl, pu16Dst, u16Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x63 + * + * @note This is a weird one. It works like a regular move instruction if + * REX.W isn't set, at least according to AMD docs (rev 3.15, 2009-11). + * @todo This definitely needs a testcase to verify the odd cases. */ +FNIEMOP_DEF(iemOp_movsxd_Gv_Ev) +{ + Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT); /* Caller branched already . */ + + IEMOP_MNEMONIC(movsxd_Gv_Ev, "movsxd Gv,Ev"); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register to register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_GREG_U32_SX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * We're loading a register from memory. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32_SX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + AssertFailedReturn(VERR_IEM_INSTR_NOT_IMPLEMENTED); +} + + +/** + * @opcode 0x64 + * @opmnemonic segfs + * @opmincpu 80386 + * @opgroup og_prefixes + */ +FNIEMOP_DEF(iemOp_seg_FS) +{ + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg fs"); + IEMOP_HLP_MIN_386(); + + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_FS; + pVCpu->iem.s.iEffSeg = X86_SREG_FS; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0x65 + * @opmnemonic seggs + * @opmincpu 80386 + * @opgroup og_prefixes + */ +FNIEMOP_DEF(iemOp_seg_GS) +{ + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg gs"); + IEMOP_HLP_MIN_386(); + + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_GS; + pVCpu->iem.s.iEffSeg = X86_SREG_GS; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0x66 + * @opmnemonic opsize + * @openc prefix + * @opmincpu 80386 + * @ophints harmless + * @opgroup og_prefixes + */ +FNIEMOP_DEF(iemOp_op_size) +{ + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("op size"); + IEMOP_HLP_MIN_386(); + + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_OP; + iemRecalEffOpSize(pVCpu); + + /* For the 4 entry opcode tables, the operand prefix doesn't not count + when REPZ or REPNZ are present. */ + if (pVCpu->iem.s.idxPrefix == 0) + pVCpu->iem.s.idxPrefix = 1; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0x67 + * @opmnemonic addrsize + * @openc prefix + * @opmincpu 80386 + * @ophints harmless + * @opgroup og_prefixes + */ +FNIEMOP_DEF(iemOp_addr_size) +{ + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("addr size"); + IEMOP_HLP_MIN_386(); + + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_ADDR; + switch (pVCpu->iem.s.enmDefAddrMode) + { + case IEMMODE_16BIT: pVCpu->iem.s.enmEffAddrMode = IEMMODE_32BIT; break; + case IEMMODE_32BIT: pVCpu->iem.s.enmEffAddrMode = IEMMODE_16BIT; break; + case IEMMODE_64BIT: pVCpu->iem.s.enmEffAddrMode = IEMMODE_32BIT; break; + default: AssertFailed(); + } + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0x68 + */ +FNIEMOP_DEF(iemOp_push_Iz) +{ + IEMOP_MNEMONIC(push_Iz, "push Iz"); + IEMOP_HLP_MIN_186(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0,0); + IEM_MC_PUSH_U16(u16Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + } + + case IEMMODE_32BIT: + { + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0,0); + IEM_MC_PUSH_U32(u32Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + } + + case IEMMODE_64BIT: + { + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0,0); + IEM_MC_PUSH_U64(u64Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x69 + */ +FNIEMOP_DEF(iemOp_imul_Gv_Ev_Iz) +{ + IEMOP_MNEMONIC(imul_Gv_Ev_Iz, "imul Gv,Ev,Iz"); /* Gv = Ev * Iz; */ + IEMOP_HLP_MIN_186(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register operand */ + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint16_t, u16Src,/*=*/ u16Imm,1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint16_t, u16Tmp); + + IEM_MC_FETCH_GREG_U16(u16Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu16Dst, u16Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u16_eflags), + pu16Dst, u16Src, pEFlags); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory operand */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint16_t, u16Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2); + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEM_MC_ASSIGN(u16Src, u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_LOCAL(pu16Dst, u16Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u16_eflags), + pu16Dst, u16Src, pEFlags); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + return VINF_SUCCESS; + } + + case IEMMODE_32BIT: + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register operand */ + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint32_t, u32Src,/*=*/ u32Imm,1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu32Dst, u32Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u32_eflags), + pu32Dst, u32Src, pEFlags); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory operand */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint32_t, u32Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4); + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEM_MC_ASSIGN(u32Src, u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_LOCAL(pu32Dst, u32Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u32_eflags), + pu32Dst, u32Src, pEFlags); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + return VINF_SUCCESS; + } + + case IEMMODE_64BIT: + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register operand */ + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint64_t, u64Src,/*=*/ u64Imm,1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu64Dst, u64Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u64_eflags), + pu64Dst, u64Src, pEFlags); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory operand */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint64_t, u64Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4); + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + IEM_MC_ASSIGN(u64Src, u64Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_LOCAL(pu64Dst, u64Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u64_eflags), + pu64Dst, u64Src, pEFlags); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + return VINF_SUCCESS; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x6a + */ +FNIEMOP_DEF(iemOp_push_Ib) +{ + IEMOP_MNEMONIC(push_Ib, "push Ib"); + IEMOP_HLP_MIN_186(); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + + IEM_MC_BEGIN(0,0); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_PUSH_U16(i8Imm); + break; + case IEMMODE_32BIT: + IEM_MC_PUSH_U32(i8Imm); + break; + case IEMMODE_64BIT: + IEM_MC_PUSH_U64(i8Imm); + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0x6b + */ +FNIEMOP_DEF(iemOp_imul_Gv_Ev_Ib) +{ + IEMOP_MNEMONIC(imul_Gv_Ev_Ib, "imul Gv,Ev,Ib"); /* Gv = Ev * Iz; */ + IEMOP_HLP_MIN_186(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register operand */ + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint16_t, u16Src,/*=*/ (int8_t)u8Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint16_t, u16Tmp); + + IEM_MC_FETCH_GREG_U16(u16Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu16Dst, u16Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u16_eflags), + pu16Dst, u16Src, pEFlags); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory operand */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint16_t, u16Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16Imm); + IEM_MC_ASSIGN(u16Src, u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_LOCAL(pu16Dst, u16Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u16_eflags), + pu16Dst, u16Src, pEFlags); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + return VINF_SUCCESS; + + case IEMMODE_32BIT: + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register operand */ + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint32_t, u32Src,/*=*/ (int8_t)u8Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu32Dst, u32Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u32_eflags), + pu32Dst, u32Src, pEFlags); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory operand */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint32_t, u32Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_S8_SX_U32(&u32Imm); + IEM_MC_ASSIGN(u32Src, u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_LOCAL(pu32Dst, u32Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u32_eflags), + pu32Dst, u32Src, pEFlags); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + return VINF_SUCCESS; + + case IEMMODE_64BIT: + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register operand */ + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint64_t, u64Src,/*=*/ (int8_t)u8Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu64Dst, u64Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u64_eflags), + pu64Dst, u64Src, pEFlags); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory operand */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(uint64_t, u64Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S8_SX_U64(&u64Imm); + IEM_MC_ASSIGN(u64Src, u64Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_LOCAL(pu64Dst, u64Tmp); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u64_eflags), + pu64Dst, u64Src, pEFlags); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + AssertFailedReturn(VERR_IEM_IPE_8); +} + + +/** + * @opcode 0x6c + */ +FNIEMOP_DEF(iemOp_insb_Yb_DX) +{ + IEMOP_HLP_MIN_186(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)) + { + IEMOP_MNEMONIC(rep_insb_Yb_DX, "rep ins Yb,DX"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op8_addr16, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op8_addr32, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op8_addr64, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + IEMOP_MNEMONIC(ins_Yb_DX, "ins Yb,DX"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op8_addr16, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op8_addr32, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op8_addr64, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0x6d + */ +FNIEMOP_DEF(iemOp_inswd_Yv_DX) +{ + IEMOP_HLP_MIN_186(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ)) + { + IEMOP_MNEMONIC(rep_ins_Yv_DX, "rep ins Yv,DX"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op16_addr16, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op16_addr32, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op16_addr64, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_64BIT: + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op32_addr16, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op32_addr32, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op32_addr64, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + IEMOP_MNEMONIC(ins_Yv_DX, "ins Yv,DX"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op16_addr16, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op16_addr32, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op16_addr64, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_64BIT: + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op32_addr16, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op32_addr32, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op32_addr64, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0x6e + */ +FNIEMOP_DEF(iemOp_outsb_Yb_DX) +{ + IEMOP_HLP_MIN_186(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)) + { + IEMOP_MNEMONIC(rep_outsb_DX_Yb, "rep outs DX,Yb"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op8_addr16, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op8_addr32, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op8_addr64, pVCpu->iem.s.iEffSeg, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + IEMOP_MNEMONIC(outs_DX_Yb, "outs DX,Yb"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op8_addr16, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op8_addr32, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op8_addr64, pVCpu->iem.s.iEffSeg, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0x6f + */ +FNIEMOP_DEF(iemOp_outswd_Yv_DX) +{ + IEMOP_HLP_MIN_186(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ)) + { + IEMOP_MNEMONIC(rep_outs_DX_Yv, "rep outs DX,Yv"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op16_addr16, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op16_addr32, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op16_addr64, pVCpu->iem.s.iEffSeg, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_64BIT: + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op32_addr16, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op32_addr32, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op32_addr64, pVCpu->iem.s.iEffSeg, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + IEMOP_MNEMONIC(outs_DX_Yv, "outs DX,Yv"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op16_addr16, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op16_addr32, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op16_addr64, pVCpu->iem.s.iEffSeg, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_64BIT: + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op32_addr16, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op32_addr32, pVCpu->iem.s.iEffSeg, false); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op32_addr64, pVCpu->iem.s.iEffSeg, false); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0x70 + */ +FNIEMOP_DEF(iemOp_jo_Jb) +{ + IEMOP_MNEMONIC(jo_Jb, "jo Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x71 + */ +FNIEMOP_DEF(iemOp_jno_Jb) +{ + IEMOP_MNEMONIC(jno_Jb, "jno Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + +/** + * @opcode 0x72 + */ +FNIEMOP_DEF(iemOp_jc_Jb) +{ + IEMOP_MNEMONIC(jc_Jb, "jc/jnae Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x73 + */ +FNIEMOP_DEF(iemOp_jnc_Jb) +{ + IEMOP_MNEMONIC(jnc_Jb, "jnc/jnb Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x74 + */ +FNIEMOP_DEF(iemOp_je_Jb) +{ + IEMOP_MNEMONIC(je_Jb, "je/jz Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x75 + */ +FNIEMOP_DEF(iemOp_jne_Jb) +{ + IEMOP_MNEMONIC(jne_Jb, "jne/jnz Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x76 + */ +FNIEMOP_DEF(iemOp_jbe_Jb) +{ + IEMOP_MNEMONIC(jbe_Jb, "jbe/jna Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x77 + */ +FNIEMOP_DEF(iemOp_jnbe_Jb) +{ + IEMOP_MNEMONIC(ja_Jb, "ja/jnbe Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x78 + */ +FNIEMOP_DEF(iemOp_js_Jb) +{ + IEMOP_MNEMONIC(js_Jb, "js Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x79 + */ +FNIEMOP_DEF(iemOp_jns_Jb) +{ + IEMOP_MNEMONIC(jns_Jb, "jns Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x7a + */ +FNIEMOP_DEF(iemOp_jp_Jb) +{ + IEMOP_MNEMONIC(jp_Jb, "jp Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x7b + */ +FNIEMOP_DEF(iemOp_jnp_Jb) +{ + IEMOP_MNEMONIC(jnp_Jb, "jnp Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x7c + */ +FNIEMOP_DEF(iemOp_jl_Jb) +{ + IEMOP_MNEMONIC(jl_Jb, "jl/jnge Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x7d + */ +FNIEMOP_DEF(iemOp_jnl_Jb) +{ + IEMOP_MNEMONIC(jge_Jb, "jnl/jge Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x7e + */ +FNIEMOP_DEF(iemOp_jle_Jb) +{ + IEMOP_MNEMONIC(jle_Jb, "jle/jng Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x7f + */ +FNIEMOP_DEF(iemOp_jnle_Jb) +{ + IEMOP_MNEMONIC(jg_Jb, "jnle/jg Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); +} + + +/** + * @opcode 0x80 + */ +FNIEMOP_DEF(iemOp_Grp1_Eb_Ib_80) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: IEMOP_MNEMONIC(add_Eb_Ib, "add Eb,Ib"); break; + case 1: IEMOP_MNEMONIC(or_Eb_Ib, "or Eb,Ib"); break; + case 2: IEMOP_MNEMONIC(adc_Eb_Ib, "adc Eb,Ib"); break; + case 3: IEMOP_MNEMONIC(sbb_Eb_Ib, "sbb Eb,Ib"); break; + case 4: IEMOP_MNEMONIC(and_Eb_Ib, "and Eb,Ib"); break; + case 5: IEMOP_MNEMONIC(sub_Eb_Ib, "sub Eb,Ib"); break; + case 6: IEMOP_MNEMONIC(xor_Eb_Ib, "xor Eb,Ib"); break; + case 7: IEMOP_MNEMONIC(cmp_Eb_Ib, "cmp Eb,Ib"); break; + } + PCIEMOPBINSIZES pImpl = g_apIemImplGrp1[IEM_GET_MODRM_REG_8(bRm)]; + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG_CONST(uint8_t, u8Src, /*=*/ u8Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + uint32_t fAccess; + if (pImpl->pfnLockedU8) + fAccess = IEM_ACCESS_DATA_RW; + else /* CMP */ + fAccess = IEM_ACCESS_DATA_R; + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEM_MC_ARG_CONST(uint8_t, u8Src, /*=*/ u8Imm, 1); + if (pImpl->pfnLockedU8) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MEM_MAP(pu8Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU8, pu8Dst, u8Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x81 + */ +FNIEMOP_DEF(iemOp_Grp1_Ev_Iz) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: IEMOP_MNEMONIC(add_Ev_Iz, "add Ev,Iz"); break; + case 1: IEMOP_MNEMONIC(or_Ev_Iz, "or Ev,Iz"); break; + case 2: IEMOP_MNEMONIC(adc_Ev_Iz, "adc Ev,Iz"); break; + case 3: IEMOP_MNEMONIC(sbb_Ev_Iz, "sbb Ev,Iz"); break; + case 4: IEMOP_MNEMONIC(and_Ev_Iz, "and Ev,Iz"); break; + case 5: IEMOP_MNEMONIC(sub_Ev_Iz, "sub Ev,Iz"); break; + case 6: IEMOP_MNEMONIC(xor_Ev_Iz, "xor Ev,Iz"); break; + case 7: IEMOP_MNEMONIC(cmp_Ev_Iz, "cmp Ev,Iz"); break; + } + PCIEMOPBINSIZES pImpl = g_apIemImplGrp1[IEM_GET_MODRM_REG_8(bRm)]; + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint16_t, u16Src, /*=*/ u16Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + uint32_t fAccess; + if (pImpl->pfnLockedU16) + fAccess = IEM_ACCESS_DATA_RW; + else /* CMP, TEST */ + fAccess = IEM_ACCESS_DATA_R; + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2); + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEM_MC_ASSIGN(u16Src, u16Imm); + if (pImpl->pfnLockedU16) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + break; + } + + case IEMMODE_32BIT: + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint32_t, u32Src, /*=*/ u32Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + if (pImpl != &g_iemAImpl_cmp) /* TEST won't get here, no need to check for it. */ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + uint32_t fAccess; + if (pImpl->pfnLockedU32) + fAccess = IEM_ACCESS_DATA_RW; + else /* CMP, TEST */ + fAccess = IEM_ACCESS_DATA_R; + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4); + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEM_MC_ASSIGN(u32Src, u32Imm); + if (pImpl->pfnLockedU32) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + break; + } + + case IEMMODE_64BIT: + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint64_t, u64Src, /*=*/ u64Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + uint32_t fAccess; + if (pImpl->pfnLockedU64) + fAccess = IEM_ACCESS_DATA_RW; + else /* CMP */ + fAccess = IEM_ACCESS_DATA_R; + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4); + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + if (pImpl->pfnLockedU64) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN(u64Src, u64Imm); + IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x82 + * @opmnemonic grp1_82 + * @opgroup og_groups + */ +FNIEMOP_DEF(iemOp_Grp1_Eb_Ib_82) +{ + IEMOP_HLP_NO_64BIT(); /** @todo do we need to decode the whole instruction or is this ok? */ + return FNIEMOP_CALL(iemOp_Grp1_Eb_Ib_80); +} + + +/** + * @opcode 0x83 + */ +FNIEMOP_DEF(iemOp_Grp1_Ev_Ib) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: IEMOP_MNEMONIC(add_Ev_Ib, "add Ev,Ib"); break; + case 1: IEMOP_MNEMONIC(or_Ev_Ib, "or Ev,Ib"); break; + case 2: IEMOP_MNEMONIC(adc_Ev_Ib, "adc Ev,Ib"); break; + case 3: IEMOP_MNEMONIC(sbb_Ev_Ib, "sbb Ev,Ib"); break; + case 4: IEMOP_MNEMONIC(and_Ev_Ib, "and Ev,Ib"); break; + case 5: IEMOP_MNEMONIC(sub_Ev_Ib, "sub Ev,Ib"); break; + case 6: IEMOP_MNEMONIC(xor_Ev_Ib, "xor Ev,Ib"); break; + case 7: IEMOP_MNEMONIC(cmp_Ev_Ib, "cmp Ev,Ib"); break; + } + /* Note! Seems the OR, AND, and XOR instructions are present on CPUs prior + to the 386 even if absent in the intel reference manuals and some + 3rd party opcode listings. */ + PCIEMOPBINSIZES pImpl = g_apIemImplGrp1[IEM_GET_MODRM_REG_8(bRm)]; + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register target + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint16_t, u16Src, /*=*/ (int8_t)u8Imm,1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + case IEMMODE_32BIT: + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint32_t, u32Src, /*=*/ (int8_t)u8Imm,1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + if (pImpl != &g_iemAImpl_cmp) /* TEST won't get here, no need to check for it. */ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + case IEMMODE_64BIT: + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint64_t, u64Src, /*=*/ (int8_t)u8Imm,1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * Memory target. + */ + uint32_t fAccess; + if (pImpl->pfnLockedU16) + fAccess = IEM_ACCESS_DATA_RW; + else /* CMP */ + fAccess = IEM_ACCESS_DATA_R; + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEM_MC_ASSIGN(u16Src, (int8_t)u8Imm); + if (pImpl->pfnLockedU16) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + case IEMMODE_32BIT: + { + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEM_MC_ASSIGN(u32Src, (int8_t)u8Imm); + if (pImpl->pfnLockedU32) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + case IEMMODE_64BIT: + { + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEM_MC_ASSIGN(u64Src, (int8_t)u8Imm); + if (pImpl->pfnLockedU64) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0x84 + */ +FNIEMOP_DEF(iemOp_test_Eb_Gb) +{ + IEMOP_MNEMONIC(test_Eb_Gb, "test Eb,Gb"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_test); +} + + +/** + * @opcode 0x85 + */ +FNIEMOP_DEF(iemOp_test_Ev_Gv) +{ + IEMOP_MNEMONIC(test_Ev_Gv, "test Ev,Gv"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_test); +} + + +/** + * @opcode 0x86 + */ +FNIEMOP_DEF(iemOp_xchg_Eb_Gb) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_MNEMONIC(xchg_Eb_Gb, "xchg Eb,Gb"); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint8_t, uTmp1); + IEM_MC_LOCAL(uint8_t, uTmp2); + + IEM_MC_FETCH_GREG_U8(uTmp1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(uTmp2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_RM(pVCpu, bRm), uTmp1); + IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), uTmp2); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * We're accessing memory. + */ +/** @todo the register must be committed separately! */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint8_t *, pu8Mem, 0); + IEM_MC_ARG(uint8_t *, pu8Reg, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu8Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_REF_GREG_U8(pu8Reg, IEM_GET_MODRM_REG(pVCpu, bRm)); + if (!pVCpu->iem.s.fDisregardLock) + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u8_locked, pu8Mem, pu8Reg); + else + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u8_unlocked, pu8Mem, pu8Reg); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Mem, IEM_ACCESS_DATA_RW); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x87 + */ +FNIEMOP_DEF(iemOp_xchg_Ev_Gv) +{ + IEMOP_MNEMONIC(xchg_Ev_Gv, "xchg Ev,Gv"); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, uTmp1); + IEM_MC_LOCAL(uint16_t, uTmp2); + + IEM_MC_FETCH_GREG_U16(uTmp1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U16(uTmp2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_RM(pVCpu, bRm), uTmp1); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), uTmp2); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, uTmp1); + IEM_MC_LOCAL(uint32_t, uTmp2); + + IEM_MC_FETCH_GREG_U32(uTmp1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U32(uTmp2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), uTmp1); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), uTmp2); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uTmp1); + IEM_MC_LOCAL(uint64_t, uTmp2); + + IEM_MC_FETCH_GREG_U64(uTmp1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U64(uTmp2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), uTmp1); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uTmp2); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're accessing memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { +/** @todo the register must be committed separately! */ + case IEMMODE_16BIT: + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint16_t *, pu16Mem, 0); + IEM_MC_ARG(uint16_t *, pu16Reg, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu16Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_REF_GREG_U16(pu16Reg, IEM_GET_MODRM_REG(pVCpu, bRm)); + if (!pVCpu->iem.s.fDisregardLock) + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u16_locked, pu16Mem, pu16Reg); + else + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u16_unlocked, pu16Mem, pu16Reg); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Mem, IEM_ACCESS_DATA_RW); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint32_t *, pu32Mem, 0); + IEM_MC_ARG(uint32_t *, pu32Reg, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu32Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_REF_GREG_U32(pu32Reg, IEM_GET_MODRM_REG(pVCpu, bRm)); + if (!pVCpu->iem.s.fDisregardLock) + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u32_locked, pu32Mem, pu32Reg); + else + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u32_unlocked, pu32Mem, pu32Reg); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Mem, IEM_ACCESS_DATA_RW); + + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Reg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint64_t *, pu64Mem, 0); + IEM_MC_ARG(uint64_t *, pu64Reg, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu64Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_REF_GREG_U64(pu64Reg, IEM_GET_MODRM_REG(pVCpu, bRm)); + if (!pVCpu->iem.s.fDisregardLock) + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u64_locked, pu64Mem, pu64Reg); + else + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u64_unlocked, pu64Mem, pu64Reg); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Mem, IEM_ACCESS_DATA_RW); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0x88 + */ +FNIEMOP_DEF(iemOp_mov_Eb_Gb) +{ + IEMOP_MNEMONIC(mov_Eb_Gb, "mov Eb,Gb"); + + uint8_t bRm; + IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint8_t, u8Value); + IEM_MC_FETCH_GREG_U8(u8Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_RM(pVCpu, bRm), u8Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * We're writing a register to memory. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint8_t, u8Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U8(u8Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u8Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x89 + */ +FNIEMOP_DEF(iemOp_mov_Ev_Gv) +{ + IEMOP_MNEMONIC(mov_Ev_Gv, "mov Ev,Gv"); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_RM(pVCpu, bRm), u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're writing a register to memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0x8a + */ +FNIEMOP_DEF(iemOp_mov_Gb_Eb) +{ + IEMOP_MNEMONIC(mov_Gb_Eb, "mov Gb,Eb"); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint8_t, u8Value); + IEM_MC_FETCH_GREG_U8(u8Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), u8Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * We're loading a register from memory. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint8_t, u8Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8(u8Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), u8Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x8b + */ +FNIEMOP_DEF(iemOp_mov_Gv_Ev) +{ + IEMOP_MNEMONIC(mov_Gv_Ev, "mov Gv,Ev"); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're loading a register from memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * opcode 0x63 + * @todo Table fixme + */ +FNIEMOP_DEF(iemOp_arpl_Ew_Gw_movsx_Gv_Ev) +{ + if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) + return FNIEMOP_CALL(iemOp_arpl_Ew_Gw); + if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT) + return FNIEMOP_CALL(iemOp_mov_Gv_Ev); + return FNIEMOP_CALL(iemOp_movsxd_Gv_Ev); +} + + +/** + * @opcode 0x8c + */ +FNIEMOP_DEF(iemOp_mov_Ev_Sw) +{ + IEMOP_MNEMONIC(mov_Ev_Sw, "mov Ev,Sw"); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * Check that the destination register exists. The REX.R prefix is ignored. + */ + uint8_t const iSegReg = IEM_GET_MODRM_REG_8(bRm); + if ( iSegReg > X86_SREG_GS) + return IEMOP_RAISE_INVALID_OPCODE(); /** @todo should probably not be raised until we've fetched all the opcode bytes? */ + + /* + * If rm is denoting a register, no more instruction bytes. + * In that case, the operand size is respected and the upper bits are + * cleared (starting with some pentium). + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_FETCH_SREG_U16(u16Value, iSegReg); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_RM(pVCpu, bRm), u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_FETCH_SREG_ZX_U32(u32Value, iSegReg); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_SREG_ZX_U64(u64Value, iSegReg); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're saving the register to memory. The access is word sized + * regardless of operand size prefixes. + */ +#if 0 /* not necessary */ + pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_16BIT; +#endif + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_SREG_U16(u16Value, iSegReg); + IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + + + +/** + * @opcode 0x8d + */ +FNIEMOP_DEF(iemOp_lea_Gv_M) +{ + IEMOP_MNEMONIC(lea_Gv_M, "lea Gv,M"); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + return IEMOP_RAISE_INVALID_OPCODE(); /* no register form */ + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint16_t, u16Cast); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN_TO_SMALLER(u16Cast, GCPtrEffSrc); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Cast); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, u32Cast); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN_TO_SMALLER(u32Cast, GCPtrEffSrc); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Cast); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), GCPtrEffSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x8e + */ +FNIEMOP_DEF(iemOp_mov_Sw_Ev) +{ + IEMOP_MNEMONIC(mov_Sw_Ev, "mov Sw,Ev"); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * The practical operand size is 16-bit. + */ +#if 0 /* not necessary */ + pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_16BIT; +#endif + + /* + * Check that the destination register exists and can be used with this + * instruction. The REX.R prefix is ignored. + */ + uint8_t const iSegReg = IEM_GET_MODRM_REG_8(bRm); + if ( iSegReg == X86_SREG_CS + || iSegReg > X86_SREG_GS) + return IEMOP_RAISE_INVALID_OPCODE(); /** @todo should probably not be raised until we've fetched all the opcode bytes? */ + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG_CONST(uint8_t, iSRegArg, iSegReg, 0); + IEM_MC_ARG(uint16_t, u16Value, 1); + IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_2(iemCImpl_load_SReg, iSRegArg, u16Value); + IEM_MC_END(); + } + else + { + /* + * We're loading the register from memory. The access is word sized + * regardless of operand size prefixes. + */ + IEM_MC_BEGIN(2, 1); + IEM_MC_ARG_CONST(uint8_t, iSRegArg, iSegReg, 0); + IEM_MC_ARG(uint16_t, u16Value, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_CALL_CIMPL_2(iemCImpl_load_SReg, iSRegArg, u16Value); + IEM_MC_END(); + } + return VINF_SUCCESS; +} + + +/** Opcode 0x8f /0. */ +FNIEMOP_DEF_1(iemOp_pop_Ev, uint8_t, bRm) +{ + /* This bugger is rather annoying as it requires rSP to be updated before + doing the effective address calculations. Will eventually require a + split between the R/M+SIB decoding and the effective address + calculation - which is something that is required for any attempt at + reusing this code for a recompiler. It may also be good to have if we + need to delay #UD exception caused by invalid lock prefixes. + + For now, we'll do a mostly safe interpreter-only implementation here. */ + /** @todo What's the deal with the 'reg' field and pop Ev? Ignorning it for + * now until tests show it's checked.. */ + IEMOP_MNEMONIC(pop_Ev, "pop Ev"); + + /* Register access is relatively easy and can share code. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + return FNIEMOP_CALL_1(iemOpCommonPopGReg, IEM_GET_MODRM_RM(pVCpu, bRm)); + + /* + * Memory target. + * + * Intel says that RSP is incremented before it's used in any effective + * address calcuations. This means some serious extra annoyance here since + * we decode and calculate the effective address in one step and like to + * delay committing registers till everything is done. + * + * So, we'll decode and calculate the effective address twice. This will + * require some recoding if turned into a recompiler. + */ + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); /* The common code does this differently. */ + +#ifndef TST_IEM_CHECK_MC + /* Calc effective address with modified ESP. */ +/** @todo testcase */ + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict; + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: rcStrict = iemOpHlpCalcRmEffAddrEx(pVCpu, bRm, 0, &GCPtrEff, 2); break; + case IEMMODE_32BIT: rcStrict = iemOpHlpCalcRmEffAddrEx(pVCpu, bRm, 0, &GCPtrEff, 4); break; + case IEMMODE_64BIT: rcStrict = iemOpHlpCalcRmEffAddrEx(pVCpu, bRm, 0, &GCPtrEff, 8); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + if (rcStrict != VINF_SUCCESS) + return rcStrict; + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* Perform the operation - this should be CImpl. */ + RTUINT64U TmpRsp; + TmpRsp.u = pVCpu->cpum.GstCtx.rsp; + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t u16Value; + rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Value, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStoreDataU16(pVCpu, pVCpu->iem.s.iEffSeg, GCPtrEff, u16Value); + break; + } + + case IEMMODE_32BIT: + { + uint32_t u32Value; + rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Value, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStoreDataU32(pVCpu, pVCpu->iem.s.iEffSeg, GCPtrEff, u32Value); + break; + } + + case IEMMODE_64BIT: + { + uint64_t u64Value; + rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Value, &TmpRsp); + if (rcStrict == VINF_SUCCESS) + rcStrict = iemMemStoreDataU64(pVCpu, pVCpu->iem.s.iEffSeg, GCPtrEff, u64Value); + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rsp = TmpRsp.u; + return iemRegUpdateRipAndFinishClearingRF(pVCpu); + } + return rcStrict; + +#else + return VERR_IEM_IPE_2; +#endif +} + + +/** + * @opcode 0x8f + */ +FNIEMOP_DEF(iemOp_Grp1A__xop) +{ + /* + * AMD has defined /1 thru /7 as XOP prefix. The prefix is similar to the + * three byte VEX prefix, except that the mmmmm field cannot have the values + * 0 thru 7, because it would then be confused with pop Ev (modrm.reg == 0). + */ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if ((bRm & X86_MODRM_REG_MASK) == (0 << X86_MODRM_REG_SHIFT)) /* /0 */ + return FNIEMOP_CALL_1(iemOp_pop_Ev, bRm); + + IEMOP_MNEMONIC(xop, "xop"); + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXop) + { + /** @todo Test when exctly the XOP conformance checks kick in during + * instruction decoding and fetching (using \#PF). */ + uint8_t bXop2; IEM_OPCODE_GET_NEXT_U8(&bXop2); + uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode); + if ( ( pVCpu->iem.s.fPrefixes + & (IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_LOCK | IEM_OP_PRF_REX)) + == 0) + { + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_XOP; + if ((bXop2 & 0x80 /* XOP.W */) && pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_W; + pVCpu->iem.s.uRexReg = (~bRm >> (7 - 3)) & 0x8; + pVCpu->iem.s.uRexIndex = (~bRm >> (6 - 3)) & 0x8; + pVCpu->iem.s.uRexB = (~bRm >> (5 - 3)) & 0x8; + pVCpu->iem.s.uVex3rdReg = (~bXop2 >> 3) & 0xf; + pVCpu->iem.s.uVexLength = (bXop2 >> 2) & 1; + pVCpu->iem.s.idxPrefix = bXop2 & 0x3; + + /** @todo XOP: Just use new tables and decoders. */ + switch (bRm & 0x1f) + { + case 8: /* xop opcode map 8. */ + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; + + case 9: /* xop opcode map 9. */ + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; + + case 10: /* xop opcode map 10. */ + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; + + default: + Log(("XOP: Invalid vvvv value: %#x!\n", bRm & 0x1f)); + return IEMOP_RAISE_INVALID_OPCODE(); + } + } + else + Log(("XOP: Invalid prefix mix!\n")); + } + else + Log(("XOP: XOP support disabled!\n")); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * Common 'xchg reg,rAX' helper. + */ +FNIEMOP_DEF_1(iemOpCommonXchgGRegRax, uint8_t, iReg) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + iReg |= pVCpu->iem.s.uRexB; + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Tmp1); + IEM_MC_LOCAL(uint16_t, u16Tmp2); + IEM_MC_FETCH_GREG_U16(u16Tmp1, iReg); + IEM_MC_FETCH_GREG_U16(u16Tmp2, X86_GREG_xAX); + IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Tmp1); + IEM_MC_STORE_GREG_U16(iReg, u16Tmp2); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Tmp1); + IEM_MC_LOCAL(uint32_t, u32Tmp2); + IEM_MC_FETCH_GREG_U32(u32Tmp1, iReg); + IEM_MC_FETCH_GREG_U32(u32Tmp2, X86_GREG_xAX); + IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u32Tmp1); + IEM_MC_STORE_GREG_U32(iReg, u32Tmp2); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Tmp1); + IEM_MC_LOCAL(uint64_t, u64Tmp2); + IEM_MC_FETCH_GREG_U64(u64Tmp1, iReg); + IEM_MC_FETCH_GREG_U64(u64Tmp2, X86_GREG_xAX); + IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Tmp1); + IEM_MC_STORE_GREG_U64(iReg, u64Tmp2); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x90 + */ +FNIEMOP_DEF(iemOp_nop) +{ + /* R8/R8D and RAX/EAX can be exchanged. */ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX_B) + { + IEMOP_MNEMONIC(xchg_r8_rAX, "xchg r8,rAX"); + return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xAX); + } + + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) + { + IEMOP_MNEMONIC(pause, "pause"); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmx) + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmx_pause); +#endif +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvm) + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_svm_pause); +#endif + } + else + IEMOP_MNEMONIC(nop, "nop"); + IEM_MC_BEGIN(0, 0); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0x91 + */ +FNIEMOP_DEF(iemOp_xchg_eCX_eAX) +{ + IEMOP_MNEMONIC(xchg_rCX_rAX, "xchg rCX,rAX"); + return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xCX); +} + + +/** + * @opcode 0x92 + */ +FNIEMOP_DEF(iemOp_xchg_eDX_eAX) +{ + IEMOP_MNEMONIC(xchg_rDX_rAX, "xchg rDX,rAX"); + return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xDX); +} + + +/** + * @opcode 0x93 + */ +FNIEMOP_DEF(iemOp_xchg_eBX_eAX) +{ + IEMOP_MNEMONIC(xchg_rBX_rAX, "xchg rBX,rAX"); + return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xBX); +} + + +/** + * @opcode 0x94 + */ +FNIEMOP_DEF(iemOp_xchg_eSP_eAX) +{ + IEMOP_MNEMONIC(xchg_rSX_rAX, "xchg rSX,rAX"); + return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xSP); +} + + +/** + * @opcode 0x95 + */ +FNIEMOP_DEF(iemOp_xchg_eBP_eAX) +{ + IEMOP_MNEMONIC(xchg_rBP_rAX, "xchg rBP,rAX"); + return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xBP); +} + + +/** + * @opcode 0x96 + */ +FNIEMOP_DEF(iemOp_xchg_eSI_eAX) +{ + IEMOP_MNEMONIC(xchg_rSI_rAX, "xchg rSI,rAX"); + return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xSI); +} + + +/** + * @opcode 0x97 + */ +FNIEMOP_DEF(iemOp_xchg_eDI_eAX) +{ + IEMOP_MNEMONIC(xchg_rDI_rAX, "xchg rDI,rAX"); + return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xDI); +} + + +/** + * @opcode 0x98 + */ +FNIEMOP_DEF(iemOp_cbw) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEMOP_MNEMONIC(cbw, "cbw"); + IEM_MC_BEGIN(0, 1); + IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 7) { + IEM_MC_OR_GREG_U16(X86_GREG_xAX, UINT16_C(0xff00)); + } IEM_MC_ELSE() { + IEM_MC_AND_GREG_U16(X86_GREG_xAX, UINT16_C(0x00ff)); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEMOP_MNEMONIC(cwde, "cwde"); + IEM_MC_BEGIN(0, 1); + IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 15) { + IEM_MC_OR_GREG_U32(X86_GREG_xAX, UINT32_C(0xffff0000)); + } IEM_MC_ELSE() { + IEM_MC_AND_GREG_U32(X86_GREG_xAX, UINT32_C(0x0000ffff)); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEMOP_MNEMONIC(cdqe, "cdqe"); + IEM_MC_BEGIN(0, 1); + IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 31) { + IEM_MC_OR_GREG_U64(X86_GREG_xAX, UINT64_C(0xffffffff00000000)); + } IEM_MC_ELSE() { + IEM_MC_AND_GREG_U64(X86_GREG_xAX, UINT64_C(0x00000000ffffffff)); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x99 + */ +FNIEMOP_DEF(iemOp_cwd) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEMOP_MNEMONIC(cwd, "cwd"); + IEM_MC_BEGIN(0, 1); + IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 15) { + IEM_MC_STORE_GREG_U16_CONST(X86_GREG_xDX, UINT16_C(0xffff)); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U16_CONST(X86_GREG_xDX, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEMOP_MNEMONIC(cdq, "cdq"); + IEM_MC_BEGIN(0, 1); + IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 31) { + IEM_MC_STORE_GREG_U32_CONST(X86_GREG_xDX, UINT32_C(0xffffffff)); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U32_CONST(X86_GREG_xDX, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEMOP_MNEMONIC(cqo, "cqo"); + IEM_MC_BEGIN(0, 1); + IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 63) { + IEM_MC_STORE_GREG_U64_CONST(X86_GREG_xDX, UINT64_C(0xffffffffffffffff)); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U64_CONST(X86_GREG_xDX, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0x9a + */ +FNIEMOP_DEF(iemOp_call_Ap) +{ + IEMOP_MNEMONIC(call_Ap, "call Ap"); + IEMOP_HLP_NO_64BIT(); + + /* Decode the far pointer address and pass it on to the far call C implementation. */ + uint32_t offSeg; + if (pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT) + IEM_OPCODE_GET_NEXT_U32(&offSeg); + else + IEM_OPCODE_GET_NEXT_U16_ZX_U32(&offSeg); + uint16_t uSel; IEM_OPCODE_GET_NEXT_U16(&uSel); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_callf, uSel, offSeg, pVCpu->iem.s.enmEffOpSize); +} + + +/** Opcode 0x9b. (aka fwait) */ +FNIEMOP_DEF(iemOp_wait) +{ + IEMOP_MNEMONIC(wait, "wait"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0x9c + */ +FNIEMOP_DEF(iemOp_pushf_Fv) +{ + IEMOP_MNEMONIC(pushf_Fv, "pushf Fv"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_pushf, pVCpu->iem.s.enmEffOpSize); +} + + +/** + * @opcode 0x9d + */ +FNIEMOP_DEF(iemOp_popf_Fv) +{ + IEMOP_MNEMONIC(popf_Fv, "popf Fv"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_popf, pVCpu->iem.s.enmEffOpSize); +} + + +/** + * @opcode 0x9e + */ +FNIEMOP_DEF(iemOp_sahf) +{ + IEMOP_MNEMONIC(sahf, "sahf"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if ( pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT + && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLahfSahf) + return IEMOP_RAISE_INVALID_OPCODE(); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Flags); + IEM_MC_LOCAL(uint32_t, EFlags); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_FETCH_GREG_U8_ZX_U32(u32Flags, X86_GREG_xSP/*=AH*/); + IEM_MC_AND_LOCAL_U32(u32Flags, X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); + IEM_MC_AND_LOCAL_U32(EFlags, UINT32_C(0xffffff00)); + IEM_MC_OR_LOCAL_U32(u32Flags, X86_EFL_1); + IEM_MC_OR_2LOCS_U32(EFlags, u32Flags); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0x9f + */ +FNIEMOP_DEF(iemOp_lahf) +{ + IEMOP_MNEMONIC(lahf, "lahf"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if ( pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT + && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLahfSahf) + return IEMOP_RAISE_INVALID_OPCODE(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint8_t, u8Flags); + IEM_MC_FETCH_EFLAGS_U8(u8Flags); + IEM_MC_STORE_GREG_U8(X86_GREG_xSP/*=AH*/, u8Flags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * Macro used by iemOp_mov_AL_Ob, iemOp_mov_rAX_Ov, iemOp_mov_Ob_AL and + * iemOp_mov_Ov_rAX to fetch the moffsXX bit of the opcode and fend off lock + * prefixes. Will return on failures. + * @param a_GCPtrMemOff The variable to store the offset in. + */ +#define IEMOP_FETCH_MOFFS_XX(a_GCPtrMemOff) \ + do \ + { \ + switch (pVCpu->iem.s.enmEffAddrMode) \ + { \ + case IEMMODE_16BIT: \ + IEM_OPCODE_GET_NEXT_U16_ZX_U64(&(a_GCPtrMemOff)); \ + break; \ + case IEMMODE_32BIT: \ + IEM_OPCODE_GET_NEXT_U32_ZX_U64(&(a_GCPtrMemOff)); \ + break; \ + case IEMMODE_64BIT: \ + IEM_OPCODE_GET_NEXT_U64(&(a_GCPtrMemOff)); \ + break; \ + IEM_NOT_REACHED_DEFAULT_CASE_RET(); \ + } \ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \ + } while (0) + +/** + * @opcode 0xa0 + */ +FNIEMOP_DEF(iemOp_mov_AL_Ob) +{ + /* + * Get the offset and fend off lock prefixes. + */ + IEMOP_MNEMONIC(mov_AL_Ob, "mov AL,Ob"); + RTGCPTR GCPtrMemOff; + IEMOP_FETCH_MOFFS_XX(GCPtrMemOff); + + /* + * Fetch AL. + */ + IEM_MC_BEGIN(0,1); + IEM_MC_LOCAL(uint8_t, u8Tmp); + IEM_MC_FETCH_MEM_U8(u8Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff); + IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0xa1 + */ +FNIEMOP_DEF(iemOp_mov_rAX_Ov) +{ + /* + * Get the offset and fend off lock prefixes. + */ + IEMOP_MNEMONIC(mov_rAX_Ov, "mov rAX,Ov"); + RTGCPTR GCPtrMemOff; + IEMOP_FETCH_MOFFS_XX(GCPtrMemOff); + + /* + * Fetch rAX. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0,1); + IEM_MC_LOCAL(uint16_t, u16Tmp); + IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff); + IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0,1); + IEM_MC_LOCAL(uint32_t, u32Tmp); + IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff); + IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u32Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0,1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff); + IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xa2 + */ +FNIEMOP_DEF(iemOp_mov_Ob_AL) +{ + /* + * Get the offset and fend off lock prefixes. + */ + IEMOP_MNEMONIC(mov_Ob_AL, "mov Ob,AL"); + RTGCPTR GCPtrMemOff; + IEMOP_FETCH_MOFFS_XX(GCPtrMemOff); + + /* + * Store AL. + */ + IEM_MC_BEGIN(0,1); + IEM_MC_LOCAL(uint8_t, u8Tmp); + IEM_MC_FETCH_GREG_U8(u8Tmp, X86_GREG_xAX); + IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u8Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0xa3 + */ +FNIEMOP_DEF(iemOp_mov_Ov_rAX) +{ + /* + * Get the offset and fend off lock prefixes. + */ + IEMOP_MNEMONIC(mov_Ov_rAX, "mov Ov,rAX"); + RTGCPTR GCPtrMemOff; + IEMOP_FETCH_MOFFS_XX(GCPtrMemOff); + + /* + * Store rAX. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0,1); + IEM_MC_LOCAL(uint16_t, u16Tmp); + IEM_MC_FETCH_GREG_U16(u16Tmp, X86_GREG_xAX); + IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u16Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0,1); + IEM_MC_LOCAL(uint32_t, u32Tmp); + IEM_MC_FETCH_GREG_U32(u32Tmp, X86_GREG_xAX); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u32Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0,1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + IEM_MC_FETCH_GREG_U64(u64Tmp, X86_GREG_xAX); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u64Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + +/** Macro used by iemOp_movsb_Xb_Yb and iemOp_movswd_Xv_Yv */ +#define IEM_MOVS_CASE(ValBits, AddrBits) \ + IEM_MC_BEGIN(0, 2); \ + IEM_MC_LOCAL(uint##ValBits##_t, uValue); \ + IEM_MC_LOCAL(RTGCPTR, uAddr); \ + IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xSI); \ + IEM_MC_FETCH_MEM_U##ValBits(uValue, pVCpu->iem.s.iEffSeg, uAddr); \ + IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \ + IEM_MC_STORE_MEM_U##ValBits(X86_SREG_ES, uAddr, uValue); \ + IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \ + IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \ + IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \ + } IEM_MC_ELSE() { \ + IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \ + IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END() + +/** + * @opcode 0xa4 + */ +FNIEMOP_DEF(iemOp_movsb_Xb_Yb) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)) + { + IEMOP_MNEMONIC(rep_movsb_Xb_Yb, "rep movsb Xb,Yb"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op8_addr16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op8_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op8_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + IEMOP_MNEMONIC(movsb_Xb_Yb, "movsb Xb,Yb"); + + /* + * Sharing case implementation with movs[wdq] below. + */ + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_MOVS_CASE(8, 16); break; + case IEMMODE_32BIT: IEM_MOVS_CASE(8, 32); break; + case IEMMODE_64BIT: IEM_MOVS_CASE(8, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xa5 + */ +FNIEMOP_DEF(iemOp_movswd_Xv_Yv) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)) + { + IEMOP_MNEMONIC(rep_movs_Xv_Yv, "rep movs Xv,Yv"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op16_addr16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op16_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op16_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op32_addr16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op32_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op32_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_6); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op64_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op64_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + IEMOP_MNEMONIC(movs_Xv_Yv, "movs Xv,Yv"); + + /* + * Annoying double switch here. + * Using ugly macro for implementing the cases, sharing it with movsb. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_MOVS_CASE(16, 16); break; + case IEMMODE_32BIT: IEM_MOVS_CASE(16, 32); break; + case IEMMODE_64BIT: IEM_MOVS_CASE(16, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_MOVS_CASE(32, 16); break; + case IEMMODE_32BIT: IEM_MOVS_CASE(32, 32); break; + case IEMMODE_64BIT: IEM_MOVS_CASE(32, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break; + case IEMMODE_32BIT: IEM_MOVS_CASE(64, 32); break; + case IEMMODE_64BIT: IEM_MOVS_CASE(64, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + +#undef IEM_MOVS_CASE + +/** Macro used by iemOp_cmpsb_Xb_Yb and iemOp_cmpswd_Xv_Yv */ +#define IEM_CMPS_CASE(ValBits, AddrBits) \ + IEM_MC_BEGIN(3, 3); \ + IEM_MC_ARG(uint##ValBits##_t *, puValue1, 0); \ + IEM_MC_ARG(uint##ValBits##_t, uValue2, 1); \ + IEM_MC_ARG(uint32_t *, pEFlags, 2); \ + IEM_MC_LOCAL(uint##ValBits##_t, uValue1); \ + IEM_MC_LOCAL(RTGCPTR, uAddr); \ + \ + IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xSI); \ + IEM_MC_FETCH_MEM_U##ValBits(uValue1, pVCpu->iem.s.iEffSeg, uAddr); \ + IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \ + IEM_MC_FETCH_MEM_U##ValBits(uValue2, X86_SREG_ES, uAddr); \ + IEM_MC_REF_LOCAL(puValue1, uValue1); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cmp_u##ValBits, puValue1, uValue2, pEFlags); \ + \ + IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \ + IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \ + IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \ + } IEM_MC_ELSE() { \ + IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \ + IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END() + +/** + * @opcode 0xa6 + */ +FNIEMOP_DEF(iemOp_cmpsb_Xb_Yb) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ) + { + IEMOP_MNEMONIC(repz_cmps_Xb_Yb, "repz cmps Xb,Yb"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op8_addr16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op8_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op8_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ) + { + IEMOP_MNEMONIC(repnz_cmps_Xb_Yb, "repnz cmps Xb,Yb"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op8_addr16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op8_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op8_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + IEMOP_MNEMONIC(cmps_Xb_Yb, "cmps Xb,Yb"); + + /* + * Sharing case implementation with cmps[wdq] below. + */ + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_CMPS_CASE(8, 16); break; + case IEMMODE_32BIT: IEM_CMPS_CASE(8, 32); break; + case IEMMODE_64BIT: IEM_CMPS_CASE(8, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xa7 + */ +FNIEMOP_DEF(iemOp_cmpswd_Xv_Yv) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ) + { + IEMOP_MNEMONIC(repe_cmps_Xv_Yv, "repe cmps Xv,Yv"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op16_addr16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op16_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op16_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op32_addr16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op32_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op32_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_4); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op64_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op64_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ) + { + IEMOP_MNEMONIC(repne_cmps_Xv_Yv, "repne cmps Xv,Yv"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op16_addr16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op16_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op16_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op32_addr16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op32_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op32_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_2); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op64_addr32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op64_addr64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + + IEMOP_MNEMONIC(cmps_Xv_Yv, "cmps Xv,Yv"); + + /* + * Annoying double switch here. + * Using ugly macro for implementing the cases, sharing it with cmpsb. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_CMPS_CASE(16, 16); break; + case IEMMODE_32BIT: IEM_CMPS_CASE(16, 32); break; + case IEMMODE_64BIT: IEM_CMPS_CASE(16, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_CMPS_CASE(32, 16); break; + case IEMMODE_32BIT: IEM_CMPS_CASE(32, 32); break; + case IEMMODE_64BIT: IEM_CMPS_CASE(32, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break; + case IEMMODE_32BIT: IEM_CMPS_CASE(64, 32); break; + case IEMMODE_64BIT: IEM_CMPS_CASE(64, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + +#undef IEM_CMPS_CASE + +/** + * @opcode 0xa8 + */ +FNIEMOP_DEF(iemOp_test_AL_Ib) +{ + IEMOP_MNEMONIC(test_al_Ib, "test al,Ib"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_test); +} + + +/** + * @opcode 0xa9 + */ +FNIEMOP_DEF(iemOp_test_eAX_Iz) +{ + IEMOP_MNEMONIC(test_rAX_Iz, "test rAX,Iz"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_test); +} + + +/** Macro used by iemOp_stosb_Yb_AL and iemOp_stoswd_Yv_eAX */ +#define IEM_STOS_CASE(ValBits, AddrBits) \ + IEM_MC_BEGIN(0, 2); \ + IEM_MC_LOCAL(uint##ValBits##_t, uValue); \ + IEM_MC_LOCAL(RTGCPTR, uAddr); \ + IEM_MC_FETCH_GREG_U##ValBits(uValue, X86_GREG_xAX); \ + IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \ + IEM_MC_STORE_MEM_U##ValBits(X86_SREG_ES, uAddr, uValue); \ + IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \ + IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \ + } IEM_MC_ELSE() { \ + IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END() + +/** + * @opcode 0xaa + */ +FNIEMOP_DEF(iemOp_stosb_Yb_AL) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)) + { + IEMOP_MNEMONIC(rep_stos_Yb_al, "rep stos Yb,al"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_al_m16); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_al_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_al_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + IEMOP_MNEMONIC(stos_Yb_al, "stos Yb,al"); + + /* + * Sharing case implementation with stos[wdq] below. + */ + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_STOS_CASE(8, 16); break; + case IEMMODE_32BIT: IEM_STOS_CASE(8, 32); break; + case IEMMODE_64BIT: IEM_STOS_CASE(8, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xab + */ +FNIEMOP_DEF(iemOp_stoswd_Yv_eAX) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)) + { + IEMOP_MNEMONIC(rep_stos_Yv_rAX, "rep stos Yv,rAX"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_ax_m16); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_ax_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_ax_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_eax_m16); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_eax_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_eax_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_9); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_rax_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_rax_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + IEMOP_MNEMONIC(stos_Yv_rAX, "stos Yv,rAX"); + + /* + * Annoying double switch here. + * Using ugly macro for implementing the cases, sharing it with stosb. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_STOS_CASE(16, 16); break; + case IEMMODE_32BIT: IEM_STOS_CASE(16, 32); break; + case IEMMODE_64BIT: IEM_STOS_CASE(16, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_STOS_CASE(32, 16); break; + case IEMMODE_32BIT: IEM_STOS_CASE(32, 32); break; + case IEMMODE_64BIT: IEM_STOS_CASE(32, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break; + case IEMMODE_32BIT: IEM_STOS_CASE(64, 32); break; + case IEMMODE_64BIT: IEM_STOS_CASE(64, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + +#undef IEM_STOS_CASE + +/** Macro used by iemOp_lodsb_AL_Xb and iemOp_lodswd_eAX_Xv */ +#define IEM_LODS_CASE(ValBits, AddrBits) \ + IEM_MC_BEGIN(0, 2); \ + IEM_MC_LOCAL(uint##ValBits##_t, uValue); \ + IEM_MC_LOCAL(RTGCPTR, uAddr); \ + IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xSI); \ + IEM_MC_FETCH_MEM_U##ValBits(uValue, pVCpu->iem.s.iEffSeg, uAddr); \ + IEM_MC_STORE_GREG_U##ValBits(X86_GREG_xAX, uValue); \ + IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \ + IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \ + } IEM_MC_ELSE() { \ + IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END() + +/** + * @opcode 0xac + */ +FNIEMOP_DEF(iemOp_lodsb_AL_Xb) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)) + { + IEMOP_MNEMONIC(rep_lodsb_AL_Xb, "rep lodsb AL,Xb"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_al_m16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_al_m32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_al_m64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + IEMOP_MNEMONIC(lodsb_AL_Xb, "lodsb AL,Xb"); + + /* + * Sharing case implementation with stos[wdq] below. + */ + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_LODS_CASE(8, 16); break; + case IEMMODE_32BIT: IEM_LODS_CASE(8, 32); break; + case IEMMODE_64BIT: IEM_LODS_CASE(8, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xad + */ +FNIEMOP_DEF(iemOp_lodswd_eAX_Xv) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)) + { + IEMOP_MNEMONIC(rep_lods_rAX_Xv, "rep lods rAX,Xv"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_ax_m16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_ax_m32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_ax_m64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_eax_m16, pVCpu->iem.s.iEffSeg); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_eax_m32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_eax_m64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_7); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_rax_m32, pVCpu->iem.s.iEffSeg); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_rax_m64, pVCpu->iem.s.iEffSeg); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + IEMOP_MNEMONIC(lods_rAX_Xv, "lods rAX,Xv"); + + /* + * Annoying double switch here. + * Using ugly macro for implementing the cases, sharing it with lodsb. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_LODS_CASE(16, 16); break; + case IEMMODE_32BIT: IEM_LODS_CASE(16, 32); break; + case IEMMODE_64BIT: IEM_LODS_CASE(16, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_LODS_CASE(32, 16); break; + case IEMMODE_32BIT: IEM_LODS_CASE(32, 32); break; + case IEMMODE_64BIT: IEM_LODS_CASE(32, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break; + case IEMMODE_32BIT: IEM_LODS_CASE(64, 32); break; + case IEMMODE_64BIT: IEM_LODS_CASE(64, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + +#undef IEM_LODS_CASE + +/** Macro used by iemOp_scasb_AL_Xb and iemOp_scaswd_eAX_Xv */ +#define IEM_SCAS_CASE(ValBits, AddrBits) \ + IEM_MC_BEGIN(3, 2); \ + IEM_MC_ARG(uint##ValBits##_t *, puRax, 0); \ + IEM_MC_ARG(uint##ValBits##_t, uValue, 1); \ + IEM_MC_ARG(uint32_t *, pEFlags, 2); \ + IEM_MC_LOCAL(RTGCPTR, uAddr); \ + \ + IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \ + IEM_MC_FETCH_MEM_U##ValBits(uValue, X86_SREG_ES, uAddr); \ + IEM_MC_REF_GREG_U##ValBits(puRax, X86_GREG_xAX); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cmp_u##ValBits, puRax, uValue, pEFlags); \ + \ + IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \ + IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \ + } IEM_MC_ELSE() { \ + IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END() + +/** + * @opcode 0xae + */ +FNIEMOP_DEF(iemOp_scasb_AL_Xb) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ) + { + IEMOP_MNEMONIC(repe_scasb_AL_Xb, "repe scasb AL,Xb"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_al_m16); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_al_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_al_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ) + { + IEMOP_MNEMONIC(repone_scasb_AL_Xb, "repne scasb AL,Xb"); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_al_m16); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_al_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_al_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + IEMOP_MNEMONIC(scasb_AL_Xb, "scasb AL,Xb"); + + /* + * Sharing case implementation with stos[wdq] below. + */ + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_SCAS_CASE(8, 16); break; + case IEMMODE_32BIT: IEM_SCAS_CASE(8, 32); break; + case IEMMODE_64BIT: IEM_SCAS_CASE(8, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xaf + */ +FNIEMOP_DEF(iemOp_scaswd_eAX_Xv) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* + * Use the C implementation if a repeat prefix is encountered. + */ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ) + { + IEMOP_MNEMONIC(repe_scas_rAX_Xv, "repe scas rAX,Xv"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_ax_m16); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_ax_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_ax_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_eax_m16); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_eax_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_eax_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_6); /** @todo It's this wrong, we can do 16-bit addressing in 64-bit mode, but not 32-bit. right? */ + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_rax_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_rax_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ) + { + IEMOP_MNEMONIC(repne_scas_rAX_Xv, "repne scas rAX,Xv"); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_ax_m16); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_ax_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_ax_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_eax_m16); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_eax_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_eax_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_5); + case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_rax_m32); + case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_rax_m64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + IEMOP_MNEMONIC(scas_rAX_Xv, "scas rAX,Xv"); + + /* + * Annoying double switch here. + * Using ugly macro for implementing the cases, sharing it with scasb. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_SCAS_CASE(16, 16); break; + case IEMMODE_32BIT: IEM_SCAS_CASE(16, 32); break; + case IEMMODE_64BIT: IEM_SCAS_CASE(16, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_32BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: IEM_SCAS_CASE(32, 16); break; + case IEMMODE_32BIT: IEM_SCAS_CASE(32, 32); break; + case IEMMODE_64BIT: IEM_SCAS_CASE(32, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + + case IEMMODE_64BIT: + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break; + case IEMMODE_32BIT: IEM_SCAS_CASE(64, 32); break; + case IEMMODE_64BIT: IEM_SCAS_CASE(64, 64); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + +#undef IEM_SCAS_CASE + +/** + * Common 'mov r8, imm8' helper. + */ +FNIEMOP_DEF_1(iemOpCommonMov_r8_Ib, uint8_t, iReg) +{ + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL_CONST(uint8_t, u8Value,/*=*/ u8Imm); + IEM_MC_STORE_GREG_U8(iReg, u8Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0xb0 + */ +FNIEMOP_DEF(iemOp_mov_AL_Ib) +{ + IEMOP_MNEMONIC(mov_AL_Ib, "mov AL,Ib"); + return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xAX | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xb1 + */ +FNIEMOP_DEF(iemOp_CL_Ib) +{ + IEMOP_MNEMONIC(mov_CL_Ib, "mov CL,Ib"); + return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xCX | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xb2 + */ +FNIEMOP_DEF(iemOp_DL_Ib) +{ + IEMOP_MNEMONIC(mov_DL_Ib, "mov DL,Ib"); + return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xDX | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xb3 + */ +FNIEMOP_DEF(iemOp_BL_Ib) +{ + IEMOP_MNEMONIC(mov_BL_Ib, "mov BL,Ib"); + return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xBX | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xb4 + */ +FNIEMOP_DEF(iemOp_mov_AH_Ib) +{ + IEMOP_MNEMONIC(mov_AH_Ib, "mov AH,Ib"); + return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xSP | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xb5 + */ +FNIEMOP_DEF(iemOp_CH_Ib) +{ + IEMOP_MNEMONIC(mov_CH_Ib, "mov CH,Ib"); + return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xBP | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xb6 + */ +FNIEMOP_DEF(iemOp_DH_Ib) +{ + IEMOP_MNEMONIC(mov_DH_Ib, "mov DH,Ib"); + return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xSI | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xb7 + */ +FNIEMOP_DEF(iemOp_BH_Ib) +{ + IEMOP_MNEMONIC(mov_BH_Ib, "mov BH,Ib"); + return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xDI | pVCpu->iem.s.uRexB); +} + + +/** + * Common 'mov regX,immX' helper. + */ +FNIEMOP_DEF_1(iemOpCommonMov_Rv_Iv, uint8_t, iReg) +{ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL_CONST(uint16_t, u16Value,/*=*/ u16Imm); + IEM_MC_STORE_GREG_U16(iReg, u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + case IEMMODE_32BIT: + { + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL_CONST(uint32_t, u32Value,/*=*/ u32Imm); + IEM_MC_STORE_GREG_U32(iReg, u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + case IEMMODE_64BIT: + { + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_U64(&u64Imm); /* 64-bit immediate! */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL_CONST(uint64_t, u64Value,/*=*/ u64Imm); + IEM_MC_STORE_GREG_U64(iReg, u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xb8 + */ +FNIEMOP_DEF(iemOp_eAX_Iv) +{ + IEMOP_MNEMONIC(mov_rAX_IV, "mov rAX,IV"); + return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xAX | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xb9 + */ +FNIEMOP_DEF(iemOp_eCX_Iv) +{ + IEMOP_MNEMONIC(mov_rCX_IV, "mov rCX,IV"); + return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xCX | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xba + */ +FNIEMOP_DEF(iemOp_eDX_Iv) +{ + IEMOP_MNEMONIC(mov_rDX_IV, "mov rDX,IV"); + return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xDX | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xbb + */ +FNIEMOP_DEF(iemOp_eBX_Iv) +{ + IEMOP_MNEMONIC(mov_rBX_IV, "mov rBX,IV"); + return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xBX | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xbc + */ +FNIEMOP_DEF(iemOp_eSP_Iv) +{ + IEMOP_MNEMONIC(mov_rSP_IV, "mov rSP,IV"); + return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xSP | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xbd + */ +FNIEMOP_DEF(iemOp_eBP_Iv) +{ + IEMOP_MNEMONIC(mov_rBP_IV, "mov rBP,IV"); + return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xBP | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xbe + */ +FNIEMOP_DEF(iemOp_eSI_Iv) +{ + IEMOP_MNEMONIC(mov_rSI_IV, "mov rSI,IV"); + return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xSI | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xbf + */ +FNIEMOP_DEF(iemOp_eDI_Iv) +{ + IEMOP_MNEMONIC(mov_rDI_IV, "mov rDI,IV"); + return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xDI | pVCpu->iem.s.uRexB); +} + + +/** + * @opcode 0xc0 + */ +FNIEMOP_DEF(iemOp_Grp2_Eb_Ib) +{ + IEMOP_HLP_MIN_186(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + PCIEMOPSHIFTSIZES pImpl; + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Eb_Ib, "rol Eb,Ib"); break; + case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Eb_Ib, "ror Eb,Ib"); break; + case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Eb_Ib, "rcl Eb,Ib"); break; + case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Eb_Ib, "rcr Eb,Ib"); break; + case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Eb_Ib, "shl Eb,Ib"); break; + case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Eb_Ib, "shr Eb,Ib"); break; + case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Eb_Ib, "sar Eb,Ib"); break; + case 6: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe stupid */ + } + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register */ + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg, cShift, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEM_MC_ASSIGN(cShiftArg, cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0xc1 + */ +FNIEMOP_DEF(iemOp_Grp2_Ev_Ib) +{ + IEMOP_HLP_MIN_186(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + PCIEMOPSHIFTSIZES pImpl; + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Ev_Ib, "rol Ev,Ib"); break; + case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Ev_Ib, "ror Ev,Ib"); break; + case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Ev_Ib, "rcl Ev,Ib"); break; + case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Ev_Ib, "rcr Ev,Ib"); break; + case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Ev_Ib, "shl Ev,Ib"); break; + case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Ev_Ib, "shr Ev,Ib"); break; + case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Ev_Ib, "sar Ev,Ib"); break; + case 6: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe stupid */ + } + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register */ + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg, cShift, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg, cShift, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg, cShift, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* memory */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEM_MC_ASSIGN(cShiftArg, cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEM_MC_ASSIGN(cShiftArg, cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEM_MC_ASSIGN(cShiftArg, cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0xc2 + */ +FNIEMOP_DEF(iemOp_retn_Iw) +{ + IEMOP_MNEMONIC(retn_Iw, "retn Iw"); + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_retn_iw_16, u16Imm); + case IEMMODE_32BIT: + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_retn_iw_32, u16Imm); + case IEMMODE_64BIT: + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_retn_iw_64, u16Imm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xc3 + */ +FNIEMOP_DEF(iemOp_retn) +{ + IEMOP_MNEMONIC(retn, "retn"); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_retn_16); + case IEMMODE_32BIT: + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_retn_32); + case IEMMODE_64BIT: + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_retn_64); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xc4 + */ +FNIEMOP_DEF(iemOp_les_Gv_Mp__vex3) +{ + /* The LDS instruction is invalid 64-bit mode. In legacy and + compatability mode it is invalid with MOD=3. + The use as a VEX prefix is made possible by assigning the inverted + REX.R and REX.X to the two MOD bits, since the REX bits are ignored + outside of 64-bit mode. VEX is not available in real or v86 mode. */ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if ( pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT + || IEM_IS_MODRM_REG_MODE(bRm) ) + { + IEMOP_MNEMONIC(vex3_prefix, "vex3"); + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx) + { + /* Note! The real mode, v8086 mode and invalid prefix checks are done once + the instruction is fully decoded. Even when XCR0=3 and CR4.OSXSAVE=0. */ + uint8_t bVex2; IEM_OPCODE_GET_NEXT_U8(&bVex2); + uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_VEX; + if ((bVex2 & 0x80 /* VEX.W */) && pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_W; + pVCpu->iem.s.uRexReg = (~bRm >> (7 - 3)) & 0x8; + pVCpu->iem.s.uRexIndex = (~bRm >> (6 - 3)) & 0x8; + pVCpu->iem.s.uRexB = (~bRm >> (5 - 3)) & 0x8; + pVCpu->iem.s.uVex3rdReg = (~bVex2 >> 3) & 0xf; + pVCpu->iem.s.uVexLength = (bVex2 >> 2) & 1; + pVCpu->iem.s.idxPrefix = bVex2 & 0x3; + + switch (bRm & 0x1f) + { + case 1: /* 0x0f lead opcode byte. */ +#ifdef IEM_WITH_VEX + return FNIEMOP_CALL(g_apfnVexMap1[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]); +#else + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +#endif + + case 2: /* 0x0f 0x38 lead opcode bytes. */ +#ifdef IEM_WITH_VEX + return FNIEMOP_CALL(g_apfnVexMap2[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]); +#else + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +#endif + + case 3: /* 0x0f 0x3a lead opcode bytes. */ +#ifdef IEM_WITH_VEX + return FNIEMOP_CALL(g_apfnVexMap3[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]); +#else + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +#endif + + default: + Log(("VEX3: Invalid vvvv value: %#x!\n", bRm & 0x1f)); + return IEMOP_RAISE_INVALID_OPCODE(); + } + } + Log(("VEX3: AVX support disabled!\n")); + return IEMOP_RAISE_INVALID_OPCODE(); + } + + IEMOP_MNEMONIC(les_Gv_Mp, "les Gv,Mp"); + return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_ES, bRm); +} + + +/** + * @opcode 0xc5 + */ +FNIEMOP_DEF(iemOp_lds_Gv_Mp__vex2) +{ + /* The LES instruction is invalid 64-bit mode. In legacy and + compatability mode it is invalid with MOD=3. + The use as a VEX prefix is made possible by assigning the inverted + REX.R to the top MOD bit, and the top bit in the inverted register + specifier to the bottom MOD bit, thereby effectively limiting 32-bit + to accessing registers 0..7 in this VEX form. */ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if ( pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT + || IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_MNEMONIC(vex2_prefix, "vex2"); + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx) + { + /* Note! The real mode, v8086 mode and invalid prefix checks are done once + the instruction is fully decoded. Even when XCR0=3 and CR4.OSXSAVE=0. */ + uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_VEX; + pVCpu->iem.s.uRexReg = (~bRm >> (7 - 3)) & 0x8; + pVCpu->iem.s.uVex3rdReg = (~bRm >> 3) & 0xf; + pVCpu->iem.s.uVexLength = (bRm >> 2) & 1; + pVCpu->iem.s.idxPrefix = bRm & 0x3; + +#ifdef IEM_WITH_VEX + return FNIEMOP_CALL(g_apfnVexMap1[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]); +#else + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +#endif + } + + /** @todo does intel completely decode the sequence with SIB/disp before \#UD? */ + Log(("VEX2: AVX support disabled!\n")); + return IEMOP_RAISE_INVALID_OPCODE(); + } + + IEMOP_MNEMONIC(lds_Gv_Mp, "lds Gv,Mp"); + return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_DS, bRm); +} + + +/** + * @opcode 0xc6 + */ +FNIEMOP_DEF(iemOp_Grp11_Eb_Ib) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if ((bRm & X86_MODRM_REG_MASK) != (0 << X86_MODRM_REG_SHIFT)) /* only mov Eb,Ib in this group. */ + return IEMOP_RAISE_INVALID_OPCODE(); + IEMOP_MNEMONIC(mov_Eb_Ib, "mov Eb,Ib"); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register access */ + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_RM(pVCpu, bRm), u8Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory access. */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u8Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0xc7 + */ +FNIEMOP_DEF(iemOp_Grp11_Ev_Iz) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if ((bRm & X86_MODRM_REG_MASK) != (0 << X86_MODRM_REG_SHIFT)) /* only mov Eb,Ib in this group. */ + return IEMOP_RAISE_INVALID_OPCODE(); + IEMOP_MNEMONIC(mov_Ev_Iz, "mov Ev,Iz"); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register access */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 0); + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_RM(pVCpu, bRm), u16Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 0); + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 0); + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* memory access. */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2); + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4); + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4); + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Imm); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + + + +/** + * @opcode 0xc8 + */ +FNIEMOP_DEF(iemOp_enter_Iw_Ib) +{ + IEMOP_MNEMONIC(enter_Iw_Ib, "enter Iw,Ib"); + IEMOP_HLP_MIN_186(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + uint16_t cbFrame; IEM_OPCODE_GET_NEXT_U16(&cbFrame); + uint8_t u8NestingLevel; IEM_OPCODE_GET_NEXT_U8(&u8NestingLevel); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_enter, pVCpu->iem.s.enmEffOpSize, cbFrame, u8NestingLevel); +} + + +/** + * @opcode 0xc9 + */ +FNIEMOP_DEF(iemOp_leave) +{ + IEMOP_MNEMONIC(leave, "leave"); + IEMOP_HLP_MIN_186(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_leave, pVCpu->iem.s.enmEffOpSize); +} + + +/** + * @opcode 0xca + */ +FNIEMOP_DEF(iemOp_retf_Iw) +{ + IEMOP_MNEMONIC(retf_Iw, "retf Iw"); + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_retf, pVCpu->iem.s.enmEffOpSize, u16Imm); +} + + +/** + * @opcode 0xcb + */ +FNIEMOP_DEF(iemOp_retf) +{ + IEMOP_MNEMONIC(retf, "retf"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_retf, pVCpu->iem.s.enmEffOpSize, 0); +} + + +/** + * @opcode 0xcc + */ +FNIEMOP_DEF(iemOp_int3) +{ + IEMOP_MNEMONIC(int3, "int3"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_int, X86_XCPT_BP, IEMINT_INT3); +} + + +/** + * @opcode 0xcd + */ +FNIEMOP_DEF(iemOp_int_Ib) +{ + IEMOP_MNEMONIC(int_Ib, "int Ib"); + uint8_t u8Int; IEM_OPCODE_GET_NEXT_U8(&u8Int); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_int, u8Int, IEMINT_INTN); +} + + +/** + * @opcode 0xce + */ +FNIEMOP_DEF(iemOp_into) +{ + IEMOP_MNEMONIC(into, "into"); + IEMOP_HLP_NO_64BIT(); + + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG_CONST(uint8_t, u8Int, /*=*/ X86_XCPT_OF, 0); + IEM_MC_ARG_CONST(IEMINT, enmInt, /*=*/ IEMINT_INTO, 1); + IEM_MC_CALL_CIMPL_2(iemCImpl_int, u8Int, enmInt); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** + * @opcode 0xcf + */ +FNIEMOP_DEF(iemOp_iret) +{ + IEMOP_MNEMONIC(iret, "iret"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_iret, pVCpu->iem.s.enmEffOpSize); +} + + +/** + * @opcode 0xd0 + */ +FNIEMOP_DEF(iemOp_Grp2_Eb_1) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + PCIEMOPSHIFTSIZES pImpl; + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Eb_1, "rol Eb,1"); break; + case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Eb_1, "ror Eb,1"); break; + case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Eb_1, "rcl Eb,1"); break; + case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Eb_1, "rcr Eb,1"); break; + case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Eb_1, "shl Eb,1"); break; + case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Eb_1, "shr Eb,1"); break; + case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Eb_1, "sar Eb,1"); break; + case 6: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe, well... */ + } + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg,/*=*/1, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg,/*=*/1, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + + +/** + * @opcode 0xd1 + */ +FNIEMOP_DEF(iemOp_Grp2_Ev_1) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + PCIEMOPSHIFTSIZES pImpl; + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Ev_1, "rol Ev,1"); break; + case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Ev_1, "ror Ev,1"); break; + case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Ev_1, "rcl Ev,1"); break; + case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Ev_1, "rcr Ev,1"); break; + case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Ev_1, "shl Ev,1"); break; + case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Ev_1, "shr Ev,1"); break; + case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Ev_1, "sar Ev,1"); break; + case 6: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe, well... */ + } + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg,/*=1*/1, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg,/*=1*/1, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg,/*=1*/1, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* memory */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg,/*=1*/1, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg,/*=1*/1, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint8_t, cShiftArg,/*=1*/1, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0xd2 + */ +FNIEMOP_DEF(iemOp_Grp2_Eb_CL) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + PCIEMOPSHIFTSIZES pImpl; + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Eb_CL, "rol Eb,CL"); break; + case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Eb_CL, "ror Eb,CL"); break; + case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Eb_CL, "rcl Eb,CL"); break; + case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Eb_CL, "rcr Eb,CL"); break; + case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Eb_CL, "shl Eb,CL"); break; + case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Eb_CL, "shr Eb,CL"); break; + case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Eb_CL, "sar Eb,CL"); break; + case 6: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc, grr. */ + } + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0xd3 + */ +FNIEMOP_DEF(iemOp_Grp2_Ev_CL) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + PCIEMOPSHIFTSIZES pImpl; + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Ev_CL, "rol Ev,CL"); break; + case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Ev_CL, "ror Ev,CL"); break; + case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Ev_CL, "rcl Ev,CL"); break; + case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Ev_CL, "rcr Ev,CL"); break; + case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Ev_CL, "shl Ev,CL"); break; + case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Ev_CL, "shr Ev,CL"); break; + case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Ev_CL, "sar Ev,CL"); break; + case 6: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe stupid */ + } + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* memory */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint8_t, cShiftArg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + +/** + * @opcode 0xd4 + */ +FNIEMOP_DEF(iemOp_aam_Ib) +{ + IEMOP_MNEMONIC(aam_Ib, "aam Ib"); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_NO_64BIT(); + if (!bImm) + return IEMOP_RAISE_DIVIDE_ERROR(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_aam, bImm); +} + + +/** + * @opcode 0xd5 + */ +FNIEMOP_DEF(iemOp_aad_Ib) +{ + IEMOP_MNEMONIC(aad_Ib, "aad Ib"); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_NO_64BIT(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_aad, bImm); +} + + +/** + * @opcode 0xd6 + */ +FNIEMOP_DEF(iemOp_salc) +{ + IEMOP_MNEMONIC(salc, "salc"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_NO_64BIT(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_STORE_GREG_U8_CONST(X86_GREG_xAX, 0xff); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(X86_GREG_xAX, 0x00); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0xd7 + */ +FNIEMOP_DEF(iemOp_xlat) +{ + IEMOP_MNEMONIC(xlat, "xlat"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_LOCAL(uint8_t, u8Tmp); + IEM_MC_LOCAL(uint16_t, u16Addr); + IEM_MC_FETCH_GREG_U8_ZX_U16(u16Addr, X86_GREG_xAX); + IEM_MC_ADD_GREG_U16_TO_LOCAL(u16Addr, X86_GREG_xBX); + IEM_MC_FETCH_MEM16_U8(u8Tmp, pVCpu->iem.s.iEffSeg, u16Addr); + IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_LOCAL(uint8_t, u8Tmp); + IEM_MC_LOCAL(uint32_t, u32Addr); + IEM_MC_FETCH_GREG_U8_ZX_U32(u32Addr, X86_GREG_xAX); + IEM_MC_ADD_GREG_U32_TO_LOCAL(u32Addr, X86_GREG_xBX); + IEM_MC_FETCH_MEM32_U8(u8Tmp, pVCpu->iem.s.iEffSeg, u32Addr); + IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_LOCAL(uint8_t, u8Tmp); + IEM_MC_LOCAL(uint64_t, u64Addr); + IEM_MC_FETCH_GREG_U8_ZX_U64(u64Addr, X86_GREG_xAX); + IEM_MC_ADD_GREG_U64_TO_LOCAL(u64Addr, X86_GREG_xBX); + IEM_MC_FETCH_MEM_U8(u8Tmp, pVCpu->iem.s.iEffSeg, u64Addr); + IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * Common worker for FPU instructions working on ST0 and STn, and storing the + * result in ST0. + * + * @param bRm Mod R/M byte. + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_2(iemOpHlpFpu_st0_stN, uint8_t, bRm, PFNIEMAIMPLFPUR80, pfnAImpl) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value2, 2); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, IEM_GET_MODRM_RM_8(bRm)) + IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr80Value2); + IEM_MC_STORE_FPU_RESULT(FpuRes, 0); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** + * Common worker for FPU instructions working on ST0 and STn, and only affecting + * flags. + * + * @param bRm Mod R/M byte. + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_2(iemOpHlpFpuNoStore_st0_stN, uint8_t, bRm, PFNIEMAIMPLFPUR80FSW, pfnAImpl) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value2, 2); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, IEM_GET_MODRM_RM_8(bRm)) + IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pu16Fsw, pr80Value1, pr80Value2); + IEM_MC_UPDATE_FSW(u16Fsw); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(UINT8_MAX); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** + * Common worker for FPU instructions working on ST0 and STn, only affecting + * flags, and popping when done. + * + * @param bRm Mod R/M byte. + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_2(iemOpHlpFpuNoStore_st0_stN_pop, uint8_t, bRm, PFNIEMAIMPLFPUR80FSW, pfnAImpl) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value2, 2); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, IEM_GET_MODRM_RM_8(bRm)) + IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pu16Fsw, pr80Value1, pr80Value2); + IEM_MC_UPDATE_FSW_THEN_POP(u16Fsw); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(UINT8_MAX); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd8 11/0. */ +FNIEMOP_DEF_1(iemOp_fadd_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fadd_st0_stN, "fadd st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fadd_r80_by_r80); +} + + +/** Opcode 0xd8 11/1. */ +FNIEMOP_DEF_1(iemOp_fmul_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fmul_st0_stN, "fmul st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fmul_r80_by_r80); +} + + +/** Opcode 0xd8 11/2. */ +FNIEMOP_DEF_1(iemOp_fcom_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcom_st0_stN, "fcom st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN, bRm, iemAImpl_fcom_r80_by_r80); +} + + +/** Opcode 0xd8 11/3. */ +FNIEMOP_DEF_1(iemOp_fcomp_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcomp_st0_stN, "fcomp st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN_pop, bRm, iemAImpl_fcom_r80_by_r80); +} + + +/** Opcode 0xd8 11/4. */ +FNIEMOP_DEF_1(iemOp_fsub_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsub_st0_stN, "fsub st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fsub_r80_by_r80); +} + + +/** Opcode 0xd8 11/5. */ +FNIEMOP_DEF_1(iemOp_fsubr_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsubr_st0_stN, "fsubr st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fsubr_r80_by_r80); +} + + +/** Opcode 0xd8 11/6. */ +FNIEMOP_DEF_1(iemOp_fdiv_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdiv_st0_stN, "fdiv st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fdiv_r80_by_r80); +} + + +/** Opcode 0xd8 11/7. */ +FNIEMOP_DEF_1(iemOp_fdivr_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdivr_st0_stN, "fdivr st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fdivr_r80_by_r80); +} + + +/** + * Common worker for FPU instructions working on ST0 and an m32r, and storing + * the result in ST0. + * + * @param bRm Mod R/M byte. + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_2(iemOpHlpFpu_st0_m32r, uint8_t, bRm, PFNIEMAIMPLFPUR32, pfnAImpl) +{ + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(RTFLOAT32U, r32Val2); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U, pr32Val2, r32Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_R32(r32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr32Val2); + IEM_MC_STORE_FPU_RESULT(FpuRes, 0); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd8 !11/0. */ +FNIEMOP_DEF_1(iemOp_fadd_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fadd_st0_m32r, "fadd st0,m32r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fadd_r80_by_r32); +} + + +/** Opcode 0xd8 !11/1. */ +FNIEMOP_DEF_1(iemOp_fmul_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fmul_st0_m32r, "fmul st0,m32r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fmul_r80_by_r32); +} + + +/** Opcode 0xd8 !11/2. */ +FNIEMOP_DEF_1(iemOp_fcom_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcom_st0_m32r, "fcom st0,m32r"); + + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_LOCAL(RTFLOAT32U, r32Val2); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U, pr32Val2, r32Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_R32(r32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r32, pu16Fsw, pr80Value1, pr32Val2); + IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd8 !11/3. */ +FNIEMOP_DEF_1(iemOp_fcomp_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcomp_st0_m32r, "fcomp st0,m32r"); + + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_LOCAL(RTFLOAT32U, r32Val2); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U, pr32Val2, r32Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_R32(r32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r32, pu16Fsw, pr80Value1, pr32Val2); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd8 !11/4. */ +FNIEMOP_DEF_1(iemOp_fsub_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsub_st0_m32r, "fsub st0,m32r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fsub_r80_by_r32); +} + + +/** Opcode 0xd8 !11/5. */ +FNIEMOP_DEF_1(iemOp_fsubr_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsubr_st0_m32r, "fsubr st0,m32r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fsubr_r80_by_r32); +} + + +/** Opcode 0xd8 !11/6. */ +FNIEMOP_DEF_1(iemOp_fdiv_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdiv_st0_m32r, "fdiv st0,m32r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fdiv_r80_by_r32); +} + + +/** Opcode 0xd8 !11/7. */ +FNIEMOP_DEF_1(iemOp_fdivr_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdivr_st0_m32r, "fdivr st0,m32r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fdivr_r80_by_r32); +} + + +/** + * @opcode 0xd8 + */ +FNIEMOP_DEF(iemOp_EscF0) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xd8 & 0x7); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fadd_stN, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fmul_stN, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fcom_stN, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fcomp_stN, bRm); + case 4: return FNIEMOP_CALL_1(iemOp_fsub_stN, bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fsubr_stN, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fdiv_stN, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fdivr_stN, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fadd_m32r, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fmul_m32r, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fcom_m32r, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fcomp_m32r, bRm); + case 4: return FNIEMOP_CALL_1(iemOp_fsub_m32r, bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fsubr_m32r, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fdiv_m32r, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fdivr_m32r, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0xd9 /0 mem32real + * @sa iemOp_fld_m64r */ +FNIEMOP_DEF_1(iemOp_fld_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fld_m32r, "fld m32r"); + + IEM_MC_BEGIN(2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(RTFLOAT32U, r32Val); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U, pr32Val, r32Val, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_R32(r32Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_IS_EMPTY(7) + IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r80_from_r32, pFpuRes, pr32Val); + IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd9 !11/2 mem32real */ +FNIEMOP_DEF_1(iemOp_fst_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fst_m32r, "fst m32r"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PRTFLOAT32U, pr32Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pr32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r32, pu16Fsw, pr32Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr32Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(pr32Dst); + IEM_MC_MEM_COMMIT_AND_UNMAP(pr32Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd9 !11/3 */ +FNIEMOP_DEF_1(iemOp_fstp_m32r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fstp_m32r, "fstp m32r"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PRTFLOAT32U, pr32Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pr32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r32, pu16Fsw, pr32Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr32Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(pr32Dst); + IEM_MC_MEM_COMMIT_AND_UNMAP(pr32Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd9 !11/4 */ +FNIEMOP_DEF_1(iemOp_fldenv, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fldenv, "fldenv m14/28byte"); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 1); + IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_fldenv, enmEffOpSize, iEffSeg, GCPtrEffSrc); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0xd9 !11/5 */ +FNIEMOP_DEF_1(iemOp_fldcw, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fldcw_m2byte, "fldcw m2byte"); + IEM_MC_BEGIN(1, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(uint16_t, u16Fsw, 0); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FETCH_MEM_U16(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_CIMPL_1(iemCImpl_fldcw, u16Fsw); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0xd9 !11/6 */ +FNIEMOP_DEF_1(iemOp_fnstenv, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fstenv, "fstenv m14/m28byte"); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 1); + IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_fnstenv, enmEffOpSize, iEffSeg, GCPtrEffDst); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0xd9 !11/7 */ +FNIEMOP_DEF_1(iemOp_fnstcw, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fnstcw_m2byte, "fnstcw m2byte"); + IEM_MC_BEGIN(2, 0); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fcw); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ(); + IEM_MC_FETCH_FCW(u16Fcw); + IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Fcw); + IEM_MC_ADVANCE_RIP_AND_FINISH(); /* C0-C3 are documented as undefined, we leave them unmodified. */ + IEM_MC_END(); +} + + +/** Opcode 0xd9 0xd0, 0xd9 0xd8-0xdf, ++?. */ +FNIEMOP_DEF(iemOp_fnop) +{ + IEMOP_MNEMONIC(fnop, "fnop"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + /** @todo Testcase: looks like FNOP leaves FOP alone but updates FPUIP. Could be + * intel optimizations. Investigate. */ + IEM_MC_UPDATE_FPU_OPCODE_IP(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); /* C0-C3 are documented as undefined, we leave them unmodified. */ + IEM_MC_END(); +} + + +/** Opcode 0xd9 11/0 stN */ +FNIEMOP_DEF_1(iemOp_fld_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fld_stN, "fld stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /** @todo Testcase: Check if this raises \#MF? Intel mentioned it not. AMD + * indicates that it does. */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80Value); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, IEM_GET_MODRM_RM_8(bRm)) + IEM_MC_SET_FPU_RESULT(FpuRes, 0 /*FSW*/, pr80Value); + IEM_MC_PUSH_FPU_RESULT(FpuRes); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_UNDERFLOW(); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xd9 11/3 stN */ +FNIEMOP_DEF_1(iemOp_fxch_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fxch_stN, "fxch stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /** @todo Testcase: Check if this raises \#MF? Intel mentioned it not. AMD + * indicates that it does. */ + IEM_MC_BEGIN(1, 3); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80Value1); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80Value2); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_ARG_CONST(uint8_t, iStReg, /*=*/ IEM_GET_MODRM_RM_8(bRm), 0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, IEM_GET_MODRM_RM_8(bRm)) + IEM_MC_SET_FPU_RESULT(FpuRes, X86_FSW_C1, pr80Value2); + IEM_MC_STORE_FPUREG_R80_SRC_REF(IEM_GET_MODRM_RM_8(bRm), pr80Value1); + IEM_MC_STORE_FPU_RESULT(FpuRes, 0); + IEM_MC_ELSE() + IEM_MC_CALL_CIMPL_1(iemCImpl_fxch_underflow, iStReg); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xd9 11/4, 0xdd 11/2. */ +FNIEMOP_DEF_1(iemOp_fstp_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fstp_st0_stN, "fstp st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + /* fstp st0, st0 is frequently used as an official 'ffreep st0' sequence. */ + uint8_t const iDstReg = IEM_GET_MODRM_RM_8(bRm); + if (!iDstReg) + { + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL_CONST(uint16_t, u16Fsw, /*=*/ 0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY(0) + IEM_MC_UPDATE_FSW_THEN_POP(u16Fsw); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(0); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80Value); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_SET_FPU_RESULT(FpuRes, 0 /*FSW*/, pr80Value); + IEM_MC_STORE_FPU_RESULT_THEN_POP(FpuRes, iDstReg); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(iDstReg); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for FPU instructions working on ST0 and replaces it with the + * result, i.e. unary operators. + * + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpFpu_st0, PFNIEMAIMPLFPUR80UNARY, pfnAImpl) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(2, 1); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 1); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_2(pfnAImpl, pFpuRes, pr80Value); + IEM_MC_STORE_FPU_RESULT(FpuRes, 0); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd9 0xe0. */ +FNIEMOP_DEF(iemOp_fchs) +{ + IEMOP_MNEMONIC(fchs_st0, "fchs st0"); + return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fchs_r80); +} + + +/** Opcode 0xd9 0xe1. */ +FNIEMOP_DEF(iemOp_fabs) +{ + IEMOP_MNEMONIC(fabs_st0, "fabs st0"); + return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fabs_r80); +} + + +/** Opcode 0xd9 0xe4. */ +FNIEMOP_DEF(iemOp_ftst) +{ + IEMOP_MNEMONIC(ftst_st0, "ftst st0"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(2, 1); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 1); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_ftst_r80, pu16Fsw, pr80Value); + IEM_MC_UPDATE_FSW(u16Fsw); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(UINT8_MAX); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd9 0xe5. */ +FNIEMOP_DEF(iemOp_fxam) +{ + IEMOP_MNEMONIC(fxam_st0, "fxam st0"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(2, 1); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 1); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_REF_FPUREG(pr80Value, 0); + IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fxam_r80, pu16Fsw, pr80Value); + IEM_MC_UPDATE_FSW(u16Fsw); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** + * Common worker for FPU instructions pushing a constant onto the FPU stack. + * + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpFpuPushConstant, PFNIEMAIMPLFPUR80LDCONST, pfnAImpl) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(1, 1); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_IS_EMPTY(7) + IEM_MC_CALL_FPU_AIMPL_1(pfnAImpl, pFpuRes); + IEM_MC_PUSH_FPU_RESULT(FpuRes); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_OVERFLOW(); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd9 0xe8. */ +FNIEMOP_DEF(iemOp_fld1) +{ + IEMOP_MNEMONIC(fld1, "fld1"); + return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fld1); +} + + +/** Opcode 0xd9 0xe9. */ +FNIEMOP_DEF(iemOp_fldl2t) +{ + IEMOP_MNEMONIC(fldl2t, "fldl2t"); + return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldl2t); +} + + +/** Opcode 0xd9 0xea. */ +FNIEMOP_DEF(iemOp_fldl2e) +{ + IEMOP_MNEMONIC(fldl2e, "fldl2e"); + return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldl2e); +} + +/** Opcode 0xd9 0xeb. */ +FNIEMOP_DEF(iemOp_fldpi) +{ + IEMOP_MNEMONIC(fldpi, "fldpi"); + return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldpi); +} + + +/** Opcode 0xd9 0xec. */ +FNIEMOP_DEF(iemOp_fldlg2) +{ + IEMOP_MNEMONIC(fldlg2, "fldlg2"); + return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldlg2); +} + +/** Opcode 0xd9 0xed. */ +FNIEMOP_DEF(iemOp_fldln2) +{ + IEMOP_MNEMONIC(fldln2, "fldln2"); + return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldln2); +} + + +/** Opcode 0xd9 0xee. */ +FNIEMOP_DEF(iemOp_fldz) +{ + IEMOP_MNEMONIC(fldz, "fldz"); + return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldz); +} + + +/** Opcode 0xd9 0xf0. + * + * The f2xm1 instruction works on values +1.0 thru -1.0, currently (the range on + * 287 & 8087 was +0.5 thru 0.0 according to docs). In addition is does appear + * to produce proper results for +Inf and -Inf. + * + * This is probably usful in the implementation pow() and similar. + */ +FNIEMOP_DEF(iemOp_f2xm1) +{ + IEMOP_MNEMONIC(f2xm1_st0, "f2xm1 st0"); + return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_f2xm1_r80); +} + + +/** + * Common worker for FPU instructions working on STn and ST0, storing the result + * in STn, and popping the stack unless IE, DE or ZE was raised. + * + * @param bRm Mod R/M byte. + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_2(iemOpHlpFpu_stN_st0_pop, uint8_t, bRm, PFNIEMAIMPLFPUR80, pfnAImpl) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value2, 2); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, IEM_GET_MODRM_RM_8(bRm), pr80Value2, 0) + IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr80Value2); + IEM_MC_STORE_FPU_RESULT_THEN_POP(FpuRes, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd9 0xf1. */ +FNIEMOP_DEF(iemOp_fyl2x) +{ + IEMOP_MNEMONIC(fyl2x_st0, "fyl2x st1,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, 1, iemAImpl_fyl2x_r80_by_r80); +} + + +/** + * Common worker for FPU instructions working on ST0 and having two outputs, one + * replacing ST0 and one pushed onto the stack. + * + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpFpuReplace_st0_push, PFNIEMAIMPLFPUR80UNARYTWO, pfnAImpl) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(2, 1); + IEM_MC_LOCAL(IEMFPURESULTTWO, FpuResTwo); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULTTWO, pFpuResTwo, FpuResTwo, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 1); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_2(pfnAImpl, pFpuResTwo, pr80Value); + IEM_MC_PUSH_FPU_RESULT_TWO(FpuResTwo); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xd9 0xf2. */ +FNIEMOP_DEF(iemOp_fptan) +{ + IEMOP_MNEMONIC(fptan_st0, "fptan st0"); + return FNIEMOP_CALL_1(iemOpHlpFpuReplace_st0_push, iemAImpl_fptan_r80_r80); +} + + +/** Opcode 0xd9 0xf3. */ +FNIEMOP_DEF(iemOp_fpatan) +{ + IEMOP_MNEMONIC(fpatan_st1_st0, "fpatan st1,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, 1, iemAImpl_fpatan_r80_by_r80); +} + + +/** Opcode 0xd9 0xf4. */ +FNIEMOP_DEF(iemOp_fxtract) +{ + IEMOP_MNEMONIC(fxtract_st0, "fxtract st0"); + return FNIEMOP_CALL_1(iemOpHlpFpuReplace_st0_push, iemAImpl_fxtract_r80_r80); +} + + +/** Opcode 0xd9 0xf5. */ +FNIEMOP_DEF(iemOp_fprem1) +{ + IEMOP_MNEMONIC(fprem1_st0_st1, "fprem1 st0,st1"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, 1, iemAImpl_fprem1_r80_by_r80); +} + + +/** Opcode 0xd9 0xf6. */ +FNIEMOP_DEF(iemOp_fdecstp) +{ + IEMOP_MNEMONIC(fdecstp, "fdecstp"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + /* Note! C0, C2 and C3 are documented as undefined, we clear them. */ + /** @todo Testcase: Check whether FOP, FPUIP and FPUCS are affected by + * FINCSTP and FDECSTP. */ + + IEM_MC_BEGIN(0,0); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_STACK_DEC_TOP(); + IEM_MC_UPDATE_FSW_CONST(0); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xd9 0xf7. */ +FNIEMOP_DEF(iemOp_fincstp) +{ + IEMOP_MNEMONIC(fincstp, "fincstp"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + /* Note! C0, C2 and C3 are documented as undefined, we clear them. */ + /** @todo Testcase: Check whether FOP, FPUIP and FPUCS are affected by + * FINCSTP and FDECSTP. */ + + IEM_MC_BEGIN(0,0); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_STACK_INC_TOP(); + IEM_MC_UPDATE_FSW_CONST(0); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xd9 0xf8. */ +FNIEMOP_DEF(iemOp_fprem) +{ + IEMOP_MNEMONIC(fprem_st0_st1, "fprem st0,st1"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, 1, iemAImpl_fprem_r80_by_r80); +} + + +/** Opcode 0xd9 0xf9. */ +FNIEMOP_DEF(iemOp_fyl2xp1) +{ + IEMOP_MNEMONIC(fyl2xp1_st1_st0, "fyl2xp1 st1,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, 1, iemAImpl_fyl2xp1_r80_by_r80); +} + + +/** Opcode 0xd9 0xfa. */ +FNIEMOP_DEF(iemOp_fsqrt) +{ + IEMOP_MNEMONIC(fsqrt_st0, "fsqrt st0"); + return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fsqrt_r80); +} + + +/** Opcode 0xd9 0xfb. */ +FNIEMOP_DEF(iemOp_fsincos) +{ + IEMOP_MNEMONIC(fsincos_st0, "fsincos st0"); + return FNIEMOP_CALL_1(iemOpHlpFpuReplace_st0_push, iemAImpl_fsincos_r80_r80); +} + + +/** Opcode 0xd9 0xfc. */ +FNIEMOP_DEF(iemOp_frndint) +{ + IEMOP_MNEMONIC(frndint_st0, "frndint st0"); + return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_frndint_r80); +} + + +/** Opcode 0xd9 0xfd. */ +FNIEMOP_DEF(iemOp_fscale) +{ + IEMOP_MNEMONIC(fscale_st0_st1, "fscale st0,st1"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, 1, iemAImpl_fscale_r80_by_r80); +} + + +/** Opcode 0xd9 0xfe. */ +FNIEMOP_DEF(iemOp_fsin) +{ + IEMOP_MNEMONIC(fsin_st0, "fsin st0"); + return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fsin_r80); +} + + +/** Opcode 0xd9 0xff. */ +FNIEMOP_DEF(iemOp_fcos) +{ + IEMOP_MNEMONIC(fcos_st0, "fcos st0"); + return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fcos_r80); +} + + +/** Used by iemOp_EscF1. */ +IEM_STATIC const PFNIEMOP g_apfnEscF1_E0toFF[32] = +{ + /* 0xe0 */ iemOp_fchs, + /* 0xe1 */ iemOp_fabs, + /* 0xe2 */ iemOp_Invalid, + /* 0xe3 */ iemOp_Invalid, + /* 0xe4 */ iemOp_ftst, + /* 0xe5 */ iemOp_fxam, + /* 0xe6 */ iemOp_Invalid, + /* 0xe7 */ iemOp_Invalid, + /* 0xe8 */ iemOp_fld1, + /* 0xe9 */ iemOp_fldl2t, + /* 0xea */ iemOp_fldl2e, + /* 0xeb */ iemOp_fldpi, + /* 0xec */ iemOp_fldlg2, + /* 0xed */ iemOp_fldln2, + /* 0xee */ iemOp_fldz, + /* 0xef */ iemOp_Invalid, + /* 0xf0 */ iemOp_f2xm1, + /* 0xf1 */ iemOp_fyl2x, + /* 0xf2 */ iemOp_fptan, + /* 0xf3 */ iemOp_fpatan, + /* 0xf4 */ iemOp_fxtract, + /* 0xf5 */ iemOp_fprem1, + /* 0xf6 */ iemOp_fdecstp, + /* 0xf7 */ iemOp_fincstp, + /* 0xf8 */ iemOp_fprem, + /* 0xf9 */ iemOp_fyl2xp1, + /* 0xfa */ iemOp_fsqrt, + /* 0xfb */ iemOp_fsincos, + /* 0xfc */ iemOp_frndint, + /* 0xfd */ iemOp_fscale, + /* 0xfe */ iemOp_fsin, + /* 0xff */ iemOp_fcos +}; + + +/** + * @opcode 0xd9 + */ +FNIEMOP_DEF(iemOp_EscF1) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xd9 & 0x7); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fld_stN, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fxch_stN, bRm); + case 2: + if (bRm == 0xd0) + return FNIEMOP_CALL(iemOp_fnop); + return IEMOP_RAISE_INVALID_OPCODE(); + case 3: return FNIEMOP_CALL_1(iemOp_fstp_stN, bRm); /* Reserved. Intel behavior seems to be FSTP ST(i) though. */ + case 4: + case 5: + case 6: + case 7: + Assert((unsigned)bRm - 0xe0U < RT_ELEMENTS(g_apfnEscF1_E0toFF)); + return FNIEMOP_CALL(g_apfnEscF1_E0toFF[bRm - 0xe0]); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fld_m32r, bRm); + case 1: return IEMOP_RAISE_INVALID_OPCODE(); + case 2: return FNIEMOP_CALL_1(iemOp_fst_m32r, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fstp_m32r, bRm); + case 4: return FNIEMOP_CALL_1(iemOp_fldenv, bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fldcw, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fnstenv, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fnstcw, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0xda 11/0. */ +FNIEMOP_DEF_1(iemOp_fcmovb_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcmovb_st0_stN, "fcmovb st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80ValueN); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0) + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) + IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN); + IEM_MC_ENDIF(); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xda 11/1. */ +FNIEMOP_DEF_1(iemOp_fcmove_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcmove_st0_stN, "fcmove st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80ValueN); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0) + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) + IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN); + IEM_MC_ENDIF(); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xda 11/2. */ +FNIEMOP_DEF_1(iemOp_fcmovbe_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcmovbe_st0_stN, "fcmovbe st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80ValueN); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0) + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) + IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN); + IEM_MC_ENDIF(); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xda 11/3. */ +FNIEMOP_DEF_1(iemOp_fcmovu_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcmovu_st0_stN, "fcmovu st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80ValueN); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0) + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) + IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN); + IEM_MC_ENDIF(); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** + * Common worker for FPU instructions working on ST0 and ST1, only affecting + * flags, and popping twice when done. + * + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpFpuNoStore_st0_st1_pop_pop, PFNIEMAIMPLFPUR80FSW, pfnAImpl) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value2, 2); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, 1) + IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pu16Fsw, pr80Value1, pr80Value2); + IEM_MC_UPDATE_FSW_THEN_POP_POP(u16Fsw); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xda 0xe9. */ +FNIEMOP_DEF(iemOp_fucompp) +{ + IEMOP_MNEMONIC(fucompp, "fucompp"); + return FNIEMOP_CALL_1(iemOpHlpFpuNoStore_st0_st1_pop_pop, iemAImpl_fucom_r80_by_r80); +} + + +/** + * Common worker for FPU instructions working on ST0 and an m32i, and storing + * the result in ST0. + * + * @param bRm Mod R/M byte. + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_2(iemOpHlpFpu_st0_m32i, uint8_t, bRm, PFNIEMAIMPLFPUI32, pfnAImpl) +{ + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(int32_t, i32Val2); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(int32_t const *, pi32Val2, i32Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_I32(i32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pi32Val2); + IEM_MC_STORE_FPU_RESULT(FpuRes, 0); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xda !11/0. */ +FNIEMOP_DEF_1(iemOp_fiadd_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fiadd_m32i, "fiadd m32i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fiadd_r80_by_i32); +} + + +/** Opcode 0xda !11/1. */ +FNIEMOP_DEF_1(iemOp_fimul_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fimul_m32i, "fimul m32i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fimul_r80_by_i32); +} + + +/** Opcode 0xda !11/2. */ +FNIEMOP_DEF_1(iemOp_ficom_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(ficom_st0_m32i, "ficom st0,m32i"); + + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_LOCAL(int32_t, i32Val2); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(int32_t const *, pi32Val2, i32Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_I32(i32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i32, pu16Fsw, pr80Value1, pi32Val2); + IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xda !11/3. */ +FNIEMOP_DEF_1(iemOp_ficomp_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(ficomp_st0_m32i, "ficomp st0,m32i"); + + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_LOCAL(int32_t, i32Val2); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(int32_t const *, pi32Val2, i32Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_I32(i32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i32, pu16Fsw, pr80Value1, pi32Val2); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xda !11/4. */ +FNIEMOP_DEF_1(iemOp_fisub_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fisub_m32i, "fisub m32i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fisub_r80_by_i32); +} + + +/** Opcode 0xda !11/5. */ +FNIEMOP_DEF_1(iemOp_fisubr_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fisubr_m32i, "fisubr m32i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fisubr_r80_by_i32); +} + + +/** Opcode 0xda !11/6. */ +FNIEMOP_DEF_1(iemOp_fidiv_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fidiv_m32i, "fidiv m32i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fidiv_r80_by_i32); +} + + +/** Opcode 0xda !11/7. */ +FNIEMOP_DEF_1(iemOp_fidivr_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fidivr_m32i, "fidivr m32i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fidivr_r80_by_i32); +} + + +/** + * @opcode 0xda + */ +FNIEMOP_DEF(iemOp_EscF2) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xda & 0x7); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fcmovb_stN, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fcmove_stN, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fcmovbe_stN, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fcmovu_stN, bRm); + case 4: return IEMOP_RAISE_INVALID_OPCODE(); + case 5: + if (bRm == 0xe9) + return FNIEMOP_CALL(iemOp_fucompp); + return IEMOP_RAISE_INVALID_OPCODE(); + case 6: return IEMOP_RAISE_INVALID_OPCODE(); + case 7: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fiadd_m32i, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fimul_m32i, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_ficom_m32i, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_ficomp_m32i, bRm); + case 4: return FNIEMOP_CALL_1(iemOp_fisub_m32i, bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fisubr_m32i, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fidiv_m32i, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fidivr_m32i, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0xdb !11/0. */ +FNIEMOP_DEF_1(iemOp_fild_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fild_m32i, "fild m32i"); + + IEM_MC_BEGIN(2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(int32_t, i32Val); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG_LOCAL_REF(int32_t const *, pi32Val, i32Val, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_I32(i32Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_IS_EMPTY(7) + IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fild_r80_from_i32, pFpuRes, pi32Val); + IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb !11/1. */ +FNIEMOP_DEF_1(iemOp_fisttp_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fisttp_m32i, "fisttp m32i"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(int32_t *, pi32Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pi32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fistt_r80_to_i32, pu16Fsw, pi32Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi32Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_I32_CONST_BY_REF(pi32Dst, INT32_MIN /* (integer indefinite) */); + IEM_MC_MEM_COMMIT_AND_UNMAP(pi32Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb !11/2. */ +FNIEMOP_DEF_1(iemOp_fist_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fist_m32i, "fist m32i"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(int32_t *, pi32Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pi32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i32, pu16Fsw, pi32Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi32Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_I32_CONST_BY_REF(pi32Dst, INT32_MIN /* (integer indefinite) */); + IEM_MC_MEM_COMMIT_AND_UNMAP(pi32Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb !11/3. */ +FNIEMOP_DEF_1(iemOp_fistp_m32i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fistp_m32i, "fistp m32i"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(int32_t *, pi32Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pi32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i32, pu16Fsw, pi32Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi32Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_I32_CONST_BY_REF(pi32Dst, INT32_MIN /* (integer indefinite) */); + IEM_MC_MEM_COMMIT_AND_UNMAP(pi32Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb !11/5. */ +FNIEMOP_DEF_1(iemOp_fld_m80r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fld_m80r, "fld m80r"); + + IEM_MC_BEGIN(2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(RTFLOAT80U, r80Val); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT80U, pr80Val, r80Val, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_R80(r80Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_IS_EMPTY(7) + IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r80_from_r80, pFpuRes, pr80Val); + IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb !11/7. */ +FNIEMOP_DEF_1(iemOp_fstp_m80r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fstp_m80r, "fstp m80r"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PRTFLOAT80U, pr80Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP_EX(pr80Dst, IEM_ACCESS_DATA_W, sizeof(*pr80Dst), pVCpu->iem.s.iEffSeg, GCPtrEffDst, 7 /*cbAlign*/, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r80, pu16Fsw, pr80Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr80Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(pr80Dst); + IEM_MC_MEM_COMMIT_AND_UNMAP(pr80Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb 11/0. */ +FNIEMOP_DEF_1(iemOp_fcmovnb_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcmovnb_st0_stN, "fcmovnb st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80ValueN); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0) + IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_CF) + IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN); + IEM_MC_ENDIF(); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb 11/1. */ +FNIEMOP_DEF_1(iemOp_fcmovne_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcmovne_st0_stN, "fcmovne st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80ValueN); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0) + IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) + IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN); + IEM_MC_ENDIF(); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb 11/2. */ +FNIEMOP_DEF_1(iemOp_fcmovnbe_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcmovnbe_st0_stN, "fcmovnbe st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80ValueN); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0) + IEM_MC_IF_EFL_NO_BITS_SET(X86_EFL_CF | X86_EFL_ZF) + IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN); + IEM_MC_ENDIF(); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb 11/3. */ +FNIEMOP_DEF_1(iemOp_fcmovnnu_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcmovnnu_st0_stN, "fcmovnnu st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80ValueN); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0) + IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_PF) + IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN); + IEM_MC_ENDIF(); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdb 0xe0. */ +FNIEMOP_DEF(iemOp_fneni) +{ + IEMOP_MNEMONIC(fneni, "fneni (8087/ign)"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0,0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xdb 0xe1. */ +FNIEMOP_DEF(iemOp_fndisi) +{ + IEMOP_MNEMONIC(fndisi, "fndisi (8087/ign)"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0,0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xdb 0xe2. */ +FNIEMOP_DEF(iemOp_fnclex) +{ + IEMOP_MNEMONIC(fnclex, "fnclex"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0,0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_CLEAR_FSW_EX(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xdb 0xe3. */ +FNIEMOP_DEF(iemOp_fninit) +{ + IEMOP_MNEMONIC(fninit, "fninit"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_finit, false /*fCheckXcpts*/); +} + + +/** Opcode 0xdb 0xe4. */ +FNIEMOP_DEF(iemOp_fnsetpm) +{ + IEMOP_MNEMONIC(fnsetpm, "fnsetpm (80287/ign)"); /* set protected mode on fpu. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0,0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xdb 0xe5. */ +FNIEMOP_DEF(iemOp_frstpm) +{ + IEMOP_MNEMONIC(frstpm, "frstpm (80287XL/ign)"); /* reset pm, back to real mode. */ +#if 0 /* #UDs on newer CPUs */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0,0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + return VINF_SUCCESS; +#else + return IEMOP_RAISE_INVALID_OPCODE(); +#endif +} + + +/** Opcode 0xdb 11/5. */ +FNIEMOP_DEF_1(iemOp_fucomi_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fucomi_st0_stN, "fucomi st0,stN"); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, IEM_GET_MODRM_RM_8(bRm), iemAImpl_fucomi_r80_by_r80, false /*fPop*/); +} + + +/** Opcode 0xdb 11/6. */ +FNIEMOP_DEF_1(iemOp_fcomi_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcomi_st0_stN, "fcomi st0,stN"); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, IEM_GET_MODRM_RM_8(bRm), iemAImpl_fcomi_r80_by_r80, false /*fPop*/); +} + + +/** + * @opcode 0xdb + */ +FNIEMOP_DEF(iemOp_EscF3) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xdb & 0x7); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fcmovnb_stN, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fcmovne_stN, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fcmovnbe_stN, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fcmovnnu_stN, bRm); + case 4: + switch (bRm) + { + case 0xe0: return FNIEMOP_CALL(iemOp_fneni); + case 0xe1: return FNIEMOP_CALL(iemOp_fndisi); + case 0xe2: return FNIEMOP_CALL(iemOp_fnclex); + case 0xe3: return FNIEMOP_CALL(iemOp_fninit); + case 0xe4: return FNIEMOP_CALL(iemOp_fnsetpm); + case 0xe5: return FNIEMOP_CALL(iemOp_frstpm); + case 0xe6: return IEMOP_RAISE_INVALID_OPCODE(); + case 0xe7: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + break; + case 5: return FNIEMOP_CALL_1(iemOp_fucomi_stN, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fcomi_stN, bRm); + case 7: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fild_m32i, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fisttp_m32i,bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fist_m32i, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fistp_m32i, bRm); + case 4: return IEMOP_RAISE_INVALID_OPCODE(); + case 5: return FNIEMOP_CALL_1(iemOp_fld_m80r, bRm); + case 6: return IEMOP_RAISE_INVALID_OPCODE(); + case 7: return FNIEMOP_CALL_1(iemOp_fstp_m80r, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * Common worker for FPU instructions working on STn and ST0, and storing the + * result in STn unless IE, DE or ZE was raised. + * + * @param bRm Mod R/M byte. + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_2(iemOpHlpFpu_stN_st0, uint8_t, bRm, PFNIEMAIMPLFPUR80, pfnAImpl) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value2, 2); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, IEM_GET_MODRM_RM_8(bRm), pr80Value2, 0) + IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr80Value2); + IEM_MC_STORE_FPU_RESULT(FpuRes, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdc 11/0. */ +FNIEMOP_DEF_1(iemOp_fadd_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fadd_stN_st0, "fadd stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fadd_r80_by_r80); +} + + +/** Opcode 0xdc 11/1. */ +FNIEMOP_DEF_1(iemOp_fmul_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fmul_stN_st0, "fmul stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fmul_r80_by_r80); +} + + +/** Opcode 0xdc 11/4. */ +FNIEMOP_DEF_1(iemOp_fsubr_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsubr_stN_st0, "fsubr stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fsubr_r80_by_r80); +} + + +/** Opcode 0xdc 11/5. */ +FNIEMOP_DEF_1(iemOp_fsub_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsub_stN_st0, "fsub stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fsub_r80_by_r80); +} + + +/** Opcode 0xdc 11/6. */ +FNIEMOP_DEF_1(iemOp_fdivr_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdivr_stN_st0, "fdivr stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fdivr_r80_by_r80); +} + + +/** Opcode 0xdc 11/7. */ +FNIEMOP_DEF_1(iemOp_fdiv_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdiv_stN_st0, "fdiv stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fdiv_r80_by_r80); +} + + +/** + * Common worker for FPU instructions working on ST0 and a 64-bit floating point + * memory operand, and storing the result in ST0. + * + * @param bRm Mod R/M byte. + * @param pfnImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_2(iemOpHlpFpu_ST0_m64r, uint8_t, bRm, PFNIEMAIMPLFPUR64, pfnImpl) +{ + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(RTFLOAT64U, r64Factor2); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Factor1, 1); + IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U, pr64Factor2, r64Factor2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_FETCH_MEM_R64(r64Factor2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Factor1, 0) + IEM_MC_CALL_FPU_AIMPL_3(pfnImpl, pFpuRes, pr80Factor1, pr64Factor2); + IEM_MC_STORE_FPU_RESULT_MEM_OP(FpuRes, 0, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(0, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdc !11/0. */ +FNIEMOP_DEF_1(iemOp_fadd_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fadd_m64r, "fadd m64r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fadd_r80_by_r64); +} + + +/** Opcode 0xdc !11/1. */ +FNIEMOP_DEF_1(iemOp_fmul_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fmul_m64r, "fmul m64r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fmul_r80_by_r64); +} + + +/** Opcode 0xdc !11/2. */ +FNIEMOP_DEF_1(iemOp_fcom_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcom_st0_m64r, "fcom st0,m64r"); + + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_LOCAL(RTFLOAT64U, r64Val2); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U, pr64Val2, r64Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_R64(r64Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r64, pu16Fsw, pr80Value1, pr64Val2); + IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdc !11/3. */ +FNIEMOP_DEF_1(iemOp_fcomp_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcomp_st0_m64r, "fcomp st0,m64r"); + + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_LOCAL(RTFLOAT64U, r64Val2); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U, pr64Val2, r64Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_R64(r64Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r64, pu16Fsw, pr80Value1, pr64Val2); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdc !11/4. */ +FNIEMOP_DEF_1(iemOp_fsub_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsub_m64r, "fsub m64r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fsub_r80_by_r64); +} + + +/** Opcode 0xdc !11/5. */ +FNIEMOP_DEF_1(iemOp_fsubr_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsubr_m64r, "fsubr m64r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fsubr_r80_by_r64); +} + + +/** Opcode 0xdc !11/6. */ +FNIEMOP_DEF_1(iemOp_fdiv_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdiv_m64r, "fdiv m64r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fdiv_r80_by_r64); +} + + +/** Opcode 0xdc !11/7. */ +FNIEMOP_DEF_1(iemOp_fdivr_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdivr_m64r, "fdivr m64r"); + return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fdivr_r80_by_r64); +} + + +/** + * @opcode 0xdc + */ +FNIEMOP_DEF(iemOp_EscF4) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xdc & 0x7); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fadd_stN_st0, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fmul_stN_st0, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fcom_stN, bRm); /* Marked reserved, intel behavior is that of FCOM ST(i). */ + case 3: return FNIEMOP_CALL_1(iemOp_fcomp_stN, bRm); /* Marked reserved, intel behavior is that of FCOMP ST(i). */ + case 4: return FNIEMOP_CALL_1(iemOp_fsubr_stN_st0, bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fsub_stN_st0, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fdivr_stN_st0, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fdiv_stN_st0, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fadd_m64r, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fmul_m64r, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fcom_m64r, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fcomp_m64r, bRm); + case 4: return FNIEMOP_CALL_1(iemOp_fsub_m64r, bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fsubr_m64r, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fdiv_m64r, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fdivr_m64r, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0xdd !11/0. + * @sa iemOp_fld_m32r */ +FNIEMOP_DEF_1(iemOp_fld_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fld_m64r, "fld m64r"); + + IEM_MC_BEGIN(2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(RTFLOAT64U, r64Val); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U, pr64Val, r64Val, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_FETCH_MEM_R64(r64Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_IS_EMPTY(7) + IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r80_from_r64, pFpuRes, pr64Val); + IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdd !11/0. */ +FNIEMOP_DEF_1(iemOp_fisttp_m64i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fisttp_m64i, "fisttp m64i"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(int64_t *, pi64Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pi64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fistt_r80_to_i64, pu16Fsw, pi64Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi64Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_I64_CONST_BY_REF(pi64Dst, INT64_MIN /* (integer indefinite) */); + IEM_MC_MEM_COMMIT_AND_UNMAP(pi64Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdd !11/0. */ +FNIEMOP_DEF_1(iemOp_fst_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fst_m64r, "fst m64r"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PRTFLOAT64U, pr64Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pr64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r64, pu16Fsw, pr64Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr64Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(pr64Dst); + IEM_MC_MEM_COMMIT_AND_UNMAP(pr64Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + + + +/** Opcode 0xdd !11/0. */ +FNIEMOP_DEF_1(iemOp_fstp_m64r, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fstp_m64r, "fstp m64r"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PRTFLOAT64U, pr64Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pr64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r64, pu16Fsw, pr64Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr64Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(pr64Dst); + IEM_MC_MEM_COMMIT_AND_UNMAP(pr64Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdd !11/0. */ +FNIEMOP_DEF_1(iemOp_frstor, uint8_t, bRm) +{ + IEMOP_MNEMONIC(frstor, "frstor m94/108byte"); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 1); + IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_frstor, enmEffOpSize, iEffSeg, GCPtrEffSrc); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0xdd !11/0. */ +FNIEMOP_DEF_1(iemOp_fnsave, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fnsave, "fnsave m94/108byte"); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 1); + IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); /* Note! Implicit fninit after the save, do not use FOR_READ here! */ + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_fnsave, enmEffOpSize, iEffSeg, GCPtrEffDst); + IEM_MC_END(); + return VINF_SUCCESS; + +} + +/** Opcode 0xdd !11/0. */ +FNIEMOP_DEF_1(iemOp_fnstsw, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fnstsw_m16, "fnstsw m16"); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + + IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ(); + IEM_MC_FETCH_FSW(u16Tmp); + IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + +/** @todo Debug / drop a hint to the verifier that things may differ + * from REM. Seen 0x4020 (iem) vs 0x4000 (rem) at 0008:801c6b88 booting + * NT4SP1. (X86_FSW_PE) */ + IEM_MC_END(); +} + + +/** Opcode 0xdd 11/0. */ +FNIEMOP_DEF_1(iemOp_ffree_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(ffree_stN, "ffree stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + /* Note! C0, C1, C2 and C3 are documented as undefined, we leave the + unmodified. */ + + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_STACK_FREE(IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xdd 11/1. */ +FNIEMOP_DEF_1(iemOp_fst_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fst_st0_stN, "fst st0,stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(PCRTFLOAT80U, pr80Value); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_SET_FPU_RESULT(FpuRes, 0 /*FSW*/, pr80Value); + IEM_MC_STORE_FPU_RESULT(FpuRes, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xdd 11/3. */ +FNIEMOP_DEF_1(iemOp_fucom_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fucom_st0_stN, "fucom st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN, bRm, iemAImpl_fucom_r80_by_r80); +} + + +/** Opcode 0xdd 11/4. */ +FNIEMOP_DEF_1(iemOp_fucomp_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fucomp_st0_stN, "fucomp st0,stN"); + return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN_pop, bRm, iemAImpl_fucom_r80_by_r80); +} + + +/** + * @opcode 0xdd + */ +FNIEMOP_DEF(iemOp_EscF5) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xdd & 0x7); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_ffree_stN, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fxch_stN, bRm); /* Reserved, intel behavior is that of XCHG ST(i). */ + case 2: return FNIEMOP_CALL_1(iemOp_fst_stN, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fstp_stN, bRm); + case 4: return FNIEMOP_CALL_1(iemOp_fucom_stN_st0,bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fucomp_stN, bRm); + case 6: return IEMOP_RAISE_INVALID_OPCODE(); + case 7: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fld_m64r, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fisttp_m64i, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fst_m64r, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fstp_m64r, bRm); + case 4: return FNIEMOP_CALL_1(iemOp_frstor, bRm); + case 5: return IEMOP_RAISE_INVALID_OPCODE(); + case 6: return FNIEMOP_CALL_1(iemOp_fnsave, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fnstsw, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0xde 11/0. */ +FNIEMOP_DEF_1(iemOp_faddp_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(faddp_stN_st0, "faddp stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fadd_r80_by_r80); +} + + +/** Opcode 0xde 11/0. */ +FNIEMOP_DEF_1(iemOp_fmulp_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fmulp_stN_st0, "fmulp stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fmul_r80_by_r80); +} + + +/** Opcode 0xde 0xd9. */ +FNIEMOP_DEF(iemOp_fcompp) +{ + IEMOP_MNEMONIC(fcompp, "fcompp"); + return FNIEMOP_CALL_1(iemOpHlpFpuNoStore_st0_st1_pop_pop, iemAImpl_fcom_r80_by_r80); +} + + +/** Opcode 0xde 11/4. */ +FNIEMOP_DEF_1(iemOp_fsubrp_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsubrp_stN_st0, "fsubrp stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fsubr_r80_by_r80); +} + + +/** Opcode 0xde 11/5. */ +FNIEMOP_DEF_1(iemOp_fsubp_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fsubp_stN_st0, "fsubp stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fsub_r80_by_r80); +} + + +/** Opcode 0xde 11/6. */ +FNIEMOP_DEF_1(iemOp_fdivrp_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdivrp_stN_st0, "fdivrp stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fdivr_r80_by_r80); +} + + +/** Opcode 0xde 11/7. */ +FNIEMOP_DEF_1(iemOp_fdivp_stN_st0, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fdivp_stN_st0, "fdivp stN,st0"); + return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fdiv_r80_by_r80); +} + + +/** + * Common worker for FPU instructions working on ST0 and an m16i, and storing + * the result in ST0. + * + * @param bRm Mod R/M byte. + * @param pfnAImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_2(iemOpHlpFpu_st0_m16i, uint8_t, bRm, PFNIEMAIMPLFPUI16, pfnAImpl) +{ + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(int16_t, i16Val2); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(int16_t const *, pi16Val2, i16Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_I16(i16Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pi16Val2); + IEM_MC_STORE_FPU_RESULT(FpuRes, 0); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW(0); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xde !11/0. */ +FNIEMOP_DEF_1(iemOp_fiadd_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fiadd_m16i, "fiadd m16i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fiadd_r80_by_i16); +} + + +/** Opcode 0xde !11/1. */ +FNIEMOP_DEF_1(iemOp_fimul_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fimul_m16i, "fimul m16i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fimul_r80_by_i16); +} + + +/** Opcode 0xde !11/2. */ +FNIEMOP_DEF_1(iemOp_ficom_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(ficom_st0_m16i, "ficom st0,m16i"); + + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_LOCAL(int16_t, i16Val2); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(int16_t const *, pi16Val2, i16Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_I16(i16Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i16, pu16Fsw, pr80Value1, pi16Val2); + IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xde !11/3. */ +FNIEMOP_DEF_1(iemOp_ficomp_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(ficomp_st0_m16i, "ficomp st0,m16i"); + + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_LOCAL(int16_t, i16Val2); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value1, 1); + IEM_MC_ARG_LOCAL_REF(int16_t const *, pi16Val2, i16Val2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_I16(i16Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i16, pu16Fsw, pr80Value1, pi16Val2); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xde !11/4. */ +FNIEMOP_DEF_1(iemOp_fisub_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fisub_m16i, "fisub m16i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fisub_r80_by_i16); +} + + +/** Opcode 0xde !11/5. */ +FNIEMOP_DEF_1(iemOp_fisubr_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fisubr_m16i, "fisubr m16i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fisubr_r80_by_i16); +} + + +/** Opcode 0xde !11/6. */ +FNIEMOP_DEF_1(iemOp_fidiv_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fidiv_m16i, "fidiv m16i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fidiv_r80_by_i16); +} + + +/** Opcode 0xde !11/7. */ +FNIEMOP_DEF_1(iemOp_fidivr_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fidivr_m16i, "fidivr m16i"); + return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fidivr_r80_by_i16); +} + + +/** + * @opcode 0xde + */ +FNIEMOP_DEF(iemOp_EscF6) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xde & 0x7); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_faddp_stN_st0, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fmulp_stN_st0, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fcomp_stN, bRm); + case 3: if (bRm == 0xd9) + return FNIEMOP_CALL(iemOp_fcompp); + return IEMOP_RAISE_INVALID_OPCODE(); + case 4: return FNIEMOP_CALL_1(iemOp_fsubrp_stN_st0, bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fsubp_stN_st0, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fdivrp_stN_st0, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fdivp_stN_st0, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fiadd_m16i, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fimul_m16i, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_ficom_m16i, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_ficomp_m16i, bRm); + case 4: return FNIEMOP_CALL_1(iemOp_fisub_m16i, bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fisubr_m16i, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fidiv_m16i, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fidivr_m16i, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0xdf 11/0. + * Undocument instruction, assumed to work like ffree + fincstp. */ +FNIEMOP_DEF_1(iemOp_ffreep_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(ffreep_stN, "ffreep stN"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_STACK_FREE(IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_FPU_STACK_INC_TOP(); + IEM_MC_UPDATE_FPU_OPCODE_IP(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xdf 0xe0. */ +FNIEMOP_DEF(iemOp_fnstsw_ax) +{ + IEMOP_MNEMONIC(fnstsw_ax, "fnstsw ax"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Tmp); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ(); + IEM_MC_FETCH_FSW(u16Tmp); + IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Tmp); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xdf 11/5. */ +FNIEMOP_DEF_1(iemOp_fucomip_st0_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fucomip_st0_stN, "fucomip st0,stN"); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, IEM_GET_MODRM_RM_8(bRm), iemAImpl_fcomi_r80_by_r80, true /*fPop*/); +} + + +/** Opcode 0xdf 11/6. */ +FNIEMOP_DEF_1(iemOp_fcomip_st0_stN, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fcomip_st0_stN, "fcomip st0,stN"); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, IEM_GET_MODRM_RM_8(bRm), iemAImpl_fcomi_r80_by_r80, true /*fPop*/); +} + + +/** Opcode 0xdf !11/0. */ +FNIEMOP_DEF_1(iemOp_fild_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fild_m16i, "fild m16i"); + + IEM_MC_BEGIN(2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(int16_t, i16Val); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG_LOCAL_REF(int16_t const *, pi16Val, i16Val, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_I16(i16Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_IS_EMPTY(7) + IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fild_r80_from_i16, pFpuRes, pi16Val); + IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdf !11/1. */ +FNIEMOP_DEF_1(iemOp_fisttp_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fisttp_m16i, "fisttp m16i"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(int16_t *, pi16Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pi16Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fistt_r80_to_i16, pu16Fsw, pi16Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi16Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_I16_CONST_BY_REF(pi16Dst, INT16_MIN /* (integer indefinite) */); + IEM_MC_MEM_COMMIT_AND_UNMAP(pi16Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdf !11/2. */ +FNIEMOP_DEF_1(iemOp_fist_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fist_m16i, "fist m16i"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(int16_t *, pi16Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pi16Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i16, pu16Fsw, pi16Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi16Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_I16_CONST_BY_REF(pi16Dst, INT16_MIN /* (integer indefinite) */); + IEM_MC_MEM_COMMIT_AND_UNMAP(pi16Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdf !11/3. */ +FNIEMOP_DEF_1(iemOp_fistp_m16i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fistp_m16i, "fistp m16i"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(int16_t *, pi16Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pi16Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i16, pu16Fsw, pi16Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi16Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_I16_CONST_BY_REF(pi16Dst, INT16_MIN /* (integer indefinite) */); + IEM_MC_MEM_COMMIT_AND_UNMAP(pi16Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdf !11/4. */ +FNIEMOP_DEF_1(iemOp_fbld_m80d, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fbld_m80d, "fbld m80d"); + + IEM_MC_BEGIN(2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(RTPBCD80U, d80Val); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG_LOCAL_REF(PCRTPBCD80U, pd80Val, d80Val, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_D80(d80Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_IS_EMPTY(7) + IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r80_from_d80, pFpuRes, pd80Val); + IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdf !11/5. */ +FNIEMOP_DEF_1(iemOp_fild_m64i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fild_m64i, "fild m64i"); + + IEM_MC_BEGIN(2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(IEMFPURESULT, FpuRes); + IEM_MC_LOCAL(int64_t, i64Val); + IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes, FpuRes, 0); + IEM_MC_ARG_LOCAL_REF(int64_t const *, pi64Val, i64Val, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_I64(i64Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_IS_EMPTY(7) + IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fild_r80_from_i64, pFpuRes, pi64Val); + IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ELSE() + IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdf !11/6. */ +FNIEMOP_DEF_1(iemOp_fbstp_m80d, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fbstp_m80d, "fbstp m80d"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(PRTPBCD80U, pd80Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP_EX(pd80Dst, IEM_ACCESS_DATA_W, sizeof(*pd80Dst), pVCpu->iem.s.iEffSeg, GCPtrEffDst, 7 /*cbAlign*/, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_d80, pu16Fsw, pd80Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pd80Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_INDEF_D80_BY_REF(pd80Dst); + IEM_MC_MEM_COMMIT_AND_UNMAP(pd80Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode 0xdf !11/7. */ +FNIEMOP_DEF_1(iemOp_fistp_m64i, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fistp_m64i, "fistp m64i"); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Fsw); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Fsw, u16Fsw, 0); + IEM_MC_ARG(int64_t *, pi64Dst, 1); + IEM_MC_ARG(PCRTFLOAT80U, pr80Value, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + + IEM_MC_MEM_MAP(pi64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0) + IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i64, pu16Fsw, pi64Dst, pr80Value); + IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi64Dst, IEM_ACCESS_DATA_W, u16Fsw); + IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ELSE() + IEM_MC_IF_FCW_IM() + IEM_MC_STORE_MEM_I64_CONST_BY_REF(pi64Dst, INT64_MIN /* (integer indefinite) */); + IEM_MC_MEM_COMMIT_AND_UNMAP(pi64Dst, IEM_ACCESS_DATA_W); + IEM_MC_ENDIF(); + IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** + * @opcode 0xdf + */ +FNIEMOP_DEF(iemOp_EscF7) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_ffreep_stN, bRm); /* ffree + pop afterwards, since forever according to AMD. */ + case 1: return FNIEMOP_CALL_1(iemOp_fxch_stN, bRm); /* Reserved, behaves like FXCH ST(i) on intel. */ + case 2: return FNIEMOP_CALL_1(iemOp_fstp_stN, bRm); /* Reserved, behaves like FSTP ST(i) on intel. */ + case 3: return FNIEMOP_CALL_1(iemOp_fstp_stN, bRm); /* Reserved, behaves like FSTP ST(i) on intel. */ + case 4: if (bRm == 0xe0) + return FNIEMOP_CALL(iemOp_fnstsw_ax); + return IEMOP_RAISE_INVALID_OPCODE(); + case 5: return FNIEMOP_CALL_1(iemOp_fucomip_st0_stN, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fcomip_st0_stN, bRm); + case 7: return IEMOP_RAISE_INVALID_OPCODE(); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: return FNIEMOP_CALL_1(iemOp_fild_m16i, bRm); + case 1: return FNIEMOP_CALL_1(iemOp_fisttp_m16i, bRm); + case 2: return FNIEMOP_CALL_1(iemOp_fist_m16i, bRm); + case 3: return FNIEMOP_CALL_1(iemOp_fistp_m16i, bRm); + case 4: return FNIEMOP_CALL_1(iemOp_fbld_m80d, bRm); + case 5: return FNIEMOP_CALL_1(iemOp_fild_m64i, bRm); + case 6: return FNIEMOP_CALL_1(iemOp_fbstp_m80d, bRm); + case 7: return FNIEMOP_CALL_1(iemOp_fistp_m64i, bRm); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * @opcode 0xe0 + */ +FNIEMOP_DEF(iemOp_loopne_Jb) +{ + IEMOP_MNEMONIC(loopne_Jb, "loopne Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_SUB_GREG_U16(X86_GREG_xCX, 1); + IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(X86_EFL_ZF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_SUB_GREG_U32(X86_GREG_xCX, 1); + IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(X86_EFL_ZF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_SUB_GREG_U64(X86_GREG_xCX, 1); + IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(X86_EFL_ZF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xe1 + */ +FNIEMOP_DEF(iemOp_loope_Jb) +{ + IEMOP_MNEMONIC(loope_Jb, "loope Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_SUB_GREG_U16(X86_GREG_xCX, 1); + IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_SUB_GREG_U32(X86_GREG_xCX, 1); + IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_SUB_GREG_U64(X86_GREG_xCX, 1); + IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xe2 + */ +FNIEMOP_DEF(iemOp_loop_Jb) +{ + IEMOP_MNEMONIC(loop_Jb, "loop Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + + /** @todo Check out the \#GP case if EIP < CS.Base or EIP > CS.Limit when + * using the 32-bit operand size override. How can that be restarted? See + * weird pseudo code in intel manual. */ + + /* NB: At least Windows for Workgroups 3.11 (NDIS.386) and Windows 95 (NDIS.VXD, IOS) + * use LOOP $-2 to implement NdisStallExecution and other CPU stall APIs. Shortcutting + * the loop causes guest crashes, but when logging it's nice to skip a few million + * lines of useless output. */ +#if defined(LOG_ENABLED) + if ((LogIs3Enabled() || LogIs4Enabled()) && (-(int8_t)IEM_GET_INSTR_LEN(pVCpu) == i8Imm)) + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_STORE_GREG_U16_CONST(X86_GREG_xCX, 0); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_STORE_GREG_U32_CONST(X86_GREG_xCX, 0); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_STORE_GREG_U64_CONST(X86_GREG_xCX, 0); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +#endif + + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0,0); + + IEM_MC_SUB_GREG_U16(X86_GREG_xCX, 1); + IEM_MC_IF_CX_IS_NZ() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_SUB_GREG_U32(X86_GREG_xCX, 1); + IEM_MC_IF_ECX_IS_NZ() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_SUB_GREG_U64(X86_GREG_xCX, 1); + IEM_MC_IF_RCX_IS_NZ() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xe3 + */ +FNIEMOP_DEF(iemOp_jecxz_Jb) +{ + IEMOP_MNEMONIC(jecxz_Jb, "jecxz Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + + switch (pVCpu->iem.s.enmEffAddrMode) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_IF_CX_IS_NZ() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_IF_ECX_IS_NZ() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0,0); + IEM_MC_IF_RCX_IS_NZ() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** Opcode 0xe4 */ +FNIEMOP_DEF(iemOp_in_AL_Ib) +{ + IEMOP_MNEMONIC(in_AL_Ib, "in AL,Ib"); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_in, u8Imm, true /* fImm */, 1); +} + + +/** Opcode 0xe5 */ +FNIEMOP_DEF(iemOp_in_eAX_Ib) +{ + IEMOP_MNEMONIC(in_eAX_Ib, "in eAX,Ib"); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_in, u8Imm, true /* fImm */, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4); +} + + +/** Opcode 0xe6 */ +FNIEMOP_DEF(iemOp_out_Ib_AL) +{ + IEMOP_MNEMONIC(out_Ib_AL, "out Ib,AL"); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_out, u8Imm, true /* fImm */, 1); +} + + +/** Opcode 0xe7 */ +FNIEMOP_DEF(iemOp_out_Ib_eAX) +{ + IEMOP_MNEMONIC(out_Ib_eAX, "out Ib,eAX"); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_out, u8Imm, true /* fImm */, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4); +} + + +/** + * @opcode 0xe8 + */ +FNIEMOP_DEF(iemOp_call_Jv) +{ + IEMOP_MNEMONIC(call_Jv, "call Jv"); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_call_rel_16, (int16_t)u16Imm); + } + + case IEMMODE_32BIT: + { + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_call_rel_32, (int32_t)u32Imm); + } + + case IEMMODE_64BIT: + { + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_call_rel_64, u64Imm); + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xe9 + */ +FNIEMOP_DEF(iemOp_jmp_Jv) +{ + IEMOP_MNEMONIC(jmp_Jv, "jmp Jv"); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEM_MC_BEGIN(0, 0); + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + IEM_MC_END(); + return VINF_SUCCESS; + } + + case IEMMODE_64BIT: + case IEMMODE_32BIT: + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEM_MC_BEGIN(0, 0); + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + IEM_MC_END(); + return VINF_SUCCESS; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xea + */ +FNIEMOP_DEF(iemOp_jmp_Ap) +{ + IEMOP_MNEMONIC(jmp_Ap, "jmp Ap"); + IEMOP_HLP_NO_64BIT(); + + /* Decode the far pointer address and pass it on to the far call C implementation. */ + uint32_t offSeg; + if (pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT) + IEM_OPCODE_GET_NEXT_U32(&offSeg); + else + IEM_OPCODE_GET_NEXT_U16_ZX_U32(&offSeg); + uint16_t uSel; IEM_OPCODE_GET_NEXT_U16(&uSel); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_FarJmp, uSel, offSeg, pVCpu->iem.s.enmEffOpSize); +} + + +/** + * @opcode 0xeb + */ +FNIEMOP_DEF(iemOp_jmp_Jb) +{ + IEMOP_MNEMONIC(jmp_Jb, "jmp Jb"); + int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0xec */ +FNIEMOP_DEF(iemOp_in_AL_DX) +{ + IEMOP_MNEMONIC(in_AL_DX, "in AL,DX"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_in_eAX_DX, 1); +} + + +/** Opcode 0xed */ +FNIEMOP_DEF(iemOp_in_eAX_DX) +{ + IEMOP_MNEMONIC(in_eAX_DX, "in eAX,DX"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_in_eAX_DX, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4); +} + + +/** Opcode 0xee */ +FNIEMOP_DEF(iemOp_out_DX_AL) +{ + IEMOP_MNEMONIC(out_DX_AL, "out DX,AL"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_out_DX_eAX, 1); +} + + +/** Opcode 0xef */ +FNIEMOP_DEF(iemOp_out_DX_eAX) +{ + IEMOP_MNEMONIC(out_DX_eAX, "out DX,eAX"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_out_DX_eAX, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4); +} + + +/** + * @opcode 0xf0 + */ +FNIEMOP_DEF(iemOp_lock) +{ + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("lock"); + if (!pVCpu->iem.s.fDisregardLock) + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_LOCK; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0xf1 + */ +FNIEMOP_DEF(iemOp_int1) +{ + IEMOP_MNEMONIC(int1, "int1"); /* icebp */ + /** @todo Does not generate \#UD on 286, or so they say... Was allegedly a + * prefix byte on 8086 and/or/maybe 80286 without meaning according to the 286 + * LOADALL memo. Needs some testing. */ + IEMOP_HLP_MIN_386(); + /** @todo testcase! */ + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_int, X86_XCPT_DB, IEMINT_INT1); +} + + +/** + * @opcode 0xf2 + */ +FNIEMOP_DEF(iemOp_repne) +{ + /* This overrides any previous REPE prefix. */ + pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REPZ; + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("repne"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REPNZ; + + /* For the 4 entry opcode tables, REPNZ overrides any previous + REPZ and operand size prefixes. */ + pVCpu->iem.s.idxPrefix = 3; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0xf3 + */ +FNIEMOP_DEF(iemOp_repe) +{ + /* This overrides any previous REPNE prefix. */ + pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REPNZ; + IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("repe"); + pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REPZ; + + /* For the 4 entry opcode tables, REPNZ overrides any previous + REPNZ and operand size prefixes. */ + pVCpu->iem.s.idxPrefix = 2; + + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnOneByteMap[b]); +} + + +/** + * @opcode 0xf4 + */ +FNIEMOP_DEF(iemOp_hlt) +{ + IEMOP_MNEMONIC(hlt, "hlt"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_hlt); +} + + +/** + * @opcode 0xf5 + */ +FNIEMOP_DEF(iemOp_cmc) +{ + IEMOP_MNEMONIC(cmc, "cmc"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_FLIP_EFL_BIT(X86_EFL_CF); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * Common implementation of 'inc/dec/not/neg Eb'. + * + * @param bRm The RM byte. + * @param pImpl The instruction implementation. + */ +FNIEMOP_DEF_2(iemOpCommonUnaryEb, uint8_t, bRm, PCIEMOPUNARYSIZES, pImpl) +{ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register access */ + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU8, pu8Dst, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory access. */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU8, pu8Dst, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU8, pu8Dst, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common implementation of 'inc/dec/not/neg Ev'. + * + * @param bRm The RM byte. + * @param pImpl The instruction implementation. + */ +FNIEMOP_DEF_2(iemOpCommonUnaryEv, uint8_t, bRm, PCIEMOPUNARYSIZES, pImpl) +{ + /* Registers are handled by a common worker. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, pImpl, IEM_GET_MODRM_RM(pVCpu, bRm)); + + /* Memory we do here. */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU16, pu16Dst, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU16, pu16Dst, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU32, pu32Dst, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU32, pu32Dst, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU64, pu64Dst, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU64, pu64Dst, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** Opcode 0xf6 /0. */ +FNIEMOP_DEF_1(iemOp_grp3_test_Eb, uint8_t, bRm) +{ + IEMOP_MNEMONIC(test_Eb_Ib, "test Eb,Ib"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register access */ + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG_CONST(uint8_t, u8Src,/*=*/u8Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u8, pu8Dst, u8Src, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory access. */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t, u8Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm); + IEM_MC_ASSIGN(u8Src, u8Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u8, pu8Dst, u8Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_R); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0xf7 /0. */ +FNIEMOP_DEF_1(iemOp_grp3_test_Ev, uint8_t, bRm) +{ + IEMOP_MNEMONIC(test_Ev_Iv, "test Ev,Iv"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register access */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint16_t, u16Src,/*=*/u16Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u16, pu16Dst, u16Src, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + case IEMMODE_32BIT: + { + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint32_t, u32Src,/*=*/u32Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u32, pu32Dst, u32Src, pEFlags); + /* No clearing the high dword here - test doesn't write back the result. */ + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + case IEMMODE_64BIT: + { + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint64_t, u64Src,/*=*/u64Imm, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u64, pu64Dst, u64Src, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* memory access. */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2); + uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm); + IEM_MC_ASSIGN(u16Src, u16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u16, pu16Dst, u16Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_R); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + case IEMMODE_32BIT: + { + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4); + uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm); + IEM_MC_ASSIGN(u32Src, u32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u32, pu32Dst, u32Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_R); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + case IEMMODE_64BIT: + { + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4); + uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm); + IEM_MC_ASSIGN(u64Src, u64Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u64, pu64Dst, u64Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_R); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0xf6 /4, /5, /6 and /7. */ +FNIEMOP_DEF_2(iemOpCommonGrp3MulDivEb, uint8_t, bRm, PFNIEMAIMPLMULDIVU8, pfnU8) +{ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register access */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t *, pu16AX, 0); + IEM_MC_ARG(uint8_t, u8Value, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(int32_t, rc); + + IEM_MC_FETCH_GREG_U8(u8Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_AIMPL_3(rc, pfnU8, pu16AX, u8Value, pEFlags); + IEM_MC_IF_LOCAL_IS_Z(rc) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_RAISE_DIVIDE_ERROR(); + } IEM_MC_ENDIF(); + + IEM_MC_END(); + } + else + { + /* memory access. */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16AX, 0); + IEM_MC_ARG(uint8_t, u8Value, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(int32_t, rc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8(u8Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_AIMPL_3(rc, pfnU8, pu16AX, u8Value, pEFlags); + IEM_MC_IF_LOCAL_IS_Z(rc) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_RAISE_DIVIDE_ERROR(); + } IEM_MC_ENDIF(); + + IEM_MC_END(); + } +} + + +/** Opcode 0xf7 /4, /5, /6 and /7. */ +FNIEMOP_DEF_2(iemOpCommonGrp3MulDivEv, uint8_t, bRm, PCIEMOPMULDIVSIZES, pImpl) +{ + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register access */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 1); + IEM_MC_ARG(uint16_t *, pu16AX, 0); + IEM_MC_ARG(uint16_t *, pu16DX, 1); + IEM_MC_ARG(uint16_t, u16Value, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_LOCAL(int32_t, rc); + + IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX); + IEM_MC_REF_GREG_U16(pu16DX, X86_GREG_xDX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU16, pu16AX, pu16DX, u16Value, pEFlags); + IEM_MC_IF_LOCAL_IS_Z(rc) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_RAISE_DIVIDE_ERROR(); + } IEM_MC_ENDIF(); + + IEM_MC_END(); + break; + } + + case IEMMODE_32BIT: + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 1); + IEM_MC_ARG(uint32_t *, pu32AX, 0); + IEM_MC_ARG(uint32_t *, pu32DX, 1); + IEM_MC_ARG(uint32_t, u32Value, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_LOCAL(int32_t, rc); + + IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32AX, X86_GREG_xAX); + IEM_MC_REF_GREG_U32(pu32DX, X86_GREG_xDX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU32, pu32AX, pu32DX, u32Value, pEFlags); + IEM_MC_IF_LOCAL_IS_Z(rc) { + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32AX); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32DX); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_RAISE_DIVIDE_ERROR(); + } IEM_MC_ENDIF(); + + IEM_MC_END(); + break; + } + + case IEMMODE_64BIT: + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 1); + IEM_MC_ARG(uint64_t *, pu64AX, 0); + IEM_MC_ARG(uint64_t *, pu64DX, 1); + IEM_MC_ARG(uint64_t, u64Value, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_LOCAL(int32_t, rc); + + IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pu64AX, X86_GREG_xAX); + IEM_MC_REF_GREG_U64(pu64DX, X86_GREG_xDX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU64, pu64AX, pu64DX, u64Value, pEFlags); + IEM_MC_IF_LOCAL_IS_Z(rc) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_RAISE_DIVIDE_ERROR(); + } IEM_MC_ENDIF(); + + IEM_MC_END(); + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* memory access. */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + IEM_MC_BEGIN(4, 2); + IEM_MC_ARG(uint16_t *, pu16AX, 0); + IEM_MC_ARG(uint16_t *, pu16DX, 1); + IEM_MC_ARG(uint16_t, u16Value, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(int32_t, rc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX); + IEM_MC_REF_GREG_U16(pu16DX, X86_GREG_xDX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU16, pu16AX, pu16DX, u16Value, pEFlags); + IEM_MC_IF_LOCAL_IS_Z(rc) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_RAISE_DIVIDE_ERROR(); + } IEM_MC_ENDIF(); + + IEM_MC_END(); + break; + } + + case IEMMODE_32BIT: + { + IEM_MC_BEGIN(4, 2); + IEM_MC_ARG(uint32_t *, pu32AX, 0); + IEM_MC_ARG(uint32_t *, pu32DX, 1); + IEM_MC_ARG(uint32_t, u32Value, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(int32_t, rc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U32(pu32AX, X86_GREG_xAX); + IEM_MC_REF_GREG_U32(pu32DX, X86_GREG_xDX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU32, pu32AX, pu32DX, u32Value, pEFlags); + IEM_MC_IF_LOCAL_IS_Z(rc) { + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32AX); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32DX); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_RAISE_DIVIDE_ERROR(); + } IEM_MC_ENDIF(); + + IEM_MC_END(); + break; + } + + case IEMMODE_64BIT: + { + IEM_MC_BEGIN(4, 2); + IEM_MC_ARG(uint64_t *, pu64AX, 0); + IEM_MC_ARG(uint64_t *, pu64DX, 1); + IEM_MC_ARG(uint64_t, u64Value, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(int32_t, rc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U64(pu64AX, X86_GREG_xAX); + IEM_MC_REF_GREG_U64(pu64DX, X86_GREG_xDX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU64, pu64AX, pu64DX, u64Value, pEFlags); + IEM_MC_IF_LOCAL_IS_Z(rc) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_RAISE_DIVIDE_ERROR(); + } IEM_MC_ENDIF(); + + IEM_MC_END(); + break; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + +/** + * @opcode 0xf6 + */ +FNIEMOP_DEF(iemOp_Grp3_Eb) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: + return FNIEMOP_CALL_1(iemOp_grp3_test_Eb, bRm); + case 1: +/** @todo testcase: Present on <=386, most 486 (not early), Pentiums, and current CPUs too. CPUUNDOC.EXE */ + return IEMOP_RAISE_INVALID_OPCODE(); + case 2: + IEMOP_MNEMONIC(not_Eb, "not Eb"); + return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_not); + case 3: + IEMOP_MNEMONIC(neg_Eb, "neg Eb"); + return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_neg); + case 4: + IEMOP_MNEMONIC(mul_Eb, "mul Eb"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_mul_u8_eflags)); + case 5: + IEMOP_MNEMONIC(imul_Eb, "imul Eb"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_u8_eflags)); + case 6: + IEMOP_MNEMONIC(div_Eb, "div Eb"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF); + return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_div_u8_eflags)); + case 7: + IEMOP_MNEMONIC(idiv_Eb, "idiv Eb"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF); + return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_idiv_u8_eflags)); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xf7 + */ +FNIEMOP_DEF(iemOp_Grp3_Ev) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: + return FNIEMOP_CALL_1(iemOp_grp3_test_Ev, bRm); + case 1: +/** @todo testcase: Present on <=386, most 486 (not early), Pentiums, and current CPUs too. CPUUNDOC.EXE */ + return IEMOP_RAISE_INVALID_OPCODE(); + case 2: + IEMOP_MNEMONIC(not_Ev, "not Ev"); + return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_not); + case 3: + IEMOP_MNEMONIC(neg_Ev, "neg Ev"); + return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_neg); + case 4: + IEMOP_MNEMONIC(mul_Ev, "mul Ev"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_mul_eflags)); + case 5: + IEMOP_MNEMONIC(imul_Ev, "imul Ev"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_eflags)); + case 6: + IEMOP_MNEMONIC(div_Ev, "div Ev"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF); + return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_div_eflags)); + case 7: + IEMOP_MNEMONIC(idiv_Ev, "idiv Ev"); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF); + return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_idiv_eflags)); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xf8 + */ +FNIEMOP_DEF(iemOp_clc) +{ + IEMOP_MNEMONIC(clc, "clc"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_CLEAR_EFL_BIT(X86_EFL_CF); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0xf9 + */ +FNIEMOP_DEF(iemOp_stc) +{ + IEMOP_MNEMONIC(stc, "stc"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_SET_EFL_BIT(X86_EFL_CF); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0xfa + */ +FNIEMOP_DEF(iemOp_cli) +{ + IEMOP_MNEMONIC(cli, "cli"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_cli); +} + + +FNIEMOP_DEF(iemOp_sti) +{ + IEMOP_MNEMONIC(sti, "sti"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_sti); +} + + +/** + * @opcode 0xfc + */ +FNIEMOP_DEF(iemOp_cld) +{ + IEMOP_MNEMONIC(cld, "cld"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_CLEAR_EFL_BIT(X86_EFL_DF); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0xfd + */ +FNIEMOP_DEF(iemOp_std) +{ + IEMOP_MNEMONIC(std, "std"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_SET_EFL_BIT(X86_EFL_DF); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** + * @opcode 0xfe + */ +FNIEMOP_DEF(iemOp_Grp4) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: + IEMOP_MNEMONIC(inc_Eb, "inc Eb"); + return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_inc); + case 1: + IEMOP_MNEMONIC(dec_Eb, "dec Eb"); + return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_dec); + default: + IEMOP_MNEMONIC(grp4_ud, "grp4-ud"); + return IEMOP_RAISE_INVALID_OPCODE(); + } +} + + +/** + * Opcode 0xff /2. + * @param bRm The RM byte. + */ +FNIEMOP_DEF_1(iemOp_Grp5_calln_Ev, uint8_t, bRm) +{ + IEMOP_MNEMONIC(calln_Ev, "calln Ev"); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* The new RIP is taken from a register. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint16_t, u16Target, 0); + IEM_MC_FETCH_GREG_U16(u16Target, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_1(iemCImpl_call_16, u16Target); + IEM_MC_END() + return VINF_SUCCESS; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint32_t, u32Target, 0); + IEM_MC_FETCH_GREG_U32(u32Target, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_1(iemCImpl_call_32, u32Target); + IEM_MC_END() + return VINF_SUCCESS; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint64_t, u64Target, 0); + IEM_MC_FETCH_GREG_U64(u64Target, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_1(iemCImpl_call_64, u64Target); + IEM_MC_END() + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* The new RIP is taken from a register. */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(1, 1); + IEM_MC_ARG(uint16_t, u16Target, 0); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_CIMPL_1(iemCImpl_call_16, u16Target); + IEM_MC_END() + return VINF_SUCCESS; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(1, 1); + IEM_MC_ARG(uint32_t, u32Target, 0); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(u32Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_CIMPL_1(iemCImpl_call_32, u32Target); + IEM_MC_END() + return VINF_SUCCESS; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(1, 1); + IEM_MC_ARG(uint64_t, u64Target, 0); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(u64Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_CIMPL_1(iemCImpl_call_64, u64Target); + IEM_MC_END() + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + +typedef IEM_CIMPL_DECL_TYPE_3(FNIEMCIMPLFARBRANCH, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmOpSize); + +FNIEMOP_DEF_2(iemOpHlp_Grp5_far_Ep, uint8_t, bRm, FNIEMCIMPLFARBRANCH *, pfnCImpl) +{ + /* Registers? How?? */ + if (RT_LIKELY(IEM_IS_MODRM_MEM_MODE(bRm))) + { /* likely */ } + else + return IEMOP_RAISE_INVALID_OPCODE(); /* callf eax is not legal */ + + /* 64-bit mode: Default is 32-bit, but only intel respects a REX.W prefix. */ + /** @todo what does VIA do? */ + if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT || pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT || IEM_IS_GUEST_CPU_INTEL(pVCpu)) + { /* likely */ } + else + pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT; + + /* Far pointer loaded from memory. */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t, u16Sel, 0); + IEM_MC_ARG(uint16_t, offSeg, 1); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize, IEMMODE_16BIT, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_MEM_U16_DISP(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 2); + IEM_MC_CALL_CIMPL_3(pfnCImpl, u16Sel, offSeg, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t, u16Sel, 0); + IEM_MC_ARG(uint32_t, offSeg, 1); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize, IEMMODE_32BIT, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_MEM_U16_DISP(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 4); + IEM_MC_CALL_CIMPL_3(pfnCImpl, u16Sel, offSeg, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_64BIT: + Assert(!IEM_IS_GUEST_CPU_AMD(pVCpu)); + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t, u16Sel, 0); + IEM_MC_ARG(uint64_t, offSeg, 1); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize, IEMMODE_64BIT, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_MEM_U16_DISP(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 8); + IEM_MC_CALL_CIMPL_3(pfnCImpl, u16Sel, offSeg, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * Opcode 0xff /3. + * @param bRm The RM byte. + */ +FNIEMOP_DEF_1(iemOp_Grp5_callf_Ep, uint8_t, bRm) +{ + IEMOP_MNEMONIC(callf_Ep, "callf Ep"); + return FNIEMOP_CALL_2(iemOpHlp_Grp5_far_Ep, bRm, iemCImpl_callf); +} + + +/** + * Opcode 0xff /4. + * @param bRm The RM byte. + */ +FNIEMOP_DEF_1(iemOp_Grp5_jmpn_Ev, uint8_t, bRm) +{ + IEMOP_MNEMONIC(jmpn_Ev, "jmpn Ev"); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* The new RIP is taken from a register. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Target); + IEM_MC_FETCH_GREG_U16(u16Target, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_SET_RIP_U16_AND_FINISH(u16Target); + IEM_MC_END() + return VINF_SUCCESS; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Target); + IEM_MC_FETCH_GREG_U32(u32Target, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_SET_RIP_U32_AND_FINISH(u32Target); + IEM_MC_END() + return VINF_SUCCESS; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Target); + IEM_MC_FETCH_GREG_U64(u64Target, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_SET_RIP_U64_AND_FINISH(u64Target); + IEM_MC_END() + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* The new RIP is taken from a memory location. */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Target); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_SET_RIP_U16_AND_FINISH(u16Target); + IEM_MC_END() + return VINF_SUCCESS; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Target); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(u32Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_SET_RIP_U32_AND_FINISH(u32Target); + IEM_MC_END() + return VINF_SUCCESS; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Target); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(u64Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_SET_RIP_U64_AND_FINISH(u64Target); + IEM_MC_END() + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * Opcode 0xff /5. + * @param bRm The RM byte. + */ +FNIEMOP_DEF_1(iemOp_Grp5_jmpf_Ep, uint8_t, bRm) +{ + IEMOP_MNEMONIC(jmpf_Ep, "jmpf Ep"); + return FNIEMOP_CALL_2(iemOpHlp_Grp5_far_Ep, bRm, iemCImpl_FarJmp); +} + + +/** + * Opcode 0xff /6. + * @param bRm The RM byte. + */ +FNIEMOP_DEF_1(iemOp_Grp5_push_Ev, uint8_t, bRm) +{ + IEMOP_MNEMONIC(push_Ev, "push Ev"); + + /* Registers are handled by a common worker. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + return FNIEMOP_CALL_1(iemOpCommonPushGReg, IEM_GET_MODRM_RM(pVCpu, bRm)); + + /* Memory we do here. */ + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Src); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_PUSH_U16(u16Src); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Src); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_PUSH_U32(u32Src); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Src); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_PUSH_U64(u64Src); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** + * @opcode 0xff + */ +FNIEMOP_DEF(iemOp_Grp5) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: + IEMOP_MNEMONIC(inc_Ev, "inc Ev"); + return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_inc); + case 1: + IEMOP_MNEMONIC(dec_Ev, "dec Ev"); + return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_dec); + case 2: + return FNIEMOP_CALL_1(iemOp_Grp5_calln_Ev, bRm); + case 3: + return FNIEMOP_CALL_1(iemOp_Grp5_callf_Ep, bRm); + case 4: + return FNIEMOP_CALL_1(iemOp_Grp5_jmpn_Ev, bRm); + case 5: + return FNIEMOP_CALL_1(iemOp_Grp5_jmpf_Ep, bRm); + case 6: + return FNIEMOP_CALL_1(iemOp_Grp5_push_Ev, bRm); + case 7: + IEMOP_MNEMONIC(grp5_ud, "grp5-ud"); + return IEMOP_RAISE_INVALID_OPCODE(); + } + AssertFailedReturn(VERR_IEM_IPE_3); +} + + + +const PFNIEMOP g_apfnOneByteMap[256] = +{ + /* 0x00 */ iemOp_add_Eb_Gb, iemOp_add_Ev_Gv, iemOp_add_Gb_Eb, iemOp_add_Gv_Ev, + /* 0x04 */ iemOp_add_Al_Ib, iemOp_add_eAX_Iz, iemOp_push_ES, iemOp_pop_ES, + /* 0x08 */ iemOp_or_Eb_Gb, iemOp_or_Ev_Gv, iemOp_or_Gb_Eb, iemOp_or_Gv_Ev, + /* 0x0c */ iemOp_or_Al_Ib, iemOp_or_eAX_Iz, iemOp_push_CS, iemOp_2byteEscape, + /* 0x10 */ iemOp_adc_Eb_Gb, iemOp_adc_Ev_Gv, iemOp_adc_Gb_Eb, iemOp_adc_Gv_Ev, + /* 0x14 */ iemOp_adc_Al_Ib, iemOp_adc_eAX_Iz, iemOp_push_SS, iemOp_pop_SS, + /* 0x18 */ iemOp_sbb_Eb_Gb, iemOp_sbb_Ev_Gv, iemOp_sbb_Gb_Eb, iemOp_sbb_Gv_Ev, + /* 0x1c */ iemOp_sbb_Al_Ib, iemOp_sbb_eAX_Iz, iemOp_push_DS, iemOp_pop_DS, + /* 0x20 */ iemOp_and_Eb_Gb, iemOp_and_Ev_Gv, iemOp_and_Gb_Eb, iemOp_and_Gv_Ev, + /* 0x24 */ iemOp_and_Al_Ib, iemOp_and_eAX_Iz, iemOp_seg_ES, iemOp_daa, + /* 0x28 */ iemOp_sub_Eb_Gb, iemOp_sub_Ev_Gv, iemOp_sub_Gb_Eb, iemOp_sub_Gv_Ev, + /* 0x2c */ iemOp_sub_Al_Ib, iemOp_sub_eAX_Iz, iemOp_seg_CS, iemOp_das, + /* 0x30 */ iemOp_xor_Eb_Gb, iemOp_xor_Ev_Gv, iemOp_xor_Gb_Eb, iemOp_xor_Gv_Ev, + /* 0x34 */ iemOp_xor_Al_Ib, iemOp_xor_eAX_Iz, iemOp_seg_SS, iemOp_aaa, + /* 0x38 */ iemOp_cmp_Eb_Gb, iemOp_cmp_Ev_Gv, iemOp_cmp_Gb_Eb, iemOp_cmp_Gv_Ev, + /* 0x3c */ iemOp_cmp_Al_Ib, iemOp_cmp_eAX_Iz, iemOp_seg_DS, iemOp_aas, + /* 0x40 */ iemOp_inc_eAX, iemOp_inc_eCX, iemOp_inc_eDX, iemOp_inc_eBX, + /* 0x44 */ iemOp_inc_eSP, iemOp_inc_eBP, iemOp_inc_eSI, iemOp_inc_eDI, + /* 0x48 */ iemOp_dec_eAX, iemOp_dec_eCX, iemOp_dec_eDX, iemOp_dec_eBX, + /* 0x4c */ iemOp_dec_eSP, iemOp_dec_eBP, iemOp_dec_eSI, iemOp_dec_eDI, + /* 0x50 */ iemOp_push_eAX, iemOp_push_eCX, iemOp_push_eDX, iemOp_push_eBX, + /* 0x54 */ iemOp_push_eSP, iemOp_push_eBP, iemOp_push_eSI, iemOp_push_eDI, + /* 0x58 */ iemOp_pop_eAX, iemOp_pop_eCX, iemOp_pop_eDX, iemOp_pop_eBX, + /* 0x5c */ iemOp_pop_eSP, iemOp_pop_eBP, iemOp_pop_eSI, iemOp_pop_eDI, + /* 0x60 */ iemOp_pusha, iemOp_popa__mvex, iemOp_bound_Gv_Ma__evex, iemOp_arpl_Ew_Gw_movsx_Gv_Ev, + /* 0x64 */ iemOp_seg_FS, iemOp_seg_GS, iemOp_op_size, iemOp_addr_size, + /* 0x68 */ iemOp_push_Iz, iemOp_imul_Gv_Ev_Iz, iemOp_push_Ib, iemOp_imul_Gv_Ev_Ib, + /* 0x6c */ iemOp_insb_Yb_DX, iemOp_inswd_Yv_DX, iemOp_outsb_Yb_DX, iemOp_outswd_Yv_DX, + /* 0x70 */ iemOp_jo_Jb, iemOp_jno_Jb, iemOp_jc_Jb, iemOp_jnc_Jb, + /* 0x74 */ iemOp_je_Jb, iemOp_jne_Jb, iemOp_jbe_Jb, iemOp_jnbe_Jb, + /* 0x78 */ iemOp_js_Jb, iemOp_jns_Jb, iemOp_jp_Jb, iemOp_jnp_Jb, + /* 0x7c */ iemOp_jl_Jb, iemOp_jnl_Jb, iemOp_jle_Jb, iemOp_jnle_Jb, + /* 0x80 */ iemOp_Grp1_Eb_Ib_80, iemOp_Grp1_Ev_Iz, iemOp_Grp1_Eb_Ib_82, iemOp_Grp1_Ev_Ib, + /* 0x84 */ iemOp_test_Eb_Gb, iemOp_test_Ev_Gv, iemOp_xchg_Eb_Gb, iemOp_xchg_Ev_Gv, + /* 0x88 */ iemOp_mov_Eb_Gb, iemOp_mov_Ev_Gv, iemOp_mov_Gb_Eb, iemOp_mov_Gv_Ev, + /* 0x8c */ iemOp_mov_Ev_Sw, iemOp_lea_Gv_M, iemOp_mov_Sw_Ev, iemOp_Grp1A__xop, + /* 0x90 */ iemOp_nop, iemOp_xchg_eCX_eAX, iemOp_xchg_eDX_eAX, iemOp_xchg_eBX_eAX, + /* 0x94 */ iemOp_xchg_eSP_eAX, iemOp_xchg_eBP_eAX, iemOp_xchg_eSI_eAX, iemOp_xchg_eDI_eAX, + /* 0x98 */ iemOp_cbw, iemOp_cwd, iemOp_call_Ap, iemOp_wait, + /* 0x9c */ iemOp_pushf_Fv, iemOp_popf_Fv, iemOp_sahf, iemOp_lahf, + /* 0xa0 */ iemOp_mov_AL_Ob, iemOp_mov_rAX_Ov, iemOp_mov_Ob_AL, iemOp_mov_Ov_rAX, + /* 0xa4 */ iemOp_movsb_Xb_Yb, iemOp_movswd_Xv_Yv, iemOp_cmpsb_Xb_Yb, iemOp_cmpswd_Xv_Yv, + /* 0xa8 */ iemOp_test_AL_Ib, iemOp_test_eAX_Iz, iemOp_stosb_Yb_AL, iemOp_stoswd_Yv_eAX, + /* 0xac */ iemOp_lodsb_AL_Xb, iemOp_lodswd_eAX_Xv, iemOp_scasb_AL_Xb, iemOp_scaswd_eAX_Xv, + /* 0xb0 */ iemOp_mov_AL_Ib, iemOp_CL_Ib, iemOp_DL_Ib, iemOp_BL_Ib, + /* 0xb4 */ iemOp_mov_AH_Ib, iemOp_CH_Ib, iemOp_DH_Ib, iemOp_BH_Ib, + /* 0xb8 */ iemOp_eAX_Iv, iemOp_eCX_Iv, iemOp_eDX_Iv, iemOp_eBX_Iv, + /* 0xbc */ iemOp_eSP_Iv, iemOp_eBP_Iv, iemOp_eSI_Iv, iemOp_eDI_Iv, + /* 0xc0 */ iemOp_Grp2_Eb_Ib, iemOp_Grp2_Ev_Ib, iemOp_retn_Iw, iemOp_retn, + /* 0xc4 */ iemOp_les_Gv_Mp__vex3, iemOp_lds_Gv_Mp__vex2, iemOp_Grp11_Eb_Ib, iemOp_Grp11_Ev_Iz, + /* 0xc8 */ iemOp_enter_Iw_Ib, iemOp_leave, iemOp_retf_Iw, iemOp_retf, + /* 0xcc */ iemOp_int3, iemOp_int_Ib, iemOp_into, iemOp_iret, + /* 0xd0 */ iemOp_Grp2_Eb_1, iemOp_Grp2_Ev_1, iemOp_Grp2_Eb_CL, iemOp_Grp2_Ev_CL, + /* 0xd4 */ iemOp_aam_Ib, iemOp_aad_Ib, iemOp_salc, iemOp_xlat, + /* 0xd8 */ iemOp_EscF0, iemOp_EscF1, iemOp_EscF2, iemOp_EscF3, + /* 0xdc */ iemOp_EscF4, iemOp_EscF5, iemOp_EscF6, iemOp_EscF7, + /* 0xe0 */ iemOp_loopne_Jb, iemOp_loope_Jb, iemOp_loop_Jb, iemOp_jecxz_Jb, + /* 0xe4 */ iemOp_in_AL_Ib, iemOp_in_eAX_Ib, iemOp_out_Ib_AL, iemOp_out_Ib_eAX, + /* 0xe8 */ iemOp_call_Jv, iemOp_jmp_Jv, iemOp_jmp_Ap, iemOp_jmp_Jb, + /* 0xec */ iemOp_in_AL_DX, iemOp_in_eAX_DX, iemOp_out_DX_AL, iemOp_out_DX_eAX, + /* 0xf0 */ iemOp_lock, iemOp_int1, iemOp_repne, iemOp_repe, + /* 0xf4 */ iemOp_hlt, iemOp_cmc, iemOp_Grp3_Eb, iemOp_Grp3_Ev, + /* 0xf8 */ iemOp_clc, iemOp_stc, iemOp_cli, iemOp_sti, + /* 0xfc */ iemOp_cld, iemOp_std, iemOp_Grp4, iemOp_Grp5, +}; + + +/** @} */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py b/src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py new file mode 100755 index 00000000..38ad56d7 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py @@ -0,0 +1,3953 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# $Id: IEMAllInstructionsPython.py $ + +""" +IEM instruction extractor. + +This script/module parses the IEMAllInstruction*.cpp.h files next to it and +collects information about the instructions. It can then be used to generate +disassembler tables and tests. +""" + +__copyright__ = \ +""" +Copyright (C) 2017-2023 Oracle and/or its affiliates. + +This file is part of VirtualBox base platform packages, as +available from https://www.virtualbox.org. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation, in version 3 of the +License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +The contents of this file may alternatively be used under the terms +of the Common Development and Distribution License Version 1.0 +(CDDL), a copy of it is provided in the "COPYING.CDDL" file included +in the VirtualBox distribution, in which case the provisions of the +CDDL are applicable instead of those of the GPL. + +You may elect to license modified versions of this file under the +terms and conditions of either the GPL or the CDDL or both. + +SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0 +""" +__version__ = "$Revision: 155244 $" + +# pylint: disable=anomalous-backslash-in-string + +# Standard python imports. +import os +import re +import sys + +## Only the main script needs to modify the path. +#g_ksValidationKitDir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), +# 'ValidationKit'); +#sys.path.append(g_ksValidationKitDir); +# +#from common import utils; - Windows build boxes doesn't have pywin32. + +# Python 3 hacks: +if sys.version_info[0] >= 3: + long = int; # pylint: disable=redefined-builtin,invalid-name + + +g_kdX86EFlagsConstants = { + 'X86_EFL_CF': 0x00000001, # RT_BIT_32(0) + 'X86_EFL_1': 0x00000002, # RT_BIT_32(1) + 'X86_EFL_PF': 0x00000004, # RT_BIT_32(2) + 'X86_EFL_AF': 0x00000010, # RT_BIT_32(4) + 'X86_EFL_ZF': 0x00000040, # RT_BIT_32(6) + 'X86_EFL_SF': 0x00000080, # RT_BIT_32(7) + 'X86_EFL_TF': 0x00000100, # RT_BIT_32(8) + 'X86_EFL_IF': 0x00000200, # RT_BIT_32(9) + 'X86_EFL_DF': 0x00000400, # RT_BIT_32(10) + 'X86_EFL_OF': 0x00000800, # RT_BIT_32(11) + 'X86_EFL_IOPL': 0x00003000, # (RT_BIT_32(12) | RT_BIT_32(13)) + 'X86_EFL_NT': 0x00004000, # RT_BIT_32(14) + 'X86_EFL_RF': 0x00010000, # RT_BIT_32(16) + 'X86_EFL_VM': 0x00020000, # RT_BIT_32(17) + 'X86_EFL_AC': 0x00040000, # RT_BIT_32(18) + 'X86_EFL_VIF': 0x00080000, # RT_BIT_32(19) + 'X86_EFL_VIP': 0x00100000, # RT_BIT_32(20) + 'X86_EFL_ID': 0x00200000, # RT_BIT_32(21) + 'X86_EFL_LIVE_MASK': 0x003f7fd5, # UINT32_C(0x003f7fd5) + 'X86_EFL_RA1_MASK': 0x00000002, # RT_BIT_32(1) +}; + +## EFlags values allowed in \@opfltest, \@opflmodify, \@opflundef, \@opflset, and \@opflclear. +g_kdEFlagsMnemonics = { + # Debugger flag notation (sorted by value): + 'cf': 'X86_EFL_CF', ##< Carry Flag. + 'nc': '!X86_EFL_CF', ##< No Carry. + + 'po': 'X86_EFL_PF', ##< Parity Pdd. + 'pe': '!X86_EFL_PF', ##< Parity Even. + + 'af': 'X86_EFL_AF', ##< Aux Flag. + 'na': '!X86_EFL_AF', ##< No Aux. + + 'zr': 'X86_EFL_ZF', ##< ZeRo. + 'nz': '!X86_EFL_ZF', ##< No Zero. + + 'ng': 'X86_EFL_SF', ##< NeGative (sign). + 'pl': '!X86_EFL_SF', ##< PLuss (sign). + + 'tf': 'X86_EFL_TF', ##< Trap flag. + + 'ei': 'X86_EFL_IF', ##< Enabled Interrupts. + 'di': '!X86_EFL_IF', ##< Disabled Interrupts. + + 'dn': 'X86_EFL_DF', ##< DowN (string op direction). + 'up': '!X86_EFL_DF', ##< UP (string op direction). + + 'ov': 'X86_EFL_OF', ##< OVerflow. + 'nv': '!X86_EFL_OF', ##< No Overflow. + + 'nt': 'X86_EFL_NT', ##< Nested Task. + 'rf': 'X86_EFL_RF', ##< Resume Flag. + 'vm': 'X86_EFL_VM', ##< Virtual-8086 Mode. + 'ac': 'X86_EFL_AC', ##< Alignment Check. + 'vif': 'X86_EFL_VIF', ##< Virtual Interrupt Flag. + 'vip': 'X86_EFL_VIP', ##< Virtual Interrupt Pending. + + # Reference manual notation not covered above (sorted by value): + 'pf': 'X86_EFL_PF', + 'zf': 'X86_EFL_ZF', + 'sf': 'X86_EFL_SF', + 'if': 'X86_EFL_IF', + 'df': 'X86_EFL_DF', + 'of': 'X86_EFL_OF', + 'iopl': 'X86_EFL_IOPL', + 'id': 'X86_EFL_ID', +}; + +## Constants and values for CR0. +g_kdX86Cr0Constants = { + 'X86_CR0_PE': 0x00000001, # RT_BIT_32(0) + 'X86_CR0_MP': 0x00000002, # RT_BIT_32(1) + 'X86_CR0_EM': 0x00000004, # RT_BIT_32(2) + 'X86_CR0_TS': 0x00000008, # RT_BIT_32(3) + 'X86_CR0_ET': 0x00000010, # RT_BIT_32(4) + 'X86_CR0_NE': 0x00000020, # RT_BIT_32(5) + 'X86_CR0_WP': 0x00010000, # RT_BIT_32(16) + 'X86_CR0_AM': 0x00040000, # RT_BIT_32(18) + 'X86_CR0_NW': 0x20000000, # RT_BIT_32(29) + 'X86_CR0_CD': 0x40000000, # RT_BIT_32(30) + 'X86_CR0_PG': 0x80000000, # RT_BIT_32(31) +}; + +## Constants and values for CR4. +g_kdX86Cr4Constants = { + 'X86_CR4_VME': 0x00000001, # RT_BIT_32(0) + 'X86_CR4_PVI': 0x00000002, # RT_BIT_32(1) + 'X86_CR4_TSD': 0x00000004, # RT_BIT_32(2) + 'X86_CR4_DE': 0x00000008, # RT_BIT_32(3) + 'X86_CR4_PSE': 0x00000010, # RT_BIT_32(4) + 'X86_CR4_PAE': 0x00000020, # RT_BIT_32(5) + 'X86_CR4_MCE': 0x00000040, # RT_BIT_32(6) + 'X86_CR4_PGE': 0x00000080, # RT_BIT_32(7) + 'X86_CR4_PCE': 0x00000100, # RT_BIT_32(8) + 'X86_CR4_OSFXSR': 0x00000200, # RT_BIT_32(9) + 'X86_CR4_OSXMMEEXCPT': 0x00000400, # RT_BIT_32(10) + 'X86_CR4_VMXE': 0x00002000, # RT_BIT_32(13) + 'X86_CR4_SMXE': 0x00004000, # RT_BIT_32(14) + 'X86_CR4_PCIDE': 0x00020000, # RT_BIT_32(17) + 'X86_CR4_OSXSAVE': 0x00040000, # RT_BIT_32(18) + 'X86_CR4_SMEP': 0x00100000, # RT_BIT_32(20) + 'X86_CR4_SMAP': 0x00200000, # RT_BIT_32(21) + 'X86_CR4_PKE': 0x00400000, # RT_BIT_32(22) +}; + +## XSAVE components (XCR0). +g_kdX86XSaveCConstants = { + 'XSAVE_C_X87': 0x00000001, + 'XSAVE_C_SSE': 0x00000002, + 'XSAVE_C_YMM': 0x00000004, + 'XSAVE_C_BNDREGS': 0x00000008, + 'XSAVE_C_BNDCSR': 0x00000010, + 'XSAVE_C_OPMASK': 0x00000020, + 'XSAVE_C_ZMM_HI256': 0x00000040, + 'XSAVE_C_ZMM_16HI': 0x00000080, + 'XSAVE_C_PKRU': 0x00000200, + 'XSAVE_C_LWP': 0x4000000000000000, + 'XSAVE_C_X': 0x8000000000000000, + 'XSAVE_C_ALL_AVX': 0x000000c4, # For clearing all AVX bits. + 'XSAVE_C_ALL_AVX_SSE': 0x000000c6, # For clearing all AVX and SSE bits. +}; + + +## \@op[1-4] locations +g_kdOpLocations = { + 'reg': [], ## modrm.reg + 'rm': [], ## modrm.rm + 'imm': [], ## immediate instruction data + 'vvvv': [], ## VEX.vvvv + + # fixed registers. + 'AL': [], + 'rAX': [], + 'rDX': [], + 'rSI': [], + 'rDI': [], + 'rFLAGS': [], + 'CS': [], + 'DS': [], + 'ES': [], + 'FS': [], + 'GS': [], + 'SS': [], +}; + +## \@op[1-4] types +## +## Value fields: +## - 0: the normal IDX_ParseXXX handler (IDX_UseModRM == IDX_ParseModRM). +## - 1: the location (g_kdOpLocations). +## - 2: disassembler format string version of the type. +## - 3: disassembler OP_PARAM_XXX (XXX only). +## - 4: IEM form matching instruction. +## +## Note! See the A.2.1 in SDM vol 2 for the type names. +g_kdOpTypes = { + # Fixed addresses + 'Ap': ( 'IDX_ParseImmAddrF', 'imm', '%Ap', 'Ap', 'FIXED', ), + + # ModR/M.rm + 'Eb': ( 'IDX_UseModRM', 'rm', '%Eb', 'Eb', 'RM', ), + 'Ed': ( 'IDX_UseModRM', 'rm', '%Ed', 'Ed', 'RM', ), + 'Ed_WO': ( 'IDX_UseModRM', 'rm', '%Ed', 'Ed', 'RM', ), + 'Eq': ( 'IDX_UseModRM', 'rm', '%Eq', 'Eq', 'RM', ), + 'Eq_WO': ( 'IDX_UseModRM', 'rm', '%Eq', 'Eq', 'RM', ), + 'Ew': ( 'IDX_UseModRM', 'rm', '%Ew', 'Ew', 'RM', ), + 'Ev': ( 'IDX_UseModRM', 'rm', '%Ev', 'Ev', 'RM', ), + 'Ey': ( 'IDX_UseModRM', 'rm', '%Ey', 'Ey', 'RM', ), + 'Qd': ( 'IDX_UseModRM', 'rm', '%Qd', 'Qd', 'RM', ), + 'Qq': ( 'IDX_UseModRM', 'rm', '%Qq', 'Qq', 'RM', ), + 'Qq_WO': ( 'IDX_UseModRM', 'rm', '%Qq', 'Qq', 'RM', ), + 'Wss': ( 'IDX_UseModRM', 'rm', '%Wss', 'Wss', 'RM', ), + 'Wss_WO': ( 'IDX_UseModRM', 'rm', '%Wss', 'Wss', 'RM', ), + 'Wsd': ( 'IDX_UseModRM', 'rm', '%Wsd', 'Wsd', 'RM', ), + 'Wsd_WO': ( 'IDX_UseModRM', 'rm', '%Wsd', 'Wsd', 'RM', ), + 'Wps': ( 'IDX_UseModRM', 'rm', '%Wps', 'Wps', 'RM', ), + 'Wps_WO': ( 'IDX_UseModRM', 'rm', '%Wps', 'Wps', 'RM', ), + 'Wpd': ( 'IDX_UseModRM', 'rm', '%Wpd', 'Wpd', 'RM', ), + 'Wpd_WO': ( 'IDX_UseModRM', 'rm', '%Wpd', 'Wpd', 'RM', ), + 'Wdq': ( 'IDX_UseModRM', 'rm', '%Wdq', 'Wdq', 'RM', ), + 'Wdq_WO': ( 'IDX_UseModRM', 'rm', '%Wdq', 'Wdq', 'RM', ), + 'Wq': ( 'IDX_UseModRM', 'rm', '%Wq', 'Wq', 'RM', ), + 'Wq_WO': ( 'IDX_UseModRM', 'rm', '%Wq', 'Wq', 'RM', ), + 'WqZxReg_WO': ( 'IDX_UseModRM', 'rm', '%Wq', 'Wq', 'RM', ), + 'Wx': ( 'IDX_UseModRM', 'rm', '%Wx', 'Wx', 'RM', ), + 'Wx_WO': ( 'IDX_UseModRM', 'rm', '%Wx', 'Wx', 'RM', ), + + # ModR/M.rm - register only. + 'Uq': ( 'IDX_UseModRM', 'rm', '%Uq', 'Uq', 'REG' ), + 'UqHi': ( 'IDX_UseModRM', 'rm', '%Uq', 'UqHi', 'REG' ), + 'Uss': ( 'IDX_UseModRM', 'rm', '%Uss', 'Uss', 'REG' ), + 'Uss_WO': ( 'IDX_UseModRM', 'rm', '%Uss', 'Uss', 'REG' ), + 'Usd': ( 'IDX_UseModRM', 'rm', '%Usd', 'Usd', 'REG' ), + 'Usd_WO': ( 'IDX_UseModRM', 'rm', '%Usd', 'Usd', 'REG' ), + 'Ux': ( 'IDX_UseModRM', 'rm', '%Ux', 'Ux', 'REG' ), + 'Nq': ( 'IDX_UseModRM', 'rm', '%Qq', 'Nq', 'REG' ), + + # ModR/M.rm - memory only. + 'Ma': ( 'IDX_UseModRM', 'rm', '%Ma', 'Ma', 'MEM', ), ##< Only used by BOUND. + 'Mb_RO': ( 'IDX_UseModRM', 'rm', '%Mb', 'Mb', 'MEM', ), + 'Md': ( 'IDX_UseModRM', 'rm', '%Md', 'Md', 'MEM', ), + 'Md_RO': ( 'IDX_UseModRM', 'rm', '%Md', 'Md', 'MEM', ), + 'Md_WO': ( 'IDX_UseModRM', 'rm', '%Md', 'Md', 'MEM', ), + 'Mdq': ( 'IDX_UseModRM', 'rm', '%Mdq', 'Mdq', 'MEM', ), + 'Mdq_WO': ( 'IDX_UseModRM', 'rm', '%Mdq', 'Mdq', 'MEM', ), + 'Mq': ( 'IDX_UseModRM', 'rm', '%Mq', 'Mq', 'MEM', ), + 'Mq_WO': ( 'IDX_UseModRM', 'rm', '%Mq', 'Mq', 'MEM', ), + 'Mps_WO': ( 'IDX_UseModRM', 'rm', '%Mps', 'Mps', 'MEM', ), + 'Mpd_WO': ( 'IDX_UseModRM', 'rm', '%Mpd', 'Mpd', 'MEM', ), + 'Mx': ( 'IDX_UseModRM', 'rm', '%Mx', 'Mx', 'MEM', ), + 'Mx_WO': ( 'IDX_UseModRM', 'rm', '%Mx', 'Mx', 'MEM', ), + 'M_RO': ( 'IDX_UseModRM', 'rm', '%M', 'M', 'MEM', ), + 'M_RW': ( 'IDX_UseModRM', 'rm', '%M', 'M', 'MEM', ), + + # ModR/M.reg + 'Gb': ( 'IDX_UseModRM', 'reg', '%Gb', 'Gb', '', ), + 'Gw': ( 'IDX_UseModRM', 'reg', '%Gw', 'Gw', '', ), + 'Gd': ( 'IDX_UseModRM', 'reg', '%Gd', 'Gd', '', ), + 'Gv': ( 'IDX_UseModRM', 'reg', '%Gv', 'Gv', '', ), + 'Gv_RO': ( 'IDX_UseModRM', 'reg', '%Gv', 'Gv', '', ), + 'Gy': ( 'IDX_UseModRM', 'reg', '%Gy', 'Gy', '', ), + 'Pd': ( 'IDX_UseModRM', 'reg', '%Pd', 'Pd', '', ), + 'PdZx_WO': ( 'IDX_UseModRM', 'reg', '%Pd', 'PdZx', '', ), + 'Pq': ( 'IDX_UseModRM', 'reg', '%Pq', 'Pq', '', ), + 'Pq_WO': ( 'IDX_UseModRM', 'reg', '%Pq', 'Pq', '', ), + 'Vd': ( 'IDX_UseModRM', 'reg', '%Vd', 'Vd', '', ), + 'Vd_WO': ( 'IDX_UseModRM', 'reg', '%Vd', 'Vd', '', ), + 'VdZx_WO': ( 'IDX_UseModRM', 'reg', '%Vd', 'Vd', '', ), + 'Vdq': ( 'IDX_UseModRM', 'reg', '%Vdq', 'Vdq', '', ), + 'Vss': ( 'IDX_UseModRM', 'reg', '%Vss', 'Vss', '', ), + 'Vss_WO': ( 'IDX_UseModRM', 'reg', '%Vss', 'Vss', '', ), + 'VssZx_WO': ( 'IDX_UseModRM', 'reg', '%Vss', 'Vss', '', ), + 'Vsd': ( 'IDX_UseModRM', 'reg', '%Vsd', 'Vsd', '', ), + 'Vsd_WO': ( 'IDX_UseModRM', 'reg', '%Vsd', 'Vsd', '', ), + 'VsdZx_WO': ( 'IDX_UseModRM', 'reg', '%Vsd', 'Vsd', '', ), + 'Vps': ( 'IDX_UseModRM', 'reg', '%Vps', 'Vps', '', ), + 'Vps_WO': ( 'IDX_UseModRM', 'reg', '%Vps', 'Vps', '', ), + 'Vpd': ( 'IDX_UseModRM', 'reg', '%Vpd', 'Vpd', '', ), + 'Vpd_WO': ( 'IDX_UseModRM', 'reg', '%Vpd', 'Vpd', '', ), + 'Vq': ( 'IDX_UseModRM', 'reg', '%Vq', 'Vq', '', ), + 'Vq_WO': ( 'IDX_UseModRM', 'reg', '%Vq', 'Vq', '', ), + 'Vdq_WO': ( 'IDX_UseModRM', 'reg', '%Vdq', 'Vdq', '', ), + 'VqHi': ( 'IDX_UseModRM', 'reg', '%Vdq', 'VdqHi', '', ), + 'VqHi_WO': ( 'IDX_UseModRM', 'reg', '%Vdq', 'VdqHi', '', ), + 'VqZx_WO': ( 'IDX_UseModRM', 'reg', '%Vq', 'VqZx', '', ), + 'Vx': ( 'IDX_UseModRM', 'reg', '%Vx', 'Vx', '', ), + 'Vx_WO': ( 'IDX_UseModRM', 'reg', '%Vx', 'Vx', '', ), + + # VEX.vvvv + 'By': ( 'IDX_UseModRM', 'vvvv', '%By', 'By', 'V', ), + 'Hps': ( 'IDX_UseModRM', 'vvvv', '%Hps', 'Hps', 'V', ), + 'Hpd': ( 'IDX_UseModRM', 'vvvv', '%Hpd', 'Hpd', 'V', ), + 'HssHi': ( 'IDX_UseModRM', 'vvvv', '%Hx', 'HssHi', 'V', ), + 'HsdHi': ( 'IDX_UseModRM', 'vvvv', '%Hx', 'HsdHi', 'V', ), + 'Hq': ( 'IDX_UseModRM', 'vvvv', '%Hq', 'Hq', 'V', ), + 'HqHi': ( 'IDX_UseModRM', 'vvvv', '%Hq', 'HqHi', 'V', ), + 'Hx': ( 'IDX_UseModRM', 'vvvv', '%Hx', 'Hx', 'V', ), + + # Immediate values. + 'Ib': ( 'IDX_ParseImmByte', 'imm', '%Ib', 'Ib', '', ), ##< NB! Could be IDX_ParseImmByteSX for some instrs. + 'Iw': ( 'IDX_ParseImmUshort', 'imm', '%Iw', 'Iw', '', ), + 'Id': ( 'IDX_ParseImmUlong', 'imm', '%Id', 'Id', '', ), + 'Iq': ( 'IDX_ParseImmQword', 'imm', '%Iq', 'Iq', '', ), + 'Iv': ( 'IDX_ParseImmV', 'imm', '%Iv', 'Iv', '', ), ##< o16: word, o32: dword, o64: qword + 'Iz': ( 'IDX_ParseImmZ', 'imm', '%Iz', 'Iz', '', ), ##< o16: word, o32|o64:dword + + # Address operands (no ModR/M). + 'Ob': ( 'IDX_ParseImmAddr', 'imm', '%Ob', 'Ob', '', ), + 'Ov': ( 'IDX_ParseImmAddr', 'imm', '%Ov', 'Ov', '', ), + + # Relative jump targets + 'Jb': ( 'IDX_ParseImmBRel', 'imm', '%Jb', 'Jb', '', ), + 'Jv': ( 'IDX_ParseImmVRel', 'imm', '%Jv', 'Jv', '', ), + + # DS:rSI + 'Xb': ( 'IDX_ParseXb', 'rSI', '%eSI', 'Xb', '', ), + 'Xv': ( 'IDX_ParseXv', 'rSI', '%eSI', 'Xv', '', ), + # ES:rDI + 'Yb': ( 'IDX_ParseYb', 'rDI', '%eDI', 'Yb', '', ), + 'Yv': ( 'IDX_ParseYv', 'rDI', '%eDI', 'Yv', '', ), + + 'Fv': ( 'IDX_ParseFixedReg', 'rFLAGS', '%Fv', 'Fv', '', ), + + # Fixed registers. + 'AL': ( 'IDX_ParseFixedReg', 'AL', 'al', 'REG_AL', '', ), + 'rAX': ( 'IDX_ParseFixedReg', 'rAX', '%eAX', 'REG_EAX', '', ), + 'rDX': ( 'IDX_ParseFixedReg', 'rDX', '%eDX', 'REG_EDX', '', ), + 'CS': ( 'IDX_ParseFixedReg', 'CS', 'cs', 'REG_CS', '', ), # 8086: push CS + 'DS': ( 'IDX_ParseFixedReg', 'DS', 'ds', 'REG_DS', '', ), + 'ES': ( 'IDX_ParseFixedReg', 'ES', 'es', 'REG_ES', '', ), + 'FS': ( 'IDX_ParseFixedReg', 'FS', 'fs', 'REG_FS', '', ), + 'GS': ( 'IDX_ParseFixedReg', 'GS', 'gs', 'REG_GS', '', ), + 'SS': ( 'IDX_ParseFixedReg', 'SS', 'ss', 'REG_SS', '', ), +}; + +# IDX_ParseFixedReg +# IDX_ParseVexDest + + +## IEMFORM_XXX mappings. +g_kdIemForms = { # sEncoding, [ sWhere1, ... ] opcodesub ), + 'RM': ( 'ModR/M', [ 'reg', 'rm' ], '', ), + 'RM_REG': ( 'ModR/M', [ 'reg', 'rm' ], '11 mr/reg', ), + 'RM_MEM': ( 'ModR/M', [ 'reg', 'rm' ], '!11 mr/reg', ), + 'RMI': ( 'ModR/M', [ 'reg', 'rm', 'imm' ], '', ), + 'RMI_REG': ( 'ModR/M', [ 'reg', 'rm', 'imm' ], '11 mr/reg', ), + 'RMI_MEM': ( 'ModR/M', [ 'reg', 'rm', 'imm' ], '!11 mr/reg', ), + 'MR': ( 'ModR/M', [ 'rm', 'reg' ], '', ), + 'MR_REG': ( 'ModR/M', [ 'rm', 'reg' ], '11 mr/reg', ), + 'MR_MEM': ( 'ModR/M', [ 'rm', 'reg' ], '!11 mr/reg', ), + 'MRI': ( 'ModR/M', [ 'rm', 'reg', 'imm' ], '', ), + 'MRI_REG': ( 'ModR/M', [ 'rm', 'reg', 'imm' ], '11 mr/reg', ), + 'MRI_MEM': ( 'ModR/M', [ 'rm', 'reg', 'imm' ], '!11 mr/reg', ), + 'M': ( 'ModR/M', [ 'rm', ], '', ), + 'M_REG': ( 'ModR/M', [ 'rm', ], '', ), + 'M_MEM': ( 'ModR/M', [ 'rm', ], '', ), + 'R': ( 'ModR/M', [ 'reg', ], '', ), + + 'VEX_RM': ( 'VEX.ModR/M', [ 'reg', 'rm' ], '', ), + 'VEX_RM_REG': ( 'VEX.ModR/M', [ 'reg', 'rm' ], '11 mr/reg', ), + 'VEX_RM_MEM': ( 'VEX.ModR/M', [ 'reg', 'rm' ], '!11 mr/reg', ), + 'VEX_MR': ( 'VEX.ModR/M', [ 'rm', 'reg' ], '', ), + 'VEX_MR_REG': ( 'VEX.ModR/M', [ 'rm', 'reg' ], '11 mr/reg', ), + 'VEX_MR_MEM': ( 'VEX.ModR/M', [ 'rm', 'reg' ], '!11 mr/reg', ), + 'VEX_M': ( 'VEX.ModR/M', [ 'rm', ], '' ), + 'VEX_M_REG': ( 'VEX.ModR/M', [ 'rm', ], '' ), + 'VEX_M_MEM': ( 'VEX.ModR/M', [ 'rm', ], '' ), + 'VEX_R': ( 'VEX.ModR/M', [ 'reg', ], '' ), + 'VEX_RVM': ( 'VEX.ModR/M', [ 'reg', 'vvvv', 'rm' ], '', ), + 'VEX_RVM_REG': ( 'VEX.ModR/M', [ 'reg', 'vvvv', 'rm' ], '11 mr/reg', ), + 'VEX_RVM_MEM': ( 'VEX.ModR/M', [ 'reg', 'vvvv', 'rm' ], '!11 mr/reg', ), + 'VEX_RMV': ( 'VEX.ModR/M', [ 'reg', 'rm', 'vvvv' ], '', ), + 'VEX_RMV_REG': ( 'VEX.ModR/M', [ 'reg', 'rm', 'vvvv' ], '11 mr/reg', ), + 'VEX_RMV_MEM': ( 'VEX.ModR/M', [ 'reg', 'rm', 'vvvv' ], '!11 mr/reg', ), + 'VEX_RMI': ( 'VEX.ModR/M', [ 'reg', 'rm', 'imm' ], '', ), + 'VEX_RMI_REG': ( 'VEX.ModR/M', [ 'reg', 'rm', 'imm' ], '11 mr/reg', ), + 'VEX_RMI_MEM': ( 'VEX.ModR/M', [ 'reg', 'rm', 'imm' ], '!11 mr/reg', ), + 'VEX_MVR': ( 'VEX.ModR/M', [ 'rm', 'vvvv', 'reg' ], '', ), + 'VEX_MVR_REG': ( 'VEX.ModR/M', [ 'rm', 'vvvv', 'reg' ], '11 mr/reg', ), + 'VEX_MVR_MEM': ( 'VEX.ModR/M', [ 'rm', 'vvvv', 'reg' ], '!11 mr/reg', ), + + 'VEX_VM': ( 'VEX.ModR/M', [ 'vvvv', 'rm' ], '', ), + 'VEX_VM_REG': ( 'VEX.ModR/M', [ 'vvvv', 'rm' ], '11 mr/reg', ), + 'VEX_VM_MEM': ( 'VEX.ModR/M', [ 'vvvv', 'rm' ], '!11 mr/reg', ), + + 'FIXED': ( 'fixed', None, '', ), +}; + +## \@oppfx values. +g_kdPrefixes = { + 'none': [], + '0x66': [], + '0xf3': [], + '0xf2': [], +}; + +## Special \@opcode tag values. +g_kdSpecialOpcodes = { + '/reg': [], + 'mr/reg': [], + '11 /reg': [], + '!11 /reg': [], + '11 mr/reg': [], + '!11 mr/reg': [], +}; + +## Special \@opcodesub tag values. +## The first value is the real value for aliases. +## The second value is for bs3cg1. +g_kdSubOpcodes = { + 'none': [ None, '', ], + '11 mr/reg': [ '11 mr/reg', '', ], + '11': [ '11 mr/reg', '', ], ##< alias + '!11 mr/reg': [ '!11 mr/reg', '', ], + '!11': [ '!11 mr/reg', '', ], ##< alias + 'rex.w=0': [ 'rex.w=0', 'WZ', ], + 'w=0': [ 'rex.w=0', '', ], ##< alias + 'rex.w=1': [ 'rex.w=1', 'WNZ', ], + 'w=1': [ 'rex.w=1', '', ], ##< alias + 'vex.l=0': [ 'vex.l=0', 'L0', ], + 'vex.l=1': [ 'vex.l=0', 'L1', ], + '11 mr/reg vex.l=0': [ '11 mr/reg vex.l=0', 'L0', ], + '11 mr/reg vex.l=1': [ '11 mr/reg vex.l=1', 'L1', ], + '!11 mr/reg vex.l=0': [ '!11 mr/reg vex.l=0', 'L0', ], + '!11 mr/reg vex.l=1': [ '!11 mr/reg vex.l=1', 'L1', ], +}; + +## Valid values for \@openc +g_kdEncodings = { + 'ModR/M': [ 'BS3CG1ENC_MODRM', ], ##< ModR/M + 'VEX.ModR/M': [ 'BS3CG1ENC_VEX_MODRM', ], ##< VEX...ModR/M + 'fixed': [ 'BS3CG1ENC_FIXED', ], ##< Fixed encoding (address, registers, unused, etc). + 'VEX.fixed': [ 'BS3CG1ENC_VEX_FIXED', ], ##< VEX + fixed encoding (address, registers, unused, etc). + 'prefix': [ None, ], ##< Prefix +}; + +## \@opunused, \@opinvalid, \@opinvlstyle +g_kdInvalidStyles = { + 'immediate': [], ##< CPU stops decoding immediately after the opcode. + 'vex.modrm': [], ##< VEX+ModR/M, everyone. + 'intel-modrm': [], ##< Intel decodes ModR/M. + 'intel-modrm-imm8': [], ##< Intel decodes ModR/M and an 8-byte immediate. + 'intel-opcode-modrm': [], ##< Intel decodes another opcode byte followed by ModR/M. (Unused extension tables.) + 'intel-opcode-modrm-imm8': [], ##< Intel decodes another opcode byte followed by ModR/M and an 8-byte immediate. +}; + +g_kdCpuNames = { + '8086': (), + '80186': (), + '80286': (), + '80386': (), + '80486': (), +}; + +## \@opcpuid +g_kdCpuIdFlags = { + 'vme': 'X86_CPUID_FEATURE_EDX_VME', + 'tsc': 'X86_CPUID_FEATURE_EDX_TSC', + 'msr': 'X86_CPUID_FEATURE_EDX_MSR', + 'cx8': 'X86_CPUID_FEATURE_EDX_CX8', + 'sep': 'X86_CPUID_FEATURE_EDX_SEP', + 'cmov': 'X86_CPUID_FEATURE_EDX_CMOV', + 'clfsh': 'X86_CPUID_FEATURE_EDX_CLFSH', + 'clflushopt': 'X86_CPUID_STEXT_FEATURE_EBX_CLFLUSHOPT', + 'mmx': 'X86_CPUID_FEATURE_EDX_MMX', + 'fxsr': 'X86_CPUID_FEATURE_EDX_FXSR', + 'sse': 'X86_CPUID_FEATURE_EDX_SSE', + 'sse2': 'X86_CPUID_FEATURE_EDX_SSE2', + 'sse3': 'X86_CPUID_FEATURE_ECX_SSE3', + 'pclmul': 'X86_CPUID_FEATURE_ECX_DTES64', + 'monitor': 'X86_CPUID_FEATURE_ECX_CPLDS', + 'vmx': 'X86_CPUID_FEATURE_ECX_VMX', + 'smx': 'X86_CPUID_FEATURE_ECX_TM2', + 'ssse3': 'X86_CPUID_FEATURE_ECX_SSSE3', + 'fma': 'X86_CPUID_FEATURE_ECX_FMA', + 'cx16': 'X86_CPUID_FEATURE_ECX_CX16', + 'pcid': 'X86_CPUID_FEATURE_ECX_PCID', + 'sse4.1': 'X86_CPUID_FEATURE_ECX_SSE4_1', + 'sse4.2': 'X86_CPUID_FEATURE_ECX_SSE4_2', + 'movbe': 'X86_CPUID_FEATURE_ECX_MOVBE', + 'popcnt': 'X86_CPUID_FEATURE_ECX_POPCNT', + 'aes': 'X86_CPUID_FEATURE_ECX_AES', + 'xsave': 'X86_CPUID_FEATURE_ECX_XSAVE', + 'avx': 'X86_CPUID_FEATURE_ECX_AVX', + 'avx2': 'X86_CPUID_STEXT_FEATURE_EBX_AVX2', + 'f16c': 'X86_CPUID_FEATURE_ECX_F16C', + 'rdrand': 'X86_CPUID_FEATURE_ECX_RDRAND', + + 'axmmx': 'X86_CPUID_AMD_FEATURE_EDX_AXMMX', + '3dnowext': 'X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX', + '3dnow': 'X86_CPUID_AMD_FEATURE_EDX_3DNOW', + 'svm': 'X86_CPUID_AMD_FEATURE_ECX_SVM', + 'cr8l': 'X86_CPUID_AMD_FEATURE_ECX_CR8L', + 'abm': 'X86_CPUID_AMD_FEATURE_ECX_ABM', + 'sse4a': 'X86_CPUID_AMD_FEATURE_ECX_SSE4A', + '3dnowprf': 'X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF', + 'xop': 'X86_CPUID_AMD_FEATURE_ECX_XOP', + 'fma4': 'X86_CPUID_AMD_FEATURE_ECX_FMA4', +}; + +## \@ophints values. +g_kdHints = { + 'invalid': 'DISOPTYPE_INVALID', ##< + 'harmless': 'DISOPTYPE_HARMLESS', ##< + 'controlflow': 'DISOPTYPE_CONTROLFLOW', ##< + 'potentially_dangerous': 'DISOPTYPE_POTENTIALLY_DANGEROUS', ##< + 'dangerous': 'DISOPTYPE_DANGEROUS', ##< + 'portio': 'DISOPTYPE_PORTIO', ##< + 'privileged': 'DISOPTYPE_PRIVILEGED', ##< + 'privileged_notrap': 'DISOPTYPE_PRIVILEGED_NOTRAP', ##< + 'uncond_controlflow': 'DISOPTYPE_UNCOND_CONTROLFLOW', ##< + 'relative_controlflow': 'DISOPTYPE_RELATIVE_CONTROLFLOW', ##< + 'cond_controlflow': 'DISOPTYPE_COND_CONTROLFLOW', ##< + 'interrupt': 'DISOPTYPE_INTERRUPT', ##< + 'illegal': 'DISOPTYPE_ILLEGAL', ##< + 'rrm_dangerous': 'DISOPTYPE_RRM_DANGEROUS', ##< Some additional dangerous ones when recompiling raw r0. + 'rrm_dangerous_16': 'DISOPTYPE_RRM_DANGEROUS_16', ##< Some additional dangerous ones when recompiling 16-bit raw r0. + 'inhibit_irqs': 'DISOPTYPE_INHIBIT_IRQS', ##< Will or can inhibit irqs (sti, pop ss, mov ss) */ + 'portio_read': 'DISOPTYPE_PORTIO_READ', ##< + 'portio_write': 'DISOPTYPE_PORTIO_WRITE', ##< + 'invalid_64': 'DISOPTYPE_INVALID_64', ##< Invalid in 64 bits mode + 'only_64': 'DISOPTYPE_ONLY_64', ##< Only valid in 64 bits mode + 'default_64_op_size': 'DISOPTYPE_DEFAULT_64_OP_SIZE', ##< Default 64 bits operand size + 'forced_64_op_size': 'DISOPTYPE_FORCED_64_OP_SIZE', ##< Forced 64 bits operand size; regardless of prefix bytes + 'rexb_extends_opreg': 'DISOPTYPE_REXB_EXTENDS_OPREG', ##< REX.B extends the register field in the opcode byte + 'mod_fixed_11': 'DISOPTYPE_MOD_FIXED_11', ##< modrm.mod is always 11b + 'forced_32_op_size_x86': 'DISOPTYPE_FORCED_32_OP_SIZE_X86', ##< Forced 32 bits operand size; regardless of prefix bytes + ## (only in 16 & 32 bits mode!) + 'avx': 'DISOPTYPE_AVX', ##< AVX,AVX2,++ instruction. Not implemented yet! + 'sse': 'DISOPTYPE_SSE', ##< SSE,SSE2,SSE3,++ instruction. Not implemented yet! + 'mmx': 'DISOPTYPE_MMX', ##< MMX,MMXExt,3DNow,++ instruction. Not implemented yet! + 'fpu': 'DISOPTYPE_FPU', ##< FPU instruction. Not implemented yet! + 'ignores_oz_pfx': '', ##< Ignores operand size prefix 66h. + 'ignores_rexw': '', ##< Ignores REX.W. + 'ignores_op_sizes': '', ##< Shorthand for "ignores_oz_pfx | ignores_op_sizes". + 'vex_l_zero': '', ##< VEX.L must be 0. + 'vex_l_ignored': '', ##< VEX.L is ignored. + 'vex_v_zero': '', ##< VEX.V must be 0. (generate sub-table?) + 'lock_allowed': '', ##< Lock prefix allowed. +}; + +## \@opxcpttype values (see SDMv2 2.4, 2.7). +g_kdXcptTypes = { + 'none': [], + '1': [], + '2': [], + '3': [], + '4': [], + '4UA': [], + '5': [], + '5LZ': [], # LZ = VEX.L must be zero. + '6': [], + '7': [], + '7LZ': [], + '8': [], + '11': [], + '12': [], + 'E1': [], + 'E1NF': [], + 'E2': [], + 'E3': [], + 'E3NF': [], + 'E4': [], + 'E4NF': [], + 'E5': [], + 'E5NF': [], + 'E6': [], + 'E6NF': [], + 'E7NF': [], + 'E9': [], + 'E9NF': [], + 'E10': [], + 'E11': [], + 'E12': [], + 'E12NF': [], +}; + + +def _isValidOpcodeByte(sOpcode): + """ + Checks if sOpcode is a valid lower case opcode byte. + Returns true/false. + """ + if len(sOpcode) == 4: + if sOpcode[:2] == '0x': + if sOpcode[2] in '0123456789abcdef': + if sOpcode[3] in '0123456789abcdef': + return True; + return False; + + +class InstructionMap(object): + """ + Instruction map. + + The opcode map provides the lead opcode bytes (empty for the one byte + opcode map). An instruction can be member of multiple opcode maps as long + as it uses the same opcode value within the map (because of VEX). + """ + + kdEncodings = { + 'legacy': [], + 'vex1': [], ##< VEX or EVEX prefix with vvvvv = 1 + 'vex2': [], ##< VEX or EVEX prefix with vvvvv = 2 + 'vex3': [], ##< VEX or EVEX prefix with vvvvv = 3 + 'xop8': [], ##< XOP prefix with vvvvv = 8 + 'xop9': [], ##< XOP prefix with vvvvv = 9 + 'xop10': [], ##< XOP prefix with vvvvv = 10 + }; + ## Selectors. + ## 1. The first value is the number of table entries required by a + ## decoder or disassembler for this type of selector. + ## 2. The second value is how many entries per opcode byte if applicable. + kdSelectors = { + 'byte': [ 256, 1, ], ##< next opcode byte selects the instruction (default). + 'byte+pfx': [ 1024, 4, ], ##< next opcode byte selects the instruction together with the 0x66, 0xf2 and 0xf3 prefixes. + '/r': [ 8, 1, ], ##< modrm.reg selects the instruction. + 'memreg /r':[ 16, 1, ], ##< modrm.reg and (modrm.mod == 3) selects the instruction. + 'mod /r': [ 32, 1, ], ##< modrm.reg and modrm.mod selects the instruction. + '!11 /r': [ 8, 1, ], ##< modrm.reg selects the instruction with modrm.mod != 0y11. + '11 /r': [ 8, 1, ], ##< modrm.reg select the instruction with modrm.mod == 0y11. + '11': [ 64, 1, ], ##< modrm.reg and modrm.rm select the instruction with modrm.mod == 0y11. + }; + + ## Define the subentry number according to the Instruction::sPrefix + ## value for 'byte+pfx' selected tables. + kiPrefixOrder = { + 'none': 0, + '0x66': 1, + '0xf3': 2, + '0xf2': 3, + }; + + def __init__(self, sName, sIemName = None, asLeadOpcodes = None, sSelector = 'byte+pfx', + sEncoding = 'legacy', sDisParse = None): + assert sSelector in self.kdSelectors; + assert sEncoding in self.kdEncodings; + if asLeadOpcodes is None: + asLeadOpcodes = []; + else: + for sOpcode in asLeadOpcodes: + assert _isValidOpcodeByte(sOpcode); + assert sDisParse is None or sDisParse.startswith('IDX_Parse'); + + self.sName = sName; + self.sIemName = sIemName; + self.asLeadOpcodes = asLeadOpcodes; ##< Lead opcode bytes formatted as hex strings like '0x0f'. + self.sSelector = sSelector; ##< The member selector, see kdSelectors. + self.sEncoding = sEncoding; ##< The encoding, see kdSelectors. + self.aoInstructions = [] # type: Instruction + self.sDisParse = sDisParse; ##< IDX_ParseXXX. + + def copy(self, sNewName, sPrefixFilter = None): + """ + Copies the table with filtering instruction by sPrefix if not None. + """ + oCopy = InstructionMap(sNewName, sIemName = self.sIemName, asLeadOpcodes = self.asLeadOpcodes, + sSelector = 'byte' if sPrefixFilter is not None and self.sSelector == 'byte+pfx' + else self.sSelector, + sEncoding = self.sEncoding, sDisParse = self.sDisParse); + if sPrefixFilter is None: + oCopy.aoInstructions = list(self.aoInstructions); + else: + oCopy.aoInstructions = [oInstr for oInstr in self.aoInstructions if oInstr.sPrefix == sPrefixFilter]; + return oCopy; + + def getTableSize(self): + """ + Number of table entries. This corresponds directly to the selector. + """ + return self.kdSelectors[self.sSelector][0]; + + def getEntriesPerByte(self): + """ + Number of table entries per opcode bytes. + + This only really makes sense for the 'byte' and 'byte+pfx' selectors, for + the others it will just return 1. + """ + return self.kdSelectors[self.sSelector][1]; + + def getInstructionIndex(self, oInstr): + """ + Returns the table index for the instruction. + """ + bOpcode = oInstr.getOpcodeByte(); + + # The byte selectors are simple. We need a full opcode byte and need just return it. + if self.sSelector == 'byte': + assert oInstr.sOpcode[:2] == '0x' and len(oInstr.sOpcode) == 4, str(oInstr); + return bOpcode; + + # The byte + prefix selector is similarly simple, though requires a prefix as well as the full opcode. + if self.sSelector == 'byte+pfx': + assert oInstr.sOpcode[:2] == '0x' and len(oInstr.sOpcode) == 4, str(oInstr); + assert self.kiPrefixOrder.get(oInstr.sPrefix, -16384) >= 0; + return bOpcode * 4 + self.kiPrefixOrder.get(oInstr.sPrefix, -16384); + + # The other selectors needs masking and shifting. + if self.sSelector == '/r': + return (bOpcode >> 3) & 0x7; + + if self.sSelector == 'mod /r': + return (bOpcode >> 3) & 0x1f; + + if self.sSelector == 'memreg /r': + return ((bOpcode >> 3) & 0x7) | (int((bOpcode >> 6) == 3) << 3); + + if self.sSelector == '!11 /r': + assert (bOpcode & 0xc0) != 0xc, str(oInstr); + return (bOpcode >> 3) & 0x7; + + if self.sSelector == '11 /r': + assert (bOpcode & 0xc0) == 0xc, str(oInstr); + return (bOpcode >> 3) & 0x7; + + if self.sSelector == '11': + assert (bOpcode & 0xc0) == 0xc, str(oInstr); + return bOpcode & 0x3f; + + assert False, self.sSelector; + return -1; + + def getInstructionsInTableOrder(self): + """ + Get instructions in table order. + + Returns array of instructions. Normally there is exactly one + instruction per entry. However the entry could also be None if + not instruction was specified for that opcode value. Or there + could be a list of instructions to deal with special encodings + where for instance prefix (e.g. REX.W) encodes a different + instruction or different CPUs have different instructions or + prefixes in the same place. + """ + # Start with empty table. + cTable = self.getTableSize(); + aoTable = [None] * cTable; + + # Insert the instructions. + for oInstr in self.aoInstructions: + if oInstr.sOpcode: + idxOpcode = self.getInstructionIndex(oInstr); + assert idxOpcode < cTable, str(idxOpcode); + + oExisting = aoTable[idxOpcode]; + if oExisting is None: + aoTable[idxOpcode] = oInstr; + elif not isinstance(oExisting, list): + aoTable[idxOpcode] = list([oExisting, oInstr]); + else: + oExisting.append(oInstr); + + return aoTable; + + + def getDisasTableName(self): + """ + Returns the disassembler table name for this map. + """ + sName = 'g_aDisas'; + for sWord in self.sName.split('_'): + if sWord == 'm': # suffix indicating modrm.mod==mem + sName += '_m'; + elif sWord == 'r': # suffix indicating modrm.mod==reg + sName += '_r'; + elif len(sWord) == 2 and re.match('^[a-f0-9][a-f0-9]$', sWord): + sName += '_' + sWord; + else: + sWord = sWord.replace('grp', 'Grp'); + sWord = sWord.replace('map', 'Map'); + sName += sWord[0].upper() + sWord[1:]; + return sName; + + def getDisasRangeName(self): + """ + Returns the disassembler table range name for this map. + """ + return self.getDisasTableName().replace('g_aDisas', 'g_Disas') + 'Range'; + + def isVexMap(self): + """ Returns True if a VEX map. """ + return self.sEncoding.startswith('vex'); + + +class TestType(object): + """ + Test value type. + + This base class deals with integer like values. The fUnsigned constructor + parameter indicates the default stance on zero vs sign extending. It is + possible to override fUnsigned=True by prefixing the value with '+' or '-'. + """ + def __init__(self, sName, acbSizes = None, fUnsigned = True): + self.sName = sName; + self.acbSizes = [1, 2, 4, 8, 16, 32] if acbSizes is None else acbSizes; # Normal sizes. + self.fUnsigned = fUnsigned; + + class BadValue(Exception): + """ Bad value exception. """ + def __init__(self, sMessage): + Exception.__init__(self, sMessage); + self.sMessage = sMessage; + + ## For ascii ~ operator. + kdHexInv = { + '0': 'f', + '1': 'e', + '2': 'd', + '3': 'c', + '4': 'b', + '5': 'a', + '6': '9', + '7': '8', + '8': '7', + '9': '6', + 'a': '5', + 'b': '4', + 'c': '3', + 'd': '2', + 'e': '1', + 'f': '0', + }; + + def get(self, sValue): + """ + Get the shortest normal sized byte representation of oValue. + + Returns ((fSignExtend, bytearray), ) or ((fSignExtend, bytearray), (fSignExtend, bytearray), ). + The latter form is for AND+OR pairs where the first entry is what to + AND with the field and the second the one or OR with. + + Raises BadValue if invalid value. + """ + if not sValue: + raise TestType.BadValue('empty value'); + + # Deal with sign and detect hexadecimal or decimal. + fSignExtend = not self.fUnsigned; + if sValue[0] == '-' or sValue[0] == '+': + fSignExtend = True; + fHex = len(sValue) > 3 and sValue[1:3].lower() == '0x'; + else: + fHex = len(sValue) > 2 and sValue[0:2].lower() == '0x'; + + # try convert it to long integer. + try: + iValue = long(sValue, 16 if fHex else 10); + except Exception as oXcpt: + raise TestType.BadValue('failed to convert "%s" to integer (%s)' % (sValue, oXcpt)); + + # Convert the hex string and pad it to a decent value. Negative values + # needs to be manually converted to something non-negative (~-n + 1). + if iValue >= 0: + sHex = hex(iValue); + if sys.version_info[0] < 3: + assert sHex[-1] == 'L'; + sHex = sHex[:-1]; + assert sHex[:2] == '0x'; + sHex = sHex[2:]; + else: + sHex = hex(-iValue - 1); + if sys.version_info[0] < 3: + assert sHex[-1] == 'L'; + sHex = sHex[:-1]; + assert sHex[:2] == '0x'; + sHex = ''.join([self.kdHexInv[sDigit] for sDigit in sHex[2:]]); + if fSignExtend and sHex[0] not in [ '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']: + sHex = 'f' + sHex; + + cDigits = len(sHex); + if cDigits <= self.acbSizes[-1] * 2: + for cb in self.acbSizes: + cNaturalDigits = cb * 2; + if cDigits <= cNaturalDigits: + break; + else: + cNaturalDigits = self.acbSizes[-1] * 2; + cNaturalDigits = int((cDigits + cNaturalDigits - 1) / cNaturalDigits) * cNaturalDigits; + assert isinstance(cNaturalDigits, int) + + if cNaturalDigits != cDigits: + cNeeded = cNaturalDigits - cDigits; + if iValue >= 0: + sHex = ('0' * cNeeded) + sHex; + else: + sHex = ('f' * cNeeded) + sHex; + + # Invert and convert to bytearray and return it. + abValue = bytearray([int(sHex[offHex - 2 : offHex], 16) for offHex in range(len(sHex), 0, -2)]); + + return ((fSignExtend, abValue),); + + def validate(self, sValue): + """ + Returns True if value is okay, error message on failure. + """ + try: + self.get(sValue); + except TestType.BadValue as oXcpt: + return oXcpt.sMessage; + return True; + + def isAndOrPair(self, sValue): + """ + Checks if sValue is a pair. + """ + _ = sValue; + return False; + + +class TestTypeEflags(TestType): + """ + Special value parsing for EFLAGS/RFLAGS/FLAGS. + """ + + kdZeroValueFlags = { 'nv': 0, 'pl': 0, 'nz': 0, 'na': 0, 'pe': 0, 'nc': 0, 'di': 0, 'up': 0 }; + + def __init__(self, sName): + TestType.__init__(self, sName, acbSizes = [1, 2, 4, 8], fUnsigned = True); + + def get(self, sValue): + fClear = 0; + fSet = 0; + for sFlag in sValue.split(','): + sConstant = g_kdEFlagsMnemonics.get(sFlag, None); + if sConstant is None: + raise self.BadValue('Unknown flag "%s" in "%s"' % (sFlag, sValue)) + if sConstant[0] == '!': + fClear |= g_kdX86EFlagsConstants[sConstant[1:]]; + else: + fSet |= g_kdX86EFlagsConstants[sConstant]; + + aoSet = TestType.get(self, '0x%x' % (fSet,)); + if fClear != 0: + aoClear = TestType.get(self, '%#x' % (fClear,)) + assert self.isAndOrPair(sValue) is True; + return (aoClear[0], aoSet[0]); + assert self.isAndOrPair(sValue) is False; + return aoSet; + + def isAndOrPair(self, sValue): + for sZeroFlag in self.kdZeroValueFlags: + if sValue.find(sZeroFlag) >= 0: + return True; + return False; + +class TestTypeFromDict(TestType): + """ + Special value parsing for CR0. + """ + + kdZeroValueFlags = { 'nv': 0, 'pl': 0, 'nz': 0, 'na': 0, 'pe': 0, 'nc': 0, 'di': 0, 'up': 0 }; + + def __init__(self, sName, kdConstantsAndValues, sConstantPrefix): + TestType.__init__(self, sName, acbSizes = [1, 2, 4, 8], fUnsigned = True); + self.kdConstantsAndValues = kdConstantsAndValues; + self.sConstantPrefix = sConstantPrefix; + + def get(self, sValue): + fValue = 0; + for sFlag in sValue.split(','): + fFlagValue = self.kdConstantsAndValues.get(self.sConstantPrefix + sFlag.upper(), None); + if fFlagValue is None: + raise self.BadValue('Unknown flag "%s" in "%s"' % (sFlag, sValue)) + fValue |= fFlagValue; + return TestType.get(self, '0x%x' % (fValue,)); + + +class TestInOut(object): + """ + One input or output state modifier. + + This should be thought as values to modify BS3REGCTX and extended (needs + to be structured) state. + """ + ## Assigned operators. + kasOperators = [ + '&|=', # Special AND(INV)+OR operator for use with EFLAGS. + '&~=', + '&=', + '|=', + '=' + ]; + ## Types + kdTypes = { + 'uint': TestType('uint', fUnsigned = True), + 'int': TestType('int'), + 'efl': TestTypeEflags('efl'), + 'cr0': TestTypeFromDict('cr0', g_kdX86Cr0Constants, 'X86_CR0_'), + 'cr4': TestTypeFromDict('cr4', g_kdX86Cr4Constants, 'X86_CR4_'), + 'xcr0': TestTypeFromDict('xcr0', g_kdX86XSaveCConstants, 'XSAVE_C_'), + }; + ## CPU context fields. + kdFields = { + # name: ( default type, [both|input|output], ) + # Operands. + 'op1': ( 'uint', 'both', ), ## \@op1 + 'op2': ( 'uint', 'both', ), ## \@op2 + 'op3': ( 'uint', 'both', ), ## \@op3 + 'op4': ( 'uint', 'both', ), ## \@op4 + # Flags. + 'efl': ( 'efl', 'both', ), + 'efl_undef': ( 'uint', 'output', ), + # 8-bit GPRs. + 'al': ( 'uint', 'both', ), + 'cl': ( 'uint', 'both', ), + 'dl': ( 'uint', 'both', ), + 'bl': ( 'uint', 'both', ), + 'ah': ( 'uint', 'both', ), + 'ch': ( 'uint', 'both', ), + 'dh': ( 'uint', 'both', ), + 'bh': ( 'uint', 'both', ), + 'r8l': ( 'uint', 'both', ), + 'r9l': ( 'uint', 'both', ), + 'r10l': ( 'uint', 'both', ), + 'r11l': ( 'uint', 'both', ), + 'r12l': ( 'uint', 'both', ), + 'r13l': ( 'uint', 'both', ), + 'r14l': ( 'uint', 'both', ), + 'r15l': ( 'uint', 'both', ), + # 16-bit GPRs. + 'ax': ( 'uint', 'both', ), + 'dx': ( 'uint', 'both', ), + 'cx': ( 'uint', 'both', ), + 'bx': ( 'uint', 'both', ), + 'sp': ( 'uint', 'both', ), + 'bp': ( 'uint', 'both', ), + 'si': ( 'uint', 'both', ), + 'di': ( 'uint', 'both', ), + 'r8w': ( 'uint', 'both', ), + 'r9w': ( 'uint', 'both', ), + 'r10w': ( 'uint', 'both', ), + 'r11w': ( 'uint', 'both', ), + 'r12w': ( 'uint', 'both', ), + 'r13w': ( 'uint', 'both', ), + 'r14w': ( 'uint', 'both', ), + 'r15w': ( 'uint', 'both', ), + # 32-bit GPRs. + 'eax': ( 'uint', 'both', ), + 'edx': ( 'uint', 'both', ), + 'ecx': ( 'uint', 'both', ), + 'ebx': ( 'uint', 'both', ), + 'esp': ( 'uint', 'both', ), + 'ebp': ( 'uint', 'both', ), + 'esi': ( 'uint', 'both', ), + 'edi': ( 'uint', 'both', ), + 'r8d': ( 'uint', 'both', ), + 'r9d': ( 'uint', 'both', ), + 'r10d': ( 'uint', 'both', ), + 'r11d': ( 'uint', 'both', ), + 'r12d': ( 'uint', 'both', ), + 'r13d': ( 'uint', 'both', ), + 'r14d': ( 'uint', 'both', ), + 'r15d': ( 'uint', 'both', ), + # 64-bit GPRs. + 'rax': ( 'uint', 'both', ), + 'rdx': ( 'uint', 'both', ), + 'rcx': ( 'uint', 'both', ), + 'rbx': ( 'uint', 'both', ), + 'rsp': ( 'uint', 'both', ), + 'rbp': ( 'uint', 'both', ), + 'rsi': ( 'uint', 'both', ), + 'rdi': ( 'uint', 'both', ), + 'r8': ( 'uint', 'both', ), + 'r9': ( 'uint', 'both', ), + 'r10': ( 'uint', 'both', ), + 'r11': ( 'uint', 'both', ), + 'r12': ( 'uint', 'both', ), + 'r13': ( 'uint', 'both', ), + 'r14': ( 'uint', 'both', ), + 'r15': ( 'uint', 'both', ), + # 16-bit, 32-bit or 64-bit registers according to operand size. + 'oz.rax': ( 'uint', 'both', ), + 'oz.rdx': ( 'uint', 'both', ), + 'oz.rcx': ( 'uint', 'both', ), + 'oz.rbx': ( 'uint', 'both', ), + 'oz.rsp': ( 'uint', 'both', ), + 'oz.rbp': ( 'uint', 'both', ), + 'oz.rsi': ( 'uint', 'both', ), + 'oz.rdi': ( 'uint', 'both', ), + 'oz.r8': ( 'uint', 'both', ), + 'oz.r9': ( 'uint', 'both', ), + 'oz.r10': ( 'uint', 'both', ), + 'oz.r11': ( 'uint', 'both', ), + 'oz.r12': ( 'uint', 'both', ), + 'oz.r13': ( 'uint', 'both', ), + 'oz.r14': ( 'uint', 'both', ), + 'oz.r15': ( 'uint', 'both', ), + # Control registers. + 'cr0': ( 'cr0', 'both', ), + 'cr4': ( 'cr4', 'both', ), + 'xcr0': ( 'xcr0', 'both', ), + # FPU Registers + 'fcw': ( 'uint', 'both', ), + 'fsw': ( 'uint', 'both', ), + 'ftw': ( 'uint', 'both', ), + 'fop': ( 'uint', 'both', ), + 'fpuip': ( 'uint', 'both', ), + 'fpucs': ( 'uint', 'both', ), + 'fpudp': ( 'uint', 'both', ), + 'fpuds': ( 'uint', 'both', ), + 'mxcsr': ( 'uint', 'both', ), + 'st0': ( 'uint', 'both', ), + 'st1': ( 'uint', 'both', ), + 'st2': ( 'uint', 'both', ), + 'st3': ( 'uint', 'both', ), + 'st4': ( 'uint', 'both', ), + 'st5': ( 'uint', 'both', ), + 'st6': ( 'uint', 'both', ), + 'st7': ( 'uint', 'both', ), + # MMX registers. + 'mm0': ( 'uint', 'both', ), + 'mm1': ( 'uint', 'both', ), + 'mm2': ( 'uint', 'both', ), + 'mm3': ( 'uint', 'both', ), + 'mm4': ( 'uint', 'both', ), + 'mm5': ( 'uint', 'both', ), + 'mm6': ( 'uint', 'both', ), + 'mm7': ( 'uint', 'both', ), + # SSE registers. + 'xmm0': ( 'uint', 'both', ), + 'xmm1': ( 'uint', 'both', ), + 'xmm2': ( 'uint', 'both', ), + 'xmm3': ( 'uint', 'both', ), + 'xmm4': ( 'uint', 'both', ), + 'xmm5': ( 'uint', 'both', ), + 'xmm6': ( 'uint', 'both', ), + 'xmm7': ( 'uint', 'both', ), + 'xmm8': ( 'uint', 'both', ), + 'xmm9': ( 'uint', 'both', ), + 'xmm10': ( 'uint', 'both', ), + 'xmm11': ( 'uint', 'both', ), + 'xmm12': ( 'uint', 'both', ), + 'xmm13': ( 'uint', 'both', ), + 'xmm14': ( 'uint', 'both', ), + 'xmm15': ( 'uint', 'both', ), + 'xmm0.lo': ( 'uint', 'both', ), + 'xmm1.lo': ( 'uint', 'both', ), + 'xmm2.lo': ( 'uint', 'both', ), + 'xmm3.lo': ( 'uint', 'both', ), + 'xmm4.lo': ( 'uint', 'both', ), + 'xmm5.lo': ( 'uint', 'both', ), + 'xmm6.lo': ( 'uint', 'both', ), + 'xmm7.lo': ( 'uint', 'both', ), + 'xmm8.lo': ( 'uint', 'both', ), + 'xmm9.lo': ( 'uint', 'both', ), + 'xmm10.lo': ( 'uint', 'both', ), + 'xmm11.lo': ( 'uint', 'both', ), + 'xmm12.lo': ( 'uint', 'both', ), + 'xmm13.lo': ( 'uint', 'both', ), + 'xmm14.lo': ( 'uint', 'both', ), + 'xmm15.lo': ( 'uint', 'both', ), + 'xmm0.hi': ( 'uint', 'both', ), + 'xmm1.hi': ( 'uint', 'both', ), + 'xmm2.hi': ( 'uint', 'both', ), + 'xmm3.hi': ( 'uint', 'both', ), + 'xmm4.hi': ( 'uint', 'both', ), + 'xmm5.hi': ( 'uint', 'both', ), + 'xmm6.hi': ( 'uint', 'both', ), + 'xmm7.hi': ( 'uint', 'both', ), + 'xmm8.hi': ( 'uint', 'both', ), + 'xmm9.hi': ( 'uint', 'both', ), + 'xmm10.hi': ( 'uint', 'both', ), + 'xmm11.hi': ( 'uint', 'both', ), + 'xmm12.hi': ( 'uint', 'both', ), + 'xmm13.hi': ( 'uint', 'both', ), + 'xmm14.hi': ( 'uint', 'both', ), + 'xmm15.hi': ( 'uint', 'both', ), + 'xmm0.lo.zx': ( 'uint', 'both', ), + 'xmm1.lo.zx': ( 'uint', 'both', ), + 'xmm2.lo.zx': ( 'uint', 'both', ), + 'xmm3.lo.zx': ( 'uint', 'both', ), + 'xmm4.lo.zx': ( 'uint', 'both', ), + 'xmm5.lo.zx': ( 'uint', 'both', ), + 'xmm6.lo.zx': ( 'uint', 'both', ), + 'xmm7.lo.zx': ( 'uint', 'both', ), + 'xmm8.lo.zx': ( 'uint', 'both', ), + 'xmm9.lo.zx': ( 'uint', 'both', ), + 'xmm10.lo.zx': ( 'uint', 'both', ), + 'xmm11.lo.zx': ( 'uint', 'both', ), + 'xmm12.lo.zx': ( 'uint', 'both', ), + 'xmm13.lo.zx': ( 'uint', 'both', ), + 'xmm14.lo.zx': ( 'uint', 'both', ), + 'xmm15.lo.zx': ( 'uint', 'both', ), + 'xmm0.dw0': ( 'uint', 'both', ), + 'xmm1.dw0': ( 'uint', 'both', ), + 'xmm2.dw0': ( 'uint', 'both', ), + 'xmm3.dw0': ( 'uint', 'both', ), + 'xmm4.dw0': ( 'uint', 'both', ), + 'xmm5.dw0': ( 'uint', 'both', ), + 'xmm6.dw0': ( 'uint', 'both', ), + 'xmm7.dw0': ( 'uint', 'both', ), + 'xmm8.dw0': ( 'uint', 'both', ), + 'xmm9.dw0': ( 'uint', 'both', ), + 'xmm10.dw0': ( 'uint', 'both', ), + 'xmm11.dw0': ( 'uint', 'both', ), + 'xmm12.dw0': ( 'uint', 'both', ), + 'xmm13.dw0': ( 'uint', 'both', ), + 'xmm14.dw0': ( 'uint', 'both', ), + 'xmm15_dw0': ( 'uint', 'both', ), + # AVX registers. + 'ymm0': ( 'uint', 'both', ), + 'ymm1': ( 'uint', 'both', ), + 'ymm2': ( 'uint', 'both', ), + 'ymm3': ( 'uint', 'both', ), + 'ymm4': ( 'uint', 'both', ), + 'ymm5': ( 'uint', 'both', ), + 'ymm6': ( 'uint', 'both', ), + 'ymm7': ( 'uint', 'both', ), + 'ymm8': ( 'uint', 'both', ), + 'ymm9': ( 'uint', 'both', ), + 'ymm10': ( 'uint', 'both', ), + 'ymm11': ( 'uint', 'both', ), + 'ymm12': ( 'uint', 'both', ), + 'ymm13': ( 'uint', 'both', ), + 'ymm14': ( 'uint', 'both', ), + 'ymm15': ( 'uint', 'both', ), + + # Special ones. + 'value.xcpt': ( 'uint', 'output', ), + }; + + def __init__(self, sField, sOp, sValue, sType): + assert sField in self.kdFields; + assert sOp in self.kasOperators; + self.sField = sField; + self.sOp = sOp; + self.sValue = sValue; + self.sType = sType; + assert isinstance(sField, str); + assert isinstance(sOp, str); + assert isinstance(sType, str); + assert isinstance(sValue, str); + + +class TestSelector(object): + """ + One selector for an instruction test. + """ + ## Selector compare operators. + kasCompareOps = [ '==', '!=' ]; + ## Selector variables and their valid values. + kdVariables = { + # Operand size. + 'size': { + 'o16': 'size_o16', + 'o32': 'size_o32', + 'o64': 'size_o64', + }, + # VEX.L value. + 'vex.l': { + '0': 'vexl_0', + '1': 'vexl_1', + }, + # Execution ring. + 'ring': { + '0': 'ring_0', + '1': 'ring_1', + '2': 'ring_2', + '3': 'ring_3', + '0..2': 'ring_0_thru_2', + '1..3': 'ring_1_thru_3', + }, + # Basic code mode. + 'codebits': { + '64': 'code_64bit', + '32': 'code_32bit', + '16': 'code_16bit', + }, + # cpu modes. + 'mode': { + 'real': 'mode_real', + 'prot': 'mode_prot', + 'long': 'mode_long', + 'v86': 'mode_v86', + 'smm': 'mode_smm', + 'vmx': 'mode_vmx', + 'svm': 'mode_svm', + }, + # paging on/off + 'paging': { + 'on': 'paging_on', + 'off': 'paging_off', + }, + # CPU vendor + 'vendor': { + 'amd': 'vendor_amd', + 'intel': 'vendor_intel', + 'via': 'vendor_via', + }, + }; + ## Selector shorthand predicates. + ## These translates into variable expressions. + kdPredicates = { + 'o16': 'size==o16', + 'o32': 'size==o32', + 'o64': 'size==o64', + 'ring0': 'ring==0', + '!ring0': 'ring==1..3', + 'ring1': 'ring==1', + 'ring2': 'ring==2', + 'ring3': 'ring==3', + 'user': 'ring==3', + 'supervisor': 'ring==0..2', + '16-bit': 'codebits==16', + '32-bit': 'codebits==32', + '64-bit': 'codebits==64', + 'real': 'mode==real', + 'prot': 'mode==prot', + 'long': 'mode==long', + 'v86': 'mode==v86', + 'smm': 'mode==smm', + 'vmx': 'mode==vmx', + 'svm': 'mode==svm', + 'paging': 'paging==on', + '!paging': 'paging==off', + 'amd': 'vendor==amd', + '!amd': 'vendor!=amd', + 'intel': 'vendor==intel', + '!intel': 'vendor!=intel', + 'via': 'vendor==via', + '!via': 'vendor!=via', + }; + + def __init__(self, sVariable, sOp, sValue): + assert sVariable in self.kdVariables; + assert sOp in self.kasCompareOps; + assert sValue in self.kdVariables[sVariable]; + self.sVariable = sVariable; + self.sOp = sOp; + self.sValue = sValue; + + +class InstructionTest(object): + """ + Instruction test. + """ + + def __init__(self, oInstr): # type: (InstructionTest, Instruction) + self.oInstr = oInstr # type: InstructionTest + self.aoInputs = [] # type: list(TestInOut) + self.aoOutputs = [] # type: list(TestInOut) + self.aoSelectors = [] # type: list(TestSelector) + + def toString(self, fRepr = False): + """ + Converts it to string representation. + """ + asWords = []; + if self.aoSelectors: + for oSelector in self.aoSelectors: + asWords.append('%s%s%s' % (oSelector.sVariable, oSelector.sOp, oSelector.sValue,)); + asWords.append('/'); + + for oModifier in self.aoInputs: + asWords.append('%s%s%s:%s' % (oModifier.sField, oModifier.sOp, oModifier.sValue, oModifier.sType,)); + + asWords.append('->'); + + for oModifier in self.aoOutputs: + asWords.append('%s%s%s:%s' % (oModifier.sField, oModifier.sOp, oModifier.sValue, oModifier.sType,)); + + if fRepr: + return '<' + ' '.join(asWords) + '>'; + return ' '.join(asWords); + + def __str__(self): + """ Provide string represenation. """ + return self.toString(False); + + def __repr__(self): + """ Provide unambigious string representation. """ + return self.toString(True); + +class Operand(object): + """ + Instruction operand. + """ + + def __init__(self, sWhere, sType): + assert sWhere in g_kdOpLocations, sWhere; + assert sType in g_kdOpTypes, sType; + self.sWhere = sWhere; ##< g_kdOpLocations + self.sType = sType; ##< g_kdOpTypes + + def usesModRM(self): + """ Returns True if using some form of ModR/M encoding. """ + return self.sType[0] in ['E', 'G', 'M']; + + + +class Instruction(object): # pylint: disable=too-many-instance-attributes + """ + Instruction. + """ + + def __init__(self, sSrcFile, iLine): + ## @name Core attributes. + ## @{ + self.oParent = None # type: Instruction + self.sMnemonic = None; + self.sBrief = None; + self.asDescSections = [] # type: list(str) + self.aoMaps = [] # type: list(InstructionMap) + self.aoOperands = [] # type: list(Operand) + self.sPrefix = None; ##< Single prefix: None, 'none', 0x66, 0xf3, 0xf2 + self.sOpcode = None # type: str + self.sSubOpcode = None # type: str + self.sEncoding = None; + self.asFlTest = None; + self.asFlModify = None; + self.asFlUndefined = None; + self.asFlSet = None; + self.asFlClear = None; + self.dHints = {}; ##< Dictionary of instruction hints, flags, whatnot. (Dictionary for speed; dummy value). + self.sDisEnum = None; ##< OP_XXXX value. Default is based on the uppercased mnemonic. + self.asCpuIds = []; ##< The CPUID feature bit names for this instruction. If multiple, assume AND. + self.asReqFeatures = []; ##< Which features are required to be enabled to run this instruction. + self.aoTests = [] # type: list(InstructionTest) + self.sMinCpu = None; ##< Indicates the minimum CPU required for the instruction. Not set when oCpuExpr is. + self.oCpuExpr = None; ##< Some CPU restriction expression... + self.sGroup = None; + self.fUnused = False; ##< Unused instruction. + self.fInvalid = False; ##< Invalid instruction (like UD2). + self.sInvalidStyle = None; ##< Invalid behviour style (g_kdInvalidStyles), + self.sXcptType = None; ##< Exception type (g_kdXcptTypes). + ## @} + + ## @name Implementation attributes. + ## @{ + self.sStats = None; + self.sFunction = None; + self.fStub = False; + self.fUdStub = False; + ## @} + + ## @name Decoding info + ## @{ + self.sSrcFile = sSrcFile; + self.iLineCreated = iLine; + self.iLineCompleted = None; + self.cOpTags = 0; + self.iLineFnIemOpMacro = -1; + self.iLineMnemonicMacro = -1; + ## @} + + ## @name Intermediate input fields. + ## @{ + self.sRawDisOpNo = None; + self.asRawDisParams = []; + self.sRawIemOpFlags = None; + self.sRawOldOpcodes = None; + self.asCopyTests = []; + ## @} + + def toString(self, fRepr = False): + """ Turn object into a string. """ + aasFields = []; + + aasFields.append(['opcode', self.sOpcode]); + if self.sPrefix: + aasFields.append(['prefix', self.sPrefix]); + aasFields.append(['mnemonic', self.sMnemonic]); + for iOperand, oOperand in enumerate(self.aoOperands): + aasFields.append(['op%u' % (iOperand + 1,), '%s:%s' % (oOperand.sWhere, oOperand.sType,)]); + if self.aoMaps: aasFields.append(['maps', ','.join([oMap.sName for oMap in self.aoMaps])]); + aasFields.append(['encoding', self.sEncoding]); + if self.dHints: aasFields.append(['hints', ','.join(self.dHints.keys())]); + aasFields.append(['disenum', self.sDisEnum]); + if self.asCpuIds: aasFields.append(['cpuid', ','.join(self.asCpuIds)]); + aasFields.append(['group', self.sGroup]); + if self.fUnused: aasFields.append(['unused', 'True']); + if self.fInvalid: aasFields.append(['invalid', 'True']); + aasFields.append(['invlstyle', self.sInvalidStyle]); + aasFields.append(['fltest', self.asFlTest]); + aasFields.append(['flmodify', self.asFlModify]); + aasFields.append(['flundef', self.asFlUndefined]); + aasFields.append(['flset', self.asFlSet]); + aasFields.append(['flclear', self.asFlClear]); + aasFields.append(['mincpu', self.sMinCpu]); + aasFields.append(['stats', self.sStats]); + aasFields.append(['sFunction', self.sFunction]); + if self.fStub: aasFields.append(['fStub', 'True']); + if self.fUdStub: aasFields.append(['fUdStub', 'True']); + if self.cOpTags: aasFields.append(['optags', str(self.cOpTags)]); + if self.iLineFnIemOpMacro != -1: aasFields.append(['FNIEMOP_XXX', str(self.iLineFnIemOpMacro)]); + if self.iLineMnemonicMacro != -1: aasFields.append(['IEMOP_MNEMMONICn', str(self.iLineMnemonicMacro)]); + + sRet = '<' if fRepr else ''; + for sField, sValue in aasFields: + if sValue is not None: + if len(sRet) > 1: + sRet += '; '; + sRet += '%s=%s' % (sField, sValue,); + if fRepr: + sRet += '>'; + + return sRet; + + def __str__(self): + """ Provide string represenation. """ + return self.toString(False); + + def __repr__(self): + """ Provide unambigious string representation. """ + return self.toString(True); + + def copy(self, oMap = None, sOpcode = None, sSubOpcode = None, sPrefix = None): + """ + Makes a copy of the object for the purpose of putting in a different map + or a different place in the current map. + """ + oCopy = Instruction(self.sSrcFile, self.iLineCreated); + + oCopy.oParent = self; + oCopy.sMnemonic = self.sMnemonic; + oCopy.sBrief = self.sBrief; + oCopy.asDescSections = list(self.asDescSections); + oCopy.aoMaps = [oMap,] if oMap else list(self.aoMaps); + oCopy.aoOperands = list(self.aoOperands); ## Deeper copy? + oCopy.sPrefix = sPrefix if sPrefix else self.sPrefix; + oCopy.sOpcode = sOpcode if sOpcode else self.sOpcode; + oCopy.sSubOpcode = sSubOpcode if sSubOpcode else self.sSubOpcode; + oCopy.sEncoding = self.sEncoding; + oCopy.asFlTest = self.asFlTest; + oCopy.asFlModify = self.asFlModify; + oCopy.asFlUndefined = self.asFlUndefined; + oCopy.asFlSet = self.asFlSet; + oCopy.asFlClear = self.asFlClear; + oCopy.dHints = dict(self.dHints); + oCopy.sDisEnum = self.sDisEnum; + oCopy.asCpuIds = list(self.asCpuIds); + oCopy.asReqFeatures = list(self.asReqFeatures); + oCopy.aoTests = list(self.aoTests); ## Deeper copy? + oCopy.sMinCpu = self.sMinCpu; + oCopy.oCpuExpr = self.oCpuExpr; + oCopy.sGroup = self.sGroup; + oCopy.fUnused = self.fUnused; + oCopy.fInvalid = self.fInvalid; + oCopy.sInvalidStyle = self.sInvalidStyle; + oCopy.sXcptType = self.sXcptType; + + oCopy.sStats = self.sStats; + oCopy.sFunction = self.sFunction; + oCopy.fStub = self.fStub; + oCopy.fUdStub = self.fUdStub; + + oCopy.iLineCompleted = self.iLineCompleted; + oCopy.cOpTags = self.cOpTags; + oCopy.iLineFnIemOpMacro = self.iLineFnIemOpMacro; + oCopy.iLineMnemonicMacro = self.iLineMnemonicMacro; + + oCopy.sRawDisOpNo = self.sRawDisOpNo; + oCopy.asRawDisParams = list(self.asRawDisParams); + oCopy.sRawIemOpFlags = self.sRawIemOpFlags; + oCopy.sRawOldOpcodes = self.sRawOldOpcodes; + oCopy.asCopyTests = list(self.asCopyTests); + + return oCopy; + + def getOpcodeByte(self): + """ + Decodes sOpcode into a byte range integer value. + Raises exception if sOpcode is None or invalid. + """ + if self.sOpcode is None: + raise Exception('No opcode byte for %s!' % (self,)); + sOpcode = str(self.sOpcode); # pylint type confusion workaround. + + # Full hex byte form. + if sOpcode[:2] == '0x': + return int(sOpcode, 16); + + # The /r form: + if len(sOpcode) == 4 and sOpcode.startswith('/') and sOpcode[-1].isdigit(): + return int(sOpcode[-1:]) << 3; + + # The 11/r form: + if len(sOpcode) == 4 and sOpcode.startswith('11/') and sOpcode[-1].isdigit(): + return (int(sOpcode[-1:]) << 3) | 0xc0; + + # The !11/r form (returns mod=1): + ## @todo this doesn't really work... + if len(sOpcode) == 5 and sOpcode.startswith('!11/') and sOpcode[-1].isdigit(): + return (int(sOpcode[-1:]) << 3) | 0x80; + + raise Exception('unsupported opcode byte spec "%s" for %s' % (sOpcode, self,)); + + @staticmethod + def _flagsToIntegerMask(asFlags): + """ + Returns the integer mask value for asFlags. + """ + uRet = 0; + if asFlags: + for sFlag in asFlags: + sConstant = g_kdEFlagsMnemonics[sFlag]; + assert sConstant[0] != '!', sConstant + uRet |= g_kdX86EFlagsConstants[sConstant]; + return uRet; + + def getTestedFlagsMask(self): + """ Returns asFlTest into a integer mask value """ + return self._flagsToIntegerMask(self.asFlTest); + + def getModifiedFlagsMask(self): + """ Returns asFlModify into a integer mask value """ + return self._flagsToIntegerMask(self.asFlModify); + + def getUndefinedFlagsMask(self): + """ Returns asFlUndefined into a integer mask value """ + return self._flagsToIntegerMask(self.asFlUndefined); + + def getSetFlagsMask(self): + """ Returns asFlSet into a integer mask value """ + return self._flagsToIntegerMask(self.asFlSet); + + def getClearedFlagsMask(self): + """ Returns asFlClear into a integer mask value """ + return self._flagsToIntegerMask(self.asFlClear); + + def onlyInVexMaps(self): + """ Returns True if only in VEX maps, otherwise False. (No maps -> False) """ + if not self.aoMaps: + return False; + for oMap in self.aoMaps: + if not oMap.isVexMap(): + return False; + return True; + + + +## All the instructions. +g_aoAllInstructions = [] # type: list(Instruction) + +## All the instructions indexed by statistics name (opstat). +g_dAllInstructionsByStat = {} # type: dict(Instruction) + +## All the instructions indexed by function name (opfunction). +g_dAllInstructionsByFunction = {} # type: dict(list(Instruction)) + +## Instructions tagged by oponlytest +g_aoOnlyTestInstructions = [] # type: list(Instruction) + +## Instruction maps. +g_aoInstructionMaps = [ + InstructionMap('one', 'g_apfnOneByteMap', sSelector = 'byte'), + InstructionMap('grp1_80', asLeadOpcodes = ['0x80',], sSelector = '/r'), + InstructionMap('grp1_81', asLeadOpcodes = ['0x81',], sSelector = '/r'), + InstructionMap('grp1_82', asLeadOpcodes = ['0x82',], sSelector = '/r'), + InstructionMap('grp1_83', asLeadOpcodes = ['0x83',], sSelector = '/r'), + InstructionMap('grp1a', asLeadOpcodes = ['0x8f',], sSelector = '/r'), + InstructionMap('grp2_c0', asLeadOpcodes = ['0xc0',], sSelector = '/r'), + InstructionMap('grp2_c1', asLeadOpcodes = ['0xc1',], sSelector = '/r'), + InstructionMap('grp2_d0', asLeadOpcodes = ['0xd0',], sSelector = '/r'), + InstructionMap('grp2_d1', asLeadOpcodes = ['0xd1',], sSelector = '/r'), + InstructionMap('grp2_d2', asLeadOpcodes = ['0xd2',], sSelector = '/r'), + InstructionMap('grp2_d3', asLeadOpcodes = ['0xd3',], sSelector = '/r'), + ## @todo g_apfnEscF1_E0toFF + InstructionMap('grp3_f6', asLeadOpcodes = ['0xf6',], sSelector = '/r'), + InstructionMap('grp3_f7', asLeadOpcodes = ['0xf7',], sSelector = '/r'), + InstructionMap('grp4', asLeadOpcodes = ['0xfe',], sSelector = '/r'), + InstructionMap('grp5', asLeadOpcodes = ['0xff',], sSelector = '/r'), + InstructionMap('grp11_c6_m', asLeadOpcodes = ['0xc6',], sSelector = '!11 /r'), + InstructionMap('grp11_c6_r', asLeadOpcodes = ['0xc6',], sSelector = '11'), # xabort + InstructionMap('grp11_c7_m', asLeadOpcodes = ['0xc7',], sSelector = '!11 /r'), + InstructionMap('grp11_c7_r', asLeadOpcodes = ['0xc7',], sSelector = '11'), # xbegin + + InstructionMap('two0f', 'g_apfnTwoByteMap', asLeadOpcodes = ['0x0f',], sDisParse = 'IDX_ParseTwoByteEsc'), + InstructionMap('grp6', 'g_apfnGroup6', asLeadOpcodes = ['0x0f', '0x00',], sSelector = '/r'), + InstructionMap('grp7_m', 'g_apfnGroup7Mem', asLeadOpcodes = ['0x0f', '0x01',], sSelector = '!11 /r'), + InstructionMap('grp7_r', asLeadOpcodes = ['0x0f', '0x01',], sSelector = '11'), + InstructionMap('grp8', asLeadOpcodes = ['0x0f', '0xba',], sSelector = '/r'), + InstructionMap('grp9', 'g_apfnGroup9RegReg', asLeadOpcodes = ['0x0f', '0xc7',], sSelector = 'mod /r'), + ## @todo What about g_apfnGroup9MemReg? + InstructionMap('grp10', None, asLeadOpcodes = ['0x0f', '0xb9',], sSelector = '/r'), # UD1 /w modr/m + InstructionMap('grp12', 'g_apfnGroup12RegReg', asLeadOpcodes = ['0x0f', '0x71',], sSelector = 'mod /r'), + InstructionMap('grp13', 'g_apfnGroup13RegReg', asLeadOpcodes = ['0x0f', '0x72',], sSelector = 'mod /r'), + InstructionMap('grp14', 'g_apfnGroup14RegReg', asLeadOpcodes = ['0x0f', '0x73',], sSelector = 'mod /r'), + InstructionMap('grp15', 'g_apfnGroup15MemReg', asLeadOpcodes = ['0x0f', '0xae',], sSelector = 'memreg /r'), + ## @todo What about g_apfnGroup15RegReg? + InstructionMap('grp16', asLeadOpcodes = ['0x0f', '0x18',], sSelector = 'mod /r'), + InstructionMap('grpA17', asLeadOpcodes = ['0x0f', '0x78',], sSelector = '/r'), # AMD: EXTRQ weirdness + InstructionMap('grpP', asLeadOpcodes = ['0x0f', '0x0d',], sSelector = '/r'), # AMD: prefetch + + InstructionMap('three0f38', 'g_apfnThreeByte0f38', asLeadOpcodes = ['0x0f', '0x38',]), + InstructionMap('three0f3a', 'g_apfnThreeByte0f3a', asLeadOpcodes = ['0x0f', '0x3a',]), + + InstructionMap('vexmap1', 'g_apfnVexMap1', sEncoding = 'vex1'), + InstructionMap('vexgrp12', 'g_apfnVexGroup12RegReg', sEncoding = 'vex1', asLeadOpcodes = ['0x71',], sSelector = 'mod /r'), + InstructionMap('vexgrp13', 'g_apfnVexGroup13RegReg', sEncoding = 'vex1', asLeadOpcodes = ['0x72',], sSelector = 'mod /r'), + InstructionMap('vexgrp14', 'g_apfnVexGroup14RegReg', sEncoding = 'vex1', asLeadOpcodes = ['0x73',], sSelector = 'mod /r'), + InstructionMap('vexgrp15', 'g_apfnVexGroup15MemReg', sEncoding = 'vex1', asLeadOpcodes = ['0xae',], sSelector = 'memreg /r'), + InstructionMap('vexgrp17', 'g_apfnVexGroup17_f3', sEncoding = 'vex1', asLeadOpcodes = ['0xf3',], sSelector = '/r'), + + InstructionMap('vexmap2', 'g_apfnVexMap2', sEncoding = 'vex2'), + InstructionMap('vexmap3', 'g_apfnVexMap3', sEncoding = 'vex3'), + + InstructionMap('3dnow', asLeadOpcodes = ['0x0f', '0x0f',]), + InstructionMap('xopmap8', sEncoding = 'xop8'), + InstructionMap('xopmap9', sEncoding = 'xop9'), + InstructionMap('xopgrp1', sEncoding = 'xop9', asLeadOpcodes = ['0x01'], sSelector = '/r'), + InstructionMap('xopgrp2', sEncoding = 'xop9', asLeadOpcodes = ['0x02'], sSelector = '/r'), + InstructionMap('xopgrp3', sEncoding = 'xop9', asLeadOpcodes = ['0x12'], sSelector = '/r'), + InstructionMap('xopmap10', sEncoding = 'xop10'), + InstructionMap('xopgrp4', sEncoding = 'xop10', asLeadOpcodes = ['0x12'], sSelector = '/r'), +]; +g_dInstructionMaps = { oMap.sName: oMap for oMap in g_aoInstructionMaps }; +g_dInstructionMapsByIemName = { oMap.sIemName: oMap for oMap in g_aoInstructionMaps }; + + + +class ParserException(Exception): + """ Parser exception """ + def __init__(self, sMessage): + Exception.__init__(self, sMessage); + + +class SimpleParser(object): + """ + Parser of IEMAllInstruction*.cpp.h instruction specifications. + """ + + ## @name Parser state. + ## @{ + kiCode = 0; + kiCommentMulti = 1; + ## @} + + def __init__(self, sSrcFile, asLines, sDefaultMap): + self.sSrcFile = sSrcFile; + self.asLines = asLines; + self.iLine = 0; + self.iState = self.kiCode; + self.sComment = ''; + self.iCommentLine = 0; + self.aoCurInstrs = []; + + assert sDefaultMap in g_dInstructionMaps; + self.oDefaultMap = g_dInstructionMaps[sDefaultMap]; + + self.cTotalInstr = 0; + self.cTotalStubs = 0; + self.cTotalTagged = 0; + + self.oReMacroName = re.compile('^[A-Za-z_][A-Za-z0-9_]*$'); + self.oReMnemonic = re.compile('^[A-Za-z_][A-Za-z0-9_]*$'); + self.oReStatsName = re.compile('^[A-Za-z_][A-Za-z0-9_]*$'); + self.oReFunctionName= re.compile('^iemOp_[A-Za-z_][A-Za-z0-9_]*$'); + self.oReGroupName = re.compile('^og_[a-z0-9]+(|_[a-z0-9]+|_[a-z0-9]+_[a-z0-9]+)$'); + self.oReDisEnum = re.compile('^OP_[A-Z0-9_]+$'); + self.oReFunTable = re.compile('^(IEM_STATIC|static) +const +PFNIEMOP +g_apfn[A-Za-z0-9_]+ *\[ *\d* *\] *= *$'); + self.oReComment = re.compile('//.*?$|/\*.*?\*/'); ## Full comments. + self.fDebug = True; + + self.dTagHandlers = { + '@opbrief': self.parseTagOpBrief, + '@opdesc': self.parseTagOpDesc, + '@opmnemonic': self.parseTagOpMnemonic, + '@op1': self.parseTagOpOperandN, + '@op2': self.parseTagOpOperandN, + '@op3': self.parseTagOpOperandN, + '@op4': self.parseTagOpOperandN, + '@oppfx': self.parseTagOpPfx, + '@opmaps': self.parseTagOpMaps, + '@opcode': self.parseTagOpcode, + '@opcodesub': self.parseTagOpcodeSub, + '@openc': self.parseTagOpEnc, + '@opfltest': self.parseTagOpEFlags, + '@opflmodify': self.parseTagOpEFlags, + '@opflundef': self.parseTagOpEFlags, + '@opflset': self.parseTagOpEFlags, + '@opflclear': self.parseTagOpEFlags, + '@ophints': self.parseTagOpHints, + '@opdisenum': self.parseTagOpDisEnum, + '@opmincpu': self.parseTagOpMinCpu, + '@opcpuid': self.parseTagOpCpuId, + '@opgroup': self.parseTagOpGroup, + '@opunused': self.parseTagOpUnusedInvalid, + '@opinvalid': self.parseTagOpUnusedInvalid, + '@opinvlstyle': self.parseTagOpUnusedInvalid, + '@optest': self.parseTagOpTest, + '@optestign': self.parseTagOpTestIgnore, + '@optestignore': self.parseTagOpTestIgnore, + '@opcopytests': self.parseTagOpCopyTests, + '@oponly': self.parseTagOpOnlyTest, + '@oponlytest': self.parseTagOpOnlyTest, + '@opxcpttype': self.parseTagOpXcptType, + '@opstats': self.parseTagOpStats, + '@opfunction': self.parseTagOpFunction, + '@opdone': self.parseTagOpDone, + }; + for i in range(48): + self.dTagHandlers['@optest%u' % (i,)] = self.parseTagOpTestNum; + self.dTagHandlers['@optest[%u]' % (i,)] = self.parseTagOpTestNum; + + self.asErrors = []; + + def raiseError(self, sMessage): + """ + Raise error prefixed with the source and line number. + """ + raise ParserException("%s:%d: error: %s" % (self.sSrcFile, self.iLine, sMessage,)); + + def raiseCommentError(self, iLineInComment, sMessage): + """ + Similar to raiseError, but the line number is iLineInComment + self.iCommentLine. + """ + raise ParserException("%s:%d: error: %s" % (self.sSrcFile, self.iCommentLine + iLineInComment, sMessage,)); + + def error(self, sMessage): + """ + Adds an error. + returns False; + """ + self.asErrors.append(u'%s:%d: error: %s\n' % (self.sSrcFile, self.iLine, sMessage,)); + return False; + + def errorOnLine(self, iLine, sMessage): + """ + Adds an error. + returns False; + """ + self.asErrors.append(u'%s:%d: error: %s\n' % (self.sSrcFile, iLine, sMessage,)); + return False; + + def errorComment(self, iLineInComment, sMessage): + """ + Adds a comment error. + returns False; + """ + self.asErrors.append(u'%s:%d: error: %s\n' % (self.sSrcFile, self.iCommentLine + iLineInComment, sMessage,)); + return False; + + def printErrors(self): + """ + Print the errors to stderr. + Returns number of errors. + """ + if self.asErrors: + sys.stderr.write(u''.join(self.asErrors)); + return len(self.asErrors); + + def debug(self, sMessage): + """ + For debugging. + """ + if self.fDebug: + print('debug: %s' % (sMessage,)); + + def stripComments(self, sLine): + """ + Returns sLine with comments stripped. + + Complains if traces of incomplete multi-line comments are encountered. + """ + sLine = self.oReComment.sub(" ", sLine); + if sLine.find('/*') >= 0 or sLine.find('*/') >= 0: + self.error('Unexpected multi-line comment will not be handled correctly. Please simplify.'); + return sLine; + + def parseFunctionTable(self, sLine): + """ + Parses a PFNIEMOP table, updating/checking the @oppfx value. + + Note! Updates iLine as it consumes the whole table. + """ + + # + # Extract the table name. + # + sName = re.search(' *([a-zA-Z_0-9]+) *\[', sLine).group(1); + oMap = g_dInstructionMapsByIemName.get(sName); + if not oMap: + self.debug('No map for PFNIEMOP table: %s' % (sName,)); + oMap = self.oDefaultMap; # This is wrong wrong wrong. + + # + # All but the g_apfnOneByteMap & g_apfnEscF1_E0toFF tables uses four + # entries per byte: + # no prefix, 066h prefix, f3h prefix, f2h prefix + # Those tables has 256 & 32 entries respectively. + # + cEntriesPerByte = 4; + cValidTableLength = 1024; + asPrefixes = ('none', '0x66', '0xf3', '0xf2'); + + oEntriesMatch = re.search('\[ *(256|32) *\]', sLine); + if oEntriesMatch: + cEntriesPerByte = 1; + cValidTableLength = int(oEntriesMatch.group(1)); + asPrefixes = (None,); + + # + # The next line should be '{' and nothing else. + # + if self.iLine >= len(self.asLines) or not re.match('^ *{ *$', self.asLines[self.iLine]): + return self.errorOnLine(self.iLine + 1, 'Expected lone "{" on line following PFNIEMOP table %s start' % (sName, )); + self.iLine += 1; + + # + # Parse till we find the end of the table. + # + iEntry = 0; + while self.iLine < len(self.asLines): + # Get the next line and strip comments and spaces (assumes no + # multi-line comments). + sLine = self.asLines[self.iLine]; + self.iLine += 1; + sLine = self.stripComments(sLine).strip(); + + # Split the line up into entries, expanding IEMOP_X4 usage. + asEntries = sLine.split(','); + for i in range(len(asEntries) - 1, -1, -1): + sEntry = asEntries[i].strip(); + if sEntry.startswith('IEMOP_X4(') and sEntry[-1] == ')': + sEntry = (sEntry[len('IEMOP_X4('):-1]).strip(); + asEntries.insert(i + 1, sEntry); + asEntries.insert(i + 1, sEntry); + asEntries.insert(i + 1, sEntry); + if sEntry: + asEntries[i] = sEntry; + else: + del asEntries[i]; + + # Process the entries. + for sEntry in asEntries: + if sEntry in ('};', '}'): + if iEntry != cValidTableLength: + return self.error('Wrong table length for %s: %#x, expected %#x' % (sName, iEntry, cValidTableLength, )); + return True; + if sEntry.startswith('iemOp_Invalid'): + pass; # skip + else: + # Look up matching instruction by function. + sPrefix = asPrefixes[iEntry % cEntriesPerByte]; + sOpcode = '%#04x' % (iEntry // cEntriesPerByte); + aoInstr = g_dAllInstructionsByFunction.get(sEntry); + if aoInstr: + if not isinstance(aoInstr, list): + aoInstr = [aoInstr,]; + oInstr = None; + for oCurInstr in aoInstr: + if oCurInstr.sOpcode == sOpcode and oCurInstr.sPrefix == sPrefix: + pass; + elif oCurInstr.sOpcode == sOpcode and oCurInstr.sPrefix is None: + oCurInstr.sPrefix = sPrefix; + elif oCurInstr.sOpcode is None and oCurInstr.sPrefix is None: + oCurInstr.sOpcode = sOpcode; + oCurInstr.sPrefix = sPrefix; + else: + continue; + oInstr = oCurInstr; + break; + if not oInstr: + oInstr = aoInstr[0].copy(oMap = oMap, sOpcode = sOpcode, sPrefix = sPrefix); + aoInstr.append(oInstr); + g_dAllInstructionsByFunction[sEntry] = aoInstr; + g_aoAllInstructions.append(oInstr); + oMap.aoInstructions.append(oInstr); + else: + self.debug('Function "%s", entry %#04x / byte %#04x in %s, is not associated with an instruction.' + % (sEntry, iEntry, iEntry // cEntriesPerByte, sName,)); + iEntry += 1; + + return self.error('Unexpected end of file in PFNIEMOP table'); + + def addInstruction(self, iLine = None): + """ + Adds an instruction. + """ + oInstr = Instruction(self.sSrcFile, self.iLine if iLine is None else iLine); + g_aoAllInstructions.append(oInstr); + self.aoCurInstrs.append(oInstr); + return oInstr; + + def deriveMnemonicAndOperandsFromStats(self, oInstr, sStats): + """ + Derives the mnemonic and operands from a IEM stats base name like string. + """ + if oInstr.sMnemonic is None: + asWords = sStats.split('_'); + oInstr.sMnemonic = asWords[0].lower(); + if len(asWords) > 1 and not oInstr.aoOperands: + for sType in asWords[1:]: + if sType in g_kdOpTypes: + oInstr.aoOperands.append(Operand(g_kdOpTypes[sType][1], sType)); + else: + #return self.error('unknown operand type: %s (instruction: %s)' % (sType, oInstr)) + return False; + return True; + + def doneInstructionOne(self, oInstr, iLine): + """ + Complete the parsing by processing, validating and expanding raw inputs. + """ + assert oInstr.iLineCompleted is None; + oInstr.iLineCompleted = iLine; + + # + # Specified instructions. + # + if oInstr.cOpTags > 0: + if oInstr.sStats is None: + pass; + + # + # Unspecified legacy stuff. We generally only got a few things to go on here. + # /** Opcode 0x0f 0x00 /0. */ + # FNIEMOPRM_DEF(iemOp_Grp6_sldt) + # + else: + #if oInstr.sRawOldOpcodes: + # + #if oInstr.sMnemonic: + pass; + + # + # Common defaults. + # + + # Guess mnemonic and operands from stats if the former is missing. + if oInstr.sMnemonic is None: + if oInstr.sStats is not None: + self.deriveMnemonicAndOperandsFromStats(oInstr, oInstr.sStats); + elif oInstr.sFunction is not None: + self.deriveMnemonicAndOperandsFromStats(oInstr, oInstr.sFunction.replace('iemOp_', '')); + + # Derive the disassembler op enum constant from the mnemonic. + if oInstr.sDisEnum is None and oInstr.sMnemonic is not None: + oInstr.sDisEnum = 'OP_' + oInstr.sMnemonic.upper(); + + # Derive the IEM statistics base name from mnemonic and operand types. + if oInstr.sStats is None: + if oInstr.sFunction is not None: + oInstr.sStats = oInstr.sFunction.replace('iemOp_', ''); + elif oInstr.sMnemonic is not None: + oInstr.sStats = oInstr.sMnemonic; + for oOperand in oInstr.aoOperands: + if oOperand.sType: + oInstr.sStats += '_' + oOperand.sType; + + # Derive the IEM function name from mnemonic and operand types. + if oInstr.sFunction is None: + if oInstr.sMnemonic is not None: + oInstr.sFunction = 'iemOp_' + oInstr.sMnemonic; + for oOperand in oInstr.aoOperands: + if oOperand.sType: + oInstr.sFunction += '_' + oOperand.sType; + elif oInstr.sStats: + oInstr.sFunction = 'iemOp_' + oInstr.sStats; + + # + # Apply default map and then add the instruction to all it's groups. + # + if not oInstr.aoMaps: + oInstr.aoMaps = [ self.oDefaultMap, ]; + for oMap in oInstr.aoMaps: + oMap.aoInstructions.append(oInstr); + + # + # Derive encoding from operands and maps. + # + if oInstr.sEncoding is None: + if not oInstr.aoOperands: + if oInstr.fUnused and oInstr.sSubOpcode: + oInstr.sEncoding = 'VEX.ModR/M' if oInstr.onlyInVexMaps() else 'ModR/M'; + else: + oInstr.sEncoding = 'VEX.fixed' if oInstr.onlyInVexMaps() else 'fixed'; + elif oInstr.aoOperands[0].usesModRM(): + if (len(oInstr.aoOperands) >= 2 and oInstr.aoOperands[1].sWhere == 'vvvv') \ + or oInstr.onlyInVexMaps(): + oInstr.sEncoding = 'VEX.ModR/M'; + else: + oInstr.sEncoding = 'ModR/M'; + + # + # Check the opstat value and add it to the opstat indexed dictionary. + # + if oInstr.sStats: + if oInstr.sStats not in g_dAllInstructionsByStat: + g_dAllInstructionsByStat[oInstr.sStats] = oInstr; + else: + self.error('Duplicate opstat value "%s"\nnew: %s\nold: %s' + % (oInstr.sStats, oInstr, g_dAllInstructionsByStat[oInstr.sStats],)); + + # + # Add to function indexed dictionary. We allow multiple instructions per function. + # + if oInstr.sFunction: + if oInstr.sFunction not in g_dAllInstructionsByFunction: + g_dAllInstructionsByFunction[oInstr.sFunction] = [oInstr,]; + else: + g_dAllInstructionsByFunction[oInstr.sFunction].append(oInstr); + + #self.debug('%d..%d: %s; %d @op tags' % (oInstr.iLineCreated, oInstr.iLineCompleted, oInstr.sFunction, oInstr.cOpTags)); + return True; + + def doneInstructions(self, iLineInComment = None): + """ + Done with current instruction. + """ + for oInstr in self.aoCurInstrs: + self.doneInstructionOne(oInstr, self.iLine if iLineInComment is None else self.iCommentLine + iLineInComment); + if oInstr.fStub: + self.cTotalStubs += 1; + + self.cTotalInstr += len(self.aoCurInstrs); + + self.sComment = ''; + self.aoCurInstrs = []; + return True; + + def setInstrunctionAttrib(self, sAttrib, oValue, fOverwrite = False): + """ + Sets the sAttrib of all current instruction to oValue. If fOverwrite + is False, only None values and empty strings are replaced. + """ + for oInstr in self.aoCurInstrs: + if fOverwrite is not True: + oOldValue = getattr(oInstr, sAttrib); + if oOldValue is not None: + continue; + setattr(oInstr, sAttrib, oValue); + + def setInstrunctionArrayAttrib(self, sAttrib, iEntry, oValue, fOverwrite = False): + """ + Sets the iEntry of the array sAttrib of all current instruction to oValue. + If fOverwrite is False, only None values and empty strings are replaced. + """ + for oInstr in self.aoCurInstrs: + aoArray = getattr(oInstr, sAttrib); + while len(aoArray) <= iEntry: + aoArray.append(None); + if fOverwrite is True or aoArray[iEntry] is None: + aoArray[iEntry] = oValue; + + def parseCommentOldOpcode(self, asLines): + """ Deals with 'Opcode 0xff /4' like comments """ + asWords = asLines[0].split(); + if len(asWords) >= 2 \ + and asWords[0] == 'Opcode' \ + and ( asWords[1].startswith('0x') + or asWords[1].startswith('0X')): + asWords = asWords[:1]; + for iWord, sWord in enumerate(asWords): + if sWord.startswith('0X'): + sWord = '0x' + sWord[:2]; + asWords[iWord] = asWords; + self.setInstrunctionAttrib('sRawOldOpcodes', ' '.join(asWords)); + + return False; + + def ensureInstructionForOpTag(self, iTagLine): + """ Ensure there is an instruction for the op-tag being parsed. """ + if not self.aoCurInstrs: + self.addInstruction(self.iCommentLine + iTagLine); + for oInstr in self.aoCurInstrs: + oInstr.cOpTags += 1; + if oInstr.cOpTags == 1: + self.cTotalTagged += 1; + return self.aoCurInstrs[-1]; + + @staticmethod + def flattenSections(aasSections): + """ + Flattens multiline sections into stripped single strings. + Returns list of strings, on section per string. + """ + asRet = []; + for asLines in aasSections: + if asLines: + asRet.append(' '.join([sLine.strip() for sLine in asLines])); + return asRet; + + @staticmethod + def flattenAllSections(aasSections, sLineSep = ' ', sSectionSep = '\n'): + """ + Flattens sections into a simple stripped string with newlines as + section breaks. The final section does not sport a trailing newline. + """ + # Typical: One section with a single line. + if len(aasSections) == 1 and len(aasSections[0]) == 1: + return aasSections[0][0].strip(); + + sRet = ''; + for iSection, asLines in enumerate(aasSections): + if asLines: + if iSection > 0: + sRet += sSectionSep; + sRet += sLineSep.join([sLine.strip() for sLine in asLines]); + return sRet; + + + + ## @name Tag parsers + ## @{ + + def parseTagOpBrief(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opbrief + Value: Text description, multiple sections, appended. + + Brief description. If not given, it's the first sentence from @opdesc. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten and validate the value. + sBrief = self.flattenAllSections(aasSections); + if not sBrief: + return self.errorComment(iTagLine, '%s: value required' % (sTag,)); + if sBrief[-1] != '.': + sBrief = sBrief + '.'; + if len(sBrief) > 180: + return self.errorComment(iTagLine, '%s: value too long (max 180 chars): %s' % (sTag, sBrief)); + offDot = sBrief.find('.'); + while 0 <= offDot < len(sBrief) - 1 and sBrief[offDot + 1] != ' ': + offDot = sBrief.find('.', offDot + 1); + if offDot >= 0 and offDot != len(sBrief) - 1: + return self.errorComment(iTagLine, '%s: only one sentence: %s' % (sTag, sBrief)); + + # Update the instruction. + if oInstr.sBrief is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite brief "%s" with "%s"' + % (sTag, oInstr.sBrief, sBrief,)); + _ = iEndLine; + return True; + + def parseTagOpDesc(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opdesc + Value: Text description, multiple sections, appended. + + It is used to describe instructions. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + if aasSections: + oInstr.asDescSections.extend(self.flattenSections(aasSections)); + return True; + + _ = sTag; _ = iEndLine; + return True; + + def parseTagOpMnemonic(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: @opmenmonic + Value: mnemonic + + The 'mnemonic' value must be a valid C identifier string. Because of + prefixes, groups and whatnot, there times when the mnemonic isn't that + of an actual assembler mnemonic. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten and validate the value. + sMnemonic = self.flattenAllSections(aasSections); + if not self.oReMnemonic.match(sMnemonic): + return self.errorComment(iTagLine, '%s: invalid menmonic name: "%s"' % (sTag, sMnemonic,)); + if oInstr.sMnemonic is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite menmonic "%s" with "%s"' + % (sTag, oInstr.sMnemonic, sMnemonic,)); + oInstr.sMnemonic = sMnemonic + + _ = iEndLine; + return True; + + def parseTagOpOperandN(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tags: \@op1, \@op2, \@op3, \@op4 + Value: [where:]type + + The 'where' value indicates where the operand is found, like the 'reg' + part of the ModR/M encoding. See Instruction.kdOperandLocations for + a list. + + The 'type' value indicates the operand type. These follow the types + given in the opcode tables in the CPU reference manuals. + See Instruction.kdOperandTypes for a list. + + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + idxOp = int(sTag[-1]) - 1; + assert 0 <= idxOp < 4; + + # flatten, split up, and validate the "where:type" value. + sFlattened = self.flattenAllSections(aasSections); + asSplit = sFlattened.split(':'); + if len(asSplit) == 1: + sType = asSplit[0]; + sWhere = None; + elif len(asSplit) == 2: + (sWhere, sType) = asSplit; + else: + return self.errorComment(iTagLine, 'expected %s value on format "[:]" not "%s"' % (sTag, sFlattened,)); + + if sType not in g_kdOpTypes: + return self.errorComment(iTagLine, '%s: invalid where value "%s", valid: %s' + % (sTag, sType, ', '.join(g_kdOpTypes.keys()),)); + if sWhere is None: + sWhere = g_kdOpTypes[sType][1]; + elif sWhere not in g_kdOpLocations: + return self.errorComment(iTagLine, '%s: invalid where value "%s", valid: %s' + % (sTag, sWhere, ', '.join(g_kdOpLocations.keys()),)); + + # Insert the operand, refusing to overwrite an existing one. + while idxOp >= len(oInstr.aoOperands): + oInstr.aoOperands.append(None); + if oInstr.aoOperands[idxOp] is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite "%s:%s" with "%s:%s"' + % ( sTag, oInstr.aoOperands[idxOp].sWhere, oInstr.aoOperands[idxOp].sType, + sWhere, sType,)); + oInstr.aoOperands[idxOp] = Operand(sWhere, sType); + + _ = iEndLine; + return True; + + def parseTagOpMaps(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opmaps + Value: map[,map2] + + Indicates which maps the instruction is in. There is a default map + associated with each input file. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten, split up and validate the value. + sFlattened = self.flattenAllSections(aasSections, sLineSep = ',', sSectionSep = ','); + asMaps = sFlattened.split(','); + if not asMaps: + return self.errorComment(iTagLine, '%s: value required' % (sTag,)); + for sMap in asMaps: + if sMap not in g_dInstructionMaps: + return self.errorComment(iTagLine, '%s: invalid map value: %s (valid values: %s)' + % (sTag, sMap, ', '.join(g_dInstructionMaps.keys()),)); + + # Add the maps to the current list. Throw errors on duplicates. + for oMap in oInstr.aoMaps: + if oMap.sName in asMaps: + return self.errorComment(iTagLine, '%s: duplicate map assignment: %s' % (sTag, oMap.sName)); + + for sMap in asMaps: + oMap = g_dInstructionMaps[sMap]; + if oMap not in oInstr.aoMaps: + oInstr.aoMaps.append(oMap); + else: + self.errorComment(iTagLine, '%s: duplicate map assignment (input): %s' % (sTag, sMap)); + + _ = iEndLine; + return True; + + def parseTagOpPfx(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@oppfx + Value: n/a|none|0x66|0xf3|0xf2 + + Required prefix for the instruction. (In a (E)VEX context this is the + value of the 'pp' field rather than an actual prefix.) + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten and validate the value. + sFlattened = self.flattenAllSections(aasSections); + asPrefixes = sFlattened.split(); + if len(asPrefixes) > 1: + return self.errorComment(iTagLine, '%s: max one prefix: %s' % (sTag, asPrefixes,)); + + sPrefix = asPrefixes[0].lower(); + if sPrefix == 'none': + sPrefix = 'none'; + elif sPrefix == 'n/a': + sPrefix = None; + else: + if len(sPrefix) == 2: + sPrefix = '0x' + sPrefix; + if not _isValidOpcodeByte(sPrefix): + return self.errorComment(iTagLine, '%s: invalid prefix: %s' % (sTag, sPrefix,)); + + if sPrefix is not None and sPrefix not in g_kdPrefixes: + return self.errorComment(iTagLine, '%s: invalid prefix: %s (valid %s)' % (sTag, sPrefix, g_kdPrefixes,)); + + # Set it. + if oInstr.sPrefix is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, oInstr.sPrefix, sPrefix,)); + oInstr.sPrefix = sPrefix; + + _ = iEndLine; + return True; + + def parseTagOpcode(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opcode + Value: 0x?? | /reg (TODO: | mr/reg | 11 /reg | !11 /reg | 11 mr/reg | !11 mr/reg) + + The opcode byte or sub-byte for the instruction in the context of a map. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten and validate the value. + sOpcode = self.flattenAllSections(aasSections); + if _isValidOpcodeByte(sOpcode): + pass; + elif len(sOpcode) == 2 and sOpcode.startswith('/') and sOpcode[-1] in '012345678': + pass; + elif len(sOpcode) == 4 and sOpcode.startswith('11/') and sOpcode[-1] in '012345678': + pass; + elif len(sOpcode) == 5 and sOpcode.startswith('!11/') and sOpcode[-1] in '012345678': + pass; + else: + return self.errorComment(iTagLine, '%s: invalid opcode: %s' % (sTag, sOpcode,)); + + # Set it. + if oInstr.sOpcode is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, oInstr.sOpcode, sOpcode,)); + oInstr.sOpcode = sOpcode; + + _ = iEndLine; + return True; + + def parseTagOpcodeSub(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opcodesub + Value: none | 11 mr/reg | !11 mr/reg | rex.w=0 | rex.w=1 | vex.l=0 | vex.l=1 + | 11 mr/reg vex.l=0 | 11 mr/reg vex.l=1 | !11 mr/reg vex.l=0 | !11 mr/reg vex.l=1 + + This is a simple way of dealing with encodings where the mod=3 and mod!=3 + represents exactly two different instructions. The more proper way would + be to go via maps with two members, but this is faster. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten and validate the value. + sSubOpcode = self.flattenAllSections(aasSections); + if sSubOpcode not in g_kdSubOpcodes: + return self.errorComment(iTagLine, '%s: invalid sub opcode: %s (valid: 11, !11, none)' % (sTag, sSubOpcode,)); + sSubOpcode = g_kdSubOpcodes[sSubOpcode][0]; + + # Set it. + if oInstr.sSubOpcode is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' + % ( sTag, oInstr.sSubOpcode, sSubOpcode,)); + oInstr.sSubOpcode = sSubOpcode; + + _ = iEndLine; + return True; + + def parseTagOpEnc(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@openc + Value: ModR/M|fixed|prefix| + + The instruction operand encoding style. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten and validate the value. + sEncoding = self.flattenAllSections(aasSections); + if sEncoding in g_kdEncodings: + pass; + elif sEncoding in g_dInstructionMaps: + pass; + elif not _isValidOpcodeByte(sEncoding): + return self.errorComment(iTagLine, '%s: invalid encoding: %s' % (sTag, sEncoding,)); + + # Set it. + if oInstr.sEncoding is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' + % ( sTag, oInstr.sEncoding, sEncoding,)); + oInstr.sEncoding = sEncoding; + + _ = iEndLine; + return True; + + ## EFlags tag to Instruction attribute name. + kdOpFlagToAttr = { + '@opfltest': 'asFlTest', + '@opflmodify': 'asFlModify', + '@opflundef': 'asFlUndefined', + '@opflset': 'asFlSet', + '@opflclear': 'asFlClear', + }; + + def parseTagOpEFlags(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tags: \@opfltest, \@opflmodify, \@opflundef, \@opflset, \@opflclear + Value: + + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten, split up and validate the values. + asFlags = self.flattenAllSections(aasSections, sLineSep = ',', sSectionSep = ',').split(','); + if len(asFlags) == 1 and asFlags[0].lower() == 'none': + asFlags = []; + else: + fRc = True; + for iFlag, sFlag in enumerate(asFlags): + if sFlag not in g_kdEFlagsMnemonics: + if sFlag.strip() in g_kdEFlagsMnemonics: + asFlags[iFlag] = sFlag.strip(); + else: + fRc = self.errorComment(iTagLine, '%s: invalid EFLAGS value: %s' % (sTag, sFlag,)); + if not fRc: + return False; + + # Set them. + asOld = getattr(oInstr, self.kdOpFlagToAttr[sTag]); + if asOld is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, asOld, asFlags,)); + setattr(oInstr, self.kdOpFlagToAttr[sTag], asFlags); + + _ = iEndLine; + return True; + + def parseTagOpHints(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@ophints + Value: Comma or space separated list of flags and hints. + + This covers the disassembler flags table and more. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten as a space separated list, split it up and validate the values. + asHints = self.flattenAllSections(aasSections, sLineSep = ' ', sSectionSep = ' ').replace(',', ' ').split(); + if len(asHints) == 1 and asHints[0].lower() == 'none': + asHints = []; + else: + fRc = True; + for iHint, sHint in enumerate(asHints): + if sHint not in g_kdHints: + if sHint.strip() in g_kdHints: + sHint[iHint] = sHint.strip(); + else: + fRc = self.errorComment(iTagLine, '%s: invalid hint value: %s' % (sTag, sHint,)); + if not fRc: + return False; + + # Append them. + for sHint in asHints: + if sHint not in oInstr.dHints: + oInstr.dHints[sHint] = True; # (dummy value, using dictionary for speed) + else: + self.errorComment(iTagLine, '%s: duplicate hint: %s' % ( sTag, sHint,)); + + _ = iEndLine; + return True; + + def parseTagOpDisEnum(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opdisenum + Value: OP_XXXX + + This is for select a specific (legacy) disassembler enum value for the + instruction. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten and split. + asWords = self.flattenAllSections(aasSections).split(); + if len(asWords) != 1: + self.errorComment(iTagLine, '%s: expected exactly one value: %s' % (sTag, asWords,)); + if not asWords: + return False; + sDisEnum = asWords[0]; + if not self.oReDisEnum.match(sDisEnum): + return self.errorComment(iTagLine, '%s: invalid disassembler OP_XXXX enum: %s (pattern: %s)' + % (sTag, sDisEnum, self.oReDisEnum.pattern)); + + # Set it. + if oInstr.sDisEnum is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % (sTag, oInstr.sDisEnum, sDisEnum,)); + oInstr.sDisEnum = sDisEnum; + + _ = iEndLine; + return True; + + def parseTagOpMinCpu(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opmincpu + Value: + + Indicates when this instruction was introduced. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten the value, split into words, make sure there's just one, valid it. + asCpus = self.flattenAllSections(aasSections).split(); + if len(asCpus) > 1: + self.errorComment(iTagLine, '%s: exactly one CPU name, please: %s' % (sTag, ' '.join(asCpus),)); + + sMinCpu = asCpus[0]; + if sMinCpu in g_kdCpuNames: + oInstr.sMinCpu = sMinCpu; + else: + return self.errorComment(iTagLine, '%s: invalid CPU name: %s (names: %s)' + % (sTag, sMinCpu, ','.join(sorted(g_kdCpuNames)),)); + + # Set it. + if oInstr.sMinCpu is None: + oInstr.sMinCpu = sMinCpu; + elif oInstr.sMinCpu != sMinCpu: + self.errorComment(iTagLine, '%s: attemting to overwrite "%s" with "%s"' % (sTag, oInstr.sMinCpu, sMinCpu,)); + + _ = iEndLine; + return True; + + def parseTagOpCpuId(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opcpuid + Value: none | + + CPUID feature bit which is required for the instruction to be present. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten as a space separated list, split it up and validate the values. + asCpuIds = self.flattenAllSections(aasSections, sLineSep = ' ', sSectionSep = ' ').replace(',', ' ').split(); + if len(asCpuIds) == 1 and asCpuIds[0].lower() == 'none': + asCpuIds = []; + else: + fRc = True; + for iCpuId, sCpuId in enumerate(asCpuIds): + if sCpuId not in g_kdCpuIdFlags: + if sCpuId.strip() in g_kdCpuIdFlags: + sCpuId[iCpuId] = sCpuId.strip(); + else: + fRc = self.errorComment(iTagLine, '%s: invalid CPUID value: %s' % (sTag, sCpuId,)); + if not fRc: + return False; + + # Append them. + for sCpuId in asCpuIds: + if sCpuId not in oInstr.asCpuIds: + oInstr.asCpuIds.append(sCpuId); + else: + self.errorComment(iTagLine, '%s: duplicate CPUID: %s' % ( sTag, sCpuId,)); + + _ = iEndLine; + return True; + + def parseTagOpGroup(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opgroup + Value: op_grp1[_subgrp2[_subsubgrp3]] + + Instruction grouping. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten as a space separated list, split it up and validate the values. + asGroups = self.flattenAllSections(aasSections).split(); + if len(asGroups) != 1: + return self.errorComment(iTagLine, '%s: exactly one group, please: %s' % (sTag, asGroups,)); + sGroup = asGroups[0]; + if not self.oReGroupName.match(sGroup): + return self.errorComment(iTagLine, '%s: invalid group name: %s (valid: %s)' + % (sTag, sGroup, self.oReGroupName.pattern)); + + # Set it. + if oInstr.sGroup is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, oInstr.sGroup, sGroup,)); + oInstr.sGroup = sGroup; + + _ = iEndLine; + return True; + + def parseTagOpUnusedInvalid(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opunused, \@opinvalid, \@opinvlstyle + Value: + + The \@opunused indicates the specification is for a currently unused + instruction encoding. + + The \@opinvalid indicates the specification is for an invalid currently + instruction encoding (like UD2). + + The \@opinvlstyle just indicates how CPUs decode the instruction when + not supported (\@opcpuid, \@opmincpu) or disabled. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten as a space separated list, split it up and validate the values. + asStyles = self.flattenAllSections(aasSections).split(); + if len(asStyles) != 1: + return self.errorComment(iTagLine, '%s: exactly one invalid behviour style, please: %s' % (sTag, asStyles,)); + sStyle = asStyles[0]; + if sStyle not in g_kdInvalidStyles: + return self.errorComment(iTagLine, '%s: invalid invalid behaviour style: %s (valid: %s)' + % (sTag, sStyle, g_kdInvalidStyles.keys(),)); + # Set it. + if oInstr.sInvalidStyle is not None: + return self.errorComment(iTagLine, + '%s: attempting to overwrite "%s" with "%s" (only one @opunused, @opinvalid, @opinvlstyle)' + % ( sTag, oInstr.sInvalidStyle, sStyle,)); + oInstr.sInvalidStyle = sStyle; + if sTag == '@opunused': + oInstr.fUnused = True; + elif sTag == '@opinvalid': + oInstr.fInvalid = True; + + _ = iEndLine; + return True; + + def parseTagOpTest(self, sTag, aasSections, iTagLine, iEndLine): # pylint: disable=too-many-locals + """ + Tag: \@optest + Value: [[ ]?] -> + Example: mode==64bit / in1=0xfffffffe:dw in2=1:dw -> out1=0xffffffff:dw outfl=a?,p? + + The main idea here is to generate basic instruction tests. + + The probably simplest way of handling the diverse input, would be to use + it to produce size optimized byte code for a simple interpreter that + modifies the register input and output states. + + An alternative to the interpreter would be creating multiple tables, + but that becomes rather complicated wrt what goes where and then to use + them in an efficient manner. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # + # Do it section by section. + # + for asSectionLines in aasSections: + # + # Sort the input into outputs, inputs and selector conditions. + # + sFlatSection = self.flattenAllSections([asSectionLines,]); + if not sFlatSection: + self.errorComment(iTagLine, '%s: missing value (dbg: aasSections=%s)' % ( sTag, aasSections)); + continue; + oTest = InstructionTest(oInstr); + + asSelectors = []; + asInputs = []; + asOutputs = []; + asCur = asOutputs; + fRc = True; + asWords = sFlatSection.split(); + for iWord in range(len(asWords) - 1, -1, -1): + sWord = asWords[iWord]; + # Check for array switchers. + if sWord == '->': + if asCur != asOutputs: + fRc = self.errorComment(iTagLine, '%s: "->" shall only occure once: %s' % (sTag, sFlatSection,)); + break; + asCur = asInputs; + elif sWord == '/': + if asCur != asInputs: + fRc = self.errorComment(iTagLine, '%s: "/" shall only occure once: %s' % (sTag, sFlatSection,)); + break; + asCur = asSelectors; + else: + asCur.insert(0, sWord); + + # + # Validate and add selectors. + # + for sCond in asSelectors: + sCondExp = TestSelector.kdPredicates.get(sCond, sCond); + oSelector = None; + for sOp in TestSelector.kasCompareOps: + off = sCondExp.find(sOp); + if off >= 0: + sVariable = sCondExp[:off]; + sValue = sCondExp[off + len(sOp):]; + if sVariable in TestSelector.kdVariables: + if sValue in TestSelector.kdVariables[sVariable]: + oSelector = TestSelector(sVariable, sOp, sValue); + else: + self.errorComment(iTagLine, '%s: invalid condition value "%s" in "%s" (valid: %s)' + % ( sTag, sValue, sCond, + TestSelector.kdVariables[sVariable].keys(),)); + else: + self.errorComment(iTagLine, '%s: invalid condition variable "%s" in "%s" (valid: %s)' + % ( sTag, sVariable, sCond, TestSelector.kdVariables.keys(),)); + break; + if oSelector is not None: + for oExisting in oTest.aoSelectors: + if oExisting.sVariable == oSelector.sVariable: + self.errorComment(iTagLine, '%s: already have a selector for variable "%s" (existing: %s, new: %s)' + % ( sTag, oSelector.sVariable, oExisting, oSelector,)); + oTest.aoSelectors.append(oSelector); + else: + fRc = self.errorComment(iTagLine, '%s: failed to parse selector: %s' % ( sTag, sCond,)); + + # + # Validate outputs and inputs, adding them to the test as we go along. + # + for asItems, sDesc, aoDst in [ (asInputs, 'input', oTest.aoInputs), (asOutputs, 'output', oTest.aoOutputs)]: + asValidFieldKinds = [ 'both', sDesc, ]; + for sItem in asItems: + oItem = None; + for sOp in TestInOut.kasOperators: + off = sItem.find(sOp); + if off < 0: + continue; + sField = sItem[:off]; + sValueType = sItem[off + len(sOp):]; + if sField in TestInOut.kdFields \ + and TestInOut.kdFields[sField][1] in asValidFieldKinds: + asSplit = sValueType.split(':', 1); + sValue = asSplit[0]; + sType = asSplit[1] if len(asSplit) > 1 else TestInOut.kdFields[sField][0]; + if sType in TestInOut.kdTypes: + oValid = TestInOut.kdTypes[sType].validate(sValue); + if oValid is True: + if not TestInOut.kdTypes[sType].isAndOrPair(sValue) or sOp == '&|=': + oItem = TestInOut(sField, sOp, sValue, sType); + else: + self.errorComment(iTagLine, '%s: and-or %s value "%s" can only be used with "&|="' + % ( sTag, sDesc, sItem, )); + else: + self.errorComment(iTagLine, '%s: invalid %s value "%s" in "%s" (type: %s): %s' + % ( sTag, sDesc, sValue, sItem, sType, oValid, )); + else: + self.errorComment(iTagLine, '%s: invalid %s type "%s" in "%s" (valid types: %s)' + % ( sTag, sDesc, sType, sItem, TestInOut.kdTypes.keys(),)); + else: + self.errorComment(iTagLine, '%s: invalid %s field "%s" in "%s"\nvalid fields: %s' + % ( sTag, sDesc, sField, sItem, + ', '.join([sKey for sKey, asVal in TestInOut.kdFields.items() + if asVal[1] in asValidFieldKinds]),)); + break; + if oItem is not None: + for oExisting in aoDst: + if oExisting.sField == oItem.sField and oExisting.sOp == oItem.sOp: + self.errorComment(iTagLine, + '%s: already have a "%s" assignment for field "%s" (existing: %s, new: %s)' + % ( sTag, oItem.sOp, oItem.sField, oExisting, oItem,)); + aoDst.append(oItem); + else: + fRc = self.errorComment(iTagLine, '%s: failed to parse assignment: %s' % ( sTag, sItem,)); + + # + # . + # + if fRc: + oInstr.aoTests.append(oTest); + else: + self.errorComment(iTagLine, '%s: failed to parse test: %s' % (sTag, ' '.join(asWords),)); + self.errorComment(iTagLine, '%s: asSelectors=%s / asInputs=%s -> asOutputs=%s' + % (sTag, asSelectors, asInputs, asOutputs,)); + + _ = iEndLine; + return True; + + def parseTagOpTestNum(self, sTag, aasSections, iTagLine, iEndLine): + """ + Numbered \@optest tag. Either \@optest42 or \@optest[42]. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + iTest = 0; + if sTag[-1] == ']': + iTest = int(sTag[8:-1]); + else: + iTest = int(sTag[7:]); + + if iTest != len(oInstr.aoTests): + self.errorComment(iTagLine, '%s: incorrect test number: %u, actual %u' % (sTag, iTest, len(oInstr.aoTests),)); + return self.parseTagOpTest(sTag, aasSections, iTagLine, iEndLine); + + def parseTagOpTestIgnore(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@optestign | \@optestignore + Value: + + This is a simple trick to ignore a test while debugging another. + + See also \@oponlytest. + """ + _ = sTag; _ = aasSections; _ = iTagLine; _ = iEndLine; + return True; + + def parseTagOpCopyTests(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opcopytests + Value: [..] + Example: \@opcopytests add_Eb_Gb + + Trick to avoid duplicating tests for different encodings of the same + operation. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten, validate and append the copy job to the instruction. We execute + # them after parsing all the input so we can handle forward references. + asToCopy = self.flattenAllSections(aasSections).split(); + if not asToCopy: + return self.errorComment(iTagLine, '%s: requires at least on reference value' % (sTag,)); + for sToCopy in asToCopy: + if sToCopy not in oInstr.asCopyTests: + if self.oReStatsName.match(sToCopy) or self.oReFunctionName.match(sToCopy): + oInstr.asCopyTests.append(sToCopy); + else: + self.errorComment(iTagLine, '%s: invalid instruction reference (opstat or function) "%s" (valid: %s or %s)' + % (sTag, sToCopy, self.oReStatsName.pattern, self.oReFunctionName.pattern)); + else: + self.errorComment(iTagLine, '%s: ignoring duplicate "%s"' % (sTag, sToCopy,)); + + _ = iEndLine; + return True; + + def parseTagOpOnlyTest(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@oponlytest | \@oponly + Value: none + + Only test instructions with this tag. This is a trick that is handy + for singling out one or two new instructions or tests. + + See also \@optestignore. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Validate and add instruction to only test dictionary. + sValue = self.flattenAllSections(aasSections).strip(); + if sValue: + return self.errorComment(iTagLine, '%s: does not take any value: %s' % (sTag, sValue)); + + if oInstr not in g_aoOnlyTestInstructions: + g_aoOnlyTestInstructions.append(oInstr); + + _ = iEndLine; + return True; + + def parseTagOpXcptType(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opxcpttype + Value: [none|1|2|3|4|4UA|5|6|7|8|11|12|E1|E1NF|E2|E3|E3NF|E4|E4NF|E5|E5NF|E6|E6NF|E7NF|E9|E9NF|E10|E11|E12|E12NF] + + Sets the SSE or AVX exception type (see SDMv2 2.4, 2.7). + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten as a space separated list, split it up and validate the values. + asTypes = self.flattenAllSections(aasSections).split(); + if len(asTypes) != 1: + return self.errorComment(iTagLine, '%s: exactly one invalid exception type, please: %s' % (sTag, asTypes,)); + sType = asTypes[0]; + if sType not in g_kdXcptTypes: + return self.errorComment(iTagLine, '%s: invalid invalid exception type: %s (valid: %s)' + % (sTag, sType, sorted(g_kdXcptTypes.keys()),)); + # Set it. + if oInstr.sXcptType is not None: + return self.errorComment(iTagLine, + '%s: attempting to overwrite "%s" with "%s" (only one @opxcpttype)' + % ( sTag, oInstr.sXcptType, sType,)); + oInstr.sXcptType = sType; + + _ = iEndLine; + return True; + + def parseTagOpFunction(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opfunction + Value: + + This is for explicitly setting the IEM function name. Normally we pick + this up from the FNIEMOP_XXX macro invocation after the description, or + generate it from the mnemonic and operands. + + It it thought it maybe necessary to set it when specifying instructions + which implementation isn't following immediately or aren't implemented yet. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten and validate the value. + sFunction = self.flattenAllSections(aasSections); + if not self.oReFunctionName.match(sFunction): + return self.errorComment(iTagLine, '%s: invalid VMM function name: "%s" (valid: %s)' + % (sTag, sFunction, self.oReFunctionName.pattern)); + + if oInstr.sFunction is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite VMM function name "%s" with "%s"' + % (sTag, oInstr.sFunction, sFunction,)); + oInstr.sFunction = sFunction; + + _ = iEndLine; + return True; + + def parseTagOpStats(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opstats + Value: + + This is for explicitly setting the statistics name. Normally we pick + this up from the IEMOP_MNEMONIC macro invocation, or generate it from + the mnemonic and operands. + + It it thought it maybe necessary to set it when specifying instructions + which implementation isn't following immediately or aren't implemented yet. + """ + oInstr = self.ensureInstructionForOpTag(iTagLine); + + # Flatten and validate the value. + sStats = self.flattenAllSections(aasSections); + if not self.oReStatsName.match(sStats): + return self.errorComment(iTagLine, '%s: invalid VMM statistics name: "%s" (valid: %s)' + % (sTag, sStats, self.oReStatsName.pattern)); + + if oInstr.sStats is not None: + return self.errorComment(iTagLine, '%s: attempting to overwrite VMM statistics base name "%s" with "%s"' + % (sTag, oInstr.sStats, sStats,)); + oInstr.sStats = sStats; + + _ = iEndLine; + return True; + + def parseTagOpDone(self, sTag, aasSections, iTagLine, iEndLine): + """ + Tag: \@opdone + Value: none + + Used to explictily flush the instructions that have been specified. + """ + sFlattened = self.flattenAllSections(aasSections); + if sFlattened != '': + return self.errorComment(iTagLine, '%s: takes no value, found: "%s"' % (sTag, sFlattened,)); + _ = sTag; _ = iEndLine; + return self.doneInstructions(); + + ## @} + + + def parseComment(self): + """ + Parse the current comment (self.sComment). + + If it's a opcode specifiying comment, we reset the macro stuff. + """ + # + # Reject if comment doesn't seem to contain anything interesting. + # + if self.sComment.find('Opcode') < 0 \ + and self.sComment.find('@') < 0: + return False; + + # + # Split the comment into lines, removing leading asterisks and spaces. + # Also remove leading and trailing empty lines. + # + asLines = self.sComment.split('\n'); + for iLine, sLine in enumerate(asLines): + asLines[iLine] = sLine.lstrip().lstrip('*').lstrip(); + + while asLines and not asLines[0]: + self.iCommentLine += 1; + asLines.pop(0); + + while asLines and not asLines[-1]: + asLines.pop(len(asLines) - 1); + + # + # Check for old style: Opcode 0x0f 0x12 + # + if asLines[0].startswith('Opcode '): + self.parseCommentOldOpcode(asLines); + + # + # Look for @op* tagged data. + # + cOpTags = 0; + sFlatDefault = None; + sCurTag = '@default'; + iCurTagLine = 0; + asCurSection = []; + aasSections = [ asCurSection, ]; + for iLine, sLine in enumerate(asLines): + if not sLine.startswith('@'): + if sLine: + asCurSection.append(sLine); + elif asCurSection: + asCurSection = []; + aasSections.append(asCurSection); + else: + # + # Process the previous tag. + # + if not asCurSection and len(aasSections) > 1: + aasSections.pop(-1); + if sCurTag in self.dTagHandlers: + self.dTagHandlers[sCurTag](sCurTag, aasSections, iCurTagLine, iLine); + cOpTags += 1; + elif sCurTag.startswith('@op'): + self.errorComment(iCurTagLine, 'Unknown tag: %s' % (sCurTag)); + elif sCurTag == '@default': + sFlatDefault = self.flattenAllSections(aasSections); + elif '@op' + sCurTag[1:] in self.dTagHandlers: + self.errorComment(iCurTagLine, 'Did you mean "@op%s" rather than "%s"?' % (sCurTag[1:], sCurTag)); + elif sCurTag in ['@encoding', '@opencoding']: + self.errorComment(iCurTagLine, 'Did you mean "@openc" rather than "%s"?' % (sCurTag,)); + + # + # New tag. + # + asSplit = sLine.split(None, 1); + sCurTag = asSplit[0].lower(); + if len(asSplit) > 1: + asCurSection = [asSplit[1],]; + else: + asCurSection = []; + aasSections = [asCurSection, ]; + iCurTagLine = iLine; + + # + # Process the final tag. + # + if not asCurSection and len(aasSections) > 1: + aasSections.pop(-1); + if sCurTag in self.dTagHandlers: + self.dTagHandlers[sCurTag](sCurTag, aasSections, iCurTagLine, iLine); + cOpTags += 1; + elif sCurTag.startswith('@op'): + self.errorComment(iCurTagLine, 'Unknown tag: %s' % (sCurTag)); + elif sCurTag == '@default': + sFlatDefault = self.flattenAllSections(aasSections); + + # + # Don't allow default text in blocks containing @op*. + # + if cOpTags > 0 and sFlatDefault: + self.errorComment(0, 'Untagged comment text is not allowed with @op*: %s' % (sFlatDefault,)); + + return True; + + def parseMacroInvocation(self, sInvocation): + """ + Parses a macro invocation. + + Returns a tuple, first element is the offset following the macro + invocation. The second element is a list of macro arguments, where the + zero'th is the macro name. + """ + # First the name. + offOpen = sInvocation.find('('); + if offOpen <= 0: + self.raiseError("macro invocation open parenthesis not found"); + sName = sInvocation[:offOpen].strip(); + if not self.oReMacroName.match(sName): + return self.error("invalid macro name '%s'" % (sName,)); + asRet = [sName, ]; + + # Arguments. + iLine = self.iLine; + cDepth = 1; + off = offOpen + 1; + offStart = off; + chQuote = None; + while cDepth > 0: + if off >= len(sInvocation): + if iLine >= len(self.asLines): + self.error('macro invocation beyond end of file'); + return (off, asRet); + sInvocation += self.asLines[iLine]; + iLine += 1; + ch = sInvocation[off]; + + if chQuote: + if ch == '\\' and off + 1 < len(sInvocation): + off += 1; + elif ch == chQuote: + chQuote = None; + elif ch in ('"', '\'',): + chQuote = ch; + elif ch in (',', ')',): + if cDepth == 1: + asRet.append(sInvocation[offStart:off].strip()); + offStart = off + 1; + if ch == ')': + cDepth -= 1; + elif ch == '(': + cDepth += 1; + off += 1; + + return (off, asRet); + + def findAndParseMacroInvocationEx(self, sCode, sMacro): + """ + Returns (len(sCode), None) if not found, parseMacroInvocation result if found. + """ + offHit = sCode.find(sMacro); + if offHit >= 0 and sCode[offHit + len(sMacro):].strip()[0] == '(': + offAfter, asRet = self.parseMacroInvocation(sCode[offHit:]) + return (offHit + offAfter, asRet); + return (len(sCode), None); + + def findAndParseMacroInvocation(self, sCode, sMacro): + """ + Returns None if not found, arguments as per parseMacroInvocation if found. + """ + return self.findAndParseMacroInvocationEx(sCode, sMacro)[1]; + + def findAndParseFirstMacroInvocation(self, sCode, asMacro): + """ + Returns same as findAndParseMacroInvocation. + """ + for sMacro in asMacro: + asRet = self.findAndParseMacroInvocation(sCode, sMacro); + if asRet is not None: + return asRet; + return None; + + def workerIemOpMnemonicEx(self, sMacro, sStats, sAsm, sForm, sUpper, sLower, # pylint: disable=too-many-arguments + sDisHints, sIemHints, asOperands): + """ + Processes one of the a IEMOP_MNEMONIC0EX, IEMOP_MNEMONIC1EX, IEMOP_MNEMONIC2EX, + IEMOP_MNEMONIC3EX, and IEMOP_MNEMONIC4EX macros. + """ + # + # Some invocation checks. + # + if sUpper != sUpper.upper(): + self.error('%s: bad a_Upper parameter: %s' % (sMacro, sUpper,)); + if sLower != sLower.lower(): + self.error('%s: bad a_Lower parameter: %s' % (sMacro, sLower,)); + if sUpper.lower() != sLower: + self.error('%s: a_Upper and a_Lower parameters does not match: %s vs %s' % (sMacro, sUpper, sLower,)); + if not self.oReMnemonic.match(sLower): + self.error('%s: invalid a_Lower: %s (valid: %s)' % (sMacro, sLower, self.oReMnemonic.pattern,)); + + # + # Check if sIemHints tells us to not consider this macro invocation. + # + if sIemHints.find('IEMOPHINT_SKIP_PYTHON') >= 0: + return True; + + # Apply to the last instruction only for now. + if not self.aoCurInstrs: + self.addInstruction(); + oInstr = self.aoCurInstrs[-1]; + if oInstr.iLineMnemonicMacro == -1: + oInstr.iLineMnemonicMacro = self.iLine; + else: + self.error('%s: already saw a IEMOP_MNEMONIC* macro on line %u for this instruction' + % (sMacro, oInstr.iLineMnemonicMacro,)); + + # Mnemonic + if oInstr.sMnemonic is None: + oInstr.sMnemonic = sLower; + elif oInstr.sMnemonic != sLower: + self.error('%s: current instruction and a_Lower does not match: %s vs %s' % (sMacro, oInstr.sMnemonic, sLower,)); + + # Process operands. + if len(oInstr.aoOperands) not in [0, len(asOperands)]: + self.error('%s: number of operands given by @opN does not match macro: %s vs %s' + % (sMacro, len(oInstr.aoOperands), len(asOperands),)); + for iOperand, sType in enumerate(asOperands): + sWhere = g_kdOpTypes.get(sType, [None, None])[1]; + if sWhere is None: + self.error('%s: unknown a_Op%u value: %s' % (sMacro, iOperand + 1, sType)); + if iOperand < len(oInstr.aoOperands): # error recovery. + sWhere = oInstr.aoOperands[iOperand].sWhere; + sType = oInstr.aoOperands[iOperand].sType; + else: + sWhere = 'reg'; + sType = 'Gb'; + if iOperand == len(oInstr.aoOperands): + oInstr.aoOperands.append(Operand(sWhere, sType)) + elif oInstr.aoOperands[iOperand].sWhere != sWhere or oInstr.aoOperands[iOperand].sType != sType: + self.error('%s: @op%u and a_Op%u mismatch: %s:%s vs %s:%s' + % (sMacro, iOperand + 1, iOperand + 1, oInstr.aoOperands[iOperand].sWhere, + oInstr.aoOperands[iOperand].sType, sWhere, sType,)); + + # Encoding. + if sForm not in g_kdIemForms: + self.error('%s: unknown a_Form value: %s' % (sMacro, sForm,)); + else: + if oInstr.sEncoding is None: + oInstr.sEncoding = g_kdIemForms[sForm][0]; + elif g_kdIemForms[sForm][0] != oInstr.sEncoding: + self.error('%s: current instruction @openc and a_Form does not match: %s vs %s (%s)' + % (sMacro, oInstr.sEncoding, g_kdIemForms[sForm], sForm)); + + # Check the parameter locations for the encoding. + if g_kdIemForms[sForm][1] is not None: + if len(g_kdIemForms[sForm][1]) > len(oInstr.aoOperands): + self.error('%s: The a_Form=%s has a different operand count: %s (form) vs %s' + % (sMacro, sForm, len(g_kdIemForms[sForm][1]), len(oInstr.aoOperands) )); + else: + for iOperand, sWhere in enumerate(g_kdIemForms[sForm][1]): + if oInstr.aoOperands[iOperand].sWhere != sWhere: + self.error('%s: current instruction @op%u and a_Form location does not match: %s vs %s (%s)' + % (sMacro, iOperand + 1, oInstr.aoOperands[iOperand].sWhere, sWhere, sForm,)); + sOpFormMatch = g_kdOpTypes[oInstr.aoOperands[iOperand].sType][4]; + if (sOpFormMatch in [ 'REG', 'MEM', ] and sForm.find('_' + sOpFormMatch) < 0) \ + or (sOpFormMatch in [ 'FIXED', ] and sForm.find(sOpFormMatch) < 0) \ + or (sOpFormMatch == 'RM' and (sForm.find('_MEM') > 0 or sForm.find('_REG') > 0) ) \ + or (sOpFormMatch == 'V' and ( not (sForm.find('VEX') > 0 or sForm.find('XOP')) \ + or sForm.replace('VEX','').find('V') < 0) ): + self.error('%s: current instruction @op%u and a_Form type does not match: %s/%s vs %s' + % (sMacro, iOperand + 1, oInstr.aoOperands[iOperand].sType, sOpFormMatch, sForm, )); + if len(g_kdIemForms[sForm][1]) < len(oInstr.aoOperands): + for iOperand in range(len(g_kdIemForms[sForm][1]), len(oInstr.aoOperands)): + if oInstr.aoOperands[iOperand].sType != 'FIXED' \ + and g_kdOpTypes[oInstr.aoOperands[iOperand].sType][0] != 'IDX_ParseFixedReg': + self.error('%s: Expected FIXED type operand #%u following operands given by a_Form=%s: %s (%s)' + % (sMacro, iOperand, sForm, oInstr.aoOperands[iOperand].sType, + oInstr.aoOperands[iOperand].sWhere)); + + + # Check @opcodesub + if oInstr.sSubOpcode \ + and g_kdIemForms[sForm][2] \ + and oInstr.sSubOpcode.find(g_kdIemForms[sForm][2]) < 0: + self.error('%s: current instruction @opcodesub and a_Form does not match: %s vs %s (%s)' + % (sMacro, oInstr.sSubOpcode, g_kdIemForms[sForm][2], sForm,)); + + # Stats. + if not self.oReStatsName.match(sStats): + self.error('%s: invalid a_Stats value: %s' % (sMacro, sStats,)); + elif oInstr.sStats is None: + oInstr.sStats = sStats; + elif oInstr.sStats != sStats: + self.error('%s: mismatching @opstats and a_Stats value: %s vs %s' + % (sMacro, oInstr.sStats, sStats,)); + + # Process the hints (simply merge with @ophints w/o checking anything). + for sHint in sDisHints.split('|'): + sHint = sHint.strip(); + if sHint.startswith('DISOPTYPE_'): + sShortHint = sHint[len('DISOPTYPE_'):].lower(); + if sShortHint in g_kdHints: + oInstr.dHints[sShortHint] = True; # (dummy value, using dictionary for speed) + else: + self.error('%s: unknown a_fDisHints value: %s' % (sMacro, sHint,)); + elif sHint != '0': + self.error('%s: expected a_fDisHints value: %s' % (sMacro, sHint,)); + + for sHint in sIemHints.split('|'): + sHint = sHint.strip(); + if sHint.startswith('IEMOPHINT_'): + sShortHint = sHint[len('IEMOPHINT_'):].lower(); + if sShortHint in g_kdHints: + oInstr.dHints[sShortHint] = True; # (dummy value, using dictionary for speed) + else: + self.error('%s: unknown a_fIemHints value: %s' % (sMacro, sHint,)); + elif sHint != '0': + self.error('%s: expected a_fIemHints value: %s' % (sMacro, sHint,)); + + _ = sAsm; + return True; + + def workerIemOpMnemonic(self, sMacro, sForm, sUpper, sLower, sDisHints, sIemHints, asOperands): + """ + Processes one of the a IEMOP_MNEMONIC0, IEMOP_MNEMONIC1, IEMOP_MNEMONIC2, + IEMOP_MNEMONIC3, and IEMOP_MNEMONIC4 macros. + """ + if not asOperands: + return self.workerIemOpMnemonicEx(sMacro, sLower, sLower, sForm, sUpper, sLower, sDisHints, sIemHints, asOperands); + return self.workerIemOpMnemonicEx(sMacro, sLower + '_' + '_'.join(asOperands), sLower + ' ' + ','.join(asOperands), + sForm, sUpper, sLower, sDisHints, sIemHints, asOperands); + + def checkCodeForMacro(self, sCode): + """ + Checks code for relevant macro invocation. + """ + # + # Scan macro invocations. + # + if sCode.find('(') > 0: + # Look for instruction decoder function definitions. ASSUME single line. + asArgs = self.findAndParseFirstMacroInvocation(sCode, + [ 'FNIEMOP_DEF', + 'FNIEMOP_STUB', + 'FNIEMOP_STUB_1', + 'FNIEMOP_UD_STUB', + 'FNIEMOP_UD_STUB_1' ]); + if asArgs is not None: + sFunction = asArgs[1]; + + if not self.aoCurInstrs: + self.addInstruction(); + for oInstr in self.aoCurInstrs: + if oInstr.iLineFnIemOpMacro == -1: + oInstr.iLineFnIemOpMacro = self.iLine; + else: + self.error('%s: already seen a FNIEMOP_XXX macro for %s' % (asArgs[0], oInstr,) ); + self.setInstrunctionAttrib('sFunction', sFunction); + self.setInstrunctionAttrib('fStub', asArgs[0].find('STUB') > 0, fOverwrite = True); + self.setInstrunctionAttrib('fUdStub', asArgs[0].find('UD_STUB') > 0, fOverwrite = True); + if asArgs[0].find('STUB') > 0: + self.doneInstructions(); + return True; + + # IEMOP_HLP_DONE_VEX_DECODING_* + asArgs = self.findAndParseFirstMacroInvocation(sCode, + [ 'IEMOP_HLP_DONE_VEX_DECODING', + 'IEMOP_HLP_DONE_VEX_DECODING_L0', + 'IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV', + 'IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV', + ]); + if asArgs is not None: + sMacro = asArgs[0]; + if sMacro in ('IEMOP_HLP_DONE_VEX_DECODING_L0', 'IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV', ): + for oInstr in self.aoCurInstrs: + if 'vex_l_zero' not in oInstr.dHints: + if oInstr.iLineMnemonicMacro >= 0: + self.errorOnLine(oInstr.iLineMnemonicMacro, + 'Missing IEMOPHINT_VEX_L_ZERO! (%s on line %d)' % (sMacro, self.iLine,)); + oInstr.dHints['vex_l_zero'] = True; + return True; + + # + # IEMOP_MNEMONIC* + # + + # IEMOP_MNEMONIC(a_Stats, a_szMnemonic) IEMOP_INC_STATS(a_Stats) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC'); + if asArgs is not None: + if len(self.aoCurInstrs) == 1: + oInstr = self.aoCurInstrs[0]; + if oInstr.sStats is None: + oInstr.sStats = asArgs[1]; + self.deriveMnemonicAndOperandsFromStats(oInstr, asArgs[1]); + + # IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC0EX'); + if asArgs is not None: + self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[6], asArgs[7], + []); + # IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC1EX'); + if asArgs is not None: + self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[7], asArgs[8], + [asArgs[6],]); + # IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC2EX'); + if asArgs is not None: + self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[8], asArgs[9], + [asArgs[6], asArgs[7]]); + # IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC3EX'); + if asArgs is not None: + self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[9], + asArgs[10], [asArgs[6], asArgs[7], asArgs[8],]); + # IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, + # a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC4EX'); + if asArgs is not None: + self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[10], + asArgs[11], [asArgs[6], asArgs[7], asArgs[8], asArgs[9],]); + + # IEMOP_MNEMONIC0(a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC0'); + if asArgs is not None: + self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], []); + # IEMOP_MNEMONIC1(a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC1'); + if asArgs is not None: + self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[5], asArgs[6], [asArgs[4],]); + # IEMOP_MNEMONIC2(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC2'); + if asArgs is not None: + self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[6], asArgs[7], + [asArgs[4], asArgs[5],]); + # IEMOP_MNEMONIC3(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC3'); + if asArgs is not None: + self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[7], asArgs[8], + [asArgs[4], asArgs[5], asArgs[6],]); + # IEMOP_MNEMONIC4(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) + asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC4'); + if asArgs is not None: + self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[8], asArgs[9], + [asArgs[4], asArgs[5], asArgs[6], asArgs[7],]); + + return False; + + + def parse(self): + """ + Parses the given file. + Returns number or errors. + Raises exception on fatal trouble. + """ + #self.debug('Parsing %s' % (self.sSrcFile,)); + + while self.iLine < len(self.asLines): + sLine = self.asLines[self.iLine]; + self.iLine += 1; + + # We only look for comments, so only lines with a slash might possibly + # influence the parser state. + offSlash = sLine.find('/'); + if offSlash >= 0: + if offSlash + 1 >= len(sLine) or sLine[offSlash + 1] != '/' or self.iState != self.kiCode: + offLine = 0; + while offLine < len(sLine): + if self.iState == self.kiCode: + offHit = sLine.find('/*', offLine); # only multiline comments for now. + if offHit >= 0: + self.checkCodeForMacro(sLine[offLine:offHit]); + self.sComment = ''; + self.iCommentLine = self.iLine; + self.iState = self.kiCommentMulti; + offLine = offHit + 2; + else: + self.checkCodeForMacro(sLine[offLine:]); + offLine = len(sLine); + + elif self.iState == self.kiCommentMulti: + offHit = sLine.find('*/', offLine); + if offHit >= 0: + self.sComment += sLine[offLine:offHit]; + self.iState = self.kiCode; + offLine = offHit + 2; + self.parseComment(); + else: + self.sComment += sLine[offLine:]; + offLine = len(sLine); + else: + assert False; + # C++ line comment. + elif offSlash > 0: + self.checkCodeForMacro(sLine[:offSlash]); + + # No slash, but append the line if in multi-line comment. + elif self.iState == self.kiCommentMulti: + #self.debug('line %d: multi' % (self.iLine,)); + self.sComment += sLine; + + # No slash, but check code line for relevant macro. + elif self.iState == self.kiCode and sLine.find('IEMOP_') >= 0: + #self.debug('line %d: macro' % (self.iLine,)); + self.checkCodeForMacro(sLine); + + # If the line is a '}' in the first position, complete the instructions. + elif self.iState == self.kiCode and sLine[0] == '}': + #self.debug('line %d: }' % (self.iLine,)); + self.doneInstructions(); + + # Look for instruction table on the form 'IEM_STATIC const PFNIEMOP g_apfnVexMap3' + # so we can check/add @oppfx info from it. + elif self.iState == self.kiCode and sLine.find('PFNIEMOP') > 0 and self.oReFunTable.match(sLine): + self.parseFunctionTable(sLine); + + self.doneInstructions(); + self.debug('%3s%% / %3s stubs out of %4s instructions in %s' + % (self.cTotalStubs * 100 // self.cTotalInstr, self.cTotalStubs, self.cTotalInstr, + os.path.basename(self.sSrcFile),)); + return self.printErrors(); + + +def __parseFileByName(sSrcFile, sDefaultMap): + """ + Parses one source file for instruction specfications. + """ + # + # Read sSrcFile into a line array. + # + try: + oFile = open(sSrcFile, "r"); # pylint: disable=consider-using-with + except Exception as oXcpt: + raise Exception("failed to open %s for reading: %s" % (sSrcFile, oXcpt,)); + try: + asLines = oFile.readlines(); + except Exception as oXcpt: + raise Exception("failed to read %s: %s" % (sSrcFile, oXcpt,)); + finally: + oFile.close(); + + # + # Do the parsing. + # + try: + cErrors = SimpleParser(sSrcFile, asLines, sDefaultMap).parse(); + except ParserException as oXcpt: + print(str(oXcpt)); + raise; + + return cErrors; + + +def __doTestCopying(): + """ + Executes the asCopyTests instructions. + """ + asErrors = []; + for oDstInstr in g_aoAllInstructions: + if oDstInstr.asCopyTests: + for sSrcInstr in oDstInstr.asCopyTests: + oSrcInstr = g_dAllInstructionsByStat.get(sSrcInstr, None); + if oSrcInstr: + aoSrcInstrs = [oSrcInstr,]; + else: + aoSrcInstrs = g_dAllInstructionsByFunction.get(sSrcInstr, []); + if aoSrcInstrs: + for oSrcInstr in aoSrcInstrs: + if oSrcInstr != oDstInstr: + oDstInstr.aoTests.extend(oSrcInstr.aoTests); + else: + asErrors.append('%s:%s: error: @opcopytests reference "%s" matches the destination\n' + % ( oDstInstr.sSrcFile, oDstInstr.iLineCreated, sSrcInstr)); + else: + asErrors.append('%s:%s: error: @opcopytests reference "%s" not found\n' + % ( oDstInstr.sSrcFile, oDstInstr.iLineCreated, sSrcInstr)); + + if asErrors: + sys.stderr.write(u''.join(asErrors)); + return len(asErrors); + + +def __applyOnlyTest(): + """ + If g_aoOnlyTestInstructions contains any instructions, drop aoTests from + all other instructions so that only these get tested. + """ + if g_aoOnlyTestInstructions: + for oInstr in g_aoAllInstructions: + if oInstr.aoTests: + if oInstr not in g_aoOnlyTestInstructions: + oInstr.aoTests = []; + return 0; + +def __parseAll(): + """ + Parses all the IEMAllInstruction*.cpp.h files. + + Raises exception on failure. + """ + sSrcDir = os.path.dirname(os.path.abspath(__file__)); + cErrors = 0; + for sDefaultMap, sName in [ + ( 'one', 'IEMAllInstructionsOneByte.cpp.h'), + ( 'two0f', 'IEMAllInstructionsTwoByte0f.cpp.h'), + ( 'three0f38', 'IEMAllInstructionsThree0f38.cpp.h'), + ( 'three0f3a', 'IEMAllInstructionsThree0f3a.cpp.h'), + ( 'vexmap1', 'IEMAllInstructionsVexMap1.cpp.h'), + ( 'vexmap2', 'IEMAllInstructionsVexMap2.cpp.h'), + ( 'vexmap3', 'IEMAllInstructionsVexMap3.cpp.h'), + ( '3dnow', 'IEMAllInstructions3DNow.cpp.h'), + ]: + cErrors += __parseFileByName(os.path.join(sSrcDir, sName), sDefaultMap); + cErrors += __doTestCopying(); + cErrors += __applyOnlyTest(); + + # Total stub stats: + cTotalStubs = 0; + for oInstr in g_aoAllInstructions: + cTotalStubs += oInstr.fStub; + print('debug: %3s%% / %3s stubs out of %4s instructions in total' + % (cTotalStubs * 100 // len(g_aoAllInstructions), cTotalStubs, len(g_aoAllInstructions),)); + + if cErrors != 0: + #raise Exception('%d parse errors' % (cErrors,)); + sys.exit(1); + return True; + + + +__parseAll(); + + +# +# Generators (may perhaps move later). +# +def __formatDisassemblerTableEntry(oInstr): + """ + """ + sMacro = 'OP'; + cMaxOperands = 3; + if len(oInstr.aoOperands) > 3: + sMacro = 'OPVEX' + cMaxOperands = 4; + assert len(oInstr.aoOperands) <= cMaxOperands; + + # + # Format string. + # + sTmp = '%s("%s' % (sMacro, oInstr.sMnemonic,); + for iOperand, oOperand in enumerate(oInstr.aoOperands): + sTmp += ' ' if iOperand == 0 else ','; + if g_kdOpTypes[oOperand.sType][2][0] != '%': ## @todo remove upper() later. + sTmp += g_kdOpTypes[oOperand.sType][2].upper(); ## @todo remove upper() later. + else: + sTmp += g_kdOpTypes[oOperand.sType][2]; + sTmp += '",'; + asColumns = [ sTmp, ]; + + # + # Decoders. + # + iStart = len(asColumns); + if oInstr.sEncoding is None: + pass; + elif oInstr.sEncoding == 'ModR/M': + # ASSUME the first operand is using the ModR/M encoding + assert len(oInstr.aoOperands) >= 1 and oInstr.aoOperands[0].usesModRM(); + asColumns.append('IDX_ParseModRM,'); + elif oInstr.sEncoding in [ 'prefix', ]: + for oOperand in oInstr.aoOperands: + asColumns.append('0,'); + elif oInstr.sEncoding in [ 'fixed', 'VEX.fixed' ]: + pass; + elif oInstr.sEncoding == 'VEX.ModR/M': + asColumns.append('IDX_ParseModRM,'); + elif oInstr.sEncoding == 'vex2': + asColumns.append('IDX_ParseVex2b,') + elif oInstr.sEncoding == 'vex3': + asColumns.append('IDX_ParseVex3b,') + elif oInstr.sEncoding in g_dInstructionMaps: + asColumns.append(g_dInstructionMaps[oInstr.sEncoding].sDisParse + ','); + else: + ## @todo + #IDX_ParseTwoByteEsc, + #IDX_ParseGrp1, + #IDX_ParseShiftGrp2, + #IDX_ParseGrp3, + #IDX_ParseGrp4, + #IDX_ParseGrp5, + #IDX_Parse3DNow, + #IDX_ParseGrp6, + #IDX_ParseGrp7, + #IDX_ParseGrp8, + #IDX_ParseGrp9, + #IDX_ParseGrp10, + #IDX_ParseGrp12, + #IDX_ParseGrp13, + #IDX_ParseGrp14, + #IDX_ParseGrp15, + #IDX_ParseGrp16, + #IDX_ParseThreeByteEsc4, + #IDX_ParseThreeByteEsc5, + #IDX_ParseModFence, + #IDX_ParseEscFP, + #IDX_ParseNopPause, + #IDX_ParseInvOpModRM, + assert False, str(oInstr); + + # Check for immediates and stuff in the remaining operands. + for oOperand in oInstr.aoOperands[len(asColumns) - iStart:]: + sIdx = g_kdOpTypes[oOperand.sType][0]; + #if sIdx != 'IDX_UseModRM': + asColumns.append(sIdx + ','); + asColumns.extend(['0,'] * (cMaxOperands - (len(asColumns) - iStart))); + + # + # Opcode and operands. + # + assert oInstr.sDisEnum, str(oInstr); + asColumns.append(oInstr.sDisEnum + ','); + iStart = len(asColumns) + for oOperand in oInstr.aoOperands: + asColumns.append('OP_PARM_' + g_kdOpTypes[oOperand.sType][3] + ','); + asColumns.extend(['OP_PARM_NONE,'] * (cMaxOperands - (len(asColumns) - iStart))); + + # + # Flags. + # + sTmp = ''; + for sHint in sorted(oInstr.dHints.keys()): + sDefine = g_kdHints[sHint]; + if sDefine.startswith('DISOPTYPE_'): + if sTmp: + sTmp += ' | ' + sDefine; + else: + sTmp += sDefine; + if sTmp: + sTmp += '),'; + else: + sTmp += '0),'; + asColumns.append(sTmp); + + # + # Format the columns into a line. + # + aoffColumns = [4, 29, 49, 65, 77, 89, 109, 125, 141, 157, 183, 199]; + sLine = ''; + for i, s in enumerate(asColumns): + if len(sLine) < aoffColumns[i]: + sLine += ' ' * (aoffColumns[i] - len(sLine)); + else: + sLine += ' '; + sLine += s; + + # OP("psrlw %Vdq,%Wdq", IDX_ParseModRM, IDX_UseModRM, 0, OP_PSRLW, OP_PARM_Vdq, OP_PARM_Wdq, OP_PARM_NONE, + # DISOPTYPE_HARMLESS), + # define OP(pszOpcode, idxParse1, idxParse2, idxParse3, opcode, param1, param2, param3, optype) \ + # { pszOpcode, idxParse1, idxParse2, idxParse3, 0, opcode, param1, param2, param3, 0, 0, optype } + return sLine; + +def __checkIfShortTable(aoTableOrdered, oMap): + """ + Returns (iInstr, cInstructions, fShortTable) + """ + + # Determin how much we can trim off. + cInstructions = len(aoTableOrdered); + while cInstructions > 0 and aoTableOrdered[cInstructions - 1] is None: + cInstructions -= 1; + + iInstr = 0; + while iInstr < cInstructions and aoTableOrdered[iInstr] is None: + iInstr += 1; + + # If we can save more than 30%, we go for the short table version. + if iInstr + len(aoTableOrdered) - cInstructions >= len(aoTableOrdered) // 30: + return (iInstr, cInstructions, True); + _ = oMap; # Use this for overriding. + + # Output the full table. + return (0, len(aoTableOrdered), False); + +def generateDisassemblerTables(oDstFile = sys.stdout): + """ + Generates disassembler tables. + """ + + # + # The disassembler uses a slightly different table layout to save space, + # since several of the prefix varia + # + aoDisasmMaps = []; + for sName, oMap in sorted(iter(g_dInstructionMaps.items()), + key = lambda aKV: aKV[1].sEncoding + ''.join(aKV[1].asLeadOpcodes)): + if oMap.sSelector != 'byte+pfx': + aoDisasmMaps.append(oMap); + else: + # Split the map by prefix. + aoDisasmMaps.append(oMap.copy(oMap.sName, 'none')); + aoDisasmMaps.append(oMap.copy(oMap.sName + '_66', '0x66')); + aoDisasmMaps.append(oMap.copy(oMap.sName + '_F3', '0xf3')); + aoDisasmMaps.append(oMap.copy(oMap.sName + '_F2', '0xf2')); + + # + # Dump each map. + # + asHeaderLines = []; + print("debug: maps=%s\n" % (', '.join([oMap.sName for oMap in aoDisasmMaps]),)); + for oMap in aoDisasmMaps: + sName = oMap.sName; + + if not sName.startswith("vex"): continue; # only looking at the vex maps at the moment. + + # + # Get the instructions for the map and see if we can do a short version or not. + # + aoTableOrder = oMap.getInstructionsInTableOrder(); + cEntriesPerByte = oMap.getEntriesPerByte(); + (iInstrStart, iInstrEnd, fShortTable) = __checkIfShortTable(aoTableOrder, oMap); + + # + # Output the table start. + # Note! Short tables are static and only accessible via the map range record. + # + asLines = []; + asLines.append('/* Generated from: %-11s Selector: %-7s Encoding: %-7s Lead bytes opcodes: %s */' + % ( oMap.sName, oMap.sSelector, oMap.sEncoding, ' '.join(oMap.asLeadOpcodes), )); + if fShortTable: + asLines.append('%sconst DISOPCODE %s[] =' % ('static ' if fShortTable else '', oMap.getDisasTableName(),)); + else: + asHeaderLines.append('extern const DISOPCODE %s[%d];' % (oMap.getDisasTableName(), iInstrEnd - iInstrStart,)); + asLines.append( 'const DISOPCODE %s[%d] =' % (oMap.getDisasTableName(), iInstrEnd - iInstrStart,)); + asLines.append('{'); + + if fShortTable and (iInstrStart & ((0x10 * cEntriesPerByte) - 1)) != 0: + asLines.append(' /* %#04x: */' % (iInstrStart,)); + + # + # Output the instructions. + # + iInstr = iInstrStart; + while iInstr < iInstrEnd: + oInstr = aoTableOrder[iInstr]; + if (iInstr & ((0x10 * cEntriesPerByte) - 1)) == 0: + if iInstr != iInstrStart: + asLines.append(''); + asLines.append(' /* %x */' % ((iInstr // cEntriesPerByte) >> 4,)); + + if oInstr is None: + # Invalid. Optimize blocks of invalid instructions. + cInvalidInstrs = 1; + while iInstr + cInvalidInstrs < len(aoTableOrder) and aoTableOrder[iInstr + cInvalidInstrs] is None: + cInvalidInstrs += 1; + if (iInstr & (0x10 * cEntriesPerByte - 1)) == 0 and cInvalidInstrs >= 0x10 * cEntriesPerByte: + asLines.append(' INVALID_OPCODE_BLOCK_%u,' % (0x10 * cEntriesPerByte,)); + iInstr += 0x10 * cEntriesPerByte - 1; + elif cEntriesPerByte > 1: + if (iInstr & (cEntriesPerByte - 1)) == 0 and cInvalidInstrs >= cEntriesPerByte: + asLines.append(' INVALID_OPCODE_BLOCK_%u,' % (cEntriesPerByte,)); + iInstr += 3; + else: + asLines.append(' /* %#04x/%d */ INVALID_OPCODE,' + % (iInstr // cEntriesPerByte, iInstr % cEntriesPerByte)); + else: + asLines.append(' /* %#04x */ INVALID_OPCODE,' % (iInstr)); + elif isinstance(oInstr, list): + if len(oInstr) != 0: + asLines.append(' /* %#04x */ ComplicatedListStuffNeedingWrapper, /* \n -- %s */' + % (iInstr, '\n -- '.join([str(oItem) for oItem in oInstr]),)); + else: + asLines.append(__formatDisassemblerTableEntry(oInstr)); + else: + asLines.append(__formatDisassemblerTableEntry(oInstr)); + + iInstr += 1; + + if iInstrStart >= iInstrEnd: + asLines.append(' /* dummy */ INVALID_OPCODE'); + + asLines.append('};'); + asLines.append('AssertCompile(RT_ELEMENTS(%s) == %s);' % (oMap.getDisasTableName(), iInstrEnd - iInstrStart,)); + + # + # We always emit a map range record, assuming the linker will eliminate the unnecessary ones. + # + asHeaderLines.append('extern const DISOPMAPDESC %sRange;' % (oMap.getDisasRangeName())); + asLines.append('const DISOPMAPDESC %s = { &%s[0], %#04x, RT_ELEMENTS(%s) };' + % (oMap.getDisasRangeName(), oMap.getDisasTableName(), iInstrStart, oMap.getDisasTableName(),)); + + # + # Write out the lines. + # + oDstFile.write('\n'.join(asLines)); + oDstFile.write('\n'); + oDstFile.write('\n'); + #break; #for now + +if __name__ == '__main__': + generateDisassemblerTables(); + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h new file mode 100644 index 00000000..a19eb792 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h @@ -0,0 +1,2187 @@ +/* $Id: IEMAllInstructionsThree0f38.cpp.h $ */ +/** @file + * IEM - Instruction Decoding and Emulation. + * + * @remarks IEMAllInstructionsVexMap2.cpp.h is a VEX mirror of this file. + * Any update here is likely needed in that file too. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/** @name Three byte opcodes with first two bytes 0x0f 0x38 + * @{ + */ + +FNIEMOP_DEF_2(iemOpCommonMmx_FullFull_To_Full_Ex, PFNIEMAIMPLMEDIAF2U64, pfnU64, bool, fSupported); /* in IEMAllInstructionsTwoByteOf.cpp.h */ + + +/** + * Common worker for SSSE3 instructions on the forms: + * pxxx xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. SSSE3 cpuid checks. + * + * @sa iemOpCommonSse2_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSsse3_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSSE3_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSSE3_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE4.1 instructions on the forms: + * pxxx xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. SSE4.1 cpuid checks. + * + * @sa iemOpCommonSse2_FullFull_To_Full, iemOpCommonSsse3_FullFull_To_Full, + * iemOpCommonSse42_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse41_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE4.1 instructions on the forms: + * pxxx xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. SSE4.1 cpuid checks. + * + * Unlike iemOpCommonSse41_FullFull_To_Full, the @a pfnU128 worker function + * takes no FXSAVE state, just the operands. + * + * @sa iemOpCommonSse2_FullFull_To_Full, iemOpCommonSsse3_FullFull_To_Full, + * iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse42_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse41Opt_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE4.2 instructions on the forms: + * pxxx xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. SSE4.2 cpuid checks. + * + * @sa iemOpCommonSse2_FullFull_To_Full, iemOpCommonSsse3_FullFull_To_Full, + * iemOpCommonSse41_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse42_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE42_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE42_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE-style AES-NI instructions of the form: + * aesxxx xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. AES-NI cpuid checks. + * + * Unlike iemOpCommonSse41_FullFull_To_Full, the @a pfnU128 worker function + * takes no FXSAVE state, just the operands. + * + * @sa iemOpCommonSse2_FullFull_To_Full, iemOpCommonSsse3_FullFull_To_Full, + * iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse42_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonAesNi_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_AESNI_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_AESNI_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x38 0x00. */ +FNIEMOP_DEF(iemOp_pshufb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSHUFB, pshufb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pshufb_u64,&iemAImpl_pshufb_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x00. */ +FNIEMOP_DEF(iemOp_pshufb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSHUFB, pshufb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pshufb_u128, iemAImpl_pshufb_u128_fallback)); + +} + + +/* Opcode 0x0f 0x38 0x01. */ +FNIEMOP_DEF(iemOp_phaddw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PHADDW, phaddw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddw_u64,&iemAImpl_phaddw_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x01. */ +FNIEMOP_DEF(iemOp_phaddw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PHADDW, phaddw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddw_u128, iemAImpl_phaddw_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x02. */ +FNIEMOP_DEF(iemOp_phaddd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PHADDD, phaddd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddd_u64,&iemAImpl_phaddd_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x02. */ +FNIEMOP_DEF(iemOp_phaddd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PHADDD, phaddd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddd_u128, iemAImpl_phaddd_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x03. */ +FNIEMOP_DEF(iemOp_phaddsw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PHADDSW, phaddsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddsw_u64,&iemAImpl_phaddsw_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x03. */ +FNIEMOP_DEF(iemOp_phaddsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PHADDSW, phaddsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddsw_u128, iemAImpl_phaddsw_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x04. */ +FNIEMOP_DEF(iemOp_pmaddubsw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMADDUBSW, pmaddubsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pmaddubsw_u64, &iemAImpl_pmaddubsw_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x04. */ +FNIEMOP_DEF(iemOp_pmaddubsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMADDUBSW, pmaddubsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pmaddubsw_u128, iemAImpl_pmaddubsw_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x05. */ +FNIEMOP_DEF(iemOp_phsubw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PHSUBW, phsubw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubw_u64,&iemAImpl_phsubw_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x05. */ +FNIEMOP_DEF(iemOp_phsubw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PHSUBW, phsubw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubw_u128, iemAImpl_phsubw_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x06. */ +FNIEMOP_DEF(iemOp_phsubd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PHSUBD, phsubd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubd_u64,&iemAImpl_phsubd_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + + +/** Opcode 0x66 0x0f 0x38 0x06. */ +FNIEMOP_DEF(iemOp_phsubd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PHSUBD, phsubd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubd_u128, iemAImpl_phsubd_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x07. */ +FNIEMOP_DEF(iemOp_phsubsw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PHSUBSW, phsubsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubsw_u64,&iemAImpl_phsubsw_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x07. */ +FNIEMOP_DEF(iemOp_phsubsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PHSUBSW, phsubsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubsw_u128, iemAImpl_phsubsw_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x08. */ +FNIEMOP_DEF(iemOp_psignb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSIGNB, psignb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignb_u64, &iemAImpl_psignb_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x08. */ +FNIEMOP_DEF(iemOp_psignb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSIGNB, psignb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignb_u128, iemAImpl_psignb_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x09. */ +FNIEMOP_DEF(iemOp_psignw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSIGNW, psignw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignw_u64, &iemAImpl_psignw_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x09. */ +FNIEMOP_DEF(iemOp_psignw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSIGNW, psignw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignw_u128, iemAImpl_psignw_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x0a. */ +FNIEMOP_DEF(iemOp_psignd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSIGND, psignd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignd_u64, &iemAImpl_psignd_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x0a. */ +FNIEMOP_DEF(iemOp_psignd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSIGND, psignd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignd_u128, iemAImpl_psignd_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x0b. */ +FNIEMOP_DEF(iemOp_pmulhrsw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMULHRSW, pmulhrsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pmulhrsw_u64, &iemAImpl_pmulhrsw_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x0b. */ +FNIEMOP_DEF(iemOp_pmulhrsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMULHRSW, pmulhrsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pmulhrsw_u128, iemAImpl_pmulhrsw_u128_fallback)); + +} + + +/* Opcode 0x0f 0x38 0x0c - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x0c - invalid (vex only). */ +/* Opcode 0x0f 0x38 0x0d - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x0d - invalid (vex only). */ +/* Opcode 0x0f 0x38 0x0e - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x0e - invalid (vex only). */ +/* Opcode 0x0f 0x38 0x0f - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x0f - invalid (vex only). */ + + +/* Opcode 0x0f 0x38 0x10 - invalid */ + + +/** Body for the *blend* instructions. */ +#define IEMOP_BODY_P_BLEND_X(a_Instr) \ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \ + if (IEM_IS_MODRM_REG_MODE(bRm)) \ + { \ + /* \ + * Register, register. \ + */ \ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \ + IEM_MC_BEGIN(3, 0); \ + IEM_MC_ARG(PRTUINT128U, puDst, 0); \ + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); \ + IEM_MC_ARG(PCRTUINT128U, puMask, 2); \ + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); \ + IEM_MC_PREPARE_SSE_USAGE(); \ + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_REF_XREG_U128_CONST(puMask, 0); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fSse41, \ + iemAImpl_ ## a_Instr ## _u128, \ + iemAImpl_ ## a_Instr ## _u128_fallback), \ + puDst, puSrc, puMask); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + /* \ + * Register, memory. \ + */ \ + IEM_MC_BEGIN(3, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_LOCAL(RTUINT128U, uSrc); \ + IEM_MC_ARG(PRTUINT128U, puDst, 0); \ + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); \ + IEM_MC_ARG(PCRTUINT128U, puMask, 2); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \ + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); \ + IEM_MC_PREPARE_SSE_USAGE(); \ + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_REF_XREG_U128_CONST(puMask, 0); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fSse41, \ + iemAImpl_ ## a_Instr ## _u128, \ + iemAImpl_ ## a_Instr ## _u128_fallback), \ + puDst, puSrc, puMask); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + (void)0 + +/** Opcode 0x66 0x0f 0x38 0x10 (legacy only). */ +FNIEMOP_DEF(iemOp_pblendvb_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, PBLENDVB, pblendvb, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); /** @todo RM0 */ + IEMOP_BODY_P_BLEND_X(pblendvb); +} + + +/* Opcode 0x0f 0x38 0x11 - invalid */ +/* Opcode 0x66 0x0f 0x38 0x11 - invalid */ +/* Opcode 0x0f 0x38 0x12 - invalid */ +/* Opcode 0x66 0x0f 0x38 0x12 - invalid */ +/* Opcode 0x0f 0x38 0x13 - invalid */ +/* Opcode 0x66 0x0f 0x38 0x13 - invalid (vex only). */ +/* Opcode 0x0f 0x38 0x14 - invalid */ + + +/** Opcode 0x66 0x0f 0x38 0x14 (legacy only). */ +FNIEMOP_DEF(iemOp_blendvps_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, BLENDVPS, blendvps, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); /** @todo RM0 */ + IEMOP_BODY_P_BLEND_X(blendvps); +} + + +/* Opcode 0x0f 0x38 0x15 - invalid */ + + +/** Opcode 0x66 0x0f 0x38 0x15 (legacy only). */ +FNIEMOP_DEF(iemOp_blendvpd_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, BLENDVPD, blendvpd, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); /** @todo RM0 */ + IEMOP_BODY_P_BLEND_X(blendvpd); +} + + +/* Opcode 0x0f 0x38 0x16 - invalid */ +/* Opcode 0x66 0x0f 0x38 0x16 - invalid (vex only). */ +/* Opcode 0x0f 0x38 0x17 - invalid */ + + +/** Opcode 0x66 0x0f 0x38 0x17 - invalid */ +FNIEMOP_DEF(iemOp_ptest_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PTEST, ptest, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc2, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_ptest_u128, puSrc1, puSrc2, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc2, uSrc2, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_ptest_u128, puSrc1, puSrc2, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode 0x0f 0x38 0x18 - invalid */ +/* Opcode 0x66 0x0f 0x38 0x18 - invalid (vex only). */ +/* Opcode 0x0f 0x38 0x19 - invalid */ +/* Opcode 0x66 0x0f 0x38 0x19 - invalid (vex only). */ +/* Opcode 0x0f 0x38 0x1a - invalid */ +/* Opcode 0x66 0x0f 0x38 0x1a - invalid (vex only). */ +/* Opcode 0x0f 0x38 0x1b - invalid */ +/* Opcode 0x66 0x0f 0x38 0x1b - invalid */ + + +/** Opcode 0x0f 0x38 0x1c. */ +FNIEMOP_DEF(iemOp_pabsb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PABSB, pabsb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsb_u64, &iemAImpl_pabsb_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x1c. */ +FNIEMOP_DEF(iemOp_pabsb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PABSB, pabsb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsb_u128, iemAImpl_pabsb_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x1d. */ +FNIEMOP_DEF(iemOp_pabsw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PABSW, pabsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsw_u64, &iemAImpl_pabsw_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x1d. */ +FNIEMOP_DEF(iemOp_pabsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PABSW, pabsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsw_u128, iemAImpl_pabsw_u128_fallback)); + +} + + +/** Opcode 0x0f 0x38 0x1e. */ +FNIEMOP_DEF(iemOp_pabsd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PABSD, pabsd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsd_u64, &iemAImpl_pabsd_u64_fallback), + IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); +} + + +/** Opcode 0x66 0x0f 0x38 0x1e. */ +FNIEMOP_DEF(iemOp_pabsd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PABSD, pabsd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsd_u128, iemAImpl_pabsd_u128_fallback)); + +} + + +/* Opcode 0x0f 0x38 0x1f - invalid */ +/* Opcode 0x66 0x0f 0x38 0x1f - invalid */ + + +/** Body for the pmov{s,z}x* instructions. */ +#define IEMOP_BODY_PMOV_S_Z(a_Instr, a_SrcWidth) \ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \ + if (IEM_IS_MODRM_REG_MODE(bRm)) \ + { \ + /* \ + * Register, register. \ + */ \ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \ + IEM_MC_BEGIN(2, 0); \ + IEM_MC_ARG(PRTUINT128U, puDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc, 1); \ + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); \ + IEM_MC_PREPARE_SSE_USAGE(); \ + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword */); \ + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse41, \ + iemAImpl_ ## a_Instr ## _u128, \ + iemAImpl_v ## a_Instr ## _u128_fallback), \ + puDst, uSrc); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + /* \ + * Register, memory. \ + */ \ + IEM_MC_BEGIN(2, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_ARG(PRTUINT128U, puDst, 0); \ + IEM_MC_ARG(uint ## a_SrcWidth ## _t, uSrc, 1); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \ + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); \ + IEM_MC_PREPARE_SSE_USAGE(); \ + IEM_MC_FETCH_MEM_U## a_SrcWidth (uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse41, \ + iemAImpl_ ## a_Instr ## _u128, \ + iemAImpl_v ## a_Instr ## _u128_fallback), \ + puDst, uSrc); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + (void)0 + + +/** Opcode 0x66 0x0f 0x38 0x20. */ +FNIEMOP_DEF(iemOp_pmovsxbw_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVSXBW, pmovsxbw, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovsxbw, 64); +} + + +/** Opcode 0x66 0x0f 0x38 0x21. */ +FNIEMOP_DEF(iemOp_pmovsxbd_Vx_UxMd) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVSXBD, pmovsxbd, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovsxbd, 32); +} + + +/** Opcode 0x66 0x0f 0x38 0x22. */ +FNIEMOP_DEF(iemOp_pmovsxbq_Vx_UxMw) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVSXBQ, pmovsxbq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovsxbq, 16); +} + + +/** Opcode 0x66 0x0f 0x38 0x23. */ +FNIEMOP_DEF(iemOp_pmovsxwd_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVSXWD, pmovsxwd, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovsxwd, 64); +} + + +/** Opcode 0x66 0x0f 0x38 0x24. */ +FNIEMOP_DEF(iemOp_pmovsxwq_Vx_UxMd) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVSXWQ, pmovsxwq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovsxwq, 32); +} + + +/** Opcode 0x66 0x0f 0x38 0x25. */ +FNIEMOP_DEF(iemOp_pmovsxdq_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVSXDQ, pmovsxdq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovsxdq, 64); +} + + +/* Opcode 0x66 0x0f 0x38 0x26 - invalid */ +/* Opcode 0x66 0x0f 0x38 0x27 - invalid */ + + +/** Opcode 0x66 0x0f 0x38 0x28. */ +FNIEMOP_DEF(iemOp_pmuldq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMULDQ, pmuldq, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41Opt_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmuldq_u128, iemAImpl_pmuldq_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x29. */ +FNIEMOP_DEF(iemOp_pcmpeqq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PCMPEQQ, pcmpeqq, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pcmpeqq_u128, iemAImpl_pcmpeqq_u128_fallback)); +} + + +/** + * @opcode 0x2a + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid sse4.1 + * @opgroup og_sse41_cachect + * @opxcpttype 1 + * @optest op1=-1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movntdqa_Vdq_Mdq) +{ + IEMOP_MNEMONIC2(RM_MEM, MOVNTDQA, movntdqa, Vdq_WO, Mdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* Register, memory. */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic ud660f382areg + * @opcode 0x2a + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid sse + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode 0x66 0x0f 0x38 0x2b. */ +FNIEMOP_DEF(iemOp_packusdw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PACKUSDW, packusdw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse41Opt_FullFull_To_Full, iemAImpl_packusdw_u128); +} + + +/* Opcode 0x66 0x0f 0x38 0x2c - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x2d - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x2e - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x2f - invalid (vex only). */ + +/** Opcode 0x66 0x0f 0x38 0x30. */ +FNIEMOP_DEF(iemOp_pmovzxbw_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVZXBW, pmovzxbw, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovzxbw, 64); +} + + +/** Opcode 0x66 0x0f 0x38 0x31. */ +FNIEMOP_DEF(iemOp_pmovzxbd_Vx_UxMd) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVZXBD, pmovzxbd, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovzxbd, 32); +} + + +/** Opcode 0x66 0x0f 0x38 0x32. */ +FNIEMOP_DEF(iemOp_pmovzxbq_Vx_UxMw) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVZXBQ, pmovzxbq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovzxbq, 16); +} + + +/** Opcode 0x66 0x0f 0x38 0x33. */ +FNIEMOP_DEF(iemOp_pmovzxwd_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVZXWD, pmovzxwd, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovzxwd, 64); +} + + +/** Opcode 0x66 0x0f 0x38 0x34. */ +FNIEMOP_DEF(iemOp_pmovzxwq_Vx_UxMd) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVZXWQ, pmovzxwq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovzxwq, 32); +} + + +/** Opcode 0x66 0x0f 0x38 0x35. */ +FNIEMOP_DEF(iemOp_pmovzxdq_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(RM, PMOVZXDQ, pmovzxdq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_BODY_PMOV_S_Z(pmovzxdq, 64); +} + + +/* Opcode 0x66 0x0f 0x38 0x36 - invalid (vex only). */ + + +/** Opcode 0x66 0x0f 0x38 0x37. */ +FNIEMOP_DEF(iemOp_pcmpgtq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PCMPGTQ, pcmpgtq, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse42_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_pcmpgtq_u128, iemAImpl_pcmpgtq_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x38. */ +FNIEMOP_DEF(iemOp_pminsb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMINSB, pminsb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pminsb_u128, iemAImpl_pminsb_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x39. */ +FNIEMOP_DEF(iemOp_pminsd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMINSD, pminsd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pminsd_u128, iemAImpl_pminsd_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x3a. */ +FNIEMOP_DEF(iemOp_pminuw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMINUW, pminuw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pminuw_u128, iemAImpl_pminuw_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x3b. */ +FNIEMOP_DEF(iemOp_pminud_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMINUD, pminud, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pminud_u128, iemAImpl_pminud_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x3c. */ +FNIEMOP_DEF(iemOp_pmaxsb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMAXSB, pmaxsb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmaxsb_u128, iemAImpl_pmaxsb_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x3d. */ +FNIEMOP_DEF(iemOp_pmaxsd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMAXSD, pmaxsd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmaxsd_u128, iemAImpl_pmaxsd_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x3e. */ +FNIEMOP_DEF(iemOp_pmaxuw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMAXUW, pmaxuw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmaxuw_u128, iemAImpl_pmaxuw_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x3f. */ +FNIEMOP_DEF(iemOp_pmaxud_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMAXUD, pmaxud, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmaxud_u128, iemAImpl_pmaxud_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x40. */ +FNIEMOP_DEF(iemOp_pmulld_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMULLD, pmulld, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmulld_u128, iemAImpl_pmulld_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0x41. */ +FNIEMOP_DEF(iemOp_phminposuw_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, PHMINPOSUW, phminposuw, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse41Opt_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_phminposuw_u128, iemAImpl_phminposuw_u128_fallback)); +} + + +/* Opcode 0x66 0x0f 0x38 0x42 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x43 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x44 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x45 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x46 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x47 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x48 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x49 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x4a - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x4b - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x4c - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x4d - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x4e - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x4f - invalid. */ + +/* Opcode 0x66 0x0f 0x38 0x50 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x51 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x52 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x53 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x54 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x55 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x56 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x57 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x58 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x59 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x5a - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x5b - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x5c - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x5d - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x5e - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x5f - invalid. */ + +/* Opcode 0x66 0x0f 0x38 0x60 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x61 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x62 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x63 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x64 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x65 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x66 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x67 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x68 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x69 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x6a - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x6b - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x6c - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x6d - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x6e - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x6f - invalid. */ + +/* Opcode 0x66 0x0f 0x38 0x70 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x71 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x72 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x73 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x74 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x75 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x76 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x77 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x78 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x79 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x7a - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x7b - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x7c - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x7d - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x7e - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x7f - invalid. */ + +/** Opcode 0x66 0x0f 0x38 0x80. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +FNIEMOP_DEF(iemOp_invept_Gy_Mdq) +{ + IEMOP_MNEMONIC(invept, "invept Gy,Mdq"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_IN_VMX_OPERATION("invept", kVmxVDiag_Invept); + IEMOP_HLP_VMX_INSTR("invept", kVmxVDiag_Invept); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* Register, memory. */ + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrInveptDesc, 1); + IEM_MC_ARG(uint64_t, uInveptType, 2); + IEM_MC_FETCH_GREG_U64(uInveptType, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrInveptDesc, bRm, 0); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_invept, iEffSeg, GCPtrInveptDesc, uInveptType); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrInveptDesc, 1); + IEM_MC_ARG(uint32_t, uInveptType, 2); + IEM_MC_FETCH_GREG_U32(uInveptType, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrInveptDesc, bRm, 0); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_invept, iEffSeg, GCPtrInveptDesc, uInveptType); + IEM_MC_END(); + } + } + Log(("iemOp_invept_Gy_Mdq: invalid encoding -> #UD\n")); + return IEMOP_RAISE_INVALID_OPCODE(); +} +#else +FNIEMOP_STUB(iemOp_invept_Gy_Mdq); +#endif + +/** Opcode 0x66 0x0f 0x38 0x81. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF(iemOp_invvpid_Gy_Mdq) +{ + IEMOP_MNEMONIC(invvpid, "invvpid Gy,Mdq"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEMOP_HLP_IN_VMX_OPERATION("invvpid", kVmxVDiag_Invvpid); + IEMOP_HLP_VMX_INSTR("invvpid", kVmxVDiag_Invvpid); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* Register, memory. */ + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrInvvpidDesc, 1); + IEM_MC_ARG(uint64_t, uInvvpidType, 2); + IEM_MC_FETCH_GREG_U64(uInvvpidType, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvvpidDesc, bRm, 0); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_invvpid, iEffSeg, GCPtrInvvpidDesc, uInvvpidType); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrInvvpidDesc, 1); + IEM_MC_ARG(uint32_t, uInvvpidType, 2); + IEM_MC_FETCH_GREG_U32(uInvvpidType, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvvpidDesc, bRm, 0); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_invvpid, iEffSeg, GCPtrInvvpidDesc, uInvvpidType); + IEM_MC_END(); + } + } + Log(("iemOp_invvpid_Gy_Mdq: invalid encoding -> #UD\n")); + return IEMOP_RAISE_INVALID_OPCODE(); +} +#else +FNIEMOP_STUB(iemOp_invvpid_Gy_Mdq); +#endif + +/** Opcode 0x66 0x0f 0x38 0x82. */ +FNIEMOP_DEF(iemOp_invpcid_Gy_Mdq) +{ + IEMOP_MNEMONIC(invpcid, "invpcid Gy,Mdq"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* Register, memory. */ + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrInvpcidDesc, 1); + IEM_MC_ARG(uint64_t, uInvpcidType, 2); + IEM_MC_FETCH_GREG_U64(uInvpcidType, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvpcidDesc, bRm, 0); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_invpcid, iEffSeg, GCPtrInvpcidDesc, uInvpcidType); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrInvpcidDesc, 1); + IEM_MC_ARG(uint32_t, uInvpcidType, 2); + IEM_MC_FETCH_GREG_U32(uInvpcidType, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvpcidDesc, bRm, 0); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_invpcid, iEffSeg, GCPtrInvpcidDesc, uInvpcidType); + IEM_MC_END(); + } + } + Log(("iemOp_invpcid_Gy_Mdq: invalid encoding -> #UD\n")); + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/* Opcode 0x66 0x0f 0x38 0x83 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x84 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x85 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x86 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x87 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x88 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x89 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x8a - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x8b - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x8c - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x8d - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x8e - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x8f - invalid. */ + +/* Opcode 0x66 0x0f 0x38 0x90 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x91 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x92 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x93 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x94 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x95 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0x96 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x97 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x98 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x99 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x9a - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x9b - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x9c - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x9d - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x9e - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0x9f - invalid (vex only). */ + +/* Opcode 0x66 0x0f 0x38 0xa0 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xa1 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xa2 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xa3 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xa4 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xa5 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xa6 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xa7 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xa8 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xa9 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xaa - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xab - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xac - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xad - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xae - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xaf - invalid (vex only). */ + +/* Opcode 0x66 0x0f 0x38 0xb0 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xb1 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xb2 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xb3 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xb4 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xb5 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xb6 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xb7 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xb8 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xb9 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xba - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xbb - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xbc - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xbd - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xbe - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xbf - invalid (vex only). */ + +/* Opcode 0x0f 0x38 0xc0 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xc0 - invalid. */ +/* Opcode 0x0f 0x38 0xc1 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xc1 - invalid. */ +/* Opcode 0x0f 0x38 0xc2 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xc2 - invalid. */ +/* Opcode 0x0f 0x38 0xc3 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xc3 - invalid. */ +/* Opcode 0x0f 0x38 0xc4 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xc4 - invalid. */ +/* Opcode 0x0f 0x38 0xc5 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xc5 - invalid. */ +/* Opcode 0x0f 0x38 0xc6 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xc6 - invalid. */ +/* Opcode 0x0f 0x38 0xc7 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xc7 - invalid. */ +/** Opcode 0x0f 0x38 0xc8. */ +FNIEMOP_STUB(iemOp_sha1nexte_Vdq_Wdq); +/* Opcode 0x66 0x0f 0x38 0xc8 - invalid. */ +/** Opcode 0x0f 0x38 0xc9. */ +FNIEMOP_STUB(iemOp_sha1msg1_Vdq_Wdq); +/* Opcode 0x66 0x0f 0x38 0xc9 - invalid. */ +/** Opcode 0x0f 0x38 0xca. */ +FNIEMOP_STUB(iemOp_sha1msg2_Vdq_Wdq); +/* Opcode 0x66 0x0f 0x38 0xca - invalid. */ +/** Opcode 0x0f 0x38 0xcb. */ +FNIEMOP_STUB(iemOp_sha256rnds2_Vdq_Wdq); +/* Opcode 0x66 0x0f 0x38 0xcb - invalid. */ +/** Opcode 0x0f 0x38 0xcc. */ +FNIEMOP_STUB(iemOp_sha256msg1_Vdq_Wdq); +/* Opcode 0x66 0x0f 0x38 0xcc - invalid. */ +/** Opcode 0x0f 0x38 0xcd. */ +FNIEMOP_STUB(iemOp_sha256msg2_Vdq_Wdq); +/* Opcode 0x66 0x0f 0x38 0xcd - invalid. */ +/* Opcode 0x0f 0x38 0xce - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xce - invalid. */ +/* Opcode 0x0f 0x38 0xcf - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xcf - invalid. */ + +/* Opcode 0x66 0x0f 0x38 0xd0 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xd1 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xd2 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xd3 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xd4 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xd5 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xd6 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xd7 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xd8 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xd9 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xda - invalid. */ + + +/** Opcode 0x66 0x0f 0x38 0xdb. */ +FNIEMOP_DEF(iemOp_aesimc_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, AESIMC, aesimc, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesimc_u128, iemAImpl_aesimc_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0xdc. */ +FNIEMOP_DEF(iemOp_aesenc_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, AESENC, aesenc, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesenc_u128, iemAImpl_aesenc_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0xdd. */ +FNIEMOP_DEF(iemOp_aesenclast_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, AESENCLAST, aesenclast, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesenclast_u128, iemAImpl_aesenclast_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0xde. */ +FNIEMOP_DEF(iemOp_aesdec_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, AESDEC, aesdec, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesdec_u128, iemAImpl_aesdec_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x38 0xdf. */ +FNIEMOP_DEF(iemOp_aesdeclast_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, AESDECLAST, aesdeclast, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesdeclast_u128, iemAImpl_aesdeclast_u128_fallback)); +} + + +/* Opcode 0x66 0x0f 0x38 0xe0 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xe1 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xe2 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xe3 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xe4 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xe5 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xe6 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xe7 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xe8 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xe9 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xea - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xeb - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xec - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xed - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xee - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xef - invalid. */ + + +/** Opcode [0x66] 0x0f 0x38 0xf0. */ +FNIEMOP_DEF(iemOp_movbe_Gv_Mv) +{ + IEMOP_MNEMONIC2(RM, MOVBE, movbe, Gv, Ev, DISOPTYPE_HARMLESS, 0); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovBe) + return iemOp_InvalidNeedRM(pVCpu); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (!IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_BSWAP_LOCAL_U16(uSrc); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_BSWAP_LOCAL_U32(uSrc); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_BSWAP_LOCAL_U64(uSrc); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* Reg/reg not supported. */ + return IEMOP_RAISE_INVALID_OPCODE(); + } +} + + +/* Opcode 0xf3 0x0f 0x38 0xf0 - invalid. */ + + +/** Opcode 0xf2 0x0f 0x38 0xf0. */ +FNIEMOP_DEF(iemOp_crc32_Gd_Eb) +{ + IEMOP_MNEMONIC2(RM, CRC32, crc32, Gd, Eb, DISOPTYPE_HARMLESS, 0); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse42) + return iemOp_InvalidNeedRM(pVCpu); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint32_t *, puDst, 0); + IEM_MC_ARG(uint8_t, uSrc, 1); + IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u8, iemAImpl_crc32_u8_fallback), puDst, uSrc); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(2, 1); + IEM_MC_ARG(uint32_t *, puDst, 0); + IEM_MC_ARG(uint8_t, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u8, iemAImpl_crc32_u8_fallback), puDst, uSrc); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode [0x66] 0x0f 0x38 0xf1. */ +FNIEMOP_DEF(iemOp_movbe_Mv_Gv) +{ + IEMOP_MNEMONIC2(MR, MOVBE, movbe, Ev, Gv, DISOPTYPE_HARMLESS, 0); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovBe) + return iemOp_InvalidNeedRM(pVCpu); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (!IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Memory, register. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_BSWAP_LOCAL_U16(u16Value); + IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_BSWAP_LOCAL_U32(u32Value); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_BSWAP_LOCAL_U64(u64Value); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* Reg/reg not supported. */ + return IEMOP_RAISE_INVALID_OPCODE(); + } +} + + +/* Opcode 0xf3 0x0f 0x38 0xf1 - invalid. */ + + +/** Opcode 0xf2 0x0f 0x38 0xf1. */ +FNIEMOP_DEF(iemOp_crc32_Gv_Ev) +{ + IEMOP_MNEMONIC2(RM, CRC32, crc32, Gd, Ev, DISOPTYPE_HARMLESS, 0); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse42) + return iemOp_InvalidNeedRM(pVCpu); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint32_t *, puDst, 0); + IEM_MC_ARG(uint16_t, uSrc, 1); + IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U16(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u16, iemAImpl_crc32_u16_fallback), + puDst, uSrc); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint32_t *, puDst, 0); + IEM_MC_ARG(uint32_t, uSrc, 1); + IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u32, iemAImpl_crc32_u32_fallback), + puDst, uSrc); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint32_t *, puDst, 0); + IEM_MC_ARG(uint64_t, uSrc, 1); + IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u64, iemAImpl_crc32_u64_fallback), + puDst, uSrc); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * Register, memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(2, 1); + IEM_MC_ARG(uint32_t *, puDst, 0); + IEM_MC_ARG(uint16_t, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u16, iemAImpl_crc32_u16_fallback), + puDst, uSrc); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(2, 1); + IEM_MC_ARG(uint32_t *, puDst, 0); + IEM_MC_ARG(uint32_t, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u32, iemAImpl_crc32_u32_fallback), + puDst, uSrc); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(2, 1); + IEM_MC_ARG(uint32_t *, puDst, 0); + IEM_MC_ARG(uint64_t, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u64, iemAImpl_crc32_u64_fallback), + puDst, uSrc); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/* Opcode 0x0f 0x38 0xf2 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xf2 - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xf2 - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xf2 - invalid. */ + +/* Opcode 0x0f 0x38 0xf3 - invalid (vex only - group 17). */ +/* Opcode 0x66 0x0f 0x38 0xf3 - invalid (vex only - group 17). */ +/* Opcode 0xf3 0x0f 0x38 0xf3 - invalid (vex only - group 17). */ +/* Opcode 0xf2 0x0f 0x38 0xf3 - invalid (vex only - group 17). */ + +/* Opcode 0x0f 0x38 0xf4 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xf4 - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xf4 - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xf4 - invalid. */ + +/* Opcode 0x0f 0x38 0xf5 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xf5 - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xf5 - invalid (vex only). */ +/* Opcode 0xf2 0x0f 0x38 0xf5 - invalid (vex only). */ + +/* Opcode 0x0f 0x38 0xf6 - invalid. */ +/** Opcode 0x66 0x0f 0x38 0xf6. */ +FNIEMOP_STUB(iemOp_adcx_Gy_Ey); +/** Opcode 0xf3 0x0f 0x38 0xf6. */ +FNIEMOP_STUB(iemOp_adox_Gy_Ey); +/* Opcode 0xf2 0x0f 0x38 0xf6 - invalid (vex only). */ + +/* Opcode 0x0f 0x38 0xf7 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x38 0xf7 - invalid (vex only). */ +/* Opcode 0xf3 0x0f 0x38 0xf7 - invalid (vex only). */ +/* Opcode 0xf2 0x0f 0x38 0xf7 - invalid (vex only). */ + +/* Opcode 0x0f 0x38 0xf8 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xf8 - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xf8 - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xf8 - invalid. */ + +/* Opcode 0x0f 0x38 0xf9 - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xf9 - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xf9 - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xf9 - invalid. */ + +/* Opcode 0x0f 0x38 0xfa - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xfa - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xfa - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xfa - invalid. */ + +/* Opcode 0x0f 0x38 0xfb - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xfb - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xfb - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xfb - invalid. */ + +/* Opcode 0x0f 0x38 0xfc - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xfc - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xfc - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xfc - invalid. */ + +/* Opcode 0x0f 0x38 0xfd - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xfd - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xfd - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xfd - invalid. */ + +/* Opcode 0x0f 0x38 0xfe - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xfe - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xfe - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xfe - invalid. */ + +/* Opcode 0x0f 0x38 0xff - invalid. */ +/* Opcode 0x66 0x0f 0x38 0xff - invalid. */ +/* Opcode 0xf3 0x0f 0x38 0xff - invalid. */ +/* Opcode 0xf2 0x0f 0x38 0xff - invalid. */ + + +/** + * Three byte opcode map, first two bytes are 0x0f 0x38. + * @sa g_apfnVexMap2 + */ +IEM_STATIC const PFNIEMOP g_apfnThreeByte0f38[] = +{ + /* no prefix, 066h prefix f3h prefix, f2h prefix */ + /* 0x00 */ iemOp_pshufb_Pq_Qq, iemOp_pshufb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x01 */ iemOp_phaddw_Pq_Qq, iemOp_phaddw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x02 */ iemOp_phaddd_Pq_Qq, iemOp_phaddd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x03 */ iemOp_phaddsw_Pq_Qq, iemOp_phaddsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x04 */ iemOp_pmaddubsw_Pq_Qq, iemOp_pmaddubsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x05 */ iemOp_phsubw_Pq_Qq, iemOp_phsubw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x06 */ iemOp_phsubd_Pq_Qq, iemOp_phsubd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x07 */ iemOp_phsubsw_Pq_Qq, iemOp_phsubsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x08 */ iemOp_psignb_Pq_Qq, iemOp_psignb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x09 */ iemOp_psignw_Pq_Qq, iemOp_psignw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x0a */ iemOp_psignd_Pq_Qq, iemOp_psignd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x0b */ iemOp_pmulhrsw_Pq_Qq, iemOp_pmulhrsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x0c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x0d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x0e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x0f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x10 */ iemOp_InvalidNeedRM, iemOp_pblendvb_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x11 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x12 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x13 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x14 */ iemOp_InvalidNeedRM, iemOp_blendvps_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x15 */ iemOp_InvalidNeedRM, iemOp_blendvpd_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x16 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x17 */ iemOp_InvalidNeedRM, iemOp_ptest_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x18 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x19 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1c */ iemOp_pabsb_Pq_Qq, iemOp_pabsb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x1d */ iemOp_pabsw_Pq_Qq, iemOp_pabsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x1e */ iemOp_pabsd_Pq_Qq, iemOp_pabsd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x1f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x20 */ iemOp_InvalidNeedRM, iemOp_pmovsxbw_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x21 */ iemOp_InvalidNeedRM, iemOp_pmovsxbd_Vx_UxMd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x22 */ iemOp_InvalidNeedRM, iemOp_pmovsxbq_Vx_UxMw, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x23 */ iemOp_InvalidNeedRM, iemOp_pmovsxwd_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x24 */ iemOp_InvalidNeedRM, iemOp_pmovsxwq_Vx_UxMd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x25 */ iemOp_InvalidNeedRM, iemOp_pmovsxdq_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x26 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x27 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x28 */ iemOp_InvalidNeedRM, iemOp_pmuldq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x29 */ iemOp_InvalidNeedRM, iemOp_pcmpeqq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2a */ iemOp_InvalidNeedRM, iemOp_movntdqa_Vdq_Mdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2b */ iemOp_InvalidNeedRM, iemOp_packusdw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x2d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x2e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x2f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x30 */ iemOp_InvalidNeedRM, iemOp_pmovzxbw_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x31 */ iemOp_InvalidNeedRM, iemOp_pmovzxbd_Vx_UxMd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x32 */ iemOp_InvalidNeedRM, iemOp_pmovzxbq_Vx_UxMw, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x33 */ iemOp_InvalidNeedRM, iemOp_pmovzxwd_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x34 */ iemOp_InvalidNeedRM, iemOp_pmovzxwq_Vx_UxMd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x35 */ iemOp_InvalidNeedRM, iemOp_pmovzxdq_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x36 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x37 */ iemOp_InvalidNeedRM, iemOp_pcmpgtq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x38 */ iemOp_InvalidNeedRM, iemOp_pminsb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x39 */ iemOp_InvalidNeedRM, iemOp_pminsd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3a */ iemOp_InvalidNeedRM, iemOp_pminuw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3b */ iemOp_InvalidNeedRM, iemOp_pminud_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3c */ iemOp_InvalidNeedRM, iemOp_pmaxsb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3d */ iemOp_InvalidNeedRM, iemOp_pmaxsd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3e */ iemOp_InvalidNeedRM, iemOp_pmaxuw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3f */ iemOp_InvalidNeedRM, iemOp_pmaxud_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0x40 */ iemOp_InvalidNeedRM, iemOp_pmulld_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x41 */ iemOp_InvalidNeedRM, iemOp_phminposuw_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x42 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x43 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x44 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x45 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x46 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x47 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x48 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x49 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x50 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x51 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x52 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x53 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x54 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x55 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x56 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x57 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x58 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x59 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x60 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x61 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x62 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x63 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x64 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x65 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x66 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x67 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x68 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x69 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x70 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x71 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x72 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x73 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x74 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x75 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x76 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x77 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x78 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x79 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x80 */ iemOp_InvalidNeedRM, iemOp_invept_Gy_Mdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x81 */ iemOp_InvalidNeedRM, iemOp_invvpid_Gy_Mdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x82 */ iemOp_InvalidNeedRM, iemOp_invpcid_Gy_Mdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x83 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x84 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x85 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x86 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x87 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x88 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x89 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x90 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x91 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x92 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x93 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x94 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x95 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x96 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x97 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x98 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x99 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xa0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xaa */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xab */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xac */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xad */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xae */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xaf */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xb0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xba */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbb */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbc */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbd */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbe */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbf */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xc0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc8 */ iemOp_sha1nexte_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xc9 */ iemOp_sha1msg1_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xca */ iemOp_sha1msg2_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xcb */ iemOp_sha256rnds2_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xcc */ iemOp_sha256msg1_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xcd */ iemOp_sha256msg2_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xce */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xcf */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xd0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xda */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xdb */ iemOp_InvalidNeedRM, iemOp_aesimc_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdc */ iemOp_InvalidNeedRM, iemOp_aesenc_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdd */ iemOp_InvalidNeedRM, iemOp_aesenclast_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xde */ iemOp_InvalidNeedRM, iemOp_aesdec_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdf */ iemOp_InvalidNeedRM, iemOp_aesdeclast_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0xe0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xea */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xeb */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xec */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xed */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xee */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xef */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xf0 */ iemOp_movbe_Gv_Mv, iemOp_movbe_Gv_Mv, iemOp_InvalidNeedRM, iemOp_crc32_Gd_Eb, + /* 0xf1 */ iemOp_movbe_Mv_Gv, iemOp_movbe_Mv_Gv, iemOp_InvalidNeedRM, iemOp_crc32_Gv_Ev, + /* 0xf2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf6 */ iemOp_InvalidNeedRM, iemOp_adcx_Gy_Ey, iemOp_adox_Gy_Ey, iemOp_InvalidNeedRM, + /* 0xf7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfa */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfb */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfc */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfd */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfe */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xff */ IEMOP_X4(iemOp_InvalidNeedRM), +}; +AssertCompile(RT_ELEMENTS(g_apfnThreeByte0f38) == 1024); + +/** @} */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h new file mode 100644 index 00000000..9809bbc8 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h @@ -0,0 +1,1549 @@ +/* $Id: IEMAllInstructionsThree0f3a.cpp.h $ */ +/** @file + * IEM - Instruction Decoding and Emulation, 0x0f 0x3a map. + * + * @remarks IEMAllInstructionsVexMap3.cpp.h is a VEX mirror of this file. + * Any update here is likely needed in that file too. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/** @name Three byte opcodes with first two bytes 0x0f 0x3a + * @{ + */ + +/** + * Common worker for SSSE3 instructions on the forms: + * pxxx xmm1, xmm2/mem128, imm8 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. SSSE3 cpuid checks. + * + * @sa iemOpCommonSse41_FullFullImm8_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSsse3_FullFullImm8_To_Full, PFNIEMAIMPLMEDIAOPTF2U128IMM8, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_SSSE3_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSSE3_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE 4.1 instructions on the forms: + * pxxx xmm1, xmm2/mem128, imm8 + * + * Proper alignment of the 128-bit operand is enforced. + * No SIMD exceptions. SSE 4.1 cpuid checks. + * + * @sa iemOpCommonSsse3_FullFullImm8_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse41_FullFullImm8_To_Full, PFNIEMAIMPLMEDIAOPTF2U128IMM8, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM, imm8 + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128], imm8. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE 4.1 instructions of the form: + * xxx xmm1, xmm2/mem128, imm8 + * + * Proper alignment of the 128-bit operand is enforced. + * MXCSR is used as input and output. + * Exceptions type 4. SSE 4.1 cpuid checks. + * + * @sa iemOpCommonSse41_FullFullImm8_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse41Fp_FullFullImm8_To_Full, FNIEMAIMPLMXCSRF2XMMIMM8, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM, imm8. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(pfnU128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128], imm8. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(Src.uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(pfnU128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE-style AES-NI instructions of the form: + * aesxxx xmm1, xmm2/mem128, imm8 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. AES-NI cpuid checks. + * + * @sa iemOpCommonSsse3_FullFullImm8_To_Full + * @sa iemOpCommonSse41_FullFullImm8_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonAesNi_FullFullImm8_To_Full, PFNIEMAIMPLMEDIAOPTF2U128IMM8, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_AESNI_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_AESNI_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x00 - invalid (vex only). */ +/** Opcode 0x66 0x0f 0x01 - invalid (vex only). */ +/** Opcode 0x66 0x0f 0x02 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x03 - invalid */ +/** Opcode 0x66 0x0f 0x04 - invalid (vex only). */ +/** Opcode 0x66 0x0f 0x05 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x06 - invalid (vex only) */ +/* Opcode 0x66 0x0f 0x07 - invalid */ +/** Opcode 0x66 0x0f 0x08. */ +FNIEMOP_DEF(iemOp_roundps_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(RMI, ROUNDPS, roundps, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse41Fp_FullFullImm8_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_roundps_u128, iemAImpl_roundps_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x09. */ +FNIEMOP_DEF(iemOp_roundpd_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(RMI, ROUNDPD, roundpd, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse41Fp_FullFullImm8_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_roundpd_u128, iemAImpl_roundpd_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x0a. */ +FNIEMOP_DEF(iemOp_roundss_Vss_Wss_Ib) +{ + /* The instruction form is very similar to CMPSS. */ + IEMOP_MNEMONIC3(RMI, ROUNDSS, roundss, Vss, Wss, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM32, XMM32. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_roundss_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_STORE_XREG_XMM_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, Dst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM32, [mem32]. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U32(Src.uSrc2, 0 /*a_iDword */, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_roundss_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_STORE_XREG_XMM_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, Dst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** Opcode 0x66 0x0f 0x0b. */ +FNIEMOP_DEF(iemOp_roundsd_Vsd_Wsd_Ib) +{ + /* The instruction form is very similar to CMPSD. */ + IEMOP_MNEMONIC3(RMI, ROUNDSD, roundsd, Vsd, Wsd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM64, XMM64, imm8. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_roundsd_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_STORE_XREG_XMM_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iQword*/, Dst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM64, [mem64], imm8. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U64(Src.uSrc2, 0 /*a_iQword */, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_roundsd_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_STORE_XREG_XMM_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iQword*/, Dst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x0c. */ +FNIEMOP_DEF(iemOp_blendps_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(RMI, BLENDPS, blendps, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFullImm8_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_blendps_u128, iemAImpl_blendps_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x0d. */ +FNIEMOP_DEF(iemOp_blendpd_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(RMI, BLENDPD, blendpd, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFullImm8_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_blendpd_u128, iemAImpl_blendpd_u128_fallback)); +} + + +/** Opcode 0x66 0x0f 0x0e. */ +FNIEMOP_DEF(iemOp_pblendw_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(RMI, PBLENDW, pblendw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse41_FullFullImm8_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pblendw_u128, iemAImpl_pblendw_u128_fallback)); +} + + +/** Opcode 0x0f 0x0f. */ +FNIEMOP_DEF(iemOp_palignr_Pq_Qq_Ib) +{ + IEMOP_MNEMONIC3(RMI, PALIGNR, palignr, Pq, Qq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */ + /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t, uSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_EX(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_FETCH_MREG_U64(uSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_palignr_u64, iemAImpl_palignr_u64_fallback), + pDst, uSrc, bImmArg); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_EX(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_palignr_u64, iemAImpl_palignr_u64_fallback), + pDst, uSrc, bImmArg); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x0f. */ +FNIEMOP_DEF(iemOp_palignr_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(RMI, PALIGNR, palignr, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFullImm8_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_palignr_u128, iemAImpl_palignr_u128_fallback)); +} + + +/* Opcode 0x66 0x0f 0x10 - invalid */ +/* Opcode 0x66 0x0f 0x11 - invalid */ +/* Opcode 0x66 0x0f 0x12 - invalid */ +/* Opcode 0x66 0x0f 0x13 - invalid */ + + +/** Opcode 0x66 0x0f 0x14. */ +FNIEMOP_DEF(iemOp_pextrb_RdMb_Vdq_Ib) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_MNEMONIC3(MRI, PEXTRB, pextrb, Ev, Vq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * greg32, XMM. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint8_t, uValue); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_AND_LOCAL_U8(bImm, 15); + IEM_MC_FETCH_XREG_U8(uValue, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iByte*/); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), uValue); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem8], XMM. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint8_t, uValue); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_AND_LOCAL_U8(bImm, 15); + IEM_MC_FETCH_XREG_U8(uValue, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iByte*/); + IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uValue); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x15. */ +FNIEMOP_DEF(iemOp_pextrw_RdMw_Vdq_Ib) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_MNEMONIC3(MRI, PEXTRW, pextrw, Ev, Vq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * greg32, XMM. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, uValue); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_AND_LOCAL_U8(bImm, 7); + IEM_MC_FETCH_XREG_U16(uValue, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iWord*/); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), uValue); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem16], XMM. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, uValue); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_AND_LOCAL_U8(bImm, 7); + IEM_MC_FETCH_XREG_U16(uValue, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iWord*/); + IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uValue); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +FNIEMOP_DEF(iemOp_pextrd_q_RdMw_Vdq_Ib) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + /** + * @opcode 0x16 + * @opcodesub rex.w=1 + * @oppfx 0x66 + * @opcpuid sse + */ + IEMOP_MNEMONIC3(MRI, PEXTRQ, pextrq, Ev, Vq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * greg64, XMM. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_AND_LOCAL_U8(bImm, 1); + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iQword*/); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem64], XMM. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_AND_LOCAL_U8(bImm, 1); + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iQword*/); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /** + * @opdone + * @opcode 0x16 + * @opcodesub rex.w=0 + * @oppfx 0x66 + * @opcpuid sse + */ + IEMOP_MNEMONIC3(MRI, PEXTRD, pextrd, Ey, Vd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * greg32, XMM. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_AND_LOCAL_U8(bImm, 3); + IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem32], XMM. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_AND_LOCAL_U8(bImm, 3); + IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode 0x66 0x0f 0x17. */ +FNIEMOP_DEF(iemOp_extractps_Ed_Vdq_Ib) +{ + IEMOP_MNEMONIC3(MRI, EXTRACTPS, extractps, Ed, Vdq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * greg32, XMM. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_AND_LOCAL_U8(bImm, 3); + IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem32], XMM. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_AND_LOCAL_U8(bImm, 3); + IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode 0x66 0x0f 0x18 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x19 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x1a - invalid */ +/* Opcode 0x66 0x0f 0x1b - invalid */ +/* Opcode 0x66 0x0f 0x1c - invalid */ +/* Opcode 0x66 0x0f 0x1d - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x1e - invalid */ +/* Opcode 0x66 0x0f 0x1f - invalid */ + + +/** Opcode 0x66 0x0f 0x20. */ +FNIEMOP_DEF(iemOp_pinsrb_Vdq_RyMb_Ib) +{ + IEMOP_MNEMONIC3(RMI, PINSRB, pinsrb, Vd, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, greg32. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint8_t, uSrc); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_GREG_U8(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_AND_LOCAL_U8(bImm, 15); + IEM_MC_STORE_XREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iByte*/, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem8]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint8_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_MEM_U8(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_AND_LOCAL_U8(bImm, 15); + IEM_MC_STORE_XREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iByte*/, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** Opcode 0x66 0x0f 0x21, */ +FNIEMOP_DEF(iemOp_insertps_Vdq_UdqMd_Ib) +{ + IEMOP_MNEMONIC3(RMI, INSERTPS, insertps, Vdq, Wdq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 3); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(uint8_t, uSrcSel); + IEM_MC_LOCAL(uint8_t, uDstSel); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_ASSIGN(uSrcSel, bImm); + IEM_MC_SHR_LOCAL_U8(uSrcSel, 6); + IEM_MC_AND_LOCAL_U8(uSrcSel, 3); + IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), uSrcSel); + IEM_MC_ASSIGN(uDstSel, bImm); + IEM_MC_SHR_LOCAL_U8(uDstSel, 4); + IEM_MC_AND_LOCAL_U8(uDstSel, 3); + IEM_MC_CLEAR_XREG_U32_MASK(IEM_GET_MODRM_REG(pVCpu, bRm), bImm); + IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), uDstSel, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem32]. + */ + IEM_MC_BEGIN(0, 3); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint8_t, uDstSel); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_ASSIGN(uDstSel, bImm); + IEM_MC_SHR_LOCAL_U8(uDstSel, 4); + IEM_MC_AND_LOCAL_U8(uDstSel, 3); + IEM_MC_CLEAR_XREG_U32_MASK(IEM_GET_MODRM_REG(pVCpu, bRm), bImm); + IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), uDstSel, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +FNIEMOP_DEF(iemOp_pinsrd_q_Vdq_Ey_Ib) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + /** + * @opcode 0x22 + * @opcodesub rex.w=1 + * @oppfx 0x66 + * @opcpuid sse + */ + IEMOP_MNEMONIC3(RMI, PINSRQ, pinsrq, Vq, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, greg64. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_GREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_AND_LOCAL_U8(bImm, 1); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iQword*/, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem64]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_AND_LOCAL_U8(bImm, 1); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iQword*/, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /** + * @opdone + * @opcode 0x22 + * @opcodesub rex.w=0 + * @oppfx 0x66 + * @opcpuid sse + */ + IEMOP_MNEMONIC3(RMI, PINSRD, pinsrd, Vd, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, greg32. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_GREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_AND_LOCAL_U8(bImm, 3); + IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem32]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_AND_LOCAL_U8(bImm, 3); + IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/, uSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/* Opcode 0x66 0x0f 0x23 - invalid */ +/* Opcode 0x66 0x0f 0x24 - invalid */ +/* Opcode 0x66 0x0f 0x25 - invalid */ +/* Opcode 0x66 0x0f 0x26 - invalid */ +/* Opcode 0x66 0x0f 0x27 - invalid */ +/* Opcode 0x66 0x0f 0x28 - invalid */ +/* Opcode 0x66 0x0f 0x29 - invalid */ +/* Opcode 0x66 0x0f 0x2a - invalid */ +/* Opcode 0x66 0x0f 0x2b - invalid */ +/* Opcode 0x66 0x0f 0x2c - invalid */ +/* Opcode 0x66 0x0f 0x2d - invalid */ +/* Opcode 0x66 0x0f 0x2e - invalid */ +/* Opcode 0x66 0x0f 0x2f - invalid */ + + +/* Opcode 0x66 0x0f 0x30 - invalid */ +/* Opcode 0x66 0x0f 0x31 - invalid */ +/* Opcode 0x66 0x0f 0x32 - invalid */ +/* Opcode 0x66 0x0f 0x33 - invalid */ +/* Opcode 0x66 0x0f 0x34 - invalid */ +/* Opcode 0x66 0x0f 0x35 - invalid */ +/* Opcode 0x66 0x0f 0x36 - invalid */ +/* Opcode 0x66 0x0f 0x37 - invalid */ +/* Opcode 0x66 0x0f 0x38 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x39 - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x3a - invalid */ +/* Opcode 0x66 0x0f 0x3b - invalid */ +/* Opcode 0x66 0x0f 0x3c - invalid */ +/* Opcode 0x66 0x0f 0x3d - invalid */ +/* Opcode 0x66 0x0f 0x3e - invalid */ +/* Opcode 0x66 0x0f 0x3f - invalid */ + + +/** Opcode 0x66 0x0f 0x40. */ +FNIEMOP_STUB(iemOp_dpps_Vx_Wx_Ib); +/** Opcode 0x66 0x0f 0x41, */ +FNIEMOP_STUB(iemOp_dppd_Vdq_Wdq_Ib); +/** Opcode 0x66 0x0f 0x42. */ +FNIEMOP_STUB(iemOp_mpsadbw_Vx_Wx_Ib); +/* Opcode 0x66 0x0f 0x43 - invalid */ + + +/** Opcode 0x66 0x0f 0x44. */ +FNIEMOP_DEF(iemOp_pclmulqdq_Vdq_Wdq_Ib) +{ + IEMOP_MNEMONIC3(RMI, PCLMULQDQ, pclmulqdq, Vdq, Wdq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_PCLMUL_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fPclMul, + iemAImpl_pclmulqdq_u128, + iemAImpl_pclmulqdq_u128_fallback), + puDst, puSrc, bImmArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_PCLMUL_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fPclMul, + iemAImpl_pclmulqdq_u128, + iemAImpl_pclmulqdq_u128_fallback), + puDst, puSrc, bImmArg); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode 0x66 0x0f 0x45 - invalid */ +/* Opcode 0x66 0x0f 0x46 - invalid (vex only) */ +/* Opcode 0x66 0x0f 0x47 - invalid */ +/* Opcode 0x66 0x0f 0x48 - invalid */ +/* Opcode 0x66 0x0f 0x49 - invalid */ +/* Opcode 0x66 0x0f 0x4a - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x4b - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x4c - invalid (vex only). */ +/* Opcode 0x66 0x0f 0x4d - invalid */ +/* Opcode 0x66 0x0f 0x4e - invalid */ +/* Opcode 0x66 0x0f 0x4f - invalid */ + + +/* Opcode 0x66 0x0f 0x50 - invalid */ +/* Opcode 0x66 0x0f 0x51 - invalid */ +/* Opcode 0x66 0x0f 0x52 - invalid */ +/* Opcode 0x66 0x0f 0x53 - invalid */ +/* Opcode 0x66 0x0f 0x54 - invalid */ +/* Opcode 0x66 0x0f 0x55 - invalid */ +/* Opcode 0x66 0x0f 0x56 - invalid */ +/* Opcode 0x66 0x0f 0x57 - invalid */ +/* Opcode 0x66 0x0f 0x58 - invalid */ +/* Opcode 0x66 0x0f 0x59 - invalid */ +/* Opcode 0x66 0x0f 0x5a - invalid */ +/* Opcode 0x66 0x0f 0x5b - invalid */ +/* Opcode 0x66 0x0f 0x5c - invalid */ +/* Opcode 0x66 0x0f 0x5d - invalid */ +/* Opcode 0x66 0x0f 0x5e - invalid */ +/* Opcode 0x66 0x0f 0x5f - invalid */ + + +/** Opcode 0x66 0x0f 0x60. */ +FNIEMOP_STUB(iemOp_pcmpestrm_Vdq_Wdq_Ib); +/** Opcode 0x66 0x0f 0x61, */ +FNIEMOP_STUB(iemOp_pcmpestri_Vdq_Wdq_Ib); +/** Opcode 0x66 0x0f 0x62. */ +FNIEMOP_STUB(iemOp_pcmpistrm_Vdq_Wdq_Ib); + + +/** Opcode 0x66 0x0f 0x63*/ +FNIEMOP_DEF(iemOp_pcmpistri_Vdq_Wdq_Ib) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 1); + IEM_MC_ARG(uint32_t *, pu32Ecx, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + IEM_MC_LOCAL(IEMPCMPISTRISRC, Src); + IEM_MC_ARG_LOCAL_REF(PIEMPCMPISTRISRC, pSrc, Src, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_SSE42_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_GREG_U32(pu32Ecx, X86_GREG_xCX); + IEM_MC_FETCH_XREG_U128(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_XREG_U128(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fSse42, + iemAImpl_pcmpistri_u128, + iemAImpl_pcmpistri_u128_fallback), + pu32Ecx, pEFlags, pSrc, bImmArg); + /** @todo testcase: High dword of RCX cleared? */ + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_ARG(uint32_t *, pu32Ecx, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + IEM_MC_LOCAL(IEMPCMPISTRISRC, Src); + IEM_MC_ARG_LOCAL_REF(PIEMPCMPISTRISRC, pSrc, Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE42_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128(Src.uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_GREG_U32(pu32Ecx, X86_GREG_xCX); + IEM_MC_FETCH_XREG_U128(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fSse42, + iemAImpl_pcmpistri_u128, + iemAImpl_pcmpistri_u128_fallback), + pu32Ecx, pEFlags, pSrc, bImmArg); + /** @todo testcase: High dword of RCX cleared? */ + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode 0x66 0x0f 0x64 - invalid */ +/* Opcode 0x66 0x0f 0x65 - invalid */ +/* Opcode 0x66 0x0f 0x66 - invalid */ +/* Opcode 0x66 0x0f 0x67 - invalid */ +/* Opcode 0x66 0x0f 0x68 - invalid */ +/* Opcode 0x66 0x0f 0x69 - invalid */ +/* Opcode 0x66 0x0f 0x6a - invalid */ +/* Opcode 0x66 0x0f 0x6b - invalid */ +/* Opcode 0x66 0x0f 0x6c - invalid */ +/* Opcode 0x66 0x0f 0x6d - invalid */ +/* Opcode 0x66 0x0f 0x6e - invalid */ +/* Opcode 0x66 0x0f 0x6f - invalid */ + +/* Opcodes 0x0f 0x70 thru 0x0f 0xb0 are unused. */ + + +/* Opcode 0x0f 0xc0 - invalid */ +/* Opcode 0x0f 0xc1 - invalid */ +/* Opcode 0x0f 0xc2 - invalid */ +/* Opcode 0x0f 0xc3 - invalid */ +/* Opcode 0x0f 0xc4 - invalid */ +/* Opcode 0x0f 0xc5 - invalid */ +/* Opcode 0x0f 0xc6 - invalid */ +/* Opcode 0x0f 0xc7 - invalid */ +/* Opcode 0x0f 0xc8 - invalid */ +/* Opcode 0x0f 0xc9 - invalid */ +/* Opcode 0x0f 0xca - invalid */ +/* Opcode 0x0f 0xcb - invalid */ +/* Opcode 0x0f 0xcc */ +FNIEMOP_STUB(iemOp_sha1rnds4_Vdq_Wdq_Ib); +/* Opcode 0x0f 0xcd - invalid */ +/* Opcode 0x0f 0xce - invalid */ +/* Opcode 0x0f 0xcf - invalid */ + + +/* Opcode 0x66 0x0f 0xd0 - invalid */ +/* Opcode 0x66 0x0f 0xd1 - invalid */ +/* Opcode 0x66 0x0f 0xd2 - invalid */ +/* Opcode 0x66 0x0f 0xd3 - invalid */ +/* Opcode 0x66 0x0f 0xd4 - invalid */ +/* Opcode 0x66 0x0f 0xd5 - invalid */ +/* Opcode 0x66 0x0f 0xd6 - invalid */ +/* Opcode 0x66 0x0f 0xd7 - invalid */ +/* Opcode 0x66 0x0f 0xd8 - invalid */ +/* Opcode 0x66 0x0f 0xd9 - invalid */ +/* Opcode 0x66 0x0f 0xda - invalid */ +/* Opcode 0x66 0x0f 0xdb - invalid */ +/* Opcode 0x66 0x0f 0xdc - invalid */ +/* Opcode 0x66 0x0f 0xdd - invalid */ +/* Opcode 0x66 0x0f 0xde - invalid */ + + +/* Opcode 0x66 0x0f 0xdf - (aeskeygenassist). */ +FNIEMOP_DEF(iemOp_aeskeygen_Vdq_Wdq_Ib) +{ + IEMOP_MNEMONIC3(RMI, AESKEYGEN, aeskeygen, Vdq, Wdq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFullImm8_To_Full, + IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aeskeygenassist_u128, iemAImpl_aeskeygenassist_u128_fallback)); +} + + +/* Opcode 0xf2 0x0f 0xf0 - invalid (vex only) */ + + +/** + * Three byte opcode map, first two bytes are 0x0f 0x3a. + * @sa g_apfnVexMap2 + */ +IEM_STATIC const PFNIEMOP g_apfnThreeByte0f3a[] = +{ + /* no prefix, 066h prefix f3h prefix, f2h prefix */ + /* 0x00 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x01 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x02 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x03 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x04 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x05 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x06 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x07 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x08 */ iemOp_InvalidNeedRMImm8, iemOp_roundps_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x09 */ iemOp_InvalidNeedRMImm8, iemOp_roundpd_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0a */ iemOp_InvalidNeedRMImm8, iemOp_roundss_Vss_Wss_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0b */ iemOp_InvalidNeedRMImm8, iemOp_roundsd_Vsd_Wsd_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0c */ iemOp_InvalidNeedRMImm8, iemOp_blendps_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0d */ iemOp_InvalidNeedRMImm8, iemOp_blendpd_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0e */ iemOp_InvalidNeedRMImm8, iemOp_pblendw_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0f */ iemOp_palignr_Pq_Qq_Ib, iemOp_palignr_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + + /* 0x10 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x11 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x12 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x13 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x14 */ iemOp_InvalidNeedRMImm8, iemOp_pextrb_RdMb_Vdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x15 */ iemOp_InvalidNeedRMImm8, iemOp_pextrw_RdMw_Vdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x16 */ iemOp_InvalidNeedRMImm8, iemOp_pextrd_q_RdMw_Vdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x17 */ iemOp_InvalidNeedRMImm8, iemOp_extractps_Ed_Vdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x18 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x19 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x20 */ iemOp_InvalidNeedRMImm8, iemOp_pinsrb_Vdq_RyMb_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x21 */ iemOp_InvalidNeedRMImm8, iemOp_insertps_Vdq_UdqMd_Ib,iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x22 */ iemOp_InvalidNeedRMImm8, iemOp_pinsrd_q_Vdq_Ey_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x23 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x24 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x25 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x26 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x27 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x28 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x29 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x30 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x31 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x32 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x33 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x34 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x35 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x36 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x37 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x38 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x39 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x40 */ iemOp_InvalidNeedRMImm8, iemOp_dpps_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x41 */ iemOp_InvalidNeedRMImm8, iemOp_dppd_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x42 */ iemOp_InvalidNeedRMImm8, iemOp_mpsadbw_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x43 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x44 */ iemOp_InvalidNeedRMImm8, iemOp_pclmulqdq_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x45 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x46 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x47 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x48 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x49 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x4a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x4b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x4c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x4d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x4e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x4f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x50 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x51 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x52 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x53 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x54 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x55 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x56 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x57 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x58 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x59 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x5a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x5b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x5c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x5d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x5e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x5f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x60 */ iemOp_InvalidNeedRMImm8, iemOp_pcmpestrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x61 */ iemOp_InvalidNeedRMImm8, iemOp_pcmpestri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x62 */ iemOp_InvalidNeedRMImm8, iemOp_pcmpistrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x63 */ iemOp_InvalidNeedRMImm8, iemOp_pcmpistri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x64 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x65 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x66 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x67 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x68 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x69 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x6a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x6b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x6c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x6d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x6e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x6f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x70 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x71 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x72 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x73 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x74 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x75 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x76 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x77 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x78 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x79 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x7a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x7b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x7c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x7d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x7e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x7f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x80 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x81 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x82 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x83 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x84 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x85 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x86 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x87 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x88 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x89 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x90 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x91 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x92 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x93 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x94 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x95 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x96 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x97 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x98 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x99 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xa0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xaa */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xab */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xac */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xad */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xae */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xaf */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xb0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xba */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbc */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbd */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbe */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbf */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xc0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xca */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xcb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xcc */ iemOp_sha1rnds4_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0xcd */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xce */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xcf */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xd0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xda */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xdb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xdc */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xdd */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xde */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xdf */ iemOp_InvalidNeedRMImm8, iemOp_aeskeygen_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + + /* 0xe0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xea */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xeb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xec */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xed */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xee */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xef */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xf0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfa */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfc */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfd */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfe */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xff */ IEMOP_X4(iemOp_InvalidNeedRMImm8), +}; +AssertCompile(RT_ELEMENTS(g_apfnThreeByte0f3a) == 1024); + +/** @} */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h new file mode 100644 index 00000000..25f2cc9f --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h @@ -0,0 +1,13975 @@ +/* $Id: IEMAllInstructionsTwoByte0f.cpp.h $ */ +/** @file + * IEM - Instruction Decoding and Emulation. + * + * @remarks IEMAllInstructionsVexMap1.cpp.h is a VEX mirror of this file. + * Any update here is likely needed in that file too. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/** @name Two byte opcodes (first byte 0x0f). + * + * @{ + */ + + +/** + * Common worker for MMX instructions on the form: + * pxxx mm1, mm2/mem64 + */ +FNIEMOP_DEF_1(iemOpCommonMmx_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U64, pfnU64) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * MMX, MMX. + */ + /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */ + /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t const *, pSrc, 1); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * MMX, [mem64]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_ARG_LOCAL_REF(uint64_t const *, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for MMX instructions on the form: + * pxxx mm1, mm2/mem64 + * + * Unlike iemOpCommonMmx_FullFull_To_Full, the @a pfnU64 worker function takes + * no FXSAVE state, just the operands. + */ +FNIEMOP_DEF_1(iemOpCommonMmxOpt_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U64, pfnU64) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * MMX, MMX. + */ + /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */ + /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t const *, pSrc, 1); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * MMX, [mem64]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_ARG_LOCAL_REF(uint64_t const *, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for MMX instructions on the form: + * pxxx mm1, mm2/mem64 + * for instructions introduced with SSE. + */ +FNIEMOP_DEF_1(iemOpCommonMmxSse_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U64, pfnU64) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * MMX, MMX. + */ + /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */ + /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t const *, pSrc, 1); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * MMX, [mem64]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_ARG_LOCAL_REF(uint64_t const *, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for MMX instructions on the form: + * pxxx mm1, mm2/mem64 + * for instructions introduced with SSE. + * + * Unlike iemOpCommonMmxSse_FullFull_To_Full, the @a pfnU64 worker function takes + * no FXSAVE state, just the operands. + */ +FNIEMOP_DEF_1(iemOpCommonMmxSseOpt_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U64, pfnU64) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * MMX, MMX. + */ + /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */ + /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t const *, pSrc, 1); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * MMX, [mem64]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_ARG_LOCAL_REF(uint64_t const *, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for MMX instructions on the form: + * pxxx mm1, mm2/mem64 + * that was introduced with SSE2. + */ +FNIEMOP_DEF_2(iemOpCommonMmx_FullFull_To_Full_Ex, PFNIEMAIMPLMEDIAF2U64, pfnU64, bool, fSupported) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * MMX, MMX. + */ + /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */ + /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t const *, pSrc, 1); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_EX(fSupported); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * MMX, [mem64]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_ARG_LOCAL_REF(uint64_t const *, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_EX(fSupported); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE instructions of the form: + * pxxx xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * SSE cpuid checks. No SIMD FP exceptions. + * + * @sa iemOpCommonSse2_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_ARG(PCRTUINT128U, pSrc, 1); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, pDst, pSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, pDst, pSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE2 instructions on the forms: + * pxxx xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. SSE2 cpuid checks. + * + * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse2_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_ARG(PCRTUINT128U, pSrc, 1); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, pDst, pSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_2(pfnU128, pDst, pSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE2 instructions on the forms: + * pxxx xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. SSE2 cpuid checks. + * + * Unlike iemOpCommonSse2_FullFull_To_Full, the @a pfnU128 worker function takes + * no FXSAVE state, just the operands. + * + * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse2Opt_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_ARG(PCRTUINT128U, pSrc, 1); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, pDst, pSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, pDst, pSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for MMX instructions on the forms: + * pxxxx mm1, mm2/mem32 + * + * The 2nd operand is the first half of a register, which in the memory case + * means a 32-bit memory access. + */ +FNIEMOP_DEF_1(iemOpCommonMmx_LowLow_To_Full, FNIEMAIMPLMEDIAOPTF2U64, pfnU64) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * MMX, MMX. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, puDst, 0); + IEM_MC_ARG(uint64_t const *, puSrc, 1); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(puDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_REF_MREG_U64_CONST(puSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU64, puDst, puSrc); + IEM_MC_MODIFIED_MREG_BY_REF(puDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * MMX, [mem32]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint64_t *, puDst, 0); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_ARG_LOCAL_REF(uint64_t const *, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U32_ZX_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(puDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU64, puDst, puSrc); + IEM_MC_MODIFIED_MREG_BY_REF(puDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE instructions on the forms: + * pxxxx xmm1, xmm2/mem128 + * + * The 2nd operand is the first half of a register, which in the memory case + * 128-bit aligned 64-bit or 128-bit memory accessed for SSE. + * + * Exceptions type 4. + */ +FNIEMOP_DEF_1(iemOpCommonSse_LowLow_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + /** @todo Most CPUs probably only read the low qword. We read everything to + * make sure we apply segmentation and alignment checks correctly. + * When we have time, it would be interesting to explore what real + * CPUs actually does and whether it will do a TLB load for the high + * part or skip any associated \#PF. Ditto for segmentation \#GPs. */ + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE2 instructions on the forms: + * pxxxx xmm1, xmm2/mem128 + * + * The 2nd operand is the first half of a register, which in the memory case + * 128-bit aligned 64-bit or 128-bit memory accessed for SSE. + * + * Exceptions type 4. + */ +FNIEMOP_DEF_1(iemOpCommonSse2_LowLow_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + /** @todo Most CPUs probably only read the low qword. We read everything to + * make sure we apply segmentation and alignment checks correctly. + * When we have time, it would be interesting to explore what real + * CPUs actually does and whether it will do a TLB load for the high + * part or skip any associated \#PF. Ditto for segmentation \#GPs. */ + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for MMX instructions on the form: + * pxxxx mm1, mm2/mem64 + * + * The 2nd operand is the second half of a register, which in the memory case + * means a 64-bit memory access for MMX. + */ +FNIEMOP_DEF_1(iemOpCommonMmx_HighHigh_To_Full, PFNIEMAIMPLMEDIAOPTF2U64, pfnU64) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * MMX, MMX. + */ + /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */ + /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, puDst, 0); + IEM_MC_ARG(uint64_t const *, puSrc, 1); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(puDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_REF_MREG_U64_CONST(puSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU64, puDst, puSrc); + IEM_MC_MODIFIED_MREG_BY_REF(puDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * MMX, [mem64]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(uint64_t *, puDst, 0); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_ARG_LOCAL_REF(uint64_t const *, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); /* intel docs this to be full 64-bit read */ + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(puDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU64, puDst, puSrc); + IEM_MC_MODIFIED_MREG_BY_REF(puDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE instructions on the form: + * pxxxx xmm1, xmm2/mem128 + * + * The 2nd operand is the second half of a register, which for SSE a 128-bit + * aligned access where it may read the full 128 bits or only the upper 64 bits. + * + * Exceptions type 4. + */ +FNIEMOP_DEF_1(iemOpCommonSse_HighHigh_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + /** @todo Most CPUs probably only read the high qword. We read everything to + * make sure we apply segmentation and alignment checks correctly. + * When we have time, it would be interesting to explore what real + * CPUs actually does and whether it will do a TLB load for the lower + * part or skip any associated \#PF. */ + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE instructions on the forms: + * pxxs xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 2. SSE cpuid checks. + * + * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSseFp_FullFull_To_Full, PFNIEMAIMPLFPSSEF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM128. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG(PCX86XMMREG, pSrc2, 2); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem128]. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, pSrc2, uSrc2, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE instructions on the forms: + * pxxs xmm1, xmm2/mem32 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 2. SSE cpuid checks. + * + * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSseFp_FullR32_To_Full, PFNIEMAIMPLFPSSEF2U128R32, pfnU128_R32) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM32. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG(PCRTFLOAT32U, pSrc2, 2); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_R32_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128_R32, pSseRes, pSrc1, pSrc2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem32]. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_LOCAL(RTFLOAT32U, r32Src2); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U, pr32Src2, r32Src2, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_FETCH_MEM_R32(r32Src2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128_R32, pSseRes, pSrc1, pr32Src2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE2 instructions on the forms: + * pxxd xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 2. SSE cpuid checks. + * + * @sa iemOpCommonSseFp_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse2Fp_FullFull_To_Full, PFNIEMAIMPLFPSSEF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM128. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG(PCX86XMMREG, pSrc2, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem128]. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, pSrc2, uSrc2, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE2 instructions on the forms: + * pxxs xmm1, xmm2/mem64 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 2. SSE2 cpuid checks. + * + * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse2Fp_FullR64_To_Full, PFNIEMAIMPLFPSSEF2U128R64, pfnU128_R64) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG(PCRTFLOAT64U, pSrc2, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_R64_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128_R64, pSseRes, pSrc1, pSrc2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem64]. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_LOCAL(RTFLOAT64U, r64Src2); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U, pr64Src2, r64Src2, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_R64(r64Src2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128_R64, pSseRes, pSrc1, pr64Src2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE2 instructions on the form: + * pxxxx xmm1, xmm2/mem128 + * + * The 2nd operand is the second half of a register, which for SSE a 128-bit + * aligned access where it may read the full 128 bits or only the upper 64 bits. + * + * Exceptions type 4. + */ +FNIEMOP_DEF_1(iemOpCommonSse2_HighHigh_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + /** @todo Most CPUs probably only read the high qword. We read everything to + * make sure we apply segmentation and alignment checks correctly. + * When we have time, it would be interesting to explore what real + * CPUs actually does and whether it will do a TLB load for the lower + * part or skip any associated \#PF. */ + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE3 instructions on the forms: + * hxxx xmm1, xmm2/mem128 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 2. SSE3 cpuid checks. + * + * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full + */ +FNIEMOP_DEF_1(iemOpCommonSse3Fp_FullFull_To_Full, PFNIEMAIMPLFPSSEF2U128, pfnU128) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG(PCX86XMMREG, pSrc2, 2); + IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(IEMSSERESULT, SseRes); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes, SseRes, 0); + IEM_MC_ARG(PCX86XMMREG, pSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, pSrc2, uSrc2, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2); + IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x00 /0. */ +FNIEMOPRM_DEF(iemOp_Grp6_sldt) +{ + IEMOP_MNEMONIC(sldt, "sldt Rv/Mw"); + IEMOP_HLP_MIN_286(); + IEMOP_HLP_NO_REAL_OR_V86_MODE(); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DECODED_NL_1(OP_SLDT, IEMOPFORM_M_REG, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_sldt_reg, IEM_GET_MODRM_RM(pVCpu, bRm), pVCpu->iem.s.enmEffOpSize); + } + + /* Ignore operand size here, memory refs are always 16-bit. */ + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint16_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DECODED_NL_1(OP_SLDT, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_sldt_mem, iEffSeg, GCPtrEffDst); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x00 /1. */ +FNIEMOPRM_DEF(iemOp_Grp6_str) +{ + IEMOP_MNEMONIC(str, "str Rv/Mw"); + IEMOP_HLP_MIN_286(); + IEMOP_HLP_NO_REAL_OR_V86_MODE(); + + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DECODED_NL_1(OP_STR, IEMOPFORM_M_REG, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_str_reg, IEM_GET_MODRM_RM(pVCpu, bRm), pVCpu->iem.s.enmEffOpSize); + } + + /* Ignore operand size here, memory refs are always 16-bit. */ + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint16_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DECODED_NL_1(OP_STR, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_str_mem, iEffSeg, GCPtrEffDst); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x00 /2. */ +FNIEMOPRM_DEF(iemOp_Grp6_lldt) +{ + IEMOP_MNEMONIC(lldt, "lldt Ew"); + IEMOP_HLP_MIN_286(); + IEMOP_HLP_NO_REAL_OR_V86_MODE(); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DECODED_NL_1(OP_LLDT, IEMOPFORM_M_REG, OP_PARM_Ew, DISOPTYPE_DANGEROUS); + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint16_t, u16Sel, 0); + IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_1(iemCImpl_lldt, u16Sel); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(1, 1); + IEM_MC_ARG(uint16_t, u16Sel, 0); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DECODED_NL_1(OP_LLDT, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS); + IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO(); /** @todo test order */ + IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_CIMPL_1(iemCImpl_lldt, u16Sel); + IEM_MC_END(); + } + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x00 /3. */ +FNIEMOPRM_DEF(iemOp_Grp6_ltr) +{ + IEMOP_MNEMONIC(ltr, "ltr Ew"); + IEMOP_HLP_MIN_286(); + IEMOP_HLP_NO_REAL_OR_V86_MODE(); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint16_t, u16Sel, 0); + IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_1(iemCImpl_ltr, u16Sel); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(1, 1); + IEM_MC_ARG(uint16_t, u16Sel, 0); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO(); /** @todo test order */ + IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_CIMPL_1(iemCImpl_ltr, u16Sel); + IEM_MC_END(); + } + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x00 /3. */ +FNIEMOP_DEF_2(iemOpCommonGrp6VerX, uint8_t, bRm, bool, fWrite) +{ + IEMOP_HLP_MIN_286(); + IEMOP_HLP_NO_REAL_OR_V86_MODE(); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DECODED_NL_1(fWrite ? OP_VERW : OP_VERR, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint16_t, u16Sel, 0); + IEM_MC_ARG_CONST(bool, fWriteArg, fWrite, 1); + IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_2(iemCImpl_VerX, u16Sel, fWriteArg); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(2, 1); + IEM_MC_ARG(uint16_t, u16Sel, 0); + IEM_MC_ARG_CONST(bool, fWriteArg, fWrite, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DECODED_NL_1(fWrite ? OP_VERW : OP_VERR, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP); + IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_CIMPL_2(iemCImpl_VerX, u16Sel, fWriteArg); + IEM_MC_END(); + } + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x00 /4. */ +FNIEMOPRM_DEF(iemOp_Grp6_verr) +{ + IEMOP_MNEMONIC(verr, "verr Ew"); + IEMOP_HLP_MIN_286(); + return FNIEMOP_CALL_2(iemOpCommonGrp6VerX, bRm, false); +} + + +/** Opcode 0x0f 0x00 /5. */ +FNIEMOPRM_DEF(iemOp_Grp6_verw) +{ + IEMOP_MNEMONIC(verw, "verw Ew"); + IEMOP_HLP_MIN_286(); + return FNIEMOP_CALL_2(iemOpCommonGrp6VerX, bRm, true); +} + + +/** + * Group 6 jump table. + */ +IEM_STATIC const PFNIEMOPRM g_apfnGroup6[8] = +{ + iemOp_Grp6_sldt, + iemOp_Grp6_str, + iemOp_Grp6_lldt, + iemOp_Grp6_ltr, + iemOp_Grp6_verr, + iemOp_Grp6_verw, + iemOp_InvalidWithRM, + iemOp_InvalidWithRM +}; + +/** Opcode 0x0f 0x00. */ +FNIEMOP_DEF(iemOp_Grp6) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + return FNIEMOP_CALL_1(g_apfnGroup6[IEM_GET_MODRM_REG_8(bRm)], bRm); +} + + +/** Opcode 0x0f 0x01 /0. */ +FNIEMOP_DEF_1(iemOp_Grp7_sgdt, uint8_t, bRm) +{ + IEMOP_MNEMONIC(sgdt, "sgdt Ms"); + IEMOP_HLP_MIN_286(); + IEMOP_HLP_64BIT_OP_SIZE(); + IEM_MC_BEGIN(2, 1); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_sgdt, iEffSeg, GCPtrEffSrc); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x01 /0. */ +FNIEMOP_DEF(iemOp_Grp7_vmcall) +{ + IEMOP_MNEMONIC(vmcall, "vmcall"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the VMX instructions. ASSUMING no lock for now. */ + + /* Note! We do not check any CPUMFEATURES::fSvm here as we (GIM) generally + want all hypercalls regardless of instruction used, and if a + hypercall isn't handled by GIM or HMSvm will raise an #UD. + (NEM/win makes ASSUMPTIONS about this behavior.) */ + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmcall); +} + + +/** Opcode 0x0f 0x01 /0. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF(iemOp_Grp7_vmlaunch) +{ + IEMOP_MNEMONIC(vmlaunch, "vmlaunch"); + IEMOP_HLP_IN_VMX_OPERATION("vmlaunch", kVmxVDiag_Vmentry); + IEMOP_HLP_VMX_INSTR("vmlaunch", kVmxVDiag_Vmentry); + IEMOP_HLP_DONE_DECODING(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmlaunch); +} +#else +FNIEMOP_DEF(iemOp_Grp7_vmlaunch) +{ + IEMOP_BITCH_ABOUT_STUB(); + return IEMOP_RAISE_INVALID_OPCODE(); +} +#endif + + +/** Opcode 0x0f 0x01 /0. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF(iemOp_Grp7_vmresume) +{ + IEMOP_MNEMONIC(vmresume, "vmresume"); + IEMOP_HLP_IN_VMX_OPERATION("vmresume", kVmxVDiag_Vmentry); + IEMOP_HLP_VMX_INSTR("vmresume", kVmxVDiag_Vmentry); + IEMOP_HLP_DONE_DECODING(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmresume); +} +#else +FNIEMOP_DEF(iemOp_Grp7_vmresume) +{ + IEMOP_BITCH_ABOUT_STUB(); + return IEMOP_RAISE_INVALID_OPCODE(); +} +#endif + + +/** Opcode 0x0f 0x01 /0. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF(iemOp_Grp7_vmxoff) +{ + IEMOP_MNEMONIC(vmxoff, "vmxoff"); + IEMOP_HLP_IN_VMX_OPERATION("vmxoff", kVmxVDiag_Vmxoff); + IEMOP_HLP_VMX_INSTR("vmxoff", kVmxVDiag_Vmxoff); + IEMOP_HLP_DONE_DECODING(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmxoff); +} +#else +FNIEMOP_DEF(iemOp_Grp7_vmxoff) +{ + IEMOP_BITCH_ABOUT_STUB(); + return IEMOP_RAISE_INVALID_OPCODE(); +} +#endif + + +/** Opcode 0x0f 0x01 /1. */ +FNIEMOP_DEF_1(iemOp_Grp7_sidt, uint8_t, bRm) +{ + IEMOP_MNEMONIC(sidt, "sidt Ms"); + IEMOP_HLP_MIN_286(); + IEMOP_HLP_64BIT_OP_SIZE(); + IEM_MC_BEGIN(2, 1); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_sidt, iEffSeg, GCPtrEffSrc); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x01 /1. */ +FNIEMOP_DEF(iemOp_Grp7_monitor) +{ + IEMOP_MNEMONIC(monitor, "monitor"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo Verify that monitor is allergic to lock prefixes. */ + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_monitor, pVCpu->iem.s.iEffSeg); +} + + +/** Opcode 0x0f 0x01 /1. */ +FNIEMOP_DEF(iemOp_Grp7_mwait) +{ + IEMOP_MNEMONIC(mwait, "mwait"); /** @todo Verify that mwait is allergic to lock prefixes. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_mwait); +} + + +/** Opcode 0x0f 0x01 /2. */ +FNIEMOP_DEF_1(iemOp_Grp7_lgdt, uint8_t, bRm) +{ + IEMOP_MNEMONIC(lgdt, "lgdt"); + IEMOP_HLP_64BIT_OP_SIZE(); + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSizeArg,/*=*/pVCpu->iem.s.enmEffOpSize, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_lgdt, iEffSeg, GCPtrEffSrc, enmEffOpSizeArg); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x01 0xd0. */ +FNIEMOP_DEF(iemOp_Grp7_xgetbv) +{ + IEMOP_MNEMONIC(xgetbv, "xgetbv"); + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor) + { + /** @todo r=ramshankar: We should use + * IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX and + * IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES here. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_xgetbv); + } + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode 0x0f 0x01 0xd1. */ +FNIEMOP_DEF(iemOp_Grp7_xsetbv) +{ + IEMOP_MNEMONIC(xsetbv, "xsetbv"); + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor) + { + /** @todo r=ramshankar: We should use + * IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX and + * IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES here. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_xsetbv); + } + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode 0x0f 0x01 /3. */ +FNIEMOP_DEF_1(iemOp_Grp7_lidt, uint8_t, bRm) +{ + IEMOP_MNEMONIC(lidt, "lidt"); + IEMMODE enmEffOpSize = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT + ? IEMMODE_64BIT + : pVCpu->iem.s.enmEffOpSize; + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSizeArg,/*=*/enmEffOpSize, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_lidt, iEffSeg, GCPtrEffSrc, enmEffOpSizeArg); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x01 0xd8. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +FNIEMOP_DEF(iemOp_Grp7_Amd_vmrun) +{ + IEMOP_MNEMONIC(vmrun, "vmrun"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */ + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmrun); +} +#else +FNIEMOP_UD_STUB(iemOp_Grp7_Amd_vmrun); +#endif + +/** Opcode 0x0f 0x01 0xd9. */ +FNIEMOP_DEF(iemOp_Grp7_Amd_vmmcall) +{ + IEMOP_MNEMONIC(vmmcall, "vmmcall"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */ + + /* Note! We do not check any CPUMFEATURES::fSvm here as we (GIM) generally + want all hypercalls regardless of instruction used, and if a + hypercall isn't handled by GIM or HMSvm will raise an #UD. + (NEM/win makes ASSUMPTIONS about this behavior.) */ + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmmcall); +} + +/** Opcode 0x0f 0x01 0xda. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +FNIEMOP_DEF(iemOp_Grp7_Amd_vmload) +{ + IEMOP_MNEMONIC(vmload, "vmload"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */ + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmload); +} +#else +FNIEMOP_UD_STUB(iemOp_Grp7_Amd_vmload); +#endif + + +/** Opcode 0x0f 0x01 0xdb. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +FNIEMOP_DEF(iemOp_Grp7_Amd_vmsave) +{ + IEMOP_MNEMONIC(vmsave, "vmsave"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */ + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmsave); +} +#else +FNIEMOP_UD_STUB(iemOp_Grp7_Amd_vmsave); +#endif + + +/** Opcode 0x0f 0x01 0xdc. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +FNIEMOP_DEF(iemOp_Grp7_Amd_stgi) +{ + IEMOP_MNEMONIC(stgi, "stgi"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */ + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stgi); +} +#else +FNIEMOP_UD_STUB(iemOp_Grp7_Amd_stgi); +#endif + + +/** Opcode 0x0f 0x01 0xdd. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +FNIEMOP_DEF(iemOp_Grp7_Amd_clgi) +{ + IEMOP_MNEMONIC(clgi, "clgi"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */ + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_clgi); +} +#else +FNIEMOP_UD_STUB(iemOp_Grp7_Amd_clgi); +#endif + + +/** Opcode 0x0f 0x01 0xdf. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +FNIEMOP_DEF(iemOp_Grp7_Amd_invlpga) +{ + IEMOP_MNEMONIC(invlpga, "invlpga"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */ + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_invlpga); +} +#else +FNIEMOP_UD_STUB(iemOp_Grp7_Amd_invlpga); +#endif + + +/** Opcode 0x0f 0x01 0xde. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM +FNIEMOP_DEF(iemOp_Grp7_Amd_skinit) +{ + IEMOP_MNEMONIC(skinit, "skinit"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */ + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_skinit); +} +#else +FNIEMOP_UD_STUB(iemOp_Grp7_Amd_skinit); +#endif + + +/** Opcode 0x0f 0x01 /4. */ +FNIEMOP_DEF_1(iemOp_Grp7_smsw, uint8_t, bRm) +{ + IEMOP_MNEMONIC(smsw, "smsw"); + IEMOP_HLP_MIN_286(); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_smsw_reg, IEM_GET_MODRM_RM(pVCpu, bRm), pVCpu->iem.s.enmEffOpSize); + } + + /* Ignore operand size here, memory refs are always 16-bit. */ + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint16_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_smsw_mem, iEffSeg, GCPtrEffDst); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x01 /6. */ +FNIEMOP_DEF_1(iemOp_Grp7_lmsw, uint8_t, bRm) +{ + /* The operand size is effectively ignored, all is 16-bit and only the + lower 3-bits are used. */ + IEMOP_MNEMONIC(lmsw, "lmsw"); + IEMOP_HLP_MIN_286(); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint16_t, u16Tmp, 0); + IEM_MC_ARG_CONST(RTGCPTR, GCPtrEffDst, NIL_RTGCPTR, 1); + IEM_MC_FETCH_GREG_U16(u16Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_2(iemCImpl_lmsw, u16Tmp, GCPtrEffDst); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint16_t, u16Tmp, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_CALL_CIMPL_2(iemCImpl_lmsw, u16Tmp, GCPtrEffDst); + IEM_MC_END(); + } + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x01 /7. */ +FNIEMOP_DEF_1(iemOp_Grp7_invlpg, uint8_t, bRm) +{ + IEMOP_MNEMONIC(invlpg, "invlpg"); + IEMOP_HLP_MIN_486(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(1, 1); + IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 0); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_CALL_CIMPL_1(iemCImpl_invlpg, GCPtrEffDst); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0x01 0xf8. */ +FNIEMOP_DEF(iemOp_Grp7_swapgs) +{ + IEMOP_MNEMONIC(swapgs, "swapgs"); + IEMOP_HLP_ONLY_64BIT(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_swapgs); +} + + +/** Opcode 0x0f 0x01 0xf9. */ +FNIEMOP_DEF(iemOp_Grp7_rdtscp) +{ + IEMOP_MNEMONIC(rdtscp, "rdtscp"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdtscp); +} + + +/** + * Group 7 jump table, memory variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnGroup7Mem[8] = +{ + iemOp_Grp7_sgdt, + iemOp_Grp7_sidt, + iemOp_Grp7_lgdt, + iemOp_Grp7_lidt, + iemOp_Grp7_smsw, + iemOp_InvalidWithRM, + iemOp_Grp7_lmsw, + iemOp_Grp7_invlpg +}; + + +/** Opcode 0x0f 0x01. */ +FNIEMOP_DEF(iemOp_Grp7) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + return FNIEMOP_CALL_1(g_apfnGroup7Mem[IEM_GET_MODRM_REG_8(bRm)], bRm); + + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: + switch (IEM_GET_MODRM_RM_8(bRm)) + { + case 1: return FNIEMOP_CALL(iemOp_Grp7_vmcall); + case 2: return FNIEMOP_CALL(iemOp_Grp7_vmlaunch); + case 3: return FNIEMOP_CALL(iemOp_Grp7_vmresume); + case 4: return FNIEMOP_CALL(iemOp_Grp7_vmxoff); + } + return IEMOP_RAISE_INVALID_OPCODE(); + + case 1: + switch (IEM_GET_MODRM_RM_8(bRm)) + { + case 0: return FNIEMOP_CALL(iemOp_Grp7_monitor); + case 1: return FNIEMOP_CALL(iemOp_Grp7_mwait); + } + return IEMOP_RAISE_INVALID_OPCODE(); + + case 2: + switch (IEM_GET_MODRM_RM_8(bRm)) + { + case 0: return FNIEMOP_CALL(iemOp_Grp7_xgetbv); + case 1: return FNIEMOP_CALL(iemOp_Grp7_xsetbv); + } + return IEMOP_RAISE_INVALID_OPCODE(); + + case 3: + switch (IEM_GET_MODRM_RM_8(bRm)) + { + case 0: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmrun); + case 1: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmmcall); + case 2: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmload); + case 3: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmsave); + case 4: return FNIEMOP_CALL(iemOp_Grp7_Amd_stgi); + case 5: return FNIEMOP_CALL(iemOp_Grp7_Amd_clgi); + case 6: return FNIEMOP_CALL(iemOp_Grp7_Amd_skinit); + case 7: return FNIEMOP_CALL(iemOp_Grp7_Amd_invlpga); + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + case 4: + return FNIEMOP_CALL_1(iemOp_Grp7_smsw, bRm); + + case 5: + return IEMOP_RAISE_INVALID_OPCODE(); + + case 6: + return FNIEMOP_CALL_1(iemOp_Grp7_lmsw, bRm); + + case 7: + switch (IEM_GET_MODRM_RM_8(bRm)) + { + case 0: return FNIEMOP_CALL(iemOp_Grp7_swapgs); + case 1: return FNIEMOP_CALL(iemOp_Grp7_rdtscp); + } + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + +/** Opcode 0x0f 0x00 /3. */ +FNIEMOP_DEF_1(iemOpCommonLarLsl_Gv_Ew, bool, fIsLar) +{ + IEMOP_HLP_NO_REAL_OR_V86_MODE(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DECODED_NL_2(fIsLar ? OP_LAR : OP_LSL, IEMOPFORM_RM_REG, OP_PARM_Gv, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Sel, 1); + IEM_MC_ARG_CONST(bool, fIsLarArg, fIsLar, 2); + + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u16, pu16Dst, u16Sel, fIsLarArg); + + IEM_MC_END(); + return VINF_SUCCESS; + } + + case IEMMODE_32BIT: + case IEMMODE_64BIT: + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint16_t, u16Sel, 1); + IEM_MC_ARG_CONST(bool, fIsLarArg, fIsLar, 2); + + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u64, pu64Dst, u16Sel, fIsLarArg); + + IEM_MC_END(); + return VINF_SUCCESS; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + { + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Sel, 1); + IEM_MC_ARG_CONST(bool, fIsLarArg, fIsLar, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DECODED_NL_2(fIsLar ? OP_LAR : OP_LSL, IEMOPFORM_RM_MEM, OP_PARM_Gv, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP); + + IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u16, pu16Dst, u16Sel, fIsLarArg); + + IEM_MC_END(); + return VINF_SUCCESS; + } + + case IEMMODE_32BIT: + case IEMMODE_64BIT: + { + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint16_t, u16Sel, 1); + IEM_MC_ARG_CONST(bool, fIsLarArg, fIsLar, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DECODED_NL_2(fIsLar ? OP_LAR : OP_LSL, IEMOPFORM_RM_MEM, OP_PARM_Gv, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP); +/** @todo testcase: make sure it's a 16-bit read. */ + + IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u64, pu64Dst, u16Sel, fIsLarArg); + + IEM_MC_END(); + return VINF_SUCCESS; + } + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + + +/** Opcode 0x0f 0x02. */ +FNIEMOP_DEF(iemOp_lar_Gv_Ew) +{ + IEMOP_MNEMONIC(lar, "lar Gv,Ew"); + return FNIEMOP_CALL_1(iemOpCommonLarLsl_Gv_Ew, true); +} + + +/** Opcode 0x0f 0x03. */ +FNIEMOP_DEF(iemOp_lsl_Gv_Ew) +{ + IEMOP_MNEMONIC(lsl, "lsl Gv,Ew"); + return FNIEMOP_CALL_1(iemOpCommonLarLsl_Gv_Ew, false); +} + + +/** Opcode 0x0f 0x05. */ +FNIEMOP_DEF(iemOp_syscall) +{ + IEMOP_MNEMONIC(syscall, "syscall"); /** @todo 286 LOADALL */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_syscall); +} + + +/** Opcode 0x0f 0x06. */ +FNIEMOP_DEF(iemOp_clts) +{ + IEMOP_MNEMONIC(clts, "clts"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_clts); +} + + +/** Opcode 0x0f 0x07. */ +FNIEMOP_DEF(iemOp_sysret) +{ + IEMOP_MNEMONIC(sysret, "sysret"); /** @todo 386 LOADALL */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_sysret); +} + + +/** Opcode 0x0f 0x08. */ +FNIEMOP_DEF(iemOp_invd) +{ + IEMOP_MNEMONIC0(FIXED, INVD, invd, DISOPTYPE_PRIVILEGED, 0); + IEMOP_HLP_MIN_486(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_invd); +} + + +/** Opcode 0x0f 0x09. */ +FNIEMOP_DEF(iemOp_wbinvd) +{ + IEMOP_MNEMONIC0(FIXED, WBINVD, wbinvd, DISOPTYPE_PRIVILEGED, 0); + IEMOP_HLP_MIN_486(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_wbinvd); +} + + +/** Opcode 0x0f 0x0b. */ +FNIEMOP_DEF(iemOp_ud2) +{ + IEMOP_MNEMONIC(ud2, "ud2"); + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/** Opcode 0x0f 0x0d. */ +FNIEMOP_DEF(iemOp_nop_Ev_GrpP) +{ + /* AMD prefetch group, Intel implements this as NOP Ev (and so do we). */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->f3DNowPrefetch) + { + IEMOP_MNEMONIC(GrpPNotSupported, "GrpP"); + return IEMOP_RAISE_INVALID_OPCODE(); + } + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_MNEMONIC(GrpPInvalid, "GrpP"); + return IEMOP_RAISE_INVALID_OPCODE(); + } + + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 2: /* Aliased to /0 for the time being. */ + case 4: /* Aliased to /0 for the time being. */ + case 5: /* Aliased to /0 for the time being. */ + case 6: /* Aliased to /0 for the time being. */ + case 7: /* Aliased to /0 for the time being. */ + case 0: IEMOP_MNEMONIC(prefetch, "prefetch"); break; + case 1: IEMOP_MNEMONIC(prefetchw_1, "prefetchw"); break; + case 3: IEMOP_MNEMONIC(prefetchw_3, "prefetchw"); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + /* Currently a NOP. */ + NOREF(GCPtrEffSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0x0f 0x0e. */ +FNIEMOP_DEF(iemOp_femms) +{ + IEMOP_MNEMONIC(femms, "femms"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0,0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_FROM_MMX_MODE(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0x0f 0x0f. */ +FNIEMOP_DEF(iemOp_3Dnow) +{ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->f3DNow) + { + IEMOP_MNEMONIC(Inv3Dnow, "3Dnow"); + return IEMOP_RAISE_INVALID_OPCODE(); + } + +#ifdef IEM_WITH_3DNOW + /* This is pretty sparse, use switch instead of table. */ + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL_1(iemOp_3DNowDispatcher, b); +#else + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +#endif +} + + +/** + * @opcode 0x10 + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ +FNIEMOP_DEF(iemOp_movups_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, MOVUPS, movups, Vps_WO, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM128. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem128]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + +} + + +/** + * @opcode 0x10 + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movupd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, MOVUPD, movupd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM128. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem128]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x10 + * @oppfx 0xf3 + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ +FNIEMOP_DEF(iemOp_movss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, MOVSS, movss, VssZx_WO, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM32, XMM32. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /*a_iDword*/ ); + IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem32]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U32_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x10 + * @oppfx 0xf2 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movsd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, MOVSD, movsd, VsdZx_WO, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM64, XMM64. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem64]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x11 + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movups_Wps_Vps) +{ + IEMOP_MNEMONIC2(MR, MOVUPS, movups, Wps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM128. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem128], XMM128. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x11 + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movupd_Wpd_Vpd) +{ + IEMOP_MNEMONIC2(MR, MOVUPD, movupd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM128. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem128], XMM128. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x11 + * @oppfx 0xf3 + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ +FNIEMOP_DEF(iemOp_movss_Wss_Vss) +{ + IEMOP_MNEMONIC2(MR, MOVSS, movss, Wss_WO, Vss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM32, XMM32. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/); + IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), 0 /*a_iDword*/, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem32], XMM32. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x11 + * @oppfx 0xf2 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movsd_Wsd_Vsd) +{ + IEMOP_MNEMONIC2(MR, MOVSD, movsd, Wsd_WO, Vsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM64, XMM64. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem64], XMM64. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +FNIEMOP_DEF(iemOp_movlps_Vq_Mq__movhlps) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /** + * @opcode 0x12 + * @opcodesub 11 mr/reg + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(RM_REG, MOVHLPS, movhlps, Vq_WO, UqHi, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 1 /* a_iQword*/); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /** + * @opdone + * @opcode 0x12 + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + * @opfunction iemOp_movlps_Vq_Mq__vmovhlps + */ + IEMOP_MNEMONIC2(RM_MEM, MOVLPS, movlps, Vq, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x12 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movlpd_Vq_Mq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(RM_MEM, MOVLPD, movlpd, Vq_WO, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic ud660f12m3 + * @opcode 0x12 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid sse + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opcode 0x12 + * @oppfx 0xf3 + * @opcpuid sse3 + * @opgroup og_sse3_pcksclr_datamove + * @opxcpttype 4 + * @optest op1=-1 op2=0xdddddddd00000002eeeeeeee00000001 -> + * op1=0x00000002000000020000000100000001 + */ +FNIEMOP_DEF(iemOp_movsldup_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, MOVSLDUP, movsldup, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTUINT128U, uSrc); + + IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 0); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 0); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 2); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 2); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 0); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 0); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 2); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 2); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x12 + * @oppfx 0xf2 + * @opcpuid sse3 + * @opgroup og_sse3_pcksclr_datamove + * @opxcpttype 5 + * @optest op1=-1 op2=0xddddddddeeeeeeee2222222211111111 -> + * op1=0x22222222111111112222222211111111 + */ +FNIEMOP_DEF(iemOp_movddup_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, MOVDDUP, movddup, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM64. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint64_t, uSrc, 0); + + IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc); + IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem64]. + */ + IEM_MC_BEGIN(1, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(uint64_t, uSrc, 0); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc); + IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x13 + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movlps_Mq_Vq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(MR_MEM, MOVLPS, movlps, Mq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic ud0f13m3 + * @opcode 0x13 + * @opcodesub 11 mr/reg + * @oppfx none + * @opunused immediate + * @opcpuid sse + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opcode 0x13 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movlpd_Mq_Vq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(MR_MEM, MOVLPD, movlpd, Mq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic ud660f13m3 + * @opcode 0x13 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid sse + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opmnemonic udf30f13 + * @opcode 0x13 + * @oppfx 0xf3 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + * @opdone + */ + +/** + * @opmnemonic udf20f13 + * @opcode 0x13 + * @oppfx 0xf2 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + * @opdone + */ + +/** Opcode 0x0f 0x14 - unpcklps Vx, Wx*/ +FNIEMOP_DEF(iemOp_unpcklps_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, UNPCKLPS, unpcklps, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, iemAImpl_unpcklps_u128); +} + + +/** Opcode 0x66 0x0f 0x14 - unpcklpd Vx, Wx */ +FNIEMOP_DEF(iemOp_unpcklpd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, UNPCKLPD, unpcklpd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_unpcklpd_u128); +} + + +/** + * @opdone + * @opmnemonic udf30f14 + * @opcode 0x14 + * @oppfx 0xf3 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + * @opdone + */ + +/** + * @opmnemonic udf20f14 + * @opcode 0x14 + * @oppfx 0xf2 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + * @opdone + */ + +/** Opcode 0x0f 0x15 - unpckhps Vx, Wx */ +FNIEMOP_DEF(iemOp_unpckhps_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, UNPCKHPS, unpckhps, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, iemAImpl_unpckhps_u128); +} + + +/** Opcode 0x66 0x0f 0x15 - unpckhpd Vx, Wx */ +FNIEMOP_DEF(iemOp_unpckhpd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, UNPCKHPD, unpckhpd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_unpckhpd_u128); +} + + +/* Opcode 0xf3 0x0f 0x15 - invalid */ +/* Opcode 0xf2 0x0f 0x15 - invalid */ + +/** + * @opdone + * @opmnemonic udf30f15 + * @opcode 0x15 + * @oppfx 0xf3 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + * @opdone + */ + +/** + * @opmnemonic udf20f15 + * @opcode 0x15 + * @oppfx 0xf2 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + * @opdone + */ + +FNIEMOP_DEF(iemOp_movhps_Vdq_Mq__movlhps_Vdq_Uq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /** + * @opcode 0x16 + * @opcodesub 11 mr/reg + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(RM_REG, MOVLHPS, movlhps, VqHi_WO, Uq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /** + * @opdone + * @opcode 0x16 + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + * @opfunction iemOp_movhps_Vdq_Mq__movlhps_Vdq_Uq + */ + IEMOP_MNEMONIC2(RM_MEM, MOVHPS, movhps, VqHi_WO, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x16 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movhpd_Vdq_Mq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(RM_MEM, MOVHPD, movhpd, VqHi_WO, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic ud660f16m3 + * @opcode 0x16 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid sse + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opcode 0x16 + * @oppfx 0xf3 + * @opcpuid sse3 + * @opgroup og_sse3_pcksclr_datamove + * @opxcpttype 4 + * @optest op1=-1 op2=0x00000002dddddddd00000001eeeeeeee -> + * op1=0x00000002000000020000000100000001 + */ +FNIEMOP_DEF(iemOp_movshdup_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, MOVSHDUP, movshdup, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM128. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTUINT128U, uSrc); + + IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 1); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 1); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 3); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 3); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem128]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 1); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 1); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 3); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 3); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** + * @opdone + * @opmnemonic udf30f16 + * @opcode 0x16 + * @oppfx 0xf2 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + * @opdone + */ + + +/** + * @opcode 0x17 + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movhps_Mq_Vq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(MR_MEM, MOVHPS, movhps, Mq_WO, VqHi, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 1 /* a_iQword*/); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic ud0f17m3 + * @opcode 0x17 + * @opcodesub 11 mr/reg + * @oppfx none + * @opunused immediate + * @opcpuid sse + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opcode 0x17 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movhpd_Mq_Vq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(MR_MEM, MOVHPD, movhpd, Mq_WO, VqHi, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 1 /* a_iQword*/); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic ud660f17m3 + * @opcode 0x17 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid sse + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opdone + * @opmnemonic udf30f17 + * @opcode 0x17 + * @oppfx 0xf3 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + * @opdone + */ + +/** + * @opmnemonic udf20f17 + * @opcode 0x17 + * @oppfx 0xf2 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + * @opdone + */ + + +/** Opcode 0x0f 0x18. */ +FNIEMOP_DEF(iemOp_prefetch_Grp16) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 4: /* Aliased to /0 for the time being according to AMD. */ + case 5: /* Aliased to /0 for the time being according to AMD. */ + case 6: /* Aliased to /0 for the time being according to AMD. */ + case 7: /* Aliased to /0 for the time being according to AMD. */ + case 0: IEMOP_MNEMONIC(prefetchNTA, "prefetchNTA m8"); break; + case 1: IEMOP_MNEMONIC(prefetchT0, "prefetchT0 m8"); break; + case 2: IEMOP_MNEMONIC(prefetchT1, "prefetchT1 m8"); break; + case 3: IEMOP_MNEMONIC(prefetchT2, "prefetchT2 m8"); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + /* Currently a NOP. */ + NOREF(GCPtrEffSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode 0x0f 0x19..0x1f. */ +FNIEMOP_DEF(iemOp_nop_Ev) +{ + IEMOP_MNEMONIC(nop_Ev, "nop Ev"); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + /* Currently a NOP. */ + NOREF(GCPtrEffSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x20. */ +FNIEMOP_DEF(iemOp_mov_Rd_Cd) +{ + /* mod is ignored, as is operand size overrides. */ + IEMOP_MNEMONIC(mov_Rd_Cd, "mov Rd,Cd"); + IEMOP_HLP_MIN_386(); + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; + else + pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT; + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + uint8_t iCrReg = IEM_GET_MODRM_REG(pVCpu, bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) + { + /* The lock prefix can be used to encode CR8 accesses on some CPUs. */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovCr8In32Bit) + return IEMOP_RAISE_INVALID_OPCODE(); /* #UD takes precedence over #GP(), see test. */ + iCrReg |= 8; + } + switch (iCrReg) + { + case 0: case 2: case 3: case 4: case 8: + break; + default: + return IEMOP_RAISE_INVALID_OPCODE(); + } + IEMOP_HLP_DONE_DECODING(); + + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Rd_Cd, IEM_GET_MODRM_RM(pVCpu, bRm), iCrReg); +} + + +/** Opcode 0x0f 0x21. */ +FNIEMOP_DEF(iemOp_mov_Rd_Dd) +{ + IEMOP_MNEMONIC(mov_Rd_Dd, "mov Rd,Dd"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX_R) + return IEMOP_RAISE_INVALID_OPCODE(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Rd_Dd, + IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG_8(bRm)); +} + + +/** Opcode 0x0f 0x22. */ +FNIEMOP_DEF(iemOp_mov_Cd_Rd) +{ + /* mod is ignored, as is operand size overrides. */ + IEMOP_MNEMONIC(mov_Cd_Rd, "mov Cd,Rd"); + IEMOP_HLP_MIN_386(); + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; + else + pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT; + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + uint8_t iCrReg = IEM_GET_MODRM_REG(pVCpu, bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK) + { + /* The lock prefix can be used to encode CR8 accesses on some CPUs. */ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovCr8In32Bit) + return IEMOP_RAISE_INVALID_OPCODE(); /* #UD takes precedence over #GP(), see test. */ + iCrReg |= 8; + } + switch (iCrReg) + { + case 0: case 2: case 3: case 4: case 8: + break; + default: + return IEMOP_RAISE_INVALID_OPCODE(); + } + IEMOP_HLP_DONE_DECODING(); + + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Cd_Rd, iCrReg, IEM_GET_MODRM_RM(pVCpu, bRm)); +} + + +/** Opcode 0x0f 0x23. */ +FNIEMOP_DEF(iemOp_mov_Dd_Rd) +{ + IEMOP_MNEMONIC(mov_Dd_Rd, "mov Dd,Rd"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX_R) + return IEMOP_RAISE_INVALID_OPCODE(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Dd_Rd, + IEM_GET_MODRM_REG_8(bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); +} + + +/** Opcode 0x0f 0x24. */ +FNIEMOP_DEF(iemOp_mov_Rd_Td) +{ + IEMOP_MNEMONIC(mov_Rd_Td, "mov Rd,Td"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (RT_LIKELY(IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_PENTIUM)) + return IEMOP_RAISE_INVALID_OPCODE(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Rd_Td, + IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG_8(bRm)); +} + + +/** Opcode 0x0f 0x26. */ +FNIEMOP_DEF(iemOp_mov_Td_Rd) +{ + IEMOP_MNEMONIC(mov_Td_Rd, "mov Td,Rd"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (RT_LIKELY(IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_PENTIUM)) + return IEMOP_RAISE_INVALID_OPCODE(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Td_Rd, + IEM_GET_MODRM_REG_8(bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); +} + + +/** + * @opcode 0x28 + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movaps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, MOVAPS, movaps, Vps_WO, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** + * @opcode 0x28 + * @oppfx 66 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movapd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, MOVAPD, movapd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/* Opcode 0xf3 0x0f 0x28 - invalid */ +/* Opcode 0xf2 0x0f 0x28 - invalid */ + +/** + * @opcode 0x29 + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_simdfp_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movaps_Wps_Vps) +{ + IEMOP_MNEMONIC2(MR, MOVAPS, movaps, Wps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** + * @opcode 0x29 + * @oppfx 66 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movapd_Wpd_Vpd) +{ + IEMOP_MNEMONIC2(MR, MOVAPD, movapd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/* Opcode 0xf3 0x0f 0x29 - invalid */ +/* Opcode 0xf2 0x0f 0x29 - invalid */ + + +/** Opcode 0x0f 0x2a - cvtpi2ps Vps, Qpi */ +FNIEMOP_DEF(iemOp_cvtpi2ps_Vps_Qpi) +{ + IEMOP_MNEMONIC2(RM, CVTPI2PS, cvtpi2ps, Vps, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, MMX + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG(uint64_t, u64Src, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); /* Need it because the high quadword remains unchanged. */ + IEM_MC_FETCH_MREG_U64(u64Src, IEM_GET_MODRM_RM_8(bRm)); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpi2ps_u128, pfMxcsr, pDst, u64Src); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem64] + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG(uint64_t, u64Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + IEM_MC_REF_MXCSR(pfMxcsr); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpi2ps_u128, pfMxcsr, pDst, u64Src); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x2a - cvtpi2pd Vpd, Qpi */ +FNIEMOP_DEF(iemOp_cvtpi2pd_Vpd_Qpi) +{ + IEMOP_MNEMONIC2(RM, CVTPI2PD, cvtpi2pd, Vps, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, MMX + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG(uint64_t, u64Src, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_MREG_U64(u64Src, IEM_GET_MODRM_RM_8(bRm)); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpi2pd_u128, pfMxcsr, pDst, u64Src); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem64] + */ + IEM_MC_BEGIN(3, 3); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG(uint64_t, u64Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + /* Doesn't cause a transition to MMX mode. */ + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpi2pd_u128, pfMxcsr, pDst, u64Src); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0xf3 0x0f 0x2a - cvtsi2ss Vss, Ey */ +FNIEMOP_DEF(iemOp_cvtsi2ss_Vss_Ey) +{ + IEMOP_MNEMONIC2(RM, CVTSI2SS, cvtsi2ss, Vss, Ey, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* XMM, greg64 */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(RTFLOAT32U, r32Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PRTFLOAT32U, pr32Dst, r32Dst, 1); + IEM_MC_ARG(const int64_t *, pi64Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_GREG_I64_CONST(pi64Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2ss_r32_i64, pfMxcsr, pr32Dst, pi64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_R32(IEM_GET_MODRM_REG(pVCpu, bRm), r32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* XMM, [mem64] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(RTFLOAT32U, r32Dst); + IEM_MC_LOCAL(int64_t, i64Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PRTFLOAT32U, pr32Dst, r32Dst, 1); + IEM_MC_ARG_LOCAL_REF(const int64_t *, pi64Src, i64Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_I64(i64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2ss_r32_i64, pfMxcsr, pr32Dst, pi64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_R32(IEM_GET_MODRM_REG(pVCpu, bRm), r32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg, XMM */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(RTFLOAT32U, r32Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PRTFLOAT32U, pr32Dst, r32Dst, 1); + IEM_MC_ARG(const int32_t *, pi32Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_GREG_I32_CONST(pi32Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2ss_r32_i32, pfMxcsr, pr32Dst, pi32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_R32(IEM_GET_MODRM_REG(pVCpu, bRm), r32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* greg, [mem32] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(RTFLOAT32U, r32Dst); + IEM_MC_LOCAL(int32_t, i32Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PRTFLOAT32U, pr32Dst, r32Dst, 1); + IEM_MC_ARG_LOCAL_REF(const int32_t *, pi32Src, i32Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_I32(i32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2ss_r32_i32, pfMxcsr, pr32Dst, pi32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_R32(IEM_GET_MODRM_REG(pVCpu, bRm), r32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode 0xf2 0x0f 0x2a - cvtsi2sd Vsd, Ey */ +FNIEMOP_DEF(iemOp_cvtsi2sd_Vsd_Ey) +{ + IEMOP_MNEMONIC2(RM, CVTSI2SD, cvtsi2sd, Vsd, Ey, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* XMM, greg64 */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(RTFLOAT64U, r64Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U, pr64Dst, r64Dst, 1); + IEM_MC_ARG(const int64_t *, pi64Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_GREG_I64_CONST(pi64Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2sd_r64_i64, pfMxcsr, pr64Dst, pi64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_R64(IEM_GET_MODRM_REG(pVCpu, bRm), r64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* XMM, [mem64] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(RTFLOAT64U, r64Dst); + IEM_MC_LOCAL(int64_t, i64Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U, pr64Dst, r64Dst, 1); + IEM_MC_ARG_LOCAL_REF(const int64_t *, pi64Src, i64Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_I64(i64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2sd_r64_i64, pfMxcsr, pr64Dst, pi64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_R64(IEM_GET_MODRM_REG(pVCpu, bRm), r64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* XMM, greg32 */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(RTFLOAT64U, r64Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U, pr64Dst, r64Dst, 1); + IEM_MC_ARG(const int32_t *, pi32Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_GREG_I32_CONST(pi32Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2sd_r64_i32, pfMxcsr, pr64Dst, pi32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_R64(IEM_GET_MODRM_REG(pVCpu, bRm), r64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* XMM, [mem32] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(RTFLOAT64U, r64Dst); + IEM_MC_LOCAL(int32_t, i32Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U, pr64Dst, r64Dst, 1); + IEM_MC_ARG_LOCAL_REF(const int32_t *, pi32Src, i32Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_I32(i32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2sd_r64_i32, pfMxcsr, pr64Dst, pi32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_R64(IEM_GET_MODRM_REG(pVCpu, bRm), r64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** + * @opcode 0x2b + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid sse + * @opgroup og_sse1_cachect + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movntps_Mps_Vps) +{ + IEMOP_MNEMONIC2(MR_MEM, MOVNTPS, movntps, Mps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* + * memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + /* The register, register encoding is invalid. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/** + * @opcode 0x2b + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_cachect + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movntpd_Mpd_Vpd) +{ + IEMOP_MNEMONIC2(MR_MEM, MOVNTPD, movntpd, Mpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* + * memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); /** @todo optimize this one day... */ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + /* The register, register encoding is invalid. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} +/* Opcode 0xf3 0x0f 0x2b - invalid */ +/* Opcode 0xf2 0x0f 0x2b - invalid */ + + +/** Opcode 0x0f 0x2c - cvttps2pi Ppi, Wps */ +FNIEMOP_DEF(iemOp_cvttps2pi_Ppi_Wps) +{ + IEMOP_MNEMONIC2(RM, CVTTPS2PI, cvttps2pi, Pq, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(uint64_t, u64Dst); + IEM_MC_ARG_LOCAL_REF(uint64_t *, pu64Dst, u64Dst, 1); + IEM_MC_ARG(uint64_t, u64Src, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_U64(u64Src, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvttps2pi_u128, pfMxcsr, pu64Dst, u64Src); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(uint64_t, u64Dst); + IEM_MC_ARG_LOCAL_REF(uint64_t *, pu64Dst, u64Dst, 1); + IEM_MC_ARG(uint64_t, u64Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + IEM_MC_REF_MXCSR(pfMxcsr); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvttps2pi_u128, pfMxcsr, pu64Dst, u64Src); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x2c - cvttpd2pi Ppi, Wpd */ +FNIEMOP_DEF(iemOp_cvttpd2pi_Ppi_Wpd) +{ + IEMOP_MNEMONIC2(RM, CVTTPD2PI, cvttpd2pi, Pq, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(uint64_t, u64Dst); + IEM_MC_ARG_LOCAL_REF(uint64_t *, pu64Dst, u64Dst, 1); + IEM_MC_ARG(PCX86XMMREG, pSrc, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvttpd2pi_u128, pfMxcsr, pu64Dst, pSrc); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 3); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(uint64_t, u64Dst); + IEM_MC_ARG_LOCAL_REF(uint64_t *, pu64Dst, u64Dst, 1); + IEM_MC_LOCAL(X86XMMREG, uSrc); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, pSrc, uSrc, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MXCSR(pfMxcsr); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvttpd2pi_u128, pfMxcsr, pu64Dst, pSrc); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0xf3 0x0f 0x2c - cvttss2si Gy, Wss */ +FNIEMOP_DEF(iemOp_cvttss2si_Gy_Wss) +{ + IEMOP_MNEMONIC2(RM, CVTTSS2SI, cvttss2si, Gy, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg64, XMM */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int64_t, i64Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int64_t *, pi64Dst, i64Dst, 1); + IEM_MC_ARG(const uint32_t *, pu32Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_XREG_U32_CONST(pu32Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttss2si_i64_r32, pfMxcsr, pi64Dst, pu32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* greg64, [mem64] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int64_t, i64Dst); + IEM_MC_LOCAL(uint32_t, u32Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int64_t *, pi64Dst, i64Dst, 1); + IEM_MC_ARG_LOCAL_REF(const uint32_t *, pu32Src, u32Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttss2si_i64_r32, pfMxcsr, pi64Dst, pu32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg, XMM */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int32_t, i32Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int32_t *, pi32Dst, i32Dst, 1); + IEM_MC_ARG(const uint32_t *, pu32Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_XREG_U32_CONST(pu32Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttss2si_i32_r32, pfMxcsr, pi32Dst, pu32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* greg, [mem] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int32_t, i32Dst); + IEM_MC_LOCAL(uint32_t, u32Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int32_t *, pi32Dst, i32Dst, 1); + IEM_MC_ARG_LOCAL_REF(const uint32_t *, pu32Src, u32Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttss2si_i32_r32, pfMxcsr, pi32Dst, pu32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode 0xf2 0x0f 0x2c - cvttsd2si Gy, Wsd */ +FNIEMOP_DEF(iemOp_cvttsd2si_Gy_Wsd) +{ + IEMOP_MNEMONIC2(RM, CVTTSD2SI, cvttsd2si, Gy, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg64, XMM */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int64_t, i64Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int64_t *, pi64Dst, i64Dst, 1); + IEM_MC_ARG(const uint64_t *, pu64Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_XREG_U64_CONST(pu64Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttsd2si_i64_r64, pfMxcsr, pi64Dst, pu64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* greg64, [mem64] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int64_t, i64Dst); + IEM_MC_LOCAL(uint64_t, u64Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int64_t *, pi64Dst, i64Dst, 1); + IEM_MC_ARG_LOCAL_REF(const uint64_t *, pu64Src, u64Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttsd2si_i64_r64, pfMxcsr, pi64Dst, pu64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg, XMM */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int32_t, i32Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int32_t *, pi32Dst, i32Dst, 1); + IEM_MC_ARG(const uint64_t *, pu64Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_XREG_U64_CONST(pu64Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttsd2si_i32_r64, pfMxcsr, pi32Dst, pu64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* greg32, [mem32] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int32_t, i32Dst); + IEM_MC_LOCAL(uint64_t, u64Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int32_t *, pi32Dst, i32Dst, 1); + IEM_MC_ARG_LOCAL_REF(const uint64_t *, pu64Src, u64Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttsd2si_i32_r64, pfMxcsr, pi32Dst, pu64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode 0x0f 0x2d - cvtps2pi Ppi, Wps */ +FNIEMOP_DEF(iemOp_cvtps2pi_Ppi_Wps) +{ + IEMOP_MNEMONIC2(RM, CVTPS2PI, cvtps2pi, Pq, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(uint64_t, u64Dst); + IEM_MC_ARG_LOCAL_REF(uint64_t *, pu64Dst, u64Dst, 1); + IEM_MC_ARG(uint64_t, u64Src, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_U64(u64Src, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtps2pi_u128, pfMxcsr, pu64Dst, u64Src); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(uint64_t, u64Dst); + IEM_MC_ARG_LOCAL_REF(uint64_t *, pu64Dst, u64Dst, 1); + IEM_MC_ARG(uint64_t, u64Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + IEM_MC_REF_MXCSR(pfMxcsr); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtps2pi_u128, pfMxcsr, pu64Dst, u64Src); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x2d - cvtpd2pi Qpi, Wpd */ +FNIEMOP_DEF(iemOp_cvtpd2pi_Qpi_Wpd) +{ + IEMOP_MNEMONIC2(RM, CVTPD2PI, cvtpd2pi, Pq, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(uint64_t, u64Dst); + IEM_MC_ARG_LOCAL_REF(uint64_t *, pu64Dst, u64Dst, 1); + IEM_MC_ARG(PCX86XMMREG, pSrc, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpd2pi_u128, pfMxcsr, pu64Dst, pSrc); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 3); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_LOCAL(uint64_t, u64Dst); + IEM_MC_ARG_LOCAL_REF(uint64_t *, pu64Dst, u64Dst, 1); + IEM_MC_LOCAL(X86XMMREG, uSrc); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, pSrc, uSrc, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MXCSR(pfMxcsr); + + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpd2pi_u128, pfMxcsr, pu64Dst, pSrc); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0xf3 0x0f 0x2d - cvtss2si Gy, Wss */ +FNIEMOP_DEF(iemOp_cvtss2si_Gy_Wss) +{ + IEMOP_MNEMONIC2(RM, CVTSS2SI, cvtss2si, Gy, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg64, XMM */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int64_t, i64Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int64_t *, pi64Dst, i64Dst, 1); + IEM_MC_ARG(const uint32_t *, pu32Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_XREG_U32_CONST(pu32Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtss2si_i64_r32, pfMxcsr, pi64Dst, pu32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* greg64, [mem64] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int64_t, i64Dst); + IEM_MC_LOCAL(uint32_t, u32Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int64_t *, pi64Dst, i64Dst, 1); + IEM_MC_ARG_LOCAL_REF(const uint32_t *, pu32Src, u32Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtss2si_i64_r32, pfMxcsr, pi64Dst, pu32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg, XMM */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int32_t, i32Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int32_t *, pi32Dst, i32Dst, 1); + IEM_MC_ARG(const uint32_t *, pu32Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_XREG_U32_CONST(pu32Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtss2si_i32_r32, pfMxcsr, pi32Dst, pu32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* greg, [mem] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int32_t, i32Dst); + IEM_MC_LOCAL(uint32_t, u32Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int32_t *, pi32Dst, i32Dst, 1); + IEM_MC_ARG_LOCAL_REF(const uint32_t *, pu32Src, u32Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtss2si_i32_r32, pfMxcsr, pi32Dst, pu32Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode 0xf2 0x0f 0x2d - cvtsd2si Gy, Wsd */ +FNIEMOP_DEF(iemOp_cvtsd2si_Gy_Wsd) +{ + IEMOP_MNEMONIC2(RM, CVTSD2SI, cvtsd2si, Gy, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg64, XMM */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int64_t, i64Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int64_t *, pi64Dst, i64Dst, 1); + IEM_MC_ARG(const uint64_t *, pu64Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_XREG_U64_CONST(pu64Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsd2si_i64_r64, pfMxcsr, pi64Dst, pu64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* greg64, [mem64] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int64_t, i64Dst); + IEM_MC_LOCAL(uint64_t, u64Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int64_t *, pi64Dst, i64Dst, 1); + IEM_MC_ARG_LOCAL_REF(const uint64_t *, pu64Src, u64Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsd2si_i64_r64, pfMxcsr, pi64Dst, pu64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg32, XMM */ + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int32_t, i32Dst); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int32_t *, pi32Dst, i32Dst, 1); + IEM_MC_ARG(const uint64_t *, pu64Src, 2); + + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_REF_XREG_U64_CONST(pu64Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsd2si_i32_r64, pfMxcsr, pi32Dst, pu64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* greg32, [mem64] */ + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, fMxcsr); + IEM_MC_LOCAL(int32_t, i32Dst); + IEM_MC_LOCAL(uint64_t, u64Src); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(int32_t *, pi32Dst, i32Dst, 1); + IEM_MC_ARG_LOCAL_REF(const uint64_t *, pu64Src, u64Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */ + + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsd2si_i32_r64, pfMxcsr, pi32Dst, pu64Src); + IEM_MC_SSE_UPDATE_MXCSR(fMxcsr); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode 0x0f 0x2e - ucomiss Vss, Wss */ +FNIEMOP_DEF(iemOp_ucomiss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, UCOMISS, ucomiss, Vss, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 1); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_ARG(PCX86XMMREG, puSrc2, 3); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_ucomiss_u128, pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, puSrc2, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_ucomiss_u128, pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x2e - ucomisd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_ucomisd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, UCOMISD, ucomisd, Vsd, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 1); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_ARG(PCX86XMMREG, puSrc2, 3); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_ucomisd_u128, pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, puSrc2, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U64(uSrc2, 0 /*a_QWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_ucomisd_u128, pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode 0xf3 0x0f 0x2e - invalid */ +/* Opcode 0xf2 0x0f 0x2e - invalid */ + + +/** Opcode 0x0f 0x2f - comiss Vss, Wss */ +FNIEMOP_DEF(iemOp_comiss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, COMISS, comiss, Vss, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 1); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_ARG(PCX86XMMREG, puSrc2, 3); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_comiss_u128, pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, puSrc2, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_comiss_u128, pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x2f - comisd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_comisd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, COMISD, comisd, Vsd, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 1); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_ARG(PCX86XMMREG, puSrc2, 3); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_comisd_u128, pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, puSrc2, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U64(uSrc2, 0 /*a_QWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_comisd_u128, pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode 0xf3 0x0f 0x2f - invalid */ +/* Opcode 0xf2 0x0f 0x2f - invalid */ + +/** Opcode 0x0f 0x30. */ +FNIEMOP_DEF(iemOp_wrmsr) +{ + IEMOP_MNEMONIC(wrmsr, "wrmsr"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_wrmsr); +} + + +/** Opcode 0x0f 0x31. */ +FNIEMOP_DEF(iemOp_rdtsc) +{ + IEMOP_MNEMONIC(rdtsc, "rdtsc"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdtsc); +} + + +/** Opcode 0x0f 0x33. */ +FNIEMOP_DEF(iemOp_rdmsr) +{ + IEMOP_MNEMONIC(rdmsr, "rdmsr"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdmsr); +} + + +/** Opcode 0x0f 0x34. */ +FNIEMOP_DEF(iemOp_rdpmc) +{ + IEMOP_MNEMONIC(rdpmc, "rdpmc"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdpmc); +} + + +/** Opcode 0x0f 0x34. */ +FNIEMOP_DEF(iemOp_sysenter) +{ + IEMOP_MNEMONIC0(FIXED, SYSENTER, sysenter, DISOPTYPE_CONTROLFLOW | DISOPTYPE_UNCOND_CONTROLFLOW, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_sysenter); +} + +/** Opcode 0x0f 0x35. */ +FNIEMOP_DEF(iemOp_sysexit) +{ + IEMOP_MNEMONIC0(FIXED, SYSEXIT, sysexit, DISOPTYPE_CONTROLFLOW | DISOPTYPE_UNCOND_CONTROLFLOW, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_sysexit, pVCpu->iem.s.enmEffOpSize); +} + +/** Opcode 0x0f 0x37. */ +FNIEMOP_STUB(iemOp_getsec); + + +/** Opcode 0x0f 0x38. */ +FNIEMOP_DEF(iemOp_3byte_Esc_0f_38) +{ +#ifdef IEM_WITH_THREE_0F_38 + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnThreeByte0f38[(uintptr_t)b * 4 + pVCpu->iem.s.idxPrefix]); +#else + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +#endif +} + + +/** Opcode 0x0f 0x3a. */ +FNIEMOP_DEF(iemOp_3byte_Esc_0f_3a) +{ +#ifdef IEM_WITH_THREE_0F_3A + uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b); + return FNIEMOP_CALL(g_apfnThreeByte0f3a[(uintptr_t)b * 4 + pVCpu->iem.s.idxPrefix]); +#else + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +#endif +} + + +/** + * Implements a conditional move. + * + * Wish there was an obvious way to do this where we could share and reduce + * code bloat. + * + * @param a_Cnd The conditional "microcode" operation. + */ +#define CMOV_X(a_Cnd) \ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \ + if (IEM_IS_MODRM_REG_MODE(bRm)) \ + { \ + switch (pVCpu->iem.s.enmEffOpSize) \ + { \ + case IEMMODE_16BIT: \ + IEM_MC_BEGIN(0, 1); \ + IEM_MC_LOCAL(uint16_t, u16Tmp); \ + a_Cnd { \ + IEM_MC_FETCH_GREG_U16(u16Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + break; \ + \ + case IEMMODE_32BIT: \ + IEM_MC_BEGIN(0, 1); \ + IEM_MC_LOCAL(uint32_t, u32Tmp); \ + a_Cnd { \ + IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); \ + } IEM_MC_ELSE() { \ + IEM_MC_CLEAR_HIGH_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm)); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + break; \ + \ + case IEMMODE_64BIT: \ + IEM_MC_BEGIN(0, 1); \ + IEM_MC_LOCAL(uint64_t, u64Tmp); \ + a_Cnd { \ + IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + break; \ + \ + IEM_NOT_REACHED_DEFAULT_CASE_RET(); \ + } \ + } \ + else \ + { \ + switch (pVCpu->iem.s.enmEffOpSize) \ + { \ + case IEMMODE_16BIT: \ + IEM_MC_BEGIN(0, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_LOCAL(uint16_t, u16Tmp); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + a_Cnd { \ + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + break; \ + \ + case IEMMODE_32BIT: \ + IEM_MC_BEGIN(0, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_LOCAL(uint32_t, u32Tmp); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + a_Cnd { \ + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); \ + } IEM_MC_ELSE() { \ + IEM_MC_CLEAR_HIGH_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm)); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + break; \ + \ + case IEMMODE_64BIT: \ + IEM_MC_BEGIN(0, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_LOCAL(uint64_t, u64Tmp); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + a_Cnd { \ + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); \ + } IEM_MC_ENDIF(); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + break; \ + \ + IEM_NOT_REACHED_DEFAULT_CASE_RET(); \ + } \ + } do {} while (0) + + + +/** Opcode 0x0f 0x40. */ +FNIEMOP_DEF(iemOp_cmovo_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovo_Gv_Ev, "cmovo Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF)); +} + + +/** Opcode 0x0f 0x41. */ +FNIEMOP_DEF(iemOp_cmovno_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovno_Gv_Ev, "cmovno Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_OF)); +} + + +/** Opcode 0x0f 0x42. */ +FNIEMOP_DEF(iemOp_cmovc_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovc_Gv_Ev, "cmovc Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF)); +} + + +/** Opcode 0x0f 0x43. */ +FNIEMOP_DEF(iemOp_cmovnc_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovnc_Gv_Ev, "cmovnc Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_CF)); +} + + +/** Opcode 0x0f 0x44. */ +FNIEMOP_DEF(iemOp_cmove_Gv_Ev) +{ + IEMOP_MNEMONIC(cmove_Gv_Ev, "cmove Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF)); +} + + +/** Opcode 0x0f 0x45. */ +FNIEMOP_DEF(iemOp_cmovne_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovne_Gv_Ev, "cmovne Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)); +} + + +/** Opcode 0x0f 0x46. */ +FNIEMOP_DEF(iemOp_cmovbe_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovbe_Gv_Ev, "cmovbe Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF)); +} + + +/** Opcode 0x0f 0x47. */ +FNIEMOP_DEF(iemOp_cmovnbe_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovnbe_Gv_Ev, "cmovnbe Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_NO_BITS_SET(X86_EFL_CF | X86_EFL_ZF)); +} + + +/** Opcode 0x0f 0x48. */ +FNIEMOP_DEF(iemOp_cmovs_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovs_Gv_Ev, "cmovs Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF)); +} + + +/** Opcode 0x0f 0x49. */ +FNIEMOP_DEF(iemOp_cmovns_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovns_Gv_Ev, "cmovns Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_SF)); +} + + +/** Opcode 0x0f 0x4a. */ +FNIEMOP_DEF(iemOp_cmovp_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovp_Gv_Ev, "cmovp Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF)); +} + + +/** Opcode 0x0f 0x4b. */ +FNIEMOP_DEF(iemOp_cmovnp_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovnp_Gv_Ev, "cmovnp Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_PF)); +} + + +/** Opcode 0x0f 0x4c. */ +FNIEMOP_DEF(iemOp_cmovl_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovl_Gv_Ev, "cmovl Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF)); +} + + +/** Opcode 0x0f 0x4d. */ +FNIEMOP_DEF(iemOp_cmovnl_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovnl_Gv_Ev, "cmovnl Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BITS_EQ(X86_EFL_SF, X86_EFL_OF)); +} + + +/** Opcode 0x0f 0x4e. */ +FNIEMOP_DEF(iemOp_cmovle_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovle_Gv_Ev, "cmovle Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF)); +} + + +/** Opcode 0x0f 0x4f. */ +FNIEMOP_DEF(iemOp_cmovnle_Gv_Ev) +{ + IEMOP_MNEMONIC(cmovnle_Gv_Ev, "cmovnle Gv,Ev"); + CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF)); +} + +#undef CMOV_X + +/** Opcode 0x0f 0x50 - movmskps Gy, Ups */ +FNIEMOP_DEF(iemOp_movmskps_Gy_Ups) +{ + IEMOP_MNEMONIC2(RM_REG, MOVMSKPS, movmskps, Gy, Ux, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /** @todo */ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 1); + IEM_MC_LOCAL(uint8_t, u8Dst); + IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst, u8Dst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_movmskps_u128, pu8Dst, puSrc); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + /* No memory operand. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode 0x66 0x0f 0x50 - movmskpd Gy, Upd */ +FNIEMOP_DEF(iemOp_movmskpd_Gy_Upd) +{ + IEMOP_MNEMONIC2(RM_REG, MOVMSKPD, movmskpd, Gy, Ux, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /** @todo */ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 1); + IEM_MC_LOCAL(uint8_t, u8Dst); + IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst, u8Dst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_movmskpd_u128, pu8Dst, puSrc); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG_8(bRm), u8Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + /* No memory operand. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); + +} + + +/* Opcode 0xf3 0x0f 0x50 - invalid */ +/* Opcode 0xf2 0x0f 0x50 - invalid */ + + +/** Opcode 0x0f 0x51 - sqrtps Vps, Wps */ +FNIEMOP_DEF(iemOp_sqrtps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, SQRTPS, sqrtps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_sqrtps_u128); +} + + +/** Opcode 0x66 0x0f 0x51 - sqrtpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_sqrtpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, SQRTPD, sqrtpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_sqrtpd_u128); +} + + +/** Opcode 0xf3 0x0f 0x51 - sqrtss Vss, Wss */ +FNIEMOP_DEF(iemOp_sqrtss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, SQRTSS, sqrtss, Vss, Wss, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_sqrtss_u128_r32); +} + + +/** Opcode 0xf2 0x0f 0x51 - sqrtsd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_sqrtsd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, SQRTSD, sqrtsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_sqrtsd_u128_r64); +} + + +/** Opcode 0x0f 0x52 - rsqrtps Vps, Wps */ +FNIEMOP_DEF(iemOp_rsqrtps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, RSQRTPS, rsqrtps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_rsqrtps_u128); +} + + +/* Opcode 0x66 0x0f 0x52 - invalid */ + + +/** Opcode 0xf3 0x0f 0x52 - rsqrtss Vss, Wss */ +FNIEMOP_DEF(iemOp_rsqrtss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, RSQRTSS, rsqrtss, Vss, Wss, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_rsqrtss_u128_r32); +} + + +/* Opcode 0xf2 0x0f 0x52 - invalid */ + +/** Opcode 0x0f 0x53 - rcpps Vps, Wps */ +FNIEMOP_STUB(iemOp_rcpps_Vps_Wps); +/* Opcode 0x66 0x0f 0x53 - invalid */ +/** Opcode 0xf3 0x0f 0x53 - rcpss Vss, Wss */ +FNIEMOP_STUB(iemOp_rcpss_Vss_Wss); +/* Opcode 0xf2 0x0f 0x53 - invalid */ + + +/** Opcode 0x0f 0x54 - andps Vps, Wps */ +FNIEMOP_DEF(iemOp_andps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, ANDPS, andps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse_FullFull_To_Full, iemAImpl_pand_u128); +} + + +/** Opcode 0x66 0x0f 0x54 - andpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_andpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, ANDPD, andpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pand_u128); +} + + +/* Opcode 0xf3 0x0f 0x54 - invalid */ +/* Opcode 0xf2 0x0f 0x54 - invalid */ + + +/** Opcode 0x0f 0x55 - andnps Vps, Wps */ +FNIEMOP_DEF(iemOp_andnps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, ANDNPS, andnps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse_FullFull_To_Full, iemAImpl_pandn_u128); +} + + +/** Opcode 0x66 0x0f 0x55 - andnpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_andnpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, ANDNPD, andnpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pandn_u128); +} + + +/* Opcode 0xf3 0x0f 0x55 - invalid */ +/* Opcode 0xf2 0x0f 0x55 - invalid */ + + +/** Opcode 0x0f 0x56 - orps Vps, Wps */ +FNIEMOP_DEF(iemOp_orps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, ORPS, orps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse_FullFull_To_Full, iemAImpl_por_u128); +} + + +/** Opcode 0x66 0x0f 0x56 - orpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_orpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, ORPD, orpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_por_u128); +} + + +/* Opcode 0xf3 0x0f 0x56 - invalid */ +/* Opcode 0xf2 0x0f 0x56 - invalid */ + + +/** Opcode 0x0f 0x57 - xorps Vps, Wps */ +FNIEMOP_DEF(iemOp_xorps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, XORPS, xorps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse_FullFull_To_Full, iemAImpl_pxor_u128); +} + + +/** Opcode 0x66 0x0f 0x57 - xorpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_xorpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, XORPD, xorpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pxor_u128); +} + + +/* Opcode 0xf3 0x0f 0x57 - invalid */ +/* Opcode 0xf2 0x0f 0x57 - invalid */ + +/** Opcode 0x0f 0x58 - addps Vps, Wps */ +FNIEMOP_DEF(iemOp_addps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, ADDPS, addps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_addps_u128); +} + + +/** Opcode 0x66 0x0f 0x58 - addpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_addpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, ADDPD, addpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_addpd_u128); +} + + +/** Opcode 0xf3 0x0f 0x58 - addss Vss, Wss */ +FNIEMOP_DEF(iemOp_addss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, ADDSS, addss, Vss, Wss, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_addss_u128_r32); +} + + +/** Opcode 0xf2 0x0f 0x58 - addsd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_addsd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, ADDSD, addsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_addsd_u128_r64); +} + + +/** Opcode 0x0f 0x59 - mulps Vps, Wps */ +FNIEMOP_DEF(iemOp_mulps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, MULPS, mulps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_mulps_u128); +} + + +/** Opcode 0x66 0x0f 0x59 - mulpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_mulpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, MULPD, mulpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_mulpd_u128); +} + + +/** Opcode 0xf3 0x0f 0x59 - mulss Vss, Wss */ +FNIEMOP_DEF(iemOp_mulss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, MULSS, mulss, Vss, Wss, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_mulss_u128_r32); +} + + +/** Opcode 0xf2 0x0f 0x59 - mulsd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_mulsd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, MULSD, mulsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_mulsd_u128_r64); +} + + +/** Opcode 0x0f 0x5a - cvtps2pd Vpd, Wps */ +FNIEMOP_DEF(iemOp_cvtps2pd_Vpd_Wps) +{ + IEMOP_MNEMONIC2(RM, CVTPS2PD, cvtps2pd, Vpd, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtps2pd_u128); +} + + +/** Opcode 0x66 0x0f 0x5a - cvtpd2ps Vps, Wpd */ +FNIEMOP_DEF(iemOp_cvtpd2ps_Vps_Wpd) +{ + IEMOP_MNEMONIC2(RM, CVTPD2PS, cvtpd2ps, Vps, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtpd2ps_u128); +} + + +/** Opcode 0xf3 0x0f 0x5a - cvtss2sd Vsd, Wss */ +FNIEMOP_DEF(iemOp_cvtss2sd_Vsd_Wss) +{ + IEMOP_MNEMONIC2(RM, CVTSS2SD, cvtss2sd, Vsd, Wss, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_cvtss2sd_u128_r32); +} + + +/** Opcode 0xf2 0x0f 0x5a - cvtsd2ss Vss, Wsd */ +FNIEMOP_DEF(iemOp_cvtsd2ss_Vss_Wsd) +{ + IEMOP_MNEMONIC2(RM, CVTSD2SS, cvtsd2ss, Vss, Wsd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_cvtsd2ss_u128_r64); +} + + +/** Opcode 0x0f 0x5b - cvtdq2ps Vps, Wdq */ +FNIEMOP_DEF(iemOp_cvtdq2ps_Vps_Wdq) +{ + IEMOP_MNEMONIC2(RM, CVTDQ2PS, cvtdq2ps, Vps, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtdq2ps_u128); +} + + +/** Opcode 0x66 0x0f 0x5b - cvtps2dq Vdq, Wps */ +FNIEMOP_DEF(iemOp_cvtps2dq_Vdq_Wps) +{ + IEMOP_MNEMONIC2(RM, CVTPS2DQ, cvtps2dq, Vdq, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtps2dq_u128); +} + + +/** Opcode 0xf3 0x0f 0x5b - cvttps2dq Vdq, Wps */ +FNIEMOP_DEF(iemOp_cvttps2dq_Vdq_Wps) +{ + IEMOP_MNEMONIC2(RM, CVTTPS2DQ, cvttps2dq, Vdq, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvttps2dq_u128); +} + + +/* Opcode 0xf2 0x0f 0x5b - invalid */ + + +/** Opcode 0x0f 0x5c - subps Vps, Wps */ +FNIEMOP_DEF(iemOp_subps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, SUBPS, subps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_subps_u128); +} + + +/** Opcode 0x66 0x0f 0x5c - subpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_subpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, SUBPD, subpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_subpd_u128); +} + + +/** Opcode 0xf3 0x0f 0x5c - subss Vss, Wss */ +FNIEMOP_DEF(iemOp_subss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, SUBSS, subss, Vss, Wss, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_subss_u128_r32); +} + + +/** Opcode 0xf2 0x0f 0x5c - subsd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_subsd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, SUBSD, subsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_subsd_u128_r64); +} + + +/** Opcode 0x0f 0x5d - minps Vps, Wps */ +FNIEMOP_DEF(iemOp_minps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, MINPS, minps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_minps_u128); +} + + +/** Opcode 0x66 0x0f 0x5d - minpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_minpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, MINPD, minpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_minpd_u128); +} + + +/** Opcode 0xf3 0x0f 0x5d - minss Vss, Wss */ +FNIEMOP_DEF(iemOp_minss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, MINSS, minss, Vss, Wss, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_minss_u128_r32); +} + + +/** Opcode 0xf2 0x0f 0x5d - minsd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_minsd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, MINSD, minsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_minsd_u128_r64); +} + + +/** Opcode 0x0f 0x5e - divps Vps, Wps */ +FNIEMOP_DEF(iemOp_divps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, DIVPS, divps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_divps_u128); +} + + +/** Opcode 0x66 0x0f 0x5e - divpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_divpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, DIVPD, divpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_divpd_u128); +} + + +/** Opcode 0xf3 0x0f 0x5e - divss Vss, Wss */ +FNIEMOP_DEF(iemOp_divss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, DIVSS, divss, Vss, Wss, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_divss_u128_r32); +} + + +/** Opcode 0xf2 0x0f 0x5e - divsd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_divsd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, DIVSD, divsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_divsd_u128_r64); +} + + +/** Opcode 0x0f 0x5f - maxps Vps, Wps */ +FNIEMOP_DEF(iemOp_maxps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, MAXPS, maxps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_maxps_u128); +} + + +/** Opcode 0x66 0x0f 0x5f - maxpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_maxpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, MAXPD, maxpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_maxpd_u128); +} + + +/** Opcode 0xf3 0x0f 0x5f - maxss Vss, Wss */ +FNIEMOP_DEF(iemOp_maxss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, MAXSS, maxss, Vss, Wss, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_maxss_u128_r32); +} + + +/** Opcode 0xf2 0x0f 0x5f - maxsd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_maxsd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, MAXSD, maxsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_maxsd_u128_r64); +} + + +/** Opcode 0x0f 0x60 - punpcklbw Pq, Qd */ +FNIEMOP_DEF(iemOp_punpcklbw_Pq_Qd) +{ + IEMOP_MNEMONIC2(RM, PUNPCKLBW, punpcklbw, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmx_LowLow_To_Full, iemAImpl_punpcklbw_u64); +} + + +/** Opcode 0x66 0x0f 0x60 - punpcklbw Vx, W */ +FNIEMOP_DEF(iemOp_punpcklbw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PUNPCKLBW, punpcklbw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_punpcklbw_u128); +} + + +/* Opcode 0xf3 0x0f 0x60 - invalid */ + + +/** Opcode 0x0f 0x61 - punpcklwd Pq, Qd */ +FNIEMOP_DEF(iemOp_punpcklwd_Pq_Qd) +{ + /** @todo AMD mark the MMX version as 3DNow!. Intel says MMX CPUID req. */ + IEMOP_MNEMONIC2(RM, PUNPCKLWD, punpcklwd, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmx_LowLow_To_Full, iemAImpl_punpcklwd_u64); +} + + +/** Opcode 0x66 0x0f 0x61 - punpcklwd Vx, Wx */ +FNIEMOP_DEF(iemOp_punpcklwd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PUNPCKLWD, punpcklwd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_punpcklwd_u128); +} + + +/* Opcode 0xf3 0x0f 0x61 - invalid */ + + +/** Opcode 0x0f 0x62 - punpckldq Pq, Qd */ +FNIEMOP_DEF(iemOp_punpckldq_Pq_Qd) +{ + IEMOP_MNEMONIC2(RM, PUNPCKLDQ, punpckldq, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmx_LowLow_To_Full, iemAImpl_punpckldq_u64); +} + + +/** Opcode 0x66 0x0f 0x62 - punpckldq Vx, Wx */ +FNIEMOP_DEF(iemOp_punpckldq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PUNPCKLDQ, punpckldq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_punpckldq_u128); +} + + +/* Opcode 0xf3 0x0f 0x62 - invalid */ + + + +/** Opcode 0x0f 0x63 - packsswb Pq, Qq */ +FNIEMOP_DEF(iemOp_packsswb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PACKSSWB, packsswb, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_packsswb_u64); +} + + +/** Opcode 0x66 0x0f 0x63 - packsswb Vx, Wx */ +FNIEMOP_DEF(iemOp_packsswb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PACKSSWB, packsswb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_packsswb_u128); +} + + +/* Opcode 0xf3 0x0f 0x63 - invalid */ + + +/** Opcode 0x0f 0x64 - pcmpgtb Pq, Qq */ +FNIEMOP_DEF(iemOp_pcmpgtb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PCMPGTB, pcmpgtb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpgtb_u64); +} + + +/** Opcode 0x66 0x0f 0x64 - pcmpgtb Vx, Wx */ +FNIEMOP_DEF(iemOp_pcmpgtb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PCMPGTB, pcmpgtb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpgtb_u128); +} + + +/* Opcode 0xf3 0x0f 0x64 - invalid */ + + +/** Opcode 0x0f 0x65 - pcmpgtw Pq, Qq */ +FNIEMOP_DEF(iemOp_pcmpgtw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PCMPGTW, pcmpgtw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpgtw_u64); +} + + +/** Opcode 0x66 0x0f 0x65 - pcmpgtw Vx, Wx */ +FNIEMOP_DEF(iemOp_pcmpgtw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PCMPGTW, pcmpgtw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpgtw_u128); +} + + +/* Opcode 0xf3 0x0f 0x65 - invalid */ + + +/** Opcode 0x0f 0x66 - pcmpgtd Pq, Qq */ +FNIEMOP_DEF(iemOp_pcmpgtd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PCMPGTD, pcmpgtd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpgtd_u64); +} + + +/** Opcode 0x66 0x0f 0x66 - pcmpgtd Vx, Wx */ +FNIEMOP_DEF(iemOp_pcmpgtd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PCMPGTD, pcmpgtd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpgtd_u128); +} + + +/* Opcode 0xf3 0x0f 0x66 - invalid */ + + +/** Opcode 0x0f 0x67 - packuswb Pq, Qq */ +FNIEMOP_DEF(iemOp_packuswb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PACKUSWB, packuswb, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_packuswb_u64); +} + + +/** Opcode 0x66 0x0f 0x67 - packuswb Vx, Wx */ +FNIEMOP_DEF(iemOp_packuswb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PACKUSWB, packuswb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_packuswb_u128); +} + + +/* Opcode 0xf3 0x0f 0x67 - invalid */ + + +/** Opcode 0x0f 0x68 - punpckhbw Pq, Qq + * @note Intel and AMD both uses Qd for the second parameter, however they + * both list it as a mmX/mem64 operand and intel describes it as being + * loaded as a qword, so it should be Qq, shouldn't it? */ +FNIEMOP_DEF(iemOp_punpckhbw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PUNPCKHBW, punpckhbw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmx_HighHigh_To_Full, iemAImpl_punpckhbw_u64); +} + + +/** Opcode 0x66 0x0f 0x68 - punpckhbw Vx, Wx */ +FNIEMOP_DEF(iemOp_punpckhbw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PUNPCKHBW, punpckhbw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_punpckhbw_u128); +} + + +/* Opcode 0xf3 0x0f 0x68 - invalid */ + + +/** Opcode 0x0f 0x69 - punpckhwd Pq, Qq + * @note Intel and AMD both uses Qd for the second parameter, however they + * both list it as a mmX/mem64 operand and intel describes it as being + * loaded as a qword, so it should be Qq, shouldn't it? */ +FNIEMOP_DEF(iemOp_punpckhwd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PUNPCKHWD, punpckhwd, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmx_HighHigh_To_Full, iemAImpl_punpckhwd_u64); +} + + +/** Opcode 0x66 0x0f 0x69 - punpckhwd Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_punpckhwd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PUNPCKHWD, punpckhwd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_punpckhwd_u128); + +} + + +/* Opcode 0xf3 0x0f 0x69 - invalid */ + + +/** Opcode 0x0f 0x6a - punpckhdq Pq, Qq + * @note Intel and AMD both uses Qd for the second parameter, however they + * both list it as a mmX/mem64 operand and intel describes it as being + * loaded as a qword, so it should be Qq, shouldn't it? */ +FNIEMOP_DEF(iemOp_punpckhdq_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PUNPCKHDQ, punpckhdq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmx_HighHigh_To_Full, iemAImpl_punpckhdq_u64); +} + + +/** Opcode 0x66 0x0f 0x6a - punpckhdq Vx, Wx */ +FNIEMOP_DEF(iemOp_punpckhdq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PUNPCKHDQ, punpckhdq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_punpckhdq_u128); +} + + +/* Opcode 0xf3 0x0f 0x6a - invalid */ + + +/** Opcode 0x0f 0x6b - packssdw Pq, Qd */ +FNIEMOP_DEF(iemOp_packssdw_Pq_Qd) +{ + IEMOP_MNEMONIC2(RM, PACKSSDW, packssdw, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_packssdw_u64); +} + + +/** Opcode 0x66 0x0f 0x6b - packssdw Vx, Wx */ +FNIEMOP_DEF(iemOp_packssdw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PACKSSDW, packssdw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_packssdw_u128); +} + + +/* Opcode 0xf3 0x0f 0x6b - invalid */ + + +/* Opcode 0x0f 0x6c - invalid */ + + +/** Opcode 0x66 0x0f 0x6c - punpcklqdq Vx, Wx */ +FNIEMOP_DEF(iemOp_punpcklqdq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PUNPCKLQDQ, punpcklqdq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_punpcklqdq_u128); +} + + +/* Opcode 0xf3 0x0f 0x6c - invalid */ +/* Opcode 0xf2 0x0f 0x6c - invalid */ + + +/* Opcode 0x0f 0x6d - invalid */ + + +/** Opcode 0x66 0x0f 0x6d - punpckhqdq Vx, Wx */ +FNIEMOP_DEF(iemOp_punpckhqdq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PUNPCKHQDQ, punpckhqdq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_punpckhqdq_u128); +} + + +/* Opcode 0xf3 0x0f 0x6d - invalid */ + + +FNIEMOP_DEF(iemOp_movd_q_Pd_Ey) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + /** + * @opcode 0x6e + * @opcodesub rex.w=1 + * @oppfx none + * @opcpuid mmx + * @opgroup og_mmx_datamove + * @opxcpttype 5 + * @optest 64-bit / op1=1 op2=2 -> op1=2 ftw=0xff + * @optest 64-bit / op1=0 op2=-42 -> op1=-42 ftw=0xff + */ + IEMOP_MNEMONIC2(RM, MOVQ, movq, Pq_WO, Eq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* MMX, greg64 */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* MMX, [mem64] */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /** + * @opdone + * @opcode 0x6e + * @opcodesub rex.w=0 + * @oppfx none + * @opcpuid mmx + * @opgroup og_mmx_datamove + * @opxcpttype 5 + * @opfunction iemOp_movd_q_Pd_Ey + * @optest op1=1 op2=2 -> op1=2 ftw=0xff + * @optest op1=0 op2=-42 -> op1=-42 ftw=0xff + */ + IEMOP_MNEMONIC2(RM, MOVD, movd, PdZx_WO, Ed, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* MMX, greg32 */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_MREG_U32_ZX_U64(IEM_GET_MODRM_REG_8(bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* MMX, [mem32] */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_MREG_U32_ZX_U64(IEM_GET_MODRM_REG_8(bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + +FNIEMOP_DEF(iemOp_movd_q_Vy_Ey) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + /** + * @opcode 0x6e + * @opcodesub rex.w=1 + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_simdint_datamove + * @opxcpttype 5 + * @optest 64-bit / op1=1 op2=2 -> op1=2 + * @optest 64-bit / op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(RM, MOVQ, movq, VqZx_WO, Eq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* XMM, greg64 */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* XMM, [mem64] */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /** + * @opdone + * @opcode 0x6e + * @opcodesub rex.w=0 + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_simdint_datamove + * @opxcpttype 5 + * @opfunction iemOp_movd_q_Vy_Ey + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(RM, MOVD, movd, VdZx_WO, Ed, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* XMM, greg32 */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_XREG_U32_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* XMM, [mem32] */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U32_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + +/* Opcode 0xf3 0x0f 0x6e - invalid */ + + +/** + * @opcode 0x6f + * @oppfx none + * @opcpuid mmx + * @opgroup og_mmx_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 ftw=0xff + * @optest op1=0 op2=-42 -> op1=-42 ftw=0xff + */ +FNIEMOP_DEF(iemOp_movq_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, MOVD, movd, Pq_WO, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** + * @opcode 0x6f + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_simdint_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movdqa_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, MOVDQA, movdqa, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** + * @opcode 0x6f + * @oppfx 0xf3 + * @opcpuid sse2 + * @opgroup og_sse2_simdint_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movdqu_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(RM, MOVDQU, movdqu, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x70 - pshufw Pq, Qq, Ib */ +FNIEMOP_DEF(iemOp_pshufw_Pq_Qq_Ib) +{ + IEMOP_MNEMONIC3(RMI, PSHUFW, pshufw, Pq, Qq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t const *, pSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pshufw_u64, pDst, pSrc, bImmArg); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_ARG_LOCAL_REF(uint64_t const *, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pshufw_u64, pDst, pSrc, bImmArg); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common worker for SSE2 instructions on the forms: + * pshufd xmm1, xmm2/mem128, imm8 + * pshufhw xmm1, xmm2/mem128, imm8 + * pshuflw xmm1, xmm2/mem128, imm8 + * + * Proper alignment of the 128-bit operand is enforced. + * Exceptions type 4. SSE2 cpuid checks. + */ +FNIEMOP_DEF_1(iemOpCommonSse2_pshufXX_Vx_Wx_Ib, PFNIEMAIMPLMEDIAPSHUFU128, pfnWorker) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnWorker, puDst, puSrc, bImmArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnWorker, puDst, puSrc, bImmArg); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0x70 - pshufd Vx, Wx, Ib */ +FNIEMOP_DEF(iemOp_pshufd_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(RMI, PSHUFD, pshufd, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_pshufXX_Vx_Wx_Ib, iemAImpl_pshufd_u128); +} + + +/** Opcode 0xf3 0x0f 0x70 - pshufhw Vx, Wx, Ib */ +FNIEMOP_DEF(iemOp_pshufhw_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(RMI, PSHUFHW, pshufhw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_pshufXX_Vx_Wx_Ib, iemAImpl_pshufhw_u128); +} + + +/** Opcode 0xf2 0x0f 0x70 - pshuflw Vx, Wx, Ib */ +FNIEMOP_DEF(iemOp_pshuflw_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(RMI, PSHUFLW, pshuflw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_pshufXX_Vx_Wx_Ib, iemAImpl_pshuflw_u128); +} + + +/** + * Common worker for MMX instructions of the form: + * psrlw mm, imm8 + * psraw mm, imm8 + * psllw mm, imm8 + * psrld mm, imm8 + * psrad mm, imm8 + * pslld mm, imm8 + * psrlq mm, imm8 + * psllq mm, imm8 + * + */ +FNIEMOP_DEF_2(iemOpCommonMmx_Shift_Imm, uint8_t, bRm, FNIEMAIMPLMEDIAPSHIFTU64, pfnU64) +{ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, immediate. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG_CONST(uint8_t, bShiftArg, /*=*/ bImm, 1); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, bShiftArg); + IEM_MC_MODIFIED_MREG_BY_REF(pDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory not supported. + */ + /// @todo Caller already enforced register mode?! + AssertFailedReturn(VINF_SUCCESS); + } +} + + +/** + * Common worker for SSE2 instructions of the form: + * psrlw xmm, imm8 + * psraw xmm, imm8 + * psllw xmm, imm8 + * psrld xmm, imm8 + * psrad xmm, imm8 + * pslld xmm, imm8 + * psrlq xmm, imm8 + * psllq xmm, imm8 + * + */ +FNIEMOP_DEF_2(iemOpCommonSse2_Shift_Imm, uint8_t, bRm, FNIEMAIMPLMEDIAPSHIFTU128, pfnU128) +{ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, immediate. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_ARG_CONST(uint8_t, bShiftArg, /*=*/ bImm, 1); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pfnU128, pDst, bShiftArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + /// @todo Caller already enforced register mode?! + AssertFailedReturn(VINF_SUCCESS); + } +} + + +/** Opcode 0x0f 0x71 11/2 - psrlw Nq, Ib */ +FNIEMOPRM_DEF(iemOp_Grp12_psrlw_Nq_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRLW, psrlw, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psrlw_imm_u64); +} + + +/** Opcode 0x66 0x0f 0x71 11/2. */ +FNIEMOPRM_DEF(iemOp_Grp12_psrlw_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRLW, psrlw, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrlw_imm_u128); +} + + +/** Opcode 0x0f 0x71 11/4. */ +FNIEMOPRM_DEF(iemOp_Grp12_psraw_Nq_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRAW, psraw, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psraw_imm_u64); +} + + +/** Opcode 0x66 0x0f 0x71 11/4. */ +FNIEMOPRM_DEF(iemOp_Grp12_psraw_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRAW, psraw, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psraw_imm_u128); +} + + +/** Opcode 0x0f 0x71 11/6. */ +FNIEMOPRM_DEF(iemOp_Grp12_psllw_Nq_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSLLW, psllw, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psllw_imm_u64); +} + + +/** Opcode 0x66 0x0f 0x71 11/6. */ +FNIEMOPRM_DEF(iemOp_Grp12_psllw_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSLLW, psllw, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psllw_imm_u128); +} + + +/** + * Group 12 jump table for register variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnGroup12RegReg[] = +{ + /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /2 */ iemOp_Grp12_psrlw_Nq_Ib, iemOp_Grp12_psrlw_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /4 */ iemOp_Grp12_psraw_Nq_Ib, iemOp_Grp12_psraw_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /6 */ iemOp_Grp12_psllw_Nq_Ib, iemOp_Grp12_psllw_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8) +}; +AssertCompile(RT_ELEMENTS(g_apfnGroup12RegReg) == 8*4); + + +/** Opcode 0x0f 0x71. */ +FNIEMOP_DEF(iemOp_Grp12) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + /* register, register */ + return FNIEMOP_CALL_1(g_apfnGroup12RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); + return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm); +} + + +/** Opcode 0x0f 0x72 11/2. */ +FNIEMOPRM_DEF(iemOp_Grp13_psrld_Nq_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRLD, psrld, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psrld_imm_u64); +} + + +/** Opcode 0x66 0x0f 0x72 11/2. */ +FNIEMOPRM_DEF(iemOp_Grp13_psrld_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRLD, psrld, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrld_imm_u128); +} + + +/** Opcode 0x0f 0x72 11/4. */ +FNIEMOPRM_DEF(iemOp_Grp13_psrad_Nq_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRAD, psrad, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psrad_imm_u64); +} + + +/** Opcode 0x66 0x0f 0x72 11/4. */ +FNIEMOPRM_DEF(iemOp_Grp13_psrad_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRAD, psrad, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrad_imm_u128); +} + + +/** Opcode 0x0f 0x72 11/6. */ +FNIEMOPRM_DEF(iemOp_Grp13_pslld_Nq_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSLLD, pslld, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_pslld_imm_u64); +} + +/** Opcode 0x66 0x0f 0x72 11/6. */ +FNIEMOPRM_DEF(iemOp_Grp13_pslld_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSLLD, pslld, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_pslld_imm_u128); +} + + +/** + * Group 13 jump table for register variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnGroup13RegReg[] = +{ + /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /2 */ iemOp_Grp13_psrld_Nq_Ib, iemOp_Grp13_psrld_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /4 */ iemOp_Grp13_psrad_Nq_Ib, iemOp_Grp13_psrad_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /6 */ iemOp_Grp13_pslld_Nq_Ib, iemOp_Grp13_pslld_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8) +}; +AssertCompile(RT_ELEMENTS(g_apfnGroup13RegReg) == 8*4); + +/** Opcode 0x0f 0x72. */ +FNIEMOP_DEF(iemOp_Grp13) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + /* register, register */ + return FNIEMOP_CALL_1(g_apfnGroup13RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); + return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm); +} + + +/** Opcode 0x0f 0x73 11/2. */ +FNIEMOPRM_DEF(iemOp_Grp14_psrlq_Nq_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRLQ, psrlq, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psrlq_imm_u64); +} + + +/** Opcode 0x66 0x0f 0x73 11/2. */ +FNIEMOPRM_DEF(iemOp_Grp14_psrlq_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRLQ, psrlq, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrlq_imm_u128); +} + + +/** Opcode 0x66 0x0f 0x73 11/3. */ +FNIEMOPRM_DEF(iemOp_Grp14_psrldq_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSRLDQ, psrldq, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrldq_imm_u128); +} + + +/** Opcode 0x0f 0x73 11/6. */ +FNIEMOPRM_DEF(iemOp_Grp14_psllq_Nq_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSLLQ, psllq, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psllq_imm_u64); +} + + +/** Opcode 0x66 0x0f 0x73 11/6. */ +FNIEMOPRM_DEF(iemOp_Grp14_psllq_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSLLQ, psllq, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psllq_imm_u128); +} + + +/** Opcode 0x66 0x0f 0x73 11/7. */ +FNIEMOPRM_DEF(iemOp_Grp14_pslldq_Ux_Ib) +{ +// IEMOP_MNEMONIC2(RI, PSLLDQ, pslldq, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_pslldq_imm_u128); +} + +/** + * Group 14 jump table for register variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnGroup14RegReg[] = +{ + /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /2 */ iemOp_Grp14_psrlq_Nq_Ib, iemOp_Grp14_psrlq_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /3 */ iemOp_InvalidWithRMNeedImm8, iemOp_Grp14_psrldq_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /4 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /6 */ iemOp_Grp14_psllq_Nq_Ib, iemOp_Grp14_psllq_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /7 */ iemOp_InvalidWithRMNeedImm8, iemOp_Grp14_pslldq_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, +}; +AssertCompile(RT_ELEMENTS(g_apfnGroup14RegReg) == 8*4); + + +/** Opcode 0x0f 0x73. */ +FNIEMOP_DEF(iemOp_Grp14) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + /* register, register */ + return FNIEMOP_CALL_1(g_apfnGroup14RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); + return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm); +} + + +/** Opcode 0x0f 0x74 - pcmpeqb Pq, Qq */ +FNIEMOP_DEF(iemOp_pcmpeqb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PCMPEQB, pcmpeqb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpeqb_u64); +} + + +/** Opcode 0x66 0x0f 0x74 - pcmpeqb Vx, Wx */ +FNIEMOP_DEF(iemOp_pcmpeqb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PCMPEQB, pcmpeqb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpeqb_u128); +} + + +/* Opcode 0xf3 0x0f 0x74 - invalid */ +/* Opcode 0xf2 0x0f 0x74 - invalid */ + + +/** Opcode 0x0f 0x75 - pcmpeqw Pq, Qq */ +FNIEMOP_DEF(iemOp_pcmpeqw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PCMPEQW, pcmpeqw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpeqw_u64); +} + + +/** Opcode 0x66 0x0f 0x75 - pcmpeqw Vx, Wx */ +FNIEMOP_DEF(iemOp_pcmpeqw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PCMPEQW, pcmpeqw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpeqw_u128); +} + + +/* Opcode 0xf3 0x0f 0x75 - invalid */ +/* Opcode 0xf2 0x0f 0x75 - invalid */ + + +/** Opcode 0x0f 0x76 - pcmpeqd Pq, Qq */ +FNIEMOP_DEF(iemOp_pcmpeqd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PCMPEQD, pcmpeqd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpeqd_u64); +} + + +/** Opcode 0x66 0x0f 0x76 - pcmpeqd Vx, Wx */ +FNIEMOP_DEF(iemOp_pcmpeqd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PCMPEQD, pcmpeqd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpeqd_u128); +} + + +/* Opcode 0xf3 0x0f 0x76 - invalid */ +/* Opcode 0xf2 0x0f 0x76 - invalid */ + + +/** Opcode 0x0f 0x77 - emms (vex has vzeroall and vzeroupper here) */ +FNIEMOP_DEF(iemOp_emms) +{ + IEMOP_MNEMONIC(emms, "emms"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0,0); + IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE(); + IEM_MC_MAYBE_RAISE_FPU_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_FROM_MMX_MODE(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + +/* Opcode 0x66 0x0f 0x77 - invalid */ +/* Opcode 0xf3 0x0f 0x77 - invalid */ +/* Opcode 0xf2 0x0f 0x77 - invalid */ + +/** Opcode 0x0f 0x78 - VMREAD Ey, Gy */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF(iemOp_vmread_Ey_Gy) +{ + IEMOP_MNEMONIC(vmread, "vmread Ey,Gy"); + IEMOP_HLP_IN_VMX_OPERATION("vmread", kVmxVDiag_Vmread); + IEMOP_HLP_VMX_INSTR("vmread", kVmxVDiag_Vmread); + IEMMODE const enmEffOpSize = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? IEMMODE_64BIT : IEMMODE_32BIT; + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES(); + if (enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Enc, 1); + IEM_MC_FETCH_GREG_U64(u64Enc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_2(iemCImpl_vmread_reg64, pu64Dst, u64Enc); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Enc, 1); + IEM_MC_FETCH_GREG_U32(u32Enc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_2(iemCImpl_vmread_reg32, pu32Dst, u32Enc); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_END(); + } + } + else + { + /* + * Memory, register. + */ + if (enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrVal, 1); + IEM_MC_ARG(uint64_t, u64Enc, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES(); + IEM_MC_FETCH_GREG_U64(u64Enc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_vmread_mem_reg64, iEffSeg, GCPtrVal, u64Enc); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrVal, 1); + IEM_MC_ARG(uint32_t, u32Enc, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES(); + IEM_MC_FETCH_GREG_U32(u32Enc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_vmread_mem_reg32, iEffSeg, GCPtrVal, u32Enc); + IEM_MC_END(); + } + } + return VINF_SUCCESS; +} +#else +FNIEMOP_STUB(iemOp_vmread_Ey_Gy); +#endif + +/* Opcode 0x66 0x0f 0x78 - AMD Group 17 */ +FNIEMOP_STUB(iemOp_AmdGrp17); +/* Opcode 0xf3 0x0f 0x78 - invalid */ +/* Opcode 0xf2 0x0f 0x78 - invalid */ + +/** Opcode 0x0f 0x79 - VMWRITE Gy, Ey */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF(iemOp_vmwrite_Gy_Ey) +{ + IEMOP_MNEMONIC(vmwrite, "vmwrite Gy,Ey"); + IEMOP_HLP_IN_VMX_OPERATION("vmwrite", kVmxVDiag_Vmwrite); + IEMOP_HLP_VMX_INSTR("vmwrite", kVmxVDiag_Vmwrite); + IEMMODE const enmEffOpSize = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? IEMMODE_64BIT : IEMMODE_32BIT; + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES(); + if (enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t, u64Val, 0); + IEM_MC_ARG(uint64_t, u64Enc, 1); + IEM_MC_FETCH_GREG_U64(u64Val, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U64(u64Enc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_2(iemCImpl_vmwrite_reg, u64Val, u64Enc); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint32_t, u32Val, 0); + IEM_MC_ARG(uint32_t, u32Enc, 1); + IEM_MC_FETCH_GREG_U32(u32Val, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U32(u32Enc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_CIMPL_2(iemCImpl_vmwrite_reg, u32Val, u32Enc); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrVal, 1); + IEM_MC_ARG(uint64_t, u64Enc, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES(); + IEM_MC_FETCH_GREG_U64(u64Enc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_vmwrite_mem, iEffSeg, GCPtrVal, u64Enc); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrVal, 1); + IEM_MC_ARG(uint32_t, u32Enc, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES(); + IEM_MC_FETCH_GREG_U32(u32Enc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_vmwrite_mem, iEffSeg, GCPtrVal, u32Enc); + IEM_MC_END(); + } + } + return VINF_SUCCESS; +} +#else +FNIEMOP_STUB(iemOp_vmwrite_Gy_Ey); +#endif +/* Opcode 0x66 0x0f 0x79 - invalid */ +/* Opcode 0xf3 0x0f 0x79 - invalid */ +/* Opcode 0xf2 0x0f 0x79 - invalid */ + +/* Opcode 0x0f 0x7a - invalid */ +/* Opcode 0x66 0x0f 0x7a - invalid */ +/* Opcode 0xf3 0x0f 0x7a - invalid */ +/* Opcode 0xf2 0x0f 0x7a - invalid */ + +/* Opcode 0x0f 0x7b - invalid */ +/* Opcode 0x66 0x0f 0x7b - invalid */ +/* Opcode 0xf3 0x0f 0x7b - invalid */ +/* Opcode 0xf2 0x0f 0x7b - invalid */ + +/* Opcode 0x0f 0x7c - invalid */ + + +/** Opcode 0x66 0x0f 0x7c - haddpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_haddpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, HADDPD, haddpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_haddpd_u128); +} + + +/* Opcode 0xf3 0x0f 0x7c - invalid */ + + +/** Opcode 0xf2 0x0f 0x7c - haddps Vps, Wps */ +FNIEMOP_DEF(iemOp_haddps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, HADDPS, haddps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_haddps_u128); +} + + +/* Opcode 0x0f 0x7d - invalid */ + + +/** Opcode 0x66 0x0f 0x7d - hsubpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_hsubpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, HSUBPD, hsubpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_hsubpd_u128); +} + + +/* Opcode 0xf3 0x0f 0x7d - invalid */ + + +/** Opcode 0xf2 0x0f 0x7d - hsubps Vps, Wps */ +FNIEMOP_DEF(iemOp_hsubps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, HSUBPS, hsubps, Vps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_hsubps_u128); +} + + +/** Opcode 0x0f 0x7e - movd_q Ey, Pd */ +FNIEMOP_DEF(iemOp_movd_q_Ey_Pd) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + /** + * @opcode 0x7e + * @opcodesub rex.w=1 + * @oppfx none + * @opcpuid mmx + * @opgroup og_mmx_datamove + * @opxcpttype 5 + * @optest 64-bit / op1=1 op2=2 -> op1=2 ftw=0xff + * @optest 64-bit / op1=0 op2=-42 -> op1=-42 ftw=0xff + */ + IEMOP_MNEMONIC2(MR, MOVQ, movq, Eq_WO, Pq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg64, MMX */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* [mem64], MMX */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /** + * @opdone + * @opcode 0x7e + * @opcodesub rex.w=0 + * @oppfx none + * @opcpuid mmx + * @opgroup og_mmx_datamove + * @opxcpttype 5 + * @opfunction iemOp_movd_q_Pd_Ey + * @optest op1=1 op2=2 -> op1=2 ftw=0xff + * @optest op1=0 op2=-42 -> op1=-42 ftw=0xff + */ + IEMOP_MNEMONIC2(MR, MOVD, movd, Ed_WO, Pd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg32, MMX */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MREG_U32(u32Tmp, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* [mem32], MMX */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MREG_U32(u32Tmp, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +FNIEMOP_DEF(iemOp_movd_q_Ey_Vy) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + /** + * @opcode 0x7e + * @opcodesub rex.w=1 + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_simdint_datamove + * @opxcpttype 5 + * @optest 64-bit / op1=1 op2=2 -> op1=2 + * @optest 64-bit / op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(MR, MOVQ, movq, Eq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg64, XMM */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U64(u64Tmp, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* [mem64], XMM */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U64(u64Tmp, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /** + * @opdone + * @opcode 0x7e + * @opcodesub rex.w=0 + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_simdint_datamove + * @opxcpttype 5 + * @opfunction iemOp_movd_q_Vy_Ey + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(MR, MOVD, movd, Ed_WO, Vd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg32, XMM */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U32(u32Tmp, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* [mem32], XMM */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U32(u32Tmp, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + +/** + * @opcode 0x7e + * @oppfx 0xf3 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype none + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movq_Vq_Wq) +{ + IEMOP_MNEMONIC2(RM, MOVQ, movq, VqZx_WO, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM128, XMM64. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM128, [mem64]. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/* Opcode 0xf2 0x0f 0x7e - invalid */ + + +/** Opcode 0x0f 0x7f - movq Qq, Pq */ +FNIEMOP_DEF(iemOp_movq_Qq_Pq) +{ + IEMOP_MNEMONIC2(MR, MOVQ, movq, Qq_WO, Pq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_IGNORES_REXW); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * MMX, MMX. + */ + /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */ + /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_RM_8(bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem64], MMX. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** Opcode 0x66 0x0f 0x7f - movdqa Wx,Vx */ +FNIEMOP_DEF(iemOp_movdqa_Wx_Vx) +{ + IEMOP_MNEMONIC2(MR, MOVDQA, movdqa, Wx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem128], XMM. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U128(u128Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** Opcode 0xf3 0x0f 0x7f - movdqu Wx,Vx */ +FNIEMOP_DEF(iemOp_movdqu_Wx_Vx) +{ + IEMOP_MNEMONIC2(MR, MOVDQU, movdqu, Wx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * [mem128], XMM. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U128(u128Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/* Opcode 0xf2 0x0f 0x7f - invalid */ + + + +/** Opcode 0x0f 0x80. */ +FNIEMOP_DEF(iemOp_jo_Jv) +{ + IEMOP_MNEMONIC(jo_Jv, "jo Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x81. */ +FNIEMOP_DEF(iemOp_jno_Jv) +{ + IEMOP_MNEMONIC(jno_Jv, "jno Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x82. */ +FNIEMOP_DEF(iemOp_jc_Jv) +{ + IEMOP_MNEMONIC(jc_Jv, "jc/jb/jnae Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x83. */ +FNIEMOP_DEF(iemOp_jnc_Jv) +{ + IEMOP_MNEMONIC(jnc_Jv, "jnc/jnb/jae Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x84. */ +FNIEMOP_DEF(iemOp_je_Jv) +{ + IEMOP_MNEMONIC(je_Jv, "je/jz Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x85. */ +FNIEMOP_DEF(iemOp_jne_Jv) +{ + IEMOP_MNEMONIC(jne_Jv, "jne/jnz Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x86. */ +FNIEMOP_DEF(iemOp_jbe_Jv) +{ + IEMOP_MNEMONIC(jbe_Jv, "jbe/jna Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x87. */ +FNIEMOP_DEF(iemOp_jnbe_Jv) +{ + IEMOP_MNEMONIC(ja_Jv, "jnbe/ja Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x88. */ +FNIEMOP_DEF(iemOp_js_Jv) +{ + IEMOP_MNEMONIC(js_Jv, "js Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x89. */ +FNIEMOP_DEF(iemOp_jns_Jv) +{ + IEMOP_MNEMONIC(jns_Jv, "jns Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x8a. */ +FNIEMOP_DEF(iemOp_jp_Jv) +{ + IEMOP_MNEMONIC(jp_Jv, "jp Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x8b. */ +FNIEMOP_DEF(iemOp_jnp_Jv) +{ + IEMOP_MNEMONIC(jnp_Jv, "jnp Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x8c. */ +FNIEMOP_DEF(iemOp_jl_Jv) +{ + IEMOP_MNEMONIC(jl_Jv, "jl/jnge Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x8d. */ +FNIEMOP_DEF(iemOp_jnl_Jv) +{ + IEMOP_MNEMONIC(jge_Jv, "jnl/jge Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x8e. */ +FNIEMOP_DEF(iemOp_jle_Jv) +{ + IEMOP_MNEMONIC(jle_Jv, "jle/jng Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ELSE() { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x8f. */ +FNIEMOP_DEF(iemOp_jnle_Jv) +{ + IEMOP_MNEMONIC(jg_Jv, "jnle/jg Jv"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT) + { + int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } + else + { + int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_ADVANCE_RIP_AND_FINISH(); + } IEM_MC_ELSE() { + IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm); + } IEM_MC_ENDIF(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x90. */ +FNIEMOP_DEF(iemOp_seto_Eb) +{ + IEMOP_MNEMONIC(seto_Eb, "seto Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x91. */ +FNIEMOP_DEF(iemOp_setno_Eb) +{ + IEMOP_MNEMONIC(setno_Eb, "setno Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x92. */ +FNIEMOP_DEF(iemOp_setc_Eb) +{ + IEMOP_MNEMONIC(setc_Eb, "setc Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x93. */ +FNIEMOP_DEF(iemOp_setnc_Eb) +{ + IEMOP_MNEMONIC(setnc_Eb, "setnc Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x94. */ +FNIEMOP_DEF(iemOp_sete_Eb) +{ + IEMOP_MNEMONIC(sete_Eb, "sete Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x95. */ +FNIEMOP_DEF(iemOp_setne_Eb) +{ + IEMOP_MNEMONIC(setne_Eb, "setne Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x96. */ +FNIEMOP_DEF(iemOp_setbe_Eb) +{ + IEMOP_MNEMONIC(setbe_Eb, "setbe Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x97. */ +FNIEMOP_DEF(iemOp_setnbe_Eb) +{ + IEMOP_MNEMONIC(setnbe_Eb, "setnbe Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x98. */ +FNIEMOP_DEF(iemOp_sets_Eb) +{ + IEMOP_MNEMONIC(sets_Eb, "sets Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x99. */ +FNIEMOP_DEF(iemOp_setns_Eb) +{ + IEMOP_MNEMONIC(setns_Eb, "setns Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x9a. */ +FNIEMOP_DEF(iemOp_setp_Eb) +{ + IEMOP_MNEMONIC(setp_Eb, "setp Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x9b. */ +FNIEMOP_DEF(iemOp_setnp_Eb) +{ + IEMOP_MNEMONIC(setnp_Eb, "setnp Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x9c. */ +FNIEMOP_DEF(iemOp_setl_Eb) +{ + IEMOP_MNEMONIC(setl_Eb, "setl Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x9d. */ +FNIEMOP_DEF(iemOp_setnl_Eb) +{ + IEMOP_MNEMONIC(setnl_Eb, "setnl Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x9e. */ +FNIEMOP_DEF(iemOp_setle_Eb) +{ + IEMOP_MNEMONIC(setle_Eb, "setle Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0x9f. */ +FNIEMOP_DEF(iemOp_setnle_Eb) +{ + IEMOP_MNEMONIC(setnle_Eb, "setnle Eb"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in + * any way. AMD says it's "unused", whatever that means. We're + * ignoring for now. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register target */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 0); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* memory target */ + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + } IEM_MC_ELSE() { + IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1); + } IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Common 'push segment-register' helper. + */ +FNIEMOP_DEF_1(iemOpCommonPushSReg, uint8_t, iReg) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + Assert(iReg < X86_SREG_FS || pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT); + IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_FETCH_SREG_U16(u16Value, iReg); + IEM_MC_PUSH_U16(u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_FETCH_SREG_ZX_U32(u32Value, iReg); + IEM_MC_PUSH_U32_SREG(u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_SREG_ZX_U64(u64Value, iReg); + IEM_MC_PUSH_U64(u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** Opcode 0x0f 0xa0. */ +FNIEMOP_DEF(iemOp_push_fs) +{ + IEMOP_MNEMONIC(push_fs, "push fs"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_FS); +} + + +/** Opcode 0x0f 0xa1. */ +FNIEMOP_DEF(iemOp_pop_fs) +{ + IEMOP_MNEMONIC(pop_fs, "pop fs"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_FS, pVCpu->iem.s.enmEffOpSize); +} + + +/** Opcode 0x0f 0xa2. */ +FNIEMOP_DEF(iemOp_cpuid) +{ + IEMOP_MNEMONIC(cpuid, "cpuid"); + IEMOP_HLP_MIN_486(); /* not all 486es. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_cpuid); +} + + +/** + * Common worker for iemOp_bt_Ev_Gv, iemOp_btc_Ev_Gv, iemOp_btr_Ev_Gv and + * iemOp_bts_Ev_Gv. + */ +FNIEMOP_DEF_1(iemOpCommonBit_Ev_Gv, PCIEMOPBINSIZES, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register destination. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_AND_LOCAL_U16(u16Src, 0xf); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_AND_LOCAL_U32(u32Src, 0x1f); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_AND_LOCAL_U64(u64Src, 0x3f); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* memory destination. */ + + uint32_t fAccess; + if (pImpl->pfnLockedU16) + fAccess = IEM_ACCESS_DATA_RW; + else /* BT */ + fAccess = IEM_ACCESS_DATA_R; + + /** @todo test negative bit offsets! */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(int16_t, i16AddrAdj); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + if (pImpl->pfnLockedU16) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ASSIGN(i16AddrAdj, u16Src); + IEM_MC_AND_ARG_U16(u16Src, 0x0f); + IEM_MC_SAR_LOCAL_S16(i16AddrAdj, 4); + IEM_MC_SHL_LOCAL_S16(i16AddrAdj, 1); + IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(GCPtrEffDst, i16AddrAdj); + IEM_MC_FETCH_EFLAGS(EFlags); + + IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess); + + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(int32_t, i32AddrAdj); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + if (pImpl->pfnLockedU16) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ASSIGN(i32AddrAdj, u32Src); + IEM_MC_AND_ARG_U32(u32Src, 0x1f); + IEM_MC_SAR_LOCAL_S32(i32AddrAdj, 5); + IEM_MC_SHL_LOCAL_S32(i32AddrAdj, 2); + IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(GCPtrEffDst, i32AddrAdj); + IEM_MC_FETCH_EFLAGS(EFlags); + + IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess); + + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(int64_t, i64AddrAdj); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + if (pImpl->pfnLockedU16) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ASSIGN(i64AddrAdj, u64Src); + IEM_MC_AND_ARG_U64(u64Src, 0x3f); + IEM_MC_SAR_LOCAL_S64(i64AddrAdj, 6); + IEM_MC_SHL_LOCAL_S64(i64AddrAdj, 3); + IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(GCPtrEffDst, i64AddrAdj); + IEM_MC_FETCH_EFLAGS(EFlags); + + IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess); + + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0x0f 0xa3. */ +FNIEMOP_DEF(iemOp_bt_Ev_Gv) +{ + IEMOP_MNEMONIC(bt_Ev_Gv, "bt Ev,Gv"); + IEMOP_HLP_MIN_386(); + return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_bt); +} + + +/** + * Common worker for iemOp_shrd_Ev_Gv_Ib and iemOp_shld_Ev_Gv_Ib. + */ +FNIEMOP_DEF_1(iemOpCommonShldShrd_Ib, PCIEMOPSHIFTDBLSIZES, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_OF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_CONST(uint8_t, cShiftArg, /*=*/cShift, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG_CONST(uint8_t, cShiftArg, /*=*/cShift, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags); + + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG_CONST(uint8_t, cShiftArg, /*=*/cShift, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(4, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint8_t, cShiftArg, 2); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEM_MC_ASSIGN(cShiftArg, cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(4, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint8_t, cShiftArg, 2); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEM_MC_ASSIGN(cShiftArg, cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(4, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint8_t, cShiftArg, 2); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift); + IEM_MC_ASSIGN(cShiftArg, cShift); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** + * Common worker for iemOp_shrd_Ev_Gv_CL and iemOp_shld_Ev_Gv_CL. + */ +FNIEMOP_DEF_1(iemOpCommonShldShrd_CL, PCIEMOPSHIFTDBLSIZES, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_OF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint8_t, cShiftArg, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint8_t, cShiftArg, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags); + + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint8_t, cShiftArg, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(4, 2); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint8_t, cShiftArg, 2); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(4, 2); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint8_t, cShiftArg, 2); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(4, 2); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint8_t, cShiftArg, 2); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + + +/** Opcode 0x0f 0xa4. */ +FNIEMOP_DEF(iemOp_shld_Ev_Gv_Ib) +{ + IEMOP_MNEMONIC(shld_Ev_Gv_Ib, "shld Ev,Gv,Ib"); + IEMOP_HLP_MIN_386(); + return FNIEMOP_CALL_1(iemOpCommonShldShrd_Ib, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shld_eflags)); +} + + +/** Opcode 0x0f 0xa5. */ +FNIEMOP_DEF(iemOp_shld_Ev_Gv_CL) +{ + IEMOP_MNEMONIC(shld_Ev_Gv_CL, "shld Ev,Gv,CL"); + IEMOP_HLP_MIN_386(); + return FNIEMOP_CALL_1(iemOpCommonShldShrd_CL, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shld_eflags)); +} + + +/** Opcode 0x0f 0xa8. */ +FNIEMOP_DEF(iemOp_push_gs) +{ + IEMOP_MNEMONIC(push_gs, "push gs"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_GS); +} + + +/** Opcode 0x0f 0xa9. */ +FNIEMOP_DEF(iemOp_pop_gs) +{ + IEMOP_MNEMONIC(pop_gs, "pop gs"); + IEMOP_HLP_MIN_386(); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_GS, pVCpu->iem.s.enmEffOpSize); +} + + +/** Opcode 0x0f 0xaa. */ +FNIEMOP_DEF(iemOp_rsm) +{ + IEMOP_MNEMONIC0(FIXED, RSM, rsm, DISOPTYPE_HARMLESS, 0); + IEMOP_HLP_MIN_386(); /* 386SL and later. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rsm); +} + + + +/** Opcode 0x0f 0xab. */ +FNIEMOP_DEF(iemOp_bts_Ev_Gv) +{ + IEMOP_MNEMONIC(bts_Ev_Gv, "bts Ev,Gv"); + IEMOP_HLP_MIN_386(); + return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_bts); +} + + +/** Opcode 0x0f 0xac. */ +FNIEMOP_DEF(iemOp_shrd_Ev_Gv_Ib) +{ + IEMOP_MNEMONIC(shrd_Ev_Gv_Ib, "shrd Ev,Gv,Ib"); + IEMOP_HLP_MIN_386(); + return FNIEMOP_CALL_1(iemOpCommonShldShrd_Ib, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shrd_eflags)); +} + + +/** Opcode 0x0f 0xad. */ +FNIEMOP_DEF(iemOp_shrd_Ev_Gv_CL) +{ + IEMOP_MNEMONIC(shrd_Ev_Gv_CL, "shrd Ev,Gv,CL"); + IEMOP_HLP_MIN_386(); + return FNIEMOP_CALL_1(iemOpCommonShldShrd_CL, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shrd_eflags)); +} + + +/** Opcode 0x0f 0xae mem/0. */ +FNIEMOP_DEF_1(iemOp_Grp15_fxsave, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fxsave, "fxsave m512"); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFxSaveRstor) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_fxsave, iEffSeg, GCPtrEff, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0xae mem/1. */ +FNIEMOP_DEF_1(iemOp_Grp15_fxrstor, uint8_t, bRm) +{ + IEMOP_MNEMONIC(fxrstor, "fxrstor m512"); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFxSaveRstor) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_fxrstor, iEffSeg, GCPtrEff, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** + * @opmaps grp15 + * @opcode !11/2 + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_mxcsrsm + * @opxcpttype 5 + * @optest op1=0 -> mxcsr=0 + * @optest op1=0x2083 -> mxcsr=0x2083 + * @optest op1=0xfffffffe -> value.xcpt=0xd + * @optest op1=0x2083 cr0|=ts -> value.xcpt=0x7 + * @optest op1=0x2083 cr0|=em -> value.xcpt=0x6 + * @optest op1=0x2083 cr0|=mp -> mxcsr=0x2083 + * @optest op1=0x2083 cr4&~=osfxsr -> value.xcpt=0x6 + * @optest op1=0x2083 cr0|=ts,em -> value.xcpt=0x6 + * @optest op1=0x2083 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6 + * @optest op1=0x2083 cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6 + * @optest op1=0x2083 cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6 + */ +FNIEMOP_DEF_1(iemOp_Grp15_ldmxcsr, uint8_t, bRm) +{ + IEMOP_MNEMONIC1(M_MEM, LDMXCSR, ldmxcsr, Md_RO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_ldmxcsr, iEffSeg, GCPtrEff); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** + * @opmaps grp15 + * @opcode !11/3 + * @oppfx none + * @opcpuid sse + * @opgroup og_sse_mxcsrsm + * @opxcpttype 5 + * @optest mxcsr=0 -> op1=0 + * @optest mxcsr=0x2083 -> op1=0x2083 + * @optest mxcsr=0x2084 cr0|=ts -> value.xcpt=0x7 + * @optest mxcsr=0x2085 cr0|=em -> value.xcpt=0x6 + * @optest mxcsr=0x2086 cr0|=mp -> op1=0x2086 + * @optest mxcsr=0x2087 cr4&~=osfxsr -> value.xcpt=0x6 + * @optest mxcsr=0x2088 cr0|=ts,em -> value.xcpt=0x6 + * @optest mxcsr=0x2089 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6 + * @optest mxcsr=0x208a cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6 + * @optest mxcsr=0x208b cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6 + */ +FNIEMOP_DEF_1(iemOp_Grp15_stmxcsr, uint8_t, bRm) +{ + IEMOP_MNEMONIC1(M_MEM, STMXCSR, stmxcsr, Md_WO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_stmxcsr, iEffSeg, GCPtrEff); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** + * @opmaps grp15 + * @opcode !11/4 + * @oppfx none + * @opcpuid xsave + * @opgroup og_system + * @opxcpttype none + */ +FNIEMOP_DEF_1(iemOp_Grp15_xsave, uint8_t, bRm) +{ + IEMOP_MNEMONIC1(M_MEM, XSAVE, xsave, M_RW, DISOPTYPE_HARMLESS, 0); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_xsave, iEffSeg, GCPtrEff, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** + * @opmaps grp15 + * @opcode !11/5 + * @oppfx none + * @opcpuid xsave + * @opgroup og_system + * @opxcpttype none + */ +FNIEMOP_DEF_1(iemOp_Grp15_xrstor, uint8_t, bRm) +{ + IEMOP_MNEMONIC1(M_MEM, XRSTOR, xrstor, M_RO, DISOPTYPE_HARMLESS, 0); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_3(iemCImpl_xrstor, iEffSeg, GCPtrEff, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; +} + +/** Opcode 0x0f 0xae mem/6. */ +FNIEMOP_STUB_1(iemOp_Grp15_xsaveopt, uint8_t, bRm); + +/** + * @opmaps grp15 + * @opcode !11/7 + * @oppfx none + * @opcpuid clfsh + * @opgroup og_cachectl + * @optest op1=1 -> + */ +FNIEMOP_DEF_1(iemOp_Grp15_clflush, uint8_t, bRm) +{ + IEMOP_MNEMONIC1(M_MEM, CLFLUSH, clflush, Mb_RO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fClFlush) + return FNIEMOP_CALL_1(iemOp_InvalidWithRMAllNeeded, bRm); + + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_clflush_clflushopt, iEffSeg, GCPtrEff); + IEM_MC_END(); + return VINF_SUCCESS; +} + +/** + * @opmaps grp15 + * @opcode !11/7 + * @oppfx 0x66 + * @opcpuid clflushopt + * @opgroup og_cachectl + * @optest op1=1 -> + */ +FNIEMOP_DEF_1(iemOp_Grp15_clflushopt, uint8_t, bRm) +{ + IEMOP_MNEMONIC1(M_MEM, CLFLUSHOPT, clflushopt, Mb_RO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fClFlushOpt) + return FNIEMOP_CALL_1(iemOp_InvalidWithRMAllNeeded, bRm); + + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_clflush_clflushopt, iEffSeg, GCPtrEff); + IEM_MC_END(); + return VINF_SUCCESS; +} + + +/** Opcode 0x0f 0xae 11b/5. */ +FNIEMOP_DEF_1(iemOp_Grp15_lfence, uint8_t, bRm) +{ + RT_NOREF_PV(bRm); + IEMOP_MNEMONIC(lfence, "lfence"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_BEGIN(0, 0); +#ifndef RT_ARCH_ARM64 + if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fSse2) +#endif + IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_lfence); +#ifndef RT_ARCH_ARM64 + else + IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_alt_mem_fence); +#endif + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0x0f 0xae 11b/6. */ +FNIEMOP_DEF_1(iemOp_Grp15_mfence, uint8_t, bRm) +{ + RT_NOREF_PV(bRm); + IEMOP_MNEMONIC(mfence, "mfence"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_BEGIN(0, 0); +#ifndef RT_ARCH_ARM64 + if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fSse2) +#endif + IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_mfence); +#ifndef RT_ARCH_ARM64 + else + IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_alt_mem_fence); +#endif + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0x0f 0xae 11b/7. */ +FNIEMOP_DEF_1(iemOp_Grp15_sfence, uint8_t, bRm) +{ + RT_NOREF_PV(bRm); + IEMOP_MNEMONIC(sfence, "sfence"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_BEGIN(0, 0); +#ifndef RT_ARCH_ARM64 + if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fSse2) +#endif + IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_sfence); +#ifndef RT_ARCH_ARM64 + else + IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_alt_mem_fence); +#endif + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); +} + + +/** Opcode 0xf3 0x0f 0xae 11b/0. */ +FNIEMOP_DEF_1(iemOp_Grp15_rdfsbase, uint8_t, bRm) +{ + IEMOP_MNEMONIC(rdfsbase, "rdfsbase Ry"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(1, 0); + IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT(); + IEM_MC_ARG(uint64_t, u64Dst, 0); + IEM_MC_FETCH_SREG_BASE_U64(u64Dst, X86_SREG_FS); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(1, 0); + IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT(); + IEM_MC_ARG(uint32_t, u32Dst, 0); + IEM_MC_FETCH_SREG_BASE_U32(u32Dst, X86_SREG_FS); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0xf3 0x0f 0xae 11b/1. */ +FNIEMOP_DEF_1(iemOp_Grp15_rdgsbase, uint8_t, bRm) +{ + IEMOP_MNEMONIC(rdgsbase, "rdgsbase Ry"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(1, 0); + IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT(); + IEM_MC_ARG(uint64_t, u64Dst, 0); + IEM_MC_FETCH_SREG_BASE_U64(u64Dst, X86_SREG_GS); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(1, 0); + IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT(); + IEM_MC_ARG(uint32_t, u32Dst, 0); + IEM_MC_FETCH_SREG_BASE_U32(u32Dst, X86_SREG_GS); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0xf3 0x0f 0xae 11b/2. */ +FNIEMOP_DEF_1(iemOp_Grp15_wrfsbase, uint8_t, bRm) +{ + IEMOP_MNEMONIC(wrfsbase, "wrfsbase Ry"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(1, 0); + IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT(); + IEM_MC_ARG(uint64_t, u64Dst, 0); + IEM_MC_FETCH_GREG_U64(u64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(u64Dst); + IEM_MC_STORE_SREG_BASE_U64(X86_SREG_FS, u64Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(1, 0); + IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT(); + IEM_MC_ARG(uint32_t, u32Dst, 0); + IEM_MC_FETCH_GREG_U32(u32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_SREG_BASE_U64(X86_SREG_FS, u32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0xf3 0x0f 0xae 11b/3. */ +FNIEMOP_DEF_1(iemOp_Grp15_wrgsbase, uint8_t, bRm) +{ + IEMOP_MNEMONIC(wrgsbase, "wrgsbase Ry"); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_BEGIN(1, 0); + IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT(); + IEM_MC_ARG(uint64_t, u64Dst, 0); + IEM_MC_FETCH_GREG_U64(u64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(u64Dst); + IEM_MC_STORE_SREG_BASE_U64(X86_SREG_GS, u64Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(1, 0); + IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT(); + IEM_MC_ARG(uint32_t, u32Dst, 0); + IEM_MC_FETCH_GREG_U32(u32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_SREG_BASE_U64(X86_SREG_GS, u32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * Group 15 jump table for register variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnGroup15RegReg[] = +{ /* pfx: none, 066h, 0f3h, 0f2h */ + /* /0 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_Grp15_rdfsbase, iemOp_InvalidWithRM, + /* /1 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_Grp15_rdgsbase, iemOp_InvalidWithRM, + /* /2 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_Grp15_wrfsbase, iemOp_InvalidWithRM, + /* /3 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_Grp15_wrgsbase, iemOp_InvalidWithRM, + /* /4 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /5 */ iemOp_Grp15_lfence, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /6 */ iemOp_Grp15_mfence, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /7 */ iemOp_Grp15_sfence, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, +}; +AssertCompile(RT_ELEMENTS(g_apfnGroup15RegReg) == 8*4); + + +/** + * Group 15 jump table for memory variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnGroup15MemReg[] = +{ /* pfx: none, 066h, 0f3h, 0f2h */ + /* /0 */ iemOp_Grp15_fxsave, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /1 */ iemOp_Grp15_fxrstor, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /2 */ iemOp_Grp15_ldmxcsr, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /3 */ iemOp_Grp15_stmxcsr, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /4 */ iemOp_Grp15_xsave, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /5 */ iemOp_Grp15_xrstor, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /6 */ iemOp_Grp15_xsaveopt, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /7 */ iemOp_Grp15_clflush, iemOp_Grp15_clflushopt, iemOp_InvalidWithRM, iemOp_InvalidWithRM, +}; +AssertCompile(RT_ELEMENTS(g_apfnGroup15MemReg) == 8*4); + + +/** Opcode 0x0f 0xae. */ +FNIEMOP_DEF(iemOp_Grp15) +{ + IEMOP_HLP_MIN_586(); /* Not entirely accurate nor needed, but useful for debugging 286 code. */ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + /* register, register */ + return FNIEMOP_CALL_1(g_apfnGroup15RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); + /* memory, register */ + return FNIEMOP_CALL_1(g_apfnGroup15MemReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); +} + + +/** Opcode 0x0f 0xaf. */ +FNIEMOP_DEF(iemOp_imul_Gv_Ev) +{ + IEMOP_MNEMONIC(imul_Gv_Ev, "imul Gv,Ev"); + IEMOP_HLP_MIN_386(); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_eflags)); +} + + +/** Opcode 0x0f 0xb0. */ +FNIEMOP_DEF(iemOp_cmpxchg_Eb_Gb) +{ + IEMOP_MNEMONIC(cmpxchg_Eb_Gb, "cmpxchg Eb,Gb"); + IEMOP_HLP_MIN_486(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING(); + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t *, pu8Al, 1); + IEM_MC_ARG(uint8_t, u8Src, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U8(pu8Al, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8, pu8Dst, pu8Al, u8Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8_locked, pu8Dst, pu8Al, u8Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(4, 3); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t *, pu8Al, 1); + IEM_MC_ARG(uint8_t, u8Src, 2); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint8_t, u8Al); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING(); + IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U8(u8Al, X86_GREG_xAX); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_REF_LOCAL(pu8Al, u8Al); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8, pu8Dst, pu8Al, u8Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8_locked, pu8Dst, pu8Al, u8Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Al); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** Opcode 0x0f 0xb1. */ +FNIEMOP_DEF(iemOp_cmpxchg_Ev_Gv) +{ + IEMOP_MNEMONIC(cmpxchg_Ev_Gv, "cmpxchg Ev,Gv"); + IEMOP_HLP_MIN_486(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t *, pu16Ax, 1); + IEM_MC_ARG(uint16_t, u16Src, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16Ax, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16, pu16Dst, pu16Ax, u16Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16_locked, pu16Dst, pu16Ax, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t *, pu32Eax, 1); + IEM_MC_ARG(uint32_t, u32Src, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32Eax, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32, pu32Dst, pu32Eax, u32Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32_locked, pu32Dst, pu32Eax, u32Src, pEFlags); + + IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) { + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + } IEM_MC_ELSE() { + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Eax); + } IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t *, pu64Rax, 1); +#ifdef RT_ARCH_X86 + IEM_MC_ARG(uint64_t *, pu64Src, 2); +#else + IEM_MC_ARG(uint64_t, u64Src, 2); +#endif + IEM_MC_ARG(uint32_t *, pEFlags, 3); + + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pu64Rax, X86_GREG_xAX); + IEM_MC_REF_EFLAGS(pEFlags); +#ifdef RT_ARCH_X86 + IEM_MC_REF_GREG_U64(pu64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, pu64Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, pu64Src, pEFlags); +#else + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, u64Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, u64Src, pEFlags); +#endif + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(4, 3); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t *, pu16Ax, 1); + IEM_MC_ARG(uint16_t, u16Src, 2); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint16_t, u16Ax); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING(); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U16(u16Ax, X86_GREG_xAX); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_REF_LOCAL(pu16Ax, u16Ax); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16, pu16Dst, pu16Ax, u16Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16_locked, pu16Dst, pu16Ax, u16Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Ax); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(4, 3); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t *, pu32Eax, 1); + IEM_MC_ARG(uint32_t, u32Src, 2); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint32_t, u32Eax); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING(); + IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U32(u32Eax, X86_GREG_xAX); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_REF_LOCAL(pu32Eax, u32Eax); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32, pu32Dst, pu32Eax, u32Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32_locked, pu32Dst, pu32Eax, u32Src, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + + IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) + IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u32Eax); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(4, 3); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t *, pu64Rax, 1); +#ifdef RT_ARCH_X86 + IEM_MC_ARG(uint64_t *, pu64Src, 2); +#else + IEM_MC_ARG(uint64_t, u64Src, 2); +#endif + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_LOCAL(uint64_t, u64Rax); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING(); + IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + IEM_MC_FETCH_GREG_U64(u64Rax, X86_GREG_xAX); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_REF_LOCAL(pu64Rax, u64Rax); +#ifdef RT_ARCH_X86 + IEM_MC_REF_GREG_U64(pu64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, pu64Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, pu64Src, pEFlags); +#else + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm)); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, u64Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, u64Src, pEFlags); +#endif + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Rax); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +FNIEMOP_DEF_2(iemOpCommonLoadSRegAndGreg, uint8_t, iSegReg, uint8_t, bRm) +{ + Assert(IEM_IS_MODRM_MEM_MODE(bRm)); /* Caller checks this */ + uint8_t const iGReg = IEM_GET_MODRM_REG(pVCpu, bRm); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(5, 1); + IEM_MC_ARG(uint16_t, uSel, 0); + IEM_MC_ARG(uint16_t, offSeg, 1); + IEM_MC_ARG_CONST(uint8_t, iSegRegArg,/*=*/iSegReg, 2); + IEM_MC_ARG_CONST(uint8_t, iGRegArg, /*=*/iGReg, 3); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEff); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff); + IEM_MC_FETCH_MEM_U16_DISP(uSel, pVCpu->iem.s.iEffSeg, GCPtrEff, 2); + IEM_MC_CALL_CIMPL_5(iemCImpl_load_SReg_Greg, uSel, offSeg, iSegRegArg, iGRegArg, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(5, 1); + IEM_MC_ARG(uint16_t, uSel, 0); + IEM_MC_ARG(uint32_t, offSeg, 1); + IEM_MC_ARG_CONST(uint8_t, iSegRegArg,/*=*/iSegReg, 2); + IEM_MC_ARG_CONST(uint8_t, iGRegArg, /*=*/iGReg, 3); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEff); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff); + IEM_MC_FETCH_MEM_U16_DISP(uSel, pVCpu->iem.s.iEffSeg, GCPtrEff, 4); + IEM_MC_CALL_CIMPL_5(iemCImpl_load_SReg_Greg, uSel, offSeg, iSegRegArg, iGRegArg, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(5, 1); + IEM_MC_ARG(uint16_t, uSel, 0); + IEM_MC_ARG(uint64_t, offSeg, 1); + IEM_MC_ARG_CONST(uint8_t, iSegRegArg,/*=*/iSegReg, 2); + IEM_MC_ARG_CONST(uint8_t, iGRegArg, /*=*/iGReg, 3); + IEM_MC_ARG_CONST(IEMMODE, enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 4); + IEM_MC_LOCAL(RTGCPTR, GCPtrEff); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (IEM_IS_GUEST_CPU_AMD(pVCpu)) /** @todo testcase: rev 3.15 of the amd manuals claims it only loads a 32-bit greg. */ + IEM_MC_FETCH_MEM_U32_SX_U64(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff); + else + IEM_MC_FETCH_MEM_U64(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff); + IEM_MC_FETCH_MEM_U16_DISP(uSel, pVCpu->iem.s.iEffSeg, GCPtrEff, 8); + IEM_MC_CALL_CIMPL_5(iemCImpl_load_SReg_Greg, uSel, offSeg, iSegRegArg, iGRegArg, enmEffOpSize); + IEM_MC_END(); + return VINF_SUCCESS; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** Opcode 0x0f 0xb2. */ +FNIEMOP_DEF(iemOp_lss_Gv_Mp) +{ + IEMOP_MNEMONIC(lss_Gv_Mp, "lss Gv,Mp"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + return IEMOP_RAISE_INVALID_OPCODE(); + return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_SS, bRm); +} + + +/** Opcode 0x0f 0xb3. */ +FNIEMOP_DEF(iemOp_btr_Ev_Gv) +{ + IEMOP_MNEMONIC(btr_Ev_Gv, "btr Ev,Gv"); + IEMOP_HLP_MIN_386(); + return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_btr); +} + + +/** Opcode 0x0f 0xb4. */ +FNIEMOP_DEF(iemOp_lfs_Gv_Mp) +{ + IEMOP_MNEMONIC(lfs_Gv_Mp, "lfs Gv,Mp"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + return IEMOP_RAISE_INVALID_OPCODE(); + return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_FS, bRm); +} + + +/** Opcode 0x0f 0xb5. */ +FNIEMOP_DEF(iemOp_lgs_Gv_Mp) +{ + IEMOP_MNEMONIC(lgs_Gv_Mp, "lgs Gv,Mp"); + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + return IEMOP_RAISE_INVALID_OPCODE(); + return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_GS, bRm); +} + + +/** Opcode 0x0f 0xb6. */ +FNIEMOP_DEF(iemOp_movzx_Gv_Eb) +{ + IEMOP_MNEMONIC(movzx_Gv_Eb, "movzx Gv,Eb"); + IEMOP_HLP_MIN_386(); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_FETCH_GREG_U8_ZX_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_FETCH_GREG_U8_ZX_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_GREG_U8_ZX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're loading a register from memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8_ZX_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8_ZX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8_ZX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0x0f 0xb7. */ +FNIEMOP_DEF(iemOp_movzx_Gv_Ew) +{ + IEMOP_MNEMONIC(movzx_Gv_Ew, "movzx Gv,Ew"); + IEMOP_HLP_MIN_386(); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Not entirely sure how the operand size prefix is handled here, + * assuming that it will be ignored. Would be nice to have a few + * test for this. */ + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT) + { + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_FETCH_GREG_U16_ZX_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_GREG_U16_ZX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * We're loading a register from memory. + */ + if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16_ZX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16_ZX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode 0x0f 0xb8 - JMPE (reserved for emulator on IPF) */ +FNIEMOP_UD_STUB(iemOp_jmpe); + + +/** Opcode 0xf3 0x0f 0xb8 - POPCNT Gv, Ev */ +FNIEMOP_DEF(iemOp_popcnt_Gv_Ev) +{ + IEMOP_MNEMONIC2(RM, POPCNT, popcnt, Gv, Ev, DISOPTYPE_HARMLESS, 0); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fPopCnt) + return iemOp_InvalidNeedRM(pVCpu); +#ifndef TST_IEM_CHECK_MC +# if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY) + static const IEMOPBINSIZES s_Native = + { NULL, NULL, iemAImpl_popcnt_u16, NULL, iemAImpl_popcnt_u32, NULL, iemAImpl_popcnt_u64, NULL }; +# endif + static const IEMOPBINSIZES s_Fallback = + { NULL, NULL, iemAImpl_popcnt_u16_fallback, NULL, iemAImpl_popcnt_u32_fallback, NULL, iemAImpl_popcnt_u64_fallback, NULL }; +#endif + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, IEM_SELECT_HOST_OR_FALLBACK(fPopCnt, &s_Native, &s_Fallback)); +} + + +/** + * @opcode 0xb9 + * @opinvalid intel-modrm + * @optest -> + */ +FNIEMOP_DEF(iemOp_Grp10) +{ + /* + * AMD does not decode beyond the 0xb9 whereas intel does the modr/m bit + * too. See bs3-cpu-decoder-1.c32. So, we can forward to iemOp_InvalidNeedRM. + */ + Log(("iemOp_Grp10 aka UD1 -> #UD\n")); + IEMOP_MNEMONIC2EX(ud1, "ud1", RM, UD1, ud1, Gb, Eb, DISOPTYPE_INVALID, IEMOPHINT_IGNORES_OP_SIZES); /* just picked Gb,Eb here. */ + return FNIEMOP_CALL(iemOp_InvalidNeedRM); +} + + +/** Opcode 0x0f 0xba. */ +FNIEMOP_DEF(iemOp_Grp8) +{ + IEMOP_HLP_MIN_386(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + PCIEMOPBINSIZES pImpl; + switch (IEM_GET_MODRM_REG_8(bRm)) + { + case 0: case 1: case 2: case 3: + /* Both AMD and Intel want full modr/m decoding and imm8. */ + return FNIEMOP_CALL_1(iemOp_InvalidWithRMAllNeedImm8, bRm); + case 4: pImpl = &g_iemAImpl_bt; IEMOP_MNEMONIC(bt_Ev_Ib, "bt Ev,Ib"); break; + case 5: pImpl = &g_iemAImpl_bts; IEMOP_MNEMONIC(bts_Ev_Ib, "bts Ev,Ib"); break; + case 6: pImpl = &g_iemAImpl_btr; IEMOP_MNEMONIC(btr_Ev_Ib, "btr Ev,Ib"); break; + case 7: pImpl = &g_iemAImpl_btc; IEMOP_MNEMONIC(btc_Ev_Ib, "btc Ev,Ib"); break; + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register destination. */ + uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG_CONST(uint16_t, u16Src, /*=*/ u8Bit & 0x0f, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG_CONST(uint32_t, u32Src, /*=*/ u8Bit & 0x1f, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG_CONST(uint64_t, u64Src, /*=*/ u8Bit & 0x3f, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* memory destination. */ + + uint32_t fAccess; + if (pImpl->pfnLockedU16) + fAccess = IEM_ACCESS_DATA_RW; + else /* BT */ + fAccess = IEM_ACCESS_DATA_R; + + /** @todo test negative bit offsets! */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit); + IEM_MC_ASSIGN(u16Src, u8Bit & 0x0f); + if (pImpl->pfnLockedU16) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess); + + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit); + IEM_MC_ASSIGN(u32Src, u8Bit & 0x1f); + if (pImpl->pfnLockedU16) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess); + + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG_LOCAL_EFLAGS( pEFlags, EFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1); + uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit); + IEM_MC_ASSIGN(u64Src, u8Bit & 0x3f); + if (pImpl->pfnLockedU16) + IEMOP_HLP_DONE_DECODING(); + else + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_EFLAGS(EFlags); + IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags); + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess); + + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0x0f 0xbb. */ +FNIEMOP_DEF(iemOp_btc_Ev_Gv) +{ + IEMOP_MNEMONIC(btc_Ev_Gv, "btc Ev,Gv"); + IEMOP_HLP_MIN_386(); + return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_btc); +} + + +/** + * Common worker for BSF and BSR instructions. + * + * These cannot use iemOpHlpBinaryOperator_rv_rm because they don't always write + * the destination register, which means that for 32-bit operations the high + * bits must be left alone. + * + * @param pImpl Pointer to the instruction implementation (assembly). + */ +FNIEMOP_DEF_1(iemOpHlpBitScanOperator_rv_rm, PCIEMOPBINSIZES, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're accessing memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t, u32Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags); + + IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0x0f 0xbc. */ +FNIEMOP_DEF(iemOp_bsf_Gv_Ev) +{ + IEMOP_MNEMONIC(bsf_Gv_Ev, "bsf Gv,Ev"); + IEMOP_HLP_MIN_386(); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); + return FNIEMOP_CALL_1(iemOpHlpBitScanOperator_rv_rm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_bsf_eflags)); +} + + +/** Opcode 0xf3 0x0f 0xbc - TZCNT Gv, Ev */ +FNIEMOP_DEF(iemOp_tzcnt_Gv_Ev) +{ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi1) + return FNIEMOP_CALL(iemOp_bsf_Gv_Ev); + IEMOP_MNEMONIC2(RM, TZCNT, tzcnt, Gv, Ev, DISOPTYPE_HARMLESS, 0); + +#ifndef TST_IEM_CHECK_MC + static const IEMOPBINSIZES s_iemAImpl_tzcnt = + { NULL, NULL, iemAImpl_tzcnt_u16, NULL, iemAImpl_tzcnt_u32, NULL, iemAImpl_tzcnt_u64, NULL }; + static const IEMOPBINSIZES s_iemAImpl_tzcnt_amd = + { NULL, NULL, iemAImpl_tzcnt_u16_amd, NULL, iemAImpl_tzcnt_u32_amd, NULL, iemAImpl_tzcnt_u64_amd, NULL }; + static const IEMOPBINSIZES s_iemAImpl_tzcnt_intel = + { NULL, NULL, iemAImpl_tzcnt_u16_intel, NULL, iemAImpl_tzcnt_u32_intel, NULL, iemAImpl_tzcnt_u64_intel, NULL }; + static const IEMOPBINSIZES * const s_iemAImpl_tzcnt_eflags[2][4] = + { + { &s_iemAImpl_tzcnt_intel, &s_iemAImpl_tzcnt_intel, &s_iemAImpl_tzcnt_amd, &s_iemAImpl_tzcnt_intel }, + { &s_iemAImpl_tzcnt, &s_iemAImpl_tzcnt_intel, &s_iemAImpl_tzcnt_amd, &s_iemAImpl_tzcnt } + }; +#endif + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, + IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(s_iemAImpl_tzcnt_eflags, IEM_GET_HOST_CPU_FEATURES(pVCpu)->fBmi1)); +} + + +/** Opcode 0x0f 0xbd. */ +FNIEMOP_DEF(iemOp_bsr_Gv_Ev) +{ + IEMOP_MNEMONIC(bsr_Gv_Ev, "bsr Gv,Ev"); + IEMOP_HLP_MIN_386(); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); + return FNIEMOP_CALL_1(iemOpHlpBitScanOperator_rv_rm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_bsr_eflags)); +} + + +/** Opcode 0xf3 0x0f 0xbd - LZCNT Gv, Ev */ +FNIEMOP_DEF(iemOp_lzcnt_Gv_Ev) +{ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi1) + return FNIEMOP_CALL(iemOp_bsr_Gv_Ev); + IEMOP_MNEMONIC2(RM, LZCNT, lzcnt, Gv, Ev, DISOPTYPE_HARMLESS, 0); + +#ifndef TST_IEM_CHECK_MC + static const IEMOPBINSIZES s_iemAImpl_lzcnt = + { NULL, NULL, iemAImpl_lzcnt_u16, NULL, iemAImpl_lzcnt_u32, NULL, iemAImpl_lzcnt_u64, NULL }; + static const IEMOPBINSIZES s_iemAImpl_lzcnt_amd = + { NULL, NULL, iemAImpl_lzcnt_u16_amd, NULL, iemAImpl_lzcnt_u32_amd, NULL, iemAImpl_lzcnt_u64_amd, NULL }; + static const IEMOPBINSIZES s_iemAImpl_lzcnt_intel = + { NULL, NULL, iemAImpl_lzcnt_u16_intel, NULL, iemAImpl_lzcnt_u32_intel, NULL, iemAImpl_lzcnt_u64_intel, NULL }; + static const IEMOPBINSIZES * const s_iemAImpl_lzcnt_eflags[2][4] = + { + { &s_iemAImpl_lzcnt_intel, &s_iemAImpl_lzcnt_intel, &s_iemAImpl_lzcnt_amd, &s_iemAImpl_lzcnt_intel }, + { &s_iemAImpl_lzcnt, &s_iemAImpl_lzcnt_intel, &s_iemAImpl_lzcnt_amd, &s_iemAImpl_lzcnt } + }; +#endif + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF); + return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, + IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(s_iemAImpl_lzcnt_eflags, IEM_GET_HOST_CPU_FEATURES(pVCpu)->fBmi1)); +} + + + +/** Opcode 0x0f 0xbe. */ +FNIEMOP_DEF(iemOp_movsx_Gv_Eb) +{ + IEMOP_MNEMONIC(movsx_Gv_Eb, "movsx Gv,Eb"); + IEMOP_HLP_MIN_386(); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_FETCH_GREG_U8_SX_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_FETCH_GREG_U8_SX_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_GREG_U8_SX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're loading a register from memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint16_t, u16Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8_SX_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8_SX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U8_SX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0x0f 0xbf. */ +FNIEMOP_DEF(iemOp_movsx_Gv_Ew) +{ + IEMOP_MNEMONIC(movsx_Gv_Ew, "movsx Gv,Ew"); + IEMOP_HLP_MIN_386(); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /** @todo Not entirely sure how the operand size prefix is handled here, + * assuming that it will be ignored. Would be nice to have a few + * test for this. */ + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT) + { + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_FETCH_GREG_U16_SX_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_FETCH_GREG_U16_SX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * We're loading a register from memory. + */ + if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16_SX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_FETCH_MEM_U16_SX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode 0x0f 0xc0. */ +FNIEMOP_DEF(iemOp_xadd_Eb_Gb) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + IEMOP_HLP_MIN_486(); + IEMOP_MNEMONIC(xadd_Eb_Gb, "xadd Eb,Gb"); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t *, pu8Reg, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U8(pu8Reg, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u8, pu8Dst, pu8Reg, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * We're accessing memory. + */ + IEM_MC_BEGIN(3, 3); + IEM_MC_ARG(uint8_t *, pu8Dst, 0); + IEM_MC_ARG(uint8_t *, pu8Reg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(uint8_t, u8RegCopy); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_GREG_U8(u8RegCopy, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu8Reg, u8RegCopy); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u8, pu8Dst, pu8Reg, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u8_locked, pu8Dst, pu8Reg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), u8RegCopy); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0xc1. */ +FNIEMOP_DEF(iemOp_xadd_Ev_Gv) +{ + IEMOP_MNEMONIC(xadd_Ev_Gv, "xadd Ev,Gv"); + IEMOP_HLP_MIN_486(); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* + * If rm is denoting a register, no more instruction bytes. + */ + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t *, pu16Reg, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U16(pu16Reg, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u16, pu16Dst, pu16Reg, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t *, pu32Reg, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pu32Reg, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u32, pu32Dst, pu32Reg, pEFlags); + + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Reg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t *, pu64Reg, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pu64Reg, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u64, pu64Dst, pu64Reg, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + { + /* + * We're accessing memory. + */ + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(3, 3); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint16_t *, pu16Reg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(uint16_t, u16RegCopy); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_GREG_U16(u16RegCopy, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu16Reg, u16RegCopy); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u16, pu16Dst, pu16Reg, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u16_locked, pu16Dst, pu16Reg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16RegCopy); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(3, 3); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t *, pu32Reg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(uint32_t, u32RegCopy); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_GREG_U32(u32RegCopy, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu32Reg, u32RegCopy); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u32, pu32Dst, pu32Reg, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u32_locked, pu32Dst, pu32Reg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32RegCopy); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(3, 3); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint64_t *, pu64Reg, 1); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2); + IEM_MC_LOCAL(uint64_t, u64RegCopy); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + IEM_MC_FETCH_GREG_U64(u64RegCopy, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_LOCAL(pu64Reg, u64RegCopy); + IEM_MC_FETCH_EFLAGS(EFlags); + if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u64, pu64Dst, pu64Reg, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u64_locked, pu64Dst, pu64Reg, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64RegCopy); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } +} + + +/** Opcode 0x0f 0xc2 - cmpps Vps,Wps,Ib */ +FNIEMOP_DEF(iemOp_cmpps_Vps_Wps_Ib) +{ + IEMOP_MNEMONIC3(RMI, CMPPS, cmpps, Vps, Wps, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpps_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(Src.uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpps_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0xc2 - cmppd Vpd,Wpd,Ib */ +FNIEMOP_DEF(iemOp_cmppd_Vpd_Wpd_Ib) +{ + IEMOP_MNEMONIC3(RMI, CMPPD, cmppd, Vpd, Wpd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmppd_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128]. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(Src.uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmppd_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0xf3 0x0f 0xc2 - cmpss Vss,Wss,Ib */ +FNIEMOP_DEF(iemOp_cmpss_Vss_Wss_Ib) +{ + IEMOP_MNEMONIC3(RMI, CMPSS, cmpss, Vss, Wss, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM32, XMM32. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpss_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM32, [mem32]. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U32(Src.uSrc2, 0 /*a_iDword */, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpss_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0xf2 0x0f 0xc2 - cmpsd Vsd,Wsd,Ib */ +FNIEMOP_DEF(iemOp_cmpsd_Vsd_Wsd_Ib) +{ + IEMOP_MNEMONIC3(RMI, CMPSD, cmpsd, Vsd, Wsd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM64, XMM64. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpsd_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iQword*/, Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM64, [mem64]. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(IEMMEDIAF2XMMSRC, Src); + IEM_MC_LOCAL(X86XMMREG, Dst); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(PX86XMMREG, pDst, Dst, 1); + IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC, pSrc, Src, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U64(Src.uSrc2, 0 /*a_iQword */, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpsd_u128, pfMxcsr, pDst, pSrc, bImmArg); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_STORE_XREG_XMM_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iQword*/, Dst); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0xc3. */ +FNIEMOP_DEF(iemOp_movnti_My_Gy) +{ + IEMOP_MNEMONIC(movnti_My_Gy, "movnti My,Gy"); + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + + /* Only the register -> memory form makes sense, assuming #UD for the other form. */ + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_32BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, u32Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, u64Value); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Value); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_16BIT: + /** @todo check this form. */ + return IEMOP_RAISE_INVALID_OPCODE(); + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/* Opcode 0x66 0x0f 0xc3 - invalid */ +/* Opcode 0xf3 0x0f 0xc3 - invalid */ +/* Opcode 0xf2 0x0f 0xc3 - invalid */ + + +/** Opcode 0x0f 0xc4 - pinsrw Pq, Ry/Mw,Ib */ +FNIEMOP_DEF(iemOp_pinsrw_Pq_RyMw_Ib) +{ + IEMOP_MNEMONIC3(RMI, PINSRW, pinsrw, Pq, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + IEM_MC_REF_MREG_U64(pu64Dst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pinsrw_u64, pu64Dst, u16Src, bImmArg); + IEM_MC_MODIFIED_MREG_BY_REF(pu64Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_MREG_U64(pu64Dst, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pinsrw_u64, pu64Dst, u16Src, bImmArg); + IEM_MC_MODIFIED_MREG_BY_REF(pu64Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0xc4 - pinsrw Vdq, Ry/Mw,Ib */ +FNIEMOP_DEF(iemOp_pinsrw_Vdq_RyMw_Ib) +{ + IEMOP_MNEMONIC3(RMI, PINSRW, pinsrw, Vq, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pinsrw_u128, puDst, u16Src, bImmArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(uint16_t, u16Src, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + + IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pinsrw_u128, puDst, u16Src, bImmArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode 0xf3 0x0f 0xc4 - invalid */ +/* Opcode 0xf2 0x0f 0xc4 - invalid */ + + +/** Opcode 0x0f 0xc5 - pextrw Gd, Nq, Ib */ +FNIEMOP_DEF(iemOp_pextrw_Gd_Nq_Ib) +{ + /*IEMOP_MNEMONIC3(RMI_REG, PEXTRW, pextrw, Gd, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);*/ /** @todo */ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Greg32, MMX, imm8. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(uint16_t, u16Dst); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Dst, u16Dst, 0); + IEM_MC_ARG(uint64_t, u64Src, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + IEM_MC_FETCH_MREG_U64(u64Src, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pextrw_u64, pu16Dst, u64Src, bImmArg); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u16Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + /* No memory operand. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode 0x66 0x0f 0xc5 - pextrw Gd, Udq, Ib */ +FNIEMOP_DEF(iemOp_pextrw_Gd_Udq_Ib) +{ + IEMOP_MNEMONIC3(RMI_REG, PEXTRW, pextrw, Gd, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Greg32, XMM, imm8. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(uint16_t, u16Dst); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Dst, u16Dst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pextrw_u128, pu16Dst, puSrc, bImmArg); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u16Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + /* No memory operand. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/* Opcode 0xf3 0x0f 0xc5 - invalid */ +/* Opcode 0xf2 0x0f 0xc5 - invalid */ + + +/** Opcode 0x0f 0xc6 - shufps Vps, Wps, Ib */ +FNIEMOP_DEF(iemOp_shufps_Vps_Wps_Ib) +{ + IEMOP_MNEMONIC3(RMI, SHUFPS, shufps, Vps, Wps, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM, imm8. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_ARG(PCRTUINT128U, pSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_shufps_u128, pDst, pSrc, bImmArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128], imm8. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_shufps_u128, pDst, pSrc, bImmArg); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x66 0x0f 0xc6 - shufpd Vpd, Wpd, Ib */ +FNIEMOP_DEF(iemOp_shufpd_Vpd_Wpd_Ib) +{ + IEMOP_MNEMONIC3(RMI, SHUFPD, shufpd, Vpd, Wpd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * XMM, XMM, imm8. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_ARG(PCRTUINT128U, pSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_shufpd_u128, pDst, pSrc, bImmArg); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * XMM, [mem128], imm8. + */ + IEM_MC_BEGIN(3, 2); + IEM_MC_ARG(PRTUINT128U, pDst, 0); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, pSrc, uSrc, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_shufpd_u128, pDst, pSrc, bImmArg); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode 0xf3 0x0f 0xc6 - invalid */ +/* Opcode 0xf2 0x0f 0xc6 - invalid */ + + +/** Opcode 0x0f 0xc7 !11/1. */ +FNIEMOP_DEF_1(iemOp_Grp9_cmpxchg8b_Mq, uint8_t, bRm) +{ + IEMOP_MNEMONIC(cmpxchg8b, "cmpxchg8b Mq"); + + IEM_MC_BEGIN(4, 3); + IEM_MC_ARG(uint64_t *, pu64MemDst, 0); + IEM_MC_ARG(PRTUINT64U, pu64EaxEdx, 1); + IEM_MC_ARG(PRTUINT64U, pu64EbxEcx, 2); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTUINT64U, u64EaxEdx); + IEM_MC_LOCAL(RTUINT64U, u64EbxEcx); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING(); + IEM_MC_MEM_MAP(pu64MemDst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + + IEM_MC_FETCH_GREG_U32(u64EaxEdx.s.Lo, X86_GREG_xAX); + IEM_MC_FETCH_GREG_U32(u64EaxEdx.s.Hi, X86_GREG_xDX); + IEM_MC_REF_LOCAL(pu64EaxEdx, u64EaxEdx); + + IEM_MC_FETCH_GREG_U32(u64EbxEcx.s.Lo, X86_GREG_xBX); + IEM_MC_FETCH_GREG_U32(u64EbxEcx.s.Hi, X86_GREG_xCX); + IEM_MC_REF_LOCAL(pu64EbxEcx, u64EbxEcx); + + IEM_MC_FETCH_EFLAGS(EFlags); + if ( !(pVCpu->iem.s.fDisregardLock) + && (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg8b_locked, pu64MemDst, pu64EaxEdx, pu64EbxEcx, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg8b, pu64MemDst, pu64EaxEdx, pu64EbxEcx, pEFlags); + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu64MemDst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) + IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u64EaxEdx.s.Lo); + IEM_MC_STORE_GREG_U32(X86_GREG_xDX, u64EaxEdx.s.Hi); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +} + + +/** Opcode REX.W 0x0f 0xc7 !11/1. */ +FNIEMOP_DEF_1(iemOp_Grp9_cmpxchg16b_Mdq, uint8_t, bRm) +{ + IEMOP_MNEMONIC(cmpxchg16b, "cmpxchg16b Mdq"); + if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovCmpXchg16b) + { +#if 0 + RT_NOREF(bRm); + IEMOP_BITCH_ABOUT_STUB(); + return VERR_IEM_INSTR_NOT_IMPLEMENTED; +#else + IEM_MC_BEGIN(4, 3); + IEM_MC_ARG(PRTUINT128U, pu128MemDst, 0); + IEM_MC_ARG(PRTUINT128U, pu128RaxRdx, 1); + IEM_MC_ARG(PRTUINT128U, pu128RbxRcx, 2); + IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 3); + IEM_MC_LOCAL(RTUINT128U, u128RaxRdx); + IEM_MC_LOCAL(RTUINT128U, u128RbxRcx); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING(); + IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(GCPtrEffDst, 16); + IEM_MC_MEM_MAP(pu128MemDst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); + + IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Lo, X86_GREG_xAX); + IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Hi, X86_GREG_xDX); + IEM_MC_REF_LOCAL(pu128RaxRdx, u128RaxRdx); + + IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Lo, X86_GREG_xBX); + IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Hi, X86_GREG_xCX); + IEM_MC_REF_LOCAL(pu128RbxRcx, u128RbxRcx); + + IEM_MC_FETCH_EFLAGS(EFlags); +# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_ARM64) +# if defined(RT_ARCH_AMD64) + if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fMovCmpXchg16b) +# endif + { + if ( !(pVCpu->iem.s.fDisregardLock) + && (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_locked, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags); + else + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags); + } +# if defined(RT_ARCH_AMD64) + else +# endif +# endif +# if !defined(RT_ARCH_ARM64) /** @todo may need this for unaligned accesses... */ + { + /* Note! The fallback for 32-bit systems and systems without CX16 is multiple + accesses and not all all atomic, which works fine on in UNI CPU guest + configuration (ignoring DMA). If guest SMP is active we have no choice + but to use a rendezvous callback here. Sigh. */ + if (pVCpu->CTX_SUFF(pVM)->cCpus == 1) + IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_fallback, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags); + else + { + IEM_MC_CALL_CIMPL_4(iemCImpl_cmpxchg16b_fallback_rendezvous, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags); + /* Does not get here, tail code is duplicated in iemCImpl_cmpxchg16b_fallback_rendezvous. */ + } + } +# endif + + IEM_MC_MEM_COMMIT_AND_UNMAP(pu128MemDst, IEM_ACCESS_DATA_RW); + IEM_MC_COMMIT_EFLAGS(EFlags); + IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) + IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u128RaxRdx.s.Lo); + IEM_MC_STORE_GREG_U64(X86_GREG_xDX, u128RaxRdx.s.Hi); + IEM_MC_ENDIF(); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + + IEM_MC_END(); +#endif + } + Log(("cmpxchg16b -> #UD\n")); + return IEMOP_RAISE_INVALID_OPCODE(); +} + +FNIEMOP_DEF_1(iemOp_Grp9_cmpxchg8bOr16b, uint8_t, bRm) +{ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + return FNIEMOP_CALL_1(iemOp_Grp9_cmpxchg16b_Mdq, bRm); + return FNIEMOP_CALL_1(iemOp_Grp9_cmpxchg8b_Mq, bRm); +} + + +/** Opcode 0x0f 0xc7 11/6. */ +FNIEMOP_DEF_1(iemOp_Grp9_rdrand_Rv, uint8_t, bRm) +{ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fRdRand) + return IEMOP_RAISE_INVALID_OPCODE(); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register destination. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdRand, iemAImpl_rdrand_u16, iemAImpl_rdrand_u16_fallback), + pu16Dst, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdRand, iemAImpl_rdrand_u32, iemAImpl_rdrand_u32_fallback), + pu32Dst, pEFlags); + + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdRand, iemAImpl_rdrand_u64, iemAImpl_rdrand_u64_fallback), + pu64Dst, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + /* Register only. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/** Opcode 0x0f 0xc7 !11/6. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF_1(iemOp_Grp9_vmptrld_Mq, uint8_t, bRm) +{ + IEMOP_MNEMONIC(vmptrld, "vmptrld"); + IEMOP_HLP_IN_VMX_OPERATION("vmptrld", kVmxVDiag_Vmptrld); + IEMOP_HLP_VMX_INSTR("vmptrld", kVmxVDiag_Vmptrld); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_vmptrld, iEffSeg, GCPtrEffSrc); + IEM_MC_END(); + return VINF_SUCCESS; +} +#else +FNIEMOP_UD_STUB_1(iemOp_Grp9_vmptrld_Mq, uint8_t, bRm); +#endif + +/** Opcode 0x66 0x0f 0xc7 !11/6. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF_1(iemOp_Grp9_vmclear_Mq, uint8_t, bRm) +{ + IEMOP_MNEMONIC(vmclear, "vmclear"); + IEMOP_HLP_IN_VMX_OPERATION("vmclear", kVmxVDiag_Vmclear); + IEMOP_HLP_VMX_INSTR("vmclear", kVmxVDiag_Vmclear); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_vmclear, iEffSeg, GCPtrEffDst); + IEM_MC_END(); + return VINF_SUCCESS; +} +#else +FNIEMOP_UD_STUB_1(iemOp_Grp9_vmclear_Mq, uint8_t, bRm); +#endif + +/** Opcode 0xf3 0x0f 0xc7 !11/6. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF_1(iemOp_Grp9_vmxon_Mq, uint8_t, bRm) +{ + IEMOP_MNEMONIC(vmxon, "vmxon"); + IEMOP_HLP_VMX_INSTR("vmxon", kVmxVDiag_Vmxon); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_vmxon, iEffSeg, GCPtrEffSrc); + IEM_MC_END(); + return VINF_SUCCESS; +} +#else +FNIEMOP_UD_STUB_1(iemOp_Grp9_vmxon_Mq, uint8_t, bRm); +#endif + +/** Opcode [0xf3] 0x0f 0xc7 !11/7. */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +FNIEMOP_DEF_1(iemOp_Grp9_vmptrst_Mq, uint8_t, bRm) +{ + IEMOP_MNEMONIC(vmptrst, "vmptrst"); + IEMOP_HLP_IN_VMX_OPERATION("vmptrst", kVmxVDiag_Vmptrst); + IEMOP_HLP_VMX_INSTR("vmptrst", kVmxVDiag_Vmptrst); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_vmptrst, iEffSeg, GCPtrEffDst); + IEM_MC_END(); + return VINF_SUCCESS; +} +#else +FNIEMOP_UD_STUB_1(iemOp_Grp9_vmptrst_Mq, uint8_t, bRm); +#endif + +/** Opcode 0x0f 0xc7 11/7. */ +FNIEMOP_DEF_1(iemOp_Grp9_rdseed_Rv, uint8_t, bRm) +{ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fRdSeed) + return IEMOP_RAISE_INVALID_OPCODE(); + + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* register destination. */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint16_t *, pu16Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + + IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdSeed, iemAImpl_rdseed_u16, iemAImpl_rdseed_u16_fallback), + pu16Dst, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + + IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdSeed, iemAImpl_rdseed_u32, iemAImpl_rdseed_u32_fallback), + pu32Dst, pEFlags); + + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_ARG(uint32_t *, pEFlags, 1); + + IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdSeed, iemAImpl_rdseed_u64, iemAImpl_rdseed_u64_fallback), + pu64Dst, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } + } + /* Register only. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/** + * Group 9 jump table for register variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnGroup9RegReg[] = +{ /* pfx: none, 066h, 0f3h, 0f2h */ + /* /0 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /1 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /2 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /3 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /4 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /5 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /6 */ iemOp_Grp9_rdrand_Rv, iemOp_Grp9_rdrand_Rv, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /7 */ iemOp_Grp9_rdseed_Rv, iemOp_Grp9_rdseed_Rv, iemOp_InvalidWithRM, iemOp_InvalidWithRM, +}; +AssertCompile(RT_ELEMENTS(g_apfnGroup9RegReg) == 8*4); + + +/** + * Group 9 jump table for memory variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnGroup9MemReg[] = +{ /* pfx: none, 066h, 0f3h, 0f2h */ + /* /0 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /1 */ iemOp_Grp9_cmpxchg8bOr16b, iemOp_Grp9_cmpxchg8bOr16b, iemOp_Grp9_cmpxchg8bOr16b, iemOp_Grp9_cmpxchg8bOr16b, /* see bs3-cpu-decoding-1 */ + /* /2 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /3 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /4 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /5 */ IEMOP_X4(iemOp_InvalidWithRM), + /* /6 */ iemOp_Grp9_vmptrld_Mq, iemOp_Grp9_vmclear_Mq, iemOp_Grp9_vmxon_Mq, iemOp_InvalidWithRM, + /* /7 */ iemOp_Grp9_vmptrst_Mq, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, +}; +AssertCompile(RT_ELEMENTS(g_apfnGroup9MemReg) == 8*4); + + +/** Opcode 0x0f 0xc7. */ +FNIEMOP_DEF(iemOp_Grp9) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_RM(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + /* register, register */ + return FNIEMOP_CALL_1(g_apfnGroup9RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); + /* memory, register */ + return FNIEMOP_CALL_1(g_apfnGroup9MemReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); +} + + +/** + * Common 'bswap register' helper. + */ +FNIEMOP_DEF_1(iemOpCommonBswapGReg, uint8_t, iReg) +{ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + switch (pVCpu->iem.s.enmEffOpSize) + { + case IEMMODE_16BIT: + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_REF_GREG_U32(pu32Dst, iReg); /* Don't clear the high dword! */ + IEM_MC_CALL_VOID_AIMPL_1(iemAImpl_bswap_u16, pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_32BIT: + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint32_t *, pu32Dst, 0); + IEM_MC_REF_GREG_U32(pu32Dst, iReg); + IEM_MC_CALL_VOID_AIMPL_1(iemAImpl_bswap_u32, pu32Dst); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + case IEMMODE_64BIT: + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint64_t *, pu64Dst, 0); + IEM_MC_REF_GREG_U64(pu64Dst, iReg); + IEM_MC_CALL_VOID_AIMPL_1(iemAImpl_bswap_u64, pu64Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + break; + + IEM_NOT_REACHED_DEFAULT_CASE_RET(); + } +} + + +/** Opcode 0x0f 0xc8. */ +FNIEMOP_DEF(iemOp_bswap_rAX_r8) +{ + IEMOP_MNEMONIC(bswap_rAX_r8, "bswap rAX/r8"); + /* Note! Intel manuals states that R8-R15 can be accessed by using a REX.X + prefix. REX.B is the correct prefix it appears. For a parallel + case, see iemOp_mov_AL_Ib and iemOp_mov_eAX_Iv. */ + IEMOP_HLP_MIN_486(); + return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xAX | pVCpu->iem.s.uRexB); +} + + +/** Opcode 0x0f 0xc9. */ +FNIEMOP_DEF(iemOp_bswap_rCX_r9) +{ + IEMOP_MNEMONIC(bswap_rCX_r9, "bswap rCX/r9"); + IEMOP_HLP_MIN_486(); + return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xCX | pVCpu->iem.s.uRexB); +} + + +/** Opcode 0x0f 0xca. */ +FNIEMOP_DEF(iemOp_bswap_rDX_r10) +{ + IEMOP_MNEMONIC(bswap_rDX_r9, "bswap rDX/r10"); + IEMOP_HLP_MIN_486(); + return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xDX | pVCpu->iem.s.uRexB); +} + + +/** Opcode 0x0f 0xcb. */ +FNIEMOP_DEF(iemOp_bswap_rBX_r11) +{ + IEMOP_MNEMONIC(bswap_rBX_r9, "bswap rBX/r11"); + IEMOP_HLP_MIN_486(); + return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xBX | pVCpu->iem.s.uRexB); +} + + +/** Opcode 0x0f 0xcc. */ +FNIEMOP_DEF(iemOp_bswap_rSP_r12) +{ + IEMOP_MNEMONIC(bswap_rSP_r12, "bswap rSP/r12"); + IEMOP_HLP_MIN_486(); + return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xSP | pVCpu->iem.s.uRexB); +} + + +/** Opcode 0x0f 0xcd. */ +FNIEMOP_DEF(iemOp_bswap_rBP_r13) +{ + IEMOP_MNEMONIC(bswap_rBP_r13, "bswap rBP/r13"); + IEMOP_HLP_MIN_486(); + return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xBP | pVCpu->iem.s.uRexB); +} + + +/** Opcode 0x0f 0xce. */ +FNIEMOP_DEF(iemOp_bswap_rSI_r14) +{ + IEMOP_MNEMONIC(bswap_rSI_r14, "bswap rSI/r14"); + IEMOP_HLP_MIN_486(); + return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xSI | pVCpu->iem.s.uRexB); +} + + +/** Opcode 0x0f 0xcf. */ +FNIEMOP_DEF(iemOp_bswap_rDI_r15) +{ + IEMOP_MNEMONIC(bswap_rDI_r15, "bswap rDI/r15"); + IEMOP_HLP_MIN_486(); + return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xDI | pVCpu->iem.s.uRexB); +} + + +/* Opcode 0x0f 0xd0 - invalid */ + + +/** Opcode 0x66 0x0f 0xd0 - addsubpd Vpd, Wpd */ +FNIEMOP_DEF(iemOp_addsubpd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(RM, ADDSUBPD, addsubpd, Vpd, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_addsubpd_u128); +} + + +/* Opcode 0xf3 0x0f 0xd0 - invalid */ + + +/** Opcode 0xf2 0x0f 0xd0 - addsubps Vps, Wps */ +FNIEMOP_DEF(iemOp_addsubps_Vps_Wps) +{ + IEMOP_MNEMONIC2(RM, ADDSUBPS, addsubps, Vps, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_addsubps_u128); +} + + + +/** Opcode 0x0f 0xd1 - psrlw Pq, Qq */ +FNIEMOP_DEF(iemOp_psrlw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSRLW, psrlw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psrlw_u64); +} + +/** Opcode 0x66 0x0f 0xd1 - psrlw Vx, Wx */ +FNIEMOP_DEF(iemOp_psrlw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSRLW, psrlw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psrlw_u128); +} + +/* Opcode 0xf3 0x0f 0xd1 - invalid */ +/* Opcode 0xf2 0x0f 0xd1 - invalid */ + +/** Opcode 0x0f 0xd2 - psrld Pq, Qq */ +FNIEMOP_DEF(iemOp_psrld_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSRLD, psrld, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psrld_u64); +} + + +/** Opcode 0x66 0x0f 0xd2 - psrld Vx, Wx */ +FNIEMOP_DEF(iemOp_psrld_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSRLD, psrld, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psrld_u128); +} + + +/* Opcode 0xf3 0x0f 0xd2 - invalid */ +/* Opcode 0xf2 0x0f 0xd2 - invalid */ + +/** Opcode 0x0f 0xd3 - psrlq Pq, Qq */ +FNIEMOP_DEF(iemOp_psrlq_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSRLQ, psrlq, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psrlq_u64); +} + + +/** Opcode 0x66 0x0f 0xd3 - psrlq Vx, Wx */ +FNIEMOP_DEF(iemOp_psrlq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSRLQ, psrlq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psrlq_u128); +} + + +/* Opcode 0xf3 0x0f 0xd3 - invalid */ +/* Opcode 0xf2 0x0f 0xd3 - invalid */ + + +/** Opcode 0x0f 0xd4 - paddq Pq, Qq */ +FNIEMOP_DEF(iemOp_paddq_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PADDQ, paddq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, iemAImpl_paddq_u64, IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2); +} + + +/** Opcode 0x66 0x0f 0xd4 - paddq Vx, Wx */ +FNIEMOP_DEF(iemOp_paddq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PADDQ, paddq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddq_u128); +} + + +/* Opcode 0xf3 0x0f 0xd4 - invalid */ +/* Opcode 0xf2 0x0f 0xd4 - invalid */ + +/** Opcode 0x0f 0xd5 - pmullw Pq, Qq */ +FNIEMOP_DEF(iemOp_pmullw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMULLW, pmullw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pmullw_u64); +} + +/** Opcode 0x66 0x0f 0xd5 - pmullw Vx, Wx */ +FNIEMOP_DEF(iemOp_pmullw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMULLW, pmullw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmullw_u128); +} + + +/* Opcode 0xf3 0x0f 0xd5 - invalid */ +/* Opcode 0xf2 0x0f 0xd5 - invalid */ + +/* Opcode 0x0f 0xd6 - invalid */ + +/** + * @opcode 0xd6 + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_pcksclr_datamove + * @opxcpttype none + * @optest op1=-1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movq_Wq_Vq) +{ + IEMOP_MNEMONIC2(MR, MOVQ, movq, WqZxReg_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_RM(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0xd6 + * @opcodesub 11 mr/reg + * @oppfx f3 + * @opcpuid sse2 + * @opgroup og_sse2_simdint_datamove + * @optest op1=1 op2=2 -> op1=2 ftw=0xff + * @optest op1=0 op2=-42 -> op1=-42 ftw=0xff + */ +FNIEMOP_DEF(iemOp_movq2dq_Vdq_Nq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_MNEMONIC2(RM_REG, MOVQ2DQ, movq2dq, VqZx_WO, Nq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MREG_U64(uSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic udf30fd6mem + * @opcode 0xd6 + * @opcodesub !11 mr/reg + * @oppfx f3 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + */ + else + return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedDecode, bRm); +} + + +/** + * @opcode 0xd6 + * @opcodesub 11 mr/reg + * @oppfx f2 + * @opcpuid sse2 + * @opgroup og_sse2_simdint_datamove + * @optest op1=1 op2=2 -> op1=2 ftw=0xff + * @optest op1=0 op2=-42 -> op1=-42 ftw=0xff + * @optest op1=0 op2=0x1123456789abcdef -> op1=0x1123456789abcdef ftw=0xff + * @optest op1=0 op2=0xfedcba9876543210 -> op1=0xfedcba9876543210 ftw=0xff + * @optest op1=-42 op2=0xfedcba9876543210 + * -> op1=0xfedcba9876543210 ftw=0xff + */ +FNIEMOP_DEF(iemOp_movdq2q_Pq_Uq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_MNEMONIC2(RM_REG, MOVDQ2Q, movdq2q, Pq_WO, Uq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, uSrc); + + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic udf20fd6mem + * @opcode 0xd6 + * @opcodesub !11 mr/reg + * @oppfx f2 + * @opunused intel-modrm + * @opcpuid sse + * @optest -> + */ + else + return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedDecode, bRm); +} + + +/** Opcode 0x0f 0xd7 - pmovmskb Gd, Nq */ +FNIEMOP_DEF(iemOp_pmovmskb_Gd_Nq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + /* Docs says register only. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) /** @todo test that this is registers only. */ + { + /* Note! Taking the lazy approch here wrt the high 32-bits of the GREG. */ + IEMOP_MNEMONIC2(RM_REG, PMOVMSKB, pmovmskb, Gd, Nq, DISOPTYPE_MMX | DISOPTYPE_HARMLESS, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, puDst, 0); + IEM_MC_ARG(uint64_t const *, puSrc, 1); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT(); + IEM_MC_PREPARE_FPU_USAGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_REF_GREG_U64(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_MREG_U64_CONST(puSrc, IEM_GET_MODRM_RM_8(bRm)); + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_pmovmskb_u64, puDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode 0x66 0x0f 0xd7 - */ +FNIEMOP_DEF(iemOp_pmovmskb_Gd_Ux) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + /* Docs says register only. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) /** @todo test that this is registers only. */ + { + /* Note! Taking the lazy approch here wrt the high 32-bits of the GREG. */ + IEMOP_MNEMONIC2(RM_REG, PMOVMSKB, pmovmskb, Gd, Ux, DISOPTYPE_SSE | DISOPTYPE_HARMLESS, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_PREPARE_SSE_USAGE(); + IEM_MC_REF_GREG_U64(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_pmovmskb_u128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/* Opcode 0xf3 0x0f 0xd7 - invalid */ +/* Opcode 0xf2 0x0f 0xd7 - invalid */ + + +/** Opcode 0x0f 0xd8 - psubusb Pq, Qq */ +FNIEMOP_DEF(iemOp_psubusb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSUBUSB, psubusb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubusb_u64); +} + + +/** Opcode 0x66 0x0f 0xd8 - psubusb Vx, Wx */ +FNIEMOP_DEF(iemOp_psubusb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSUBUSB, psubusb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubusb_u128); +} + + +/* Opcode 0xf3 0x0f 0xd8 - invalid */ +/* Opcode 0xf2 0x0f 0xd8 - invalid */ + +/** Opcode 0x0f 0xd9 - psubusw Pq, Qq */ +FNIEMOP_DEF(iemOp_psubusw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSUBUSW, psubusw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubusw_u64); +} + + +/** Opcode 0x66 0x0f 0xd9 - psubusw Vx, Wx */ +FNIEMOP_DEF(iemOp_psubusw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSUBUSW, psubusw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubusw_u128); +} + + +/* Opcode 0xf3 0x0f 0xd9 - invalid */ +/* Opcode 0xf2 0x0f 0xd9 - invalid */ + +/** Opcode 0x0f 0xda - pminub Pq, Qq */ +FNIEMOP_DEF(iemOp_pminub_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMINUB, pminub, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxSse_FullFull_To_Full, iemAImpl_pminub_u64); +} + + +/** Opcode 0x66 0x0f 0xda - pminub Vx, Wx */ +FNIEMOP_DEF(iemOp_pminub_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMINUB, pminub, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pminub_u128); +} + +/* Opcode 0xf3 0x0f 0xda - invalid */ +/* Opcode 0xf2 0x0f 0xda - invalid */ + +/** Opcode 0x0f 0xdb - pand Pq, Qq */ +FNIEMOP_DEF(iemOp_pand_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PAND, pand, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pand_u64); +} + + +/** Opcode 0x66 0x0f 0xdb - pand Vx, Wx */ +FNIEMOP_DEF(iemOp_pand_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PAND, pand, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pand_u128); +} + + +/* Opcode 0xf3 0x0f 0xdb - invalid */ +/* Opcode 0xf2 0x0f 0xdb - invalid */ + +/** Opcode 0x0f 0xdc - paddusb Pq, Qq */ +FNIEMOP_DEF(iemOp_paddusb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PADDUSB, paddusb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddusb_u64); +} + + +/** Opcode 0x66 0x0f 0xdc - paddusb Vx, Wx */ +FNIEMOP_DEF(iemOp_paddusb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PADDUSB, paddusb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddusb_u128); +} + + +/* Opcode 0xf3 0x0f 0xdc - invalid */ +/* Opcode 0xf2 0x0f 0xdc - invalid */ + +/** Opcode 0x0f 0xdd - paddusw Pq, Qq */ +FNIEMOP_DEF(iemOp_paddusw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PADDUSW, paddusw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddusw_u64); +} + + +/** Opcode 0x66 0x0f 0xdd - paddusw Vx, Wx */ +FNIEMOP_DEF(iemOp_paddusw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PADDUSW, paddusw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddusw_u128); +} + + +/* Opcode 0xf3 0x0f 0xdd - invalid */ +/* Opcode 0xf2 0x0f 0xdd - invalid */ + +/** Opcode 0x0f 0xde - pmaxub Pq, Qq */ +FNIEMOP_DEF(iemOp_pmaxub_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMAXUB, pmaxub, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxSse_FullFull_To_Full, iemAImpl_pmaxub_u64); +} + + +/** Opcode 0x66 0x0f 0xde - pmaxub Vx, W */ +FNIEMOP_DEF(iemOp_pmaxub_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMAXUB, pmaxub, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmaxub_u128); +} + +/* Opcode 0xf3 0x0f 0xde - invalid */ +/* Opcode 0xf2 0x0f 0xde - invalid */ + + +/** Opcode 0x0f 0xdf - pandn Pq, Qq */ +FNIEMOP_DEF(iemOp_pandn_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PANDN, pandn, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pandn_u64); +} + + +/** Opcode 0x66 0x0f 0xdf - pandn Vx, Wx */ +FNIEMOP_DEF(iemOp_pandn_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PANDN, pandn, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pandn_u128); +} + + +/* Opcode 0xf3 0x0f 0xdf - invalid */ +/* Opcode 0xf2 0x0f 0xdf - invalid */ + +/** Opcode 0x0f 0xe0 - pavgb Pq, Qq */ +FNIEMOP_DEF(iemOp_pavgb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PAVGB, pavgb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxSseOpt_FullFull_To_Full, iemAImpl_pavgb_u64); +} + + +/** Opcode 0x66 0x0f 0xe0 - pavgb Vx, Wx */ +FNIEMOP_DEF(iemOp_pavgb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PAVGB, pavgb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_pavgb_u128); +} + + +/* Opcode 0xf3 0x0f 0xe0 - invalid */ +/* Opcode 0xf2 0x0f 0xe0 - invalid */ + +/** Opcode 0x0f 0xe1 - psraw Pq, Qq */ +FNIEMOP_DEF(iemOp_psraw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSRAW, psraw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psraw_u64); +} + + +/** Opcode 0x66 0x0f 0xe1 - psraw Vx, Wx */ +FNIEMOP_DEF(iemOp_psraw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSRAW, psraw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psraw_u128); +} + + +/* Opcode 0xf3 0x0f 0xe1 - invalid */ +/* Opcode 0xf2 0x0f 0xe1 - invalid */ + +/** Opcode 0x0f 0xe2 - psrad Pq, Qq */ +FNIEMOP_DEF(iemOp_psrad_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSRAD, psrad, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psrad_u64); +} + + +/** Opcode 0x66 0x0f 0xe2 - psrad Vx, Wx */ +FNIEMOP_DEF(iemOp_psrad_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSRAD, psrad, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psrad_u128); +} + + +/* Opcode 0xf3 0x0f 0xe2 - invalid */ +/* Opcode 0xf2 0x0f 0xe2 - invalid */ + +/** Opcode 0x0f 0xe3 - pavgw Pq, Qq */ +FNIEMOP_DEF(iemOp_pavgw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PAVGW, pavgw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxSseOpt_FullFull_To_Full, iemAImpl_pavgw_u64); +} + + +/** Opcode 0x66 0x0f 0xe3 - pavgw Vx, Wx */ +FNIEMOP_DEF(iemOp_pavgw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PAVGW, pavgw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_pavgw_u128); +} + + +/* Opcode 0xf3 0x0f 0xe3 - invalid */ +/* Opcode 0xf2 0x0f 0xe3 - invalid */ + +/** Opcode 0x0f 0xe4 - pmulhuw Pq, Qq */ +FNIEMOP_DEF(iemOp_pmulhuw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMULHUW, pmulhuw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxSseOpt_FullFull_To_Full, iemAImpl_pmulhuw_u64); +} + + +/** Opcode 0x66 0x0f 0xe4 - pmulhuw Vx, Wx */ +FNIEMOP_DEF(iemOp_pmulhuw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMULHUW, pmulhuw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_pmulhuw_u128); +} + + +/* Opcode 0xf3 0x0f 0xe4 - invalid */ +/* Opcode 0xf2 0x0f 0xe4 - invalid */ + +/** Opcode 0x0f 0xe5 - pmulhw Pq, Qq */ +FNIEMOP_DEF(iemOp_pmulhw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMULHW, pmulhw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pmulhw_u64); +} + + +/** Opcode 0x66 0x0f 0xe5 - pmulhw Vx, Wx */ +FNIEMOP_DEF(iemOp_pmulhw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMULHW, pmulhw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmulhw_u128); +} + + +/* Opcode 0xf3 0x0f 0xe5 - invalid */ +/* Opcode 0xf2 0x0f 0xe5 - invalid */ +/* Opcode 0x0f 0xe6 - invalid */ + + +/** Opcode 0x66 0x0f 0xe6 - cvttpd2dq Vx, Wpd */ +FNIEMOP_DEF(iemOp_cvttpd2dq_Vx_Wpd) +{ + IEMOP_MNEMONIC2(RM, CVTTPD2DQ, cvttpd2dq, Vx, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvttpd2dq_u128); +} + + +/** Opcode 0xf3 0x0f 0xe6 - cvtdq2pd Vx, Wpd */ +FNIEMOP_DEF(iemOp_cvtdq2pd_Vx_Wpd) +{ + IEMOP_MNEMONIC2(RM, CVTDQ2PD, cvtdq2pd, Vx, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtdq2pd_u128); +} + + +/** Opcode 0xf2 0x0f 0xe6 - cvtpd2dq Vx, Wpd */ +FNIEMOP_DEF(iemOp_cvtpd2dq_Vx_Wpd) +{ + IEMOP_MNEMONIC2(RM, CVTPD2DQ, cvtpd2dq, Vx, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtpd2dq_u128); +} + + +/** + * @opcode 0xe7 + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid sse + * @opgroup og_sse1_cachect + * @opxcpttype none + * @optest op1=-1 op2=2 -> op1=2 ftw=0xff + * @optest op1=0 op2=-42 -> op1=-42 ftw=0xff + */ +FNIEMOP_DEF(iemOp_movntq_Mq_Pq) +{ + IEMOP_MNEMONIC2(MR_MEM, MOVNTQ, movntq, Mq_WO, Pq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* Register, memory. */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); + IEM_MC_FPU_TO_MMX_MODE(); + + IEM_MC_FETCH_MREG_U64(uSrc, IEM_GET_MODRM_REG_8(bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + /** + * @opdone + * @opmnemonic ud0fe7reg + * @opcode 0xe7 + * @opcodesub 11 mr/reg + * @oppfx none + * @opunused immediate + * @opcpuid sse + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/** + * @opcode 0xe7 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid sse2 + * @opgroup og_sse2_cachect + * @opxcpttype 1 + * @optest op1=-1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_movntdq_Mdq_Vdq) +{ + IEMOP_MNEMONIC2(MR_MEM, MOVNTDQ, movntdq, Mdq_WO, Vdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* Register, memory. */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic ud660fe7reg + * @opcode 0xe7 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid sse + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/* Opcode 0xf3 0x0f 0xe7 - invalid */ +/* Opcode 0xf2 0x0f 0xe7 - invalid */ + + +/** Opcode 0x0f 0xe8 - psubsb Pq, Qq */ +FNIEMOP_DEF(iemOp_psubsb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSUBSB, psubsb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubsb_u64); +} + + +/** Opcode 0x66 0x0f 0xe8 - psubsb Vx, Wx */ +FNIEMOP_DEF(iemOp_psubsb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSUBSB, psubsb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubsb_u128); +} + + +/* Opcode 0xf3 0x0f 0xe8 - invalid */ +/* Opcode 0xf2 0x0f 0xe8 - invalid */ + +/** Opcode 0x0f 0xe9 - psubsw Pq, Qq */ +FNIEMOP_DEF(iemOp_psubsw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSUBSW, psubsw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubsw_u64); +} + + +/** Opcode 0x66 0x0f 0xe9 - psubsw Vx, Wx */ +FNIEMOP_DEF(iemOp_psubsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSUBSW, psubsw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubsw_u128); +} + + +/* Opcode 0xf3 0x0f 0xe9 - invalid */ +/* Opcode 0xf2 0x0f 0xe9 - invalid */ + + +/** Opcode 0x0f 0xea - pminsw Pq, Qq */ +FNIEMOP_DEF(iemOp_pminsw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMINSW, pminsw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxSse_FullFull_To_Full, iemAImpl_pminsw_u64); +} + + +/** Opcode 0x66 0x0f 0xea - pminsw Vx, Wx */ +FNIEMOP_DEF(iemOp_pminsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMINSW, pminsw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pminsw_u128); +} + + +/* Opcode 0xf3 0x0f 0xea - invalid */ +/* Opcode 0xf2 0x0f 0xea - invalid */ + + +/** Opcode 0x0f 0xeb - por Pq, Qq */ +FNIEMOP_DEF(iemOp_por_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, POR, por, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_por_u64); +} + + +/** Opcode 0x66 0x0f 0xeb - por Vx, Wx */ +FNIEMOP_DEF(iemOp_por_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, POR, por, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_por_u128); +} + + +/* Opcode 0xf3 0x0f 0xeb - invalid */ +/* Opcode 0xf2 0x0f 0xeb - invalid */ + +/** Opcode 0x0f 0xec - paddsb Pq, Qq */ +FNIEMOP_DEF(iemOp_paddsb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PADDSB, paddsb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddsb_u64); +} + + +/** Opcode 0x66 0x0f 0xec - paddsb Vx, Wx */ +FNIEMOP_DEF(iemOp_paddsb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PADDSB, paddsb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddsb_u128); +} + + +/* Opcode 0xf3 0x0f 0xec - invalid */ +/* Opcode 0xf2 0x0f 0xec - invalid */ + +/** Opcode 0x0f 0xed - paddsw Pq, Qq */ +FNIEMOP_DEF(iemOp_paddsw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PADDSW, paddsw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddsw_u64); +} + + +/** Opcode 0x66 0x0f 0xed - paddsw Vx, Wx */ +FNIEMOP_DEF(iemOp_paddsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PADDSW, paddsw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddsw_u128); +} + + +/* Opcode 0xf3 0x0f 0xed - invalid */ +/* Opcode 0xf2 0x0f 0xed - invalid */ + + +/** Opcode 0x0f 0xee - pmaxsw Pq, Qq */ +FNIEMOP_DEF(iemOp_pmaxsw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMAXSW, pmaxsw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxSse_FullFull_To_Full, iemAImpl_pmaxsw_u64); +} + + +/** Opcode 0x66 0x0f 0xee - pmaxsw Vx, Wx */ +FNIEMOP_DEF(iemOp_pmaxsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMAXSW, pmaxsw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmaxsw_u128); +} + + +/* Opcode 0xf3 0x0f 0xee - invalid */ +/* Opcode 0xf2 0x0f 0xee - invalid */ + + +/** Opcode 0x0f 0xef - pxor Pq, Qq */ +FNIEMOP_DEF(iemOp_pxor_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PXOR, pxor, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pxor_u64); +} + + +/** Opcode 0x66 0x0f 0xef - pxor Vx, Wx */ +FNIEMOP_DEF(iemOp_pxor_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PXOR, pxor, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pxor_u128); +} + + +/* Opcode 0xf3 0x0f 0xef - invalid */ +/* Opcode 0xf2 0x0f 0xef - invalid */ + +/* Opcode 0x0f 0xf0 - invalid */ +/* Opcode 0x66 0x0f 0xf0 - invalid */ + + +/** Opcode 0xf2 0x0f 0xf0 - lddqu Vx, Mx */ +FNIEMOP_DEF(iemOp_lddqu_Vx_Mx) +{ + IEMOP_MNEMONIC2(RM_MEM, LDDQU, lddqu, Vdq_WO, Mx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register - (not implemented, assuming it raises \#UD). + */ + return IEMOP_RAISE_INVALID_OPCODE(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); + IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE(); + IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode 0x0f 0xf1 - psllw Pq, Qq */ +FNIEMOP_DEF(iemOp_psllw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSLLW, psllw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psllw_u64); +} + + +/** Opcode 0x66 0x0f 0xf1 - psllw Vx, Wx */ +FNIEMOP_DEF(iemOp_psllw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSLLW, psllw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psllw_u128); +} + + +/* Opcode 0xf2 0x0f 0xf1 - invalid */ + +/** Opcode 0x0f 0xf2 - pslld Pq, Qq */ +FNIEMOP_DEF(iemOp_pslld_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSLLD, pslld, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_pslld_u64); +} + + +/** Opcode 0x66 0x0f 0xf2 - pslld Vx, Wx */ +FNIEMOP_DEF(iemOp_pslld_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSLLD, pslld, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_pslld_u128); +} + + +/* Opcode 0xf2 0x0f 0xf2 - invalid */ + +/** Opcode 0x0f 0xf3 - psllq Pq, Qq */ +FNIEMOP_DEF(iemOp_psllq_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSLLQ, psllq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psllq_u64); +} + + +/** Opcode 0x66 0x0f 0xf3 - psllq Vx, Wx */ +FNIEMOP_DEF(iemOp_psllq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSLLQ, psllq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psllq_u128); +} + +/* Opcode 0xf2 0x0f 0xf3 - invalid */ + +/** Opcode 0x0f 0xf4 - pmuludq Pq, Qq */ +FNIEMOP_DEF(iemOp_pmuludq_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMULUDQ, pmuludq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pmuludq_u64); +} + + +/** Opcode 0x66 0x0f 0xf4 - pmuludq Vx, W */ +FNIEMOP_DEF(iemOp_pmuludq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMULUDQ, pmuludq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmuludq_u128); +} + + +/* Opcode 0xf2 0x0f 0xf4 - invalid */ + +/** Opcode 0x0f 0xf5 - pmaddwd Pq, Qq */ +FNIEMOP_DEF(iemOp_pmaddwd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PMADDWD, pmaddwd, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pmaddwd_u64); +} + + +/** Opcode 0x66 0x0f 0xf5 - pmaddwd Vx, Wx */ +FNIEMOP_DEF(iemOp_pmaddwd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PMADDWD, pmaddwd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmaddwd_u128); +} + +/* Opcode 0xf2 0x0f 0xf5 - invalid */ + +/** Opcode 0x0f 0xf6 - psadbw Pq, Qq */ +FNIEMOP_DEF(iemOp_psadbw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSADBW, psadbw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmxSseOpt_FullFull_To_Full, iemAImpl_psadbw_u64); +} + + +/** Opcode 0x66 0x0f 0xf6 - psadbw Vx, Wx */ +FNIEMOP_DEF(iemOp_psadbw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSADBW, psadbw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psadbw_u128); +} + + +/* Opcode 0xf2 0x0f 0xf6 - invalid */ + +/** Opcode 0x0f 0xf7 - maskmovq Pq, Nq */ +FNIEMOP_STUB(iemOp_maskmovq_Pq_Nq); +/** Opcode 0x66 0x0f 0xf7 - maskmovdqu Vdq, Udq */ +FNIEMOP_STUB(iemOp_maskmovdqu_Vdq_Udq); +/* Opcode 0xf2 0x0f 0xf7 - invalid */ + + +/** Opcode 0x0f 0xf8 - psubb Pq, Qq */ +FNIEMOP_DEF(iemOp_psubb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSUBB, psubb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubb_u64); +} + + +/** Opcode 0x66 0x0f 0xf8 - psubb Vx, Wx */ +FNIEMOP_DEF(iemOp_psubb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSUBB, psubb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubb_u128); +} + + +/* Opcode 0xf2 0x0f 0xf8 - invalid */ + + +/** Opcode 0x0f 0xf9 - psubw Pq, Qq */ +FNIEMOP_DEF(iemOp_psubw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSUBW, psubw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubw_u64); +} + + +/** Opcode 0x66 0x0f 0xf9 - psubw Vx, Wx */ +FNIEMOP_DEF(iemOp_psubw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSUBW, psubw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubw_u128); +} + + +/* Opcode 0xf2 0x0f 0xf9 - invalid */ + + +/** Opcode 0x0f 0xfa - psubd Pq, Qq */ +FNIEMOP_DEF(iemOp_psubd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSUBD, psubd, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubd_u64); +} + + +/** Opcode 0x66 0x0f 0xfa - psubd Vx, Wx */ +FNIEMOP_DEF(iemOp_psubd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSUBD, psubd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubd_u128); +} + + +/* Opcode 0xf2 0x0f 0xfa - invalid */ + + +/** Opcode 0x0f 0xfb - psubq Pq, Qq */ +FNIEMOP_DEF(iemOp_psubq_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PSUBQ, psubq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, iemAImpl_psubq_u64, IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2); +} + + +/** Opcode 0x66 0x0f 0xfb - psubq Vx, Wx */ +FNIEMOP_DEF(iemOp_psubq_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PSUBQ, psubq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubq_u128); +} + + +/* Opcode 0xf2 0x0f 0xfb - invalid */ + + +/** Opcode 0x0f 0xfc - paddb Pq, Qq */ +FNIEMOP_DEF(iemOp_paddb_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PADDB, paddb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddb_u64); +} + + +/** Opcode 0x66 0x0f 0xfc - paddb Vx, Wx */ +FNIEMOP_DEF(iemOp_paddb_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PADDB, paddb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddb_u128); +} + + +/* Opcode 0xf2 0x0f 0xfc - invalid */ + + +/** Opcode 0x0f 0xfd - paddw Pq, Qq */ +FNIEMOP_DEF(iemOp_paddw_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PADDW, paddw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddw_u64); +} + + +/** Opcode 0x66 0x0f 0xfd - paddw Vx, Wx */ +FNIEMOP_DEF(iemOp_paddw_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PADDW, paddw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddw_u128); +} + + +/* Opcode 0xf2 0x0f 0xfd - invalid */ + + +/** Opcode 0x0f 0xfe - paddd Pq, Qq */ +FNIEMOP_DEF(iemOp_paddd_Pq_Qq) +{ + IEMOP_MNEMONIC2(RM, PADDD, paddd, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddd_u64); +} + + +/** Opcode 0x66 0x0f 0xfe - paddd Vx, W */ +FNIEMOP_DEF(iemOp_paddd_Vx_Wx) +{ + IEMOP_MNEMONIC2(RM, PADDD, paddd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddd_u128); +} + + +/* Opcode 0xf2 0x0f 0xfe - invalid */ + + +/** Opcode **** 0x0f 0xff - UD0 */ +FNIEMOP_DEF(iemOp_ud0) +{ + IEMOP_MNEMONIC(ud0, "ud0"); + if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL) + { + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm); +#ifndef TST_IEM_CHECK_MC + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } +#endif + IEMOP_HLP_DONE_DECODING(); + } + return IEMOP_RAISE_INVALID_OPCODE(); +} + + + +/** + * Two byte opcode map, first byte 0x0f. + * + * @remarks The g_apfnVexMap1 table is currently a subset of this one, so please + * check if it needs updating as well when making changes. + */ +IEM_STATIC const PFNIEMOP g_apfnTwoByteMap[] = +{ + /* no prefix, 066h prefix f3h prefix, f2h prefix */ + /* 0x00 */ IEMOP_X4(iemOp_Grp6), + /* 0x01 */ IEMOP_X4(iemOp_Grp7), + /* 0x02 */ IEMOP_X4(iemOp_lar_Gv_Ew), + /* 0x03 */ IEMOP_X4(iemOp_lsl_Gv_Ew), + /* 0x04 */ IEMOP_X4(iemOp_Invalid), + /* 0x05 */ IEMOP_X4(iemOp_syscall), + /* 0x06 */ IEMOP_X4(iemOp_clts), + /* 0x07 */ IEMOP_X4(iemOp_sysret), + /* 0x08 */ IEMOP_X4(iemOp_invd), + /* 0x09 */ IEMOP_X4(iemOp_wbinvd), + /* 0x0a */ IEMOP_X4(iemOp_Invalid), + /* 0x0b */ IEMOP_X4(iemOp_ud2), + /* 0x0c */ IEMOP_X4(iemOp_Invalid), + /* 0x0d */ IEMOP_X4(iemOp_nop_Ev_GrpP), + /* 0x0e */ IEMOP_X4(iemOp_femms), + /* 0x0f */ IEMOP_X4(iemOp_3Dnow), + + /* 0x10 */ iemOp_movups_Vps_Wps, iemOp_movupd_Vpd_Wpd, iemOp_movss_Vss_Wss, iemOp_movsd_Vsd_Wsd, + /* 0x11 */ iemOp_movups_Wps_Vps, iemOp_movupd_Wpd_Vpd, iemOp_movss_Wss_Vss, iemOp_movsd_Wsd_Vsd, + /* 0x12 */ iemOp_movlps_Vq_Mq__movhlps, iemOp_movlpd_Vq_Mq, iemOp_movsldup_Vdq_Wdq, iemOp_movddup_Vdq_Wdq, + /* 0x13 */ iemOp_movlps_Mq_Vq, iemOp_movlpd_Mq_Vq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x14 */ iemOp_unpcklps_Vx_Wx, iemOp_unpcklpd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x15 */ iemOp_unpckhps_Vx_Wx, iemOp_unpckhpd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x16 */ iemOp_movhps_Vdq_Mq__movlhps_Vdq_Uq, iemOp_movhpd_Vdq_Mq, iemOp_movshdup_Vdq_Wdq, iemOp_InvalidNeedRM, + /* 0x17 */ iemOp_movhps_Mq_Vq, iemOp_movhpd_Mq_Vq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x18 */ IEMOP_X4(iemOp_prefetch_Grp16), + /* 0x19 */ IEMOP_X4(iemOp_nop_Ev), + /* 0x1a */ IEMOP_X4(iemOp_nop_Ev), + /* 0x1b */ IEMOP_X4(iemOp_nop_Ev), + /* 0x1c */ IEMOP_X4(iemOp_nop_Ev), + /* 0x1d */ IEMOP_X4(iemOp_nop_Ev), + /* 0x1e */ IEMOP_X4(iemOp_nop_Ev), + /* 0x1f */ IEMOP_X4(iemOp_nop_Ev), + + /* 0x20 */ iemOp_mov_Rd_Cd, iemOp_mov_Rd_Cd, iemOp_mov_Rd_Cd, iemOp_mov_Rd_Cd, + /* 0x21 */ iemOp_mov_Rd_Dd, iemOp_mov_Rd_Dd, iemOp_mov_Rd_Dd, iemOp_mov_Rd_Dd, + /* 0x22 */ iemOp_mov_Cd_Rd, iemOp_mov_Cd_Rd, iemOp_mov_Cd_Rd, iemOp_mov_Cd_Rd, + /* 0x23 */ iemOp_mov_Dd_Rd, iemOp_mov_Dd_Rd, iemOp_mov_Dd_Rd, iemOp_mov_Dd_Rd, + /* 0x24 */ iemOp_mov_Rd_Td, iemOp_mov_Rd_Td, iemOp_mov_Rd_Td, iemOp_mov_Rd_Td, + /* 0x25 */ iemOp_Invalid, iemOp_Invalid, iemOp_Invalid, iemOp_Invalid, + /* 0x26 */ iemOp_mov_Td_Rd, iemOp_mov_Td_Rd, iemOp_mov_Td_Rd, iemOp_mov_Td_Rd, + /* 0x27 */ iemOp_Invalid, iemOp_Invalid, iemOp_Invalid, iemOp_Invalid, + /* 0x28 */ iemOp_movaps_Vps_Wps, iemOp_movapd_Vpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x29 */ iemOp_movaps_Wps_Vps, iemOp_movapd_Wpd_Vpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2a */ iemOp_cvtpi2ps_Vps_Qpi, iemOp_cvtpi2pd_Vpd_Qpi, iemOp_cvtsi2ss_Vss_Ey, iemOp_cvtsi2sd_Vsd_Ey, + /* 0x2b */ iemOp_movntps_Mps_Vps, iemOp_movntpd_Mpd_Vpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2c */ iemOp_cvttps2pi_Ppi_Wps, iemOp_cvttpd2pi_Ppi_Wpd, iemOp_cvttss2si_Gy_Wss, iemOp_cvttsd2si_Gy_Wsd, + /* 0x2d */ iemOp_cvtps2pi_Ppi_Wps, iemOp_cvtpd2pi_Qpi_Wpd, iemOp_cvtss2si_Gy_Wss, iemOp_cvtsd2si_Gy_Wsd, + /* 0x2e */ iemOp_ucomiss_Vss_Wss, iemOp_ucomisd_Vsd_Wsd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2f */ iemOp_comiss_Vss_Wss, iemOp_comisd_Vsd_Wsd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0x30 */ IEMOP_X4(iemOp_wrmsr), + /* 0x31 */ IEMOP_X4(iemOp_rdtsc), + /* 0x32 */ IEMOP_X4(iemOp_rdmsr), + /* 0x33 */ IEMOP_X4(iemOp_rdpmc), + /* 0x34 */ IEMOP_X4(iemOp_sysenter), + /* 0x35 */ IEMOP_X4(iemOp_sysexit), + /* 0x36 */ IEMOP_X4(iemOp_Invalid), + /* 0x37 */ IEMOP_X4(iemOp_getsec), + /* 0x38 */ IEMOP_X4(iemOp_3byte_Esc_0f_38), + /* 0x39 */ IEMOP_X4(iemOp_InvalidNeed3ByteEscRM), + /* 0x3a */ IEMOP_X4(iemOp_3byte_Esc_0f_3a), + /* 0x3b */ IEMOP_X4(iemOp_InvalidNeed3ByteEscRMImm8), + /* 0x3c */ IEMOP_X4(iemOp_InvalidNeed3ByteEscRM), + /* 0x3d */ IEMOP_X4(iemOp_InvalidNeed3ByteEscRM), + /* 0x3e */ IEMOP_X4(iemOp_InvalidNeed3ByteEscRMImm8), + /* 0x3f */ IEMOP_X4(iemOp_InvalidNeed3ByteEscRMImm8), + + /* 0x40 */ IEMOP_X4(iemOp_cmovo_Gv_Ev), + /* 0x41 */ IEMOP_X4(iemOp_cmovno_Gv_Ev), + /* 0x42 */ IEMOP_X4(iemOp_cmovc_Gv_Ev), + /* 0x43 */ IEMOP_X4(iemOp_cmovnc_Gv_Ev), + /* 0x44 */ IEMOP_X4(iemOp_cmove_Gv_Ev), + /* 0x45 */ IEMOP_X4(iemOp_cmovne_Gv_Ev), + /* 0x46 */ IEMOP_X4(iemOp_cmovbe_Gv_Ev), + /* 0x47 */ IEMOP_X4(iemOp_cmovnbe_Gv_Ev), + /* 0x48 */ IEMOP_X4(iemOp_cmovs_Gv_Ev), + /* 0x49 */ IEMOP_X4(iemOp_cmovns_Gv_Ev), + /* 0x4a */ IEMOP_X4(iemOp_cmovp_Gv_Ev), + /* 0x4b */ IEMOP_X4(iemOp_cmovnp_Gv_Ev), + /* 0x4c */ IEMOP_X4(iemOp_cmovl_Gv_Ev), + /* 0x4d */ IEMOP_X4(iemOp_cmovnl_Gv_Ev), + /* 0x4e */ IEMOP_X4(iemOp_cmovle_Gv_Ev), + /* 0x4f */ IEMOP_X4(iemOp_cmovnle_Gv_Ev), + + /* 0x50 */ iemOp_movmskps_Gy_Ups, iemOp_movmskpd_Gy_Upd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x51 */ iemOp_sqrtps_Vps_Wps, iemOp_sqrtpd_Vpd_Wpd, iemOp_sqrtss_Vss_Wss, iemOp_sqrtsd_Vsd_Wsd, + /* 0x52 */ iemOp_rsqrtps_Vps_Wps, iemOp_InvalidNeedRM, iemOp_rsqrtss_Vss_Wss, iemOp_InvalidNeedRM, + /* 0x53 */ iemOp_rcpps_Vps_Wps, iemOp_InvalidNeedRM, iemOp_rcpss_Vss_Wss, iemOp_InvalidNeedRM, + /* 0x54 */ iemOp_andps_Vps_Wps, iemOp_andpd_Vpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x55 */ iemOp_andnps_Vps_Wps, iemOp_andnpd_Vpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x56 */ iemOp_orps_Vps_Wps, iemOp_orpd_Vpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x57 */ iemOp_xorps_Vps_Wps, iemOp_xorpd_Vpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x58 */ iemOp_addps_Vps_Wps, iemOp_addpd_Vpd_Wpd, iemOp_addss_Vss_Wss, iemOp_addsd_Vsd_Wsd, + /* 0x59 */ iemOp_mulps_Vps_Wps, iemOp_mulpd_Vpd_Wpd, iemOp_mulss_Vss_Wss, iemOp_mulsd_Vsd_Wsd, + /* 0x5a */ iemOp_cvtps2pd_Vpd_Wps, iemOp_cvtpd2ps_Vps_Wpd, iemOp_cvtss2sd_Vsd_Wss, iemOp_cvtsd2ss_Vss_Wsd, + /* 0x5b */ iemOp_cvtdq2ps_Vps_Wdq, iemOp_cvtps2dq_Vdq_Wps, iemOp_cvttps2dq_Vdq_Wps, iemOp_InvalidNeedRM, + /* 0x5c */ iemOp_subps_Vps_Wps, iemOp_subpd_Vpd_Wpd, iemOp_subss_Vss_Wss, iemOp_subsd_Vsd_Wsd, + /* 0x5d */ iemOp_minps_Vps_Wps, iemOp_minpd_Vpd_Wpd, iemOp_minss_Vss_Wss, iemOp_minsd_Vsd_Wsd, + /* 0x5e */ iemOp_divps_Vps_Wps, iemOp_divpd_Vpd_Wpd, iemOp_divss_Vss_Wss, iemOp_divsd_Vsd_Wsd, + /* 0x5f */ iemOp_maxps_Vps_Wps, iemOp_maxpd_Vpd_Wpd, iemOp_maxss_Vss_Wss, iemOp_maxsd_Vsd_Wsd, + + /* 0x60 */ iemOp_punpcklbw_Pq_Qd, iemOp_punpcklbw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x61 */ iemOp_punpcklwd_Pq_Qd, iemOp_punpcklwd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x62 */ iemOp_punpckldq_Pq_Qd, iemOp_punpckldq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x63 */ iemOp_packsswb_Pq_Qq, iemOp_packsswb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x64 */ iemOp_pcmpgtb_Pq_Qq, iemOp_pcmpgtb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x65 */ iemOp_pcmpgtw_Pq_Qq, iemOp_pcmpgtw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x66 */ iemOp_pcmpgtd_Pq_Qq, iemOp_pcmpgtd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x67 */ iemOp_packuswb_Pq_Qq, iemOp_packuswb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x68 */ iemOp_punpckhbw_Pq_Qq, iemOp_punpckhbw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x69 */ iemOp_punpckhwd_Pq_Qq, iemOp_punpckhwd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6a */ iemOp_punpckhdq_Pq_Qq, iemOp_punpckhdq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6b */ iemOp_packssdw_Pq_Qd, iemOp_packssdw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6c */ iemOp_InvalidNeedRM, iemOp_punpcklqdq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6d */ iemOp_InvalidNeedRM, iemOp_punpckhqdq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6e */ iemOp_movd_q_Pd_Ey, iemOp_movd_q_Vy_Ey, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6f */ iemOp_movq_Pq_Qq, iemOp_movdqa_Vdq_Wdq, iemOp_movdqu_Vdq_Wdq, iemOp_InvalidNeedRM, + + /* 0x70 */ iemOp_pshufw_Pq_Qq_Ib, iemOp_pshufd_Vx_Wx_Ib, iemOp_pshufhw_Vx_Wx_Ib, iemOp_pshuflw_Vx_Wx_Ib, + /* 0x71 */ IEMOP_X4(iemOp_Grp12), + /* 0x72 */ IEMOP_X4(iemOp_Grp13), + /* 0x73 */ IEMOP_X4(iemOp_Grp14), + /* 0x74 */ iemOp_pcmpeqb_Pq_Qq, iemOp_pcmpeqb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x75 */ iemOp_pcmpeqw_Pq_Qq, iemOp_pcmpeqw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x76 */ iemOp_pcmpeqd_Pq_Qq, iemOp_pcmpeqd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x77 */ iemOp_emms, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0x78 */ iemOp_vmread_Ey_Gy, iemOp_AmdGrp17, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x79 */ iemOp_vmwrite_Gy_Ey, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x7a */ iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x7b */ iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x7c */ iemOp_InvalidNeedRM, iemOp_haddpd_Vpd_Wpd, iemOp_InvalidNeedRM, iemOp_haddps_Vps_Wps, + /* 0x7d */ iemOp_InvalidNeedRM, iemOp_hsubpd_Vpd_Wpd, iemOp_InvalidNeedRM, iemOp_hsubps_Vps_Wps, + /* 0x7e */ iemOp_movd_q_Ey_Pd, iemOp_movd_q_Ey_Vy, iemOp_movq_Vq_Wq, iemOp_InvalidNeedRM, + /* 0x7f */ iemOp_movq_Qq_Pq, iemOp_movdqa_Wx_Vx, iemOp_movdqu_Wx_Vx, iemOp_InvalidNeedRM, + + /* 0x80 */ IEMOP_X4(iemOp_jo_Jv), + /* 0x81 */ IEMOP_X4(iemOp_jno_Jv), + /* 0x82 */ IEMOP_X4(iemOp_jc_Jv), + /* 0x83 */ IEMOP_X4(iemOp_jnc_Jv), + /* 0x84 */ IEMOP_X4(iemOp_je_Jv), + /* 0x85 */ IEMOP_X4(iemOp_jne_Jv), + /* 0x86 */ IEMOP_X4(iemOp_jbe_Jv), + /* 0x87 */ IEMOP_X4(iemOp_jnbe_Jv), + /* 0x88 */ IEMOP_X4(iemOp_js_Jv), + /* 0x89 */ IEMOP_X4(iemOp_jns_Jv), + /* 0x8a */ IEMOP_X4(iemOp_jp_Jv), + /* 0x8b */ IEMOP_X4(iemOp_jnp_Jv), + /* 0x8c */ IEMOP_X4(iemOp_jl_Jv), + /* 0x8d */ IEMOP_X4(iemOp_jnl_Jv), + /* 0x8e */ IEMOP_X4(iemOp_jle_Jv), + /* 0x8f */ IEMOP_X4(iemOp_jnle_Jv), + + /* 0x90 */ IEMOP_X4(iemOp_seto_Eb), + /* 0x91 */ IEMOP_X4(iemOp_setno_Eb), + /* 0x92 */ IEMOP_X4(iemOp_setc_Eb), + /* 0x93 */ IEMOP_X4(iemOp_setnc_Eb), + /* 0x94 */ IEMOP_X4(iemOp_sete_Eb), + /* 0x95 */ IEMOP_X4(iemOp_setne_Eb), + /* 0x96 */ IEMOP_X4(iemOp_setbe_Eb), + /* 0x97 */ IEMOP_X4(iemOp_setnbe_Eb), + /* 0x98 */ IEMOP_X4(iemOp_sets_Eb), + /* 0x99 */ IEMOP_X4(iemOp_setns_Eb), + /* 0x9a */ IEMOP_X4(iemOp_setp_Eb), + /* 0x9b */ IEMOP_X4(iemOp_setnp_Eb), + /* 0x9c */ IEMOP_X4(iemOp_setl_Eb), + /* 0x9d */ IEMOP_X4(iemOp_setnl_Eb), + /* 0x9e */ IEMOP_X4(iemOp_setle_Eb), + /* 0x9f */ IEMOP_X4(iemOp_setnle_Eb), + + /* 0xa0 */ IEMOP_X4(iemOp_push_fs), + /* 0xa1 */ IEMOP_X4(iemOp_pop_fs), + /* 0xa2 */ IEMOP_X4(iemOp_cpuid), + /* 0xa3 */ IEMOP_X4(iemOp_bt_Ev_Gv), + /* 0xa4 */ IEMOP_X4(iemOp_shld_Ev_Gv_Ib), + /* 0xa5 */ IEMOP_X4(iemOp_shld_Ev_Gv_CL), + /* 0xa6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa8 */ IEMOP_X4(iemOp_push_gs), + /* 0xa9 */ IEMOP_X4(iemOp_pop_gs), + /* 0xaa */ IEMOP_X4(iemOp_rsm), + /* 0xab */ IEMOP_X4(iemOp_bts_Ev_Gv), + /* 0xac */ IEMOP_X4(iemOp_shrd_Ev_Gv_Ib), + /* 0xad */ IEMOP_X4(iemOp_shrd_Ev_Gv_CL), + /* 0xae */ IEMOP_X4(iemOp_Grp15), + /* 0xaf */ IEMOP_X4(iemOp_imul_Gv_Ev), + + /* 0xb0 */ IEMOP_X4(iemOp_cmpxchg_Eb_Gb), + /* 0xb1 */ IEMOP_X4(iemOp_cmpxchg_Ev_Gv), + /* 0xb2 */ IEMOP_X4(iemOp_lss_Gv_Mp), + /* 0xb3 */ IEMOP_X4(iemOp_btr_Ev_Gv), + /* 0xb4 */ IEMOP_X4(iemOp_lfs_Gv_Mp), + /* 0xb5 */ IEMOP_X4(iemOp_lgs_Gv_Mp), + /* 0xb6 */ IEMOP_X4(iemOp_movzx_Gv_Eb), + /* 0xb7 */ IEMOP_X4(iemOp_movzx_Gv_Ew), + /* 0xb8 */ iemOp_jmpe, iemOp_InvalidNeedRM, iemOp_popcnt_Gv_Ev, iemOp_InvalidNeedRM, + /* 0xb9 */ IEMOP_X4(iemOp_Grp10), + /* 0xba */ IEMOP_X4(iemOp_Grp8), + /* 0xbb */ IEMOP_X4(iemOp_btc_Ev_Gv), // 0xf3? + /* 0xbc */ iemOp_bsf_Gv_Ev, iemOp_bsf_Gv_Ev, iemOp_tzcnt_Gv_Ev, iemOp_bsf_Gv_Ev, + /* 0xbd */ iemOp_bsr_Gv_Ev, iemOp_bsr_Gv_Ev, iemOp_lzcnt_Gv_Ev, iemOp_bsr_Gv_Ev, + /* 0xbe */ IEMOP_X4(iemOp_movsx_Gv_Eb), + /* 0xbf */ IEMOP_X4(iemOp_movsx_Gv_Ew), + + /* 0xc0 */ IEMOP_X4(iemOp_xadd_Eb_Gb), + /* 0xc1 */ IEMOP_X4(iemOp_xadd_Ev_Gv), + /* 0xc2 */ iemOp_cmpps_Vps_Wps_Ib, iemOp_cmppd_Vpd_Wpd_Ib, iemOp_cmpss_Vss_Wss_Ib, iemOp_cmpsd_Vsd_Wsd_Ib, + /* 0xc3 */ iemOp_movnti_My_Gy, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xc4 */ iemOp_pinsrw_Pq_RyMw_Ib, iemOp_pinsrw_Vdq_RyMw_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0xc5 */ iemOp_pextrw_Gd_Nq_Ib, iemOp_pextrw_Gd_Udq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0xc6 */ iemOp_shufps_Vps_Wps_Ib, iemOp_shufpd_Vpd_Wpd_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0xc7 */ IEMOP_X4(iemOp_Grp9), + /* 0xc8 */ IEMOP_X4(iemOp_bswap_rAX_r8), + /* 0xc9 */ IEMOP_X4(iemOp_bswap_rCX_r9), + /* 0xca */ IEMOP_X4(iemOp_bswap_rDX_r10), + /* 0xcb */ IEMOP_X4(iemOp_bswap_rBX_r11), + /* 0xcc */ IEMOP_X4(iemOp_bswap_rSP_r12), + /* 0xcd */ IEMOP_X4(iemOp_bswap_rBP_r13), + /* 0xce */ IEMOP_X4(iemOp_bswap_rSI_r14), + /* 0xcf */ IEMOP_X4(iemOp_bswap_rDI_r15), + + /* 0xd0 */ iemOp_InvalidNeedRM, iemOp_addsubpd_Vpd_Wpd, iemOp_InvalidNeedRM, iemOp_addsubps_Vps_Wps, + /* 0xd1 */ iemOp_psrlw_Pq_Qq, iemOp_psrlw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd2 */ iemOp_psrld_Pq_Qq, iemOp_psrld_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd3 */ iemOp_psrlq_Pq_Qq, iemOp_psrlq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd4 */ iemOp_paddq_Pq_Qq, iemOp_paddq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd5 */ iemOp_pmullw_Pq_Qq, iemOp_pmullw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd6 */ iemOp_InvalidNeedRM, iemOp_movq_Wq_Vq, iemOp_movq2dq_Vdq_Nq, iemOp_movdq2q_Pq_Uq, + /* 0xd7 */ iemOp_pmovmskb_Gd_Nq, iemOp_pmovmskb_Gd_Ux, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd8 */ iemOp_psubusb_Pq_Qq, iemOp_psubusb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd9 */ iemOp_psubusw_Pq_Qq, iemOp_psubusw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xda */ iemOp_pminub_Pq_Qq, iemOp_pminub_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdb */ iemOp_pand_Pq_Qq, iemOp_pand_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdc */ iemOp_paddusb_Pq_Qq, iemOp_paddusb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdd */ iemOp_paddusw_Pq_Qq, iemOp_paddusw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xde */ iemOp_pmaxub_Pq_Qq, iemOp_pmaxub_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdf */ iemOp_pandn_Pq_Qq, iemOp_pandn_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0xe0 */ iemOp_pavgb_Pq_Qq, iemOp_pavgb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe1 */ iemOp_psraw_Pq_Qq, iemOp_psraw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe2 */ iemOp_psrad_Pq_Qq, iemOp_psrad_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe3 */ iemOp_pavgw_Pq_Qq, iemOp_pavgw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe4 */ iemOp_pmulhuw_Pq_Qq, iemOp_pmulhuw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe5 */ iemOp_pmulhw_Pq_Qq, iemOp_pmulhw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe6 */ iemOp_InvalidNeedRM, iemOp_cvttpd2dq_Vx_Wpd, iemOp_cvtdq2pd_Vx_Wpd, iemOp_cvtpd2dq_Vx_Wpd, + /* 0xe7 */ iemOp_movntq_Mq_Pq, iemOp_movntdq_Mdq_Vdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe8 */ iemOp_psubsb_Pq_Qq, iemOp_psubsb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe9 */ iemOp_psubsw_Pq_Qq, iemOp_psubsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xea */ iemOp_pminsw_Pq_Qq, iemOp_pminsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xeb */ iemOp_por_Pq_Qq, iemOp_por_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xec */ iemOp_paddsb_Pq_Qq, iemOp_paddsb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xed */ iemOp_paddsw_Pq_Qq, iemOp_paddsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xee */ iemOp_pmaxsw_Pq_Qq, iemOp_pmaxsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xef */ iemOp_pxor_Pq_Qq, iemOp_pxor_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0xf0 */ iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_lddqu_Vx_Mx, + /* 0xf1 */ iemOp_psllw_Pq_Qq, iemOp_psllw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf2 */ iemOp_pslld_Pq_Qq, iemOp_pslld_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf3 */ iemOp_psllq_Pq_Qq, iemOp_psllq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf4 */ iemOp_pmuludq_Pq_Qq, iemOp_pmuludq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf5 */ iemOp_pmaddwd_Pq_Qq, iemOp_pmaddwd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf6 */ iemOp_psadbw_Pq_Qq, iemOp_psadbw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf7 */ iemOp_maskmovq_Pq_Nq, iemOp_maskmovdqu_Vdq_Udq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf8 */ iemOp_psubb_Pq_Qq, iemOp_psubb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf9 */ iemOp_psubw_Pq_Qq, iemOp_psubw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfa */ iemOp_psubd_Pq_Qq, iemOp_psubd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfb */ iemOp_psubq_Pq_Qq, iemOp_psubq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfc */ iemOp_paddb_Pq_Qq, iemOp_paddb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfd */ iemOp_paddw_Pq_Qq, iemOp_paddw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfe */ iemOp_paddd_Pq_Qq, iemOp_paddd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xff */ IEMOP_X4(iemOp_ud0), +}; +AssertCompile(RT_ELEMENTS(g_apfnTwoByteMap) == 1024); + +/** @} */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h new file mode 100644 index 00000000..f60f30ea --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h @@ -0,0 +1,5642 @@ +/* $Id: IEMAllInstructionsVexMap1.cpp.h $ */ +/** @file + * IEM - Instruction Decoding and Emulation. + * + * @remarks IEMAllInstructionsTwoByte0f.cpp.h is a legacy mirror of this file. + * Any update here is likely needed in that file too. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/** @name VEX Opcode Map 1 + * @{ + */ + +/** + * Common worker for AVX2 instructions on the forms: + * - vpxxx xmm0, xmm1, xmm2/mem128 + * - vpxxx ymm0, ymm1, ymm2/mem256 + * + * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit. + */ +FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, PCIEMOPMEDIAF3, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + if (pVCpu->iem.s.uVexLength) + { + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 2); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 3); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_YREG_U256(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_AVX_AIMPL_3(pImpl->pfnU256, puDst, puSrc1, puSrc2); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_BEGIN(4, 0); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG(PRTUINT128U, puDst, 1); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 2); + IEM_MC_ARG(PCRTUINT128U, puSrc2, 3); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_AVX_AIMPL_3(pImpl->pfnU128, puDst, puSrc1, puSrc2); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength) + { + IEM_MC_BEGIN(4, 4); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 2); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 3); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_CALL_AVX_AIMPL_3(pImpl->pfnU256, puDst, puSrc1, puSrc2); + IEM_MC_STORE_YREG_U256_ZX_VLMAX( IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG(PRTUINT128U, puDst, 1); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 2); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc2, uSrc2, 3); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_CALL_AVX_AIMPL_3(pImpl->pfnU128, puDst, puSrc1, puSrc2); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** + * Common worker for AVX2 instructions on the forms: + * - vpxxx xmm0, xmm1, xmm2/mem128 + * - vpxxx ymm0, ymm1, ymm2/mem256 + * + * Takes function table for function w/o implicit state parameter. + * + * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit. + */ +FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, PCIEMOPMEDIAOPTF3, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + if (pVCpu->iem.s.uVexLength) + { + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_YREG_U256(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnU256, puDst, puSrc1, puSrc2); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); + IEM_MC_ARG(PCRTUINT128U, puSrc2, 2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnU128, puDst, puSrc1, puSrc2); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength) + { + IEM_MC_BEGIN(3, 4); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnU256, puDst, puSrc1, puSrc2); + IEM_MC_STORE_YREG_U256_ZX_VLMAX( IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc2, uSrc2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnU128, puDst, puSrc1, puSrc2); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** + * Common worker for AVX2 instructions on the forms: + * - vpunpckhxx xmm0, xmm1, xmm2/mem128 + * - vpunpckhxx ymm0, ymm1, ymm2/mem256 + * + * The 128-bit memory version of this instruction may elect to skip fetching the + * lower 64 bits of the operand. We, however, do not. + * + * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit. + */ +FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, PCIEMOPMEDIAOPTF3, pImpl) +{ + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, pImpl); +} + + +/** + * Common worker for AVX2 instructions on the forms: + * - vpunpcklxx xmm0, xmm1, xmm2/mem128 + * - vpunpcklxx ymm0, ymm1, ymm2/mem256 + * + * The 128-bit memory version of this instruction may elect to skip fetching the + * higher 64 bits of the operand. We, however, do not. + * + * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit. + */ +FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, PCIEMOPMEDIAOPTF3, pImpl) +{ + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, pImpl); +} + + +/** + * Common worker for AVX2 instructions on the forms: + * - vpxxx xmm0, xmm1/mem128 + * - vpxxx ymm0, ymm1/mem256 + * + * Takes function table for function w/o implicit state parameter. + * + * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit. + */ +FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Wx_Opt, PCIEMOPMEDIAOPTF2, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + if (pVCpu->iem.s.uVexLength) + { + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_BEGIN(2, 2); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 1); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnU256, puDst, puSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnU128, puDst, puSrc); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength) + { + IEM_MC_BEGIN(2, 3); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnU256, puDst, puSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX( IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(2, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnU128, puDst, puSrc); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/* Opcode VEX.0F 0x00 - invalid */ +/* Opcode VEX.0F 0x01 - invalid */ +/* Opcode VEX.0F 0x02 - invalid */ +/* Opcode VEX.0F 0x03 - invalid */ +/* Opcode VEX.0F 0x04 - invalid */ +/* Opcode VEX.0F 0x05 - invalid */ +/* Opcode VEX.0F 0x06 - invalid */ +/* Opcode VEX.0F 0x07 - invalid */ +/* Opcode VEX.0F 0x08 - invalid */ +/* Opcode VEX.0F 0x09 - invalid */ +/* Opcode VEX.0F 0x0a - invalid */ + +/** Opcode VEX.0F 0x0b. */ +FNIEMOP_DEF(iemOp_vud2) +{ + IEMOP_MNEMONIC(vud2, "vud2"); + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/* Opcode VEX.0F 0x0c - invalid */ +/* Opcode VEX.0F 0x0d - invalid */ +/* Opcode VEX.0F 0x0e - invalid */ +/* Opcode VEX.0F 0x0f - invalid */ + + +/** + * @opcode 0x10 + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ +FNIEMOP_DEF(iemOp_vmovups_Vps_Wps) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVUPS, vmovups, Vps_WO, Wps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else if (pVCpu->iem.s.uVexLength == 0) + { + /* + * 128-bit: Register, Memory + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * 256-bit: Register, Memory + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x10 + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ +FNIEMOP_DEF(iemOp_vmovupd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVUPD, vmovupd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else if (pVCpu->iem.s.uVexLength == 0) + { + /* + * 128-bit: Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * 256-bit: Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +FNIEMOP_DEF(iemOp_vmovss_Vss_Hss_Wss) +{ + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /** + * @opcode 0x10 + * @oppfx 0xf3 + * @opcodesub 11 mr/reg + * @opcpuid avx + * @opgroup og_avx_simdfp_datamerge + * @opxcpttype 5 + * @optest op1=1 op2=0 op3=2 -> op1=2 + * @optest op1=0 op2=0 op3=-22 -> op1=0xffffffea + * @optest op1=3 op2=-1 op3=0x77 -> op1=-4294967177 + * @optest op1=3 op2=-2 op3=0x77 -> op1=-8589934473 + * @note HssHi refers to bits 127:32. + */ + IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVSS, vmovss, Vss_WO, HssHi, Uss, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED); + IEMOP_HLP_DONE_VEX_DECODING(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm) /*U32*/, + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /** + * @opdone + * @opcode 0x10 + * @oppfx 0xf3 + * @opcodesub !11 mr/reg + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 5 + * @opfunction iemOp_vmovss_Vss_Hss_Wss + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ + IEMOP_MNEMONIC2(VEX_RM_MEM, VMOVSS, vmovss, VssZx_WO, Md, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +FNIEMOP_DEF(iemOp_vmovsd_Vsd_Hsd_Wsd) +{ + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /** + * @opcode 0x10 + * @oppfx 0xf2 + * @opcodesub 11 mr/reg + * @opcpuid avx + * @opgroup og_avx_simdfp_datamerge + * @opxcpttype 5 + * @optest op1=1 op2=0 op3=2 -> op1=2 + * @optest op1=0 op2=0 op3=-22 -> op1=0xffffffffffffffea + * @optest op1=3 op2=-1 op3=0x77 -> + * op1=0xffffffffffffffff0000000000000077 + * @optest op1=3 op2=0x42 op3=0x77 -> op1=0x420000000000000077 + */ + IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVSD, vmovsd, Vsd_WO, HsdHi, Usd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED); + IEMOP_HLP_DONE_VEX_DECODING(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm) /*U32*/, + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /** + * @opdone + * @opcode 0x10 + * @oppfx 0xf2 + * @opcodesub !11 mr/reg + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 5 + * @opfunction iemOp_vmovsd_Vsd_Hsd_Wsd + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ + IEMOP_MNEMONIC2(VEX_RM_MEM, VMOVSD, vmovsd, VsdZx_WO, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x11 + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ +FNIEMOP_DEF(iemOp_vmovups_Wps_Vps) +{ + IEMOP_MNEMONIC2(VEX_MR, VMOVUPS, vmovups, Wps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else if (pVCpu->iem.s.uVexLength == 0) + { + /* + * 128-bit: Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * 256-bit: Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U256(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x11 + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ +FNIEMOP_DEF(iemOp_vmovupd_Wpd_Vpd) +{ + IEMOP_MNEMONIC2(VEX_MR, VMOVUPD, vmovupd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else if (pVCpu->iem.s.uVexLength == 0) + { + /* + * 128-bit: Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * 256-bit: Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U256(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +FNIEMOP_DEF(iemOp_vmovss_Wss_Hss_Vss) +{ + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /** + * @opcode 0x11 + * @oppfx 0xf3 + * @opcodesub 11 mr/reg + * @opcpuid avx + * @opgroup og_avx_simdfp_datamerge + * @opxcpttype 5 + * @optest op1=1 op2=0 op3=2 -> op1=2 + * @optest op1=0 op2=0 op3=-22 -> op1=0xffffffea + * @optest op1=3 op2=-1 op3=0x77 -> op1=-4294967177 + * @optest op1=3 op2=0x42 op3=0x77 -> op1=0x4200000077 + */ + IEMOP_MNEMONIC3(VEX_MVR_REG, VMOVSS, vmovss, Uss_WO, HssHi, Vss, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED); + IEMOP_HLP_DONE_VEX_DECODING(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm) /*U32*/, + IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /** + * @opdone + * @opcode 0x11 + * @oppfx 0xf3 + * @opcodesub !11 mr/reg + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 5 + * @opfunction iemOp_vmovss_Vss_Hss_Wss + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ + IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVSS, vmovss, Md_WO, Vss, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint32_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +FNIEMOP_DEF(iemOp_vmovsd_Wsd_Hsd_Vsd) +{ + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /** + * @opcode 0x11 + * @oppfx 0xf2 + * @opcodesub 11 mr/reg + * @opcpuid avx + * @opgroup og_avx_simdfp_datamerge + * @opxcpttype 5 + * @optest op1=1 op2=0 op3=2 -> op1=2 + * @optest op1=0 op2=0 op3=-22 -> op1=0xffffffffffffffea + * @optest op1=3 op2=-1 op3=0x77 -> + * op1=0xffffffffffffffff0000000000000077 + * @optest op2=0x42 op3=0x77 -> op1=0x420000000000000077 + */ + IEMOP_MNEMONIC3(VEX_MVR_REG, VMOVSD, vmovsd, Usd_WO, HsdHi, Vsd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED); + IEMOP_HLP_DONE_VEX_DECODING(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /** + * @opdone + * @opcode 0x11 + * @oppfx 0xf2 + * @opcodesub !11 mr/reg + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 5 + * @opfunction iemOp_vmovsd_Wsd_Hsd_Vsd + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-22 -> op1=-22 + */ + IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVSD, vmovsd, Mq_WO, Vsd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +FNIEMOP_DEF(iemOp_vmovlps_Vq_Hq_Mq__vmovhlps) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /** + * @opcode 0x12 + * @opcodesub 11 mr/reg + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_simdfp_datamerge + * @opxcpttype 7LZ + * @optest op2=0x2200220122022203 + * op3=0x3304330533063307 + * -> op1=0x22002201220222033304330533063307 + * @optest op2=-1 op3=-42 -> op1=-42 + * @note op3 and op2 are only the 8-byte high XMM register halfs. + */ + IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVHLPS, vmovhlps, Vq_WO, HqHi, UqHi, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /** + * @opdone + * @opcode 0x12 + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 5LZ + * @opfunction iemOp_vmovlps_Vq_Hq_Mq__vmovhlps + * @optest op1=1 op2=0 op3=0 -> op1=0 + * @optest op1=0 op2=-1 op3=-1 -> op1=-1 + * @optest op1=1 op2=2 op3=3 -> op1=0x20000000000000003 + * @optest op2=-1 op3=0x42 -> op1=0xffffffffffffffff0000000000000042 + */ + IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVLPS, vmovlps, Vq_WO, HqHi, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + uSrc, + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x12 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamerge + * @opxcpttype 5LZ + * @optest op2=0 op3=2 -> op1=2 + * @optest op2=0x22 op3=0x33 -> op1=0x220000000000000033 + * @optest op2=0xfffffff0fffffff1 op3=0xeeeeeee8eeeeeee9 + * -> op1=0xfffffff0fffffff1eeeeeee8eeeeeee9 + */ +FNIEMOP_DEF(iemOp_vmovlpd_Vq_Hq_Mq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVLPD, vmovlpd, Vq_WO, HqHi, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + uSrc, + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic udvex660f12m3 + * @opcode 0x12 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid avx + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opcode 0x12 + * @oppfx 0xf3 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype 4 + * @optest vex.l==0 / op1=-1 op2=0xdddddddd00000002eeeeeeee00000001 + * -> op1=0x00000002000000020000000100000001 + * @optest vex.l==1 / + * op2=0xbbbbbbbb00000004cccccccc00000003dddddddd00000002eeeeeeee00000001 + * -> op1=0x0000000400000004000000030000000300000002000000020000000100000001 + */ +FNIEMOP_DEF(iemOp_vmovsldup_Vx_Wx) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVSLDUP, vmovsldup, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTUINT128U, uSrc); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 0); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 0); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 2); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 2); + IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 0); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG_CONST(uint8_t, iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1); + IEM_MC_ARG_CONST(uint8_t, iYRegSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 2); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovsldup_256_rr, iYRegDst, iYRegSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 0); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 0); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 2); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 2); + IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG_CONST(uint8_t, iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovsldup_256_rm, iYRegDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** + * @opcode 0x12 + * @oppfx 0xf2 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype 5 + * @optest vex.l==0 / op2=0xddddddddeeeeeeee2222222211111111 + * -> op1=0x22222222111111112222222211111111 + * @optest vex.l==1 / op2=0xbbbbbbbbcccccccc4444444433333333ddddddddeeeeeeee2222222211111111 + * -> op1=0x4444444433333333444444443333333322222222111111112222222211111111 + */ +FNIEMOP_DEF(iemOp_vmovddup_Vx_Wx) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVDDUP, vmovddup, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(1, 0); + IEM_MC_ARG(uint64_t, uSrc, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 1 /* a_iQword*/, uSrc); + IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 0); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG_CONST(uint8_t, iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1); + IEM_MC_ARG_CONST(uint8_t, iYRegSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 2); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovddup_256_rr, iYRegDst, iYRegSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(1, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(uint64_t, uSrc, 0); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc); + IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 1 /* a_iQword*/, uSrc); + IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG_CONST(uint8_t, iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovddup_256_rm, iYRegDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** + * @opcode 0x13 + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_vmovlps_Mq_Vq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVLPS, vmovlps, Mq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic udvex0f13m3 + * @opcode 0x13 + * @opcodesub 11 mr/reg + * @oppfx none + * @opunused immediate + * @opcpuid avx + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opcode 0x13 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype 5 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_vmovlpd_Mq_Vq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVLPD, vmovlpd, Mq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic udvex660f13m3 + * @opcode 0x13 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid avx + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/* Opcode VEX.F3.0F 0x13 - invalid */ +/* Opcode VEX.F2.0F 0x13 - invalid */ + +/** Opcode VEX.0F 0x14 - vunpcklps Vx, Hx, Wx*/ +FNIEMOP_DEF(iemOp_vunpcklps_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VUNPCKLPS, vunpcklps, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vunpcklps); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F 0x14 - vunpcklpd Vx,Hx,Wx */ +FNIEMOP_DEF(iemOp_vunpcklpd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VUNPCKLPD, vunpcklpd, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vunpcklpd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x14 - invalid */ +/* Opcode VEX.F2.0F 0x14 - invalid */ + + +/** Opcode VEX.0F 0x15 - vunpckhps Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vunpckhps_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VUNPCKHPS, vunpckhps, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vunpckhps); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F 0x15 - vunpckhpd Vx,Hx,Wx */ +FNIEMOP_DEF(iemOp_vunpckhpd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VUNPCKHPD, vunpckhpd, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vunpckhpd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x15 - invalid */ +/* Opcode VEX.F2.0F 0x15 - invalid */ + + +FNIEMOP_DEF(iemOp_vmovhps_Vdq_Hq_Mq__vmovlhps_Vdq_Hq_Uq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /** + * @opcode 0x16 + * @opcodesub 11 mr/reg + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_simdfp_datamerge + * @opxcpttype 7LZ + */ + IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVLHPS, vmovlhps, Vq_WO, Hq, Uq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /** + * @opdone + * @opcode 0x16 + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 5LZ + * @opfunction iemOp_vmovhps_Vdq_Hq_Mq__vmovlhps_Vdq_Hq_Uq + */ + IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVHPS, vmovhps, Vq_WO, Hq, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/, + uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x16 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamerge + * @opxcpttype 5LZ + */ +FNIEMOP_DEF(iemOp_vmovhpd_Vdq_Hq_Mq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVHPD, vmovhpd, Vq_WO, Hq, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/, + uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic udvex660f16m3 + * @opcode 0x12 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid avx + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode VEX.F3.0F 0x16 - vmovshdup Vx, Wx */ +/** + * @opcode 0x16 + * @oppfx 0xf3 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype 4 + */ +FNIEMOP_DEF(iemOp_vmovshdup_Vx_Wx) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVSHDUP, vmovshdup, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(RTUINT128U, uSrc); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 1); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 1); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 3); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 3); + IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 0); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG_CONST(uint8_t, iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1); + IEM_MC_ARG_CONST(uint8_t, iYRegSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 2); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovshdup_256_rr, iYRegDst, iYRegSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 1); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 1); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 3); + IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 3); + IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_IMPLICIT_AVX_AIMPL_ARGS(); + IEM_MC_ARG_CONST(uint8_t, iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovshdup_256_rm, iYRegDst, puSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/* Opcode VEX.F2.0F 0x16 - invalid */ + + +/** + * @opcode 0x17 + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_simdfp_datamove + * @opxcpttype 5 + */ +FNIEMOP_DEF(iemOp_vmovhps_Mq_Vq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVHPS, vmovhps, Mq_WO, VqHi, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_2ND_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic udvex0f17m3 + * @opcode 0x17 + * @opcodesub 11 mr/reg + * @oppfx none + * @opunused immediate + * @opcpuid avx + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** + * @opcode 0x17 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype 5 + */ +FNIEMOP_DEF(iemOp_vmovhpd_Mq_Vq) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVHPD, vmovhpd, Mq_WO, VqHi, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_2ND_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + + /** + * @opdone + * @opmnemonic udvex660f17m3 + * @opcode 0x17 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid avx + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/* Opcode VEX.F3.0F 0x17 - invalid */ +/* Opcode VEX.F2.0F 0x17 - invalid */ + + +/* Opcode VEX.0F 0x18 - invalid */ +/* Opcode VEX.0F 0x19 - invalid */ +/* Opcode VEX.0F 0x1a - invalid */ +/* Opcode VEX.0F 0x1b - invalid */ +/* Opcode VEX.0F 0x1c - invalid */ +/* Opcode VEX.0F 0x1d - invalid */ +/* Opcode VEX.0F 0x1e - invalid */ +/* Opcode VEX.0F 0x1f - invalid */ + +/* Opcode VEX.0F 0x20 - invalid */ +/* Opcode VEX.0F 0x21 - invalid */ +/* Opcode VEX.0F 0x22 - invalid */ +/* Opcode VEX.0F 0x23 - invalid */ +/* Opcode VEX.0F 0x24 - invalid */ +/* Opcode VEX.0F 0x25 - invalid */ +/* Opcode VEX.0F 0x26 - invalid */ +/* Opcode VEX.0F 0x27 - invalid */ + +/** + * @opcode 0x28 + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + * @note Almost identical to vmovapd. + */ +FNIEMOP_DEF(iemOp_vmovaps_Vps_Wps) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVAPS, vmovaps, Vps_WO, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + Assert(pVCpu->iem.s.uVexLength <= 1); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(1, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(RTUINT128U, uSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(RTUINT256U, uSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U256_ALIGN_AVX(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** + * @opcode 0x28 + * @oppfx 66 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + * @note Almost identical to vmovaps + */ +FNIEMOP_DEF(iemOp_vmovapd_Vpd_Wpd) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVAPD, vmovapd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + Assert(pVCpu->iem.s.uVexLength <= 1); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(1, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(RTUINT128U, uSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(RTUINT256U, uSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U256_ALIGN_AVX(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + +/** + * @opmnemonic udvexf30f28 + * @opcode 0x28 + * @oppfx 0xf3 + * @opunused vex.modrm + * @opcpuid avx + * @optest -> + * @opdone + */ + +/** + * @opmnemonic udvexf20f28 + * @opcode 0x28 + * @oppfx 0xf2 + * @opunused vex.modrm + * @opcpuid avx + * @optest -> + * @opdone + */ + +/** + * @opcode 0x29 + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + * @note Almost identical to vmovapd. + */ +FNIEMOP_DEF(iemOp_vmovaps_Wps_Vps) +{ + IEMOP_MNEMONIC2(VEX_MR, VMOVAPS, vmovaps, Wps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + Assert(pVCpu->iem.s.uVexLength <= 1); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(1, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(RTUINT128U, uSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(RTUINT256U, uSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + +/** + * @opcode 0x29 + * @oppfx 66 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + * @note Almost identical to vmovaps + */ +FNIEMOP_DEF(iemOp_vmovapd_Wpd_Vpd) +{ + IEMOP_MNEMONIC2(VEX_MR, VMOVAPD, vmovapd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(1, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(RTUINT128U, uSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(RTUINT256U, uSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** + * @opmnemonic udvexf30f29 + * @opcode 0x29 + * @oppfx 0xf3 + * @opunused vex.modrm + * @opcpuid avx + * @optest -> + * @opdone + */ + +/** + * @opmnemonic udvexf20f29 + * @opcode 0x29 + * @oppfx 0xf2 + * @opunused vex.modrm + * @opcpuid avx + * @optest -> + * @opdone + */ + + +/** Opcode VEX.0F 0x2a - invalid */ +/** Opcode VEX.66.0F 0x2a - invalid */ +/** Opcode VEX.F3.0F 0x2a - vcvtsi2ss Vss, Hss, Ey */ +FNIEMOP_STUB(iemOp_vcvtsi2ss_Vss_Hss_Ey); +/** Opcode VEX.F2.0F 0x2a - vcvtsi2sd Vsd, Hsd, Ey */ +FNIEMOP_STUB(iemOp_vcvtsi2sd_Vsd_Hsd_Ey); + + +/** + * @opcode 0x2b + * @opcodesub !11 mr/reg + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_cachect + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + * @note Identical implementation to vmovntpd + */ +FNIEMOP_DEF(iemOp_vmovntps_Mps_Vps) +{ + IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVNTPS, vmovntps, Mps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* + * memory, register. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + /* The register, register encoding is invalid. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/** + * @opcode 0x2b + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_cachect + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + * @note Identical implementation to vmovntps + */ +FNIEMOP_DEF(iemOp_vmovntpd_Mpd_Vpd) +{ + IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVNTPD, vmovntpd, Mpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + /* + * memory, register. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + /* The register, register encoding is invalid. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/** + * @opmnemonic udvexf30f2b + * @opcode 0x2b + * @oppfx 0xf3 + * @opunused vex.modrm + * @opcpuid avx + * @optest -> + * @opdone + */ + +/** + * @opmnemonic udvexf20f2b + * @opcode 0x2b + * @oppfx 0xf2 + * @opunused vex.modrm + * @opcpuid avx + * @optest -> + * @opdone + */ + + +/* Opcode VEX.0F 0x2c - invalid */ +/* Opcode VEX.66.0F 0x2c - invalid */ +/** Opcode VEX.F3.0F 0x2c - vcvttss2si Gy, Wss */ +FNIEMOP_STUB(iemOp_vcvttss2si_Gy_Wss); +/** Opcode VEX.F2.0F 0x2c - vcvttsd2si Gy, Wsd */ +FNIEMOP_STUB(iemOp_vcvttsd2si_Gy_Wsd); + +/* Opcode VEX.0F 0x2d - invalid */ +/* Opcode VEX.66.0F 0x2d - invalid */ +/** Opcode VEX.F3.0F 0x2d - vcvtss2si Gy, Wss */ +FNIEMOP_STUB(iemOp_vcvtss2si_Gy_Wss); +/** Opcode VEX.F2.0F 0x2d - vcvtsd2si Gy, Wsd */ +FNIEMOP_STUB(iemOp_vcvtsd2si_Gy_Wsd); + + +/** Opcode VEX.0F 0x2e - vucomiss Vss, Wss */ +FNIEMOP_DEF(iemOp_vucomiss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, VUCOMISS, vucomiss, Vss, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(4, 1); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_ARG(PCX86XMMREG, puSrc2, 3); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vucomiss_u128, iemAImpl_vucomiss_u128_fallback), + pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, puSrc2, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vucomiss_u128, iemAImpl_vucomiss_u128_fallback), + pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode VEX.66.0F 0x2e - vucomisd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_vucomisd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, VUCOMISD, vucomisd, Vsd, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(4, 1); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_ARG(PCX86XMMREG, puSrc2, 3); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vucomisd_u128, iemAImpl_vucomisd_u128_fallback), + pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, puSrc2, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vucomisd_u128, iemAImpl_vucomisd_u128_fallback), + pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode VEX.F3.0F 0x2e - invalid */ +/* Opcode VEX.F2.0F 0x2e - invalid */ + +/** Opcode VEX.0F 0x2f - vcomiss Vss, Wss */ +FNIEMOP_DEF(iemOp_vcomiss_Vss_Wss) +{ + IEMOP_MNEMONIC2(RM, VCOMISS, vcomiss, Vss, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(4, 1); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_ARG(PCX86XMMREG, puSrc2, 3); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vcomiss_u128, iemAImpl_vcomiss_u128_fallback), + pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, puSrc2, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vcomiss_u128, iemAImpl_vcomiss_u128_fallback), + pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** Opcode VEX.66.0F 0x2f - vcomisd Vsd, Wsd */ +FNIEMOP_DEF(iemOp_vcomisd_Vsd_Wsd) +{ + IEMOP_MNEMONIC2(RM, VCOMISD, vcomisd, Vsd, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(4, 1); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_ARG(PCX86XMMREG, puSrc2, 3); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_XMM_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vcomisd_u128, iemAImpl_vcomisd_u128_fallback), + pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(uint32_t, fEFlags); + IEM_MC_ARG(uint32_t *, pfMxcsr, 0); + IEM_MC_ARG_LOCAL_REF(uint32_t *, pEFlags, fEFlags, 1); + IEM_MC_ARG(PCX86XMMREG, puSrc1, 2); + IEM_MC_LOCAL(X86XMMREG, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCX86XMMREG, puSrc2, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_EFLAGS(fEFlags); + IEM_MC_REF_MXCSR(pfMxcsr); + IEM_MC_REF_XREG_XMM_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vcomisd_u128, iemAImpl_vcomisd_u128_fallback), + pfMxcsr, pEFlags, puSrc1, puSrc2); + IEM_MC_IF_MXCSR_XCPT_PENDING() + IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT(); + IEM_MC_ELSE() + IEM_MC_COMMIT_EFLAGS(fEFlags); + IEM_MC_ENDIF(); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode VEX.F3.0F 0x2f - invalid */ +/* Opcode VEX.F2.0F 0x2f - invalid */ + +/* Opcode VEX.0F 0x30 - invalid */ +/* Opcode VEX.0F 0x31 - invalid */ +/* Opcode VEX.0F 0x32 - invalid */ +/* Opcode VEX.0F 0x33 - invalid */ +/* Opcode VEX.0F 0x34 - invalid */ +/* Opcode VEX.0F 0x35 - invalid */ +/* Opcode VEX.0F 0x36 - invalid */ +/* Opcode VEX.0F 0x37 - invalid */ +/* Opcode VEX.0F 0x38 - invalid */ +/* Opcode VEX.0F 0x39 - invalid */ +/* Opcode VEX.0F 0x3a - invalid */ +/* Opcode VEX.0F 0x3b - invalid */ +/* Opcode VEX.0F 0x3c - invalid */ +/* Opcode VEX.0F 0x3d - invalid */ +/* Opcode VEX.0F 0x3e - invalid */ +/* Opcode VEX.0F 0x3f - invalid */ +/* Opcode VEX.0F 0x40 - invalid */ +/* Opcode VEX.0F 0x41 - invalid */ +/* Opcode VEX.0F 0x42 - invalid */ +/* Opcode VEX.0F 0x43 - invalid */ +/* Opcode VEX.0F 0x44 - invalid */ +/* Opcode VEX.0F 0x45 - invalid */ +/* Opcode VEX.0F 0x46 - invalid */ +/* Opcode VEX.0F 0x47 - invalid */ +/* Opcode VEX.0F 0x48 - invalid */ +/* Opcode VEX.0F 0x49 - invalid */ +/* Opcode VEX.0F 0x4a - invalid */ +/* Opcode VEX.0F 0x4b - invalid */ +/* Opcode VEX.0F 0x4c - invalid */ +/* Opcode VEX.0F 0x4d - invalid */ +/* Opcode VEX.0F 0x4e - invalid */ +/* Opcode VEX.0F 0x4f - invalid */ + + +/** Opcode VEX.0F 0x50 - vmovmskps Gy, Ups */ +FNIEMOP_DEF(iemOp_vmovmskps_Gy_Ups) +{ + IEMOP_MNEMONIC2(VEX_RM_REG, VMOVMSKPS, vmovmskps, Gd, Ux, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEMOP_HLP_DONE_VEX_DECODING(); + IEM_MC_BEGIN(2, 1); + IEM_MC_LOCAL(uint8_t, u8Dst); + IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst, u8Dst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vmovmskps_u128, iemAImpl_vmovmskps_u128_fallback), + pu8Dst, puSrc); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEMOP_HLP_DONE_VEX_DECODING(); + IEM_MC_BEGIN(2, 2); + IEM_MC_LOCAL(uint8_t, u8Dst); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst, u8Dst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 1); + + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vmovmskps_u256, iemAImpl_vmovmskps_u256_fallback), + pu8Dst, puSrc); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + /* No memory operand. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode VEX.66.0F 0x50 - vmovmskpd Gy,Upd */ +FNIEMOP_DEF(iemOp_vmovmskpd_Gy_Upd) +{ + IEMOP_MNEMONIC2(VEX_RM_REG, VMOVMSKPD, vmovmskpd, Gd, Ux, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + if (pVCpu->iem.s.uVexLength == 0) + { + IEMOP_HLP_DONE_VEX_DECODING(); + IEM_MC_BEGIN(2, 1); + IEM_MC_LOCAL(uint8_t, u8Dst); + IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst, u8Dst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vmovmskpd_u128, iemAImpl_vmovmskpd_u128_fallback), + pu8Dst, puSrc); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEMOP_HLP_DONE_VEX_DECODING(); + IEM_MC_BEGIN(2, 2); + IEM_MC_LOCAL(uint8_t, u8Dst); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst, u8Dst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 1); + + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vmovmskpd_u256, iemAImpl_vmovmskpd_u256_fallback), + pu8Dst, puSrc); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + /* No memory operand. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/* Opcode VEX.F3.0F 0x50 - invalid */ +/* Opcode VEX.F2.0F 0x50 - invalid */ + +/** Opcode VEX.0F 0x51 - vsqrtps Vps, Wps */ +FNIEMOP_STUB(iemOp_vsqrtps_Vps_Wps); +/** Opcode VEX.66.0F 0x51 - vsqrtpd Vpd, Wpd */ +FNIEMOP_STUB(iemOp_vsqrtpd_Vpd_Wpd); +/** Opcode VEX.F3.0F 0x51 - vsqrtss Vss, Hss, Wss */ +FNIEMOP_STUB(iemOp_vsqrtss_Vss_Hss_Wss); +/** Opcode VEX.F2.0F 0x51 - vsqrtsd Vsd, Hsd, Wsd */ +FNIEMOP_STUB(iemOp_vsqrtsd_Vsd_Hsd_Wsd); + +/** Opcode VEX.0F 0x52 - vrsqrtps Vps, Wps */ +FNIEMOP_STUB(iemOp_vrsqrtps_Vps_Wps); +/* Opcode VEX.66.0F 0x52 - invalid */ +/** Opcode VEX.F3.0F 0x52 - vrsqrtss Vss, Hss, Wss */ +FNIEMOP_STUB(iemOp_vrsqrtss_Vss_Hss_Wss); +/* Opcode VEX.F2.0F 0x52 - invalid */ + +/** Opcode VEX.0F 0x53 - vrcpps Vps, Wps */ +FNIEMOP_STUB(iemOp_vrcpps_Vps_Wps); +/* Opcode VEX.66.0F 0x53 - invalid */ +/** Opcode VEX.F3.0F 0x53 - vrcpss Vss, Hss, Wss */ +FNIEMOP_STUB(iemOp_vrcpss_Vss_Hss_Wss); +/* Opcode VEX.F2.0F 0x53 - invalid */ + + +/** Opcode VEX.0F 0x54 - vandps Vps, Hps, Wps */ +FNIEMOP_DEF(iemOp_vandps_Vps_Hps_Wps) +{ + IEMOP_MNEMONIC3(VEX_RVM, VANDPS, vandps, Vps, Hps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpand, &g_iemAImpl_vpand_fallback)); +} + + +/** Opcode VEX.66.0F 0x54 - vandpd Vpd, Hpd, Wpd */ +FNIEMOP_DEF(iemOp_vandpd_Vpd_Hpd_Wpd) +{ + IEMOP_MNEMONIC3(VEX_RVM, VANDPD, vandpd, Vpd, Hpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpand, &g_iemAImpl_vpand_fallback)); +} + + +/* Opcode VEX.F3.0F 0x54 - invalid */ +/* Opcode VEX.F2.0F 0x54 - invalid */ + + +/** Opcode VEX.0F 0x55 - vandnps Vps, Hps, Wps */ +FNIEMOP_DEF(iemOp_vandnps_Vps_Hps_Wps) +{ + IEMOP_MNEMONIC3(VEX_RVM, VANDNPS, vandnps, Vps, Hps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpandn, &g_iemAImpl_vpandn_fallback)); +} + + +/** Opcode VEX.66.0F 0x55 - vandnpd Vpd, Hpd, Wpd */ +FNIEMOP_DEF(iemOp_vandnpd_Vpd_Hpd_Wpd) +{ + IEMOP_MNEMONIC3(VEX_RVM, VANDNPD, vandnpd, Vpd, Hpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpandn, &g_iemAImpl_vpandn_fallback)); +} + + +/* Opcode VEX.F3.0F 0x55 - invalid */ +/* Opcode VEX.F2.0F 0x55 - invalid */ + +/** Opcode VEX.0F 0x56 - vorps Vps, Hps, Wps */ +FNIEMOP_DEF(iemOp_vorps_Vps_Hps_Wps) +{ + IEMOP_MNEMONIC3(VEX_RVM, VORPS, vorps, Vps, Hps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpor, &g_iemAImpl_vpor_fallback)); +} + + +/** Opcode VEX.66.0F 0x56 - vorpd Vpd, Hpd, Wpd */ +FNIEMOP_DEF(iemOp_vorpd_Vpd_Hpd_Wpd) +{ + IEMOP_MNEMONIC3(VEX_RVM, VORPD, vorpd, Vpd, Hpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpor, &g_iemAImpl_vpor_fallback)); +} + + +/* Opcode VEX.F3.0F 0x56 - invalid */ +/* Opcode VEX.F2.0F 0x56 - invalid */ + + +/** Opcode VEX.0F 0x57 - vxorps Vps, Hps, Wps */ +FNIEMOP_DEF(iemOp_vxorps_Vps_Hps_Wps) +{ + IEMOP_MNEMONIC3(VEX_RVM, VXORPS, vxorps, Vps, Hps, Wps, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpxor, &g_iemAImpl_vpxor_fallback)); +} + + +/** Opcode VEX.66.0F 0x57 - vxorpd Vpd, Hpd, Wpd */ +FNIEMOP_DEF(iemOp_vxorpd_Vpd_Hpd_Wpd) +{ + IEMOP_MNEMONIC3(VEX_RVM, VXORPD, vxorpd, Vpd, Hpd, Wpd, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpxor, &g_iemAImpl_vpxor_fallback)); +} + + +/* Opcode VEX.F3.0F 0x57 - invalid */ +/* Opcode VEX.F2.0F 0x57 - invalid */ + +/** Opcode VEX.0F 0x58 - vaddps Vps, Hps, Wps */ +FNIEMOP_STUB(iemOp_vaddps_Vps_Hps_Wps); +/** Opcode VEX.66.0F 0x58 - vaddpd Vpd, Hpd, Wpd */ +FNIEMOP_STUB(iemOp_vaddpd_Vpd_Hpd_Wpd); +/** Opcode VEX.F3.0F 0x58 - vaddss Vss, Hss, Wss */ +FNIEMOP_STUB(iemOp_vaddss_Vss_Hss_Wss); +/** Opcode VEX.F2.0F 0x58 - vaddsd Vsd, Hsd, Wsd */ +FNIEMOP_STUB(iemOp_vaddsd_Vsd_Hsd_Wsd); + +/** Opcode VEX.0F 0x59 - vmulps Vps, Hps, Wps */ +FNIEMOP_STUB(iemOp_vmulps_Vps_Hps_Wps); +/** Opcode VEX.66.0F 0x59 - vmulpd Vpd, Hpd, Wpd */ +FNIEMOP_STUB(iemOp_vmulpd_Vpd_Hpd_Wpd); +/** Opcode VEX.F3.0F 0x59 - vmulss Vss, Hss, Wss */ +FNIEMOP_STUB(iemOp_vmulss_Vss_Hss_Wss); +/** Opcode VEX.F2.0F 0x59 - vmulsd Vsd, Hsd, Wsd */ +FNIEMOP_STUB(iemOp_vmulsd_Vsd_Hsd_Wsd); + +/** Opcode VEX.0F 0x5a - vcvtps2pd Vpd, Wps */ +FNIEMOP_STUB(iemOp_vcvtps2pd_Vpd_Wps); +/** Opcode VEX.66.0F 0x5a - vcvtpd2ps Vps, Wpd */ +FNIEMOP_STUB(iemOp_vcvtpd2ps_Vps_Wpd); +/** Opcode VEX.F3.0F 0x5a - vcvtss2sd Vsd, Hx, Wss */ +FNIEMOP_STUB(iemOp_vcvtss2sd_Vsd_Hx_Wss); +/** Opcode VEX.F2.0F 0x5a - vcvtsd2ss Vss, Hx, Wsd */ +FNIEMOP_STUB(iemOp_vcvtsd2ss_Vss_Hx_Wsd); + +/** Opcode VEX.0F 0x5b - vcvtdq2ps Vps, Wdq */ +FNIEMOP_STUB(iemOp_vcvtdq2ps_Vps_Wdq); +/** Opcode VEX.66.0F 0x5b - vcvtps2dq Vdq, Wps */ +FNIEMOP_STUB(iemOp_vcvtps2dq_Vdq_Wps); +/** Opcode VEX.F3.0F 0x5b - vcvttps2dq Vdq, Wps */ +FNIEMOP_STUB(iemOp_vcvttps2dq_Vdq_Wps); +/* Opcode VEX.F2.0F 0x5b - invalid */ + +/** Opcode VEX.0F 0x5c - vsubps Vps, Hps, Wps */ +FNIEMOP_STUB(iemOp_vsubps_Vps_Hps_Wps); +/** Opcode VEX.66.0F 0x5c - vsubpd Vpd, Hpd, Wpd */ +FNIEMOP_STUB(iemOp_vsubpd_Vpd_Hpd_Wpd); +/** Opcode VEX.F3.0F 0x5c - vsubss Vss, Hss, Wss */ +FNIEMOP_STUB(iemOp_vsubss_Vss_Hss_Wss); +/** Opcode VEX.F2.0F 0x5c - vsubsd Vsd, Hsd, Wsd */ +FNIEMOP_STUB(iemOp_vsubsd_Vsd_Hsd_Wsd); + +/** Opcode VEX.0F 0x5d - vminps Vps, Hps, Wps */ +FNIEMOP_STUB(iemOp_vminps_Vps_Hps_Wps); +/** Opcode VEX.66.0F 0x5d - vminpd Vpd, Hpd, Wpd */ +FNIEMOP_STUB(iemOp_vminpd_Vpd_Hpd_Wpd); +/** Opcode VEX.F3.0F 0x5d - vminss Vss, Hss, Wss */ +FNIEMOP_STUB(iemOp_vminss_Vss_Hss_Wss); +/** Opcode VEX.F2.0F 0x5d - vminsd Vsd, Hsd, Wsd */ +FNIEMOP_STUB(iemOp_vminsd_Vsd_Hsd_Wsd); + +/** Opcode VEX.0F 0x5e - vdivps Vps, Hps, Wps */ +FNIEMOP_STUB(iemOp_vdivps_Vps_Hps_Wps); +/** Opcode VEX.66.0F 0x5e - vdivpd Vpd, Hpd, Wpd */ +FNIEMOP_STUB(iemOp_vdivpd_Vpd_Hpd_Wpd); +/** Opcode VEX.F3.0F 0x5e - vdivss Vss, Hss, Wss */ +FNIEMOP_STUB(iemOp_vdivss_Vss_Hss_Wss); +/** Opcode VEX.F2.0F 0x5e - vdivsd Vsd, Hsd, Wsd */ +FNIEMOP_STUB(iemOp_vdivsd_Vsd_Hsd_Wsd); + +/** Opcode VEX.0F 0x5f - vmaxps Vps, Hps, Wps */ +FNIEMOP_STUB(iemOp_vmaxps_Vps_Hps_Wps); +/** Opcode VEX.66.0F 0x5f - vmaxpd Vpd, Hpd, Wpd */ +FNIEMOP_STUB(iemOp_vmaxpd_Vpd_Hpd_Wpd); +/** Opcode VEX.F3.0F 0x5f - vmaxss Vss, Hss, Wss */ +FNIEMOP_STUB(iemOp_vmaxss_Vss_Hss_Wss); +/** Opcode VEX.F2.0F 0x5f - vmaxsd Vsd, Hsd, Wsd */ +FNIEMOP_STUB(iemOp_vmaxsd_Vsd_Hsd_Wsd); + + +/* Opcode VEX.0F 0x60 - invalid */ + + +/** Opcode VEX.66.0F 0x60 - vpunpcklbw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpunpcklbw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKLBW, vpunpcklbw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpunpcklbw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x60 - invalid */ + + +/* Opcode VEX.0F 0x61 - invalid */ + + +/** Opcode VEX.66.0F 0x61 - vpunpcklwd Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpunpcklwd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKLWD, vpunpcklwd, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpunpcklwd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x61 - invalid */ + + +/* Opcode VEX.0F 0x62 - invalid */ + +/** Opcode VEX.66.0F 0x62 - vpunpckldq Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpunpckldq_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKLDQ, vpunpckldq, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpunpckldq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x62 - invalid */ + + + +/* Opcode VEX.0F 0x63 - invalid */ + + +/** Opcode VEX.66.0F 0x63 - vpacksswb Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpacksswb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPACKSSWB, vpacksswb, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpacksswb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x63 - invalid */ + +/* Opcode VEX.0F 0x64 - invalid */ + + +/** Opcode VEX.66.0F 0x64 - vpcmpgtb Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpcmpgtb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPCMPGTB, vpcmpgtb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpcmpgtb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x64 - invalid */ + +/* Opcode VEX.0F 0x65 - invalid */ + + +/** Opcode VEX.66.0F 0x65 - vpcmpgtw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpcmpgtw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPCMPGTW, vpcmpgtw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpcmpgtw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x65 - invalid */ + +/* Opcode VEX.0F 0x66 - invalid */ + + +/** Opcode VEX.66.0F 0x66 - vpcmpgtd Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpcmpgtd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPCMPGTD, vpcmpgtd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpcmpgtd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x66 - invalid */ + +/* Opcode VEX.0F 0x67 - invalid */ + + +/** Opcode VEX.66.0F 0x67 - vpackuswb Vx, Hx, W */ +FNIEMOP_DEF(iemOp_vpackuswb_Vx_Hx_W) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPACKUSWB, vpackuswb, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpackuswb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x67 - invalid */ + + +///** +// * Common worker for SSE2 instructions on the form: +// * pxxxx xmm1, xmm2/mem128 +// * +// * The 2nd operand is the second half of a register, which in the memory case +// * means a 64-bit memory access for MMX, and for SSE a 128-bit aligned access +// * where it may read the full 128 bits or only the upper 64 bits. +// * +// * Exceptions type 4. +// */ +//FNIEMOP_DEF_1(iemOpCommonSse_HighHigh_To_Full, PCIEMOPMEDIAF1H1, pImpl) +//{ +// uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); +// if (IEM_IS_MODRM_REG_MODE(bRm)) +// { +// /* +// * Register, register. +// */ +// IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); +// IEM_MC_BEGIN(2, 0); +// IEM_MC_ARG(PRTUINT128U, pDst, 0); +// IEM_MC_ARG(PCRTUINT128U, pSrc, 1); +// IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); +// IEM_MC_PREPARE_SSE_USAGE(); +// IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); +// IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); +// IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc); +// IEM_MC_ADVANCE_RIP_AND_FINISH(); +// IEM_MC_END(); +// } +// else +// { +// /* +// * Register, memory. +// */ +// IEM_MC_BEGIN(2, 2); +// IEM_MC_ARG(PRTUINT128U, pDst, 0); +// IEM_MC_LOCAL(RTUINT128U, uSrc); +// IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, pSrc, uSrc, 1); +// IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); +// +// IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); +// IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); +// IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT(); +// IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); /* Most CPUs probably only right high qword */ +// +// IEM_MC_PREPARE_SSE_USAGE(); +// IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); +// IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc); +// +// IEM_MC_ADVANCE_RIP_AND_FINISH(); +// IEM_MC_END(); +// } +// return VINF_SUCCESS; +//} + + +/* Opcode VEX.0F 0x68 - invalid */ + +/** Opcode VEX.66.0F 0x68 - vpunpckhbw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpunpckhbw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKHBW, vpunpckhbw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpunpckhbw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x68 - invalid */ + + +/* Opcode VEX.0F 0x69 - invalid */ + + +/** Opcode VEX.66.0F 0x69 - vpunpckhwd Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpunpckhwd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKHWD, vpunpckhwd, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpunpckhwd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x69 - invalid */ + + +/* Opcode VEX.0F 0x6a - invalid */ + + +/** Opcode VEX.66.0F 0x6a - vpunpckhdq Vx, Hx, W */ +FNIEMOP_DEF(iemOp_vpunpckhdq_Vx_Hx_W) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKHDQ, vpunpckhdq, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpunpckhdq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x6a - invalid */ + + +/* Opcode VEX.0F 0x6b - invalid */ + + +/** Opcode VEX.66.0F 0x6b - vpackssdw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpackssdw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPACKSSDW, vpackssdw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpackssdw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x6b - invalid */ + + +/* Opcode VEX.0F 0x6c - invalid */ + + +/** Opcode VEX.66.0F 0x6c - vpunpcklqdq Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpunpcklqdq_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKLQDQ, vpunpcklqdq, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpunpcklqdq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x6c - invalid */ +/* Opcode VEX.F2.0F 0x6c - invalid */ + + +/* Opcode VEX.0F 0x6d - invalid */ + + +/** Opcode VEX.66.0F 0x6d - vpunpckhqdq Vx, Hx, W */ +FNIEMOP_DEF(iemOp_vpunpckhqdq_Vx_Hx_W) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKHQDQ, vpunpckhqdq, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpunpckhqdq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x6d - invalid */ + + +/* Opcode VEX.0F 0x6e - invalid */ + +FNIEMOP_DEF(iemOp_vmovd_q_Vy_Ey) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + /** + * @opcode 0x6e + * @opcodesub rex.w=1 + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_simdint_datamov + * @opxcpttype 5 + * @optest 64-bit / op1=1 op2=2 -> op1=2 + * @optest 64-bit / op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(VEX_RM, VMOVQ, vmovq, Vq_WO, Eq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* XMM, greg64 */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* XMM, [mem64] */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /** + * @opdone + * @opcode 0x6e + * @opcodesub rex.w=0 + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_simdint_datamov + * @opxcpttype 5 + * @opfunction iemOp_vmovd_q_Vy_Ey + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(VEX_RM, VMOVD, vmovd, Vd_WO, Ed, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* XMM, greg32 */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* XMM, [mem32] */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/* Opcode VEX.F3.0F 0x6e - invalid */ + + +/* Opcode VEX.0F 0x6f - invalid */ + +/** + * @opcode 0x6f + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_simdint_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_vmovdqa_Vx_Wx) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVDQA, vmovdqa, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else if (pVCpu->iem.s.uVexLength == 0) + { + /* + * Register, memory128. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory256. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, u256Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U256_ALIGN_AVX(u256Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u256Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/** + * @opcode 0x6f + * @oppfx 0xf3 + * @opcpuid avx + * @opgroup og_avx_simdint_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_vmovdqu_Vx_Wx) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVDQU, vmovdqu, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else if (pVCpu->iem.s.uVexLength == 0) + { + /* + * Register, memory128. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory256. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, u256Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U256(u256Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u256Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode VEX.0F 0x70 - invalid */ + + +/** + * Common worker for AVX/AVX2 instructions on the forms: + * - vpxxx xmm0, xmm2/mem128, imm8 + * - vpxxx ymm0, ymm2/mem256, imm8 + * + * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit. + */ +FNIEMOP_DEF_2(iemOpCommonAvxAvx2_vpshufXX_Vx_Wx_Ib, PFNIEMAIMPLMEDIAPSHUFU128, pfnU128, PFNIEMAIMPLMEDIAPSHUFU256, pfnU256) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + if (pVCpu->iem.s.uVexLength) + { + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx2); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnU256, puDst, puSrc, bImmArg); + IEM_MC_STORE_YREG_U256_ZX_VLMAX( IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength) + { + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_CALL_VOID_AIMPL_3(pfnU256, puDst, puSrc, bImmArg); + IEM_MC_STORE_YREG_U256_ZX_VLMAX( IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode VEX.66.0F 0x70 - vpshufd Vx, Wx, Ib */ +FNIEMOP_DEF(iemOp_vpshufd_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(VEX_RMI, VPSHUFD, vpshufd, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + return FNIEMOP_CALL_2(iemOpCommonAvxAvx2_vpshufXX_Vx_Wx_Ib, iemAImpl_pshufd_u128, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vpshufd_u256, iemAImpl_vpshufd_u256_fallback)); + +} + + +/** Opcode VEX.F3.0F 0x70 - vpshufhw Vx, Wx, Ib */ +FNIEMOP_DEF(iemOp_vpshufhw_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(VEX_RMI, VPSHUFHW, vpshufhw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + return FNIEMOP_CALL_2(iemOpCommonAvxAvx2_vpshufXX_Vx_Wx_Ib, iemAImpl_pshufhw_u128, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vpshufhw_u256, iemAImpl_vpshufhw_u256_fallback)); + +} + + +/** Opcode VEX.F2.0F 0x70 - vpshuflw Vx, Wx, Ib */ +FNIEMOP_DEF(iemOp_vpshuflw_Vx_Wx_Ib) +{ + IEMOP_MNEMONIC3(VEX_RMI, VPSHUFLW, vpshuflw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + return FNIEMOP_CALL_2(iemOpCommonAvxAvx2_vpshufXX_Vx_Wx_Ib, iemAImpl_pshuflw_u128, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vpshuflw_u256, iemAImpl_vpshuflw_u256_fallback)); +} + + +/* Opcode VEX.0F 0x71 11/2 - invalid. */ +/** Opcode VEX.66.0F 0x71 11/2. */ +FNIEMOP_STUB_1(iemOp_VGrp12_vpsrlw_Hx_Ux_Ib, uint8_t, bRm); + +/* Opcode VEX.0F 0x71 11/4 - invalid */ +/** Opcode VEX.66.0F 0x71 11/4. */ +FNIEMOP_STUB_1(iemOp_VGrp12_vpsraw_Hx_Ux_Ib, uint8_t, bRm); + +/* Opcode VEX.0F 0x71 11/6 - invalid */ +/** Opcode VEX.66.0F 0x71 11/6. */ +FNIEMOP_STUB_1(iemOp_VGrp12_vpsllw_Hx_Ux_Ib, uint8_t, bRm); + + +/** + * VEX Group 12 jump table for register variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnVexGroup12RegReg[] = +{ + /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /2 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp12_vpsrlw_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /4 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp12_vpsraw_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /6 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp12_vpsllw_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8) +}; +AssertCompile(RT_ELEMENTS(g_apfnVexGroup12RegReg) == 8*4); + + +/** Opcode VEX.0F 0x71. */ +FNIEMOP_DEF(iemOp_VGrp12) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + /* register, register */ + return FNIEMOP_CALL_1(g_apfnVexGroup12RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); + return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm); +} + + +/* Opcode VEX.0F 0x72 11/2 - invalid. */ +/** Opcode VEX.66.0F 0x72 11/2. */ +FNIEMOP_STUB_1(iemOp_VGrp13_vpsrld_Hx_Ux_Ib, uint8_t, bRm); + +/* Opcode VEX.0F 0x72 11/4 - invalid. */ +/** Opcode VEX.66.0F 0x72 11/4. */ +FNIEMOP_STUB_1(iemOp_VGrp13_vpsrad_Hx_Ux_Ib, uint8_t, bRm); + +/* Opcode VEX.0F 0x72 11/6 - invalid. */ +/** Opcode VEX.66.0F 0x72 11/6. */ +FNIEMOP_STUB_1(iemOp_VGrp13_vpslld_Hx_Ux_Ib, uint8_t, bRm); + + +/** + * Group 13 jump table for register variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnVexGroup13RegReg[] = +{ + /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /2 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp13_vpsrld_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /4 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp13_vpsrad_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /6 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp13_vpslld_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8) +}; +AssertCompile(RT_ELEMENTS(g_apfnVexGroup13RegReg) == 8*4); + +/** Opcode VEX.0F 0x72. */ +FNIEMOP_DEF(iemOp_VGrp13) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + /* register, register */ + return FNIEMOP_CALL_1(g_apfnVexGroup13RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); + return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm); +} + + +/* Opcode VEX.0F 0x73 11/2 - invalid. */ +/** Opcode VEX.66.0F 0x73 11/2. */ +FNIEMOP_STUB_1(iemOp_VGrp14_vpsrlq_Hx_Ux_Ib, uint8_t, bRm); + +/** Opcode VEX.66.0F 0x73 11/3. */ +FNIEMOP_STUB_1(iemOp_VGrp14_vpsrldq_Hx_Ux_Ib, uint8_t, bRm); + +/* Opcode VEX.0F 0x73 11/6 - invalid. */ +/** Opcode VEX.66.0F 0x73 11/6. */ +FNIEMOP_STUB_1(iemOp_VGrp14_vpsllq_Hx_Ux_Ib, uint8_t, bRm); + +/** Opcode VEX.66.0F 0x73 11/7. */ +FNIEMOP_STUB_1(iemOp_VGrp14_vpslldq_Hx_Ux_Ib, uint8_t, bRm); + +/** + * Group 14 jump table for register variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnVexGroup14RegReg[] = +{ + /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /2 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpsrlq_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /3 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpsrldq_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /4 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8), + /* /6 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpsllq_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, + /* /7 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpslldq_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8, +}; +AssertCompile(RT_ELEMENTS(g_apfnVexGroup14RegReg) == 8*4); + + +/** Opcode VEX.0F 0x73. */ +FNIEMOP_DEF(iemOp_VGrp14) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + /* register, register */ + return FNIEMOP_CALL_1(g_apfnVexGroup14RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); + return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm); +} + + +/* Opcode VEX.0F 0x74 - invalid */ + + +/** Opcode VEX.66.0F 0x74 - vpcmpeqb Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpcmpeqb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPCMPEQB, vpcmpeqb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpcmpeqb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + +/* Opcode VEX.F3.0F 0x74 - invalid */ +/* Opcode VEX.F2.0F 0x74 - invalid */ + + +/* Opcode VEX.0F 0x75 - invalid */ + + +/** Opcode VEX.66.0F 0x75 - vpcmpeqw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpcmpeqw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPCMPEQW, vpcmpeqw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpcmpeqw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x75 - invalid */ +/* Opcode VEX.F2.0F 0x75 - invalid */ + + +/* Opcode VEX.0F 0x76 - invalid */ + + +/** Opcode VEX.66.0F 0x76 - vpcmpeqd Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpcmpeqd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPCMPEQD, vpcmpeqd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpcmpeqd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0x76 - invalid */ +/* Opcode VEX.F2.0F 0x76 - invalid */ + + +/** Opcode VEX.0F 0x77 - vzeroupperv vzeroallv */ +FNIEMOP_DEF(iemOp_vzeroupperv__vzeroallv) +{ + Assert(pVCpu->iem.s.uVexLength <= 1); + if (pVCpu->iem.s.uVexLength == 0) + { + /* + * 128-bit: vzeroupper + */ + IEMOP_MNEMONIC(vzeroupper, "vzeroupper"); + IEM_MC_BEGIN(0, 0); + + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_CLEAR_YREG_128_UP(0); + IEM_MC_CLEAR_YREG_128_UP(1); + IEM_MC_CLEAR_YREG_128_UP(2); + IEM_MC_CLEAR_YREG_128_UP(3); + IEM_MC_CLEAR_YREG_128_UP(4); + IEM_MC_CLEAR_YREG_128_UP(5); + IEM_MC_CLEAR_YREG_128_UP(6); + IEM_MC_CLEAR_YREG_128_UP(7); + + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_CLEAR_YREG_128_UP( 8); + IEM_MC_CLEAR_YREG_128_UP( 9); + IEM_MC_CLEAR_YREG_128_UP(10); + IEM_MC_CLEAR_YREG_128_UP(11); + IEM_MC_CLEAR_YREG_128_UP(12); + IEM_MC_CLEAR_YREG_128_UP(13); + IEM_MC_CLEAR_YREG_128_UP(14); + IEM_MC_CLEAR_YREG_128_UP(15); + } + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * 256-bit: vzeroall + */ + IEMOP_MNEMONIC(vzeroall, "vzeroall"); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, uZero); + + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_ASSIGN(uZero, 0); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(0, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(1, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(2, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(3, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(4, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(5, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(6, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(7, uZero); + + if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT) + { + IEM_MC_STORE_YREG_U32_ZX_VLMAX( 8, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX( 9, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(10, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(11, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(12, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(13, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(14, uZero); + IEM_MC_STORE_YREG_U32_ZX_VLMAX(15, uZero); + } + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode VEX.66.0F 0x77 - invalid */ +/* Opcode VEX.F3.0F 0x77 - invalid */ +/* Opcode VEX.F2.0F 0x77 - invalid */ + +/* Opcode VEX.0F 0x78 - invalid */ +/* Opcode VEX.66.0F 0x78 - invalid */ +/* Opcode VEX.F3.0F 0x78 - invalid */ +/* Opcode VEX.F2.0F 0x78 - invalid */ + +/* Opcode VEX.0F 0x79 - invalid */ +/* Opcode VEX.66.0F 0x79 - invalid */ +/* Opcode VEX.F3.0F 0x79 - invalid */ +/* Opcode VEX.F2.0F 0x79 - invalid */ + +/* Opcode VEX.0F 0x7a - invalid */ +/* Opcode VEX.66.0F 0x7a - invalid */ +/* Opcode VEX.F3.0F 0x7a - invalid */ +/* Opcode VEX.F2.0F 0x7a - invalid */ + +/* Opcode VEX.0F 0x7b - invalid */ +/* Opcode VEX.66.0F 0x7b - invalid */ +/* Opcode VEX.F3.0F 0x7b - invalid */ +/* Opcode VEX.F2.0F 0x7b - invalid */ + +/* Opcode VEX.0F 0x7c - invalid */ +/** Opcode VEX.66.0F 0x7c - vhaddpd Vpd, Hpd, Wpd */ +FNIEMOP_STUB(iemOp_vhaddpd_Vpd_Hpd_Wpd); +/* Opcode VEX.F3.0F 0x7c - invalid */ +/** Opcode VEX.F2.0F 0x7c - vhaddps Vps, Hps, Wps */ +FNIEMOP_STUB(iemOp_vhaddps_Vps_Hps_Wps); + +/* Opcode VEX.0F 0x7d - invalid */ +/** Opcode VEX.66.0F 0x7d - vhsubpd Vpd, Hpd, Wpd */ +FNIEMOP_STUB(iemOp_vhsubpd_Vpd_Hpd_Wpd); +/* Opcode VEX.F3.0F 0x7d - invalid */ +/** Opcode VEX.F2.0F 0x7d - vhsubps Vps, Hps, Wps */ +FNIEMOP_STUB(iemOp_vhsubps_Vps_Hps_Wps); + + +/* Opcode VEX.0F 0x7e - invalid */ + +FNIEMOP_DEF(iemOp_vmovd_q_Ey_Vy) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + /** + * @opcode 0x7e + * @opcodesub rex.w=1 + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_simdint_datamov + * @opxcpttype 5 + * @optest 64-bit / op1=1 op2=2 -> op1=2 + * @optest 64-bit / op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(VEX_MR, VMOVQ, vmovq, Eq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg64, XMM */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U64(u64Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* [mem64], XMM */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint64_t, u64Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U64(u64Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /** + * @opdone + * @opcode 0x7e + * @opcodesub rex.w=0 + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_simdint_datamov + * @opxcpttype 5 + * @opfunction iemOp_vmovd_q_Vy_Ey + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ + IEMOP_MNEMONIC2(VEX_MR, VMOVD, vmovd, Ed_WO, Vd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* greg32, XMM */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(0, 1); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U32(u32Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* [mem32], XMM */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_LOCAL(uint32_t, u32Tmp); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U32(u32Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u32Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** + * @opcode 0x7e + * @oppfx 0xf3 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype none + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_vmovq_Vq_Wq) +{ + IEMOP_MNEMONIC2(VEX_RM, VMOVQ, vmovq, Vq_WO, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_COPY_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), + IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + +} +/* Opcode VEX.F2.0F 0x7e - invalid */ + + +/* Opcode VEX.0F 0x7f - invalid */ + +/** + * @opcode 0x7f + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_simdint_datamove + * @opxcpttype 1 + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_vmovdqa_Wx_Vx) +{ + IEMOP_MNEMONIC2(VEX_MR, VMOVDQA, vmovdqa, Wx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else if (pVCpu->iem.s.uVexLength == 0) + { + /* + * Register, memory128. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U128(u128Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory256. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, u256Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U256(u256Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u256Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/** + * @opcode 0x7f + * @oppfx 0xf3 + * @opcpuid avx + * @opgroup og_avx_simdint_datamove + * @opxcpttype 4UA + * @optest op1=1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_vmovdqu_Wx_Vx) +{ + IEMOP_MNEMONIC2(VEX_MR, VMOVDQU, vmovdqu, Wx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + if (pVCpu->iem.s.uVexLength == 0) + IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + else + IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else if (pVCpu->iem.s.uVexLength == 0) + { + /* + * Register, memory128. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U128(u128Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory256. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, u256Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U256(u256Tmp, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U256(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u256Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/* Opcode VEX.F2.0F 0x7f - invalid */ + + +/* Opcode VEX.0F 0x80 - invalid */ +/* Opcode VEX.0F 0x81 - invalid */ +/* Opcode VEX.0F 0x82 - invalid */ +/* Opcode VEX.0F 0x83 - invalid */ +/* Opcode VEX.0F 0x84 - invalid */ +/* Opcode VEX.0F 0x85 - invalid */ +/* Opcode VEX.0F 0x86 - invalid */ +/* Opcode VEX.0F 0x87 - invalid */ +/* Opcode VEX.0F 0x88 - invalid */ +/* Opcode VEX.0F 0x89 - invalid */ +/* Opcode VEX.0F 0x8a - invalid */ +/* Opcode VEX.0F 0x8b - invalid */ +/* Opcode VEX.0F 0x8c - invalid */ +/* Opcode VEX.0F 0x8d - invalid */ +/* Opcode VEX.0F 0x8e - invalid */ +/* Opcode VEX.0F 0x8f - invalid */ +/* Opcode VEX.0F 0x90 - invalid */ +/* Opcode VEX.0F 0x91 - invalid */ +/* Opcode VEX.0F 0x92 - invalid */ +/* Opcode VEX.0F 0x93 - invalid */ +/* Opcode VEX.0F 0x94 - invalid */ +/* Opcode VEX.0F 0x95 - invalid */ +/* Opcode VEX.0F 0x96 - invalid */ +/* Opcode VEX.0F 0x97 - invalid */ +/* Opcode VEX.0F 0x98 - invalid */ +/* Opcode VEX.0F 0x99 - invalid */ +/* Opcode VEX.0F 0x9a - invalid */ +/* Opcode VEX.0F 0x9b - invalid */ +/* Opcode VEX.0F 0x9c - invalid */ +/* Opcode VEX.0F 0x9d - invalid */ +/* Opcode VEX.0F 0x9e - invalid */ +/* Opcode VEX.0F 0x9f - invalid */ +/* Opcode VEX.0F 0xa0 - invalid */ +/* Opcode VEX.0F 0xa1 - invalid */ +/* Opcode VEX.0F 0xa2 - invalid */ +/* Opcode VEX.0F 0xa3 - invalid */ +/* Opcode VEX.0F 0xa4 - invalid */ +/* Opcode VEX.0F 0xa5 - invalid */ +/* Opcode VEX.0F 0xa6 - invalid */ +/* Opcode VEX.0F 0xa7 - invalid */ +/* Opcode VEX.0F 0xa8 - invalid */ +/* Opcode VEX.0F 0xa9 - invalid */ +/* Opcode VEX.0F 0xaa - invalid */ +/* Opcode VEX.0F 0xab - invalid */ +/* Opcode VEX.0F 0xac - invalid */ +/* Opcode VEX.0F 0xad - invalid */ + + +/* Opcode VEX.0F 0xae mem/0 - invalid. */ +/* Opcode VEX.0F 0xae mem/1 - invalid. */ + +/** + * @ opmaps grp15 + * @ opcode !11/2 + * @ oppfx none + * @ opcpuid sse + * @ opgroup og_sse_mxcsrsm + * @ opxcpttype 5 + * @ optest op1=0 -> mxcsr=0 + * @ optest op1=0x2083 -> mxcsr=0x2083 + * @ optest op1=0xfffffffe -> value.xcpt=0xd + * @ optest op1=0x2083 cr0|=ts -> value.xcpt=0x7 + * @ optest op1=0x2083 cr0|=em -> value.xcpt=0x6 + * @ optest op1=0x2083 cr0|=mp -> mxcsr=0x2083 + * @ optest op1=0x2083 cr4&~=osfxsr -> value.xcpt=0x6 + * @ optest op1=0x2083 cr0|=ts,em -> value.xcpt=0x6 + * @ optest op1=0x2083 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6 + * @ optest op1=0x2083 cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6 + * @ optest op1=0x2083 cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6 + */ +FNIEMOP_STUB_1(iemOp_VGrp15_vldmxcsr, uint8_t, bRm); +//FNIEMOP_DEF_1(iemOp_VGrp15_vldmxcsr, uint8_t, bRm) +//{ +// IEMOP_MNEMONIC1(M_MEM, VLDMXCSR, vldmxcsr, MdRO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES); +// if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse) +// return IEMOP_RAISE_INVALID_OPCODE(); +// +// IEM_MC_BEGIN(2, 0); +// IEM_MC_ARG(uint8_t, iEffSeg, 0); +// IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); +// IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); +// IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); +// IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); +// IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); +// IEM_MC_CALL_CIMPL_2(iemCImpl_ldmxcsr, iEffSeg, GCPtrEff); +// IEM_MC_END(); +// return VINF_SUCCESS; +//} + + +/** + * @opmaps vexgrp15 + * @opcode !11/3 + * @oppfx none + * @opcpuid avx + * @opgroup og_avx_mxcsrsm + * @opxcpttype 5 + * @optest mxcsr=0 -> op1=0 + * @optest mxcsr=0x2083 -> op1=0x2083 + * @optest mxcsr=0x2084 cr0|=ts -> value.xcpt=0x7 + * @optest !amd / mxcsr=0x2085 cr0|=em -> op1=0x2085 + * @optest amd / mxcsr=0x2085 cr0|=em -> value.xcpt=0x6 + * @optest mxcsr=0x2086 cr0|=mp -> op1=0x2086 + * @optest mxcsr=0x2087 cr4&~=osfxsr -> op1=0x2087 + * @optest mxcsr=0x208f cr4&~=osxsave -> value.xcpt=0x6 + * @optest mxcsr=0x2087 cr4&~=osfxsr,osxsave -> value.xcpt=0x6 + * @optest !amd / mxcsr=0x2088 cr0|=ts,em -> value.xcpt=0x7 + * @optest amd / mxcsr=0x2088 cr0|=ts,em -> value.xcpt=0x6 + * @optest !amd / mxcsr=0x2089 cr0|=em cr4&~=osfxsr -> op1=0x2089 + * @optest amd / mxcsr=0x2089 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6 + * @optest !amd / mxcsr=0x208a cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x7 + * @optest amd / mxcsr=0x208a cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6 + * @optest !amd / mxcsr=0x208b cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x7 + * @optest amd / mxcsr=0x208b cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6 + * @optest !amd / mxcsr=0x208c xcr0&~=all_avx -> value.xcpt=0x6 + * @optest amd / mxcsr=0x208c xcr0&~=all_avx -> op1=0x208c + * @optest !amd / mxcsr=0x208d xcr0&~=all_avx_sse -> value.xcpt=0x6 + * @optest amd / mxcsr=0x208d xcr0&~=all_avx_sse -> op1=0x208d + * @optest !amd / mxcsr=0x208e xcr0&~=all_avx cr0|=ts -> value.xcpt=0x6 + * @optest amd / mxcsr=0x208e xcr0&~=all_avx cr0|=ts -> value.xcpt=0x7 + * @optest mxcsr=0x2082 cr0|=ts cr4&~=osxsave -> value.xcpt=0x6 + * @optest mxcsr=0x2081 xcr0&~=all_avx cr0|=ts cr4&~=osxsave + * -> value.xcpt=0x6 + * @remarks AMD Jaguar CPU (f0x16,m0,s1) \#UD when CR0.EM is set. It also + * doesn't seem to check XCR0[2:1] != 11b. This does not match the + * APMv4 rev 3.17 page 509. + * @todo Test this instruction on AMD Ryzen. + */ +FNIEMOP_DEF_1(iemOp_VGrp15_vstmxcsr, uint8_t, bRm) +{ + IEMOP_MNEMONIC1(VEX_M_MEM, VSTMXCSR, vstmxcsr, Md_WO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint8_t, iEffSeg, 0); + IEM_MC_ARG(RTGCPTR, GCPtrEff, 1); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ(); + IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg); + IEM_MC_CALL_CIMPL_2(iemCImpl_vstmxcsr, iEffSeg, GCPtrEff); + IEM_MC_END(); + return VINF_SUCCESS; +} + +/* Opcode VEX.0F 0xae mem/4 - invalid. */ +/* Opcode VEX.0F 0xae mem/5 - invalid. */ +/* Opcode VEX.0F 0xae mem/6 - invalid. */ +/* Opcode VEX.0F 0xae mem/7 - invalid. */ + +/* Opcode VEX.0F 0xae 11b/0 - invalid. */ +/* Opcode VEX.0F 0xae 11b/1 - invalid. */ +/* Opcode VEX.0F 0xae 11b/2 - invalid. */ +/* Opcode VEX.0F 0xae 11b/3 - invalid. */ +/* Opcode VEX.0F 0xae 11b/4 - invalid. */ +/* Opcode VEX.0F 0xae 11b/5 - invalid. */ +/* Opcode VEX.0F 0xae 11b/6 - invalid. */ +/* Opcode VEX.0F 0xae 11b/7 - invalid. */ + +/** + * Vex group 15 jump table for memory variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnVexGroup15MemReg[] = +{ /* pfx: none, 066h, 0f3h, 0f2h */ + /* /0 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /1 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /2 */ iemOp_VGrp15_vldmxcsr, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /3 */ iemOp_VGrp15_vstmxcsr, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /4 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /5 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /6 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, + /* /7 */ iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, iemOp_InvalidWithRM, +}; +AssertCompile(RT_ELEMENTS(g_apfnVexGroup15MemReg) == 8*4); + + +/** Opcode vex. 0xae. */ +FNIEMOP_DEF(iemOp_VGrp15) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + /* register, register */ + return FNIEMOP_CALL_1(iemOp_InvalidWithRM, bRm); + + /* memory, register */ + return FNIEMOP_CALL_1(g_apfnVexGroup15MemReg[ IEM_GET_MODRM_REG_8(bRm) * 4 + + pVCpu->iem.s.idxPrefix], bRm); +} + + +/* Opcode VEX.0F 0xaf - invalid. */ + +/* Opcode VEX.0F 0xb0 - invalid. */ +/* Opcode VEX.0F 0xb1 - invalid. */ +/* Opcode VEX.0F 0xb2 - invalid. */ +/* Opcode VEX.0F 0xb2 - invalid. */ +/* Opcode VEX.0F 0xb3 - invalid. */ +/* Opcode VEX.0F 0xb4 - invalid. */ +/* Opcode VEX.0F 0xb5 - invalid. */ +/* Opcode VEX.0F 0xb6 - invalid. */ +/* Opcode VEX.0F 0xb7 - invalid. */ +/* Opcode VEX.0F 0xb8 - invalid. */ +/* Opcode VEX.0F 0xb9 - invalid. */ +/* Opcode VEX.0F 0xba - invalid. */ +/* Opcode VEX.0F 0xbb - invalid. */ +/* Opcode VEX.0F 0xbc - invalid. */ +/* Opcode VEX.0F 0xbd - invalid. */ +/* Opcode VEX.0F 0xbe - invalid. */ +/* Opcode VEX.0F 0xbf - invalid. */ + +/* Opcode VEX.0F 0xc0 - invalid. */ +/* Opcode VEX.66.0F 0xc0 - invalid. */ +/* Opcode VEX.F3.0F 0xc0 - invalid. */ +/* Opcode VEX.F2.0F 0xc0 - invalid. */ + +/* Opcode VEX.0F 0xc1 - invalid. */ +/* Opcode VEX.66.0F 0xc1 - invalid. */ +/* Opcode VEX.F3.0F 0xc1 - invalid. */ +/* Opcode VEX.F2.0F 0xc1 - invalid. */ + +/** Opcode VEX.0F 0xc2 - vcmpps Vps,Hps,Wps,Ib */ +FNIEMOP_STUB(iemOp_vcmpps_Vps_Hps_Wps_Ib); +/** Opcode VEX.66.0F 0xc2 - vcmppd Vpd,Hpd,Wpd,Ib */ +FNIEMOP_STUB(iemOp_vcmppd_Vpd_Hpd_Wpd_Ib); +/** Opcode VEX.F3.0F 0xc2 - vcmpss Vss,Hss,Wss,Ib */ +FNIEMOP_STUB(iemOp_vcmpss_Vss_Hss_Wss_Ib); +/** Opcode VEX.F2.0F 0xc2 - vcmpsd Vsd,Hsd,Wsd,Ib */ +FNIEMOP_STUB(iemOp_vcmpsd_Vsd_Hsd_Wsd_Ib); + +/* Opcode VEX.0F 0xc3 - invalid */ +/* Opcode VEX.66.0F 0xc3 - invalid */ +/* Opcode VEX.F3.0F 0xc3 - invalid */ +/* Opcode VEX.F2.0F 0xc3 - invalid */ + +/* Opcode VEX.0F 0xc4 - invalid */ + + +/** Opcode VEX.66.0F 0xc4 - vpinsrw Vdq,Hdq,Ry/Mw,Ib */ +FNIEMOP_DEF(iemOp_vpinsrw_Vdq_Hdq_RyMw_Ib) +{ + /*IEMOP_MNEMONIC4(VEX_RMV, VPINSRW, vpinsrw, Vdq, Vdq, Ey, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);*/ /** @todo */ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx); + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG(uint16_t, u16Src, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vpinsrw_u128, iemAImpl_vpinsrw_u128_fallback), + puDst, puSrc, u16Src, bImmArg); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG(uint16_t, u16Src, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vpinsrw_u128, iemAImpl_vpinsrw_u128_fallback), + puDst, puSrc, u16Src, bImmArg); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode VEX.F3.0F 0xc4 - invalid */ +/* Opcode VEX.F2.0F 0xc4 - invalid */ + +/* Opcode VEX.0F 0xc5 - invlid */ + + +/** Opcode VEX.66.0F 0xc5 - vpextrw Gd, Udq, Ib */ +FNIEMOP_DEF(iemOp_vpextrw_Gd_Udq_Ib) +{ + IEMOP_MNEMONIC3(VEX_RMI_REG, VPEXTRW, vpextrw, Gd, Ux, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx); + IEM_MC_BEGIN(3, 1); + IEM_MC_LOCAL(uint16_t, u16Dst); + IEM_MC_ARG_LOCAL_REF(uint16_t *, pu16Dst, u16Dst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 2); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vpextrw_u128, iemAImpl_vpextrw_u128_fallback), + pu16Dst, puSrc, bImmArg); + IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u16Dst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + /* No memory operand. */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/* Opcode VEX.F3.0F 0xc5 - invalid */ +/* Opcode VEX.F2.0F 0xc5 - invalid */ + + +#define VSHUFP_X(a_Instr) \ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \ + if (IEM_IS_MODRM_REG_MODE(bRm)) \ + { \ + /* \ + * Register, register. \ + */ \ + if (pVCpu->iem.s.uVexLength) \ + { \ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); \ + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); \ + IEM_MC_BEGIN(4, 3); \ + IEM_MC_LOCAL(RTUINT256U, uDst); \ + IEM_MC_LOCAL(RTUINT256U, uSrc1); \ + IEM_MC_LOCAL(RTUINT256U, uSrc2); \ + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); \ + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 1); \ + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 2); \ + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); \ + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \ + IEM_MC_PREPARE_AVX_USAGE(); \ + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_FETCH_YREG_U256(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u256, \ + iemAImpl_ ## a_Instr ## _u256_fallback), puDst, puSrc1, puSrc2, bImmArg); \ + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); \ + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); \ + IEM_MC_BEGIN(4, 0); \ + IEM_MC_ARG(PRTUINT128U, puDst, 0); \ + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); \ + IEM_MC_ARG(PCRTUINT128U, puSrc2, 2); \ + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); \ + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \ + IEM_MC_PREPARE_AVX_USAGE(); \ + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u128, \ + iemAImpl_ ## a_Instr ## _u128_fallback), puDst, puSrc1, puSrc2, bImmArg); \ + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + else \ + { \ + /* \ + * Register, memory. \ + */ \ + if (pVCpu->iem.s.uVexLength) \ + { \ + IEM_MC_BEGIN(4, 4); \ + IEM_MC_LOCAL(RTUINT256U, uDst); \ + IEM_MC_LOCAL(RTUINT256U, uSrc1); \ + IEM_MC_LOCAL(RTUINT256U, uSrc2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); \ + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 1); \ + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 2); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); \ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); \ + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); \ + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); \ + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \ + IEM_MC_PREPARE_AVX_USAGE(); \ + IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u256, \ + iemAImpl_ ## a_Instr ## _u256_fallback), puDst, puSrc1, puSrc2, bImmArg); \ + IEM_MC_STORE_YREG_U256_ZX_VLMAX( IEM_GET_MODRM_REG(pVCpu, bRm), uDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(4, 2); \ + IEM_MC_LOCAL(RTUINT128U, uSrc2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_ARG(PRTUINT128U, puDst, 0); \ + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); \ + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc2, uSrc2, 2); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); \ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); \ + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); \ + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); \ + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \ + IEM_MC_PREPARE_AVX_USAGE(); \ + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u128, \ + iemAImpl_ ## a_Instr ## _u128_fallback), puDst, puSrc1, puSrc2, bImmArg); \ + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + (void)0 + +/** Opcode VEX.0F 0xc6 - vshufps Vps,Hps,Wps,Ib */ +FNIEMOP_DEF(iemOp_vshufps_Vps_Hps_Wps_Ib) +{ + IEMOP_MNEMONIC4(VEX_RMI, VSHUFPS, vshufps, Vpd, Hpd, Wpd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_SKIP_PYTHON); /** @todo */ + VSHUFP_X(vshufps); +} + + +/** Opcode VEX.66.0F 0xc6 - vshufpd Vpd,Hpd,Wpd,Ib */ +FNIEMOP_DEF(iemOp_vshufpd_Vpd_Hpd_Wpd_Ib) +{ + IEMOP_MNEMONIC4(VEX_RMI, VSHUFPD, vshufpd, Vpd, Hpd, Wpd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_SKIP_PYTHON); /** @todo */ + VSHUFP_X(vshufpd); +} +#undef VSHUFP_X + + +/* Opcode VEX.F3.0F 0xc6 - invalid */ +/* Opcode VEX.F2.0F 0xc6 - invalid */ + +/* Opcode VEX.0F 0xc7 - invalid */ +/* Opcode VEX.66.0F 0xc7 - invalid */ +/* Opcode VEX.F3.0F 0xc7 - invalid */ +/* Opcode VEX.F2.0F 0xc7 - invalid */ + +/* Opcode VEX.0F 0xc8 - invalid */ +/* Opcode VEX.0F 0xc9 - invalid */ +/* Opcode VEX.0F 0xca - invalid */ +/* Opcode VEX.0F 0xcb - invalid */ +/* Opcode VEX.0F 0xcc - invalid */ +/* Opcode VEX.0F 0xcd - invalid */ +/* Opcode VEX.0F 0xce - invalid */ +/* Opcode VEX.0F 0xcf - invalid */ + + +/* Opcode VEX.0F 0xd0 - invalid */ +/** Opcode VEX.66.0F 0xd0 - vaddsubpd Vpd, Hpd, Wpd */ +FNIEMOP_STUB(iemOp_vaddsubpd_Vpd_Hpd_Wpd); +/* Opcode VEX.F3.0F 0xd0 - invalid */ +/** Opcode VEX.F2.0F 0xd0 - vaddsubps Vps, Hps, Wps */ +FNIEMOP_STUB(iemOp_vaddsubps_Vps_Hps_Wps); + +/* Opcode VEX.0F 0xd1 - invalid */ +/** Opcode VEX.66.0F 0xd1 - vpsrlw Vx, Hx, W */ +FNIEMOP_STUB(iemOp_vpsrlw_Vx_Hx_W); +/* Opcode VEX.F3.0F 0xd1 - invalid */ +/* Opcode VEX.F2.0F 0xd1 - invalid */ + +/* Opcode VEX.0F 0xd2 - invalid */ +/** Opcode VEX.66.0F 0xd2 - vpsrld Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpsrld_Vx_Hx_Wx); +/* Opcode VEX.F3.0F 0xd2 - invalid */ +/* Opcode VEX.F2.0F 0xd2 - invalid */ + +/* Opcode VEX.0F 0xd3 - invalid */ +/** Opcode VEX.66.0F 0xd3 - vpsrlq Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpsrlq_Vx_Hx_Wx); +/* Opcode VEX.F3.0F 0xd3 - invalid */ +/* Opcode VEX.F2.0F 0xd3 - invalid */ + +/* Opcode VEX.0F 0xd4 - invalid */ + + +/** Opcode VEX.66.0F 0xd4 - vpaddq Vx, Hx, W */ +FNIEMOP_DEF(iemOp_vpaddq_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPADDQ, vpaddq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpaddq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xd4 - invalid */ +/* Opcode VEX.F2.0F 0xd4 - invalid */ + +/* Opcode VEX.0F 0xd5 - invalid */ + + +/** Opcode VEX.66.0F 0xd5 - vpmullw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpmullw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMULLW, vpmullw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpmullw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xd5 - invalid */ +/* Opcode VEX.F2.0F 0xd5 - invalid */ + +/* Opcode VEX.0F 0xd6 - invalid */ + +/** + * @opcode 0xd6 + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_pcksclr_datamove + * @opxcpttype none + * @optest op1=-1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_vmovq_Wq_Vq) +{ + IEMOP_MNEMONIC2(VEX_MR, VMOVQ, vmovq, Wq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_BEGIN(0, 0); + + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_COPY_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm), + IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(uint64_t, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + +/* Opcode VEX.F3.0F 0xd6 - invalid */ +/* Opcode VEX.F2.0F 0xd6 - invalid */ + + +/* Opcode VEX.0F 0xd7 - invalid */ + +/** Opcode VEX.66.0F 0xd7 - */ +FNIEMOP_DEF(iemOp_vpmovmskb_Gd_Ux) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + /* Docs says register only. */ + if (IEM_IS_MODRM_REG_MODE(bRm)) /** @todo test that this is registers only. */ + { + /* Note! Taking the lazy approch here wrt the high 32-bits of the GREG. */ + IEMOP_MNEMONIC2(RM_REG, VPMOVMSKB, vpmovmskb, Gd, Ux, DISOPTYPE_SSE | DISOPTYPE_HARMLESS, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + if (pVCpu->iem.s.uVexLength) + { + IEM_MC_BEGIN(2, 1); + IEM_MC_ARG(uint64_t *, puDst, 0); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc, uSrc, 1); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_GREG_U64(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vpmovmskb_u256, + iemAImpl_vpmovmskb_u256_fallback), puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(uint64_t *, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_GREG_U64(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_pmovmskb_u128, puDst, puSrc); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/* Opcode VEX.F3.0F 0xd7 - invalid */ +/* Opcode VEX.F2.0F 0xd7 - invalid */ + + +/* Opcode VEX.0F 0xd8 - invalid */ +/** Opcode VEX.66.0F 0xd8 - vpsubusb Vx, Hx, W */ +FNIEMOP_STUB(iemOp_vpsubusb_Vx_Hx_W); +/* Opcode VEX.F3.0F 0xd8 - invalid */ +/* Opcode VEX.F2.0F 0xd8 - invalid */ + +/* Opcode VEX.0F 0xd9 - invalid */ +/** Opcode VEX.66.0F 0xd9 - vpsubusw Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpsubusw_Vx_Hx_Wx); +/* Opcode VEX.F3.0F 0xd9 - invalid */ +/* Opcode VEX.F2.0F 0xd9 - invalid */ + +/* Opcode VEX.0F 0xda - invalid */ + + +/** Opcode VEX.66.0F 0xda - vpminub Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpminub_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMINUB, vpminub, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpminub); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xda - invalid */ +/* Opcode VEX.F2.0F 0xda - invalid */ + +/* Opcode VEX.0F 0xdb - invalid */ + + +/** Opcode VEX.66.0F 0xdb - vpand Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpand_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPAND, vpand, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpand, &g_iemAImpl_vpand_fallback)); +} + + +/* Opcode VEX.F3.0F 0xdb - invalid */ +/* Opcode VEX.F2.0F 0xdb - invalid */ + +/* Opcode VEX.0F 0xdc - invalid */ +/** Opcode VEX.66.0F 0xdc - vpaddusb Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpaddusb_Vx_Hx_Wx); +/* Opcode VEX.F3.0F 0xdc - invalid */ +/* Opcode VEX.F2.0F 0xdc - invalid */ + +/* Opcode VEX.0F 0xdd - invalid */ +/** Opcode VEX.66.0F 0xdd - vpaddusw Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpaddusw_Vx_Hx_Wx); +/* Opcode VEX.F3.0F 0xdd - invalid */ +/* Opcode VEX.F2.0F 0xdd - invalid */ + +/* Opcode VEX.0F 0xde - invalid */ + + +/** Opcode VEX.66.0F 0xde - vpmaxub Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpmaxub_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMAXUB, vpmaxub, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpmaxub); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xde - invalid */ +/* Opcode VEX.F2.0F 0xde - invalid */ + +/* Opcode VEX.0F 0xdf - invalid */ + + +/** Opcode VEX.66.0F 0xdf - vpandn Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpandn_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPANDN, vpandn, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpandn, &g_iemAImpl_vpandn_fallback)); +} + + +/* Opcode VEX.F3.0F 0xdf - invalid */ +/* Opcode VEX.F2.0F 0xdf - invalid */ + +/* Opcode VEX.0F 0xe0 - invalid */ + + +/** Opcode VEX.66.0F 0xe0 - vpavgb Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpavgb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPAVGB, vpavgb, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpavgb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xe0 - invalid */ +/* Opcode VEX.F2.0F 0xe0 - invalid */ + +/* Opcode VEX.0F 0xe1 - invalid */ +/** Opcode VEX.66.0F 0xe1 - vpsraw Vx, Hx, W */ +FNIEMOP_STUB(iemOp_vpsraw_Vx_Hx_W); +/* Opcode VEX.F3.0F 0xe1 - invalid */ +/* Opcode VEX.F2.0F 0xe1 - invalid */ + +/* Opcode VEX.0F 0xe2 - invalid */ +/** Opcode VEX.66.0F 0xe2 - vpsrad Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpsrad_Vx_Hx_Wx); +/* Opcode VEX.F3.0F 0xe2 - invalid */ +/* Opcode VEX.F2.0F 0xe2 - invalid */ + +/* Opcode VEX.0F 0xe3 - invalid */ + + +/** Opcode VEX.66.0F 0xe3 - vpavgw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpavgw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPAVGW, vpavgw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpavgw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xe3 - invalid */ +/* Opcode VEX.F2.0F 0xe3 - invalid */ + +/* Opcode VEX.0F 0xe4 - invalid */ + + +/** Opcode VEX.66.0F 0xe4 - vpmulhuw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpmulhuw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMULHUW, vpmulhuw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpmulhuw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xe4 - invalid */ +/* Opcode VEX.F2.0F 0xe4 - invalid */ + +/* Opcode VEX.0F 0xe5 - invalid */ + + +/** Opcode VEX.66.0F 0xe5 - vpmulhw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpmulhw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMULHW, vpmulhw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpmulhw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xe5 - invalid */ +/* Opcode VEX.F2.0F 0xe5 - invalid */ + +/* Opcode VEX.0F 0xe6 - invalid */ +/** Opcode VEX.66.0F 0xe6 - vcvttpd2dq Vx, Wpd */ +FNIEMOP_STUB(iemOp_vcvttpd2dq_Vx_Wpd); +/** Opcode VEX.F3.0F 0xe6 - vcvtdq2pd Vx, Wpd */ +FNIEMOP_STUB(iemOp_vcvtdq2pd_Vx_Wpd); +/** Opcode VEX.F2.0F 0xe6 - vcvtpd2dq Vx, Wpd */ +FNIEMOP_STUB(iemOp_vcvtpd2dq_Vx_Wpd); + + +/* Opcode VEX.0F 0xe7 - invalid */ + +/** + * @opcode 0xe7 + * @opcodesub !11 mr/reg + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_cachect + * @opxcpttype 1 + * @optest op1=-1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ +FNIEMOP_DEF(iemOp_vmovntdq_Mx_Vx) +{ + IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVNTDQ, vmovntdq, Mx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + if (pVCpu->iem.s.uVexLength == 0) + { + /* + * 128-bit: Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * 256-bit: Memory, register. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ(); + + IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + /** + * @opdone + * @opmnemonic udvex660fe7reg + * @opcode 0xe7 + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid avx + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + +/* Opcode VEX.F3.0F 0xe7 - invalid */ +/* Opcode VEX.F2.0F 0xe7 - invalid */ + + +/* Opcode VEX.0F 0xe8 - invalid */ +/** Opcode VEX.66.0F 0xe8 - vpsubsb Vx, Hx, W */ +FNIEMOP_STUB(iemOp_vpsubsb_Vx_Hx_W); +/* Opcode VEX.F3.0F 0xe8 - invalid */ +/* Opcode VEX.F2.0F 0xe8 - invalid */ + +/* Opcode VEX.0F 0xe9 - invalid */ +/** Opcode VEX.66.0F 0xe9 - vpsubsw Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpsubsw_Vx_Hx_Wx); +/* Opcode VEX.F3.0F 0xe9 - invalid */ +/* Opcode VEX.F2.0F 0xe9 - invalid */ + +/* Opcode VEX.0F 0xea - invalid */ + + +/** Opcode VEX.66.0F 0xea - vpminsw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpminsw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMINSW, vpminsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpminsw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xea - invalid */ +/* Opcode VEX.F2.0F 0xea - invalid */ + +/* Opcode VEX.0F 0xeb - invalid */ + + +/** Opcode VEX.66.0F 0xeb - vpor Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpor_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPOR, vpor, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpor, &g_iemAImpl_vpor_fallback)); +} + + + +/* Opcode VEX.F3.0F 0xeb - invalid */ +/* Opcode VEX.F2.0F 0xeb - invalid */ + +/* Opcode VEX.0F 0xec - invalid */ +/** Opcode VEX.66.0F 0xec - vpaddsb Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpaddsb_Vx_Hx_Wx); +/* Opcode VEX.F3.0F 0xec - invalid */ +/* Opcode VEX.F2.0F 0xec - invalid */ + +/* Opcode VEX.0F 0xed - invalid */ +/** Opcode VEX.66.0F 0xed - vpaddsw Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpaddsw_Vx_Hx_Wx); +/* Opcode VEX.F3.0F 0xed - invalid */ +/* Opcode VEX.F2.0F 0xed - invalid */ + +/* Opcode VEX.0F 0xee - invalid */ + + +/** Opcode VEX.66.0F 0xee - vpmaxsw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpmaxsw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMAXSW, vpmaxsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpmaxsw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F3.0F 0xee - invalid */ +/* Opcode VEX.F2.0F 0xee - invalid */ + + +/* Opcode VEX.0F 0xef - invalid */ + + +/** Opcode VEX.66.0F 0xef - vpxor Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpxor_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPXOR, vpxor, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, + IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpxor, &g_iemAImpl_vpxor_fallback)); +} + + +/* Opcode VEX.F3.0F 0xef - invalid */ +/* Opcode VEX.F2.0F 0xef - invalid */ + +/* Opcode VEX.0F 0xf0 - invalid */ +/* Opcode VEX.66.0F 0xf0 - invalid */ + + +/** Opcode VEX.F2.0F 0xf0 - vlddqu Vx, Mx */ +FNIEMOP_DEF(iemOp_vlddqu_Vx_Mx) +{ + IEMOP_MNEMONIC2(VEX_RM_MEM, VLDDQU, vlddqu, Vx_WO, Mx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register - (not implemented, assuming it raises \#UD). + */ + return IEMOP_RAISE_INVALID_OPCODE(); + } + else if (pVCpu->iem.s.uVexLength == 0) + { + /* + * Register, memory128. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, u128Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory256. + */ + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, u256Tmp); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U256(u256Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u256Tmp); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode VEX.0F 0xf1 - invalid */ +/** Opcode VEX.66.0F 0xf1 - vpsllw Vx, Hx, W */ +FNIEMOP_STUB(iemOp_vpsllw_Vx_Hx_W); +/* Opcode VEX.F2.0F 0xf1 - invalid */ + +/* Opcode VEX.0F 0xf2 - invalid */ +/** Opcode VEX.66.0F 0xf2 - vpslld Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpslld_Vx_Hx_Wx); +/* Opcode VEX.F2.0F 0xf2 - invalid */ + +/* Opcode VEX.0F 0xf3 - invalid */ +/** Opcode VEX.66.0F 0xf3 - vpsllq Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpsllq_Vx_Hx_Wx); +/* Opcode VEX.F2.0F 0xf3 - invalid */ + +/* Opcode VEX.0F 0xf4 - invalid */ + + +/** Opcode VEX.66.0F 0xf4 - vpmuludq Vx, Hx, W */ +FNIEMOP_DEF(iemOp_vpmuludq_Vx_Hx_W) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMULUDQ, vpmuludq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpmuludq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F2.0F 0xf4 - invalid */ + +/* Opcode VEX.0F 0xf5 - invalid */ +/** Opcode VEX.66.0F 0xf5 - vpmaddwd Vx, Hx, Wx */ +FNIEMOP_STUB(iemOp_vpmaddwd_Vx_Hx_Wx); +/* Opcode VEX.F2.0F 0xf5 - invalid */ + +/* Opcode VEX.0F 0xf6 - invalid */ + + +/** Opcode VEX.66.0F 0xf6 - vpsadbw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpsadbw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPSADBW, vpsadbw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpsadbw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F2.0F 0xf6 - invalid */ + +/* Opcode VEX.0F 0xf7 - invalid */ +/** Opcode VEX.66.0F 0xf7 - vmaskmovdqu Vdq, Udq */ +FNIEMOP_STUB(iemOp_vmaskmovdqu_Vdq_Udq); +/* Opcode VEX.F2.0F 0xf7 - invalid */ + +/* Opcode VEX.0F 0xf8 - invalid */ + + +/** Opcode VEX.66.0F 0xf8 - vpsubb Vx, Hx, W */ +FNIEMOP_DEF(iemOp_vpsubb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPSUBB, vpsubb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpsubb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F2.0F 0xf8 - invalid */ + +/* Opcode VEX.0F 0xf9 - invalid */ + + +/** Opcode VEX.66.0F 0xf9 - vpsubw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpsubw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPSUBW, vpsubw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpsubw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F2.0F 0xf9 - invalid */ + +/* Opcode VEX.0F 0xfa - invalid */ + + +/** Opcode VEX.66.0F 0xfa - vpsubd Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpsubd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPSUBD, vpsubd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpsubd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F2.0F 0xfa - invalid */ + +/* Opcode VEX.0F 0xfb - invalid */ + + +/** Opcode VEX.66.0F 0xfb - vpsubq Vx, Hx, W */ +FNIEMOP_DEF(iemOp_vpsubq_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPSUBQ, vpsubq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpsubq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F2.0F 0xfb - invalid */ + +/* Opcode VEX.0F 0xfc - invalid */ + + +/** Opcode VEX.66.0F 0xfc - vpaddb Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpaddb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPADDB, vpaddb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpaddb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F2.0F 0xfc - invalid */ + +/* Opcode VEX.0F 0xfd - invalid */ + + +/** Opcode VEX.66.0F 0xfd - vpaddw Vx, Hx, Wx */ +FNIEMOP_DEF(iemOp_vpaddw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPADDW, vpaddw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpaddw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F2.0F 0xfd - invalid */ + +/* Opcode VEX.0F 0xfe - invalid */ + + +/** Opcode VEX.66.0F 0xfe - vpaddd Vx, Hx, W */ +FNIEMOP_DEF(iemOp_vpaddd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPADDD, vpaddd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS( vpaddd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.F2.0F 0xfe - invalid */ + + +/** Opcode **** 0x0f 0xff - UD0 */ +FNIEMOP_DEF(iemOp_vud0) +{ + IEMOP_MNEMONIC(vud0, "vud0"); + if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL) + { + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm); +#ifndef TST_IEM_CHECK_MC + RTGCPTR GCPtrEff; + VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); + if (rcStrict != VINF_SUCCESS) + return rcStrict; +#endif + IEMOP_HLP_DONE_DECODING(); + } + return IEMOP_RAISE_INVALID_OPCODE(); +} + + + +/** + * VEX opcode map \#1. + * + * @sa g_apfnTwoByteMap + */ +IEM_STATIC const PFNIEMOP g_apfnVexMap1[] = +{ + /* no prefix, 066h prefix f3h prefix, f2h prefix */ + /* 0x00 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x01 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x02 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x03 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x04 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x05 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x06 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x07 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x08 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x09 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x0a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x0b */ IEMOP_X4(iemOp_vud2), /* ?? */ + /* 0x0c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x0d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x0e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x0f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x10 */ iemOp_vmovups_Vps_Wps, iemOp_vmovupd_Vpd_Wpd, iemOp_vmovss_Vss_Hss_Wss, iemOp_vmovsd_Vsd_Hsd_Wsd, + /* 0x11 */ iemOp_vmovups_Wps_Vps, iemOp_vmovupd_Wpd_Vpd, iemOp_vmovss_Wss_Hss_Vss, iemOp_vmovsd_Wsd_Hsd_Vsd, + /* 0x12 */ iemOp_vmovlps_Vq_Hq_Mq__vmovhlps, iemOp_vmovlpd_Vq_Hq_Mq, iemOp_vmovsldup_Vx_Wx, iemOp_vmovddup_Vx_Wx, + /* 0x13 */ iemOp_vmovlps_Mq_Vq, iemOp_vmovlpd_Mq_Vq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x14 */ iemOp_vunpcklps_Vx_Hx_Wx, iemOp_vunpcklpd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x15 */ iemOp_vunpckhps_Vx_Hx_Wx, iemOp_vunpckhpd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x16 */ iemOp_vmovhps_Vdq_Hq_Mq__vmovlhps_Vdq_Hq_Uq, iemOp_vmovhpd_Vdq_Hq_Mq, iemOp_vmovshdup_Vx_Wx, iemOp_InvalidNeedRM, + /* 0x17 */ iemOp_vmovhps_Mq_Vq, iemOp_vmovhpd_Mq_Vq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x18 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x19 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x20 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x21 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x22 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x23 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x24 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x25 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x26 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x27 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x28 */ iemOp_vmovaps_Vps_Wps, iemOp_vmovapd_Vpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x29 */ iemOp_vmovaps_Wps_Vps, iemOp_vmovapd_Wpd_Vpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2a */ iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_vcvtsi2ss_Vss_Hss_Ey, iemOp_vcvtsi2sd_Vsd_Hsd_Ey, + /* 0x2b */ iemOp_vmovntps_Mps_Vps, iemOp_vmovntpd_Mpd_Vpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2c */ iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_vcvttss2si_Gy_Wss, iemOp_vcvttsd2si_Gy_Wsd, + /* 0x2d */ iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_vcvtss2si_Gy_Wss, iemOp_vcvtsd2si_Gy_Wsd, + /* 0x2e */ iemOp_vucomiss_Vss_Wss, iemOp_vucomisd_Vsd_Wsd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2f */ iemOp_vcomiss_Vss_Wss, iemOp_vcomisd_Vsd_Wsd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0x30 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x31 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x32 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x33 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x34 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x35 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x36 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x37 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x38 */ IEMOP_X4(iemOp_InvalidNeedRM), /** @todo check that there is no escape table stuff here */ + /* 0x39 */ IEMOP_X4(iemOp_InvalidNeedRM), /** @todo check that there is no escape table stuff here */ + /* 0x3a */ IEMOP_X4(iemOp_InvalidNeedRM), /** @todo check that there is no escape table stuff here */ + /* 0x3b */ IEMOP_X4(iemOp_InvalidNeedRM), /** @todo check that there is no escape table stuff here */ + /* 0x3c */ IEMOP_X4(iemOp_InvalidNeedRM), /** @todo check that there is no escape table stuff here */ + /* 0x3d */ IEMOP_X4(iemOp_InvalidNeedRM), /** @todo check that there is no escape table stuff here */ + /* 0x3e */ IEMOP_X4(iemOp_InvalidNeedRM), /** @todo check that there is no escape table stuff here */ + /* 0x3f */ IEMOP_X4(iemOp_InvalidNeedRM), /** @todo check that there is no escape table stuff here */ + + /* 0x40 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x41 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x42 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x43 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x44 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x45 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x46 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x47 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x48 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x49 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x50 */ iemOp_vmovmskps_Gy_Ups, iemOp_vmovmskpd_Gy_Upd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x51 */ iemOp_vsqrtps_Vps_Wps, iemOp_vsqrtpd_Vpd_Wpd, iemOp_vsqrtss_Vss_Hss_Wss, iemOp_vsqrtsd_Vsd_Hsd_Wsd, + /* 0x52 */ iemOp_vrsqrtps_Vps_Wps, iemOp_InvalidNeedRM, iemOp_vrsqrtss_Vss_Hss_Wss, iemOp_InvalidNeedRM, + /* 0x53 */ iemOp_vrcpps_Vps_Wps, iemOp_InvalidNeedRM, iemOp_vrcpss_Vss_Hss_Wss, iemOp_InvalidNeedRM, + /* 0x54 */ iemOp_vandps_Vps_Hps_Wps, iemOp_vandpd_Vpd_Hpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x55 */ iemOp_vandnps_Vps_Hps_Wps, iemOp_vandnpd_Vpd_Hpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x56 */ iemOp_vorps_Vps_Hps_Wps, iemOp_vorpd_Vpd_Hpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x57 */ iemOp_vxorps_Vps_Hps_Wps, iemOp_vxorpd_Vpd_Hpd_Wpd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x58 */ iemOp_vaddps_Vps_Hps_Wps, iemOp_vaddpd_Vpd_Hpd_Wpd, iemOp_vaddss_Vss_Hss_Wss, iemOp_vaddsd_Vsd_Hsd_Wsd, + /* 0x59 */ iemOp_vmulps_Vps_Hps_Wps, iemOp_vmulpd_Vpd_Hpd_Wpd, iemOp_vmulss_Vss_Hss_Wss, iemOp_vmulsd_Vsd_Hsd_Wsd, + /* 0x5a */ iemOp_vcvtps2pd_Vpd_Wps, iemOp_vcvtpd2ps_Vps_Wpd, iemOp_vcvtss2sd_Vsd_Hx_Wss, iemOp_vcvtsd2ss_Vss_Hx_Wsd, + /* 0x5b */ iemOp_vcvtdq2ps_Vps_Wdq, iemOp_vcvtps2dq_Vdq_Wps, iemOp_vcvttps2dq_Vdq_Wps, iemOp_InvalidNeedRM, + /* 0x5c */ iemOp_vsubps_Vps_Hps_Wps, iemOp_vsubpd_Vpd_Hpd_Wpd, iemOp_vsubss_Vss_Hss_Wss, iemOp_vsubsd_Vsd_Hsd_Wsd, + /* 0x5d */ iemOp_vminps_Vps_Hps_Wps, iemOp_vminpd_Vpd_Hpd_Wpd, iemOp_vminss_Vss_Hss_Wss, iemOp_vminsd_Vsd_Hsd_Wsd, + /* 0x5e */ iemOp_vdivps_Vps_Hps_Wps, iemOp_vdivpd_Vpd_Hpd_Wpd, iemOp_vdivss_Vss_Hss_Wss, iemOp_vdivsd_Vsd_Hsd_Wsd, + /* 0x5f */ iemOp_vmaxps_Vps_Hps_Wps, iemOp_vmaxpd_Vpd_Hpd_Wpd, iemOp_vmaxss_Vss_Hss_Wss, iemOp_vmaxsd_Vsd_Hsd_Wsd, + + /* 0x60 */ iemOp_InvalidNeedRM, iemOp_vpunpcklbw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x61 */ iemOp_InvalidNeedRM, iemOp_vpunpcklwd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x62 */ iemOp_InvalidNeedRM, iemOp_vpunpckldq_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x63 */ iemOp_InvalidNeedRM, iemOp_vpacksswb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x64 */ iemOp_InvalidNeedRM, iemOp_vpcmpgtb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x65 */ iemOp_InvalidNeedRM, iemOp_vpcmpgtw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x66 */ iemOp_InvalidNeedRM, iemOp_vpcmpgtd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x67 */ iemOp_InvalidNeedRM, iemOp_vpackuswb_Vx_Hx_W, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x68 */ iemOp_InvalidNeedRM, iemOp_vpunpckhbw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x69 */ iemOp_InvalidNeedRM, iemOp_vpunpckhwd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6a */ iemOp_InvalidNeedRM, iemOp_vpunpckhdq_Vx_Hx_W, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6b */ iemOp_InvalidNeedRM, iemOp_vpackssdw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6c */ iemOp_InvalidNeedRM, iemOp_vpunpcklqdq_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6d */ iemOp_InvalidNeedRM, iemOp_vpunpckhqdq_Vx_Hx_W, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6e */ iemOp_InvalidNeedRM, iemOp_vmovd_q_Vy_Ey, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x6f */ iemOp_InvalidNeedRM, iemOp_vmovdqa_Vx_Wx, iemOp_vmovdqu_Vx_Wx, iemOp_InvalidNeedRM, + + /* 0x70 */ iemOp_InvalidNeedRM, iemOp_vpshufd_Vx_Wx_Ib, iemOp_vpshufhw_Vx_Wx_Ib, iemOp_vpshuflw_Vx_Wx_Ib, + /* 0x71 */ iemOp_InvalidNeedRM, iemOp_VGrp12, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x72 */ iemOp_InvalidNeedRM, iemOp_VGrp13, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x73 */ iemOp_InvalidNeedRM, iemOp_VGrp14, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x74 */ iemOp_InvalidNeedRM, iemOp_vpcmpeqb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x75 */ iemOp_InvalidNeedRM, iemOp_vpcmpeqw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x76 */ iemOp_InvalidNeedRM, iemOp_vpcmpeqd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x77 */ iemOp_vzeroupperv__vzeroallv, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x78 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x79 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7c */ iemOp_InvalidNeedRM, iemOp_vhaddpd_Vpd_Hpd_Wpd, iemOp_InvalidNeedRM, iemOp_vhaddps_Vps_Hps_Wps, + /* 0x7d */ iemOp_InvalidNeedRM, iemOp_vhsubpd_Vpd_Hpd_Wpd, iemOp_InvalidNeedRM, iemOp_vhsubps_Vps_Hps_Wps, + /* 0x7e */ iemOp_InvalidNeedRM, iemOp_vmovd_q_Ey_Vy, iemOp_vmovq_Vq_Wq, iemOp_InvalidNeedRM, + /* 0x7f */ iemOp_InvalidNeedRM, iemOp_vmovdqa_Wx_Vx, iemOp_vmovdqu_Wx_Vx, iemOp_InvalidNeedRM, + + /* 0x80 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x81 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x82 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x83 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x84 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x85 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x86 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x87 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x88 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x89 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x90 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x91 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x92 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x93 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x94 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x95 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x96 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x97 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x98 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x99 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x9f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xa0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xaa */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xab */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xac */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xad */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xae */ IEMOP_X4(iemOp_VGrp15), + /* 0xaf */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xb0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xba */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbb */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbc */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbd */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbe */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xbf */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xc0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc2 */ iemOp_vcmpps_Vps_Hps_Wps_Ib, iemOp_vcmppd_Vpd_Hpd_Wpd_Ib, iemOp_vcmpss_Vss_Hss_Wss_Ib, iemOp_vcmpsd_Vsd_Hsd_Wsd_Ib, + /* 0xc3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc4 */ iemOp_InvalidNeedRM, iemOp_vpinsrw_Vdq_Hdq_RyMw_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0xc5 */ iemOp_InvalidNeedRM, iemOp_vpextrw_Gd_Udq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0xc6 */ iemOp_vshufps_Vps_Hps_Wps_Ib, iemOp_vshufpd_Vpd_Hpd_Wpd_Ib, iemOp_InvalidNeedRMImm8,iemOp_InvalidNeedRMImm8, + /* 0xc7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xca */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xcb */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xcc */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xcd */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xce */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xcf */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xd0 */ iemOp_InvalidNeedRM, iemOp_vaddsubpd_Vpd_Hpd_Wpd, iemOp_InvalidNeedRM, iemOp_vaddsubps_Vps_Hps_Wps, + /* 0xd1 */ iemOp_InvalidNeedRM, iemOp_vpsrlw_Vx_Hx_W, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd2 */ iemOp_InvalidNeedRM, iemOp_vpsrld_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd3 */ iemOp_InvalidNeedRM, iemOp_vpsrlq_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd4 */ iemOp_InvalidNeedRM, iemOp_vpaddq_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd5 */ iemOp_InvalidNeedRM, iemOp_vpmullw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd6 */ iemOp_InvalidNeedRM, iemOp_vmovq_Wq_Vq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd7 */ iemOp_InvalidNeedRM, iemOp_vpmovmskb_Gd_Ux, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd8 */ iemOp_InvalidNeedRM, iemOp_vpsubusb_Vx_Hx_W, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xd9 */ iemOp_InvalidNeedRM, iemOp_vpsubusw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xda */ iemOp_InvalidNeedRM, iemOp_vpminub_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdb */ iemOp_InvalidNeedRM, iemOp_vpand_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdc */ iemOp_InvalidNeedRM, iemOp_vpaddusb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdd */ iemOp_InvalidNeedRM, iemOp_vpaddusw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xde */ iemOp_InvalidNeedRM, iemOp_vpmaxub_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdf */ iemOp_InvalidNeedRM, iemOp_vpandn_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0xe0 */ iemOp_InvalidNeedRM, iemOp_vpavgb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe1 */ iemOp_InvalidNeedRM, iemOp_vpsraw_Vx_Hx_W, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe2 */ iemOp_InvalidNeedRM, iemOp_vpsrad_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe3 */ iemOp_InvalidNeedRM, iemOp_vpavgw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe4 */ iemOp_InvalidNeedRM, iemOp_vpmulhuw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe5 */ iemOp_InvalidNeedRM, iemOp_vpmulhw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe6 */ iemOp_InvalidNeedRM, iemOp_vcvttpd2dq_Vx_Wpd, iemOp_vcvtdq2pd_Vx_Wpd, iemOp_vcvtpd2dq_Vx_Wpd, + /* 0xe7 */ iemOp_InvalidNeedRM, iemOp_vmovntdq_Mx_Vx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe8 */ iemOp_InvalidNeedRM, iemOp_vpsubsb_Vx_Hx_W, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xe9 */ iemOp_InvalidNeedRM, iemOp_vpsubsw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xea */ iemOp_InvalidNeedRM, iemOp_vpminsw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xeb */ iemOp_InvalidNeedRM, iemOp_vpor_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xec */ iemOp_InvalidNeedRM, iemOp_vpaddsb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xed */ iemOp_InvalidNeedRM, iemOp_vpaddsw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xee */ iemOp_InvalidNeedRM, iemOp_vpmaxsw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xef */ iemOp_InvalidNeedRM, iemOp_vpxor_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0xf0 */ iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_vlddqu_Vx_Mx, + /* 0xf1 */ iemOp_InvalidNeedRM, iemOp_vpsllw_Vx_Hx_W, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf2 */ iemOp_InvalidNeedRM, iemOp_vpslld_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf3 */ iemOp_InvalidNeedRM, iemOp_vpsllq_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf4 */ iemOp_InvalidNeedRM, iemOp_vpmuludq_Vx_Hx_W, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf5 */ iemOp_InvalidNeedRM, iemOp_vpmaddwd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf6 */ iemOp_InvalidNeedRM, iemOp_vpsadbw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf7 */ iemOp_InvalidNeedRM, iemOp_vmaskmovdqu_Vdq_Udq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf8 */ iemOp_InvalidNeedRM, iemOp_vpsubb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf9 */ iemOp_InvalidNeedRM, iemOp_vpsubw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfa */ iemOp_InvalidNeedRM, iemOp_vpsubd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfb */ iemOp_InvalidNeedRM, iemOp_vpsubq_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfc */ iemOp_InvalidNeedRM, iemOp_vpaddb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfd */ iemOp_InvalidNeedRM, iemOp_vpaddw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xfe */ iemOp_InvalidNeedRM, iemOp_vpaddd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xff */ IEMOP_X4(iemOp_vud0) /* ?? */ +}; +AssertCompile(RT_ELEMENTS(g_apfnVexMap1) == 1024); +/** @} */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h new file mode 100644 index 00000000..02fa3df9 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h @@ -0,0 +1,2107 @@ +/* $Id: IEMAllInstructionsVexMap2.cpp.h $ */ +/** @file + * IEM - Instruction Decoding and Emulation. + * + * @remarks IEMAllInstructionsThree0f38.cpp.h is a VEX mirror of this file. + * Any update here is likely needed in that file too. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/** @name VEX Opcode Map 2 + * @{ + */ + +/* Opcode VEX.0F38 0x00 - invalid. */ + + +/** Opcode VEX.66.0F38 0x00. */ +FNIEMOP_DEF(iemOp_vpshufb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPSHUFB, vpshufb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpshufb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x01 - invalid. */ + + +/** Opcode VEX.66.0F38 0x01. */ +FNIEMOP_DEF(iemOp_vphaddw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPHADDW, vphaddw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vphaddw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x02 - invalid. */ + + +/** Opcode VEX.66.0F38 0x02. */ +FNIEMOP_DEF(iemOp_vphaddd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPHADDD, vphaddd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vphaddd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x03 - invalid. */ + + +/** Opcode VEX.66.0F38 0x03. */ +FNIEMOP_DEF(iemOp_vphaddsw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPHADDSW, vphaddsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vphaddsw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x04 - invalid. */ + + +/** Opcode VEX.66.0F38 0x04. */ +FNIEMOP_DEF(iemOp_vpmaddubsw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMADDUBSW, vpmaddubsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpmaddubsw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x05 - invalid. */ + + +/** Opcode VEX.66.0F38 0x05. */ +FNIEMOP_DEF(iemOp_vphsubw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPHSUBW, vphsubw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vphsubw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x06 - invalid. */ + + +/** Opcode VEX.66.0F38 0x06. */ +FNIEMOP_DEF(iemOp_vphsubd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPHSUBD, vphsubd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vphsubd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x07 - invalid. */ + + +/** Opcode VEX.66.0F38 0x07. */ +FNIEMOP_DEF(iemOp_vphsubsw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPHSUBSW, vphsubsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vphsubsw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x08 - invalid. */ + + +/** Opcode VEX.66.0F38 0x08. */ +FNIEMOP_DEF(iemOp_vpsignb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPSIGNB, vpsignb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpsignb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x09 - invalid. */ + + +/** Opcode VEX.66.0F38 0x09. */ +FNIEMOP_DEF(iemOp_vpsignw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPSIGNW, vpsignw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpsignw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x0a - invalid. */ + + +/** Opcode VEX.66.0F38 0x0a. */ +FNIEMOP_DEF(iemOp_vpsignd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPSIGND, vpsignd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpsignd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x0b - invalid. */ + + +/** Opcode VEX.66.0F38 0x0b. */ +FNIEMOP_DEF(iemOp_vpmulhrsw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMULHRSW, vpmulhrsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpmulhrsw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x0c - invalid. */ +/** Opcode VEX.66.0F38 0x0c. */ +FNIEMOP_STUB(iemOp_vpermilps_Vx_Hx_Wx); +/* Opcode VEX.0F38 0x0d - invalid. */ +/** Opcode VEX.66.0F38 0x0d. */ +FNIEMOP_STUB(iemOp_vpermilpd_Vx_Hx_Wx); +/* Opcode VEX.0F38 0x0e - invalid. */ +/** Opcode VEX.66.0F38 0x0e. */ +FNIEMOP_STUB(iemOp_vtestps_Vx_Wx); +/* Opcode VEX.0F38 0x0f - invalid. */ +/** Opcode VEX.66.0F38 0x0f. */ +FNIEMOP_STUB(iemOp_vtestpd_Vx_Wx); + + +/* Opcode VEX.0F38 0x10 - invalid */ +/* Opcode VEX.66.0F38 0x10 - invalid (legacy only). */ +/* Opcode VEX.0F38 0x11 - invalid */ +/* Opcode VEX.66.0F38 0x11 - invalid */ +/* Opcode VEX.0F38 0x12 - invalid */ +/* Opcode VEX.66.0F38 0x12 - invalid */ +/* Opcode VEX.0F38 0x13 - invalid */ +/* Opcode VEX.66.0F38 0x13 - invalid (vex only). */ +/* Opcode VEX.0F38 0x14 - invalid */ +/* Opcode VEX.66.0F38 0x14 - invalid (legacy only). */ +/* Opcode VEX.0F38 0x15 - invalid */ +/* Opcode VEX.66.0F38 0x15 - invalid (legacy only). */ +/* Opcode VEX.0F38 0x16 - invalid */ +/** Opcode VEX.66.0F38 0x16. */ +FNIEMOP_STUB(iemOp_vpermps_Vqq_Hqq_Wqq); +/* Opcode VEX.0F38 0x17 - invalid */ + + +/** Opcode VEX.66.0F38 0x17 - invalid */ +FNIEMOP_DEF(iemOp_vptest_Vx_Wx) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + if (pVCpu->iem.s.uVexLength) + { + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_YREG_U256(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vptest_u256, iemAImpl_vptest_u256_fallback), + puSrc1, puSrc2, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc2, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_ptest_u128, puSrc1, puSrc2, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength) + { + IEM_MC_BEGIN(3, 3); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vptest_u256, iemAImpl_vptest_u256_fallback), + puSrc1, puSrc2, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc2, uSrc2, 1); + IEM_MC_ARG(uint32_t *, pEFlags, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_ptest_u128, puSrc1, puSrc2, pEFlags); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/* Opcode VEX.0F38 0x18 - invalid */ +/** Opcode VEX.66.0F38 0x18. */ +FNIEMOP_STUB(iemOp_vbroadcastss_Vx_Wd); +/* Opcode VEX.0F38 0x19 - invalid */ +/** Opcode VEX.66.0F38 0x19. */ +FNIEMOP_STUB(iemOp_vbroadcastsd_Vqq_Wq); +/* Opcode VEX.0F38 0x1a - invalid */ +/** Opcode VEX.66.0F38 0x1a. */ +FNIEMOP_STUB(iemOp_vbroadcastf128_Vqq_Mdq); +/* Opcode VEX.0F38 0x1b - invalid */ +/* Opcode VEX.66.0F38 0x1b - invalid */ +/* Opcode VEX.0F38 0x1c - invalid. */ + + +/** Opcode VEX.66.0F38 0x1c. */ +FNIEMOP_DEF(iemOp_vpabsb_Vx_Wx) +{ + IEMOP_MNEMONIC2(VEX_RM, VPABSB, vpabsb, Vx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF2_INIT_VARS(vpabsb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x1d - invalid. */ + + +/** Opcode VEX.66.0F38 0x1d. */ +FNIEMOP_DEF(iemOp_vpabsw_Vx_Wx) +{ + IEMOP_MNEMONIC2(VEX_RM, VPABSW, vpabsw, Vx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF2_INIT_VARS(vpabsw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + +/* Opcode VEX.0F38 0x1e - invalid. */ + + +/** Opcode VEX.66.0F38 0x1e. */ +FNIEMOP_DEF(iemOp_vpabsd_Vx_Wx) +{ + IEMOP_MNEMONIC2(VEX_RM, VPABSD, vpabsd, Vx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF2_INIT_VARS(vpabsd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.0F38 0x1f - invalid */ +/* Opcode VEX.66.0F38 0x1f - invalid */ + + +/** Body for the vpmov{s,z}x* instructions. */ +#define IEMOP_BODY_VPMOV_S_Z(a_Instr, a_SrcWidth) \ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \ + if (IEM_IS_MODRM_REG_MODE(bRm)) \ + { \ + /* \ + * Register, register. \ + */ \ + if (pVCpu->iem.s.uVexLength) \ + { \ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx2); \ + IEM_MC_BEGIN(2, 1); \ + IEM_MC_LOCAL(RTUINT256U, uDst); \ + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); \ + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); \ + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \ + IEM_MC_PREPARE_AVX_USAGE(); \ + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u256, \ + iemAImpl_ ## a_Instr ## _u256_fallback), \ + puDst, puSrc); \ + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx); \ + IEM_MC_BEGIN(2, 0); \ + IEM_MC_ARG(PRTUINT128U, puDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc, 1); \ + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \ + IEM_MC_PREPARE_AVX_USAGE(); \ + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); \ + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u128, \ + iemAImpl_## a_Instr ## _u128_fallback), \ + puDst, uSrc); \ + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + else \ + { \ + /* \ + * Register, memory. \ + */ \ + if (pVCpu->iem.s.uVexLength) \ + { \ + IEM_MC_BEGIN(2, 3); \ + IEM_MC_LOCAL(RTUINT256U, uDst); \ + IEM_MC_LOCAL(RTUINT128U, uSrc); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); \ + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx2); \ + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \ + IEM_MC_PREPARE_AVX_USAGE(); \ + IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u256, \ + iemAImpl_ ## a_Instr ## _u256_fallback), \ + puDst, puSrc); \ + IEM_MC_STORE_YREG_U256_ZX_VLMAX( IEM_GET_MODRM_REG(pVCpu, bRm), uDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(2, 1); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_ARG(PRTUINT128U, puDst, 0); \ + IEM_MC_ARG(uint ## a_SrcWidth ##_t, uSrc, 1); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx); \ + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \ + IEM_MC_PREPARE_AVX_USAGE(); \ + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_FETCH_MEM_U ## a_SrcWidth (uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u128, \ + iemAImpl_ ## a_Instr ## _u128_fallback), \ + puDst, uSrc); \ + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + (void)0 + +/** Opcode VEX.66.0F38 0x20. */ +FNIEMOP_DEF(iemOp_vpmovsxbw_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVSXBW, vpmovsxbw, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovsxbw, 64); +} + + +/** Opcode VEX.66.0F38 0x21. */ +FNIEMOP_DEF(iemOp_vpmovsxbd_Vx_UxMd) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVSXBD, vpmovsxbd, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovsxbd, 32); +} + + +/** Opcode VEX.66.0F38 0x22. */ +FNIEMOP_DEF(iemOp_vpmovsxbq_Vx_UxMw) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVSXBQ, vpmovsxbq, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovsxbq, 16); +} + + +/** Opcode VEX.66.0F38 0x23. */ +FNIEMOP_DEF(iemOp_vpmovsxwd_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVSXWD, vpmovsxwd, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovsxwd, 64); +} + + +/** Opcode VEX.66.0F38 0x24. */ +FNIEMOP_DEF(iemOp_vpmovsxwq_Vx_UxMd) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVSXWQ, vpmovsxwq, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovsxwq, 32); +} + + +/** Opcode VEX.66.0F38 0x25. */ +FNIEMOP_DEF(iemOp_vpmovsxdq_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVSXDQ, vpmovsxdq, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovsxdq, 64); +} + + +/* Opcode VEX.66.0F38 0x26 - invalid */ +/* Opcode VEX.66.0F38 0x27 - invalid */ + + +/** Opcode VEX.66.0F38 0x28. */ +FNIEMOP_DEF(iemOp_vpmuldq_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMULDQ, vpmuldq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpmuldq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x29. */ +FNIEMOP_DEF(iemOp_vpcmpeqq_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPCMPEQQ, vpcmpeqq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpcmpeqq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +FNIEMOP_DEF(iemOp_vmovntdqa_Vx_Mx) +{ + Assert(pVCpu->iem.s.uVexLength <= 1); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_MEM_MODE(bRm)) + { + if (pVCpu->iem.s.uVexLength == 0) + { + /** + * @opcode 0x2a + * @opcodesub !11 mr/reg vex.l=0 + * @oppfx 0x66 + * @opcpuid avx + * @opgroup og_avx_cachect + * @opxcpttype 1 + * @optest op1=-1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ + /* 128-bit: Memory, register. */ + IEMOP_MNEMONIC2EX(vmovntdqa_Vdq_WO_Mdq_L0, "vmovntdqa, Vdq_WO, Mdq", VEX_RM_MEM, VMOVNTDQA, vmovntdqa, Vx_WO, Mx, + DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /** + * @opdone + * @opcode 0x2a + * @opcodesub !11 mr/reg vex.l=1 + * @oppfx 0x66 + * @opcpuid avx2 + * @opgroup og_avx2_cachect + * @opxcpttype 1 + * @optest op1=-1 op2=2 -> op1=2 + * @optest op1=0 op2=-42 -> op1=-42 + */ + /* 256-bit: Memory, register. */ + IEMOP_MNEMONIC2EX(vmovntdqa_Vqq_WO_Mqq_L1, "vmovntdqa, Vqq_WO,Mqq", VEX_RM_MEM, VMOVNTDQA, vmovntdqa, Vx_WO, Mx, + DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES); + IEM_MC_BEGIN(0, 2); + IEM_MC_LOCAL(RTUINT256U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV(); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE(); + + IEM_MC_FETCH_MEM_U256_ALIGN_AVX(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + + /** + * @opdone + * @opmnemonic udvex660f382arg + * @opcode 0x2a + * @opcodesub 11 mr/reg + * @oppfx 0x66 + * @opunused immediate + * @opcpuid avx + * @optest -> + */ + else + return IEMOP_RAISE_INVALID_OPCODE(); +} + + +/** Opcode VEX.66.0F38 0x2b. */ +FNIEMOP_DEF(iemOp_vpackusdw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPACKUSDW, vpackusdw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0); + IEMOPMEDIAOPTF3_INIT_VARS( vpackusdw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x2c. */ +FNIEMOP_STUB(iemOp_vmaskmovps_Vx_Hx_Mx); +/** Opcode VEX.66.0F38 0x2d. */ +FNIEMOP_STUB(iemOp_vmaskmovpd_Vx_Hx_Mx); +/** Opcode VEX.66.0F38 0x2e. */ +FNIEMOP_STUB(iemOp_vmaskmovps_Mx_Hx_Vx); +/** Opcode VEX.66.0F38 0x2f. */ +FNIEMOP_STUB(iemOp_vmaskmovpd_Mx_Hx_Vx); + + +/** Opcode VEX.66.0F38 0x30. */ +FNIEMOP_DEF(iemOp_vpmovzxbw_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVZXBW, vpmovzxbw, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovzxbw, 64); +} + + +/** Opcode VEX.66.0F38 0x31. */ +FNIEMOP_DEF(iemOp_vpmovzxbd_Vx_UxMd) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVZXBD, vpmovzxbd, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovzxbd, 32); +} + + +/** Opcode VEX.66.0F38 0x32. */ +FNIEMOP_DEF(iemOp_vpmovzxbq_Vx_UxMw) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVZXBQ, vpmovzxbq, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovzxbq, 16); +} + + +/** Opcode VEX.66.0F38 0x33. */ +FNIEMOP_DEF(iemOp_vpmovzxwd_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVZXWD, vpmovzxwd, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovzxwd, 64); +} + + +/** Opcode VEX.66.0F38 0x34. */ +FNIEMOP_DEF(iemOp_vpmovzxwq_Vx_UxMd) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVZXWQ, vpmovzxwq, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovzxwq, 32); +} + + +/** Opcode VEX.66.0F38 0x35. */ +FNIEMOP_DEF(iemOp_vpmovzxdq_Vx_UxMq) +{ + /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */ + IEMOP_MNEMONIC2(VEX_RM, VPMOVZXDQ, vpmovzxdq, Vx, Wq, DISOPTYPE_HARMLESS, 0); + IEMOP_BODY_VPMOV_S_Z(vpmovzxdq, 64); +} + + +/* Opcode VEX.66.0F38 0x36. */ +FNIEMOP_STUB(iemOp_vpermd_Vqq_Hqq_Wqq); + + +/** Opcode VEX.66.0F38 0x37. */ +FNIEMOP_DEF(iemOp_vpcmpgtq_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPCMPGTQ, vpcmpgtq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpcmpgtq); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x38. */ +FNIEMOP_DEF(iemOp_vpminsb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMINSB, vpminsb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpminsb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x39. */ +FNIEMOP_DEF(iemOp_vpminsd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMINSD, vpminsd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpminsd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x3a. */ +FNIEMOP_DEF(iemOp_vpminuw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMINUW, vpminuw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpminuw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x3b. */ +FNIEMOP_DEF(iemOp_vpminud_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMINUD, vpminud, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpminud); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x3c. */ +FNIEMOP_DEF(iemOp_vpmaxsb_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMAXSB, vpmaxsb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpmaxsb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x3d. */ +FNIEMOP_DEF(iemOp_vpmaxsd_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMAXSD, vpmaxsd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpmaxsd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x3e. */ +FNIEMOP_DEF(iemOp_vpmaxuw_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMAXUW, vpmaxuw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpmaxuw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x3f. */ +FNIEMOP_DEF(iemOp_vpmaxud_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMAXUD, vpmaxud, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAF3_INIT_VARS(vpmaxud); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x40. */ +FNIEMOP_DEF(iemOp_vpmulld_Vx_Hx_Wx) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPMULLD, vpmulld, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); + IEMOPMEDIAOPTF3_INIT_VARS(vpmulld); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F38 0x41. */ +FNIEMOP_DEF(iemOp_vphminposuw_Vdq_Wdq) +{ + IEMOP_MNEMONIC2(VEX_RM, VPHMINPOSUW, vphminposuw, Vdq, Wdq, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx); + IEM_MC_BEGIN(2, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc, 1); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vphminposuw_u128, iemAImpl_vphminposuw_u128_fallback), + puDst, puSrc); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(2, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vphminposuw_u128, iemAImpl_vphminposuw_u128_fallback), + puDst, puSrc); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode VEX.66.0F38 0x42 - invalid. */ +/* Opcode VEX.66.0F38 0x43 - invalid. */ +/* Opcode VEX.66.0F38 0x44 - invalid. */ +/** Opcode VEX.66.0F38 0x45. */ +FNIEMOP_STUB(iemOp_vpsrlvd_q_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x46. */ +FNIEMOP_STUB(iemOp_vsravd_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x47. */ +FNIEMOP_STUB(iemOp_vpsllvd_q_Vx_Hx_Wx); +/* Opcode VEX.66.0F38 0x48 - invalid. */ +/* Opcode VEX.66.0F38 0x49 - invalid. */ +/* Opcode VEX.66.0F38 0x4a - invalid. */ +/* Opcode VEX.66.0F38 0x4b - invalid. */ +/* Opcode VEX.66.0F38 0x4c - invalid. */ +/* Opcode VEX.66.0F38 0x4d - invalid. */ +/* Opcode VEX.66.0F38 0x4e - invalid. */ +/* Opcode VEX.66.0F38 0x4f - invalid. */ + +/* Opcode VEX.66.0F38 0x50 - invalid. */ +/* Opcode VEX.66.0F38 0x51 - invalid. */ +/* Opcode VEX.66.0F38 0x52 - invalid. */ +/* Opcode VEX.66.0F38 0x53 - invalid. */ +/* Opcode VEX.66.0F38 0x54 - invalid. */ +/* Opcode VEX.66.0F38 0x55 - invalid. */ +/* Opcode VEX.66.0F38 0x56 - invalid. */ +/* Opcode VEX.66.0F38 0x57 - invalid. */ +/** Opcode VEX.66.0F38 0x58. */ +FNIEMOP_STUB(iemOp_vpbroadcastd_Vx_Wx); +/** Opcode VEX.66.0F38 0x59. */ +FNIEMOP_STUB(iemOp_vpbroadcastq_Vx_Wx); +/** Opcode VEX.66.0F38 0x5a. */ +FNIEMOP_STUB(iemOp_vbroadcasti128_Vqq_Mdq); +/* Opcode VEX.66.0F38 0x5b - invalid. */ +/* Opcode VEX.66.0F38 0x5c - invalid. */ +/* Opcode VEX.66.0F38 0x5d - invalid. */ +/* Opcode VEX.66.0F38 0x5e - invalid. */ +/* Opcode VEX.66.0F38 0x5f - invalid. */ + +/* Opcode VEX.66.0F38 0x60 - invalid. */ +/* Opcode VEX.66.0F38 0x61 - invalid. */ +/* Opcode VEX.66.0F38 0x62 - invalid. */ +/* Opcode VEX.66.0F38 0x63 - invalid. */ +/* Opcode VEX.66.0F38 0x64 - invalid. */ +/* Opcode VEX.66.0F38 0x65 - invalid. */ +/* Opcode VEX.66.0F38 0x66 - invalid. */ +/* Opcode VEX.66.0F38 0x67 - invalid. */ +/* Opcode VEX.66.0F38 0x68 - invalid. */ +/* Opcode VEX.66.0F38 0x69 - invalid. */ +/* Opcode VEX.66.0F38 0x6a - invalid. */ +/* Opcode VEX.66.0F38 0x6b - invalid. */ +/* Opcode VEX.66.0F38 0x6c - invalid. */ +/* Opcode VEX.66.0F38 0x6d - invalid. */ +/* Opcode VEX.66.0F38 0x6e - invalid. */ +/* Opcode VEX.66.0F38 0x6f - invalid. */ + +/* Opcode VEX.66.0F38 0x70 - invalid. */ +/* Opcode VEX.66.0F38 0x71 - invalid. */ +/* Opcode VEX.66.0F38 0x72 - invalid. */ +/* Opcode VEX.66.0F38 0x73 - invalid. */ +/* Opcode VEX.66.0F38 0x74 - invalid. */ +/* Opcode VEX.66.0F38 0x75 - invalid. */ +/* Opcode VEX.66.0F38 0x76 - invalid. */ +/* Opcode VEX.66.0F38 0x77 - invalid. */ +/** Opcode VEX.66.0F38 0x78. */ +FNIEMOP_STUB(iemOp_vpboardcastb_Vx_Wx); +/** Opcode VEX.66.0F38 0x79. */ +FNIEMOP_STUB(iemOp_vpboardcastw_Vx_Wx); +/* Opcode VEX.66.0F38 0x7a - invalid. */ +/* Opcode VEX.66.0F38 0x7b - invalid. */ +/* Opcode VEX.66.0F38 0x7c - invalid. */ +/* Opcode VEX.66.0F38 0x7d - invalid. */ +/* Opcode VEX.66.0F38 0x7e - invalid. */ +/* Opcode VEX.66.0F38 0x7f - invalid. */ + +/* Opcode VEX.66.0F38 0x80 - invalid (legacy only). */ +/* Opcode VEX.66.0F38 0x81 - invalid (legacy only). */ +/* Opcode VEX.66.0F38 0x82 - invalid (legacy only). */ +/* Opcode VEX.66.0F38 0x83 - invalid. */ +/* Opcode VEX.66.0F38 0x84 - invalid. */ +/* Opcode VEX.66.0F38 0x85 - invalid. */ +/* Opcode VEX.66.0F38 0x86 - invalid. */ +/* Opcode VEX.66.0F38 0x87 - invalid. */ +/* Opcode VEX.66.0F38 0x88 - invalid. */ +/* Opcode VEX.66.0F38 0x89 - invalid. */ +/* Opcode VEX.66.0F38 0x8a - invalid. */ +/* Opcode VEX.66.0F38 0x8b - invalid. */ +/** Opcode VEX.66.0F38 0x8c. */ +FNIEMOP_STUB(iemOp_vpmaskmovd_q_Vx_Hx_Mx); +/* Opcode VEX.66.0F38 0x8d - invalid. */ +/** Opcode VEX.66.0F38 0x8e. */ +FNIEMOP_STUB(iemOp_vpmaskmovd_q_Mx_Vx_Hx); +/* Opcode VEX.66.0F38 0x8f - invalid. */ + +/** Opcode VEX.66.0F38 0x90 (vex only). */ +FNIEMOP_STUB(iemOp_vgatherdd_q_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x91 (vex only). */ +FNIEMOP_STUB(iemOp_vgatherqd_q_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x92 (vex only). */ +FNIEMOP_STUB(iemOp_vgatherdps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x93 (vex only). */ +FNIEMOP_STUB(iemOp_vgatherqps_d_Vx_Hx_Wx); +/* Opcode VEX.66.0F38 0x94 - invalid. */ +/* Opcode VEX.66.0F38 0x95 - invalid. */ +/** Opcode VEX.66.0F38 0x96 (vex only). */ +FNIEMOP_STUB(iemOp_vfmaddsub132ps_q_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x97 (vex only). */ +FNIEMOP_STUB(iemOp_vfmsubadd132ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x98 (vex only). */ +FNIEMOP_STUB(iemOp_vfmadd132ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x99 (vex only). */ +FNIEMOP_STUB(iemOp_vfmadd132ss_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x9a (vex only). */ +FNIEMOP_STUB(iemOp_vfmsub132ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x9b (vex only). */ +FNIEMOP_STUB(iemOp_vfmsub132ss_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x9c (vex only). */ +FNIEMOP_STUB(iemOp_vfnmadd132ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x9d (vex only). */ +FNIEMOP_STUB(iemOp_vfnmadd132ss_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x9e (vex only). */ +FNIEMOP_STUB(iemOp_vfnmsub132ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0x9f (vex only). */ +FNIEMOP_STUB(iemOp_vfnmsub132ss_d_Vx_Hx_Wx); + +/* Opcode VEX.66.0F38 0xa0 - invalid. */ +/* Opcode VEX.66.0F38 0xa1 - invalid. */ +/* Opcode VEX.66.0F38 0xa2 - invalid. */ +/* Opcode VEX.66.0F38 0xa3 - invalid. */ +/* Opcode VEX.66.0F38 0xa4 - invalid. */ +/* Opcode VEX.66.0F38 0xa5 - invalid. */ +/** Opcode VEX.66.0F38 0xa6 (vex only). */ +FNIEMOP_STUB(iemOp_vfmaddsub213ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xa7 (vex only). */ +FNIEMOP_STUB(iemOp_vfmsubadd213ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xa8 (vex only). */ +FNIEMOP_STUB(iemOp_vfmadd213ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xa9 (vex only). */ +FNIEMOP_STUB(iemOp_vfmadd213ss_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xaa (vex only). */ +FNIEMOP_STUB(iemOp_vfmsub213ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xab (vex only). */ +FNIEMOP_STUB(iemOp_vfmsub213ss_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xac (vex only). */ +FNIEMOP_STUB(iemOp_vfnmadd213ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xad (vex only). */ +FNIEMOP_STUB(iemOp_vfnmadd213ss_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xae (vex only). */ +FNIEMOP_STUB(iemOp_vfnmsub213ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xaf (vex only). */ +FNIEMOP_STUB(iemOp_vfnmsub213ss_d_Vx_Hx_Wx); + +/* Opcode VEX.66.0F38 0xb0 - invalid. */ +/* Opcode VEX.66.0F38 0xb1 - invalid. */ +/* Opcode VEX.66.0F38 0xb2 - invalid. */ +/* Opcode VEX.66.0F38 0xb3 - invalid. */ +/* Opcode VEX.66.0F38 0xb4 - invalid. */ +/* Opcode VEX.66.0F38 0xb5 - invalid. */ +/** Opcode VEX.66.0F38 0xb6 (vex only). */ +FNIEMOP_STUB(iemOp_vfmaddsub231ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xb7 (vex only). */ +FNIEMOP_STUB(iemOp_vfmsubadd231ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xb8 (vex only). */ +FNIEMOP_STUB(iemOp_vfmadd231ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xb9 (vex only). */ +FNIEMOP_STUB(iemOp_vfmadd231ss_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xba (vex only). */ +FNIEMOP_STUB(iemOp_vfmsub231ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xbb (vex only). */ +FNIEMOP_STUB(iemOp_vfmsub231ss_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xbc (vex only). */ +FNIEMOP_STUB(iemOp_vfnmadd231ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xbd (vex only). */ +FNIEMOP_STUB(iemOp_vfnmadd231ss_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xbe (vex only). */ +FNIEMOP_STUB(iemOp_vfnmsub231ps_d_Vx_Hx_Wx); +/** Opcode VEX.66.0F38 0xbf (vex only). */ +FNIEMOP_STUB(iemOp_vfnmsub231ss_d_Vx_Hx_Wx); + +/* Opcode VEX.0F38 0xc0 - invalid. */ +/* Opcode VEX.66.0F38 0xc0 - invalid. */ +/* Opcode VEX.0F38 0xc1 - invalid. */ +/* Opcode VEX.66.0F38 0xc1 - invalid. */ +/* Opcode VEX.0F38 0xc2 - invalid. */ +/* Opcode VEX.66.0F38 0xc2 - invalid. */ +/* Opcode VEX.0F38 0xc3 - invalid. */ +/* Opcode VEX.66.0F38 0xc3 - invalid. */ +/* Opcode VEX.0F38 0xc4 - invalid. */ +/* Opcode VEX.66.0F38 0xc4 - invalid. */ +/* Opcode VEX.0F38 0xc5 - invalid. */ +/* Opcode VEX.66.0F38 0xc5 - invalid. */ +/* Opcode VEX.0F38 0xc6 - invalid. */ +/* Opcode VEX.66.0F38 0xc6 - invalid. */ +/* Opcode VEX.0F38 0xc7 - invalid. */ +/* Opcode VEX.66.0F38 0xc7 - invalid. */ +/** Opcode VEX.0F38 0xc8. */ +FNIEMOP_STUB(iemOp_vsha1nexte_Vdq_Wdq); +/* Opcode VEX.66.0F38 0xc8 - invalid. */ +/** Opcode VEX.0F38 0xc9. */ +FNIEMOP_STUB(iemOp_vsha1msg1_Vdq_Wdq); +/* Opcode VEX.66.0F38 0xc9 - invalid. */ +/** Opcode VEX.0F38 0xca. */ +FNIEMOP_STUB(iemOp_vsha1msg2_Vdq_Wdq); +/* Opcode VEX.66.0F38 0xca - invalid. */ +/** Opcode VEX.0F38 0xcb. */ +FNIEMOP_STUB(iemOp_vsha256rnds2_Vdq_Wdq); +/* Opcode VEX.66.0F38 0xcb - invalid. */ +/** Opcode VEX.0F38 0xcc. */ +FNIEMOP_STUB(iemOp_vsha256msg1_Vdq_Wdq); +/* Opcode VEX.66.0F38 0xcc - invalid. */ +/** Opcode VEX.0F38 0xcd. */ +FNIEMOP_STUB(iemOp_vsha256msg2_Vdq_Wdq); +/* Opcode VEX.66.0F38 0xcd - invalid. */ +/* Opcode VEX.0F38 0xce - invalid. */ +/* Opcode VEX.66.0F38 0xce - invalid. */ +/* Opcode VEX.0F38 0xcf - invalid. */ +/* Opcode VEX.66.0F38 0xcf - invalid. */ + +/* Opcode VEX.66.0F38 0xd0 - invalid. */ +/* Opcode VEX.66.0F38 0xd1 - invalid. */ +/* Opcode VEX.66.0F38 0xd2 - invalid. */ +/* Opcode VEX.66.0F38 0xd3 - invalid. */ +/* Opcode VEX.66.0F38 0xd4 - invalid. */ +/* Opcode VEX.66.0F38 0xd5 - invalid. */ +/* Opcode VEX.66.0F38 0xd6 - invalid. */ +/* Opcode VEX.66.0F38 0xd7 - invalid. */ +/* Opcode VEX.66.0F38 0xd8 - invalid. */ +/* Opcode VEX.66.0F38 0xd9 - invalid. */ +/* Opcode VEX.66.0F38 0xda - invalid. */ +/** Opcode VEX.66.0F38 0xdb. */ +FNIEMOP_STUB(iemOp_vaesimc_Vdq_Wdq); +/** Opcode VEX.66.0F38 0xdc. */ +FNIEMOP_STUB(iemOp_vaesenc_Vdq_Wdq); +/** Opcode VEX.66.0F38 0xdd. */ +FNIEMOP_STUB(iemOp_vaesenclast_Vdq_Wdq); +/** Opcode VEX.66.0F38 0xde. */ +FNIEMOP_STUB(iemOp_vaesdec_Vdq_Wdq); +/** Opcode VEX.66.0F38 0xdf. */ +FNIEMOP_STUB(iemOp_vaesdeclast_Vdq_Wdq); + +/* Opcode VEX.66.0F38 0xe0 - invalid. */ +/* Opcode VEX.66.0F38 0xe1 - invalid. */ +/* Opcode VEX.66.0F38 0xe2 - invalid. */ +/* Opcode VEX.66.0F38 0xe3 - invalid. */ +/* Opcode VEX.66.0F38 0xe4 - invalid. */ +/* Opcode VEX.66.0F38 0xe5 - invalid. */ +/* Opcode VEX.66.0F38 0xe6 - invalid. */ +/* Opcode VEX.66.0F38 0xe7 - invalid. */ +/* Opcode VEX.66.0F38 0xe8 - invalid. */ +/* Opcode VEX.66.0F38 0xe9 - invalid. */ +/* Opcode VEX.66.0F38 0xea - invalid. */ +/* Opcode VEX.66.0F38 0xeb - invalid. */ +/* Opcode VEX.66.0F38 0xec - invalid. */ +/* Opcode VEX.66.0F38 0xed - invalid. */ +/* Opcode VEX.66.0F38 0xee - invalid. */ +/* Opcode VEX.66.0F38 0xef - invalid. */ + + +/* Opcode VEX.0F38 0xf0 - invalid (legacy only). */ +/* Opcode VEX.66.0F38 0xf0 - invalid (legacy only). */ +/* Opcode VEX.F3.0F38 0xf0 - invalid. */ +/* Opcode VEX.F2.0F38 0xf0 - invalid (legacy only). */ + +/* Opcode VEX.0F38 0xf1 - invalid (legacy only). */ +/* Opcode VEX.66.0F38 0xf1 - invalid (legacy only). */ +/* Opcode VEX.F3.0F38 0xf1 - invalid. */ +/* Opcode VEX.F2.0F38 0xf1 - invalid (legacy only). */ + +/** Opcode VEX.0F38 0xf2 - ANDN (vex only). */ +FNIEMOP_DEF(iemOp_andn_Gy_By_Ey) +{ + IEMOP_MNEMONIC3(VEX_RVM, ANDN, andn, Gy, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi1) + return iemOp_InvalidNeedRM(pVCpu); + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_PF); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_L0(); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t, uSrc1, 1); + IEM_MC_ARG(uint64_t, uSrc2, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_andn_u64, iemAImpl_andn_u64_fallback), + pDst, uSrc1, uSrc2, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint32_t *, pDst, 0); + IEM_MC_ARG(uint32_t, uSrc1, 1); + IEM_MC_ARG(uint32_t, uSrc2, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_andn_u32, iemAImpl_andn_u32_fallback), + pDst, uSrc1, uSrc2, pEFlags); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + IEM_MC_BEGIN(4, 1); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t, uSrc1, 1); + IEM_MC_ARG(uint64_t, uSrc2, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_FETCH_MEM_U64(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_andn_u64, iemAImpl_andn_u64_fallback), + pDst, uSrc1, uSrc2, pEFlags); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(4, 1); + IEM_MC_ARG(uint32_t *, pDst, 0); + IEM_MC_ARG(uint32_t, uSrc1, 1); + IEM_MC_ARG(uint32_t, uSrc2, 2); + IEM_MC_ARG(uint32_t *, pEFlags, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_FETCH_MEM_U32(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_EFLAGS(pEFlags); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_andn_u32, iemAImpl_andn_u32_fallback), + pDst, uSrc1, uSrc2, pEFlags); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + +/* Opcode VEX.66.0F38 0xf2 - invalid. */ +/* Opcode VEX.F3.0F38 0xf2 - invalid. */ +/* Opcode VEX.F2.0F38 0xf2 - invalid. */ + + +/* Opcode VEX.0F38 0xf3 - invalid. */ +/* Opcode VEX.66.0F38 0xf3 - invalid. */ + +/* Opcode VEX.F3.0F38 0xf3 /0 - invalid. */ + +/** Body for the vex group 17 instructions. */ +#define IEMOP_BODY_By_Ey(a_Instr) \ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi1) \ + return iemOp_InvalidWithRM(pVCpu, bRm); /* decode memory variant? */ \ + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_PF); \ + if (IEM_IS_MODRM_REG_MODE(bRm)) \ + { \ + /* \ + * Register, register. \ + */ \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \ + { \ + IEM_MC_BEGIN(3, 0); \ + IEM_MC_ARG(uint64_t *, pDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc, 1); \ + IEM_MC_ARG(uint32_t *, pEFlags, 2); \ + IEM_MC_REF_GREG_U64(pDst, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_FETCH_GREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_ ## a_Instr ## _u64, \ + iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc, pEFlags); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(3, 0); \ + IEM_MC_ARG(uint32_t *, pDst, 0); \ + IEM_MC_ARG(uint32_t, uSrc, 1); \ + IEM_MC_ARG(uint32_t *, pEFlags, 2); \ + IEM_MC_REF_GREG_U32(pDst, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_FETCH_GREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_ ## a_Instr ## _u32, \ + iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc, pEFlags); \ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + else \ + { \ + /* \ + * Register, memory. \ + */ \ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \ + { \ + IEM_MC_BEGIN(3, 1); \ + IEM_MC_ARG(uint64_t *, pDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc, 1); \ + IEM_MC_ARG(uint32_t *, pEFlags, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_REF_GREG_U64(pDst, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_ ## a_Instr ## _u64, \ + iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc, pEFlags); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(3, 1); \ + IEM_MC_ARG(uint32_t *, pDst, 0); \ + IEM_MC_ARG(uint32_t, uSrc, 1); \ + IEM_MC_ARG(uint32_t *, pEFlags, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_REF_GREG_U32(pDst, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_ ## a_Instr ## _u32, \ + iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc, pEFlags); \ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + (void)0 + + +/* Opcode VEX.F3.0F38 0xf3 /1. */ +/** @opcode /1 + * @opmaps vexgrp17 */ +FNIEMOP_DEF_1(iemOp_VGrp17_blsr_By_Ey, uint8_t, bRm) +{ + IEMOP_MNEMONIC2(VEX_VM, BLSR, blsr, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_By_Ey(blsr); +} + + +/* Opcode VEX.F3.0F38 0xf3 /2. */ +/** @opcode /2 + * @opmaps vexgrp17 */ +FNIEMOP_DEF_1(iemOp_VGrp17_blsmsk_By_Ey, uint8_t, bRm) +{ + IEMOP_MNEMONIC2(VEX_VM, BLSMSK, blsmsk, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_By_Ey(blsmsk); +} + + +/* Opcode VEX.F3.0F38 0xf3 /3. */ +/** @opcode /3 + * @opmaps vexgrp17 */ +FNIEMOP_DEF_1(iemOp_VGrp17_blsi_By_Ey, uint8_t, bRm) +{ + IEMOP_MNEMONIC2(VEX_VM, BLSI, blsi, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_By_Ey(blsi); +} + + +/* Opcode VEX.F3.0F38 0xf3 /4 - invalid. */ +/* Opcode VEX.F3.0F38 0xf3 /5 - invalid. */ +/* Opcode VEX.F3.0F38 0xf3 /6 - invalid. */ +/* Opcode VEX.F3.0F38 0xf3 /7 - invalid. */ + +/** + * Group 17 jump table for the VEX.F3 variant. + */ +IEM_STATIC const PFNIEMOPRM g_apfnVexGroup17_f3[] = +{ + /* /0 */ iemOp_InvalidWithRM, + /* /1 */ iemOp_VGrp17_blsr_By_Ey, + /* /2 */ iemOp_VGrp17_blsmsk_By_Ey, + /* /3 */ iemOp_VGrp17_blsi_By_Ey, + /* /4 */ iemOp_InvalidWithRM, + /* /5 */ iemOp_InvalidWithRM, + /* /6 */ iemOp_InvalidWithRM, + /* /7 */ iemOp_InvalidWithRM +}; +AssertCompile(RT_ELEMENTS(g_apfnVexGroup17_f3) == 8); + +/** Opcode VEX.F3.0F38 0xf3 - invalid (vex only - group 17). */ +FNIEMOP_DEF(iemOp_VGrp17_f3) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + return FNIEMOP_CALL_1(g_apfnVexGroup17_f3[IEM_GET_MODRM_REG_8(bRm)], bRm); +} + +/* Opcode VEX.F2.0F38 0xf3 - invalid (vex only - group 17). */ + + +/* Opcode VEX.0F38 0xf4 - invalid. */ +/* Opcode VEX.66.0F38 0xf4 - invalid. */ +/* Opcode VEX.F3.0F38 0xf4 - invalid. */ +/* Opcode VEX.F2.0F38 0xf4 - invalid. */ + +/** Body for BZHI, BEXTR, ++; assumes VEX.L must be 0. */ +#define IEMOP_BODY_Gy_Ey_By(a_Instr, a_fFeatureMember, a_fUndefFlags) \ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->a_fFeatureMember) \ + return iemOp_InvalidNeedRM(pVCpu); \ + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(a_fUndefFlags); \ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \ + if (IEM_IS_MODRM_REG_MODE(bRm)) \ + { \ + /* \ + * Register, register. \ + */ \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \ + { \ + IEM_MC_BEGIN(4, 0); \ + IEM_MC_ARG(uint64_t *, pDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc1, 1); \ + IEM_MC_ARG(uint64_t, uSrc2, 2); \ + IEM_MC_ARG(uint32_t *, pEFlags, 3); \ + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u64, \ + iemAImpl_ ## a_Instr ## _u64_fallback), \ + pDst, uSrc1, uSrc2, pEFlags); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(4, 0); \ + IEM_MC_ARG(uint32_t *, pDst, 0); \ + IEM_MC_ARG(uint32_t, uSrc1, 1); \ + IEM_MC_ARG(uint32_t, uSrc2, 2); \ + IEM_MC_ARG(uint32_t *, pEFlags, 3); \ + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u32, \ + iemAImpl_ ## a_Instr ## _u32_fallback), \ + pDst, uSrc1, uSrc2, pEFlags); \ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + else \ + { \ + /* \ + * Register, memory. \ + */ \ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \ + { \ + IEM_MC_BEGIN(4, 1); \ + IEM_MC_ARG(uint64_t *, pDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc1, 1); \ + IEM_MC_ARG(uint64_t, uSrc2, 2); \ + IEM_MC_ARG(uint32_t *, pEFlags, 3); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + IEM_MC_FETCH_MEM_U64(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u64, \ + iemAImpl_ ## a_Instr ## _u64_fallback), \ + pDst, uSrc1, uSrc2, pEFlags); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(4, 1); \ + IEM_MC_ARG(uint32_t *, pDst, 0); \ + IEM_MC_ARG(uint32_t, uSrc1, 1); \ + IEM_MC_ARG(uint32_t, uSrc2, 2); \ + IEM_MC_ARG(uint32_t *, pEFlags, 3); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + IEM_MC_FETCH_MEM_U32(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_REF_EFLAGS(pEFlags); \ + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u32, \ + iemAImpl_ ## a_Instr ## _u32_fallback), \ + pDst, uSrc1, uSrc2, pEFlags); \ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + (void)0 + +/** Body for SARX, SHLX, SHRX; assumes VEX.L must be 0. */ +#define IEMOP_BODY_Gy_Ey_By_NoEflags(a_Instr, a_fFeatureMember, a_fUndefFlags) \ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->a_fFeatureMember) \ + return iemOp_InvalidNeedRM(pVCpu); \ + IEMOP_VERIFICATION_UNDEFINED_EFLAGS(a_fUndefFlags); \ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \ + if (IEM_IS_MODRM_REG_MODE(bRm)) \ + { \ + /* \ + * Register, register. \ + */ \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \ + { \ + IEM_MC_BEGIN(3, 0); \ + IEM_MC_ARG(uint64_t *, pDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc1, 1); \ + IEM_MC_ARG(uint64_t, uSrc2, 2); \ + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u64, \ + iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc1, uSrc2); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(3, 0); \ + IEM_MC_ARG(uint32_t *, pDst, 0); \ + IEM_MC_ARG(uint32_t, uSrc1, 1); \ + IEM_MC_ARG(uint32_t, uSrc2, 2); \ + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u32, \ + iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc1, uSrc2); \ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + else \ + { \ + /* \ + * Register, memory. \ + */ \ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \ + { \ + IEM_MC_BEGIN(3, 1); \ + IEM_MC_ARG(uint64_t *, pDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc1, 1); \ + IEM_MC_ARG(uint64_t, uSrc2, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + IEM_MC_FETCH_MEM_U64(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u64, \ + iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc1, uSrc2); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(3, 1); \ + IEM_MC_ARG(uint32_t *, pDst, 0); \ + IEM_MC_ARG(uint32_t, uSrc1, 1); \ + IEM_MC_ARG(uint32_t, uSrc2, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + IEM_MC_FETCH_MEM_U32(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u32, \ + iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc1, uSrc2); \ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + (void)0 + +/** Opcode VEX.0F38 0xf5 (vex only). */ +FNIEMOP_DEF(iemOp_bzhi_Gy_Ey_By) +{ + IEMOP_MNEMONIC3(VEX_RMV, BZHI, bzhi, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_Gy_Ey_By(bzhi, fBmi2, X86_EFL_AF | X86_EFL_PF); +} + +/* Opcode VEX.66.0F38 0xf5 - invalid. */ + +/** Body for PDEP and PEXT (similar to ANDN, except no EFLAGS). */ +#define IEMOP_BODY_Gy_By_Ey_NoEflags(a_Instr, a_fFeatureMember) \ + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->a_fFeatureMember) \ + return iemOp_InvalidNeedRM(pVCpu); \ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \ + if (IEM_IS_MODRM_REG_MODE(bRm)) \ + { \ + /* \ + * Register, register. \ + */ \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \ + { \ + IEM_MC_BEGIN(3, 0); \ + IEM_MC_ARG(uint64_t *, pDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc1, 1); \ + IEM_MC_ARG(uint64_t, uSrc2, 2); \ + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, \ + iemAImpl_ ## a_Instr ## _u64, \ + iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc1, uSrc2); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(3, 0); \ + IEM_MC_ARG(uint32_t *, pDst, 0); \ + IEM_MC_ARG(uint32_t, uSrc1, 1); \ + IEM_MC_ARG(uint32_t, uSrc2, 2); \ + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, \ + iemAImpl_ ## a_Instr ## _u32, \ + iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc1, uSrc2); \ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + else \ + { \ + /* \ + * Register, memory. \ + */ \ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \ + { \ + IEM_MC_BEGIN(3, 1); \ + IEM_MC_ARG(uint64_t *, pDst, 0); \ + IEM_MC_ARG(uint64_t, uSrc1, 1); \ + IEM_MC_ARG(uint64_t, uSrc2, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + IEM_MC_FETCH_MEM_U64(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, \ + iemAImpl_ ## a_Instr ## _u64, \ + iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc1, uSrc2); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + else \ + { \ + IEM_MC_BEGIN(3, 1); \ + IEM_MC_ARG(uint32_t *, pDst, 0); \ + IEM_MC_ARG(uint32_t, uSrc1, 1); \ + IEM_MC_ARG(uint32_t, uSrc2, 2); \ + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); \ + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \ + IEMOP_HLP_DONE_VEX_DECODING_L0(); \ + IEM_MC_FETCH_MEM_U32(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \ + IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \ + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \ + IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, \ + iemAImpl_ ## a_Instr ## _u32, \ + iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc1, uSrc2); \ + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \ + IEM_MC_ADVANCE_RIP_AND_FINISH(); \ + IEM_MC_END(); \ + } \ + } \ + (void)0 + + +/** Opcode VEX.F3.0F38 0xf5 (vex only). */ +FNIEMOP_DEF(iemOp_pext_Gy_By_Ey) +{ + IEMOP_MNEMONIC3(VEX_RVM, PEXT, pext, Gy, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_Gy_By_Ey_NoEflags(pext, fBmi2); +} + + +/** Opcode VEX.F2.0F38 0xf5 (vex only). */ +FNIEMOP_DEF(iemOp_pdep_Gy_By_Ey) +{ + IEMOP_MNEMONIC3(VEX_RVM, PDEP, pdep, Gy, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_Gy_By_Ey_NoEflags(pdep, fBmi2); +} + + +/* Opcode VEX.0F38 0xf6 - invalid. */ +/* Opcode VEX.66.0F38 0xf6 - invalid (legacy only). */ +/* Opcode VEX.F3.0F38 0xf6 - invalid (legacy only). */ + + +/** Opcode VEX.F2.0F38 0xf6 (vex only) */ +FNIEMOP_DEF(iemOp_mulx_By_Gy_rDX_Ey) +{ + IEMOP_MNEMONIC4(VEX_RVM, MULX, mulx, Gy, By, Ey, rDX, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi2) + return iemOp_InvalidNeedRM(pVCpu); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + IEMOP_HLP_DONE_VEX_DECODING_L0(); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint64_t *, pDst1, 0); + IEM_MC_ARG(uint64_t *, pDst2, 1); + IEM_MC_ARG(uint64_t, uSrc1, 2); + IEM_MC_ARG(uint64_t, uSrc2, 3); + IEM_MC_REF_GREG_U64(pDst1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U64(pDst2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_GREG_U64(uSrc1, X86_GREG_xDX); + IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi2, iemAImpl_mulx_u64, iemAImpl_mulx_u64_fallback), + pDst1, pDst2, uSrc1, uSrc2); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(uint32_t *, pDst1, 0); + IEM_MC_ARG(uint32_t *, pDst2, 1); + IEM_MC_ARG(uint32_t, uSrc1, 2); + IEM_MC_ARG(uint32_t, uSrc2, 3); + IEM_MC_REF_GREG_U32(pDst1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_GREG_U32(pDst2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_GREG_U32(uSrc1, X86_GREG_xDX); + IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi2, iemAImpl_mulx_u32, iemAImpl_mulx_u32_fallback), + pDst1, pDst2, uSrc1, uSrc2); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst2); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst1); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + IEM_MC_BEGIN(4, 1); + IEM_MC_ARG(uint64_t *, pDst1, 0); + IEM_MC_ARG(uint64_t *, pDst2, 1); + IEM_MC_ARG(uint64_t, uSrc1, 2); + IEM_MC_ARG(uint64_t, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_FETCH_MEM_U64(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_GREG_U64(uSrc1, X86_GREG_xDX); + IEM_MC_REF_GREG_U64(pDst2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_GREG_U64(pDst1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi2, iemAImpl_mulx_u64, iemAImpl_mulx_u64_fallback), + pDst1, pDst2, uSrc1, uSrc2); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(4, 1); + IEM_MC_ARG(uint32_t *, pDst1, 0); + IEM_MC_ARG(uint32_t *, pDst2, 1); + IEM_MC_ARG(uint32_t, uSrc1, 2); + IEM_MC_ARG(uint32_t, uSrc2, 3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + IEMOP_HLP_DONE_VEX_DECODING_L0(); + IEM_MC_FETCH_MEM_U32(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_GREG_U32(uSrc1, X86_GREG_xDX); + IEM_MC_REF_GREG_U32(pDst2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_GREG_U32(pDst1, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi2, iemAImpl_mulx_u32, iemAImpl_mulx_u32_fallback), + pDst1, pDst2, uSrc1, uSrc2); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst2); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst1); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode VEX.0F38 0xf7 (vex only). */ +FNIEMOP_DEF(iemOp_bextr_Gy_Ey_By) +{ + IEMOP_MNEMONIC3(VEX_RMV, BEXTR, bextr, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_Gy_Ey_By(bextr, fBmi1, X86_EFL_SF | X86_EFL_AF | X86_EFL_PF); +} + + +/** Opcode VEX.66.0F38 0xf7 (vex only). */ +FNIEMOP_DEF(iemOp_shlx_Gy_Ey_By) +{ + IEMOP_MNEMONIC3(VEX_RMV, SHLX, shlx, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_Gy_Ey_By_NoEflags(shlx, fBmi2, 0); +} + + +/** Opcode VEX.F3.0F38 0xf7 (vex only). */ +FNIEMOP_DEF(iemOp_sarx_Gy_Ey_By) +{ + IEMOP_MNEMONIC3(VEX_RMV, SARX, sarx, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_Gy_Ey_By_NoEflags(sarx, fBmi2, 0); +} + + +/** Opcode VEX.F2.0F38 0xf7 (vex only). */ +FNIEMOP_DEF(iemOp_shrx_Gy_Ey_By) +{ + IEMOP_MNEMONIC3(VEX_RMV, SHRX, shrx, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO); + IEMOP_BODY_Gy_Ey_By_NoEflags(shrx, fBmi2, 0); +} + +/* Opcode VEX.0F38 0xf8 - invalid. */ +/* Opcode VEX.66.0F38 0xf8 - invalid. */ +/* Opcode VEX.F3.0F38 0xf8 - invalid. */ +/* Opcode VEX.F2.0F38 0xf8 - invalid. */ + +/* Opcode VEX.0F38 0xf9 - invalid. */ +/* Opcode VEX.66.0F38 0xf9 - invalid. */ +/* Opcode VEX.F3.0F38 0xf9 - invalid. */ +/* Opcode VEX.F2.0F38 0xf9 - invalid. */ + +/* Opcode VEX.0F38 0xfa - invalid. */ +/* Opcode VEX.66.0F38 0xfa - invalid. */ +/* Opcode VEX.F3.0F38 0xfa - invalid. */ +/* Opcode VEX.F2.0F38 0xfa - invalid. */ + +/* Opcode VEX.0F38 0xfb - invalid. */ +/* Opcode VEX.66.0F38 0xfb - invalid. */ +/* Opcode VEX.F3.0F38 0xfb - invalid. */ +/* Opcode VEX.F2.0F38 0xfb - invalid. */ + +/* Opcode VEX.0F38 0xfc - invalid. */ +/* Opcode VEX.66.0F38 0xfc - invalid. */ +/* Opcode VEX.F3.0F38 0xfc - invalid. */ +/* Opcode VEX.F2.0F38 0xfc - invalid. */ + +/* Opcode VEX.0F38 0xfd - invalid. */ +/* Opcode VEX.66.0F38 0xfd - invalid. */ +/* Opcode VEX.F3.0F38 0xfd - invalid. */ +/* Opcode VEX.F2.0F38 0xfd - invalid. */ + +/* Opcode VEX.0F38 0xfe - invalid. */ +/* Opcode VEX.66.0F38 0xfe - invalid. */ +/* Opcode VEX.F3.0F38 0xfe - invalid. */ +/* Opcode VEX.F2.0F38 0xfe - invalid. */ + +/* Opcode VEX.0F38 0xff - invalid. */ +/* Opcode VEX.66.0F38 0xff - invalid. */ +/* Opcode VEX.F3.0F38 0xff - invalid. */ +/* Opcode VEX.F2.0F38 0xff - invalid. */ + + +/** + * VEX opcode map \#2. + * + * @sa g_apfnThreeByte0f38 + */ +IEM_STATIC const PFNIEMOP g_apfnVexMap2[] = +{ + /* no prefix, 066h prefix f3h prefix, f2h prefix */ + /* 0x00 */ iemOp_InvalidNeedRM, iemOp_vpshufb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x01 */ iemOp_InvalidNeedRM, iemOp_vphaddw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x02 */ iemOp_InvalidNeedRM, iemOp_vphaddd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x03 */ iemOp_InvalidNeedRM, iemOp_vphaddsw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x04 */ iemOp_InvalidNeedRM, iemOp_vpmaddubsw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x05 */ iemOp_InvalidNeedRM, iemOp_vphsubw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x06 */ iemOp_InvalidNeedRM, iemOp_vphsubd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x07 */ iemOp_InvalidNeedRM, iemOp_vphsubsw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x08 */ iemOp_InvalidNeedRM, iemOp_vpsignb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x09 */ iemOp_InvalidNeedRM, iemOp_vpsignw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x0a */ iemOp_InvalidNeedRM, iemOp_vpsignd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x0b */ iemOp_InvalidNeedRM, iemOp_vpmulhrsw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x0c */ iemOp_InvalidNeedRM, iemOp_vpermilps_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x0d */ iemOp_InvalidNeedRM, iemOp_vpermilpd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x0e */ iemOp_InvalidNeedRM, iemOp_vtestps_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x0f */ iemOp_InvalidNeedRM, iemOp_vtestpd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0x10 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x11 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x12 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x13 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x14 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x15 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x16 */ iemOp_InvalidNeedRM, iemOp_vpermps_Vqq_Hqq_Wqq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x17 */ iemOp_InvalidNeedRM, iemOp_vptest_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x18 */ iemOp_InvalidNeedRM, iemOp_vbroadcastss_Vx_Wd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x19 */ iemOp_InvalidNeedRM, iemOp_vbroadcastsd_Vqq_Wq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x1a */ iemOp_InvalidNeedRM, iemOp_vbroadcastf128_Vqq_Mdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x1b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x1c */ iemOp_InvalidNeedRM, iemOp_vpabsb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x1d */ iemOp_InvalidNeedRM, iemOp_vpabsw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x1e */ iemOp_InvalidNeedRM, iemOp_vpabsd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x1f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x20 */ iemOp_InvalidNeedRM, iemOp_vpmovsxbw_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x21 */ iemOp_InvalidNeedRM, iemOp_vpmovsxbd_Vx_UxMd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x22 */ iemOp_InvalidNeedRM, iemOp_vpmovsxbq_Vx_UxMw, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x23 */ iemOp_InvalidNeedRM, iemOp_vpmovsxwd_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x24 */ iemOp_InvalidNeedRM, iemOp_vpmovsxwq_Vx_UxMd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x25 */ iemOp_InvalidNeedRM, iemOp_vpmovsxdq_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x26 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x27 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x28 */ iemOp_InvalidNeedRM, iemOp_vpmuldq_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x29 */ iemOp_InvalidNeedRM, iemOp_vpcmpeqq_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2a */ iemOp_InvalidNeedRM, iemOp_vmovntdqa_Vx_Mx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2b */ iemOp_InvalidNeedRM, iemOp_vpackusdw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2c */ iemOp_InvalidNeedRM, iemOp_vmaskmovps_Vx_Hx_Mx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2d */ iemOp_InvalidNeedRM, iemOp_vmaskmovpd_Vx_Hx_Mx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2e */ iemOp_InvalidNeedRM, iemOp_vmaskmovps_Mx_Hx_Vx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x2f */ iemOp_InvalidNeedRM, iemOp_vmaskmovpd_Mx_Hx_Vx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0x30 */ iemOp_InvalidNeedRM, iemOp_vpmovzxbw_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x31 */ iemOp_InvalidNeedRM, iemOp_vpmovzxbd_Vx_UxMd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x32 */ iemOp_InvalidNeedRM, iemOp_vpmovzxbq_Vx_UxMw, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x33 */ iemOp_InvalidNeedRM, iemOp_vpmovzxwd_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x34 */ iemOp_InvalidNeedRM, iemOp_vpmovzxwq_Vx_UxMd, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x35 */ iemOp_InvalidNeedRM, iemOp_vpmovzxdq_Vx_UxMq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x36 */ iemOp_InvalidNeedRM, iemOp_vpermd_Vqq_Hqq_Wqq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x37 */ iemOp_InvalidNeedRM, iemOp_vpcmpgtq_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x38 */ iemOp_InvalidNeedRM, iemOp_vpminsb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x39 */ iemOp_InvalidNeedRM, iemOp_vpminsd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3a */ iemOp_InvalidNeedRM, iemOp_vpminuw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3b */ iemOp_InvalidNeedRM, iemOp_vpminud_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3c */ iemOp_InvalidNeedRM, iemOp_vpmaxsb_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3d */ iemOp_InvalidNeedRM, iemOp_vpmaxsd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3e */ iemOp_InvalidNeedRM, iemOp_vpmaxuw_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x3f */ iemOp_InvalidNeedRM, iemOp_vpmaxud_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0x40 */ iemOp_InvalidNeedRM, iemOp_vpmulld_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x41 */ iemOp_InvalidNeedRM, iemOp_vphminposuw_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x42 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x43 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x44 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x45 */ iemOp_InvalidNeedRM, iemOp_vpsrlvd_q_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x46 */ iemOp_InvalidNeedRM, iemOp_vsravd_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x47 */ iemOp_InvalidNeedRM, iemOp_vpsllvd_q_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x48 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x49 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x4f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x50 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x51 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x52 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x53 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x54 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x55 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x56 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x57 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x58 */ iemOp_InvalidNeedRM, iemOp_vpbroadcastd_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x59 */ iemOp_InvalidNeedRM, iemOp_vpbroadcastq_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x5a */ iemOp_InvalidNeedRM, iemOp_vbroadcasti128_Vqq_Mdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x5b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x5f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x60 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x61 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x62 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x63 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x64 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x65 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x66 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x67 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x68 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x69 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x6f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x70 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x71 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x72 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x73 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x74 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x75 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x76 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x77 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x78 */ iemOp_InvalidNeedRM, iemOp_vpboardcastb_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x79 */ iemOp_InvalidNeedRM, iemOp_vpboardcastw_Vx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x7a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7c */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7e */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x7f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x80 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x81 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x82 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x83 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x84 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x85 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x86 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x87 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x88 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x89 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8a */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8b */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8c */ iemOp_InvalidNeedRM, iemOp_vpmaskmovd_q_Vx_Hx_Mx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x8d */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x8e */ iemOp_InvalidNeedRM, iemOp_vpmaskmovd_q_Mx_Vx_Hx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x8f */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0x90 */ iemOp_InvalidNeedRM, iemOp_vgatherdd_q_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x91 */ iemOp_InvalidNeedRM, iemOp_vgatherqd_q_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x92 */ iemOp_InvalidNeedRM, iemOp_vgatherdps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x93 */ iemOp_InvalidNeedRM, iemOp_vgatherqps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x94 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x95 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0x96 */ iemOp_InvalidNeedRM, iemOp_vfmaddsub132ps_q_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x97 */ iemOp_InvalidNeedRM, iemOp_vfmsubadd132ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x98 */ iemOp_InvalidNeedRM, iemOp_vfmadd132ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x99 */ iemOp_InvalidNeedRM, iemOp_vfmadd132ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x9a */ iemOp_InvalidNeedRM, iemOp_vfmsub132ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x9b */ iemOp_InvalidNeedRM, iemOp_vfmsub132ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x9c */ iemOp_InvalidNeedRM, iemOp_vfnmadd132ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x9d */ iemOp_InvalidNeedRM, iemOp_vfnmadd132ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x9e */ iemOp_InvalidNeedRM, iemOp_vfnmsub132ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0x9f */ iemOp_InvalidNeedRM, iemOp_vfnmsub132ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0xa0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xa6 */ iemOp_InvalidNeedRM, iemOp_vfmaddsub213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xa7 */ iemOp_InvalidNeedRM, iemOp_vfmsubadd213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xa8 */ iemOp_InvalidNeedRM, iemOp_vfmadd213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xa9 */ iemOp_InvalidNeedRM, iemOp_vfmadd213ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xaa */ iemOp_InvalidNeedRM, iemOp_vfmsub213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xab */ iemOp_InvalidNeedRM, iemOp_vfmsub213ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xac */ iemOp_InvalidNeedRM, iemOp_vfnmadd213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xad */ iemOp_InvalidNeedRM, iemOp_vfnmadd213ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xae */ iemOp_InvalidNeedRM, iemOp_vfnmsub213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xaf */ iemOp_InvalidNeedRM, iemOp_vfnmsub213ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0xb0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xb6 */ iemOp_InvalidNeedRM, iemOp_vfmaddsub231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xb7 */ iemOp_InvalidNeedRM, iemOp_vfmsubadd231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xb8 */ iemOp_InvalidNeedRM, iemOp_vfmadd231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xb9 */ iemOp_InvalidNeedRM, iemOp_vfmadd231ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xba */ iemOp_InvalidNeedRM, iemOp_vfmsub231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xbb */ iemOp_InvalidNeedRM, iemOp_vfmsub231ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xbc */ iemOp_InvalidNeedRM, iemOp_vfnmadd231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xbd */ iemOp_InvalidNeedRM, iemOp_vfnmadd231ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xbe */ iemOp_InvalidNeedRM, iemOp_vfnmsub231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xbf */ iemOp_InvalidNeedRM, iemOp_vfnmsub231ss_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0xc0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xc8 */ iemOp_vsha1nexte_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xc9 */ iemOp_vsha1msg1_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xca */ iemOp_vsha1msg2_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xcb */ iemOp_vsha256rnds2_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xcc */ iemOp_vsha256msg1_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xcd */ iemOp_vsha256msg2_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xce */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xcf */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xd0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xd9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xda */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xdb */ iemOp_InvalidNeedRM, iemOp_vaesimc_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdc */ iemOp_InvalidNeedRM, iemOp_vaesenc_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdd */ iemOp_InvalidNeedRM, iemOp_vaesenclast_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xde */ iemOp_InvalidNeedRM, iemOp_vaesdec_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xdf */ iemOp_InvalidNeedRM, iemOp_vaesdeclast_Vdq_Wdq, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + + /* 0xe0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe2 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe3 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe5 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe6 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe7 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xe9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xea */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xeb */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xec */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xed */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xee */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xef */ IEMOP_X4(iemOp_InvalidNeedRM), + + /* 0xf0 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf1 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf2 */ iemOp_andn_Gy_By_Ey, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf3 */ iemOp_VGrp17_f3, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, + /* 0xf4 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf5 */ iemOp_bzhi_Gy_Ey_By, iemOp_InvalidNeedRM, iemOp_pext_Gy_By_Ey, iemOp_pdep_Gy_By_Ey, + /* 0xf6 */ iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM, iemOp_mulx_By_Gy_rDX_Ey, + /* 0xf7 */ iemOp_bextr_Gy_Ey_By, iemOp_shlx_Gy_Ey_By, iemOp_sarx_Gy_Ey_By, iemOp_shrx_Gy_Ey_By, + /* 0xf8 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xf9 */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfa */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfb */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfc */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfd */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xfe */ IEMOP_X4(iemOp_InvalidNeedRM), + /* 0xff */ IEMOP_X4(iemOp_InvalidNeedRM), +}; +AssertCompile(RT_ELEMENTS(g_apfnVexMap2) == 1024); + +/** @} */ + diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h new file mode 100644 index 00000000..6b6e5af4 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h @@ -0,0 +1,1004 @@ +/* $Id: IEMAllInstructionsVexMap3.cpp.h $ */ +/** @file + * IEM - Instruction Decoding and Emulation, 0x0f 0x3a map. + * + * @remarks IEMAllInstructionsThree0f3a.cpp.h is a VEX mirror of this file. + * Any update here is likely needed in that file too. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/** @name VEX Opcode Map 3 + * @{ + */ + +/** + * Common worker for AVX2 instructions on the forms: + * - vpxxx xmm0, xmm1, xmm2/mem128, imm8 + * - vpxxx ymm0, ymm1, ymm2/mem256, imm8 + * + * Takes function table for function w/o implicit state parameter. + * + * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit. + */ +FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, PCIEMOPMEDIAOPTF3IMM8, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + if (pVCpu->iem.s.uVexLength) + { + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_BEGIN(4, 3); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_YREG_U256(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU256, puDst, puSrc1, puSrc2, bImmArg); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); + IEM_MC_ARG(PCRTUINT128U, puSrc2, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU128, puDst, puSrc1, puSrc2, bImmArg); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength) + { + IEM_MC_BEGIN(4, 4); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU256, puDst, puSrc1, puSrc2, bImmArg); + IEM_MC_STORE_YREG_U256_ZX_VLMAX( IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc2, uSrc2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU128, puDst, puSrc1, puSrc2, bImmArg); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode VEX.66.0F3A 0x00. */ +FNIEMOP_STUB(iemOp_vpermq_Vqq_Wqq_Ib); +/** Opcode VEX.66.0F3A 0x01. */ +FNIEMOP_STUB(iemOp_vpermqd_Vqq_Wqq_Ib); +/** Opcode VEX.66.0F3A 0x02. */ +FNIEMOP_STUB(iemOp_vpblendd_Vx_Wx_Ib); +/* Opcode VEX.66.0F3A 0x03 - invalid */ +/** Opcode VEX.66.0F3A 0x04. */ +FNIEMOP_STUB(iemOp_vpermilps_Vx_Wx_Ib); +/** Opcode VEX.66.0F3A 0x05. */ +FNIEMOP_STUB(iemOp_vpermilpd_Vx_Wx_Ib); +/** Opcode VEX.66.0F3A 0x06 (vex only) */ +FNIEMOP_STUB(iemOp_vperm2f128_Vqq_Hqq_Wqq_Ib); +/* Opcode VEX.66.0F3A 0x07 - invalid */ +/** Opcode VEX.66.0F3A 0x08. */ +FNIEMOP_STUB(iemOp_vroundps_Vx_Wx_Ib); +/** Opcode VEX.66.0F3A 0x09. */ +FNIEMOP_STUB(iemOp_vroundpd_Vx_Wx_Ib); +/** Opcode VEX.66.0F3A 0x0a. */ +FNIEMOP_STUB(iemOp_vroundss_Vss_Wss_Ib); +/** Opcode VEX.66.0F3A 0x0b. */ +FNIEMOP_STUB(iemOp_vroundsd_Vsd_Wsd_Ib); + + +/** Opcode VEX.66.0F3A 0x0c. */ +FNIEMOP_DEF(iemOp_vblendps_Vx_Hx_Wx_Ib) +{ + IEMOP_MNEMONIC3(VEX_RVM, VBLENDPS, vblendps, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); /* @todo */ + IEMOPMEDIAOPTF3IMM8_INIT_VARS(vblendps); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F3A 0x0d. */ +FNIEMOP_DEF(iemOp_vblendpd_Vx_Hx_Wx_Ib) +{ + IEMOP_MNEMONIC3(VEX_RVM, VBLENDPD, vblendpd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); /* @todo */ + IEMOPMEDIAOPTF3IMM8_INIT_VARS(vblendpd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F3A 0x0e. */ +FNIEMOP_DEF(iemOp_vpblendw_Vx_Hx_Wx_Ib) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPBLENDW, vpblendw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); /* @todo */ + IEMOPMEDIAOPTF3IMM8_INIT_VARS(vpblendw); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.0F3A 0x0f - invalid. */ + + +/** Opcode VEX.66.0F3A 0x0f. */ +FNIEMOP_DEF(iemOp_vpalignr_Vx_Hx_Wx_Ib) +{ + IEMOP_MNEMONIC3(VEX_RVM, VPALIGNR, vpalignr, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); /* @todo */ + IEMOPMEDIAOPTF3IMM8_INIT_VARS(vpalignr); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.66.0F3A 0x10 - invalid */ +/* Opcode VEX.66.0F3A 0x11 - invalid */ +/* Opcode VEX.66.0F3A 0x12 - invalid */ +/* Opcode VEX.66.0F3A 0x13 - invalid */ +/** Opcode VEX.66.0F3A 0x14. */ +FNIEMOP_STUB(iemOp_vpextrb_RdMb_Vdq_Ib); +/** Opcode VEX.66.0F3A 0x15. */ +FNIEMOP_STUB(iemOp_vpextrw_RdMw_Vdq_Ib); +/** Opcode VEX.66.0F3A 0x16. */ +FNIEMOP_STUB(iemOp_vpextrd_q_RdMw_Vdq_Ib); +/** Opcode VEX.66.0F3A 0x17. */ +FNIEMOP_STUB(iemOp_vextractps_Ed_Vdq_Ib); +/** Opcode VEX.66.0F3A 0x18 (vex only). */ +FNIEMOP_STUB(iemOp_vinsertf128_Vqq_Hqq_Wqq_Ib); +/** Opcode VEX.66.0F3A 0x19 (vex only). */ +FNIEMOP_STUB(iemOp_vextractf128_Wdq_Vqq_Ib); +/* Opcode VEX.66.0F3A 0x1a - invalid */ +/* Opcode VEX.66.0F3A 0x1b - invalid */ +/* Opcode VEX.66.0F3A 0x1c - invalid */ +/** Opcode VEX.66.0F3A 0x1d (vex only). */ +FNIEMOP_STUB(iemOp_vcvtps2ph_Wx_Vx_Ib); +/* Opcode VEX.66.0F3A 0x1e - invalid */ +/* Opcode VEX.66.0F3A 0x1f - invalid */ + + +/** Opcode VEX.66.0F3A 0x20. */ +FNIEMOP_STUB(iemOp_vpinsrb_Vdq_Hdq_RyMb_Ib); +/** Opcode VEX.66.0F3A 0x21, */ +FNIEMOP_STUB(iemOp_vinsertps_Vdq_Hdq_UdqMd_Ib); +/** Opcode VEX.66.0F3A 0x22. */ +FNIEMOP_STUB(iemOp_vpinsrd_q_Vdq_Hdq_Ey_Ib); +/* Opcode VEX.66.0F3A 0x23 - invalid */ +/* Opcode VEX.66.0F3A 0x24 - invalid */ +/* Opcode VEX.66.0F3A 0x25 - invalid */ +/* Opcode VEX.66.0F3A 0x26 - invalid */ +/* Opcode VEX.66.0F3A 0x27 - invalid */ +/* Opcode VEX.66.0F3A 0x28 - invalid */ +/* Opcode VEX.66.0F3A 0x29 - invalid */ +/* Opcode VEX.66.0F3A 0x2a - invalid */ +/* Opcode VEX.66.0F3A 0x2b - invalid */ +/* Opcode VEX.66.0F3A 0x2c - invalid */ +/* Opcode VEX.66.0F3A 0x2d - invalid */ +/* Opcode VEX.66.0F3A 0x2e - invalid */ +/* Opcode VEX.66.0F3A 0x2f - invalid */ + + +/* Opcode VEX.66.0F3A 0x30 - invalid */ +/* Opcode VEX.66.0F3A 0x31 - invalid */ +/* Opcode VEX.66.0F3A 0x32 - invalid */ +/* Opcode VEX.66.0F3A 0x33 - invalid */ +/* Opcode VEX.66.0F3A 0x34 - invalid */ +/* Opcode VEX.66.0F3A 0x35 - invalid */ +/* Opcode VEX.66.0F3A 0x36 - invalid */ +/* Opcode VEX.66.0F3A 0x37 - invalid */ +/** Opcode VEX.66.0F3A 0x38 (vex only). */ +FNIEMOP_STUB(iemOp_vinserti128_Vqq_Hqq_Wqq_Ib); +/** Opcode VEX.66.0F3A 0x39 (vex only). */ +FNIEMOP_STUB(iemOp_vextracti128_Wdq_Vqq_Ib); +/* Opcode VEX.66.0F3A 0x3a - invalid */ +/* Opcode VEX.66.0F3A 0x3b - invalid */ +/* Opcode VEX.66.0F3A 0x3c - invalid */ +/* Opcode VEX.66.0F3A 0x3d - invalid */ +/* Opcode VEX.66.0F3A 0x3e - invalid */ +/* Opcode VEX.66.0F3A 0x3f - invalid */ + + +/** Opcode VEX.66.0F3A 0x40. */ +FNIEMOP_STUB(iemOp_vdpps_Vx_Hx_Wx_Ib); +/** Opcode VEX.66.0F3A 0x41, */ +FNIEMOP_STUB(iemOp_vdppd_Vdq_Hdq_Wdq_Ib); +/** Opcode VEX.66.0F3A 0x42. */ +FNIEMOP_STUB(iemOp_vmpsadbw_Vx_Hx_Wx_Ib); +/* Opcode VEX.66.0F3A 0x43 - invalid */ + + +/** Opcode VEX.66.0F3A 0x44. */ +FNIEMOP_DEF(iemOp_vpclmulqdq_Vdq_Hdq_Wdq_Ib) +{ + //IEMOP_MNEMONIC3(VEX_RVM, VPCLMULQDQ, vpclmulqdq, Vdq, Hdq, Wdq, DISOPTYPE_HARMLESS, 0); /* @todo */ + + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fPclMul); + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); + IEM_MC_ARG(PCRTUINT128U, puSrc2, 2); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fPclMul, iemAImpl_vpclmulqdq_u128, iemAImpl_vpclmulqdq_u128_fallback), + puDst, puSrc1, puSrc2, bImmArg); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + /* + * Register, memory. + */ + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc2, uSrc2, 2); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); + IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm, 3); + IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fPclMul); + IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fPclMul, iemAImpl_vpclmulqdq_u128, iemAImpl_vpclmulqdq_u128_fallback), + puDst, puSrc1, puSrc2, bImmArg); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } +} + + +/* Opcode VEX.66.0F3A 0x45 - invalid */ +/** Opcode VEX.66.0F3A 0x46 (vex only) */ +FNIEMOP_STUB(iemOp_vperm2i128_Vqq_Hqq_Wqq_Ib); +/* Opcode VEX.66.0F3A 0x47 - invalid */ +/** Opcode VEX.66.0F3A 0x48 (AMD tables only). */ +FNIEMOP_STUB(iemOp_vperlmilzz2ps_Vx_Hx_Wp_Lx); +/** Opcode VEX.66.0F3A 0x49 (AMD tables only). */ +FNIEMOP_STUB(iemOp_vperlmilzz2pd_Vx_Hx_Wp_Lx); + + +/** + * Common worker for AVX2 instructions on the forms: + * - vpxxx xmm0, xmm1, xmm2/mem128, xmm4 + * - vpxxx ymm0, ymm1, ymm2/mem256, ymm4 + * + * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit. + */ +FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Lx, PCIEMOPBLENDOP, pImpl) +{ + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + if (pVCpu->iem.s.uVexLength) + { + uint8_t bOp4; IEM_OPCODE_GET_NEXT_U8(&bOp4); + + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_BEGIN(4, 4); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_LOCAL(RTUINT256U, uSrc3); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 2); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc3, uSrc3, 3); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_YREG_U256(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_FETCH_YREG_U256(uSrc3, bOp4 >> 4); /** @todo Ignore MSB in 32-bit mode. */ + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU256, puDst, puSrc1, puSrc2, puSrc3); + IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + uint8_t bOp4; IEM_OPCODE_GET_NEXT_U8(&bOp4); + + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_BEGIN(4, 0); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); + IEM_MC_ARG(PCRTUINT128U, puSrc2, 2); + IEM_MC_ARG(PCRTUINT128U, puSrc3, 3); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc3, bOp4 >> 4); /** @todo Ignore MSB in 32-bit mode. */ + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU128, puDst, puSrc1, puSrc2, puSrc3); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.uVexLength) + { + IEM_MC_BEGIN(4, 5); + IEM_MC_LOCAL(RTUINT256U, uDst); + IEM_MC_LOCAL(RTUINT256U, uSrc1); + IEM_MC_LOCAL(RTUINT256U, uSrc2); + IEM_MC_LOCAL(RTUINT256U, uSrc3); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG_LOCAL_REF(PRTUINT256U, puDst, uDst, 0); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc1, uSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc2, uSrc2, 2); + IEM_MC_ARG_LOCAL_REF(PCRTUINT256U, puSrc3, uSrc3, 3); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + uint8_t bOp4; IEM_OPCODE_GET_NEXT_U8(&bOp4); + + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_FETCH_YREG_U256(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_YREG_U256(uSrc3, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_FETCH_YREG_U256(uSrc3, bOp4 >> 4); /** @todo Ignore MSB in 32-bit mode. */ + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU256, puDst, puSrc1, puSrc2, puSrc3); + IEM_MC_STORE_YREG_U256_ZX_VLMAX( IEM_GET_MODRM_REG(pVCpu, bRm), uDst); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(4, 2); + IEM_MC_LOCAL(RTUINT128U, uSrc2); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_ARG(PRTUINT128U, puDst, 0); + IEM_MC_ARG(PCRTUINT128U, puSrc1, 1); + IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc2, uSrc2, 2); + IEM_MC_ARG(PCRTUINT128U, puSrc3, 3); + + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); + uint8_t bOp4; IEM_OPCODE_GET_NEXT_U8(&bOp4); + + IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); + IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); + IEM_MC_PREPARE_AVX_USAGE(); + + IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); + IEM_MC_REF_XREG_U128_CONST(puSrc3, bOp4 >> 4); /** @todo Ignore MSB in 32-bit mode. */ + IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU128, puDst, puSrc1, puSrc2, puSrc3); + IEM_MC_CLEAR_YREG_128_UP( IEM_GET_MODRM_REG(pVCpu, bRm)); + + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** Opcode VEX.66.0F3A 0x4a (vex only). */ +FNIEMOP_DEF(iemOp_vblendvps_Vx_Hx_Wx_Lx) +{ + //IEMOP_MNEMONIC4(VEX_RVM, VBLENDVPS, vpblendvps, Vx, Hx, Wx, Lx, DISOPTYPE_HARMLESS, 0); @todo + IEMOPBLENDOP_INIT_VARS(vblendvps); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Lx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F3A 0x4b (vex only). */ +FNIEMOP_DEF(iemOp_vblendvpd_Vx_Hx_Wx_Lx) +{ + //IEMOP_MNEMONIC4(VEX_RVM, VPBLENDVPD, blendvpd, Vx, Hx, Wx, Lx, DISOPTYPE_HARMLESS, 0); @todo + IEMOPBLENDOP_INIT_VARS(vblendvpd); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Lx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/** Opcode VEX.66.0F3A 0x4c (vex only). */ +FNIEMOP_DEF(iemOp_vpblendvb_Vx_Hx_Wx_Lx) +{ + //IEMOP_MNEMONIC4(VEX_RVM, VPBLENDVB, vpblendvb, Vx, Hx, Wx, Lx, DISOPTYPE_HARMLESS, 0); @todo + IEMOPBLENDOP_INIT_VARS(vpblendvb); + return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Lx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback)); +} + + +/* Opcode VEX.66.0F3A 0x4d - invalid */ +/* Opcode VEX.66.0F3A 0x4e - invalid */ +/* Opcode VEX.66.0F3A 0x4f - invalid */ + + +/* Opcode VEX.66.0F3A 0x50 - invalid */ +/* Opcode VEX.66.0F3A 0x51 - invalid */ +/* Opcode VEX.66.0F3A 0x52 - invalid */ +/* Opcode VEX.66.0F3A 0x53 - invalid */ +/* Opcode VEX.66.0F3A 0x54 - invalid */ +/* Opcode VEX.66.0F3A 0x55 - invalid */ +/* Opcode VEX.66.0F3A 0x56 - invalid */ +/* Opcode VEX.66.0F3A 0x57 - invalid */ +/* Opcode VEX.66.0F3A 0x58 - invalid */ +/* Opcode VEX.66.0F3A 0x59 - invalid */ +/* Opcode VEX.66.0F3A 0x5a - invalid */ +/* Opcode VEX.66.0F3A 0x5b - invalid */ +/** Opcode VEX.66.0F3A 0x5c (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmaddsubps_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x5d (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmaddsubpd_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x5e (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmsubaddps_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x5f (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmsubaddpd_Vx_Lx_Wx_Hx); + + +/** Opcode VEX.66.0F3A 0x60. */ +FNIEMOP_STUB(iemOp_vpcmpestrm_Vdq_Wdq_Ib); +/** Opcode VEX.66.0F3A 0x61, */ +FNIEMOP_STUB(iemOp_vpcmpestri_Vdq_Wdq_Ib); +/** Opcode VEX.66.0F3A 0x62. */ +FNIEMOP_STUB(iemOp_vpcmpistrm_Vdq_Wdq_Ib); +/** Opcode VEX.66.0F3A 0x63*/ +FNIEMOP_STUB(iemOp_vpcmpistri_Vdq_Wdq_Ib); +/* Opcode VEX.66.0F3A 0x64 - invalid */ +/* Opcode VEX.66.0F3A 0x65 - invalid */ +/* Opcode VEX.66.0F3A 0x66 - invalid */ +/* Opcode VEX.66.0F3A 0x67 - invalid */ +/** Opcode VEX.66.0F3A 0x68 (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmaddps_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x69 (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmaddpd_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x6a (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmaddss_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x6b (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmaddsd_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x6c (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmsubps_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x6d (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmsubpd_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x6e (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmsubss_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x6f (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfmsubsd_Vx_Lx_Wx_Hx); + +/* Opcode VEX.66.0F3A 0x70 - invalid */ +/* Opcode VEX.66.0F3A 0x71 - invalid */ +/* Opcode VEX.66.0F3A 0x72 - invalid */ +/* Opcode VEX.66.0F3A 0x73 - invalid */ +/* Opcode VEX.66.0F3A 0x74 - invalid */ +/* Opcode VEX.66.0F3A 0x75 - invalid */ +/* Opcode VEX.66.0F3A 0x76 - invalid */ +/* Opcode VEX.66.0F3A 0x77 - invalid */ +/** Opcode VEX.66.0F3A 0x78 (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfnmaddps_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x79 (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfnmaddpd_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x7a (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfnmaddss_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x7b (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfnmaddsd_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x7c (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfnmsubps_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x7d (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfnmsubpd_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x7e (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfnmsubss_Vx_Lx_Wx_Hx); +/** Opcode VEX.66.0F3A 0x7f (AMD tables only). */ +FNIEMOP_STUB(iemOp_vfnmsubsd_Vx_Lx_Wx_Hx); + +/* Opcodes 0x0f 0x80 thru 0x0f 0xb0 are unused. */ + + +/* Opcode 0x0f 0xc0 - invalid */ +/* Opcode 0x0f 0xc1 - invalid */ +/* Opcode 0x0f 0xc2 - invalid */ +/* Opcode 0x0f 0xc3 - invalid */ +/* Opcode 0x0f 0xc4 - invalid */ +/* Opcode 0x0f 0xc5 - invalid */ +/* Opcode 0x0f 0xc6 - invalid */ +/* Opcode 0x0f 0xc7 - invalid */ +/* Opcode 0x0f 0xc8 - invalid */ +/* Opcode 0x0f 0xc9 - invalid */ +/* Opcode 0x0f 0xca - invalid */ +/* Opcode 0x0f 0xcb - invalid */ +/* Opcode 0x0f 0xcc */ +FNIEMOP_STUB(iemOp_vsha1rnds4_Vdq_Wdq_Ib); +/* Opcode 0x0f 0xcd - invalid */ +/* Opcode 0x0f 0xce - invalid */ +/* Opcode 0x0f 0xcf - invalid */ + + +/* Opcode VEX.66.0F3A 0xd0 - invalid */ +/* Opcode VEX.66.0F3A 0xd1 - invalid */ +/* Opcode VEX.66.0F3A 0xd2 - invalid */ +/* Opcode VEX.66.0F3A 0xd3 - invalid */ +/* Opcode VEX.66.0F3A 0xd4 - invalid */ +/* Opcode VEX.66.0F3A 0xd5 - invalid */ +/* Opcode VEX.66.0F3A 0xd6 - invalid */ +/* Opcode VEX.66.0F3A 0xd7 - invalid */ +/* Opcode VEX.66.0F3A 0xd8 - invalid */ +/* Opcode VEX.66.0F3A 0xd9 - invalid */ +/* Opcode VEX.66.0F3A 0xda - invalid */ +/* Opcode VEX.66.0F3A 0xdb - invalid */ +/* Opcode VEX.66.0F3A 0xdc - invalid */ +/* Opcode VEX.66.0F3A 0xdd - invalid */ +/* Opcode VEX.66.0F3A 0xde - invalid */ +/* Opcode VEX.66.0F3A 0xdf - (aeskeygenassist). */ +FNIEMOP_STUB(iemOp_vaeskeygen_Vdq_Wdq_Ib); + + +/** Opcode VEX.F2.0F3A (vex only) */ +FNIEMOP_DEF(iemOp_rorx_Gy_Ey_Ib) +{ + IEMOP_MNEMONIC3(VEX_RMI, RORX, rorx, Gy, Ey, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO | IEMOPHINT_VEX_V_ZERO); + if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi2) + return iemOp_InvalidNeedRMImm8(pVCpu); + uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); + if (IEM_IS_MODRM_REG_MODE(bRm)) + { + /* + * Register, register. + */ + uint8_t bImm8; IEM_OPCODE_GET_NEXT_U8(&bImm8); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t, uSrc1, 1); + IEM_MC_ARG_CONST(uint64_t, uSrc2, bImm8, 2); + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_rorx_u64, pDst, uSrc1, uSrc2); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 0); + IEM_MC_ARG(uint32_t *, pDst, 0); + IEM_MC_ARG(uint32_t, uSrc1, 1); + IEM_MC_ARG_CONST(uint32_t, uSrc2, bImm8, 2); + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_rorx_u32, pDst, uSrc1, uSrc2); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } + else + { + /* + * Register, memory. + */ + if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) + { + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint64_t *, pDst, 0); + IEM_MC_ARG(uint64_t, uSrc1, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm8; IEM_OPCODE_GET_NEXT_U8(&bImm8); + IEM_MC_ARG_CONST(uint64_t, uSrc2, bImm8, 2); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_FETCH_MEM_U64(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_rorx_u64, pDst, uSrc1, uSrc2); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + else + { + IEM_MC_BEGIN(3, 1); + IEM_MC_ARG(uint32_t *, pDst, 0); + IEM_MC_ARG(uint32_t, uSrc1, 1); + IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc); + IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); + uint8_t bImm8; IEM_OPCODE_GET_NEXT_U8(&bImm8); + IEM_MC_ARG_CONST(uint32_t, uSrc2, bImm8, 2); + IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV(); + IEM_MC_FETCH_MEM_U32(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); + IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm)); + IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_rorx_u32, pDst, uSrc1, uSrc2); + IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); + IEM_MC_ADVANCE_RIP_AND_FINISH(); + IEM_MC_END(); + } + } +} + + +/** + * VEX opcode map \#3. + * + * @sa g_apfnThreeByte0f3a + */ +IEM_STATIC const PFNIEMOP g_apfnVexMap3[] = +{ + /* no prefix, 066h prefix f3h prefix, f2h prefix */ + /* 0x00 */ iemOp_InvalidNeedRMImm8, iemOp_vpermq_Vqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x01 */ iemOp_InvalidNeedRMImm8, iemOp_vpermqd_Vqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x02 */ iemOp_InvalidNeedRMImm8, iemOp_vpblendd_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x03 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x04 */ iemOp_InvalidNeedRMImm8, iemOp_vpermilps_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x05 */ iemOp_InvalidNeedRMImm8, iemOp_vpermilpd_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x06 */ iemOp_InvalidNeedRMImm8, iemOp_vperm2f128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x07 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x08 */ iemOp_InvalidNeedRMImm8, iemOp_vroundps_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x09 */ iemOp_InvalidNeedRMImm8, iemOp_vroundpd_Vx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0a */ iemOp_InvalidNeedRMImm8, iemOp_vroundss_Vss_Wss_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0b */ iemOp_InvalidNeedRMImm8, iemOp_vroundsd_Vsd_Wsd_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0c */ iemOp_InvalidNeedRMImm8, iemOp_vblendps_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0d */ iemOp_InvalidNeedRMImm8, iemOp_vblendpd_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0e */ iemOp_InvalidNeedRMImm8, iemOp_vpblendw_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x0f */ iemOp_InvalidNeedRMImm8, iemOp_vpalignr_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + + /* 0x10 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x11 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x12 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x13 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x14 */ iemOp_InvalidNeedRMImm8, iemOp_vpextrb_RdMb_Vdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x15 */ iemOp_InvalidNeedRMImm8, iemOp_vpextrw_RdMw_Vdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x16 */ iemOp_InvalidNeedRMImm8, iemOp_vpextrd_q_RdMw_Vdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x17 */ iemOp_InvalidNeedRMImm8, iemOp_vextractps_Ed_Vdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x18 */ iemOp_InvalidNeedRMImm8, iemOp_vinsertf128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x19 */ iemOp_InvalidNeedRMImm8, iemOp_vextractf128_Wdq_Vqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x1a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1d */ iemOp_InvalidNeedRMImm8, iemOp_vcvtps2ph_Wx_Vx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x1e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x1f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x20 */ iemOp_InvalidNeedRMImm8, iemOp_vpinsrb_Vdq_Hdq_RyMb_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x21 */ iemOp_InvalidNeedRMImm8, iemOp_vinsertps_Vdq_Hdq_UdqMd_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x22 */ iemOp_InvalidNeedRMImm8, iemOp_vpinsrd_q_Vdq_Hdq_Ey_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x23 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x24 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x25 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x26 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x27 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x28 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x29 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x2f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x30 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x31 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x32 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x33 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x34 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x35 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x36 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x37 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x38 */ iemOp_InvalidNeedRMImm8, iemOp_vinserti128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x39 */ iemOp_InvalidNeedRMImm8, iemOp_vextracti128_Wdq_Vqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x3a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x3f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x40 */ iemOp_InvalidNeedRMImm8, iemOp_vdpps_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x41 */ iemOp_InvalidNeedRMImm8, iemOp_vdppd_Vdq_Hdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x42 */ iemOp_InvalidNeedRMImm8, iemOp_vmpsadbw_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x43 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x44 */ iemOp_InvalidNeedRMImm8, iemOp_vpclmulqdq_Vdq_Hdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x45 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x46 */ iemOp_InvalidNeedRMImm8, iemOp_vperm2i128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x47 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x48 */ iemOp_InvalidNeedRMImm8, iemOp_vperlmilzz2ps_Vx_Hx_Wp_Lx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x49 */ iemOp_InvalidNeedRMImm8, iemOp_vperlmilzz2pd_Vx_Hx_Wp_Lx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x4a */ iemOp_InvalidNeedRMImm8, iemOp_vblendvps_Vx_Hx_Wx_Lx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x4b */ iemOp_InvalidNeedRMImm8, iemOp_vblendvpd_Vx_Hx_Wx_Lx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x4c */ iemOp_InvalidNeedRMImm8, iemOp_vpblendvb_Vx_Hx_Wx_Lx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x4d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x4e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x4f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x50 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x51 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x52 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x53 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x54 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x55 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x56 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x57 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x58 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x59 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x5a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x5b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x5c */ iemOp_InvalidNeedRMImm8, iemOp_vfmaddsubps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x5d */ iemOp_InvalidNeedRMImm8, iemOp_vfmaddsubpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x5e */ iemOp_InvalidNeedRMImm8, iemOp_vfmsubaddps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x5f */ iemOp_InvalidNeedRMImm8, iemOp_vfmsubaddpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + + /* 0x60 */ iemOp_InvalidNeedRMImm8, iemOp_vpcmpestrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x61 */ iemOp_InvalidNeedRMImm8, iemOp_vpcmpestri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x62 */ iemOp_InvalidNeedRMImm8, iemOp_vpcmpistrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x63 */ iemOp_InvalidNeedRMImm8, iemOp_vpcmpistri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x64 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x65 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x66 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x67 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x68 */ iemOp_InvalidNeedRMImm8, iemOp_vfmaddps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x69 */ iemOp_InvalidNeedRMImm8, iemOp_vfmaddpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x6a */ iemOp_InvalidNeedRMImm8, iemOp_vfmaddss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x6b */ iemOp_InvalidNeedRMImm8, iemOp_vfmaddsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x6c */ iemOp_InvalidNeedRMImm8, iemOp_vfmsubps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x6d */ iemOp_InvalidNeedRMImm8, iemOp_vfmsubpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x6e */ iemOp_InvalidNeedRMImm8, iemOp_vfmsubss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x6f */ iemOp_InvalidNeedRMImm8, iemOp_vfmsubsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + + /* 0x70 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x71 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x72 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x73 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x74 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x75 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x76 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x77 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x78 */ iemOp_InvalidNeedRMImm8, iemOp_vfnmaddps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x79 */ iemOp_InvalidNeedRMImm8, iemOp_vfnmaddpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x7a */ iemOp_InvalidNeedRMImm8, iemOp_vfnmaddss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x7b */ iemOp_InvalidNeedRMImm8, iemOp_vfnmaddsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x7c */ iemOp_InvalidNeedRMImm8, iemOp_vfnmsubps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x7d */ iemOp_InvalidNeedRMImm8, iemOp_vfnmsubpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x7e */ iemOp_InvalidNeedRMImm8, iemOp_vfnmsubss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0x7f */ iemOp_InvalidNeedRMImm8, iemOp_vfnmsubsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + + /* 0x80 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x81 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x82 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x83 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x84 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x85 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x86 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x87 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x88 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x89 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x8f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0x90 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x91 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x92 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x93 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x94 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x95 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x96 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x97 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x98 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x99 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9a */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9b */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9c */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9d */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9e */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0x9f */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xa0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xa9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xaa */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xab */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xac */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xad */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xae */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xaf */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xb0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xb9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xba */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbc */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbd */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbe */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xbf */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xc0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xc9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xca */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xcb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xcc */ iemOp_vsha1rnds4_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + /* 0xcd */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xce */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xcf */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xd0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xd9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xda */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xdb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xdc */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xdd */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xde */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xdf */ iemOp_vaeskeygen_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, + + /* 0xe0 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xe9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xea */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xeb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xec */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xed */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xee */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xef */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + + /* 0xf0 */ iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8, iemOp_rorx_Gy_Ey_Ib, + /* 0xf1 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf2 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf3 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf4 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf5 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf6 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf7 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf8 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xf9 */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfa */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfb */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfc */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfd */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xfe */ IEMOP_X4(iemOp_InvalidNeedRMImm8), + /* 0xff */ IEMOP_X4(iemOp_InvalidNeedRMImm8), +}; +AssertCompile(RT_ELEMENTS(g_apfnVexMap3) == 1024); + +/** @} */ + diff --git a/src/VBox/VMM/VMMAll/IOMAll.cpp b/src/VBox/VMM/VMMAll/IOMAll.cpp new file mode 100644 index 00000000..522156b5 --- /dev/null +++ b/src/VBox/VMM/VMMAll/IOMAll.cpp @@ -0,0 +1,618 @@ +/* $Id: IOMAll.cpp $ */ +/** @file + * IOM - Input / Output Monitor - Any Context. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_IOM_IOPORT +#include +#include +#include +#include "IOMInternal.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "IOMInline.h" + + +/** + * Reads an I/O port register. + * + * @returns Strict VBox status code. Informational status codes other than the one documented + * here are to be treated as internal failure. Use IOM_SUCCESS() to check for success. + * @retval VINF_SUCCESS Success. + * @retval VINF_EM_FIRST-VINF_EM_LAST Success with some exceptions (see IOM_SUCCESS()), the + * status code must be passed on to EM. + * @retval VINF_IOM_R3_IOPORT_READ Defer the read to ring-3. (R0/RC only) + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param Port The port to read. + * @param pu32Value Where to store the value read. + * @param cbValue The size of the register to read in bytes. 1, 2 or 4 bytes. + */ +VMMDECL(VBOXSTRICTRC) IOMIOPortRead(PVMCC pVM, PVMCPU pVCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue) +{ + STAM_COUNTER_INC(&pVM->iom.s.StatIoPortIn); + Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0); + +/** @todo should initialize *pu32Value here because it can happen that some + * handle is buggy and doesn't handle all cases. */ + + /* For lookups we need to share lock IOM. */ + int rc2 = IOM_LOCK_SHARED(pVM); + if (RT_SUCCESS(rc2)) + { /* likely */ } +#ifndef IN_RING3 + else if (rc2 == VERR_SEM_BUSY) + return VINF_IOM_R3_IOPORT_READ; +#endif + else + AssertMsgFailedReturn(("rc2=%Rrc\n", rc2), rc2); + + /* + * Get the entry for the current context. + */ + uint16_t offPort; + CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, Port, &offPort, &pVCpu->iom.s.idxIoPortLastRead); + if (pRegEntry) + { +#ifdef VBOX_WITH_STATISTICS + PIOMIOPORTSTATSENTRY pStats = iomIoPortGetStats(pVM, pRegEntry, offPort); +#endif + + /* + * Found an entry, get the data so we can leave the IOM lock. + */ + uint16_t const fFlags = pRegEntry->fFlags; + PFNIOMIOPORTNEWIN pfnInCallback = pRegEntry->pfnInCallback; + PPDMDEVINS pDevIns = pRegEntry->pDevIns; +#ifndef IN_RING3 + if ( pfnInCallback + && pDevIns + && pRegEntry->cPorts > 0) + { /* likely */ } + else + { + STAM_COUNTER_INC(&pStats->InRZToR3); + IOM_UNLOCK_SHARED(pVM); + return VINF_IOM_R3_IOPORT_READ; + } +#endif + void *pvUser = pRegEntry->pvUser; + IOM_UNLOCK_SHARED(pVM); + AssertPtr(pDevIns); + AssertPtr(pfnInCallback); + + /* + * Call the device. + */ + VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_READ); + if (rcStrict == VINF_SUCCESS) + { + STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfIn), a); + rcStrict = pfnInCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? Port : offPort, pu32Value, (unsigned)cbValue); + STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfIn), a); + PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo)); + +#ifndef IN_RING3 + if (rcStrict == VINF_IOM_R3_IOPORT_READ) + STAM_COUNTER_INC(&pStats->InRZToR3); + else +#endif + { + STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(In)); + STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total); + if (rcStrict == VERR_IOM_IOPORT_UNUSED) + { + /* make return value */ + rcStrict = VINF_SUCCESS; + switch (cbValue) + { + case 1: *(uint8_t *)pu32Value = 0xff; break; + case 2: *(uint16_t *)pu32Value = 0xffff; break; + case 4: *(uint32_t *)pu32Value = UINT32_C(0xffffffff); break; + default: + AssertMsgFailedReturn(("Invalid I/O port size %d. Port=%d\n", cbValue, Port), VERR_IOM_INVALID_IOPORT_SIZE); + } + } + } + Log3(("IOMIOPortRead: Port=%RTiop *pu32=%08RX32 cb=%d rc=%Rrc\n", Port, *pu32Value, cbValue, VBOXSTRICTRC_VAL(rcStrict))); + STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total); + } + else + STAM_COUNTER_INC(&pStats->InRZToR3); + return rcStrict; + } + + /* + * Ok, no handler for this port. + */ + IOM_UNLOCK_SHARED(pVM); + switch (cbValue) + { + case 1: *(uint8_t *)pu32Value = 0xff; break; + case 2: *(uint16_t *)pu32Value = 0xffff; break; + case 4: *(uint32_t *)pu32Value = UINT32_C(0xffffffff); break; + default: + AssertMsgFailedReturn(("Invalid I/O port size %d. Port=%d\n", cbValue, Port), VERR_IOM_INVALID_IOPORT_SIZE); + } + Log3(("IOMIOPortRead: Port=%RTiop *pu32=%08RX32 cb=%d rc=VINF_SUCCESS\n", Port, *pu32Value, cbValue)); + return VINF_SUCCESS; +} + + +/** + * Reads the string buffer of an I/O port register. + * + * @returns Strict VBox status code. Informational status codes other than the one documented + * here are to be treated as internal failure. Use IOM_SUCCESS() to check for success. + * @retval VINF_SUCCESS Success or no string I/O callback in + * this context. + * @retval VINF_EM_FIRST-VINF_EM_LAST Success with some exceptions (see IOM_SUCCESS()), the + * status code must be passed on to EM. + * @retval VINF_IOM_R3_IOPORT_READ Defer the read to ring-3. (R0/RC only) + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uPort The port to read. + * @param pvDst Pointer to the destination buffer. + * @param pcTransfers Pointer to the number of transfer units to read, on return remaining transfer units. + * @param cb Size of the transfer unit (1, 2 or 4 bytes). + */ +VMM_INT_DECL(VBOXSTRICTRC) IOMIOPortReadString(PVMCC pVM, PVMCPU pVCpu, RTIOPORT uPort, + void *pvDst, uint32_t *pcTransfers, unsigned cb) +{ + STAM_COUNTER_INC(&pVM->iom.s.StatIoPortInS); + Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0); + + /* For lookups we need to share lock IOM. */ + int rc2 = IOM_LOCK_SHARED(pVM); + if (RT_SUCCESS(rc2)) + { /* likely */ } +#ifndef IN_RING3 + else if (rc2 == VERR_SEM_BUSY) + return VINF_IOM_R3_IOPORT_READ; +#endif + else + AssertMsgFailedReturn(("rc2=%Rrc\n", rc2), rc2); + + const uint32_t cRequestedTransfers = *pcTransfers; + Assert(cRequestedTransfers > 0); + + /* + * Get the entry for the current context. + */ + uint16_t offPort; + CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, uPort, &offPort, &pVCpu->iom.s.idxIoPortLastReadStr); + if (pRegEntry) + { +#ifdef VBOX_WITH_STATISTICS + PIOMIOPORTSTATSENTRY pStats = iomIoPortGetStats(pVM, pRegEntry, offPort); +#endif + + /* + * Found an entry, get the data so we can leave the IOM lock. + */ + uint16_t const fFlags = pRegEntry->fFlags; + PFNIOMIOPORTNEWINSTRING pfnInStrCallback = pRegEntry->pfnInStrCallback; + PFNIOMIOPORTNEWIN pfnInCallback = pRegEntry->pfnInCallback; + PPDMDEVINS pDevIns = pRegEntry->pDevIns; +#ifndef IN_RING3 + if ( pfnInCallback + && pDevIns + && pRegEntry->cPorts > 0) + { /* likely */ } + else + { + STAM_COUNTER_INC(&pStats->InRZToR3); + IOM_UNLOCK_SHARED(pVM); + return VINF_IOM_R3_IOPORT_READ; + } +#endif + void *pvUser = pRegEntry->pvUser; + IOM_UNLOCK_SHARED(pVM); + AssertPtr(pDevIns); + AssertPtr(pfnInCallback); + + /* + * Call the device. + */ + VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_READ); + if (rcStrict == VINF_SUCCESS) + { + /* + * First using the string I/O callback. + */ + if (pfnInStrCallback) + { + STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfIn), a); + rcStrict = pfnInStrCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort, + (uint8_t *)pvDst, pcTransfers, cb); + STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfIn), a); + } + + /* + * Then doing the single I/O fallback. + */ + if ( *pcTransfers > 0 + && rcStrict == VINF_SUCCESS) + { + pvDst = (uint8_t *)pvDst + (cRequestedTransfers - *pcTransfers) * cb; + do + { + uint32_t u32Value = 0; + STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfIn), a); + rcStrict = pfnInCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort, &u32Value, cb); + STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfIn), a); + if (rcStrict == VERR_IOM_IOPORT_UNUSED) + { + u32Value = UINT32_MAX; + rcStrict = VINF_SUCCESS; + } + if (IOM_SUCCESS(rcStrict)) + { + switch (cb) + { + case 4: *(uint32_t *)pvDst = u32Value; pvDst = (uint8_t *)pvDst + 4; break; + case 2: *(uint16_t *)pvDst = (uint16_t)u32Value; pvDst = (uint8_t *)pvDst + 2; break; + case 1: *(uint8_t *)pvDst = (uint8_t )u32Value; pvDst = (uint8_t *)pvDst + 1; break; + default: AssertFailed(); + } + *pcTransfers -= 1; + } + } while ( *pcTransfers > 0 + && rcStrict == VINF_SUCCESS); + } + PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo)); + +#ifdef VBOX_WITH_STATISTICS +# ifndef IN_RING3 + if (rcStrict == VINF_IOM_R3_IOPORT_READ) + STAM_COUNTER_INC(&pStats->InRZToR3); + else +# endif + { + STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(In)); + STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total); + } +#endif + Log3(("IOMIOPortReadStr: uPort=%RTiop pvDst=%p pcTransfer=%p:{%#x->%#x} cb=%d rc=%Rrc\n", + uPort, pvDst, pcTransfers, cRequestedTransfers, *pcTransfers, cb, VBOXSTRICTRC_VAL(rcStrict))); + } +#ifndef IN_RING3 + else + STAM_COUNTER_INC(&pStats->InRZToR3); +#endif + return rcStrict; + } + + /* + * Ok, no handler for this port. + */ + IOM_UNLOCK_SHARED(pVM); + *pcTransfers = 0; + memset(pvDst, 0xff, cRequestedTransfers * cb); + Log3(("IOMIOPortReadStr: uPort=%RTiop (unused) pvDst=%p pcTransfer=%p:{%#x->%#x} cb=%d rc=VINF_SUCCESS\n", + uPort, pvDst, pcTransfers, cRequestedTransfers, *pcTransfers, cb)); + return VINF_SUCCESS; +} + + +#ifndef IN_RING3 +/** + * Defers a pending I/O port write to ring-3. + * + * @returns VINF_IOM_R3_IOPORT_COMMIT_WRITE + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param Port The port to write to. + * @param u32Value The value to write. + * @param cbValue The size of the value (1, 2, 4). + */ +static VBOXSTRICTRC iomIOPortRing3WritePending(PVMCPU pVCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue) +{ + Log5(("iomIOPortRing3WritePending: %#x LB %u -> %RTiop\n", u32Value, cbValue, Port)); + AssertReturn(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0, VERR_IOM_IOPORT_IPE_1); + pVCpu->iom.s.PendingIOPortWrite.IOPort = Port; + pVCpu->iom.s.PendingIOPortWrite.u32Value = u32Value; + pVCpu->iom.s.PendingIOPortWrite.cbValue = (uint32_t)cbValue; + VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM); + return VINF_IOM_R3_IOPORT_COMMIT_WRITE; +} +#endif + + +/** + * Writes to an I/O port register. + * + * @returns Strict VBox status code. Informational status codes other than the one documented + * here are to be treated as internal failure. Use IOM_SUCCESS() to check for success. + * @retval VINF_SUCCESS Success. + * @retval VINF_EM_FIRST-VINF_EM_LAST Success with some exceptions (see IOM_SUCCESS()), the + * status code must be passed on to EM. + * @retval VINF_IOM_R3_IOPORT_WRITE Defer the write to ring-3. (R0/RC only) + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param Port The port to write to. + * @param u32Value The value to write. + * @param cbValue The size of the register to read in bytes. 1, 2 or 4 bytes. + */ +VMMDECL(VBOXSTRICTRC) IOMIOPortWrite(PVMCC pVM, PVMCPU pVCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue) +{ + STAM_COUNTER_INC(&pVM->iom.s.StatIoPortOut); +#ifndef IN_RING3 + Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0); +#endif + + /* For lookups we need to share lock IOM. */ + int rc2 = IOM_LOCK_SHARED(pVM); + if (RT_SUCCESS(rc2)) + { /* likely */ } +#ifndef IN_RING3 + else if (rc2 == VERR_SEM_BUSY) + return iomIOPortRing3WritePending(pVCpu, Port, u32Value, cbValue); +#endif + else + AssertMsgFailedReturn(("rc2=%Rrc\n", rc2), rc2); + + /* + * Get the entry for the current context. + */ + uint16_t offPort; + CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, Port, &offPort, &pVCpu->iom.s.idxIoPortLastWrite); + if (pRegEntry) + { +#ifdef VBOX_WITH_STATISTICS + PIOMIOPORTSTATSENTRY pStats = iomIoPortGetStats(pVM, pRegEntry, offPort); +#endif + + /* + * Found an entry, get the data so we can leave the IOM lock. + */ + uint16_t const fFlags = pRegEntry->fFlags; + PFNIOMIOPORTNEWOUT pfnOutCallback = pRegEntry->pfnOutCallback; + PPDMDEVINS pDevIns = pRegEntry->pDevIns; +#ifndef IN_RING3 + if ( pfnOutCallback + && pDevIns + && pRegEntry->cPorts > 0) + { /* likely */ } + else + { + IOM_UNLOCK_SHARED(pVM); + STAM_COUNTER_INC(&pStats->OutRZToR3); + return iomIOPortRing3WritePending(pVCpu, Port, u32Value, cbValue); + } +#endif + void *pvUser = pRegEntry->pvUser; + IOM_UNLOCK_SHARED(pVM); + AssertPtr(pDevIns); + AssertPtr(pfnOutCallback); + + /* + * Call the device. + */ + VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_WRITE); + if (rcStrict == VINF_SUCCESS) + { + STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfOut), a); + rcStrict = pfnOutCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? Port : offPort, u32Value, (unsigned)cbValue); + STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfOut), a); + + PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo)); + +#ifdef VBOX_WITH_STATISTICS +# ifndef IN_RING3 + if (rcStrict != VINF_IOM_R3_IOPORT_WRITE) +# endif + { + STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(Out)); + STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total); + } +#endif + Log3(("IOMIOPortWrite: Port=%RTiop u32=%08RX32 cb=%d rc=%Rrc\n", Port, u32Value, cbValue, VBOXSTRICTRC_VAL(rcStrict))); + } +#ifndef IN_RING3 + if (rcStrict == VINF_IOM_R3_IOPORT_WRITE) + { + STAM_COUNTER_INC(&pStats->OutRZToR3); + return iomIOPortRing3WritePending(pVCpu, Port, u32Value, cbValue); + } +#endif + return rcStrict; + } + + /* + * Ok, no handler for that port. + */ + IOM_UNLOCK_SHARED(pVM); + Log3(("IOMIOPortWrite: Port=%RTiop u32=%08RX32 cb=%d nop\n", Port, u32Value, cbValue)); + return VINF_SUCCESS; +} + + +/** + * Writes the string buffer of an I/O port register. + * + * @returns Strict VBox status code. Informational status codes other than the one documented + * here are to be treated as internal failure. Use IOM_SUCCESS() to check for success. + * @retval VINF_SUCCESS Success or no string I/O callback in + * this context. + * @retval VINF_EM_FIRST-VINF_EM_LAST Success with some exceptions (see IOM_SUCCESS()), the + * status code must be passed on to EM. + * @retval VINF_IOM_R3_IOPORT_WRITE Defer the write to ring-3. (R0/RC only) + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uPort The port to write to. + * @param pvSrc The guest page to read from. + * @param pcTransfers Pointer to the number of transfer units to write, on + * return remaining transfer units. + * @param cb Size of the transfer unit (1, 2 or 4 bytes). + */ +VMM_INT_DECL(VBOXSTRICTRC) IOMIOPortWriteString(PVMCC pVM, PVMCPU pVCpu, RTIOPORT uPort, void const *pvSrc, + uint32_t *pcTransfers, unsigned cb) +{ + STAM_COUNTER_INC(&pVM->iom.s.StatIoPortOutS); + Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0); + Assert(cb == 1 || cb == 2 || cb == 4); + + /* Take the IOM lock before performing any device I/O. */ + int rc2 = IOM_LOCK_SHARED(pVM); + if (RT_SUCCESS(rc2)) + { /* likely */ } +#ifndef IN_RING3 + else if (rc2 == VERR_SEM_BUSY) + return VINF_IOM_R3_IOPORT_WRITE; +#endif + else + AssertMsgFailedReturn(("rc2=%Rrc\n", rc2), rc2); + + const uint32_t cRequestedTransfers = *pcTransfers; + Assert(cRequestedTransfers > 0); + + /* + * Get the entry for the current context. + */ + uint16_t offPort; + CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, uPort, &offPort, &pVCpu->iom.s.idxIoPortLastWriteStr); + if (pRegEntry) + { +#ifdef VBOX_WITH_STATISTICS + PIOMIOPORTSTATSENTRY pStats = iomIoPortGetStats(pVM, pRegEntry, offPort); +#endif + + /* + * Found an entry, get the data so we can leave the IOM lock. + */ + uint16_t const fFlags = pRegEntry->fFlags; + PFNIOMIOPORTNEWOUTSTRING pfnOutStrCallback = pRegEntry->pfnOutStrCallback; + PFNIOMIOPORTNEWOUT pfnOutCallback = pRegEntry->pfnOutCallback; + PPDMDEVINS pDevIns = pRegEntry->pDevIns; +#ifndef IN_RING3 + if ( pfnOutCallback + && pDevIns + && pRegEntry->cPorts > 0) + { /* likely */ } + else + { + IOM_UNLOCK_SHARED(pVM); + STAM_COUNTER_INC(&pStats->OutRZToR3); + return VINF_IOM_R3_IOPORT_WRITE; + } +#endif + void *pvUser = pRegEntry->pvUser; + IOM_UNLOCK_SHARED(pVM); + AssertPtr(pDevIns); + AssertPtr(pfnOutCallback); + + /* + * Call the device. + */ + VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_WRITE); + if (rcStrict == VINF_SUCCESS) + { + /* + * First using string I/O if possible. + */ + if (pfnOutStrCallback) + { + STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfOut), a); + rcStrict = pfnOutStrCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort, + (uint8_t const *)pvSrc, pcTransfers, cb); + STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfOut), a); + } + + /* + * Then doing the single I/O fallback. + */ + if ( *pcTransfers > 0 + && rcStrict == VINF_SUCCESS) + { + pvSrc = (uint8_t *)pvSrc + (cRequestedTransfers - *pcTransfers) * cb; + do + { + uint32_t u32Value; + switch (cb) + { + case 4: u32Value = *(uint32_t *)pvSrc; pvSrc = (uint8_t const *)pvSrc + 4; break; + case 2: u32Value = *(uint16_t *)pvSrc; pvSrc = (uint8_t const *)pvSrc + 2; break; + case 1: u32Value = *(uint8_t *)pvSrc; pvSrc = (uint8_t const *)pvSrc + 1; break; + default: AssertFailed(); u32Value = UINT32_MAX; + } + STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfOut), a); + rcStrict = pfnOutCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort, u32Value, cb); + STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfOut), a); + if (IOM_SUCCESS(rcStrict)) + *pcTransfers -= 1; + } while ( *pcTransfers > 0 + && rcStrict == VINF_SUCCESS); + } + + PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo)); + +#ifdef VBOX_WITH_STATISTICS +# ifndef IN_RING3 + if (rcStrict == VINF_IOM_R3_IOPORT_WRITE) + STAM_COUNTER_INC(&pStats->OutRZToR3); + else +# endif + { + STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(Out)); + STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total); + } +#endif + Log3(("IOMIOPortWriteStr: uPort=%RTiop pvSrc=%p pcTransfer=%p:{%#x->%#x} cb=%d rcStrict=%Rrc\n", + uPort, pvSrc, pcTransfers, cRequestedTransfers, *pcTransfers, cb, VBOXSTRICTRC_VAL(rcStrict))); + } +#ifndef IN_RING3 + else + STAM_COUNTER_INC(&pStats->OutRZToR3); +#endif + return rcStrict; + } + + /* + * Ok, no handler for this port. + */ + IOM_UNLOCK_SHARED(pVM); + *pcTransfers = 0; + Log3(("IOMIOPortWriteStr: uPort=%RTiop (unused) pvSrc=%p pcTransfer=%p:{%#x->%#x} cb=%d rc=VINF_SUCCESS\n", + uPort, pvSrc, pcTransfers, cRequestedTransfers, *pcTransfers, cb)); + return VINF_SUCCESS; +} + diff --git a/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp b/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp new file mode 100644 index 00000000..c737bf1f --- /dev/null +++ b/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp @@ -0,0 +1,1320 @@ +/* $Id: IOMAllMmioNew.cpp $ */ +/** @file + * IOM - Input / Output Monitor - Any Context, MMIO & String I/O. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_IOM_MMIO +#define VMCPU_INCL_CPUM_GST_CTX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "IOMInternal.h" +#include +#include +#include +#include "IOMInline.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** @def IOM_MMIO_STATS_COMMA_DECL + * Parameter list declaration for statistics entry pointer. */ +/** @def IOM_MMIO_STATS_COMMA_ARG + * Statistics entry pointer argument. */ +#if defined(VBOX_WITH_STATISTICS) || defined(DOXYGEN_RUNNING) +# define IOM_MMIO_STATS_COMMA_DECL , PIOMMMIOSTATSENTRY pStats +# define IOM_MMIO_STATS_COMMA_ARG , pStats +#else +# define IOM_MMIO_STATS_COMMA_DECL +# define IOM_MMIO_STATS_COMMA_ARG +#endif + + + +#ifndef IN_RING3 +/** + * Defers a pending MMIO write to ring-3. + * + * @returns VINF_IOM_R3_MMIO_COMMIT_WRITE + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPhys The write address. + * @param pvBuf The bytes being written. + * @param cbBuf How many bytes. + * @param idxRegEntry The MMIO registration index (handle) if available, + * otherwise UINT32_MAX. + */ +static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf, + uint32_t idxRegEntry) +{ + Log5(("iomMmioRing3WritePending: %RGp LB %#x (idx=%#x)\n", GCPhys, cbBuf, idxRegEntry)); + if (pVCpu->iom.s.PendingMmioWrite.cbValue == 0) + { + pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys; + AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2); + pVCpu->iom.s.PendingMmioWrite.cbValue = (uint32_t)cbBuf; + pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = idxRegEntry; + memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf); + } + else + { + /* + * Join with pending if adjecent. + * + * This may happen if the stack overflows into MMIO territory and RSP/ESP/SP + * isn't aligned. IEM will bounce buffer the access and do one write for each + * page. We get here when the 2nd page part is written. + */ + uint32_t const cbOldValue = pVCpu->iom.s.PendingMmioWrite.cbValue; + AssertMsgReturn(GCPhys == pVCpu->iom.s.PendingMmioWrite.GCPhys + cbOldValue, + ("pending %RGp LB %#x; incoming %RGp LB %#x\n", + pVCpu->iom.s.PendingMmioWrite.GCPhys, cbOldValue, GCPhys, cbBuf), + VERR_IOM_MMIO_IPE_1); + AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue) - cbOldValue, VERR_IOM_MMIO_IPE_2); + pVCpu->iom.s.PendingMmioWrite.cbValue = cbOldValue + (uint32_t)cbBuf; + memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[cbOldValue], pvBuf, cbBuf); + } + + VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM); + return VINF_IOM_R3_MMIO_COMMIT_WRITE; +} +#endif + + +/** + * Deals with complicated MMIO writes. + * + * Complicated means unaligned or non-dword/qword sized accesses depending on + * the MMIO region's access mode flags. + * + * @returns Strict VBox status code. Any EM scheduling status code, + * VINF_IOM_R3_MMIO_WRITE, VINF_IOM_R3_MMIO_READ_WRITE or + * VINF_IOM_R3_MMIO_READ may be returned. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pRegEntry The MMIO entry for the current context. + * @param GCPhys The physical address to start writing. + * @param offRegion MMIO region offset corresponding to @a GCPhys. + * @param pvValue Where to store the value. + * @param cbValue The size of the value to write. + * @param pStats Pointer to the statistics (never NULL). + */ +static VBOXSTRICTRC iomMmioDoComplicatedWrite(PVM pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, + RTGCPHYS GCPhys, RTGCPHYS offRegion, + void const *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL) +{ + AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU + && (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING, + VERR_IOM_MMIO_IPE_1); + AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2); + RTGCPHYS const GCPhysStart = GCPhys; NOREF(GCPhysStart); + bool const fReadMissing = (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING + || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING; + RT_NOREF_PV(pVCpu); /* ring-3 */ + + /* + * Do debug stop if requested. + */ + VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM); +#ifdef VBOX_STRICT + if (!(pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE)) + { /* likely */ } + else + { +# ifdef IN_RING3 + LogRel(("IOM: Complicated write %#x byte at %RGp to %s, initiating debugger intervention\n", cbValue, GCPhys, + R3STRING(pRegEntry->pszDesc))); + rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS, + "Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, pRegEntry->pszDesc); + if (rc == VERR_DBGF_NOT_ATTACHED) + rc = VINF_SUCCESS; +# else + return VINF_IOM_R3_MMIO_WRITE; +# endif + } +#endif + + STAM_COUNTER_INC(&pStats->ComplicatedWrites); + + /* + * Check if we should ignore the write. + */ + if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD) + { + Assert(cbValue != 4 || (GCPhys & 3)); + return VINF_SUCCESS; + } + if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD) + { + Assert((cbValue != 4 && cbValue != 8) || (GCPhys & (cbValue - 1))); + return VINF_SUCCESS; + } + + /* + * Split and conquer. + */ + for (;;) + { + unsigned const offAccess = GCPhys & 3; + unsigned cbThisPart = 4 - offAccess; + if (cbThisPart > cbValue) + cbThisPart = cbValue; + + /* + * Get the missing bits (if any). + */ + uint32_t u32MissingValue = 0; + if (fReadMissing && cbThisPart != 4) + { + VBOXSTRICTRC rc2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser, + !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) + ? offRegion & ~(RTGCPHYS)3 : (GCPhys & ~(RTGCPHYS)3), + &u32MissingValue, sizeof(u32MissingValue)); + switch (VBOXSTRICTRC_VAL(rc2)) + { + case VINF_SUCCESS: + break; + case VINF_IOM_MMIO_UNUSED_FF: + STAM_COUNTER_INC(&pStats->FFor00Reads); + u32MissingValue = UINT32_C(0xffffffff); + break; + case VINF_IOM_MMIO_UNUSED_00: + STAM_COUNTER_INC(&pStats->FFor00Reads); + u32MissingValue = 0; + break; +#ifndef IN_RING3 + case VINF_IOM_R3_MMIO_READ: + case VINF_IOM_R3_MMIO_READ_WRITE: + case VINF_IOM_R3_MMIO_WRITE: + LogFlow(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2))); + rc2 = iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRegEntry->idxSelf); + if (rc == VINF_SUCCESS || rc2 < rc) + rc = rc2; + return rc; +#endif + default: + if (RT_FAILURE(rc2)) + { + Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2))); + return rc2; + } + AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS); + if (rc == VINF_SUCCESS || rc2 < rc) + rc = rc2; + break; + } + } + + /* + * Merge missing and given bits. + */ + uint32_t u32GivenMask; + uint32_t u32GivenValue; + switch (cbThisPart) + { + case 1: + u32GivenValue = *(uint8_t const *)pvValue; + u32GivenMask = UINT32_C(0x000000ff); + break; + case 2: + u32GivenValue = *(uint16_t const *)pvValue; + u32GivenMask = UINT32_C(0x0000ffff); + break; + case 3: + u32GivenValue = RT_MAKE_U32_FROM_U8(((uint8_t const *)pvValue)[0], ((uint8_t const *)pvValue)[1], + ((uint8_t const *)pvValue)[2], 0); + u32GivenMask = UINT32_C(0x00ffffff); + break; + case 4: + u32GivenValue = *(uint32_t const *)pvValue; + u32GivenMask = UINT32_C(0xffffffff); + break; + default: + AssertFailedReturn(VERR_IOM_MMIO_IPE_3); + } + if (offAccess) + { + u32GivenValue <<= offAccess * 8; + u32GivenMask <<= offAccess * 8; + } + + uint32_t u32Value = (u32MissingValue & ~u32GivenMask) + | (u32GivenValue & u32GivenMask); + + /* + * Do DWORD write to the device. + */ + VBOXSTRICTRC rc2 = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser, + !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) + ? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3, + &u32Value, sizeof(u32Value)); + switch (VBOXSTRICTRC_VAL(rc2)) + { + case VINF_SUCCESS: + break; +#ifndef IN_RING3 + case VINF_IOM_R3_MMIO_READ: + case VINF_IOM_R3_MMIO_READ_WRITE: + case VINF_IOM_R3_MMIO_WRITE: + Log3(("iomMmioDoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2))); + AssertReturn(pVCpu->iom.s.PendingMmioWrite.cbValue == 0, VERR_IOM_MMIO_IPE_1); + AssertReturn(cbValue + (GCPhys & 3) <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2); + pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys & ~(RTGCPHYS)3; + pVCpu->iom.s.PendingMmioWrite.cbValue = cbValue + (GCPhys & 3); + *(uint32_t *)pVCpu->iom.s.PendingMmioWrite.abValue = u32Value; + if (cbValue > cbThisPart) + memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[4], + (uint8_t const *)pvValue + cbThisPart, cbValue - cbThisPart); + VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM); + if (rc == VINF_SUCCESS) + rc = VINF_IOM_R3_MMIO_COMMIT_WRITE; + return rc; +#endif + default: + if (RT_FAILURE(rc2)) + { + Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2))); + return rc2; + } + AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS); + if (rc == VINF_SUCCESS || rc2 < rc) + rc = rc2; + break; + } + + /* + * Advance. + */ + cbValue -= cbThisPart; + if (!cbValue) + break; + GCPhys += cbThisPart; + offRegion += cbThisPart; + pvValue = (uint8_t const *)pvValue + cbThisPart; + } + + return rc; +} + + + + +/** + * Wrapper which does the write. + */ +DECLINLINE(VBOXSTRICTRC) iomMmioDoWrite(PVMCC pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, + RTGCPHYS GCPhys, RTGCPHYS offRegion, + const void *pvData, uint32_t cb IOM_MMIO_STATS_COMMA_DECL) +{ + VBOXSTRICTRC rcStrict; + if (RT_LIKELY(pRegEntry->pfnWriteCallback)) + { + if ( (cb == 4 && !(GCPhys & 3)) + || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU + || (cb == 8 && !(GCPhys & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRegEntry->fFlags)) ) + rcStrict = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser, + !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvData, cb); + else + rcStrict = iomMmioDoComplicatedWrite(pVM, pVCpu, pRegEntry, GCPhys, offRegion, pvData, cb IOM_MMIO_STATS_COMMA_ARG); + } + else + rcStrict = VINF_SUCCESS; + return rcStrict; +} + + +#ifdef IN_RING3 +/** + * Helper for IOMR3ProcessForceFlag() that lives here to utilize iomMmioDoWrite et al. + */ +VBOXSTRICTRC iomR3MmioCommitWorker(PVM pVM, PVMCPU pVCpu, PIOMMMIOENTRYR3 pRegEntry, RTGCPHYS offRegion) +{ +# ifdef VBOX_WITH_STATISTICS + STAM_PROFILE_START(UnusedMacroArg, Prf); + PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry); +# endif + PPDMDEVINS const pDevIns = pRegEntry->pDevIns; + int rc = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VERR_IGNORED); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, pVCpu->iom.s.PendingMmioWrite.GCPhys, offRegion, + pVCpu->iom.s.PendingMmioWrite.abValue, pVCpu->iom.s.PendingMmioWrite.cbValue + IOM_MMIO_STATS_COMMA_ARG); + + PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo)); + STAM_PROFILE_STOP(&pStats->ProfWriteR3, Prf); + return rcStrict; +} +#endif /* IN_RING3 */ + + +/** + * Deals with complicated MMIO reads. + * + * Complicated means unaligned or non-dword/qword sized accesses depending on + * the MMIO region's access mode flags. + * + * @returns Strict VBox status code. Any EM scheduling status code, + * VINF_IOM_R3_MMIO_READ, VINF_IOM_R3_MMIO_READ_WRITE or + * VINF_IOM_R3_MMIO_WRITE may be returned. + * + * @param pVM The cross context VM structure. + * @param pRegEntry The MMIO entry for the current context. + * @param GCPhys The physical address to start reading. + * @param offRegion MMIO region offset corresponding to @a GCPhys. + * @param pvValue Where to store the value. + * @param cbValue The size of the value to read. + * @param pStats Pointer to the statistics (never NULL). + */ +static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion, + void *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL) +{ + AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD + || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD, + VERR_IOM_MMIO_IPE_1); + AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2); +#ifdef LOG_ENABLED + RTGCPHYS const GCPhysStart = GCPhys; +#endif + + /* + * Do debug stop if requested. + */ + VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM); +#ifdef VBOX_STRICT + if (pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ) + { +# ifdef IN_RING3 + rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS, + "Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRegEntry->pszDesc)); + if (rc == VERR_DBGF_NOT_ATTACHED) + rc = VINF_SUCCESS; +# else + return VINF_IOM_R3_MMIO_READ; +# endif + } +#endif + + STAM_COUNTER_INC(&pStats->ComplicatedReads); + + /* + * Split and conquer. + */ + for (;;) + { + /* + * Do DWORD read from the device. + */ + uint32_t u32Value; + VBOXSTRICTRC rcStrict2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser, + !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) + ? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3, + &u32Value, sizeof(u32Value)); + switch (VBOXSTRICTRC_VAL(rcStrict2)) + { + case VINF_SUCCESS: + break; + case VINF_IOM_MMIO_UNUSED_FF: + STAM_COUNTER_INC(&pStats->FFor00Reads); + u32Value = UINT32_C(0xffffffff); + break; + case VINF_IOM_MMIO_UNUSED_00: + STAM_COUNTER_INC(&pStats->FFor00Reads); + u32Value = 0; + break; + case VINF_IOM_R3_MMIO_READ: + case VINF_IOM_R3_MMIO_READ_WRITE: + case VINF_IOM_R3_MMIO_WRITE: + /** @todo What if we've split a transfer and already read + * something? Since reads can have sideeffects we could be + * kind of screwed here... */ + LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n", + GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2))); + return rcStrict2; + default: + if (RT_FAILURE(rcStrict2)) + { + Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n", + GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2))); + return rcStrict2; + } + AssertMsgReturn(rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)), + VERR_IPE_UNEXPECTED_INFO_STATUS); + if (rc == VINF_SUCCESS || rcStrict2 < rc) + rc = rcStrict2; + break; + } + u32Value >>= (GCPhys & 3) * 8; + + /* + * Write what we've read. + */ + unsigned cbThisPart = 4 - (GCPhys & 3); + if (cbThisPart > cbValue) + cbThisPart = cbValue; + + switch (cbThisPart) + { + case 1: + *(uint8_t *)pvValue = (uint8_t)u32Value; + break; + case 2: + *(uint16_t *)pvValue = (uint16_t)u32Value; + break; + case 3: + ((uint8_t *)pvValue)[0] = RT_BYTE1(u32Value); + ((uint8_t *)pvValue)[1] = RT_BYTE2(u32Value); + ((uint8_t *)pvValue)[2] = RT_BYTE3(u32Value); + break; + case 4: + *(uint32_t *)pvValue = u32Value; + break; + } + + /* + * Advance. + */ + cbValue -= cbThisPart; + if (!cbValue) + break; + GCPhys += cbThisPart; + offRegion += cbThisPart; + pvValue = (uint8_t *)pvValue + cbThisPart; + } + + return rc; +} + + +/** + * Implements VINF_IOM_MMIO_UNUSED_FF. + * + * @returns VINF_SUCCESS. + * @param pvValue Where to store the zeros. + * @param cbValue How many bytes to read. + * @param pStats Pointer to the statistics (never NULL). + */ +static int iomMMIODoReadFFs(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL) +{ + switch (cbValue) + { + case 1: *(uint8_t *)pvValue = UINT8_C(0xff); break; + case 2: *(uint16_t *)pvValue = UINT16_C(0xffff); break; + case 4: *(uint32_t *)pvValue = UINT32_C(0xffffffff); break; + case 8: *(uint64_t *)pvValue = UINT64_C(0xffffffffffffffff); break; + default: + { + uint8_t *pb = (uint8_t *)pvValue; + while (cbValue--) + *pb++ = UINT8_C(0xff); + break; + } + } + STAM_COUNTER_INC(&pStats->FFor00Reads); + return VINF_SUCCESS; +} + + +/** + * Implements VINF_IOM_MMIO_UNUSED_00. + * + * @returns VINF_SUCCESS. + * @param pvValue Where to store the zeros. + * @param cbValue How many bytes to read. + * @param pStats Pointer to the statistics (never NULL). + */ +static int iomMMIODoRead00s(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL) +{ + switch (cbValue) + { + case 1: *(uint8_t *)pvValue = UINT8_C(0x00); break; + case 2: *(uint16_t *)pvValue = UINT16_C(0x0000); break; + case 4: *(uint32_t *)pvValue = UINT32_C(0x00000000); break; + case 8: *(uint64_t *)pvValue = UINT64_C(0x0000000000000000); break; + default: + { + uint8_t *pb = (uint8_t *)pvValue; + while (cbValue--) + *pb++ = UINT8_C(0x00); + break; + } + } + STAM_COUNTER_INC(&pStats->FFor00Reads); + return VINF_SUCCESS; +} + + +/** + * Wrapper which does the read. + */ +DECLINLINE(VBOXSTRICTRC) iomMmioDoRead(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion, + void *pvValue, uint32_t cbValue IOM_MMIO_STATS_COMMA_DECL) +{ + VBOXSTRICTRC rcStrict; + if (RT_LIKELY(pRegEntry->pfnReadCallback)) + { + if ( ( cbValue == 4 + && !(GCPhys & 3)) + || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU + || ( cbValue == 8 + && !(GCPhys & 7) + && (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) ) + rcStrict = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser, + !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvValue, cbValue); + else + rcStrict = iomMMIODoComplicatedRead(pVM, pRegEntry, GCPhys, offRegion, pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); + } + else + rcStrict = VINF_IOM_MMIO_UNUSED_FF; + if (rcStrict != VINF_SUCCESS) + { + switch (VBOXSTRICTRC_VAL(rcStrict)) + { + case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break; + case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break; + } + } + return rcStrict; +} + +#ifndef IN_RING3 + +/** + * Checks if we can handle an MMIO \#PF in R0/RC. + */ +DECLINLINE(bool) iomMmioCanHandlePfInRZ(PVMCC pVM, uint32_t uErrorCode, CTX_SUFF(PIOMMMIOENTRY) pRegEntry) +{ + if (pRegEntry->cbRegion > 0) + { + if ( pRegEntry->pfnWriteCallback + && pRegEntry->pfnReadCallback) + return true; + + PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[pRegEntry->idxSelf]; + if ( uErrorCode == UINT32_MAX + ? pRegEntryR3->pfnWriteCallback || pRegEntryR3->pfnReadCallback + : uErrorCode & X86_TRAP_PF_RW + ? !pRegEntry->pfnWriteCallback && pRegEntryR3->pfnWriteCallback + : !pRegEntry->pfnReadCallback && pRegEntryR3->pfnReadCallback) + return false; + + return true; + } + return false; +} + + +/** + * Common worker for the \#PF handler and IOMMMIOPhysHandler (APIC+VT-x). + * + * @returns VBox status code (appropriate for GC return). + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uErrorCode CPU Error code. This is UINT32_MAX when we don't have + * any error code (the EPT misconfig hack). + * @param GCPhysFault The GC physical address corresponding to pvFault. + * @param pRegEntry The MMIO entry for the current context. + */ +DECLINLINE(VBOXSTRICTRC) iomMmioCommonPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, + RTGCPHYS GCPhysFault, CTX_SUFF(PIOMMMIOENTRY) pRegEntry) +{ + Log(("iomMmioCommonPfHandler: GCPhysFault=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, CPUMGetGuestRIP(pVCpu) )); + RT_NOREF(GCPhysFault, uErrorCode); + + VBOXSTRICTRC rcStrict; + +#ifndef IN_RING3 + /* + * Should we defer the request right away? This isn't usually the case, so + * do the simple test first and the try deal with uErrorCode being N/A. + */ + PPDMDEVINS const pDevIns = pRegEntry->pDevIns; + if (RT_LIKELY( pDevIns + && iomMmioCanHandlePfInRZ(pVM, uErrorCode, pRegEntry))) + { + /* + * Enter the device critsect prior to engaging IOM in case of lock contention. + * Note! Perhaps not a good move? + */ + rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE); + if (rcStrict == VINF_SUCCESS) + { +#endif /* !IN_RING3 */ + + /* + * Let IEM call us back via iomMmioHandler. + */ + rcStrict = IEMExecOne(pVCpu); + +#ifndef IN_RING3 + PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo)); +#endif + if (RT_SUCCESS(rcStrict)) + { /* likely */ } + else if ( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED + || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED) + { + Log(("IOM: Hit unsupported IEM feature!\n")); + rcStrict = VINF_EM_RAW_EMULATE_INSTR; + } +#ifndef IN_RING3 + return rcStrict; + } + STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0); + } + else + rcStrict = VINF_IOM_R3_MMIO_READ_WRITE; + +# ifdef VBOX_WITH_STATISTICS + if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE) + { + PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry); + if (uErrorCode & X86_TRAP_PF_RW) + { + STAM_COUNTER_INC(&pStats->WriteRZToR3); + STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3); + } + else + { + STAM_COUNTER_INC(&pStats->ReadRZToR3); + STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3); + } + } +# endif +#else /* IN_RING3 */ + RT_NOREF(pVM, pRegEntry); +#endif /* IN_RING3 */ + return rcStrict; +} + + +/** + * @callback_method_impl{FNPGMRZPHYSPFHANDLER, + * \#PF access handler callback for MMIO pages.} + * + * @remarks The @a uUser argument is the MMIO handle. + */ +DECLCALLBACK(VBOXSTRICTRC) iomMmioPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx, + RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser) +{ + STAM_PROFILE_START(&pVM->iom.s.StatMmioPfHandler, Prf); + LogFlow(("iomMmioPfHandlerNew: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n", + GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtx->rip)); + RT_NOREF(pvFault, pCtx); + + /* Translate the MMIO handle to a registration entry for the current context. */ + AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE); +# ifdef IN_RING0 + AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE); + CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iomr0.s.paMmioRegs[uUser]; +# else + CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iom.s.paMmioRegs[uUser]; +# endif + + VBOXSTRICTRC rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry); + + STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPfHandler, Prf); + return rcStrict; +} + +#endif /* !IN_RING3 */ + +#ifdef IN_RING0 +/** + * Physical access handler for MMIO ranges. + * + * This is actually only used by VT-x for APIC page accesses. + * + * @returns VBox status code (appropriate for GC return). + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uErrorCode CPU Error code. + * @param GCPhysFault The GC physical address. + */ +VMM_INT_DECL(VBOXSTRICTRC) IOMR0MmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault) +{ + STAM_PROFILE_START(&pVM->iom.s.StatMmioPhysHandler, Prf); + + /* + * We don't have a range here, so look it up before calling the common function. + */ + VBOXSTRICTRC rcStrict = IOM_LOCK_SHARED(pVM); + if (RT_SUCCESS(rcStrict)) + { + RTGCPHYS offRegion; + CTX_SUFF(PIOMMMIOENTRY) pRegEntry = iomMmioGetEntry(pVM, GCPhysFault, &offRegion, &pVCpu->iom.s.idxMmioLastPhysHandler); + IOM_UNLOCK_SHARED(pVM); + if (RT_LIKELY(pRegEntry)) + rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry); + else + rcStrict = VERR_IOM_MMIO_RANGE_NOT_FOUND; + } + else if (rcStrict == VERR_SEM_BUSY) + rcStrict = VINF_IOM_R3_MMIO_READ_WRITE; + + STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPhysHandler, Prf); + return rcStrict; +} +#endif /* IN_RING0 */ + + +/** + * @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses} + * + * @remarks The @a uUser argument is the MMIO handle. + */ +DECLCALLBACK(VBOXSTRICTRC) iomMmioHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys, void *pvBuf, + size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser) +{ + STAM_PROFILE_START(UnusedMacroArg, Prf); + STAM_COUNTER_INC(&pVM->iom.s.CTX_SUFF(StatMmioHandler)); + Log4(("iomMmioHandlerNew: GCPhysFault=%RGp cbBuf=%#x enmAccessType=%d enmOrigin=%d uUser=%p\n", GCPhysFault, cbBuf, enmAccessType, enmOrigin, uUser)); + + Assert(enmAccessType == PGMACCESSTYPE_READ || enmAccessType == PGMACCESSTYPE_WRITE); + AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf)); + NOREF(pvPhys); NOREF(enmOrigin); + +#ifdef IN_RING3 + int const rcToRing3 = VERR_IOM_MMIO_IPE_3; +#else + int const rcToRing3 = enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ : VINF_IOM_R3_MMIO_WRITE; +#endif + + /* + * Translate uUser to an MMIO registration table entry. We can do this + * without any locking as the data is static after VM creation. + */ + AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE); +#ifdef IN_RING0 + AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE); + CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iomr0.s.paMmioRegs[uUser]; + PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[uUser]; +#else + CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iom.s.paMmioRegs[uUser]; +#endif +#ifdef VBOX_WITH_STATISTICS + PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry); /* (Works even without ring-0 device setup.) */ +#endif + PPDMDEVINS const pDevIns = pRegEntry->pDevIns; + +#ifdef VBOX_STRICT + /* + * Assert the right entry in strict builds. This may yield a false positive + * for SMP VMs if we're unlucky and the guest isn't well behaved. + */ +# ifdef IN_RING0 + Assert(pRegEntry && (GCPhysFault - pRegEntryR3->GCPhysMapping < pRegEntryR3->cbRegion || !pRegEntryR3->fMapped)); +# else + Assert(pRegEntry && (GCPhysFault - pRegEntry->GCPhysMapping < pRegEntry->cbRegion || !pRegEntry->fMapped)); +# endif +#endif + +#ifndef IN_RING3 + /* + * If someone is doing FXSAVE, FXRSTOR, XSAVE, XRSTOR or other stuff dealing with + * large amounts of data, just go to ring-3 where we don't need to deal with partial + * successes. No chance any of these will be problematic read-modify-write stuff. + * + * Also drop back if the ring-0 registration entry isn't actually used. + */ + if ( RT_LIKELY(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)) + && pRegEntry->cbRegion != 0 + && ( enmAccessType == PGMACCESSTYPE_READ + ? pRegEntry->pfnReadCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL + : pRegEntry->pfnWriteCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL) + && pDevIns ) + { /* likely */ } + else + { + Log4(("iomMmioHandlerNew: to ring-3: to-big=%RTbool zero-size=%RTbool no-callback=%RTbool pDevIns=%p hRegion=%#RX64\n", + !(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)), !(pRegEntry->cbRegion != 0), + !( enmAccessType == PGMACCESSTYPE_READ + ? pRegEntry->pfnReadCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL + : pRegEntry->pfnWriteCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL), + pDevIns, uUser)); + STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3); + STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3); + return rcToRing3; + } +#endif /* !IN_RING3 */ + + /* + * If we've got an offset that's outside the region, defer to ring-3 if we + * can, or pretend there is nothing there. This shouldn't happen, but can + * if we're unlucky with an SMP VM and the guest isn't behaving very well. + */ +#ifdef IN_RING0 + RTGCPHYS const GCPhysMapping = pRegEntryR3->GCPhysMapping; +#else + RTGCPHYS const GCPhysMapping = pRegEntry->GCPhysMapping; +#endif + RTGCPHYS const offRegion = GCPhysFault - GCPhysMapping; + if (RT_LIKELY(offRegion < pRegEntry->cbRegion && GCPhysMapping != NIL_RTGCPHYS)) + { /* likely */ } + else + { + STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioStaleMappings); + LogRelMax(64, ("iomMmioHandlerNew: Stale access at %#RGp to range #%#x currently residing at %RGp LB %RGp\n", + GCPhysFault, pRegEntry->idxSelf, GCPhysMapping, pRegEntry->cbRegion)); +#ifdef IN_RING3 + if (enmAccessType == PGMACCESSTYPE_READ) + iomMMIODoReadFFs(pvBuf, cbBuf IOM_MMIO_STATS_COMMA_ARG); + return VINF_SUCCESS; +#else + STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3); + STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3); + return rcToRing3; +#endif + } + + /* + * Guard against device configurations causing recursive MMIO accesses + * (see @bugref{10315}). + */ + uint8_t const idxDepth = pVCpu->iom.s.cMmioRecursionDepth; + if (RT_LIKELY(idxDepth < RT_ELEMENTS(pVCpu->iom.s.apMmioRecursionStack))) + { + pVCpu->iom.s.cMmioRecursionDepth = idxDepth + 1; + /** @todo Add iomr0 with a apMmioRecursionStack for ring-0. */ +#ifdef IN_RING3 + pVCpu->iom.s.apMmioRecursionStack[idxDepth] = pDevIns; +#endif + } + else + { + STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioTooDeepRecursion); +#ifdef IN_RING3 + AssertCompile(RT_ELEMENTS(pVCpu->iom.s.apMmioRecursionStack) == 2); + LogRelMax(64, ("iomMmioHandlerNew: Too deep recursion %RGp LB %#zx: %p (%s); %p (%s); %p (%s)\n", + GCPhysFault, cbBuf, pDevIns, pDevIns->pReg->szName, + pVCpu->iom.s.apMmioRecursionStack[1], pVCpu->iom.s.apMmioRecursionStack[1]->pReg->szName, + pVCpu->iom.s.apMmioRecursionStack[0], pVCpu->iom.s.apMmioRecursionStack[0]->pReg->szName)); +#else + LogRelMax(64, ("iomMmioHandlerNew: Too deep recursion %RGp LB %#zx!: %p (%s)\n", + GCPhysFault, cbBuf, pDevIns, pDevIns->pReg->szName)); +#endif + return VINF_PGM_HANDLER_DO_DEFAULT; + } + + + /* + * Perform locking and the access. + * + * Writes requiring a return to ring-3 are buffered by IOM so IEM can + * commit the instruction. + * + * Note! We may end up locking the device even when the relevant callback is + * NULL. This is supposed to be an unlikely case, so not optimized yet. + * + * Note! All returns goes thru the one return statement at the end of the + * function in order to correctly maintaint the recursion counter. + */ + VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), rcToRing3); + if (rcStrict == VINF_SUCCESS) + { + if (enmAccessType == PGMACCESSTYPE_READ) + { + /* + * Read. + */ + rcStrict = iomMmioDoRead(pVM, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG); + + PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo)); +#ifndef IN_RING3 + if (rcStrict == VINF_IOM_R3_MMIO_READ) + { + STAM_COUNTER_INC(&pStats->ReadRZToR3); + STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3); + } + else +#endif + STAM_COUNTER_INC(&pStats->Reads); + STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), Prf); + } + else + { + /* + * Write. + */ + rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG); + PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo)); +#ifndef IN_RING3 + if (rcStrict == VINF_IOM_R3_MMIO_WRITE) + rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf); + if (rcStrict == VINF_IOM_R3_MMIO_WRITE) + { + STAM_COUNTER_INC(&pStats->WriteRZToR3); + STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3); + } + else if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE) + { + STAM_COUNTER_INC(&pStats->CommitRZToR3); + STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3); + } + else +#endif + STAM_COUNTER_INC(&pStats->Writes); + STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), Prf); + } + + /* + * Check the return code. + */ +#ifdef IN_RING3 + AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc - Access type %d - %RGp - %s\n", + VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRegEntry->pszDesc)); +#else + AssertMsg( rcStrict == VINF_SUCCESS + || rcStrict == rcToRing3 + || (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE && enmAccessType == PGMACCESSTYPE_WRITE) + || rcStrict == VINF_EM_DBG_STOP + || rcStrict == VINF_EM_DBG_EVENT + || rcStrict == VINF_EM_DBG_BREAKPOINT + || rcStrict == VINF_EM_OFF + || rcStrict == VINF_EM_SUSPEND + || rcStrict == VINF_EM_RESET + //|| rcStrict == VINF_EM_HALT /* ?? */ + //|| rcStrict == VINF_EM_NO_MEMORY /* ?? */ + , ("%Rrc - Access type %d - %RGp - %s #%u\n", + VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns->pReg->szName, pDevIns->iInstance)); +#endif + } + /* + * Deal with enter-critsect failures. + */ +#ifndef IN_RING3 + else if (rcStrict == VINF_IOM_R3_MMIO_WRITE) + { + Assert(enmAccessType == PGMACCESSTYPE_WRITE); + rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf); + if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE) + { + STAM_COUNTER_INC(&pStats->CommitRZToR3); + STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3); + } + else + { + STAM_COUNTER_INC(&pStats->WriteRZToR3); + STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3); + } + STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0); + } + else if (rcStrict == VINF_IOM_R3_MMIO_READ) + { + Assert(enmAccessType == PGMACCESSTYPE_READ); + STAM_COUNTER_INC(&pStats->ReadRZToR3); + STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0); + } +#endif + else + AssertMsg(RT_FAILURE_NP(rcStrict), ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + + pVCpu->iom.s.cMmioRecursionDepth = idxDepth; + return rcStrict; +} + + +/** + * Mapping an MMIO2 page in place of an MMIO page for direct access. + * + * This is a special optimization used by the VGA device. Call + * IOMMmioResetRegion() to undo the mapping. + * + * @returns VBox status code. This API may return VINF_SUCCESS even if no + * remapping is made. + * @retval VERR_SEM_BUSY in ring-0 if we cannot get the IOM lock. + * + * @param pVM The cross context VM structure. + * @param pDevIns The device instance @a hRegion and @a hMmio2 are + * associated with. + * @param hRegion The handle to the MMIO region. + * @param offRegion The offset into @a hRegion of the page to be + * remapped. + * @param hMmio2 The MMIO2 handle. + * @param offMmio2 Offset into @a hMmio2 of the page to be use for the + * mapping. + * @param fPageFlags Page flags to set. Must be (X86_PTE_RW | X86_PTE_P) + * for the time being. + */ +VMMDECL(int) IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion, + uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags) +{ + /* Currently only called from the VGA device during MMIO. */ + Log(("IOMMmioMapMmio2Page %#RX64/%RGp -> %#RX64/%RGp flags=%RX64\n", hRegion, offRegion, hMmio2, offMmio2, fPageFlags)); + AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER); + AssertReturn(pDevIns, VERR_INVALID_POINTER); + +/** @todo Why is this restricted to protected mode??? Try it in all modes! */ + PVMCPUCC pVCpu = VMMGetCpu(pVM); + + /* This currently only works in real mode, protected mode without paging or with nested paging. */ + /** @todo NEM: MMIO page aliasing. */ + if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */ + || ( CPUMIsGuestInPagedProtectedMode(pVCpu) + && !HMIsNestedPagingActive(pVM))) + return VINF_SUCCESS; /* ignore */ /** @todo return some indicator if we fail here */ + + /* + * Translate the handle into an entry and check the region offset. + */ + AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE); +#ifdef IN_RING0 + AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE); + PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion]; + AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE); + AssertReturn(offRegion < pVM->iomr0.s.paMmioRegs[hRegion].cbRegion, VERR_OUT_OF_RANGE); + AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns + || ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL + && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED); +#else + PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion]; + AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE); + AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED); +#endif + AssertReturn(offRegion < pRegEntry->cbRegion, VERR_OUT_OF_RANGE); + Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0); + + /* + * When getting and using the mapping address, we must sit on the IOM lock + * to prevent remapping. Shared suffices as we change nothing. + */ + int rc = IOM_LOCK_SHARED(pVM); + if (rc == VINF_SUCCESS) + { + RTGCPHYS const GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS; + if (GCPhys != NIL_RTGCPHYS) + { + Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK)); + + /* + * Do the aliasing; page align the addresses since PGM is picky. + */ + rc = PGMHandlerPhysicalPageAliasMmio2(pVM, GCPhys, GCPhys + (offRegion & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK), + pDevIns, hMmio2, offMmio2); + } + else + AssertFailedStmt(rc = VERR_IOM_MMIO_REGION_NOT_MAPPED); + + IOM_UNLOCK_SHARED(pVM); + } + +/** @todo either ditch this or replace it with something that works in the + * nested case, since we really only care about nested paging! */ +#if 0 + /* + * Modify the shadow page table. Since it's an MMIO page it won't be present and we + * can simply prefetch it. + * + * Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page. + */ +# if 0 /* The assertion is wrong for the PGM_SYNC_CLEAR_PGM_POOL and VINF_PGM_HANDLER_ALREADY_ALIASED cases. */ +# ifdef VBOX_STRICT + uint64_t fFlags; + RTHCPHYS HCPhys; + rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys); + Assert(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT); +# endif +# endif + rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys); + Assert(rc == VINF_SUCCESS || rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT); +#endif + return rc; +} + + +#ifdef IN_RING0 /* VT-x ring-0 only, move to IOMR0Mmio.cpp later. */ +/** + * Mapping a HC page in place of an MMIO page for direct access. + * + * This is a special optimization used by the APIC in the VT-x case. This VT-x + * code uses PGMHandlerPhysicalReset rather than IOMMmioResetRegion() to undo + * the effects here. + * + * @todo Make VT-x usage more consistent. + * + * @returns VBox status code. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhys The address of the MMIO page to be changed. + * @param HCPhys The address of the host physical page. + * @param fPageFlags Page flags to set. Must be (X86_PTE_RW | X86_PTE_P) + * for the time being. + */ +VMMR0_INT_DECL(int) IOMR0MmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags) +{ + /* Currently only called from VT-x code during a page fault. */ + Log(("IOMR0MmioMapMmioHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags)); + + AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER); + /** @todo NEM: MMIO page aliasing?? */ + Assert(HMIsEnabled(pVM)); + +# ifdef VBOX_STRICT + /* + * Check input address (it's HM calling, not the device, so no region handle). + */ + int rcSem = IOM_LOCK_SHARED(pVM); + if (rcSem == VINF_SUCCESS) + { + RTGCPHYS offIgn; + uint16_t idxIgn = UINT16_MAX; + PIOMMMIOENTRYR0 pRegEntry = iomMmioGetEntry(pVM, GCPhys, &offIgn, &idxIgn); + IOM_UNLOCK_SHARED(pVM); + Assert(pRegEntry); + Assert(pRegEntry && !(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK)); + } +# endif + + /* + * Do the aliasing; page align the addresses since PGM is picky. + */ + GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + HCPhys &= ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK; + + int rc = PGMHandlerPhysicalPageAliasHC(pVM, GCPhys, GCPhys, HCPhys); + AssertRCReturn(rc, rc); + +/** @todo either ditch this or replace it with something that works in the + * nested case, since we really only care about nested paging! */ + + /* + * Modify the shadow page table. Since it's an MMIO page it won't be present and we + * can simply prefetch it. + * + * Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page. + */ + rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys); + Assert(rc == VINF_SUCCESS || rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT); + return VINF_SUCCESS; +} +#endif + + +/** + * Reset a previously modified MMIO region; restore the access flags. + * + * This undoes the effects of IOMMmioMapMmio2Page() and is currently only + * intended for some ancient VGA hack. However, it would be great to extend it + * beyond VT-x and/or nested-paging. + * + * @returns VBox status code. + * + * @param pVM The cross context VM structure. + * @param pDevIns The device instance @a hRegion is associated with. + * @param hRegion The handle to the MMIO region. + */ +VMMDECL(int) IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion) +{ + Log(("IOMMMIOResetRegion %#RX64\n", hRegion)); + AssertReturn(pDevIns, VERR_INVALID_POINTER); + +/** @todo Get rid of this this real/protected or nested paging restriction, + * it probably shouldn't be here and would be nasty when the CPU + * changes mode while we have the hack enabled... */ + PVMCPUCC pVCpu = VMMGetCpu(pVM); + + /* This currently only works in real mode, protected mode without paging or with nested paging. */ + /** @todo NEM: MMIO page aliasing. */ + if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */ + || ( CPUMIsGuestInPagedProtectedMode(pVCpu) + && !HMIsNestedPagingActive(pVM))) + return VINF_SUCCESS; /* ignore */ + + /* + * Translate the handle into an entry and mapping address for PGM. + * We have to take the lock to safely access the mapping address here. + */ + AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE); +#ifdef IN_RING0 + AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE); + PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion]; + AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE); + AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns + || ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL + && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED); +#else + PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion]; + AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE); + AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED); +#endif + Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0); + + int rcSem = IOM_LOCK_SHARED(pVM); + RTGCPHYS GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS; + if (rcSem == VINF_SUCCESS) + IOM_UNLOCK_SHARED(pVM); + + Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK)); + Assert(!(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK)); + + /* + * Call PGM to do the job work. + * + * After the call, all the pages should be non-present, unless there is + * a page pool flush pending (unlikely). + */ + int rc = PGMHandlerPhysicalReset(pVM, GCPhys); + AssertRC(rc); + +# ifdef VBOX_STRICT + if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)) + { + RTGCPHYS cb = pRegEntry->cbRegion; + while (cb) + { + uint64_t fFlags; + RTHCPHYS HCPhys; + rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys); + Assert(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT); + cb -= RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE); + GCPhys += RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE); + } + } +# endif + return rc; +} + diff --git a/src/VBox/VMM/VMMAll/MMAll.cpp b/src/VBox/VMM/VMMAll/MMAll.cpp new file mode 100644 index 00000000..856beb00 --- /dev/null +++ b/src/VBox/VMM/VMMAll/MMAll.cpp @@ -0,0 +1,160 @@ +/* $Id: MMAll.cpp $ */ +/** @file + * MM - Memory Manager - Any Context. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_MM_HYPER +#include +#include +#include "MMInternal.h" +#include +#include +#include +#include +#include + + +/** + * Gets the string name of a memory tag. + * + * @returns name of enmTag. + * @param enmTag The tag. + */ +const char *mmGetTagName(MMTAG enmTag) +{ + switch (enmTag) + { + #define TAG2STR(tag) case MM_TAG_##tag: return #tag + + TAG2STR(CFGM); + TAG2STR(CFGM_BYTES); + TAG2STR(CFGM_STRING); + TAG2STR(CFGM_USER); + + TAG2STR(CPUM_CTX); + TAG2STR(CPUM_CPUID); + TAG2STR(CPUM_MSRS); + + TAG2STR(CSAM); + TAG2STR(CSAM_PATCH); + + TAG2STR(DBGF); + TAG2STR(DBGF_AS); + TAG2STR(DBGF_FLOWTRACE); + TAG2STR(DBGF_INFO); + TAG2STR(DBGF_LINE); + TAG2STR(DBGF_LINE_DUP); + TAG2STR(DBGF_MODULE); + TAG2STR(DBGF_OS); + TAG2STR(DBGF_REG); + TAG2STR(DBGF_STACK); + TAG2STR(DBGF_SYMBOL); + TAG2STR(DBGF_SYMBOL_DUP); + TAG2STR(DBGF_TYPE); + TAG2STR(DBGF_TRACER); + + TAG2STR(EM); + + TAG2STR(IEM); + + TAG2STR(IOM); + TAG2STR(IOM_STATS); + + TAG2STR(MM); + TAG2STR(MM_LOOKUP_GUEST); + TAG2STR(MM_LOOKUP_PHYS); + TAG2STR(MM_LOOKUP_VIRT); + TAG2STR(MM_PAGE); + + TAG2STR(PARAV); + + TAG2STR(PATM); + TAG2STR(PATM_PATCH); + + TAG2STR(PDM); + TAG2STR(PDM_DEVICE); + TAG2STR(PDM_DEVICE_DESC); + TAG2STR(PDM_DEVICE_USER); + TAG2STR(PDM_DRIVER); + TAG2STR(PDM_DRIVER_DESC); + TAG2STR(PDM_DRIVER_USER); + TAG2STR(PDM_USB); + TAG2STR(PDM_USB_DESC); + TAG2STR(PDM_USB_USER); + TAG2STR(PDM_LUN); + TAG2STR(PDM_QUEUE); + TAG2STR(PDM_THREAD); + TAG2STR(PDM_ASYNC_COMPLETION); +#ifdef VBOX_WITH_NETSHAPER + TAG2STR(PDM_NET_SHAPER); +#endif /* VBOX_WITH_NETSHAPER */ + + TAG2STR(PGM); + TAG2STR(PGM_CHUNK_MAPPING); + TAG2STR(PGM_HANDLERS); + TAG2STR(PGM_HANDLER_TYPES); + TAG2STR(PGM_MAPPINGS); + TAG2STR(PGM_PHYS); + TAG2STR(PGM_POOL); + + TAG2STR(REM); + + TAG2STR(SELM); + + TAG2STR(SSM); + + TAG2STR(STAM); + + TAG2STR(TM); + + TAG2STR(TRPM); + + TAG2STR(VM); + TAG2STR(VM_REQ); + + TAG2STR(VMM); + + TAG2STR(HM); + + #undef TAG2STR + + default: + { + AssertMsgFailed(("Unknown tag %d! forgot to add it to the switch?\n", enmTag)); +#ifdef IN_RING3 + static char sz[48]; + RTStrPrintf(sz, sizeof(sz), "%d", enmTag); + return sz; +#else + return "unknown tag!"; +#endif + } + } +} + diff --git a/src/VBox/VMM/VMMAll/Makefile.kup b/src/VBox/VMM/VMMAll/Makefile.kup new file mode 100644 index 00000000..e69de29b diff --git a/src/VBox/VMM/VMMAll/NEMAll.cpp b/src/VBox/VMM/VMMAll/NEMAll.cpp new file mode 100644 index 00000000..1a44e573 --- /dev/null +++ b/src/VBox/VMM/VMMAll/NEMAll.cpp @@ -0,0 +1,136 @@ +/* $Id: NEMAll.cpp $ */ +/** @file + * NEM - Native execution manager, R0 and R3 context code. + */ + +/* + * Copyright (C) 2018-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_NEM +#include +#include "NEMInternal.h" +#include +#include + + +/** + * Checks if this VM is in NEM mode and is long-mode capable. + * + * Use VMR3IsLongModeAllowed() instead of this, when possible. + * + * @returns true if long mode is allowed, false otherwise. + * @param pVM The cross context VM structure. + * @sa VMR3IsLongModeAllowed, HMIsLongModeAllowed + */ +VMM_INT_DECL(bool) NEMHCIsLongModeAllowed(PVMCC pVM) +{ + return pVM->nem.s.fAllow64BitGuests && VM_IS_NEM_ENABLED(pVM); +} + + +/** + * Physical access handler registration notification. + * + * @param pVM The cross context VM structure. + * @param enmKind The kind of access handler. + * @param GCPhys Start of the access handling range. + * @param cb Length of the access handling range. + * + * @note Called while holding down the PGM lock. + */ +VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb) +{ +#ifdef VBOX_WITH_NATIVE_NEM + if (VM_IS_NEM_ENABLED(pVM)) + nemHCNativeNotifyHandlerPhysicalRegister(pVM, enmKind, GCPhys, cb); +#else + RT_NOREF(pVM, enmKind, GCPhys, cb); +#endif +} + + +VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld, + RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM) +{ +#ifdef VBOX_WITH_NATIVE_NEM + if (VM_IS_NEM_ENABLED(pVM)) + nemHCNativeNotifyHandlerPhysicalModify(pVM, enmKind, GCPhysOld, GCPhysNew, cb, fRestoreAsRAM); +#else + RT_NOREF(pVM, enmKind, GCPhysOld, GCPhysNew, cb, fRestoreAsRAM); +#endif +} + + +VMM_INT_DECL(int) NEMHCNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt, + PGMPAGETYPE enmType, uint8_t *pu2State) +{ + Assert(VM_IS_NEM_ENABLED(pVM)); +#ifdef VBOX_WITH_NATIVE_NEM + return nemHCNativeNotifyPhysPageAllocated(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State); +#else + RT_NOREF(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State); + return VINF_SUCCESS; +#endif +} + + +#ifndef VBOX_WITH_NATIVE_NEM +VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM) +{ + RT_NOREF(pVM); + return 0; +} +#endif + + +#ifndef VBOX_WITH_NATIVE_NEM +VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat) +{ + RT_NOREF(pVCpu, fWhat); + return VERR_NEM_IPE_9; +} +#endif + + +#ifndef VBOX_WITH_NATIVE_NEM +VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux) +{ + RT_NOREF(pVCpu, pcTicks, puAux); + AssertFailed(); + return VERR_NEM_IPE_9; +} +#endif + + +#ifndef VBOX_WITH_NATIVE_NEM +VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue) +{ + RT_NOREF(pVM, pVCpu, uPausedTscValue); + AssertFailed(); + return VERR_NEM_IPE_9; +} +#endif + diff --git a/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h b/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h new file mode 100644 index 00000000..f3493079 --- /dev/null +++ b/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h @@ -0,0 +1,3038 @@ +/* $Id: NEMAllNativeTemplate-win.cpp.h $ */ +/** @file + * NEM - Native execution manager, Windows code template ring-0/3. + */ + +/* + * Copyright (C) 2018-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + +#ifndef IN_RING3 +# error "This is ring-3 only now" +#endif + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** Copy back a segment from hyper-V. */ +#define NEM_WIN_COPY_BACK_SEG(a_Dst, a_Src) \ + do { \ + (a_Dst).u64Base = (a_Src).Base; \ + (a_Dst).u32Limit = (a_Src).Limit; \ + (a_Dst).ValidSel = (a_Dst).Sel = (a_Src).Selector; \ + (a_Dst).Attr.u = (a_Src).Attributes; \ + (a_Dst).fFlags = CPUMSELREG_FLAGS_VALID; \ + } while (0) + +/** @def NEMWIN_ASSERT_MSG_REG_VAL + * Asserts the correctness of a register value in a message/context. + */ +#if 0 +# define NEMWIN_NEED_GET_REGISTER +# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) \ + do { \ + WHV_REGISTER_VALUE TmpVal; \ + nemR3WinGetRegister(a_pVCpu, a_enmReg, &TmpVal); \ + AssertMsg(a_Expr, a_Msg); \ + } while (0) +#else +# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) do { } while (0) +#endif + +/** @def NEMWIN_ASSERT_MSG_REG_VAL + * Asserts the correctness of a 64-bit register value in a message/context. + */ +#define NEMWIN_ASSERT_MSG_REG_VAL64(a_pVCpu, a_enmReg, a_u64Val) \ + NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, (a_u64Val) == TmpVal.Reg64, \ + (#a_u64Val "=%#RX64, expected %#RX64\n", (a_u64Val), TmpVal.Reg64)) +/** @def NEMWIN_ASSERT_MSG_REG_VAL + * Asserts the correctness of a segment register value in a message/context. + */ +#define NEMWIN_ASSERT_MSG_REG_SEG(a_pVCpu, a_enmReg, a_SReg) \ + NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, \ + (a_SReg).Base == TmpVal.Segment.Base \ + && (a_SReg).Limit == TmpVal.Segment.Limit \ + && (a_SReg).Selector == TmpVal.Segment.Selector \ + && (a_SReg).Attributes == TmpVal.Segment.Attributes, \ + ( #a_SReg "=%#RX16 {%#RX64 LB %#RX32,%#RX16} expected %#RX16 {%#RX64 LB %#RX32,%#RX16}\n", \ + (a_SReg).Selector, (a_SReg).Base, (a_SReg).Limit, (a_SReg).Attributes, \ + TmpVal.Segment.Selector, TmpVal.Segment.Base, TmpVal.Segment.Limit, TmpVal.Segment.Attributes)) + + +#ifndef NTDDI_WIN10_19H1 +# define NTDDI_WIN10_19H1 0x0a000007 +#endif + +/** WHvRegisterPendingEvent0 was renamed to WHvRegisterPendingEvent between + * SDK 17134 and 18362. */ +#if WDK_NTDDI_VERSION < NTDDI_WIN10_19H1 +# define WHvRegisterPendingEvent WHvRegisterPendingEvent0 +#endif + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +/** NEM_WIN_PAGE_STATE_XXX names. */ +NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" }; + +/** HV_INTERCEPT_ACCESS_TYPE names. */ +static const char * const g_apszHvInterceptAccessTypes[4] = { "read", "write", "exec", "!undefined!" }; + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, + uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged); + + + +NEM_TMPL_STATIC int nemHCWinCopyStateToHyperV(PVMCC pVM, PVMCPUCC pVCpu) +{ + /* + * The following is very similar to what nemR0WinExportState() does. + */ + WHV_REGISTER_NAME aenmNames[128]; + WHV_REGISTER_VALUE aValues[128]; + + uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK); + if ( !fWhat + && pVCpu->nem.s.fCurrentInterruptWindows == pVCpu->nem.s.fDesiredInterruptWindows) + return VINF_SUCCESS; + uintptr_t iReg = 0; + +#define ADD_REG64(a_enmName, a_uValue) do { \ + aenmNames[iReg] = (a_enmName); \ + aValues[iReg].Reg128.High64 = 0; \ + aValues[iReg].Reg64 = (a_uValue); \ + iReg++; \ + } while (0) +#define ADD_REG128(a_enmName, a_uValueLo, a_uValueHi) do { \ + aenmNames[iReg] = (a_enmName); \ + aValues[iReg].Reg128.Low64 = (a_uValueLo); \ + aValues[iReg].Reg128.High64 = (a_uValueHi); \ + iReg++; \ + } while (0) + + /* GPRs */ + if (fWhat & CPUMCTX_EXTRN_GPRS_MASK) + { + if (fWhat & CPUMCTX_EXTRN_RAX) + ADD_REG64(WHvX64RegisterRax, pVCpu->cpum.GstCtx.rax); + if (fWhat & CPUMCTX_EXTRN_RCX) + ADD_REG64(WHvX64RegisterRcx, pVCpu->cpum.GstCtx.rcx); + if (fWhat & CPUMCTX_EXTRN_RDX) + ADD_REG64(WHvX64RegisterRdx, pVCpu->cpum.GstCtx.rdx); + if (fWhat & CPUMCTX_EXTRN_RBX) + ADD_REG64(WHvX64RegisterRbx, pVCpu->cpum.GstCtx.rbx); + if (fWhat & CPUMCTX_EXTRN_RSP) + ADD_REG64(WHvX64RegisterRsp, pVCpu->cpum.GstCtx.rsp); + if (fWhat & CPUMCTX_EXTRN_RBP) + ADD_REG64(WHvX64RegisterRbp, pVCpu->cpum.GstCtx.rbp); + if (fWhat & CPUMCTX_EXTRN_RSI) + ADD_REG64(WHvX64RegisterRsi, pVCpu->cpum.GstCtx.rsi); + if (fWhat & CPUMCTX_EXTRN_RDI) + ADD_REG64(WHvX64RegisterRdi, pVCpu->cpum.GstCtx.rdi); + if (fWhat & CPUMCTX_EXTRN_R8_R15) + { + ADD_REG64(WHvX64RegisterR8, pVCpu->cpum.GstCtx.r8); + ADD_REG64(WHvX64RegisterR9, pVCpu->cpum.GstCtx.r9); + ADD_REG64(WHvX64RegisterR10, pVCpu->cpum.GstCtx.r10); + ADD_REG64(WHvX64RegisterR11, pVCpu->cpum.GstCtx.r11); + ADD_REG64(WHvX64RegisterR12, pVCpu->cpum.GstCtx.r12); + ADD_REG64(WHvX64RegisterR13, pVCpu->cpum.GstCtx.r13); + ADD_REG64(WHvX64RegisterR14, pVCpu->cpum.GstCtx.r14); + ADD_REG64(WHvX64RegisterR15, pVCpu->cpum.GstCtx.r15); + } + } + + /* RIP & Flags */ + if (fWhat & CPUMCTX_EXTRN_RIP) + ADD_REG64(WHvX64RegisterRip, pVCpu->cpum.GstCtx.rip); + if (fWhat & CPUMCTX_EXTRN_RFLAGS) + ADD_REG64(WHvX64RegisterRflags, pVCpu->cpum.GstCtx.rflags.u); + + /* Segments */ +#define ADD_SEG(a_enmName, a_SReg) \ + do { \ + aenmNames[iReg] = a_enmName; \ + aValues[iReg].Segment.Base = (a_SReg).u64Base; \ + aValues[iReg].Segment.Limit = (a_SReg).u32Limit; \ + aValues[iReg].Segment.Selector = (a_SReg).Sel; \ + aValues[iReg].Segment.Attributes = (a_SReg).Attr.u; \ + iReg++; \ + } while (0) + if (fWhat & CPUMCTX_EXTRN_SREG_MASK) + { + if (fWhat & CPUMCTX_EXTRN_ES) + ADD_SEG(WHvX64RegisterEs, pVCpu->cpum.GstCtx.es); + if (fWhat & CPUMCTX_EXTRN_CS) + ADD_SEG(WHvX64RegisterCs, pVCpu->cpum.GstCtx.cs); + if (fWhat & CPUMCTX_EXTRN_SS) + ADD_SEG(WHvX64RegisterSs, pVCpu->cpum.GstCtx.ss); + if (fWhat & CPUMCTX_EXTRN_DS) + ADD_SEG(WHvX64RegisterDs, pVCpu->cpum.GstCtx.ds); + if (fWhat & CPUMCTX_EXTRN_FS) + ADD_SEG(WHvX64RegisterFs, pVCpu->cpum.GstCtx.fs); + if (fWhat & CPUMCTX_EXTRN_GS) + ADD_SEG(WHvX64RegisterGs, pVCpu->cpum.GstCtx.gs); + } + + /* Descriptor tables & task segment. */ + if (fWhat & CPUMCTX_EXTRN_TABLE_MASK) + { + if (fWhat & CPUMCTX_EXTRN_LDTR) + ADD_SEG(WHvX64RegisterLdtr, pVCpu->cpum.GstCtx.ldtr); + if (fWhat & CPUMCTX_EXTRN_TR) + ADD_SEG(WHvX64RegisterTr, pVCpu->cpum.GstCtx.tr); + if (fWhat & CPUMCTX_EXTRN_IDTR) + { + aenmNames[iReg] = WHvX64RegisterIdtr; + aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.idtr.cbIdt; + aValues[iReg].Table.Base = pVCpu->cpum.GstCtx.idtr.pIdt; + iReg++; + } + if (fWhat & CPUMCTX_EXTRN_GDTR) + { + aenmNames[iReg] = WHvX64RegisterGdtr; + aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.gdtr.cbGdt; + aValues[iReg].Table.Base = pVCpu->cpum.GstCtx.gdtr.pGdt; + iReg++; + } + } + + /* Control registers. */ + if (fWhat & CPUMCTX_EXTRN_CR_MASK) + { + if (fWhat & CPUMCTX_EXTRN_CR0) + ADD_REG64(WHvX64RegisterCr0, pVCpu->cpum.GstCtx.cr0); + if (fWhat & CPUMCTX_EXTRN_CR2) + ADD_REG64(WHvX64RegisterCr2, pVCpu->cpum.GstCtx.cr2); + if (fWhat & CPUMCTX_EXTRN_CR3) + ADD_REG64(WHvX64RegisterCr3, pVCpu->cpum.GstCtx.cr3); + if (fWhat & CPUMCTX_EXTRN_CR4) + ADD_REG64(WHvX64RegisterCr4, pVCpu->cpum.GstCtx.cr4); + } + if (fWhat & CPUMCTX_EXTRN_APIC_TPR) + ADD_REG64(WHvX64RegisterCr8, CPUMGetGuestCR8(pVCpu)); + + /* Debug registers. */ +/** @todo fixme. Figure out what the hyper-v version of KVM_SET_GUEST_DEBUG would be. */ + if (fWhat & CPUMCTX_EXTRN_DR0_DR3) + { + ADD_REG64(WHvX64RegisterDr0, pVCpu->cpum.GstCtx.dr[0]); // CPUMGetHyperDR0(pVCpu)); + ADD_REG64(WHvX64RegisterDr1, pVCpu->cpum.GstCtx.dr[1]); // CPUMGetHyperDR1(pVCpu)); + ADD_REG64(WHvX64RegisterDr2, pVCpu->cpum.GstCtx.dr[2]); // CPUMGetHyperDR2(pVCpu)); + ADD_REG64(WHvX64RegisterDr3, pVCpu->cpum.GstCtx.dr[3]); // CPUMGetHyperDR3(pVCpu)); + } + if (fWhat & CPUMCTX_EXTRN_DR6) + ADD_REG64(WHvX64RegisterDr6, pVCpu->cpum.GstCtx.dr[6]); // CPUMGetHyperDR6(pVCpu)); + if (fWhat & CPUMCTX_EXTRN_DR7) + ADD_REG64(WHvX64RegisterDr7, pVCpu->cpum.GstCtx.dr[7]); // CPUMGetHyperDR7(pVCpu)); + + /* Floating point state. */ + if (fWhat & CPUMCTX_EXTRN_X87) + { + ADD_REG128(WHvX64RegisterFpMmx0, pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[1]); + ADD_REG128(WHvX64RegisterFpMmx1, pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[1]); + ADD_REG128(WHvX64RegisterFpMmx2, pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[1]); + ADD_REG128(WHvX64RegisterFpMmx3, pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[1]); + ADD_REG128(WHvX64RegisterFpMmx4, pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[1]); + ADD_REG128(WHvX64RegisterFpMmx5, pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[1]); + ADD_REG128(WHvX64RegisterFpMmx6, pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[1]); + ADD_REG128(WHvX64RegisterFpMmx7, pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[1]); + + aenmNames[iReg] = WHvX64RegisterFpControlStatus; + aValues[iReg].FpControlStatus.FpControl = pVCpu->cpum.GstCtx.XState.x87.FCW; + aValues[iReg].FpControlStatus.FpStatus = pVCpu->cpum.GstCtx.XState.x87.FSW; + aValues[iReg].FpControlStatus.FpTag = pVCpu->cpum.GstCtx.XState.x87.FTW; + aValues[iReg].FpControlStatus.Reserved = pVCpu->cpum.GstCtx.XState.x87.FTW >> 8; + aValues[iReg].FpControlStatus.LastFpOp = pVCpu->cpum.GstCtx.XState.x87.FOP; + aValues[iReg].FpControlStatus.LastFpRip = (pVCpu->cpum.GstCtx.XState.x87.FPUIP) + | ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.CS << 32) + | ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.Rsrvd1 << 48); + iReg++; + + aenmNames[iReg] = WHvX64RegisterXmmControlStatus; + aValues[iReg].XmmControlStatus.LastFpRdp = (pVCpu->cpum.GstCtx.XState.x87.FPUDP) + | ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.DS << 32) + | ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.Rsrvd2 << 48); + aValues[iReg].XmmControlStatus.XmmStatusControl = pVCpu->cpum.GstCtx.XState.x87.MXCSR; + aValues[iReg].XmmControlStatus.XmmStatusControlMask = pVCpu->cpum.GstCtx.XState.x87.MXCSR_MASK; /** @todo ??? (Isn't this an output field?) */ + iReg++; + } + + /* Vector state. */ + if (fWhat & CPUMCTX_EXTRN_SSE_AVX) + { + ADD_REG128(WHvX64RegisterXmm0, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm1, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm2, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm3, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm4, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm5, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm6, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm7, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm8, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm9, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm11, pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm12, pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm13, pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm14, pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Hi); + ADD_REG128(WHvX64RegisterXmm15, pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Hi); + } + + /* MSRs */ + // WHvX64RegisterTsc - don't touch + if (fWhat & CPUMCTX_EXTRN_EFER) + ADD_REG64(WHvX64RegisterEfer, pVCpu->cpum.GstCtx.msrEFER); + if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE) + ADD_REG64(WHvX64RegisterKernelGsBase, pVCpu->cpum.GstCtx.msrKERNELGSBASE); + if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS) + { + ADD_REG64(WHvX64RegisterSysenterCs, pVCpu->cpum.GstCtx.SysEnter.cs); + ADD_REG64(WHvX64RegisterSysenterEip, pVCpu->cpum.GstCtx.SysEnter.eip); + ADD_REG64(WHvX64RegisterSysenterEsp, pVCpu->cpum.GstCtx.SysEnter.esp); + } + if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS) + { + ADD_REG64(WHvX64RegisterStar, pVCpu->cpum.GstCtx.msrSTAR); + ADD_REG64(WHvX64RegisterLstar, pVCpu->cpum.GstCtx.msrLSTAR); + ADD_REG64(WHvX64RegisterCstar, pVCpu->cpum.GstCtx.msrCSTAR); + ADD_REG64(WHvX64RegisterSfmask, pVCpu->cpum.GstCtx.msrSFMASK); + } + if (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS)) + { + PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu); + if (fWhat & CPUMCTX_EXTRN_TSC_AUX) + ADD_REG64(WHvX64RegisterTscAux, pCtxMsrs->msr.TscAux); + if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS) + { + ADD_REG64(WHvX64RegisterApicBase, APICGetBaseMsrNoCheck(pVCpu)); + ADD_REG64(WHvX64RegisterPat, pVCpu->cpum.GstCtx.msrPAT); +#if 0 /** @todo check if WHvX64RegisterMsrMtrrCap works here... */ + ADD_REG64(WHvX64RegisterMsrMtrrCap, CPUMGetGuestIa32MtrrCap(pVCpu)); +#endif + ADD_REG64(WHvX64RegisterMsrMtrrDefType, pCtxMsrs->msr.MtrrDefType); + ADD_REG64(WHvX64RegisterMsrMtrrFix64k00000, pCtxMsrs->msr.MtrrFix64K_00000); + ADD_REG64(WHvX64RegisterMsrMtrrFix16k80000, pCtxMsrs->msr.MtrrFix16K_80000); + ADD_REG64(WHvX64RegisterMsrMtrrFix16kA0000, pCtxMsrs->msr.MtrrFix16K_A0000); + ADD_REG64(WHvX64RegisterMsrMtrrFix4kC0000, pCtxMsrs->msr.MtrrFix4K_C0000); + ADD_REG64(WHvX64RegisterMsrMtrrFix4kC8000, pCtxMsrs->msr.MtrrFix4K_C8000); + ADD_REG64(WHvX64RegisterMsrMtrrFix4kD0000, pCtxMsrs->msr.MtrrFix4K_D0000); + ADD_REG64(WHvX64RegisterMsrMtrrFix4kD8000, pCtxMsrs->msr.MtrrFix4K_D8000); + ADD_REG64(WHvX64RegisterMsrMtrrFix4kE0000, pCtxMsrs->msr.MtrrFix4K_E0000); + ADD_REG64(WHvX64RegisterMsrMtrrFix4kE8000, pCtxMsrs->msr.MtrrFix4K_E8000); + ADD_REG64(WHvX64RegisterMsrMtrrFix4kF0000, pCtxMsrs->msr.MtrrFix4K_F0000); + ADD_REG64(WHvX64RegisterMsrMtrrFix4kF8000, pCtxMsrs->msr.MtrrFix4K_F8000); +#if 0 /** @todo these registers aren't available? Might explain something.. .*/ + const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM); + if (enmCpuVendor != CPUMCPUVENDOR_AMD) + { + ADD_REG64(HvX64RegisterIa32MiscEnable, pCtxMsrs->msr.MiscEnable); + ADD_REG64(HvX64RegisterIa32FeatureControl, CPUMGetGuestIa32FeatureControl(pVCpu)); + } +#endif + } + } + + /* event injection (clear it). */ + if (fWhat & CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT) + ADD_REG64(WHvRegisterPendingInterruption, 0); + + /* Interruptibility state. This can get a little complicated since we get + half of the state via HV_X64_VP_EXECUTION_STATE. */ + if ( (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI)) + == (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI) ) + { + ADD_REG64(WHvRegisterInterruptState, 0); + if (CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)) + aValues[iReg - 1].InterruptState.InterruptShadow = 1; + aValues[iReg - 1].InterruptState.NmiMasked = CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx); + } + else if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT) + { + if ( pVCpu->nem.s.fLastInterruptShadow + || CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)) + { + ADD_REG64(WHvRegisterInterruptState, 0); + if (CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)) + aValues[iReg - 1].InterruptState.InterruptShadow = 1; + /** @todo Retrieve NMI state, currently assuming it's zero. (yes this may happen on I/O) */ + //if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)) + // aValues[iReg - 1].InterruptState.NmiMasked = 1; + } + } + else + Assert(!(fWhat & CPUMCTX_EXTRN_INHIBIT_NMI)); + + /* Interrupt windows. Always set if active as Hyper-V seems to be forgetful. */ + uint8_t const fDesiredIntWin = pVCpu->nem.s.fDesiredInterruptWindows; + if ( fDesiredIntWin + || pVCpu->nem.s.fCurrentInterruptWindows != fDesiredIntWin) + { + pVCpu->nem.s.fCurrentInterruptWindows = pVCpu->nem.s.fDesiredInterruptWindows; + Log8(("Setting WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin=%X\n", fDesiredIntWin)); + ADD_REG64(WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin); + Assert(aValues[iReg - 1].DeliverabilityNotifications.NmiNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_NMI)); + Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_REGULAR)); + Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptPriority == (unsigned)((fDesiredIntWin & NEM_WIN_INTW_F_PRIO_MASK) >> NEM_WIN_INTW_F_PRIO_SHIFT)); + } + + /// @todo WHvRegisterPendingEvent + +#undef ADD_REG64 +#undef ADD_REG128 +#undef ADD_SEG + + /* + * Set the registers. + */ + Assert(iReg < RT_ELEMENTS(aValues)); + Assert(iReg < RT_ELEMENTS(aenmNames)); +#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS + Log12(("Calling WHvSetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n", + pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues)); +#endif + HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues); + if (SUCCEEDED(hrc)) + { + pVCpu->cpum.GstCtx.fExtrn |= CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK | CPUMCTX_EXTRN_KEEPER_NEM; + return VINF_SUCCESS; + } + AssertLogRelMsgFailed(("WHvSetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n", + pVM->nem.s.hPartition, pVCpu->idCpu, iReg, + hrc, RTNtLastStatusValue(), RTNtLastErrorValue())); + return VERR_INTERNAL_ERROR; +} + + +NEM_TMPL_STATIC int nemHCWinCopyStateFromHyperV(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat) +{ + WHV_REGISTER_NAME aenmNames[128]; + + fWhat &= pVCpu->cpum.GstCtx.fExtrn; + uintptr_t iReg = 0; + + /* GPRs */ + if (fWhat & CPUMCTX_EXTRN_GPRS_MASK) + { + if (fWhat & CPUMCTX_EXTRN_RAX) + aenmNames[iReg++] = WHvX64RegisterRax; + if (fWhat & CPUMCTX_EXTRN_RCX) + aenmNames[iReg++] = WHvX64RegisterRcx; + if (fWhat & CPUMCTX_EXTRN_RDX) + aenmNames[iReg++] = WHvX64RegisterRdx; + if (fWhat & CPUMCTX_EXTRN_RBX) + aenmNames[iReg++] = WHvX64RegisterRbx; + if (fWhat & CPUMCTX_EXTRN_RSP) + aenmNames[iReg++] = WHvX64RegisterRsp; + if (fWhat & CPUMCTX_EXTRN_RBP) + aenmNames[iReg++] = WHvX64RegisterRbp; + if (fWhat & CPUMCTX_EXTRN_RSI) + aenmNames[iReg++] = WHvX64RegisterRsi; + if (fWhat & CPUMCTX_EXTRN_RDI) + aenmNames[iReg++] = WHvX64RegisterRdi; + if (fWhat & CPUMCTX_EXTRN_R8_R15) + { + aenmNames[iReg++] = WHvX64RegisterR8; + aenmNames[iReg++] = WHvX64RegisterR9; + aenmNames[iReg++] = WHvX64RegisterR10; + aenmNames[iReg++] = WHvX64RegisterR11; + aenmNames[iReg++] = WHvX64RegisterR12; + aenmNames[iReg++] = WHvX64RegisterR13; + aenmNames[iReg++] = WHvX64RegisterR14; + aenmNames[iReg++] = WHvX64RegisterR15; + } + } + + /* RIP & Flags */ + if (fWhat & CPUMCTX_EXTRN_RIP) + aenmNames[iReg++] = WHvX64RegisterRip; + if (fWhat & CPUMCTX_EXTRN_RFLAGS) + aenmNames[iReg++] = WHvX64RegisterRflags; + + /* Segments */ + if (fWhat & CPUMCTX_EXTRN_SREG_MASK) + { + if (fWhat & CPUMCTX_EXTRN_ES) + aenmNames[iReg++] = WHvX64RegisterEs; + if (fWhat & CPUMCTX_EXTRN_CS) + aenmNames[iReg++] = WHvX64RegisterCs; + if (fWhat & CPUMCTX_EXTRN_SS) + aenmNames[iReg++] = WHvX64RegisterSs; + if (fWhat & CPUMCTX_EXTRN_DS) + aenmNames[iReg++] = WHvX64RegisterDs; + if (fWhat & CPUMCTX_EXTRN_FS) + aenmNames[iReg++] = WHvX64RegisterFs; + if (fWhat & CPUMCTX_EXTRN_GS) + aenmNames[iReg++] = WHvX64RegisterGs; + } + + /* Descriptor tables. */ + if (fWhat & CPUMCTX_EXTRN_TABLE_MASK) + { + if (fWhat & CPUMCTX_EXTRN_LDTR) + aenmNames[iReg++] = WHvX64RegisterLdtr; + if (fWhat & CPUMCTX_EXTRN_TR) + aenmNames[iReg++] = WHvX64RegisterTr; + if (fWhat & CPUMCTX_EXTRN_IDTR) + aenmNames[iReg++] = WHvX64RegisterIdtr; + if (fWhat & CPUMCTX_EXTRN_GDTR) + aenmNames[iReg++] = WHvX64RegisterGdtr; + } + + /* Control registers. */ + if (fWhat & CPUMCTX_EXTRN_CR_MASK) + { + if (fWhat & CPUMCTX_EXTRN_CR0) + aenmNames[iReg++] = WHvX64RegisterCr0; + if (fWhat & CPUMCTX_EXTRN_CR2) + aenmNames[iReg++] = WHvX64RegisterCr2; + if (fWhat & CPUMCTX_EXTRN_CR3) + aenmNames[iReg++] = WHvX64RegisterCr3; + if (fWhat & CPUMCTX_EXTRN_CR4) + aenmNames[iReg++] = WHvX64RegisterCr4; + } + if (fWhat & CPUMCTX_EXTRN_APIC_TPR) + aenmNames[iReg++] = WHvX64RegisterCr8; + + /* Debug registers. */ + if (fWhat & CPUMCTX_EXTRN_DR7) + aenmNames[iReg++] = WHvX64RegisterDr7; + if (fWhat & CPUMCTX_EXTRN_DR0_DR3) + { + if (!(fWhat & CPUMCTX_EXTRN_DR7) && (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_DR7)) + { + fWhat |= CPUMCTX_EXTRN_DR7; + aenmNames[iReg++] = WHvX64RegisterDr7; + } + aenmNames[iReg++] = WHvX64RegisterDr0; + aenmNames[iReg++] = WHvX64RegisterDr1; + aenmNames[iReg++] = WHvX64RegisterDr2; + aenmNames[iReg++] = WHvX64RegisterDr3; + } + if (fWhat & CPUMCTX_EXTRN_DR6) + aenmNames[iReg++] = WHvX64RegisterDr6; + + /* Floating point state. */ + if (fWhat & CPUMCTX_EXTRN_X87) + { + aenmNames[iReg++] = WHvX64RegisterFpMmx0; + aenmNames[iReg++] = WHvX64RegisterFpMmx1; + aenmNames[iReg++] = WHvX64RegisterFpMmx2; + aenmNames[iReg++] = WHvX64RegisterFpMmx3; + aenmNames[iReg++] = WHvX64RegisterFpMmx4; + aenmNames[iReg++] = WHvX64RegisterFpMmx5; + aenmNames[iReg++] = WHvX64RegisterFpMmx6; + aenmNames[iReg++] = WHvX64RegisterFpMmx7; + aenmNames[iReg++] = WHvX64RegisterFpControlStatus; + } + if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX)) + aenmNames[iReg++] = WHvX64RegisterXmmControlStatus; + + /* Vector state. */ + if (fWhat & CPUMCTX_EXTRN_SSE_AVX) + { + aenmNames[iReg++] = WHvX64RegisterXmm0; + aenmNames[iReg++] = WHvX64RegisterXmm1; + aenmNames[iReg++] = WHvX64RegisterXmm2; + aenmNames[iReg++] = WHvX64RegisterXmm3; + aenmNames[iReg++] = WHvX64RegisterXmm4; + aenmNames[iReg++] = WHvX64RegisterXmm5; + aenmNames[iReg++] = WHvX64RegisterXmm6; + aenmNames[iReg++] = WHvX64RegisterXmm7; + aenmNames[iReg++] = WHvX64RegisterXmm8; + aenmNames[iReg++] = WHvX64RegisterXmm9; + aenmNames[iReg++] = WHvX64RegisterXmm10; + aenmNames[iReg++] = WHvX64RegisterXmm11; + aenmNames[iReg++] = WHvX64RegisterXmm12; + aenmNames[iReg++] = WHvX64RegisterXmm13; + aenmNames[iReg++] = WHvX64RegisterXmm14; + aenmNames[iReg++] = WHvX64RegisterXmm15; + } + + /* MSRs */ + // WHvX64RegisterTsc - don't touch + if (fWhat & CPUMCTX_EXTRN_EFER) + aenmNames[iReg++] = WHvX64RegisterEfer; + if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE) + aenmNames[iReg++] = WHvX64RegisterKernelGsBase; + if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS) + { + aenmNames[iReg++] = WHvX64RegisterSysenterCs; + aenmNames[iReg++] = WHvX64RegisterSysenterEip; + aenmNames[iReg++] = WHvX64RegisterSysenterEsp; + } + if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS) + { + aenmNames[iReg++] = WHvX64RegisterStar; + aenmNames[iReg++] = WHvX64RegisterLstar; + aenmNames[iReg++] = WHvX64RegisterCstar; + aenmNames[iReg++] = WHvX64RegisterSfmask; + } + +//#ifdef LOG_ENABLED +// const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM); +//#endif + if (fWhat & CPUMCTX_EXTRN_TSC_AUX) + aenmNames[iReg++] = WHvX64RegisterTscAux; + if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS) + { + aenmNames[iReg++] = WHvX64RegisterApicBase; /// @todo APIC BASE + aenmNames[iReg++] = WHvX64RegisterPat; +#if 0 /*def LOG_ENABLED*/ /** @todo Check if WHvX64RegisterMsrMtrrCap works... */ + aenmNames[iReg++] = WHvX64RegisterMsrMtrrCap; +#endif + aenmNames[iReg++] = WHvX64RegisterMsrMtrrDefType; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix64k00000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16k80000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16kA0000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC0000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC8000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD0000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD8000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE0000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE8000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF0000; + aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF8000; + /** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */ +//#ifdef LOG_ENABLED +// if (enmCpuVendor != CPUMCPUVENDOR_AMD) +// aenmNames[iReg++] = HvX64RegisterIa32FeatureControl; +//#endif + } + + /* Interruptibility. */ + if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI)) + { + aenmNames[iReg++] = WHvRegisterInterruptState; + aenmNames[iReg++] = WHvX64RegisterRip; + } + + /* event injection */ + aenmNames[iReg++] = WHvRegisterPendingInterruption; + aenmNames[iReg++] = WHvRegisterPendingEvent; + + size_t const cRegs = iReg; + Assert(cRegs < RT_ELEMENTS(aenmNames)); + + /* + * Get the registers. + */ + WHV_REGISTER_VALUE aValues[128]; + RT_ZERO(aValues); + Assert(RT_ELEMENTS(aValues) >= cRegs); + Assert(RT_ELEMENTS(aenmNames) >= cRegs); +#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS + Log12(("Calling WHvGetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n", + pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues)); +#endif + HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, (uint32_t)cRegs, aValues); + AssertLogRelMsgReturn(SUCCEEDED(hrc), + ("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n", + pVM->nem.s.hPartition, pVCpu->idCpu, cRegs, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()) + , VERR_NEM_GET_REGISTERS_FAILED); + + iReg = 0; +#define GET_REG64(a_DstVar, a_enmName) do { \ + Assert(aenmNames[iReg] == (a_enmName)); \ + (a_DstVar) = aValues[iReg].Reg64; \ + iReg++; \ + } while (0) +#define GET_REG64_LOG7(a_DstVar, a_enmName, a_szLogName) do { \ + Assert(aenmNames[iReg] == (a_enmName)); \ + if ((a_DstVar) != aValues[iReg].Reg64) \ + Log7(("NEM/%u: " a_szLogName " changed %RX64 -> %RX64\n", pVCpu->idCpu, (a_DstVar), aValues[iReg].Reg64)); \ + (a_DstVar) = aValues[iReg].Reg64; \ + iReg++; \ + } while (0) +#define GET_REG128(a_DstVarLo, a_DstVarHi, a_enmName) do { \ + Assert(aenmNames[iReg] == a_enmName); \ + (a_DstVarLo) = aValues[iReg].Reg128.Low64; \ + (a_DstVarHi) = aValues[iReg].Reg128.High64; \ + iReg++; \ + } while (0) +#define GET_SEG(a_SReg, a_enmName) do { \ + Assert(aenmNames[iReg] == (a_enmName)); \ + NEM_WIN_COPY_BACK_SEG(a_SReg, aValues[iReg].Segment); \ + iReg++; \ + } while (0) + + /* GPRs */ + if (fWhat & CPUMCTX_EXTRN_GPRS_MASK) + { + if (fWhat & CPUMCTX_EXTRN_RAX) + GET_REG64(pVCpu->cpum.GstCtx.rax, WHvX64RegisterRax); + if (fWhat & CPUMCTX_EXTRN_RCX) + GET_REG64(pVCpu->cpum.GstCtx.rcx, WHvX64RegisterRcx); + if (fWhat & CPUMCTX_EXTRN_RDX) + GET_REG64(pVCpu->cpum.GstCtx.rdx, WHvX64RegisterRdx); + if (fWhat & CPUMCTX_EXTRN_RBX) + GET_REG64(pVCpu->cpum.GstCtx.rbx, WHvX64RegisterRbx); + if (fWhat & CPUMCTX_EXTRN_RSP) + GET_REG64(pVCpu->cpum.GstCtx.rsp, WHvX64RegisterRsp); + if (fWhat & CPUMCTX_EXTRN_RBP) + GET_REG64(pVCpu->cpum.GstCtx.rbp, WHvX64RegisterRbp); + if (fWhat & CPUMCTX_EXTRN_RSI) + GET_REG64(pVCpu->cpum.GstCtx.rsi, WHvX64RegisterRsi); + if (fWhat & CPUMCTX_EXTRN_RDI) + GET_REG64(pVCpu->cpum.GstCtx.rdi, WHvX64RegisterRdi); + if (fWhat & CPUMCTX_EXTRN_R8_R15) + { + GET_REG64(pVCpu->cpum.GstCtx.r8, WHvX64RegisterR8); + GET_REG64(pVCpu->cpum.GstCtx.r9, WHvX64RegisterR9); + GET_REG64(pVCpu->cpum.GstCtx.r10, WHvX64RegisterR10); + GET_REG64(pVCpu->cpum.GstCtx.r11, WHvX64RegisterR11); + GET_REG64(pVCpu->cpum.GstCtx.r12, WHvX64RegisterR12); + GET_REG64(pVCpu->cpum.GstCtx.r13, WHvX64RegisterR13); + GET_REG64(pVCpu->cpum.GstCtx.r14, WHvX64RegisterR14); + GET_REG64(pVCpu->cpum.GstCtx.r15, WHvX64RegisterR15); + } + } + + /* RIP & Flags */ + if (fWhat & CPUMCTX_EXTRN_RIP) + GET_REG64(pVCpu->cpum.GstCtx.rip, WHvX64RegisterRip); + if (fWhat & CPUMCTX_EXTRN_RFLAGS) + GET_REG64(pVCpu->cpum.GstCtx.rflags.u, WHvX64RegisterRflags); + + /* Segments */ + if (fWhat & CPUMCTX_EXTRN_SREG_MASK) + { + if (fWhat & CPUMCTX_EXTRN_ES) + GET_SEG(pVCpu->cpum.GstCtx.es, WHvX64RegisterEs); + if (fWhat & CPUMCTX_EXTRN_CS) + GET_SEG(pVCpu->cpum.GstCtx.cs, WHvX64RegisterCs); + if (fWhat & CPUMCTX_EXTRN_SS) + GET_SEG(pVCpu->cpum.GstCtx.ss, WHvX64RegisterSs); + if (fWhat & CPUMCTX_EXTRN_DS) + GET_SEG(pVCpu->cpum.GstCtx.ds, WHvX64RegisterDs); + if (fWhat & CPUMCTX_EXTRN_FS) + GET_SEG(pVCpu->cpum.GstCtx.fs, WHvX64RegisterFs); + if (fWhat & CPUMCTX_EXTRN_GS) + GET_SEG(pVCpu->cpum.GstCtx.gs, WHvX64RegisterGs); + } + + /* Descriptor tables and the task segment. */ + if (fWhat & CPUMCTX_EXTRN_TABLE_MASK) + { + if (fWhat & CPUMCTX_EXTRN_LDTR) + GET_SEG(pVCpu->cpum.GstCtx.ldtr, WHvX64RegisterLdtr); + + if (fWhat & CPUMCTX_EXTRN_TR) + { + /* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So, + avoid to trigger sanity assertions around the code, always fix this. */ + GET_SEG(pVCpu->cpum.GstCtx.tr, WHvX64RegisterTr); + switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type) + { + case X86_SEL_TYPE_SYS_386_TSS_BUSY: + case X86_SEL_TYPE_SYS_286_TSS_BUSY: + break; + case X86_SEL_TYPE_SYS_386_TSS_AVAIL: + pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY; + break; + case X86_SEL_TYPE_SYS_286_TSS_AVAIL: + pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY; + break; + } + } + if (fWhat & CPUMCTX_EXTRN_IDTR) + { + Assert(aenmNames[iReg] == WHvX64RegisterIdtr); + pVCpu->cpum.GstCtx.idtr.cbIdt = aValues[iReg].Table.Limit; + pVCpu->cpum.GstCtx.idtr.pIdt = aValues[iReg].Table.Base; + iReg++; + } + if (fWhat & CPUMCTX_EXTRN_GDTR) + { + Assert(aenmNames[iReg] == WHvX64RegisterGdtr); + pVCpu->cpum.GstCtx.gdtr.cbGdt = aValues[iReg].Table.Limit; + pVCpu->cpum.GstCtx.gdtr.pGdt = aValues[iReg].Table.Base; + iReg++; + } + } + + /* Control registers. */ + bool fMaybeChangedMode = false; + bool fUpdateCr3 = false; + if (fWhat & CPUMCTX_EXTRN_CR_MASK) + { + if (fWhat & CPUMCTX_EXTRN_CR0) + { + Assert(aenmNames[iReg] == WHvX64RegisterCr0); + if (pVCpu->cpum.GstCtx.cr0 != aValues[iReg].Reg64) + { + CPUMSetGuestCR0(pVCpu, aValues[iReg].Reg64); + fMaybeChangedMode = true; + } + iReg++; + } + if (fWhat & CPUMCTX_EXTRN_CR2) + GET_REG64(pVCpu->cpum.GstCtx.cr2, WHvX64RegisterCr2); + if (fWhat & CPUMCTX_EXTRN_CR3) + { + if (pVCpu->cpum.GstCtx.cr3 != aValues[iReg].Reg64) + { + CPUMSetGuestCR3(pVCpu, aValues[iReg].Reg64); + fUpdateCr3 = true; + } + iReg++; + } + if (fWhat & CPUMCTX_EXTRN_CR4) + { + if (pVCpu->cpum.GstCtx.cr4 != aValues[iReg].Reg64) + { + CPUMSetGuestCR4(pVCpu, aValues[iReg].Reg64); + fMaybeChangedMode = true; + } + iReg++; + } + } + if (fWhat & CPUMCTX_EXTRN_APIC_TPR) + { + Assert(aenmNames[iReg] == WHvX64RegisterCr8); + APICSetTpr(pVCpu, (uint8_t)aValues[iReg].Reg64 << 4); + iReg++; + } + + /* Debug registers. */ + if (fWhat & CPUMCTX_EXTRN_DR7) + { + Assert(aenmNames[iReg] == WHvX64RegisterDr7); + if (pVCpu->cpum.GstCtx.dr[7] != aValues[iReg].Reg64) + CPUMSetGuestDR7(pVCpu, aValues[iReg].Reg64); + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */ + iReg++; + } + if (fWhat & CPUMCTX_EXTRN_DR0_DR3) + { + Assert(aenmNames[iReg] == WHvX64RegisterDr0); + Assert(aenmNames[iReg+3] == WHvX64RegisterDr3); + if (pVCpu->cpum.GstCtx.dr[0] != aValues[iReg].Reg64) + CPUMSetGuestDR0(pVCpu, aValues[iReg].Reg64); + iReg++; + if (pVCpu->cpum.GstCtx.dr[1] != aValues[iReg].Reg64) + CPUMSetGuestDR1(pVCpu, aValues[iReg].Reg64); + iReg++; + if (pVCpu->cpum.GstCtx.dr[2] != aValues[iReg].Reg64) + CPUMSetGuestDR2(pVCpu, aValues[iReg].Reg64); + iReg++; + if (pVCpu->cpum.GstCtx.dr[3] != aValues[iReg].Reg64) + CPUMSetGuestDR3(pVCpu, aValues[iReg].Reg64); + iReg++; + } + if (fWhat & CPUMCTX_EXTRN_DR6) + { + Assert(aenmNames[iReg] == WHvX64RegisterDr6); + if (pVCpu->cpum.GstCtx.dr[6] != aValues[iReg].Reg64) + CPUMSetGuestDR6(pVCpu, aValues[iReg].Reg64); + iReg++; + } + + /* Floating point state. */ + if (fWhat & CPUMCTX_EXTRN_X87) + { + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[1], WHvX64RegisterFpMmx0); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[1], WHvX64RegisterFpMmx1); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[1], WHvX64RegisterFpMmx2); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[1], WHvX64RegisterFpMmx3); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[1], WHvX64RegisterFpMmx4); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[1], WHvX64RegisterFpMmx5); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[1], WHvX64RegisterFpMmx6); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[1], WHvX64RegisterFpMmx7); + + Assert(aenmNames[iReg] == WHvX64RegisterFpControlStatus); + pVCpu->cpum.GstCtx.XState.x87.FCW = aValues[iReg].FpControlStatus.FpControl; + pVCpu->cpum.GstCtx.XState.x87.FSW = aValues[iReg].FpControlStatus.FpStatus; + pVCpu->cpum.GstCtx.XState.x87.FTW = aValues[iReg].FpControlStatus.FpTag + /*| (aValues[iReg].FpControlStatus.Reserved << 8)*/; + pVCpu->cpum.GstCtx.XState.x87.FOP = aValues[iReg].FpControlStatus.LastFpOp; + pVCpu->cpum.GstCtx.XState.x87.FPUIP = (uint32_t)aValues[iReg].FpControlStatus.LastFpRip; + pVCpu->cpum.GstCtx.XState.x87.CS = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 32); + pVCpu->cpum.GstCtx.XState.x87.Rsrvd1 = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 48); + iReg++; + } + + if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX)) + { + Assert(aenmNames[iReg] == WHvX64RegisterXmmControlStatus); + if (fWhat & CPUMCTX_EXTRN_X87) + { + pVCpu->cpum.GstCtx.XState.x87.FPUDP = (uint32_t)aValues[iReg].XmmControlStatus.LastFpRdp; + pVCpu->cpum.GstCtx.XState.x87.DS = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 32); + pVCpu->cpum.GstCtx.XState.x87.Rsrvd2 = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 48); + } + pVCpu->cpum.GstCtx.XState.x87.MXCSR = aValues[iReg].XmmControlStatus.XmmStatusControl; + pVCpu->cpum.GstCtx.XState.x87.MXCSR_MASK = aValues[iReg].XmmControlStatus.XmmStatusControlMask; /** @todo ??? (Isn't this an output field?) */ + iReg++; + } + + /* Vector state. */ + if (fWhat & CPUMCTX_EXTRN_SSE_AVX) + { + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Hi, WHvX64RegisterXmm0); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Hi, WHvX64RegisterXmm1); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Hi, WHvX64RegisterXmm2); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Hi, WHvX64RegisterXmm3); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Hi, WHvX64RegisterXmm4); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Hi, WHvX64RegisterXmm5); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Hi, WHvX64RegisterXmm6); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Hi, WHvX64RegisterXmm7); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Hi, WHvX64RegisterXmm8); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Hi, WHvX64RegisterXmm9); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Hi, WHvX64RegisterXmm10); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Hi, WHvX64RegisterXmm11); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Hi, WHvX64RegisterXmm12); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Hi, WHvX64RegisterXmm13); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Hi, WHvX64RegisterXmm14); + GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Hi, WHvX64RegisterXmm15); + } + + /* MSRs */ + // WHvX64RegisterTsc - don't touch + if (fWhat & CPUMCTX_EXTRN_EFER) + { + Assert(aenmNames[iReg] == WHvX64RegisterEfer); + if (aValues[iReg].Reg64 != pVCpu->cpum.GstCtx.msrEFER) + { + Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, aValues[iReg].Reg64)); + if ((aValues[iReg].Reg64 ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE) + PGMNotifyNxeChanged(pVCpu, RT_BOOL(aValues[iReg].Reg64 & MSR_K6_EFER_NXE)); + pVCpu->cpum.GstCtx.msrEFER = aValues[iReg].Reg64; + fMaybeChangedMode = true; + } + iReg++; + } + if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE) + GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrKERNELGSBASE, WHvX64RegisterKernelGsBase, "MSR KERNEL_GS_BASE"); + if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS) + { + GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.cs, WHvX64RegisterSysenterCs, "MSR SYSENTER.CS"); + GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.eip, WHvX64RegisterSysenterEip, "MSR SYSENTER.EIP"); + GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.esp, WHvX64RegisterSysenterEsp, "MSR SYSENTER.ESP"); + } + if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS) + { + GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSTAR, WHvX64RegisterStar, "MSR STAR"); + GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrLSTAR, WHvX64RegisterLstar, "MSR LSTAR"); + GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrCSTAR, WHvX64RegisterCstar, "MSR CSTAR"); + GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSFMASK, WHvX64RegisterSfmask, "MSR SFMASK"); + } + if (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS)) + { + PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu); + if (fWhat & CPUMCTX_EXTRN_TSC_AUX) + GET_REG64_LOG7(pCtxMsrs->msr.TscAux, WHvX64RegisterTscAux, "MSR TSC_AUX"); + if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS) + { + Assert(aenmNames[iReg] == WHvX64RegisterApicBase); + const uint64_t uOldBase = APICGetBaseMsrNoCheck(pVCpu); + if (aValues[iReg].Reg64 != uOldBase) + { + Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n", + pVCpu->idCpu, uOldBase, aValues[iReg].Reg64, aValues[iReg].Reg64 ^ uOldBase)); + int rc2 = APICSetBaseMsr(pVCpu, aValues[iReg].Reg64); + AssertLogRelMsg(rc2 == VINF_SUCCESS, ("%Rrc %RX64\n", rc2, aValues[iReg].Reg64)); + } + iReg++; + + GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterPat, "MSR PAT"); +#if 0 /*def LOG_ENABLED*/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */ + GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterMsrMtrrCap); +#endif + GET_REG64_LOG7(pCtxMsrs->msr.MtrrDefType, WHvX64RegisterMsrMtrrDefType, "MSR MTRR_DEF_TYPE"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix64K_00000, WHvX64RegisterMsrMtrrFix64k00000, "MSR MTRR_FIX_64K_00000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_80000, WHvX64RegisterMsrMtrrFix16k80000, "MSR MTRR_FIX_16K_80000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_A0000, WHvX64RegisterMsrMtrrFix16kA0000, "MSR MTRR_FIX_16K_A0000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C0000, WHvX64RegisterMsrMtrrFix4kC0000, "MSR MTRR_FIX_4K_C0000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C8000, WHvX64RegisterMsrMtrrFix4kC8000, "MSR MTRR_FIX_4K_C8000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D0000, WHvX64RegisterMsrMtrrFix4kD0000, "MSR MTRR_FIX_4K_D0000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D8000, WHvX64RegisterMsrMtrrFix4kD8000, "MSR MTRR_FIX_4K_D8000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E0000, WHvX64RegisterMsrMtrrFix4kE0000, "MSR MTRR_FIX_4K_E0000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E8000, WHvX64RegisterMsrMtrrFix4kE8000, "MSR MTRR_FIX_4K_E8000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F0000, WHvX64RegisterMsrMtrrFix4kF0000, "MSR MTRR_FIX_4K_F0000"); + GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F8000, WHvX64RegisterMsrMtrrFix4kF8000, "MSR MTRR_FIX_4K_F8000"); + /** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */ + } + } + + /* Interruptibility. */ + if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI)) + { + Assert(aenmNames[iReg] == WHvRegisterInterruptState); + Assert(aenmNames[iReg + 1] == WHvX64RegisterRip); + + if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_INT)) + pVCpu->nem.s.fLastInterruptShadow = CPUMUpdateInterruptShadowEx(&pVCpu->cpum.GstCtx, + aValues[iReg].InterruptState.InterruptShadow, + aValues[iReg + 1].Reg64); + + if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_NMI)) + CPUMUpdateInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx, aValues[iReg].InterruptState.NmiMasked); + + fWhat |= CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI; + iReg += 2; + } + + /* Event injection. */ + /// @todo WHvRegisterPendingInterruption + Assert(aenmNames[iReg] == WHvRegisterPendingInterruption); + if (aValues[iReg].PendingInterruption.InterruptionPending) + { + Log7(("PendingInterruption: type=%u vector=%#x errcd=%RTbool/%#x instr-len=%u nested=%u\n", + aValues[iReg].PendingInterruption.InterruptionType, aValues[iReg].PendingInterruption.InterruptionVector, + aValues[iReg].PendingInterruption.DeliverErrorCode, aValues[iReg].PendingInterruption.ErrorCode, + aValues[iReg].PendingInterruption.InstructionLength, aValues[iReg].PendingInterruption.NestedEvent)); + AssertMsg((aValues[iReg].PendingInterruption.AsUINT64 & UINT64_C(0xfc00)) == 0, + ("%#RX64\n", aValues[iReg].PendingInterruption.AsUINT64)); + } + + /// @todo WHvRegisterPendingEvent + + /* Almost done, just update extrn flags and maybe change PGM mode. */ + pVCpu->cpum.GstCtx.fExtrn &= ~fWhat; + if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))) + pVCpu->cpum.GstCtx.fExtrn = 0; + + /* Typical. */ + if (!fMaybeChangedMode && !fUpdateCr3) + return VINF_SUCCESS; + + /* + * Slow. + */ + if (fMaybeChangedMode) + { + int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER, + false /* fForce */); + AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1); + } + + if (fUpdateCr3) + { + int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3); + if (rc == VINF_SUCCESS) + { /* likely */ } + else + AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2); + } + + return VINF_SUCCESS; +} + + +/** + * Interface for importing state on demand (used by IEM). + * + * @returns VBox status code. + * @param pVCpu The cross context CPU structure. + * @param fWhat What to import, CPUMCTX_EXTRN_XXX. + */ +VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat) +{ + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand); + return nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat); +} + + +/** + * Query the CPU tick counter and optionally the TSC_AUX MSR value. + * + * @returns VBox status code. + * @param pVCpu The cross context CPU structure. + * @param pcTicks Where to return the CPU tick count. + * @param puAux Where to return the TSC_AUX register value. + */ +VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux) +{ + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT); + AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9); + + /* Call the offical API. */ + WHV_REGISTER_NAME aenmNames[2] = { WHvX64RegisterTsc, WHvX64RegisterTscAux }; + WHV_REGISTER_VALUE aValues[2] = { { {0, 0} }, { {0, 0} } }; + Assert(RT_ELEMENTS(aenmNames) == RT_ELEMENTS(aValues)); + HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, 2, aValues); + AssertLogRelMsgReturn(SUCCEEDED(hrc), + ("WHvGetVirtualProcessorRegisters(%p, %u,{tsc,tsc_aux},2,) -> %Rhrc (Last=%#x/%u)\n", + pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()) + , VERR_NEM_GET_REGISTERS_FAILED); + *pcTicks = aValues[0].Reg64; + if (puAux) + *puAux = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX ? aValues[1].Reg64 : CPUMGetGuestTscAux(pVCpu); + return VINF_SUCCESS; +} + + +/** + * Resumes CPU clock (TSC) on all virtual CPUs. + * + * This is called by TM when the VM is started, restored, resumed or similar. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context CPU structure of the calling EMT. + * @param uPausedTscValue The TSC value at the time of pausing. + */ +VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue) +{ + VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT); + AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9); + + /* + * Call the offical API to do the job. + */ + if (pVM->cCpus > 1) + RTThreadYield(); /* Try decrease the chance that we get rescheduled in the middle. */ + + /* Start with the first CPU. */ + WHV_REGISTER_NAME enmName = WHvX64RegisterTsc; + WHV_REGISTER_VALUE Value = { {0, 0} }; + Value.Reg64 = uPausedTscValue; + uint64_t const uFirstTsc = ASMReadTSC(); + HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, 0 /*iCpu*/, &enmName, 1, &Value); + AssertLogRelMsgReturn(SUCCEEDED(hrc), + ("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64) -> %Rhrc (Last=%#x/%u)\n", + pVM->nem.s.hPartition, uPausedTscValue, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()) + , VERR_NEM_SET_TSC); + + /* Do the other CPUs, adjusting for elapsed TSC and keeping finger crossed + that we don't introduce too much drift here. */ + for (VMCPUID iCpu = 1; iCpu < pVM->cCpus; iCpu++) + { + Assert(enmName == WHvX64RegisterTsc); + const uint64_t offDelta = (ASMReadTSC() - uFirstTsc); + Value.Reg64 = uPausedTscValue + offDelta; + hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, iCpu, &enmName, 1, &Value); + AssertLogRelMsgReturn(SUCCEEDED(hrc), + ("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64 + %#RX64) -> %Rhrc (Last=%#x/%u)\n", + pVM->nem.s.hPartition, iCpu, uPausedTscValue, offDelta, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()) + , VERR_NEM_SET_TSC); + } + + return VINF_SUCCESS; +} + +#ifdef LOG_ENABLED + +/** + * Get the virtual processor running status. + */ +DECLINLINE(VID_PROCESSOR_STATUS) nemHCWinCpuGetRunningStatus(PVMCPUCC pVCpu) +{ + RTERRVARS Saved; + RTErrVarsSave(&Saved); + + /* + * This API is disabled in release builds, it seems. On build 17101 it requires + * the following patch to be enabled (windbg): eb vid+12180 0f 84 98 00 00 00 + */ + VID_PROCESSOR_STATUS enmCpuStatus = VidProcessorStatusUndefined; + NTSTATUS rcNt = g_pfnVidGetVirtualProcessorRunningStatus(pVCpu->pVMR3->nem.s.hPartitionDevice, pVCpu->idCpu, &enmCpuStatus); + AssertRC(rcNt); + + RTErrVarsRestore(&Saved); + return enmCpuStatus; +} + + +/** + * Logs the current CPU state. + */ +NEM_TMPL_STATIC void nemHCWinLogState(PVMCC pVM, PVMCPUCC pVCpu) +{ + if (LogIs3Enabled()) + { +# if 0 // def IN_RING3 - causes lazy state import assertions all over CPUM. + char szRegs[4096]; + DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs), + "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n" + "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n" + "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n" + "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n" + "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n" + "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n" + "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n" + "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n" + "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n" + "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n" + "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n" + "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n" + "dr6=%016VR{dr6} dr7=%016VR{dr7}\n" + "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n" + "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n" + "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n" + " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n" + " efer=%016VR{efer}\n" + " pat=%016VR{pat}\n" + " sf_mask=%016VR{sf_mask}\n" + "krnl_gs_base=%016VR{krnl_gs_base}\n" + " lstar=%016VR{lstar}\n" + " star=%016VR{star} cstar=%016VR{cstar}\n" + "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n" + ); + + char szInstr[256]; + DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0, + DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE, + szInstr, sizeof(szInstr), NULL); + Log3(("%s%s\n", szRegs, szInstr)); +# else + /** @todo stat logging in ring-0 */ + RT_NOREF(pVM, pVCpu); +# endif + } +} + +#endif /* LOG_ENABLED */ + +/** + * Translates the execution stat bitfield into a short log string, WinHv version. + * + * @returns Read-only log string. + * @param pExitCtx The exit context which state to summarize. + */ +static const char *nemR3WinExecStateToLogStr(WHV_VP_EXIT_CONTEXT const *pExitCtx) +{ + unsigned u = (unsigned)pExitCtx->ExecutionState.InterruptionPending + | ((unsigned)pExitCtx->ExecutionState.DebugActive << 1) + | ((unsigned)pExitCtx->ExecutionState.InterruptShadow << 2); +#define SWITCH_IT(a_szPrefix) \ + do \ + switch (u)\ + { \ + case 0x00: return a_szPrefix ""; \ + case 0x01: return a_szPrefix ",Pnd"; \ + case 0x02: return a_szPrefix ",Dbg"; \ + case 0x03: return a_szPrefix ",Pnd,Dbg"; \ + case 0x04: return a_szPrefix ",Shw"; \ + case 0x05: return a_szPrefix ",Pnd,Shw"; \ + case 0x06: return a_szPrefix ",Shw,Dbg"; \ + case 0x07: return a_szPrefix ",Pnd,Shw,Dbg"; \ + default: AssertFailedReturn("WTF?"); \ + } \ + while (0) + if (pExitCtx->ExecutionState.EferLma) + SWITCH_IT("LM"); + else if (pExitCtx->ExecutionState.Cr0Pe) + SWITCH_IT("PM"); + else + SWITCH_IT("RM"); +#undef SWITCH_IT +} + + +/** + * Advances the guest RIP and clear EFLAGS.RF, WinHv version. + * + * This may clear VMCPU_FF_INHIBIT_INTERRUPTS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pExitCtx The exit context. + * @param cbMinInstr The minimum instruction length, or 1 if not unknown. + */ +DECLINLINE(void) nemR3WinAdvanceGuestRipAndClearRF(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx, uint8_t cbMinInstr) +{ + Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS))); + + /* Advance the RIP. */ + Assert(pExitCtx->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr); + pVCpu->cpum.GstCtx.rip += pExitCtx->InstructionLength; + pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0; + CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); +} + + +/** + * State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess + * and nemHCWinHandleMemoryAccessPageCheckerCallback. + */ +typedef struct NEMHCWINHMACPCCSTATE +{ + /** Input: Write access. */ + bool fWriteAccess; + /** Output: Set if we did something. */ + bool fDidSomething; + /** Output: Set it we should resume. */ + bool fCanResume; +} NEMHCWINHMACPCCSTATE; + +/** + * @callback_method_impl{FNPGMPHYSNEMCHECKPAGE, + * Worker for nemR3WinHandleMemoryAccess; pvUser points to a + * NEMHCWINHMACPCCSTATE structure. } + */ +NEM_TMPL_STATIC DECLCALLBACK(int) +nemHCWinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser) +{ + NEMHCWINHMACPCCSTATE *pState = (NEMHCWINHMACPCCSTATE *)pvUser; + pState->fDidSomething = false; + pState->fCanResume = false; + + /* If A20 is disabled, we may need to make another query on the masked + page to get the correct protection information. */ + uint8_t u2State = pInfo->u2NemState; + RTGCPHYS GCPhysSrc; +#ifdef NEM_WIN_WITH_A20 + if ( pVM->nem.s.fA20Enabled + || !NEM_WIN_IS_SUBJECT_TO_A20(GCPhys)) +#endif + GCPhysSrc = GCPhys; +#ifdef NEM_WIN_WITH_A20 + else + { + GCPhysSrc = GCPhys & ~(RTGCPHYS)RT_BIT_32(20); + PGMPHYSNEMPAGEINFO Info2; + int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhysSrc, pState->fWriteAccess, &Info2, NULL, NULL); + AssertRCReturn(rc, rc); + + *pInfo = Info2; + pInfo->u2NemState = u2State; + } +#endif + + /* + * Consolidate current page state with actual page protection and access type. + * We don't really consider downgrades here, as they shouldn't happen. + */ + /** @todo Someone at microsoft please explain: + * I'm not sure WTF was going on, but I ended up in a loop if I remapped a + * readonly page as writable (unmap, then map again). Specifically, this was an + * issue with the big VRAM mapping at 0xe0000000 when booing DSL 4.4.1. So, in + * a hope to work around that we no longer pre-map anything, just unmap stuff + * and do it lazily here. And here we will first unmap, restart, and then remap + * with new protection or backing. + */ + int rc; + switch (u2State) + { + case NEM_WIN_PAGE_STATE_UNMAPPED: + case NEM_WIN_PAGE_STATE_NOT_SET: + if (pInfo->fNemProt == NEM_PAGE_PROT_NONE) + { + Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys)); + return VINF_SUCCESS; + } + + /* Don't bother remapping it if it's a write request to a non-writable page. */ + if ( pState->fWriteAccess + && !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)) + { + Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys)); + return VINF_SUCCESS; + } + + /* Map the page. */ + rc = nemHCNativeSetPhysPage(pVM, + pVCpu, + GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, + GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, + pInfo->fNemProt, + &u2State, + true /*fBackingState*/); + pInfo->u2NemState = u2State; + Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n", + GCPhys, g_apszPageStates[u2State], rc)); + pState->fDidSomething = true; + pState->fCanResume = true; + return rc; + + case NEM_WIN_PAGE_STATE_READABLE: + if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE) + && (pInfo->fNemProt & (NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE))) + { + Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys)); + return VINF_SUCCESS; + } + + break; + + case NEM_WIN_PAGE_STATE_WRITABLE: + if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE) + { + if (pInfo->u2OldNemState == NEM_WIN_PAGE_STATE_WRITABLE) + Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3a\n", GCPhys)); + else + { + pState->fCanResume = true; + Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3b (%s -> %s)\n", + GCPhys, g_apszPageStates[pInfo->u2OldNemState], g_apszPageStates[u2State])); + } + return VINF_SUCCESS; + } + break; + + default: + AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4); + } + + /* + * Unmap and restart the instruction. + * If this fails, which it does every so often, just unmap everything for now. + */ + /** @todo figure out whether we mess up the state or if it's WHv. */ + STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a); + HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE); + STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a); + if (SUCCEEDED(hrc)) + { + pState->fDidSomething = true; + pState->fCanResume = true; + pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED; + STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage); + uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages); + Log5(("NEM GPA unmapped/exit: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[u2State], cMappedPages)); + return VINF_SUCCESS; + } + STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed); + LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s hrc=%Rhrc (%#x)\n", + GCPhys, g_apszPageStates[u2State], hrc, hrc)); + return VERR_NEM_UNMAP_PAGES_FAILED; +} + + +/** + * Wrapper around nemHCWinCopyStateFromHyperV. + * + * Unlike the wrapped APIs, this checks whether it's necessary. + * + * @returns VBox strict status code. + * @param pVCpu The cross context per CPU structure. + * @param fWhat What to import. + * @param pszCaller Who is doing the importing. + */ +DECLINLINE(VBOXSTRICTRC) nemHCWinImportStateIfNeededStrict(PVMCPUCC pVCpu, uint64_t fWhat, const char *pszCaller) +{ + if (pVCpu->cpum.GstCtx.fExtrn & fWhat) + { + RT_NOREF(pszCaller); + int rc = nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat); + AssertRCReturn(rc, rc); + } + return VINF_SUCCESS; +} + + +/** + * Copies register state from the (common) exit context. + * + * ASSUMES no state copied yet. + * + * @param pVCpu The cross context per CPU structure. + * @param pExitCtx The common exit context. + * @sa nemHCWinCopyStateFromX64Header + */ +DECLINLINE(void) nemR3WinCopyStateFromX64Header(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx) +{ + Assert( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_INHIBIT_INT)) + == (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_INHIBIT_INT)); + + NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pExitCtx->Cs); + pVCpu->cpum.GstCtx.rip = pExitCtx->Rip; + pVCpu->cpum.GstCtx.rflags.u = pExitCtx->Rflags; + pVCpu->nem.s.fLastInterruptShadow = CPUMUpdateInterruptShadowEx(&pVCpu->cpum.GstCtx, + pExitCtx->ExecutionState.InterruptShadow, + pExitCtx->Rip); + APICSetTpr(pVCpu, pExitCtx->Cr8 << 4); + + pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_APIC_TPR); +} + + +/** + * Deals with memory access exits (WHvRunVpExitReasonMemoryAccess). + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + * @param pExit The VM exit information to handle. + * @sa nemHCWinHandleMessageMemory + */ +NEM_TMPL_STATIC VBOXSTRICTRC +nemR3WinHandleExitMemory(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit) +{ + uint64_t const uHostTsc = ASMReadTSC(); + Assert(pExit->MemoryAccess.AccessInfo.AccessType != 3); + + /* + * Whatever we do, we must clear pending event injection upon resume. + */ + if (pExit->VpContext.ExecutionState.InterruptionPending) + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; + + /* + * Ask PGM for information about the given GCPhys. We need to check if we're + * out of sync first. + */ + NEMHCWINHMACPCCSTATE State = { pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite, false, false }; + PGMPHYSNEMPAGEINFO Info; + int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pExit->MemoryAccess.Gpa, State.fWriteAccess, &Info, + nemHCWinHandleMemoryAccessPageCheckerCallback, &State); + if (RT_SUCCESS(rc)) + { + if (Info.fNemProt & ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite + ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ)) + { + if (State.fCanResume) + { + Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt, + Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "", + State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType])); + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc); + return VINF_SUCCESS; + } + } + Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt, + Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "", + State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType])); + } + else + Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->MemoryAccess.Gpa, rc, State.fDidSomething ? " modified-backing" : "", + g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType])); + + /* + * Emulate the memory access, either access handler or special memory. + */ + PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, + pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite + ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE) + : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc); + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_DS | CPUMCTX_EXTRN_ES); + AssertRCReturn(rc, rc); + if (pExit->VpContext.ExecutionState.Reserved0 || pExit->VpContext.ExecutionState.Reserved1) + Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pExit->VpContext.ExecutionState.Reserved0, pExit->VpContext.ExecutionState.Reserved1)); + + VBOXSTRICTRC rcStrict; + if (!pExitRec) + { + //if (pMsg->InstructionByteCount > 0) + // Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes)); + if (pExit->MemoryAccess.InstructionByteCount > 0) + rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, pExit->VpContext.Rip, pExit->MemoryAccess.InstructionBytes, pExit->MemoryAccess.InstructionByteCount); + else + rcStrict = IEMExecOne(pVCpu); + /** @todo do we need to do anything wrt debugging here? */ + } + else + { + /* Frequent access or probing. */ + rcStrict = EMHistoryExec(pVCpu, pExitRec, 0); + Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + } + return rcStrict; +} + + +/** + * Deals with I/O port access exits (WHvRunVpExitReasonX64IoPortAccess). + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + * @param pExit The VM exit information to handle. + * @sa nemHCWinHandleMessageIoPort + */ +NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitIoPort(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit) +{ + Assert( pExit->IoPortAccess.AccessInfo.AccessSize == 1 + || pExit->IoPortAccess.AccessInfo.AccessSize == 2 + || pExit->IoPortAccess.AccessInfo.AccessSize == 4); + + /* + * Whatever we do, we must clear pending event injection upon resume. + */ + if (pExit->VpContext.ExecutionState.InterruptionPending) + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; + + /* + * Add history first to avoid two paths doing EMHistoryExec calls. + */ + PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, + !pExit->IoPortAccess.AccessInfo.StringOp + ? ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite + ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE) + : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ)) + : ( pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite + ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE) + : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); + if (!pExitRec) + { + VBOXSTRICTRC rcStrict; + if (!pExit->IoPortAccess.AccessInfo.StringOp) + { + /* + * Simple port I/O. + */ + static uint32_t const s_fAndMask[8] = + { UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX }; + uint32_t const fAndMask = s_fAndMask[pExit->IoPortAccess.AccessInfo.AccessSize]; + if (pExit->IoPortAccess.AccessInfo.IsWrite) + { + rcStrict = IOMIOPortWrite(pVM, pVCpu, pExit->IoPortAccess.PortNumber, + (uint32_t)pExit->IoPortAccess.Rax & fAndMask, + pExit->IoPortAccess.AccessInfo.AccessSize); + Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->IoPortAccess.PortNumber, (uint32_t)pExit->IoPortAccess.Rax & fAndMask, + pExit->IoPortAccess.AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) )); + if (IOM_SUCCESS(rcStrict)) + { + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1); + } + } + else + { + uint32_t uValue = 0; + rcStrict = IOMIOPortRead(pVM, pVCpu, pExit->IoPortAccess.PortNumber, &uValue, + pExit->IoPortAccess.AccessInfo.AccessSize); + Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) )); + if (IOM_SUCCESS(rcStrict)) + { + if (pExit->IoPortAccess.AccessInfo.AccessSize != 4) + pVCpu->cpum.GstCtx.rax = (pExit->IoPortAccess.Rax & ~(uint64_t)fAndMask) | (uValue & fAndMask); + else + pVCpu->cpum.GstCtx.rax = uValue; + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX; + Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pExit->IoPortAccess.Rax, pVCpu->cpum.GstCtx.rax)); + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1); + } + } + } + else + { + /* + * String port I/O. + */ + /** @todo Someone at Microsoft please explain how we can get the address mode + * from the IoPortAccess.VpContext. CS.Attributes is only sufficient for + * getting the default mode, it can always be overridden by a prefix. This + * forces us to interpret the instruction from opcodes, which is suboptimal. + * Both AMD-V and VT-x includes the address size in the exit info, at least on + * CPUs that are reasonably new. + * + * Of course, it's possible this is an undocumented and we just need to do some + * experiments to figure out how it's communicated. Alternatively, we can scan + * the opcode bytes for possible evil prefixes. + */ + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDI | CPUMCTX_EXTRN_RSI + | CPUMCTX_EXTRN_DS | CPUMCTX_EXTRN_ES); + NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds); + NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es); + pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax; + pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx; + pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi; + pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi; + int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM); + AssertRCReturn(rc, rc); + + Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "", + pExit->IoPortAccess.AccessInfo.IsWrite ? "OUTS" : "INS", + pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize )); + rcStrict = IEMExecOne(pVCpu); + } + if (IOM_SUCCESS(rcStrict)) + { + /* + * Do debug checks. + */ + if ( pExit->VpContext.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */ + || (pExit->VpContext.Rflags & X86_EFL_TF) + || DBGFBpIsHwIoArmed(pVM) ) + { + /** @todo Debugging. */ + } + } + return rcStrict; + } + + /* + * Frequent exit or something needing probing. + * Get state and call EMHistoryExec. + */ + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + if (!pExit->IoPortAccess.AccessInfo.StringOp) + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX; + else + { + pVCpu->cpum.GstCtx.fExtrn &= ~( CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDI | CPUMCTX_EXTRN_RSI + | CPUMCTX_EXTRN_DS | CPUMCTX_EXTRN_ES); + NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds); + NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es); + pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx; + pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi; + pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi; + } + pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax; + int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM); + AssertRCReturn(rc, rc); + Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "", + pExit->IoPortAccess.AccessInfo.IsWrite ? "OUT" : "IN", + pExit->IoPortAccess.AccessInfo.StringOp ? "S" : "", + pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize)); + VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0); + Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + return rcStrict; +} + + +/** + * Deals with interrupt window exits (WHvRunVpExitReasonX64InterruptWindow). + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + * @param pExit The VM exit information to handle. + * @sa nemHCWinHandleMessageInterruptWindow + */ +NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitInterruptWindow(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit) +{ + /* + * Assert message sanity. + */ + AssertMsg( pExit->InterruptWindow.DeliverableType == WHvX64PendingInterrupt + || pExit->InterruptWindow.DeliverableType == WHvX64PendingNmi, + ("%#x\n", pExit->InterruptWindow.DeliverableType)); + + /* + * Just copy the state we've got and handle it in the loop for now. + */ + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); + + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d CR8=%#x\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->InterruptWindow.DeliverableType, RT_BOOL(pExit->VpContext.Rflags & X86_EFL_IF), + pExit->VpContext.ExecutionState.InterruptShadow, pExit->VpContext.Cr8)); + + /** @todo call nemHCWinHandleInterruptFF */ + RT_NOREF(pVM); + return VINF_SUCCESS; +} + + +/** + * Deals with CPUID exits (WHvRunVpExitReasonX64Cpuid). + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + * @param pExit The VM exit information to handle. + * @sa nemHCWinHandleMessageCpuId + */ +NEM_TMPL_STATIC VBOXSTRICTRC +nemR3WinHandleExitCpuId(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit) +{ + PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); + if (!pExitRec) + { + /* + * Soak up state and execute the instruction. + */ + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, + IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK + | CPUMCTX_EXTRN_CR3, /* May call PGMChangeMode() requiring cr3 (due to cr0 being imported). */ + "CPUID"); + if (rcStrict == VINF_SUCCESS) + { + /* Copy in the low register values (top is always cleared). */ + pVCpu->cpum.GstCtx.rax = (uint32_t)pExit->CpuidAccess.Rax; + pVCpu->cpum.GstCtx.rcx = (uint32_t)pExit->CpuidAccess.Rcx; + pVCpu->cpum.GstCtx.rdx = (uint32_t)pExit->CpuidAccess.Rdx; + pVCpu->cpum.GstCtx.rbx = (uint32_t)pExit->CpuidAccess.Rbx; + pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX); + + /* Execute the decoded instruction. */ + rcStrict = IEMExecDecodedCpuid(pVCpu, pExit->VpContext.InstructionLength); + + Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->CpuidAccess.Rax, pExit->CpuidAccess.Rcx, pExit->CpuidAccess.Rdx, pExit->CpuidAccess.Rbx, + pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.ebx, + pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx)); + } + + RT_NOREF_PV(pVM); + return rcStrict; + } + + /* + * Frequent exit or something needing probing. + * Get state and call EMHistoryExec. + */ + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + pVCpu->cpum.GstCtx.rax = pExit->CpuidAccess.Rax; + pVCpu->cpum.GstCtx.rcx = pExit->CpuidAccess.Rcx; + pVCpu->cpum.GstCtx.rdx = pExit->CpuidAccess.Rdx; + pVCpu->cpum.GstCtx.rbx = pExit->CpuidAccess.Rbx; + pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX); + Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->CpuidAccess.Rax, pExit->CpuidAccess.Rcx, pExit->CpuidAccess.Rdx, pExit->CpuidAccess.Rbx, + pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx)); + int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM); + AssertRCReturn(rc, rc); + VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0); + Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + return rcStrict; +} + + +/** + * Deals with MSR access exits (WHvRunVpExitReasonX64MsrAccess). + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + * @param pExit The VM exit information to handle. + * @sa nemHCWinHandleMessageMsr + */ +NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitMsr(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit) +{ + /* + * Check CPL as that's common to both RDMSR and WRMSR. + */ + VBOXSTRICTRC rcStrict; + if (pExit->VpContext.ExecutionState.Cpl == 0) + { + /* + * Get all the MSR state. Since we're getting EFER, we also need to + * get CR0, CR4 and CR3. + */ + PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, + pExit->MsrAccess.AccessInfo.IsWrite + ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE) + : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, + (!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK) + | CPUMCTX_EXTRN_ALL_MSRS | CPUMCTX_EXTRN_CR0 + | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4, + "MSRs"); + if (rcStrict == VINF_SUCCESS) + { + if (!pExitRec) + { + /* + * Handle writes. + */ + if (pExit->MsrAccess.AccessInfo.IsWrite) + { + rcStrict = CPUMSetGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber, + RT_MAKE_U64((uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx)); + Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber, + (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx, VBOXSTRICTRC_VAL(rcStrict) )); + if (rcStrict == VINF_SUCCESS) + { + nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2); + return VINF_SUCCESS; + } + LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n", pVCpu->idCpu, + pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx, + VBOXSTRICTRC_VAL(rcStrict) )); + } + /* + * Handle reads. + */ + else + { + uint64_t uValue = 0; + rcStrict = CPUMQueryGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber, &uValue); + Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu, + pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->MsrAccess.MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) )); + if (rcStrict == VINF_SUCCESS) + { + pVCpu->cpum.GstCtx.rax = (uint32_t)uValue; + pVCpu->cpum.GstCtx.rdx = uValue >> 32; + pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX); + nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2); + return VINF_SUCCESS; + } + LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber, + uValue, VBOXSTRICTRC_VAL(rcStrict) )); + } + } + else + { + /* + * Handle frequent exit or something needing probing. + */ + Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber)); + rcStrict = EMHistoryExec(pVCpu, pExitRec, 0); + Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + return rcStrict; + } + } + else + { + LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber, VBOXSTRICTRC_VAL(rcStrict) )); + return rcStrict; + } + } + else if (pExit->MsrAccess.AccessInfo.IsWrite) + Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl, + pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx )); + else + Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl, + pExit->MsrAccess.MsrNumber)); + + /* + * If we get down here, we're supposed to #GP(0). + */ + rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_ALL_MSRS, "MSR"); + if (rcStrict == VINF_SUCCESS) + { + rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0); + if (rcStrict == VINF_IEM_RAISED_XCPT) + rcStrict = VINF_SUCCESS; + else if (rcStrict != VINF_SUCCESS) + Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) )); + } + + RT_NOREF_PV(pVM); + return rcStrict; +} + + +/** + * Worker for nemHCWinHandleMessageException & nemR3WinHandleExitException that + * checks if the given opcodes are of interest at all. + * + * @returns true if interesting, false if not. + * @param cbOpcodes Number of opcode bytes available. + * @param pbOpcodes The opcode bytes. + * @param f64BitMode Whether we're in 64-bit mode. + */ +DECLINLINE(bool) nemHcWinIsInterestingUndefinedOpcode(uint8_t cbOpcodes, uint8_t const *pbOpcodes, bool f64BitMode) +{ + /* + * Currently only interested in VMCALL and VMMCALL. + */ + while (cbOpcodes >= 3) + { + switch (pbOpcodes[0]) + { + case 0x0f: + switch (pbOpcodes[1]) + { + case 0x01: + switch (pbOpcodes[2]) + { + case 0xc1: /* 0f 01 c1 VMCALL */ + return true; + case 0xd9: /* 0f 01 d9 VMMCALL */ + return true; + default: + break; + } + break; + } + break; + + default: + return false; + + /* prefixes */ + case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47: + case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f: + if (!f64BitMode) + return false; + RT_FALL_THRU(); + case X86_OP_PRF_CS: + case X86_OP_PRF_SS: + case X86_OP_PRF_DS: + case X86_OP_PRF_ES: + case X86_OP_PRF_FS: + case X86_OP_PRF_GS: + case X86_OP_PRF_SIZE_OP: + case X86_OP_PRF_SIZE_ADDR: + case X86_OP_PRF_LOCK: + case X86_OP_PRF_REPZ: + case X86_OP_PRF_REPNZ: + cbOpcodes--; + pbOpcodes++; + continue; + } + break; + } + return false; +} + + +/** + * Copies state included in a exception intercept exit. + * + * @param pVCpu The cross context per CPU structure. + * @param pExit The VM exit information. + * @param fClearXcpt Clear pending exception. + */ +DECLINLINE(void) nemR3WinCopyStateFromExceptionMessage(PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit, bool fClearXcpt) +{ + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + if (fClearXcpt) + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; +} + + +/** + * Advances the guest RIP by the number of bytes specified in @a cb. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cb RIP increment value in bytes. + */ +DECLINLINE(void) nemHcWinAdvanceRip(PVMCPUCC pVCpu, uint32_t cb) +{ + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + pCtx->rip += cb; + /** @todo Why not clear RF too? */ + CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); +} + + +/** + * Hacks its way around the lovely mesa driver's backdoor accesses. + * + * @sa hmR0VmxHandleMesaDrvGp + * @sa hmR0SvmHandleMesaDrvGp + */ +static int nemHcWinHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx) +{ + Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_GPRS_MASK))); + RT_NOREF(pCtx); + + /* For now we'll just skip the instruction. */ + nemHcWinAdvanceRip(pVCpu, 1); + return VINF_SUCCESS; +} + + +/** + * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely + * backdoor logging w/o checking what it is running inside. + * + * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the + * backdoor port and magic numbers loaded in registers. + * + * @returns true if it is, false if it isn't. + * @sa hmR0VmxIsMesaDrvGp + * @sa hmR0SvmIsMesaDrvGp + */ +DECLINLINE(bool) nemHcWinIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, const uint8_t *pbInsn, uint32_t cbInsn) +{ + /* #GP(0) is already checked by caller. */ + + /* Check magic and port. */ + Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RAX))); + if (pCtx->dx != UINT32_C(0x5658)) + return false; + if (pCtx->rax != UINT32_C(0x564d5868)) + return false; + + /* Flat ring-3 CS. */ + if (CPUMGetGuestCPL(pVCpu) != 3) + return false; + if (pCtx->cs.u64Base != 0) + return false; + + /* 0xed: IN eAX,dx */ + if (cbInsn < 1) /* Play safe (shouldn't happen). */ + { + uint8_t abInstr[1]; + int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr)); + if (RT_FAILURE(rc)) + return false; + if (abInstr[0] != 0xed) + return false; + } + else + { + if (pbInsn[0] != 0xed) + return false; + } + + return true; +} + + +/** + * Deals with MSR access exits (WHvRunVpExitReasonException). + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + * @param pExit The VM exit information to handle. + * @sa nemR3WinHandleExitException + */ +NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit) +{ + /* + * Get most of the register state since we'll end up making IEM inject the + * event. The exception isn't normally flaged as a pending event, so duh. + * + * Note! We can optimize this later with event injection. + */ + Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType, + pExit->VpException.ErrorCode, pExit->VpException.ExceptionParameter )); + nemR3WinCopyStateFromExceptionMessage(pVCpu, pExit, true /*fClearXcpt*/); + uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM; + if (pExit->VpException.ExceptionType == X86_XCPT_DB) + fWhat |= CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR7 | CPUMCTX_EXTRN_DR6; + VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, fWhat, "Xcpt"); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + + /* + * Handle the intercept. + */ + TRPMEVENT enmEvtType = TRPM_TRAP; + switch (pExit->VpException.ExceptionType) + { + /* + * We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions + * and need to turn them over to GIM. + * + * Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants + * #UD for handling non-native hypercall instructions. (IEM will + * decode both and let the GIM provider decide whether to accept it.) + */ + case X86_XCPT_UD: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd); + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); + if (nemHcWinIsInterestingUndefinedOpcode(pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes, + pExit->VpContext.ExecutionState.EferLma && pExit->VpContext.Cs.Long )) + { + rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, pExit->VpContext.Rip, + pExit->VpException.InstructionBytes, + pExit->VpException.InstructionByteCount); + Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, + nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) )); + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled); + return rcStrict; + } + + Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu, + pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), + pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes )); + break; + + /* + * Workaround the lovely mesa driver assuming that vmsvga means vmware + * hypervisor and tries to log stuff to the host. + */ + case X86_XCPT_GP: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGp); + /** @todo r=bird: Need workaround in IEM for this, right? + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_GP), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); */ + if ( !pVCpu->nem.s.fTrapXcptGpForLovelyMesaDrv + || !nemHcWinIsMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx, pExit->VpException.InstructionBytes, + pExit->VpException.InstructionByteCount)) + { +#if 1 /** @todo Need to emulate instruction or we get a triple fault when trying to inject the \#GP... */ + rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, pExit->VpContext.Rip, + pExit->VpException.InstructionBytes, + pExit->VpException.InstructionByteCount); + Log4(("XcptExit/%u: %04x:%08RX64/%s: #GP -> emulated -> %Rrc\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, + nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) )); + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled); + return rcStrict; +#else + break; +#endif + } + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGpMesa); + return nemHcWinHandleMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx); + + /* + * Filter debug exceptions. + */ + case X86_XCPT_DB: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb); + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); + Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext) )); + break; + + case X86_XCPT_BP: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp); + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); + Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.InstructionLength)); + enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */ + break; + + /* This shouldn't happen. */ + default: + AssertLogRelMsgFailedReturn(("ExceptionType=%#x\n", pExit->VpException.ExceptionType), VERR_IEM_IPE_6); + } + + /* + * Inject it. + */ + rcStrict = IEMInjectTrap(pVCpu, pExit->VpException.ExceptionType, enmEvtType, pExit->VpException.ErrorCode, + pExit->VpException.ExceptionParameter /*??*/, pExit->VpContext.InstructionLength); + Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n", + pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, + nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType, VBOXSTRICTRC_VAL(rcStrict) )); + + RT_NOREF_PV(pVM); + return rcStrict; +} + + +/** + * Deals with MSR access exits (WHvRunVpExitReasonUnrecoverableException). + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + * @param pExit The VM exit information to handle. + * @sa nemHCWinHandleMessageUnrecoverableException + */ +NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitUnrecoverableException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit) +{ +#if 0 + /* + * Just copy the state we've got and handle it in the loop for now. + */ + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags)); + RT_NOREF_PV(pVM); + return VINF_EM_TRIPLE_FAULT; +#else + /* + * Let IEM decide whether this is really it. + */ + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); + nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext); + VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_ALL, "TripleExit"); + if (rcStrict == VINF_SUCCESS) + { + rcStrict = IEMExecOne(pVCpu); + if (rcStrict == VINF_SUCCESS) + { + Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags)); + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */ + return VINF_SUCCESS; + } + if (rcStrict == VINF_EM_TRIPLE_FAULT) + Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) )); + else + Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) )); + } + else + Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector, + pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) )); + RT_NOREF_PV(pVM); + return rcStrict; +#endif +} + + +/** + * Handles VM exits. + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + * @param pExit The VM exit information to handle. + * @sa nemHCWinHandleMessage + */ +NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExit(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit) +{ + switch (pExit->ExitReason) + { + case WHvRunVpExitReasonMemoryAccess: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped); + return nemR3WinHandleExitMemory(pVM, pVCpu, pExit); + + case WHvRunVpExitReasonX64IoPortAccess: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo); + return nemR3WinHandleExitIoPort(pVM, pVCpu, pExit); + + case WHvRunVpExitReasonX64Halt: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt); + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT), + pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); + Log4(("HaltExit/%u\n", pVCpu->idCpu)); + return VINF_EM_HALT; + + case WHvRunVpExitReasonCanceled: + Log4(("CanceledExit/%u\n", pVCpu->idCpu)); + return VINF_SUCCESS; + + case WHvRunVpExitReasonX64InterruptWindow: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow); + return nemR3WinHandleExitInterruptWindow(pVM, pVCpu, pExit); + + case WHvRunVpExitReasonX64Cpuid: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId); + return nemR3WinHandleExitCpuId(pVM, pVCpu, pExit); + + case WHvRunVpExitReasonX64MsrAccess: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr); + return nemR3WinHandleExitMsr(pVM, pVCpu, pExit); + + case WHvRunVpExitReasonException: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException); + return nemR3WinHandleExitException(pVM, pVCpu, pExit); + + case WHvRunVpExitReasonUnrecoverableException: + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable); + return nemR3WinHandleExitUnrecoverableException(pVM, pVCpu, pExit); + + case WHvRunVpExitReasonUnsupportedFeature: + case WHvRunVpExitReasonInvalidVpRegisterValue: + LogRel(("Unimplemented exit:\n%.*Rhxd\n", (int)sizeof(*pExit), pExit)); + AssertLogRelMsgFailedReturn(("Unexpected exit on CPU #%u: %#x\n%.32Rhxd\n", + pVCpu->idCpu, pExit->ExitReason, pExit), VERR_NEM_IPE_3); + + /* Undesired exits: */ + case WHvRunVpExitReasonNone: + default: + LogRel(("Unknown exit:\n%.*Rhxd\n", (int)sizeof(*pExit), pExit)); + AssertLogRelMsgFailedReturn(("Unknown exit on CPU #%u: %#x!\n", pVCpu->idCpu, pExit->ExitReason), VERR_NEM_IPE_3); + } +} + + +/** + * Deals with pending interrupt related force flags, may inject interrupt. + * + * @returns VBox strict status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + * @param pfInterruptWindows Where to return interrupt window flags. + */ +NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleInterruptFF(PVMCC pVM, PVMCPUCC pVCpu, uint8_t *pfInterruptWindows) +{ + Assert(!TRPMHasTrap(pVCpu)); + RT_NOREF_PV(pVM); + + /* + * First update APIC. We ASSUME this won't need TPR/CR8. + */ + if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC)) + { + APICUpdatePendingInterrupts(pVCpu); + if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC + | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI)) + return VINF_SUCCESS; + } + + /* + * We don't currently implement SMIs. + */ + AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0); + + /* + * Check if we've got the minimum of state required for deciding whether we + * can inject interrupts and NMIs. If we don't have it, get all we might require + * for injection via IEM. + */ + bool const fPendingNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI); + uint64_t fNeedExtrn = CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS + | (fPendingNmi ? CPUMCTX_EXTRN_INHIBIT_NMI : 0); + if (pVCpu->cpum.GstCtx.fExtrn & fNeedExtrn) + { + VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "IntFF"); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + /* + * NMI? Try deliver it first. + */ + if (fPendingNmi) + { + if ( !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx) + && !CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx)) + { + VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI"); + if (rcStrict == VINF_SUCCESS) + { + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI); + rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_NMI, TRPM_HARDWARE_INT, 0, 0, 0); + Log8(("Injected NMI on %u (%d)\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) )); + } + return rcStrict; + } + *pfInterruptWindows |= NEM_WIN_INTW_F_NMI; + Log8(("NMI window pending on %u\n", pVCpu->idCpu)); + } + + /* + * APIC or PIC interrupt? + */ + if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)) + { + /** @todo check NMI inhibiting here too! */ + if ( !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx) + && pVCpu->cpum.GstCtx.rflags.Bits.u1IF) + { + AssertCompile(NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT & CPUMCTX_EXTRN_APIC_TPR); + VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI"); + if (rcStrict == VINF_SUCCESS) + { + uint8_t bInterrupt; + int rc = PDMGetInterrupt(pVCpu, &bInterrupt); + if (RT_SUCCESS(rc)) + { + Log8(("Injecting interrupt %#x on %u: %04x:%08RX64 efl=%#x\n", bInterrupt, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eflags.u)); + rcStrict = IEMInjectTrap(pVCpu, bInterrupt, TRPM_HARDWARE_INT, 0, 0, 0); + Log8(("Injected interrupt %#x on %u (%d)\n", bInterrupt, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) )); + } + else if (rc == VERR_APIC_INTR_MASKED_BY_TPR) + { + *pfInterruptWindows |= ((bInterrupt >> 4) << NEM_WIN_INTW_F_PRIO_SHIFT) | NEM_WIN_INTW_F_REGULAR; + Log8(("VERR_APIC_INTR_MASKED_BY_TPR: *pfInterruptWindows=%#x\n", *pfInterruptWindows)); + } + else + Log8(("PDMGetInterrupt failed -> %Rrc\n", rc)); + } + return rcStrict; + } + + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC) && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC)) + { + /* If only an APIC interrupt is pending, we need to know its priority. Otherwise we'll + * likely get pointless deliverability notifications with IF=1 but TPR still too high. + */ + bool fPendingIntr = false; + uint8_t bTpr = 0; + uint8_t bPendingIntr = 0; + int rc = APICGetTpr(pVCpu, &bTpr, &fPendingIntr, &bPendingIntr); + AssertRC(rc); + *pfInterruptWindows |= ((bPendingIntr >> 4) << NEM_WIN_INTW_F_PRIO_SHIFT) | NEM_WIN_INTW_F_REGULAR; + Log8(("Interrupt window pending on %u: %#x (bTpr=%#x fPendingIntr=%d bPendingIntr=%#x)\n", + pVCpu->idCpu, *pfInterruptWindows, bTpr, fPendingIntr, bPendingIntr)); + } + else + { + *pfInterruptWindows |= NEM_WIN_INTW_F_REGULAR; + Log8(("Interrupt window pending on %u: %#x\n", pVCpu->idCpu, *pfInterruptWindows)); + } + } + + return VINF_SUCCESS; +} + + +/** + * Inner NEM runloop for windows. + * + * @returns Strict VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. + */ +NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinRunGC(PVMCC pVM, PVMCPUCC pVCpu) +{ + LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u)); +#ifdef LOG_ENABLED + if (LogIs3Enabled()) + nemHCWinLogState(pVM, pVCpu); +#endif + + /* + * Try switch to NEM runloop state. + */ + if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED)) + { /* likely */ } + else + { + VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED); + LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu)); + return VINF_SUCCESS; + } + + /* + * The run loop. + * + * Current approach to state updating to use the sledgehammer and sync + * everything every time. This will be optimized later. + */ + const bool fSingleStepping = DBGFIsStepping(pVCpu); +// const uint32_t fCheckVmFFs = !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK +// : VM_FF_HP_R0_PRE_HM_STEP_MASK; +// const uint32_t fCheckCpuFFs = !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK; + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + for (unsigned iLoop = 0;; iLoop++) + { + /* + * Pending interrupts or such? Need to check and deal with this prior + * to the state syncing. + */ + pVCpu->nem.s.fDesiredInterruptWindows = 0; + if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_PIC + | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI)) + { + /* Try inject interrupt. */ + rcStrict = nemHCWinHandleInterruptFF(pVM, pVCpu, &pVCpu->nem.s.fDesiredInterruptWindows); + if (rcStrict == VINF_SUCCESS) + { /* likely */ } + else + { + LogFlow(("NEM/%u: breaking: nemHCWinHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) )); + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus); + break; + } + } + +#ifndef NEM_WIN_WITH_A20 + /* + * Do not execute in hyper-V if the A20 isn't enabled. + */ + if (PGMPhysIsA20Enabled(pVCpu)) + { /* likely */ } + else + { + rcStrict = VINF_EM_RESCHEDULE_REM; + LogFlow(("NEM/%u: breaking: A20 disabled\n", pVCpu->idCpu)); + break; + } +#endif + + /* + * Ensure that hyper-V has the whole state. + * (We always update the interrupt windows settings when active as hyper-V seems + * to forget about it after an exit.) + */ + if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK)) + != (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK) + || ( ( pVCpu->nem.s.fDesiredInterruptWindows + || pVCpu->nem.s.fCurrentInterruptWindows != pVCpu->nem.s.fDesiredInterruptWindows) ) ) + { + int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu); + AssertRCReturn(rc2, rc2); + } + + /* + * Poll timers and run for a bit. + * + * With the VID approach (ring-0 or ring-3) we can specify a timeout here, + * so we take the time of the next timer event and uses that as a deadline. + * The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine. + */ + /** @todo See if we cannot optimize this TMTimerPollGIP by only redoing + * the whole polling job when timers have changed... */ + uint64_t offDeltaIgnored; + uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt); + if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC) + && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK)) + { + if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM)) + { +#ifdef LOG_ENABLED + if (LogIsFlowEnabled()) + { + static const WHV_REGISTER_NAME s_aNames[6] = { WHvX64RegisterCs, WHvX64RegisterRip, WHvX64RegisterRflags, + WHvX64RegisterSs, WHvX64RegisterRsp, WHvX64RegisterCr0 }; + WHV_REGISTER_VALUE aRegs[RT_ELEMENTS(s_aNames)] = { {{0, 0} } }; + WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, s_aNames, RT_ELEMENTS(s_aNames), aRegs); + LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n", + pVCpu->idCpu, aRegs[0].Segment.Selector, aRegs[1].Reg64, RT_BOOL(aRegs[2].Reg64 & X86_EFL_IF), + aRegs[2].Reg64, aRegs[3].Segment.Selector, aRegs[4].Reg64, aRegs[5].Reg64)); + } +#endif + WHV_RUN_VP_EXIT_CONTEXT ExitReason = {0}; + TMNotifyStartOfExecution(pVM, pVCpu); + + HRESULT hrc = WHvRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, &ExitReason, sizeof(ExitReason)); + + VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT); + TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC()); +#ifdef LOG_ENABLED + LogFlow(("NEM/%u: Exit @ %04X:%08RX64 IF=%d CR8=%#x Reason=%#x\n", pVCpu->idCpu, ExitReason.VpContext.Cs.Selector, + ExitReason.VpContext.Rip, RT_BOOL(ExitReason.VpContext.Rflags & X86_EFL_IF), ExitReason.VpContext.Cr8, + ExitReason.ExitReason)); +#endif + if (SUCCEEDED(hrc)) + { + /* + * Deal with the message. + */ + rcStrict = nemR3WinHandleExit(pVM, pVCpu, &ExitReason); + if (rcStrict == VINF_SUCCESS) + { /* hopefully likely */ } + else + { + LogFlow(("NEM/%u: breaking: nemHCWinHandleMessage -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) )); + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus); + break; + } + } + else + AssertLogRelMsgFailedReturn(("WHvRunVirtualProcessor failed for CPU #%u: %#x (%u)\n", + pVCpu->idCpu, hrc, GetLastError()), + VERR_NEM_IPE_0); + + /* + * If no relevant FFs are pending, loop. + */ + if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK) + && !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) ) + continue; + + /** @todo Try handle pending flags, not just return to EM loops. Take care + * not to set important RCs here unless we've handled a message. */ + LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n", + pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions)); + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost); + } + else + { + LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) )); + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel); + } + } + else + { + LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu)); + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre); + } + break; + } /* the run loop */ + + + /* + * If the CPU is running, make sure to stop it before we try sync back the + * state and return to EM. We don't sync back the whole state if we can help it. + */ + if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM)) + VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED); + + if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))) + { + /* Try anticipate what we might need. */ + uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI; + if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST) + || RT_FAILURE(rcStrict)) + fImport = CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT); + else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_INTERRUPT_APIC + | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI)) + fImport |= IEM_CPUMCTX_EXTRN_XCPT_MASK; + + if (pVCpu->cpum.GstCtx.fExtrn & fImport) + { + int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, fImport | CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT); + if (RT_SUCCESS(rc2)) + pVCpu->cpum.GstCtx.fExtrn &= ~fImport; + else if (RT_SUCCESS(rcStrict)) + rcStrict = rc2; + if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))) + pVCpu->cpum.GstCtx.fExtrn = 0; + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn); + } + else + { + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped); + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; + } + } + else + { + STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped); + pVCpu->cpum.GstCtx.fExtrn = 0; + } + + LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, + pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, VBOXSTRICTRC_VAL(rcStrict) )); + return rcStrict; +} + + +/** + * @callback_method_impl{FNPGMPHYSNEMCHECKPAGE} + */ +NEM_TMPL_STATIC DECLCALLBACK(int) nemHCWinUnsetForA20CheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, + PPGMPHYSNEMPAGEINFO pInfo, void *pvUser) +{ + /* We'll just unmap the memory. */ + if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED) + { + HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE); + if (SUCCEEDED(hrc)) + { + STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage); + uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages); + Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages)); + pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED; + } + else + { + STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed); + LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n", + GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())); + return VERR_NEM_IPE_2; + } + } + RT_NOREF(pVCpu, pvUser); + return VINF_SUCCESS; +} + + +/** + * Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior. + * + * @returns The PGMPhysNemQueryPageInfo result. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhys The page to unmap. + */ +NEM_TMPL_STATIC int nemHCWinUnmapPageForA20Gate(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys) +{ + PGMPHYSNEMPAGEINFO Info; + return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /*fMakeWritable*/, &Info, + nemHCWinUnsetForA20CheckerCallback, NULL); +} + + +void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb) +{ + Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind)); + NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb); +} + + +VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb, + RTR3PTR pvMemR3, uint8_t *pu2State) +{ + Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n", + GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State)); + + *pu2State = UINT8_MAX; + if (pvMemR3) + { + STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRange, a); + HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvMemR3, GCPhys, cb, + WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute | WHvMapGpaRangeFlagWrite); + STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRange, a); + if (SUCCEEDED(hrc)) + *pu2State = NEM_WIN_PAGE_STATE_WRITABLE; + else + AssertLogRelMsgFailed(("NEMHCNotifyHandlerPhysicalDeregister: WHvMapGpaRange(,%p,%RGp,%RGp,) -> %Rhrc\n", + pvMemR3, GCPhys, cb, hrc)); + } + RT_NOREF(enmKind); +} + + +void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld, + RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM) +{ + Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n", + GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM)); + NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM); +} + + +/** + * Worker that maps pages into Hyper-V. + * + * This is used by the PGM physical page notifications as well as the memory + * access VMEXIT handlers. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the + * calling EMT. + * @param GCPhysSrc The source page address. + * @param GCPhysDst The hyper-V destination page. This may differ from + * GCPhysSrc when A20 is disabled. + * @param fPageProt NEM_PAGE_PROT_XXX. + * @param pu2State Our page state (input/output). + * @param fBackingChanged Set if the page backing is being changed. + * @thread EMT(pVCpu) + */ +NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst, + uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged) +{ + /* + * Looks like we need to unmap a page before we can change the backing + * or even modify the protection. This is going to be *REALLY* efficient. + * PGM lends us two bits to keep track of the state here. + */ + RT_NOREF(pVCpu); + uint8_t const u2OldState = *pu2State; + uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_WIN_PAGE_STATE_WRITABLE + : fPageProt & NEM_PAGE_PROT_READ ? NEM_WIN_PAGE_STATE_READABLE : NEM_WIN_PAGE_STATE_UNMAPPED; + if ( fBackingChanged + || u2NewState != u2OldState) + { + if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED) + { + STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a); + HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst, X86_PAGE_SIZE); + STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a); + if (SUCCEEDED(hrc)) + { + *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED; + STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage); + uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages); + if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED) + { + Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n", + GCPhysDst, g_apszPageStates[u2OldState], cMappedPages)); + return VINF_SUCCESS; + } + } + else + { + STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed); + LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n", + GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())); + return VERR_NEM_INIT_FAILED; + } + } + } + + /* + * Writeable mapping? + */ + if (fPageProt & NEM_PAGE_PROT_WRITE) + { + void *pvPage; + int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage); + if (RT_SUCCESS(rc)) + { + HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPage, GCPhysDst, X86_PAGE_SIZE, + WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute | WHvMapGpaRangeFlagWrite); + if (SUCCEEDED(hrc)) + { + *pu2State = NEM_WIN_PAGE_STATE_WRITABLE; + STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage); + uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages); + Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n", + GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages)); + return VINF_SUCCESS; + } + STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed); + LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n", + GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())); + return VERR_NEM_INIT_FAILED; + } + LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc)); + return rc; + } + + if (fPageProt & NEM_PAGE_PROT_READ) + { + const void *pvPage; + int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage); + if (RT_SUCCESS(rc)) + { + STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRangePage, a); + HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhysDst, X86_PAGE_SIZE, + WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute); + STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRangePage, a); + if (SUCCEEDED(hrc)) + { + *pu2State = NEM_WIN_PAGE_STATE_READABLE; + STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage); + uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages); + Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n", + GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages)); + return VINF_SUCCESS; + } + STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed); + LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n", + GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())); + return VERR_NEM_INIT_FAILED; + } + LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc)); + return rc; + } + + /* We already unmapped it above. */ + *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED; + return VINF_SUCCESS; +} + + +NEM_TMPL_STATIC int nemHCJustUnmapPageFromHyperV(PVMCC pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State) +{ + if (*pu2State <= NEM_WIN_PAGE_STATE_UNMAPPED) + { + Log5(("nemHCJustUnmapPageFromHyperV: %RGp == unmapped\n", GCPhysDst)); + *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED; + return VINF_SUCCESS; + } + + STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a); + HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE); + STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a); + if (SUCCEEDED(hrc)) + { + STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage); + uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages); + *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED; + Log5(("nemHCJustUnmapPageFromHyperV: %RGp => unmapped (total %u)\n", GCPhysDst, cMappedPages)); + return VINF_SUCCESS; + } + STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed); + LogRel(("nemHCJustUnmapPageFromHyperV(%RGp): failed! hrc=%Rhrc (%#x) Last=%#x/%u\n", + GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())); + return VERR_NEM_IPE_6; +} + + +int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt, + PGMPAGETYPE enmType, uint8_t *pu2State) +{ + Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n", + GCPhys, HCPhys, fPageProt, enmType, *pu2State)); + RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType); + + int rc; + RT_NOREF_PV(fPageProt); +#ifdef NEM_WIN_WITH_A20 + if ( pVM->nem.s.fA20Enabled + || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys)) +#endif + rc = nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State); +#ifdef NEM_WIN_WITH_A20 + else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys)) + rc = nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State); + else + rc = VINF_SUCCESS; /* ignore since we've got the alias page at this address. */ +#endif + return rc; +} + + +VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt, + PGMPAGETYPE enmType, uint8_t *pu2State) +{ + Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n", + GCPhys, HCPhys, fPageProt, enmType, *pu2State)); + Assert(VM_IS_NEM_ENABLED(pVM)); + RT_NOREF(HCPhys, enmType, pvR3); + + RT_NOREF_PV(fPageProt); +#ifdef NEM_WIN_WITH_A20 + if ( pVM->nem.s.fA20Enabled + || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys)) +#endif + nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State); +#ifdef NEM_WIN_WITH_A20 + else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys)) + nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State); + /* else: ignore since we've got the alias page at this address. */ +#endif +} + + +VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew, + RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State) +{ + Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp pvNewR3=%p fPageProt=%#x enmType=%d *pu2State=%d\n", + GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, *pu2State)); + Assert(VM_IS_NEM_ENABLED(pVM)); + RT_NOREF(HCPhysPrev, HCPhysNew, pvNewR3, enmType); + + RT_NOREF_PV(fPageProt); +#ifdef NEM_WIN_WITH_A20 + if ( pVM->nem.s.fA20Enabled + || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys)) +#endif + nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State); +#ifdef NEM_WIN_WITH_A20 + else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys)) + nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State); + /* else: ignore since we've got the alias page at this address. */ +#endif +} + + +/** + * Returns features supported by the NEM backend. + * + * @returns Flags of features supported by the native NEM backend. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM) +{ + RT_NOREF(pVM); + /** @todo Make use of the WHvGetVirtualProcessorXsaveState/WHvSetVirtualProcessorXsaveState + * interface added in 2019 to enable passthrough of xsave/xrstor (and depending) features to the guest. */ + /** @todo Is NEM_FEAT_F_FULL_GST_EXEC always true? */ + return NEM_FEAT_F_NESTED_PAGING | NEM_FEAT_F_FULL_GST_EXEC; +} + diff --git a/src/VBox/VMM/VMMAll/PDMAll.cpp b/src/VBox/VMM/VMMAll/PDMAll.cpp new file mode 100644 index 00000000..79301070 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PDMAll.cpp @@ -0,0 +1,428 @@ +/* $Id: PDMAll.cpp $ */ +/** @file + * PDM Critical Sections + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PDM +#include "PDMInternal.h" +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "PDMInline.h" +#include "dtrace/VBoxVMM.h" + + + +/** + * Gets the pending interrupt. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_APIC_INTR_MASKED_BY_TPR when an APIC interrupt is pending but + * can't be delivered due to TPR priority. + * @retval VERR_NO_DATA if there is no interrupt to be delivered (either APIC + * has been software-disabled since it flagged something was pending, + * or other reasons). + * + * @param pVCpu The cross context virtual CPU structure. + * @param pu8Interrupt Where to store the interrupt. + */ +VMMDECL(int) PDMGetInterrupt(PVMCPUCC pVCpu, uint8_t *pu8Interrupt) +{ + /* + * The local APIC has a higher priority than the PIC. + */ + int rc = VERR_NO_DATA; + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC)) + { + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC); + uint32_t uTagSrc; + rc = APICGetInterrupt(pVCpu, pu8Interrupt, &uTagSrc); + if (RT_SUCCESS(rc)) + { + VBOXVMM_PDM_IRQ_GET(pVCpu, RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc), *pu8Interrupt); + Log8(("PDMGetInterrupt: irq=%#x tag=%#x (apic)\n", *pu8Interrupt, uTagSrc)); + return VINF_SUCCESS; + } + /* else if it's masked by TPR/PPR/whatever, go ahead checking the PIC. Such masked + interrupts shouldn't prevent ExtINT from being delivered. */ + } + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + pdmLock(pVM); + + /* + * Check the PIC. + */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC)) + { + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC); + Assert(pVM->pdm.s.Pic.CTX_SUFF(pDevIns)); + Assert(pVM->pdm.s.Pic.CTX_SUFF(pfnGetInterrupt)); + uint32_t uTagSrc; + int i = pVM->pdm.s.Pic.CTX_SUFF(pfnGetInterrupt)(pVM->pdm.s.Pic.CTX_SUFF(pDevIns), &uTagSrc); + AssertMsg(i <= 255 && i >= 0, ("i=%d\n", i)); + if (i >= 0) + { + pdmUnlock(pVM); + *pu8Interrupt = (uint8_t)i; + VBOXVMM_PDM_IRQ_GET(pVCpu, RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc), i); + Log8(("PDMGetInterrupt: irq=%#x tag=%#x (pic)\n", i, uTagSrc)); + return VINF_SUCCESS; + } + } + + /* + * One scenario where we may possibly get here is if the APIC signaled a pending interrupt, + * got an APIC MMIO/MSR VM-exit which disabled the APIC. We could, in theory, clear the APIC + * force-flag from all the places which disables the APIC but letting PDMGetInterrupt() fail + * without returning a valid interrupt still needs to be handled for the TPR masked case, + * so we shall just handle it here regardless if we choose to update the APIC code in the future. + */ + + pdmUnlock(pVM); + return rc; +} + + +/** + * Sets the pending interrupt coming from ISA source or HPET. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param u8Irq The IRQ line. + * @param u8Level The new level. + * @param uTagSrc The IRQ tag and source tracer ID. + */ +VMMDECL(int) PDMIsaSetIrq(PVMCC pVM, uint8_t u8Irq, uint8_t u8Level, uint32_t uTagSrc) +{ + pdmLock(pVM); + + /** @todo put the IRQ13 code elsewhere to avoid this unnecessary bloat. */ + if (!uTagSrc && (u8Level & PDM_IRQ_LEVEL_HIGH)) /* FPU IRQ */ + { + if (u8Level == PDM_IRQ_LEVEL_HIGH) + VBOXVMM_PDM_IRQ_HIGH(VMMGetCpu(pVM), 0, 0); + else + VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pVM), 0, 0); + } + Log9(("PDMIsaSetIrq: irq=%#x lvl=%u tag=%#x\n", u8Irq, u8Level, uTagSrc)); + + int rc = VERR_PDM_NO_PIC_INSTANCE; +/** @todo r=bird: This code is incorrect, as it ASSUMES the PIC and I/O APIC + * are always ring-0 enabled! */ + if (pVM->pdm.s.Pic.CTX_SUFF(pDevIns)) + { + Assert(pVM->pdm.s.Pic.CTX_SUFF(pfnSetIrq)); + pVM->pdm.s.Pic.CTX_SUFF(pfnSetIrq)(pVM->pdm.s.Pic.CTX_SUFF(pDevIns), u8Irq, u8Level, uTagSrc); + rc = VINF_SUCCESS; + } + + if (pVM->pdm.s.IoApic.CTX_SUFF(pDevIns)) + { + Assert(pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq)); + + /* + * Apply Interrupt Source Override rules. + * See ACPI 4.0 specification 5.2.12.4 and 5.2.12.5 for details on + * interrupt source override. + * Shortly, ISA IRQ0 is electically connected to pin 2 on IO-APIC, and some OSes, + * notably recent OS X rely upon this configuration. + * If changing, also update override rules in MADT and MPS. + */ + /* ISA IRQ0 routed to pin 2, all others ISA sources are identity mapped */ + if (u8Irq == 0) + u8Irq = 2; + + pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq)(pVM->pdm.s.IoApic.CTX_SUFF(pDevIns), NIL_PCIBDF, u8Irq, u8Level, uTagSrc); + rc = VINF_SUCCESS; + } + + if (!uTagSrc && u8Level == PDM_IRQ_LEVEL_LOW) + VBOXVMM_PDM_IRQ_LOW(VMMGetCpu(pVM), 0, 0); + pdmUnlock(pVM); + return rc; +} + + +/** + * Sets the pending I/O APIC interrupt. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param u8Irq The IRQ line. + * @param uBusDevFn The bus:device:function of the device initiating the IRQ. + * Pass NIL_PCIBDF when it's not a PCI device or interrupt. + * @param u8Level The new level. + * @param uTagSrc The IRQ tag and source tracer ID. + */ +VMM_INT_DECL(int) PDMIoApicSetIrq(PVM pVM, PCIBDF uBusDevFn, uint8_t u8Irq, uint8_t u8Level, uint32_t uTagSrc) +{ + Log9(("PDMIoApicSetIrq: irq=%#x lvl=%u tag=%#x src=%#x\n", u8Irq, u8Level, uTagSrc, uBusDevFn)); + if (pVM->pdm.s.IoApic.CTX_SUFF(pDevIns)) + { + Assert(pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq)); + pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq)(pVM->pdm.s.IoApic.CTX_SUFF(pDevIns), uBusDevFn, u8Irq, u8Level, uTagSrc); + return VINF_SUCCESS; + } + return VERR_PDM_NO_PIC_INSTANCE; +} + + +/** + * Broadcasts an EOI to the I/O APIC(s). + * + * @param pVM The cross context VM structure. + * @param uVector The interrupt vector corresponding to the EOI. + */ +VMM_INT_DECL(void) PDMIoApicBroadcastEoi(PVMCC pVM, uint8_t uVector) +{ + /* + * At present, we support only a maximum of one I/O APIC per-VM. If we ever implement having + * multiple I/O APICs per-VM, we'll have to broadcast this EOI to all of the I/O APICs. + */ + PCPDMIOAPIC pIoApic = &pVM->pdm.s.IoApic; +#ifdef IN_RING0 + if (pIoApic->pDevInsR0) + { + Assert(pIoApic->pfnSetEoiR0); + pIoApic->pfnSetEoiR0(pIoApic->pDevInsR0, uVector); + } + else if (pIoApic->pDevInsR3) + { + /* Queue for ring-3 execution. */ + PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)PDMQueueAlloc(pVM, pVM->pdm.s.hDevHlpQueue, pVM); + if (pTask) + { + pTask->enmOp = PDMDEVHLPTASKOP_IOAPIC_SET_EOI; + pTask->pDevInsR3 = NIL_RTR3PTR; /* not required */ + pTask->u.IoApicSetEoi.uVector = uVector; + PDMQueueInsert(pVM, pVM->pdm.s.hDevHlpQueue, pVM, &pTask->Core); + } + else + AssertMsgFailed(("We're out of devhlp queue items!!!\n")); + } +#else + if (pIoApic->pDevInsR3) + { + Assert(pIoApic->pfnSetEoiR3); + pIoApic->pfnSetEoiR3(pIoApic->pDevInsR3, uVector); + } +#endif +} + + +/** + * Send a MSI to an I/O APIC. + * + * @param pVM The cross context VM structure. + * @param uBusDevFn The bus:device:function of the device initiating the MSI. + * @param pMsi The MSI to send. + * @param uTagSrc The IRQ tag and source tracer ID. + */ +VMM_INT_DECL(void) PDMIoApicSendMsi(PVMCC pVM, PCIBDF uBusDevFn, PCMSIMSG pMsi, uint32_t uTagSrc) +{ + Log9(("PDMIoApicSendMsi: addr=%#RX64 data=%#RX32 tag=%#x src=%#x\n", pMsi->Addr.u64, pMsi->Data.u32, uTagSrc, uBusDevFn)); + PCPDMIOAPIC pIoApic = &pVM->pdm.s.IoApic; +#ifdef IN_RING0 + if (pIoApic->pDevInsR0) + pIoApic->pfnSendMsiR0(pIoApic->pDevInsR0, uBusDevFn, pMsi, uTagSrc); + else if (pIoApic->pDevInsR3) + { + /* Queue for ring-3 execution. */ + PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)PDMQueueAlloc(pVM, pVM->pdm.s.hDevHlpQueue, pVM); + if (pTask) + { + pTask->enmOp = PDMDEVHLPTASKOP_IOAPIC_SEND_MSI; + pTask->pDevInsR3 = NIL_RTR3PTR; /* not required */ + pTask->u.IoApicSendMsi.uBusDevFn = uBusDevFn; + pTask->u.IoApicSendMsi.Msi = *pMsi; + pTask->u.IoApicSendMsi.uTagSrc = uTagSrc; + PDMQueueInsert(pVM, pVM->pdm.s.hDevHlpQueue, pVM, &pTask->Core); + } + else + AssertMsgFailed(("We're out of devhlp queue items!!!\n")); + } +#else + if (pIoApic->pDevInsR3) + { + Assert(pIoApic->pfnSendMsiR3); + pIoApic->pfnSendMsiR3(pIoApic->pDevInsR3, uBusDevFn, pMsi, uTagSrc); + } +#endif +} + + + +/** + * Returns the presence of an IO-APIC. + * + * @returns true if an IO-APIC is present. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) PDMHasIoApic(PVM pVM) +{ + return pVM->pdm.s.IoApic.pDevInsR3 != NULL; +} + + +/** + * Returns the presence of an APIC. + * + * @returns true if an APIC is present. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) PDMHasApic(PVM pVM) +{ + return pVM->pdm.s.Apic.pDevInsR3 != NIL_RTR3PTR; +} + + +/** + * Translates a ring-0 device instance index to a pointer. + * + * This is used by PGM for device access handlers. + * + * @returns Device instance pointer if valid index, otherwise NULL (asserted). + * @param pVM The cross context VM structure. + * @param idxR0Device The ring-0 device instance index. + */ +VMM_INT_DECL(PPDMDEVINS) PDMDeviceRing0IdxToInstance(PVMCC pVM, uint64_t idxR0Device) +{ +#ifdef IN_RING0 + AssertMsgReturn(idxR0Device < RT_ELEMENTS(pVM->pdmr0.s.apDevInstances), ("%#RX64\n", idxR0Device), NULL); + PPDMDEVINS pDevIns = pVM->pdmr0.s.apDevInstances[idxR0Device]; +#elif defined(IN_RING3) + AssertMsgReturn(idxR0Device < RT_ELEMENTS(pVM->pdm.s.apDevRing0Instances), ("%#RX64\n", idxR0Device), NULL); + PPDMDEVINS pDevIns = pVM->pdm.s.apDevRing0Instances[idxR0Device]; +#else +# error "Unsupported context" +#endif + AssertMsg(pDevIns, ("%#RX64\n", idxR0Device)); + return pDevIns; +} + + +/** + * Locks PDM. + * + * This might block. + * + * @param pVM The cross context VM structure. + */ +void pdmLock(PVMCC pVM) +{ + int rc = PDMCritSectEnter(pVM, &pVM->pdm.s.CritSect, VINF_SUCCESS); + PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pdm.s.CritSect, rc); +} + + +/** + * Locks PDM but don't go to ring-3 if it's owned by someone. + * + * @returns VINF_SUCCESS on success. + * @returns rc if we're in GC or R0 and can't get the lock. + * @param pVM The cross context VM structure. + * @param rcBusy The RC to return in GC or R0 when we can't get the lock. + */ +int pdmLockEx(PVMCC pVM, int rcBusy) +{ + return PDMCritSectEnter(pVM, &pVM->pdm.s.CritSect, rcBusy); +} + + +/** + * Unlocks PDM. + * + * @param pVM The cross context VM structure. + */ +void pdmUnlock(PVMCC pVM) +{ + PDMCritSectLeave(pVM, &pVM->pdm.s.CritSect); +} + + +/** + * Checks if this thread is owning the PDM lock. + * + * @returns @c true if the lock is taken, @c false otherwise. + * @param pVM The cross context VM structure. + */ +bool pdmLockIsOwner(PVMCC pVM) +{ + return PDMCritSectIsOwner(pVM, &pVM->pdm.s.CritSect); +} + + +/** + * Converts ring 3 VMM heap pointer to a guest physical address + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pv Ring-3 pointer. + * @param pGCPhys GC phys address (out). + */ +VMM_INT_DECL(int) PDMVmmDevHeapR3ToGCPhys(PVM pVM, RTR3PTR pv, RTGCPHYS *pGCPhys) +{ + if (RT_LIKELY(pVM->pdm.s.GCPhysVMMDevHeap != NIL_RTGCPHYS)) + { + RTR3UINTPTR const offHeap = (RTR3UINTPTR)pv - (RTR3UINTPTR)pVM->pdm.s.pvVMMDevHeap; + if (RT_LIKELY(offHeap < pVM->pdm.s.cbVMMDevHeap)) + { + *pGCPhys = pVM->pdm.s.GCPhysVMMDevHeap + offHeap; + return VINF_SUCCESS; + } + + /* Don't assert here as this is called before we can catch ring-0 assertions. */ + Log(("PDMVmmDevHeapR3ToGCPhys: pv=%p pvVMMDevHeap=%p cbVMMDevHeap=%#x\n", + pv, pVM->pdm.s.pvVMMDevHeap, pVM->pdm.s.cbVMMDevHeap)); + } + else + Log(("PDMVmmDevHeapR3ToGCPhys: GCPhysVMMDevHeap=%RGp (pv=%p)\n", pVM->pdm.s.GCPhysVMMDevHeap, pv)); + return VERR_PDM_DEV_HEAP_R3_TO_GCPHYS; +} + + +/** + * Checks if the vmm device heap is enabled (== vmm device's pci region mapped) + * + * @returns dev heap enabled status (true/false) + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) PDMVmmDevHeapIsEnabled(PVM pVM) +{ + return pVM->pdm.s.GCPhysVMMDevHeap != NIL_RTGCPHYS; +} diff --git a/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp b/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp new file mode 100644 index 00000000..2a190f90 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp @@ -0,0 +1,1208 @@ +/* $Id: PDMAllCritSect.cpp $ */ +/** @file + * PDM - Write-Only Critical Section, All Contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PDM_CRITSECT +#include "PDMInternal.h" +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#ifdef IN_RING3 +# include +#endif +#if defined(IN_RING3) || defined(IN_RING0) +# include +#endif +#ifdef IN_RING0 +# include +#endif +#if defined(IN_RING3) || defined(IN_RING0) +# include +#endif + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** The number loops to spin for in ring-3. */ +#define PDMCRITSECT_SPIN_COUNT_R3 20 +/** The number loops to spin for in ring-0. */ +#define PDMCRITSECT_SPIN_COUNT_R0 256 +/** The number loops to spin for in the raw-mode context. */ +#define PDMCRITSECT_SPIN_COUNT_RC 256 + + +/** Skips some of the overly paranoid atomic updates. + * Makes some assumptions about cache coherence, though not brave enough not to + * always end with an atomic update. */ +#define PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + +/* Undefine the automatic VBOX_STRICT API mappings. */ +#undef PDMCritSectEnter +#undef PDMCritSectTryEnter + + +/** + * Gets the ring-3 native thread handle of the calling thread. + * + * @returns native thread handle (ring-3). + * @param pVM The cross context VM structure. + * @param pCritSect The critical section. This is used in R0 and RC. + */ +DECL_FORCE_INLINE(RTNATIVETHREAD) pdmCritSectGetNativeSelf(PVMCC pVM, PCPDMCRITSECT pCritSect) +{ +#ifdef IN_RING3 + RT_NOREF(pVM, pCritSect); + RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf(); + +#elif defined(IN_RING0) + AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%RX32\n", pCritSect->s.Core.u32Magic), + NIL_RTNATIVETHREAD); + RTNATIVETHREAD hNativeSelf = GVMMR0GetRing3ThreadForSelf(pVM); + Assert(hNativeSelf != NIL_RTNATIVETHREAD); + +#else +# error "Invalid context" +#endif + return hNativeSelf; +} + + +#ifdef IN_RING0 +/** + * Marks the critical section as corrupted. + */ +DECL_NO_INLINE(static, int) pdmCritSectCorrupted(PPDMCRITSECT pCritSect, const char *pszMsg) +{ + ASMAtomicWriteU32(&pCritSect->s.Core.u32Magic, PDMCRITSECT_MAGIC_CORRUPTED); + LogRel(("PDMCritSect: %s pCritSect=%p\n", pszMsg, pCritSect)); + return VERR_PDM_CRITSECT_IPE; +} +#endif + + +/** + * Tail code called when we've won the battle for the lock. + * + * @returns VINF_SUCCESS. + * + * @param pCritSect The critical section. + * @param hNativeSelf The native handle of this thread. + * @param pSrcPos The source position of the lock operation. + */ +DECL_FORCE_INLINE(int) pdmCritSectEnterFirst(PPDMCRITSECT pCritSect, RTNATIVETHREAD hNativeSelf, PCRTLOCKVALSRCPOS pSrcPos) +{ + Assert(hNativeSelf != NIL_RTNATIVETHREAD); + AssertMsg(pCritSect->s.Core.NativeThreadOwner == NIL_RTNATIVETHREAD, ("NativeThreadOwner=%p\n", pCritSect->s.Core.NativeThreadOwner)); + Assert(!(pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK)); + +# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + pCritSect->s.Core.cNestings = 1; +# else + ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1); +# endif + Assert(pCritSect->s.Core.cNestings == 1); + ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeSelf); + +# ifdef PDMCRITSECT_STRICT + RTLockValidatorRecExclSetOwner(pCritSect->s.Core.pValidatorRec, NIL_RTTHREAD, pSrcPos, true); +# else + NOREF(pSrcPos); +# endif + if (pSrcPos) + Log12Func(("%p: uId=%p ln=%u fn=%s\n", pCritSect, pSrcPos->uId, pSrcPos->uLine, pSrcPos->pszFunction)); + else + Log12Func(("%p\n", pCritSect)); + + STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l); + return VINF_SUCCESS; +} + + +#if defined(IN_RING3) || defined(IN_RING0) +/** + * Deals with the contended case in ring-3 and ring-0. + * + * @retval VINF_SUCCESS on success. + * @retval VERR_SEM_DESTROYED if destroyed. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure if ring-0 and on + * an EMT, otherwise NULL. + * @param pCritSect The critsect. + * @param hNativeSelf The native thread handle. + * @param pSrcPos The source position of the lock operation. + * @param rcBusy The status code to return when we're in RC or R0 + */ +static int pdmR3R0CritSectEnterContended(PVMCC pVM, PVMCPU pVCpu, PPDMCRITSECT pCritSect, RTNATIVETHREAD hNativeSelf, + PCRTLOCKVALSRCPOS pSrcPos, int rcBusy) +{ +# ifdef IN_RING0 + /* + * If we've got queued critical section leave operations and rcBusy isn't + * VINF_SUCCESS, return to ring-3 immediately to avoid deadlocks. + */ + if ( !pVCpu + || !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT) + || rcBusy == VINF_SUCCESS ) + { /* likely */ } + else + { + /** @todo statistics. */ + STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock); + return rcBusy; + } +# endif + + /* + * Start waiting. + */ + if (ASMAtomicIncS32(&pCritSect->s.Core.cLockers) == 0) + return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); +# ifdef IN_RING3 + STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionR3); +# else + STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock); +# endif + + /* + * The wait loop. + * + * This handles VERR_TIMEOUT and VERR_INTERRUPTED. + */ + STAM_REL_PROFILE_START(&pCritSect->s.CTX_MID_Z(StatContention,Wait), a); + PSUPDRVSESSION const pSession = pVM->pSession; + SUPSEMEVENT const hEvent = (SUPSEMEVENT)pCritSect->s.Core.EventSem; +# ifdef IN_RING3 +# ifdef PDMCRITSECT_STRICT + RTTHREAD const hThreadSelf = RTThreadSelfAutoAdopt(); + int rc2 = RTLockValidatorRecExclCheckOrder(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); + if (RT_FAILURE(rc2)) + return rc2; +# else + RTTHREAD const hThreadSelf = RTThreadSelf(); +# endif +# else /* IN_RING0 */ + uint64_t const tsStart = RTTimeNanoTS(); + uint64_t const cNsMaxTotalDef = RT_NS_5MIN; + uint64_t cNsMaxTotal = cNsMaxTotalDef; + uint64_t const cNsMaxRetry = RT_NS_15SEC; + uint32_t cMsMaxOne = RT_MS_5SEC; + bool fNonInterruptible = false; +# endif + for (;;) + { + /* + * Do the wait. + * + * In ring-3 this gets cluttered by lock validation and thread state + * maintainence. + * + * In ring-0 we have to deal with the possibility that the thread has + * been signalled and the interruptible wait function returning + * immediately. In that case we do normal R0/RC rcBusy handling. + * + * We always do a timed wait here, so the event handle is revalidated + * regularly and we won't end up stuck waiting for a destroyed critsect. + */ + /** @todo Make SUPSemEventClose wake up all waiters. */ +# ifdef IN_RING3 +# ifdef PDMCRITSECT_STRICT + int rc9 = RTLockValidatorRecExclCheckBlocking(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos, + !(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NO_NESTING), + RT_INDEFINITE_WAIT, RTTHREADSTATE_CRITSECT, true); + if (RT_FAILURE(rc9)) + return rc9; +# else + RTThreadBlocking(hThreadSelf, RTTHREADSTATE_CRITSECT, true); +# endif + int const rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_MS_5SEC); + RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_CRITSECT); +# else /* IN_RING0 */ + int const rc = !fNonInterruptible + ? SUPSemEventWaitNoResume(pSession, hEvent, cMsMaxOne) + : SUPSemEventWait(pSession, hEvent, cMsMaxOne); + Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hOwner=%p)\n", + pCritSect, rc, RTTimeNanoTS() - tsStart, cMsMaxOne, pCritSect->s.Core.NativeThreadOwner)); +# endif /* IN_RING0 */ + + /* + * Make sure the critical section hasn't been delete before continuing. + */ + if (RT_LIKELY(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC)) + { /* likely */ } + else + { + LogRel(("PDMCritSectEnter: Destroyed while waiting; pCritSect=%p rc=%Rrc\n", pCritSect, rc)); + return VERR_SEM_DESTROYED; + } + + /* + * Most likely we're here because we got signalled. + */ + if (rc == VINF_SUCCESS) + { + STAM_REL_PROFILE_STOP(&pCritSect->s.CTX_MID_Z(StatContention,Wait), a); + return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); + } + + /* + * Timeout and interrupted waits needs careful handling in ring-0 + * because we're cooperating with ring-3 on this critical section + * and thus need to make absolutely sure we won't get stuck here. + * + * The r0 interrupted case means something is pending (termination, + * signal, APC, debugger, whatever), so we must try our best to + * return to the caller and to ring-3 so it can be dealt with. + */ + if (RT_LIKELY(rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED)) + { +# ifdef IN_RING0 + uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart; + int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD); + AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING, + ("rcTerm=%Rrc\n", rcTerm)); + if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef) + cNsMaxTotal = RT_NS_1MIN; + + if (rc == VERR_TIMEOUT) + { + /* Try return get out of here with a non-VINF_SUCCESS status if + the thread is terminating or if the timeout has been exceeded. */ + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectVerrTimeout); + if ( rcTerm != VINF_THREAD_IS_TERMINATING + && cNsElapsed <= cNsMaxTotal) + continue; + } + else + { + /* For interrupt cases, we must return if we can. If rcBusy is VINF_SUCCESS, + we will try non-interruptible sleep for a while to help resolve the issue + w/o guru'ing. */ + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectVerrInterrupted); + if ( rcTerm != VINF_THREAD_IS_TERMINATING + && rcBusy == VINF_SUCCESS + && pVCpu != NULL + && cNsElapsed <= cNsMaxTotal) + { + if (!fNonInterruptible) + { + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectNonInterruptibleWaits); + fNonInterruptible = true; + cMsMaxOne = 32; + uint64_t cNsLeft = cNsMaxTotal - cNsElapsed; + if (cNsLeft > RT_NS_10SEC) + cNsMaxTotal = cNsElapsed + RT_NS_10SEC; + } + continue; + } + } + + /* + * Let try get out of here. We must very carefully undo the + * cLockers increment we did using compare-and-exchange so that + * we don't race the semaphore signalling in PDMCritSectLeave + * and end up with spurious wakeups and two owners at once. + */ + uint32_t cNoIntWaits = 0; + uint32_t cCmpXchgs = 0; + int32_t cLockers = ASMAtomicReadS32(&pCritSect->s.Core.cLockers); + for (;;) + { + if (pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC) + { + if (cLockers > 0 && cCmpXchgs < _64M) + { + bool fRc = ASMAtomicCmpXchgExS32(&pCritSect->s.Core.cLockers, cLockers - 1, cLockers, &cLockers); + if (fRc) + { + LogFunc(("Aborting wait on %p (rc=%Rrc rcTerm=%Rrc cNsElapsed=%'RU64) -> %Rrc\n", pCritSect, + rc, rcTerm, cNsElapsed, rcBusy != VINF_SUCCESS ? rcBusy : rc)); + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatAbortedCritSectEnters); + return rcBusy != VINF_SUCCESS ? rcBusy : rc; + } + cCmpXchgs++; + if ((cCmpXchgs & 0xffff) == 0) + Log11Func(("%p: cLockers=%d cCmpXchgs=%u (hOwner=%p)\n", + pCritSect, cLockers, cCmpXchgs, pCritSect->s.Core.NativeThreadOwner)); + ASMNopPause(); + continue; + } + + if (cLockers == 0) + { + /* + * We are racing someone in PDMCritSectLeave. + * + * For the VERR_TIMEOUT case we'll just retry taking it the normal + * way for a while. For VERR_INTERRUPTED we're in for more fun as + * the previous owner might not have signalled the semaphore yet, + * so we'll do a short non-interruptible wait instead and then guru. + */ + if ( rc == VERR_TIMEOUT + && RTTimeNanoTS() - tsStart <= cNsMaxTotal + cNsMaxRetry) + break; + + if ( rc == VERR_INTERRUPTED + && ( cNoIntWaits == 0 + || RTTimeNanoTS() - (tsStart + cNsElapsed) < RT_NS_100MS)) + { + int const rc2 = SUPSemEventWait(pSession, hEvent, 1 /*ms*/); + if (rc2 == VINF_SUCCESS) + { + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectEntersWhileAborting); + STAM_REL_PROFILE_STOP(&pCritSect->s.CTX_MID_Z(StatContention,Wait), a); + return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); + } + cNoIntWaits++; + cLockers = ASMAtomicReadS32(&pCritSect->s.Core.cLockers); + continue; + } + } + else + LogFunc(("Critical section %p has a broken cLockers count. Aborting.\n", pCritSect)); + + /* Sabotage the critical section and return error to caller. */ + ASMAtomicWriteU32(&pCritSect->s.Core.u32Magic, PDMCRITSECT_MAGIC_FAILED_ABORT); + LogRel(("PDMCritSectEnter: Failed to abort wait on pCritSect=%p (rc=%Rrc rcTerm=%Rrc)\n", + pCritSect, rc, rcTerm)); + return VERR_PDM_CRITSECT_ABORT_FAILED; + } + LogRel(("PDMCritSectEnter: Destroyed while aborting wait; pCritSect=%p/%#x rc=%Rrc rcTerm=%Rrc\n", + pCritSect, pCritSect->s.Core.u32Magic, rc, rcTerm)); + return VERR_SEM_DESTROYED; + } + + /* We get here if we timed out. Just retry now that it + appears someone left already. */ + Assert(rc == VERR_TIMEOUT); + cMsMaxOne = 10 /*ms*/; + +# else /* IN_RING3 */ + RT_NOREF(pVM, pVCpu, rcBusy); +# endif /* IN_RING3 */ + } + /* + * Any other return code is fatal. + */ + else + { + AssertMsgFailed(("rc=%Rrc\n", rc)); + return RT_FAILURE_NP(rc) ? rc : -rc; + } + } + /* won't get here */ +} +#endif /* IN_RING3 || IN_RING0 */ + + +/** + * Common worker for the debug and normal APIs. + * + * @returns VINF_SUCCESS if entered successfully. + * @returns rcBusy when encountering a busy critical section in RC/R0. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pCritSect The PDM critical section to enter. + * @param rcBusy The status code to return when we're in RC or R0 + * @param pSrcPos The source position of the lock operation. + */ +DECL_FORCE_INLINE(int) pdmCritSectEnter(PVMCC pVM, PPDMCRITSECT pCritSect, int rcBusy, PCRTLOCKVALSRCPOS pSrcPos) +{ + Assert(pCritSect->s.Core.cNestings < 8); /* useful to catch incorrect locking */ + Assert(pCritSect->s.Core.cNestings >= 0); +#if defined(VBOX_STRICT) && defined(IN_RING0) + /* Hope we're not messing with critical sections while in the no-block + zone, that would complicate things a lot. */ + PVMCPUCC pVCpuAssert = VMMGetCpu(pVM); + Assert(pVCpuAssert && VMMRZCallRing3IsEnabled(pVCpuAssert)); +#endif + + /* + * If the critical section has already been destroyed, then inform the caller. + */ + AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, + ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic), + VERR_SEM_DESTROYED); + + /* + * See if we're lucky. + */ + /* NOP ... */ + if (!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP)) + { /* We're more likely to end up here with real critsects than a NOP one. */ } + else + return VINF_SUCCESS; + + RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pVM, pCritSect); + AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT); + /* ... not owned ... */ + if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1)) + return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); + + /* ... or nested. */ + if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf) + { + Assert(pCritSect->s.Core.cNestings >= 1); +# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + pCritSect->s.Core.cNestings += 1; +# else + ASMAtomicIncS32(&pCritSect->s.Core.cNestings); +# endif + ASMAtomicIncS32(&pCritSect->s.Core.cLockers); + Log12Func(("%p: cNestings=%d cLockers=%d\n", pCritSect, pCritSect->s.Core.cNestings, pCritSect->s.Core.cLockers)); + return VINF_SUCCESS; + } + + /* + * Spin for a bit without incrementing the counter. + */ + /** @todo Move this to cfgm variables since it doesn't make sense to spin on UNI + * cpu systems. */ + int32_t cSpinsLeft = CTX_SUFF(PDMCRITSECT_SPIN_COUNT_); + while (cSpinsLeft-- > 0) + { + if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1)) + return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); + ASMNopPause(); + /** @todo Should use monitor/mwait on e.g. &cLockers here, possibly with a + cli'ed pendingpreemption check up front using sti w/ instruction fusing + for avoiding races. Hmm ... This is assuming the other party is actually + executing code on another CPU ... which we could keep track of if we + wanted. */ + } + +#ifdef IN_RING3 + /* + * Take the slow path. + */ + NOREF(rcBusy); + return pdmR3R0CritSectEnterContended(pVM, NULL, pCritSect, hNativeSelf, pSrcPos, rcBusy); + +#elif defined(IN_RING0) +# if 1 /* new code */ + /* + * In ring-0 context we have to take the special VT-x/AMD-V HM context into + * account when waiting on contended locks. + * + * While we usually (it can be VINF_SUCCESS) have the option of returning + * rcBusy and force the caller to go back to ring-3 and to re-start the work + * there, it's almost always more efficient to try wait for the lock here. + * The rcBusy will be used if we encounter an VERR_INTERRUPTED situation + * though. + */ + PVMCPUCC pVCpu = VMMGetCpu(pVM); + if (pVCpu) + { + VMMR0EMTBLOCKCTX Ctx; + int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pCritSect, &Ctx); + if (rc == VINF_SUCCESS) + { + Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + + rc = pdmR3R0CritSectEnterContended(pVM, pVCpu, pCritSect, hNativeSelf, pSrcPos, rcBusy); + + VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); + } + else + STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLockBusy); + return rc; + } + + /* Non-EMT. */ + Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + return pdmR3R0CritSectEnterContended(pVM, NULL, pCritSect, hNativeSelf, pSrcPos, rcBusy); + +# else /* old code: */ + /* + * We preemption hasn't been disabled, we can block here in ring-0. + */ + if ( RTThreadPreemptIsEnabled(NIL_RTTHREAD) + && ASMIntAreEnabled()) + return pdmR3R0CritSectEnterContended(pVM, VMMGetCpu(pVM), pCritSect, hNativeSelf, pSrcPos, rcBusy); + + STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock); + + /* + * Call ring-3 to acquire the critical section? + */ + if (rcBusy == VINF_SUCCESS) + { + PVMCPUCC pVCpu = VMMGetCpu(pVM); + AssertReturn(pVCpu, VERR_PDM_CRITSECT_IPE); + return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_ENTER, MMHyperCCToR3(pVM, pCritSect)); + } + + /* + * Return busy. + */ + LogFlow(("PDMCritSectEnter: locked => R3 (%Rrc)\n", rcBusy)); + return rcBusy; +# endif /* old code */ +#else +# error "Unsupported context" +#endif +} + + +/** + * Enters a PDM critical section. + * + * @returns VINF_SUCCESS if entered successfully. + * @returns rcBusy when encountering a busy critical section in RC/R0. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pCritSect The PDM critical section to enter. + * @param rcBusy The status code to return when we're in RC or R0 + * and the section is busy. Pass VINF_SUCCESS to + * acquired the critical section thru a ring-3 + * call if necessary. + * + * @note Even callers setting @a rcBusy to VINF_SUCCESS must either handle + * possible failures in ring-0 or apply + * PDM_CRITSECT_RELEASE_ASSERT_RC(), + * PDM_CRITSECT_RELEASE_ASSERT_RC_DEV(), + * PDM_CRITSECT_RELEASE_ASSERT_RC_DRV() or + * PDM_CRITSECT_RELEASE_ASSERT_RC_USB() to the return value of this + * function. + */ +VMMDECL(DECL_CHECK_RETURN_NOT_R3(int)) PDMCritSectEnter(PVMCC pVM, PPDMCRITSECT pCritSect, int rcBusy) +{ +#ifndef PDMCRITSECT_STRICT + return pdmCritSectEnter(pVM, pCritSect, rcBusy, NULL); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); + return pdmCritSectEnter(pVM, pCritSect, rcBusy, &SrcPos); +#endif +} + + +/** + * Enters a PDM critical section, with location information for debugging. + * + * @returns VINF_SUCCESS if entered successfully. + * @returns rcBusy when encountering a busy critical section in RC/R0. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pCritSect The PDM critical section to enter. + * @param rcBusy The status code to return when we're in RC or R0 + * and the section is busy. Pass VINF_SUCCESS to + * acquired the critical section thru a ring-3 + * call if necessary. + * @param uId Some kind of locking location ID. Typically a + * return address up the stack. Optional (0). + * @param SRC_POS The source position where to lock is being + * acquired from. Optional. + */ +VMMDECL(DECL_CHECK_RETURN_NOT_R3(int)) +PDMCritSectEnterDebug(PVMCC pVM, PPDMCRITSECT pCritSect, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL) +{ +#ifdef PDMCRITSECT_STRICT + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); + return pdmCritSectEnter(pVM, pCritSect, rcBusy, &SrcPos); +#else + NOREF(uId); RT_SRC_POS_NOREF(); + return pdmCritSectEnter(pVM, pCritSect, rcBusy, NULL); +#endif +} + + +/** + * Common worker for the debug and normal APIs. + * + * @retval VINF_SUCCESS on success. + * @retval VERR_SEM_BUSY if the critsect was owned. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pCritSect The critical section. + * @param pSrcPos The source position of the lock operation. + */ +static int pdmCritSectTryEnter(PVMCC pVM, PPDMCRITSECT pCritSect, PCRTLOCKVALSRCPOS pSrcPos) +{ + /* + * If the critical section has already been destroyed, then inform the caller. + */ + AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, + ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic), + VERR_SEM_DESTROYED); + + /* + * See if we're lucky. + */ + /* NOP ... */ + if (!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP)) + { /* We're more likely to end up here with real critsects than a NOP one. */ } + else + return VINF_SUCCESS; + + RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pVM, pCritSect); + AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT); + /* ... not owned ... */ + if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1)) + return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos); + + /* ... or nested. */ + if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf) + { + Assert(pCritSect->s.Core.cNestings >= 1); +# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + pCritSect->s.Core.cNestings += 1; +# else + ASMAtomicIncS32(&pCritSect->s.Core.cNestings); +# endif + ASMAtomicIncS32(&pCritSect->s.Core.cLockers); + Log12Func(("%p: cNestings=%d cLockers=%d\n", pCritSect, pCritSect->s.Core.cNestings, pCritSect->s.Core.cLockers)); + return VINF_SUCCESS; + } + + /* no spinning */ + + /* + * Return busy. + */ +#ifdef IN_RING3 + STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionR3); +#else + STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLockBusy); +#endif + LogFlow(("PDMCritSectTryEnter: locked\n")); + return VERR_SEM_BUSY; +} + + +/** + * Try enter a critical section. + * + * @retval VINF_SUCCESS on success. + * @retval VERR_SEM_BUSY if the critsect was owned. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pCritSect The critical section. + */ +VMMDECL(DECL_CHECK_RETURN(int)) PDMCritSectTryEnter(PVMCC pVM, PPDMCRITSECT pCritSect) +{ +#ifndef PDMCRITSECT_STRICT + return pdmCritSectTryEnter(pVM, pCritSect, NULL); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); + return pdmCritSectTryEnter(pVM, pCritSect, &SrcPos); +#endif +} + + +/** + * Try enter a critical section, with location information for debugging. + * + * @retval VINF_SUCCESS on success. + * @retval VERR_SEM_BUSY if the critsect was owned. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pCritSect The critical section. + * @param uId Some kind of locking location ID. Typically a + * return address up the stack. Optional (0). + * @param SRC_POS The source position where to lock is being + * acquired from. Optional. + */ +VMMDECL(DECL_CHECK_RETURN(int)) +PDMCritSectTryEnterDebug(PVMCC pVM, PPDMCRITSECT pCritSect, RTHCUINTPTR uId, RT_SRC_POS_DECL) +{ +#ifdef PDMCRITSECT_STRICT + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); + return pdmCritSectTryEnter(pVM, pCritSect, &SrcPos); +#else + NOREF(uId); RT_SRC_POS_NOREF(); + return pdmCritSectTryEnter(pVM, pCritSect, NULL); +#endif +} + + +#ifdef IN_RING3 +/** + * Enters a PDM critical section. + * + * @returns VINF_SUCCESS if entered successfully. + * @returns rcBusy when encountering a busy critical section in GC/R0. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pCritSect The PDM critical section to enter. + * @param fCallRing3 Whether this is a VMMRZCallRing3()request. + */ +VMMR3DECL(int) PDMR3CritSectEnterEx(PVM pVM, PPDMCRITSECT pCritSect, bool fCallRing3) +{ + int rc = PDMCritSectEnter(pVM, pCritSect, VERR_IGNORED); + if ( rc == VINF_SUCCESS + && fCallRing3 + && pCritSect->s.Core.pValidatorRec + && pCritSect->s.Core.pValidatorRec->hThread != NIL_RTTHREAD) + RTLockValidatorRecExclReleaseOwnerUnchecked(pCritSect->s.Core.pValidatorRec); + return rc; +} +#endif /* IN_RING3 */ + + +/** + * Leaves a critical section entered with PDMCritSectEnter(). + * + * @returns Indication whether we really exited the critical section. + * @retval VINF_SUCCESS if we really exited. + * @retval VINF_SEM_NESTED if we only reduced the nesting count. + * @retval VERR_NOT_OWNER if you somehow ignore release assertions. + * + * @param pVM The cross context VM structure. + * @param pCritSect The PDM critical section to leave. + * + * @remarks Can be called from no-ring-3-call context in ring-0 (TM/VirtualSync) + * where we'll queue leaving operation for ring-3 processing. + */ +VMMDECL(int) PDMCritSectLeave(PVMCC pVM, PPDMCRITSECT pCritSect) +{ + AssertMsg(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic)); + Assert(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC); + + /* + * Check for NOP sections before asserting ownership. + */ + if (!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP)) + { /* We're more likely to end up here with real critsects than a NOP one. */ } + else + return VINF_SUCCESS; + + /* + * Always check that the caller is the owner (screw performance). + */ + RTNATIVETHREAD const hNativeSelf = pdmCritSectGetNativeSelf(pVM, pCritSect); + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, pCritSect->s.Core.NativeThreadOwner == hNativeSelf && hNativeSelf != NIL_RTNATIVETHREAD, + ("%p %s: %p != %p; cLockers=%d cNestings=%d\n", pCritSect, R3STRING(pCritSect->s.pszName), + pCritSect->s.Core.NativeThreadOwner, hNativeSelf, + pCritSect->s.Core.cLockers, pCritSect->s.Core.cNestings), + VERR_NOT_OWNER); + + /* + * Nested leave. + */ + int32_t const cNestings = pCritSect->s.Core.cNestings; + Assert(cNestings >= 1); + if (cNestings > 1) + { +#ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + pCritSect->s.Core.cNestings = cNestings - 1; +#else + ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, cNestings - 1); +#endif + int32_t const cLockers = ASMAtomicDecS32(&pCritSect->s.Core.cLockers); + Assert(cLockers >= 0); RT_NOREF(cLockers); + Log12Func(("%p: cNestings=%d cLockers=%d\n", pCritSect, cNestings - 1, cLockers)); + return VINF_SEM_NESTED; + } + + Log12Func(("%p: cNestings=%d cLockers=%d hOwner=%p - leave for real\n", + pCritSect, cNestings, pCritSect->s.Core.cLockers, pCritSect->s.Core.NativeThreadOwner)); + +#ifdef IN_RING3 + /* + * Ring-3: Leave for real. + */ + SUPSEMEVENT const hEventToSignal = pCritSect->s.hEventToSignal; + pCritSect->s.hEventToSignal = NIL_SUPSEMEVENT; + +# if defined(PDMCRITSECT_STRICT) + if (pCritSect->s.Core.pValidatorRec->hThread != NIL_RTTHREAD) + RTLockValidatorRecExclReleaseOwnerUnchecked(pCritSect->s.Core.pValidatorRec); +# endif + Assert(!pCritSect->s.Core.pValidatorRec || pCritSect->s.Core.pValidatorRec->hThread == NIL_RTTHREAD); + +# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + //pCritSect->s.Core.cNestings = 0; /* not really needed */ + pCritSect->s.Core.NativeThreadOwner = NIL_RTNATIVETHREAD; +# else + ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0); + ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD); +# endif + ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK); + + /* Stop profiling and decrement lockers. */ + STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l); + ASMCompilerBarrier(); + int32_t const cLockers = ASMAtomicDecS32(&pCritSect->s.Core.cLockers); + if (cLockers < 0) + AssertMsg(cLockers == -1, ("cLockers=%d\n", cLockers)); + else + { + /* Someone is waiting, wake up one of them. */ + Assert(cLockers < _8K); + Log8(("PDMCritSectLeave: Waking up %p (cLockers=%u)\n", pCritSect, cLockers)); + SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->s.Core.EventSem; + int rc = SUPSemEventSignal(pVM->pSession, hEvent); + AssertRC(rc); + } + + /* Signal exit event. */ + if (RT_LIKELY(hEventToSignal == NIL_SUPSEMEVENT)) + { /* likely */ } + else + { + Log8(("PDMCritSectLeave: Signalling %#p (%p)\n", hEventToSignal, pCritSect)); + int rc = SUPSemEventSignal(pVM->pSession, hEventToSignal); + AssertRC(rc); + } + + return VINF_SUCCESS; + + +#elif defined(IN_RING0) + /* + * Ring-0: Try leave for real, depends on host and context. + */ + SUPSEMEVENT const hEventToSignal = pCritSect->s.hEventToSignal; + pCritSect->s.hEventToSignal = NIL_SUPSEMEVENT; + PVMCPUCC pVCpu = VMMGetCpu(pVM); + bool fQueueOnTrouble = false; /* Set this to true to test queueing. */ + if ( pVCpu == NULL /* non-EMT access, if we implement it must be able to block */ + || VMMRZCallRing3IsEnabled(pVCpu) + || RTSemEventIsSignalSafe() + || ( VMMR0ThreadCtxHookIsEnabled(pVCpu) /* Doesn't matter if Signal() blocks if we have hooks, ... */ + && RTThreadPreemptIsEnabled(NIL_RTTHREAD) /* ... and preemption is still enabled, */ + && ASMIntAreEnabled()) /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */ + || (fQueueOnTrouble = ( hEventToSignal == NIL_SUPSEMEVENT + && ASMAtomicUoReadS32(&pCritSect->s.Core.cLockers) == 0)) ) + { + pCritSect->s.hEventToSignal = NIL_SUPSEMEVENT; + +# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + //pCritSect->s.Core.cNestings = 0; /* not really needed */ + pCritSect->s.Core.NativeThreadOwner = NIL_RTNATIVETHREAD; +# else + ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0); + ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD); +# endif + ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK); + + /* + * Stop profiling and decrement lockers. + */ + STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l); + ASMCompilerBarrier(); + + bool fQueueIt = false; + int32_t cLockers; + if (!fQueueOnTrouble) + cLockers = ASMAtomicDecS32(&pCritSect->s.Core.cLockers); + else + { + cLockers = -1; + if (!ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, -1, 0)) + fQueueIt = true; + } + if (!fQueueIt) + { + VMMR0EMTBLOCKCTX Ctx; + bool fLeaveCtx = false; + if (cLockers < 0) + AssertMsg(cLockers == -1, ("cLockers=%d\n", cLockers)); + else + { + /* Someone is waiting, wake up one of them. */ + Assert(cLockers < _8K); + SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->s.Core.EventSem; + if (!RTSemEventIsSignalSafe() && (pVCpu = VMMGetCpu(pVM)) != NULL) + { + int rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pCritSect, &Ctx); + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc); + fLeaveCtx = true; + } + int rc = SUPSemEventSignal(pVM->pSession, hEvent); + AssertRC(rc); + } + + /* + * Signal exit event. + */ + if (RT_LIKELY(hEventToSignal == NIL_SUPSEMEVENT)) + { /* likely */ } + else + { + if (!fLeaveCtx && pVCpu != NULL && !RTSemEventIsSignalSafe() && (pVCpu = VMMGetCpu(pVM)) != NULL) + { + int rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pCritSect, &Ctx); + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc); + fLeaveCtx = true; + } + Log8(("Signalling %#p\n", hEventToSignal)); + int rc = SUPSemEventSignal(pVM->pSession, hEventToSignal); + AssertRC(rc); + } + + /* + * Restore HM context if needed. + */ + if (!fLeaveCtx) + { /* contention should be unlikely */ } + else + VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); + +# ifdef DEBUG_bird + VMMTrashVolatileXMMRegs(); +# endif + return VINF_SUCCESS; + } + + /* + * Darn, someone raced in on us. Restore the state (this works only + * because the semaphore is effectively controlling ownership). + */ + bool fRc; + RTNATIVETHREAD hMessOwner = NIL_RTNATIVETHREAD; + ASMAtomicCmpXchgExHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeSelf, NIL_RTNATIVETHREAD, fRc, &hMessOwner); + AssertLogRelMsgReturn(fRc, ("pCritSect=%p hMessOwner=%p\n", pCritSect, hMessOwner), + pdmCritSectCorrupted(pCritSect, "owner race")); + STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l); +# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + //pCritSect->s.Core.cNestings = 1; + Assert(pCritSect->s.Core.cNestings == 1); +# else + //Assert(pCritSect->s.Core.cNestings == 0); + ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1); +# endif + Assert(hEventToSignal == NIL_SUPSEMEVENT); + } + + +#else /* IN_RC */ + /* + * Raw-mode: Try leave it. + */ +# error "This context is not use..." + if (pCritSect->s.Core.cLockers == 0) + { +# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + //pCritSect->s.Core.cNestings = 0; /* not really needed */ +# else + ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0); +# endif + ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK); + STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l); + + ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD); + if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, -1, 0)) + return VINF_SUCCESS; + + /* + * Darn, someone raced in on us. Restore the state (this works only + * because the semaphore is effectively controlling ownership). + */ + bool fRc; + RTNATIVETHREAD hMessOwner = NIL_RTNATIVETHREAD; + ASMAtomicCmpXchgExHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeSelf, NIL_RTNATIVETHREAD, fRc, &hMessOwner); + AssertLogRelMsgReturn(fRc, ("pCritSect=%p hMessOwner=%p\n", pCritSect, hMessOwner), + pdmCritSectCorrupted(pCritSect, "owner race")); + STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l); +# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF + //pCritSect->s.Core.cNestings = 1; + Assert(pCritSect->s.Core.cNestings == 1); +# else + //Assert(pCritSect->s.Core.cNestings == 0); + ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1); +# endif + } +#endif /* IN_RC */ + + +#ifndef IN_RING3 + /* + * Ring-0/raw-mode: Unable to leave. Queue the leave for ring-3. + */ + ASMAtomicOrU32(&pCritSect->s.Core.fFlags, PDMCRITSECT_FLAGS_PENDING_UNLOCK); +# ifndef IN_RING0 + PVMCPUCC pVCpu = VMMGetCpu(pVM); +# endif + uint32_t i = pVCpu->pdm.s.cQueuedCritSectLeaves++; + LogFlow(("PDMCritSectLeave: [%d]=%p => R3\n", i, pCritSect)); + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectLeaves), ("%d\n", i), VERR_PDM_CRITSECT_IPE); + pVCpu->pdm.s.apQueuedCritSectLeaves[i] = pCritSect->s.pSelfR3; + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, + RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectLeaves[i]) + && ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectLeaves[i] & HOST_PAGE_OFFSET_MASK) + == ((uintptr_t)pCritSect & HOST_PAGE_OFFSET_MASK), + ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectLeaves[i], pCritSect), + pdmCritSectCorrupted(pCritSect, "Invalid pSelfR3 value")); + VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT); /** @todo handle VMCPU_FF_PDM_CRITSECT in ring-0 outside the no-call-ring-3 part. */ + VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3); /* unnecessary paranoia */ + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves); + STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZUnlock); + + return VINF_SUCCESS; +#endif /* IN_RING3 */ +} + + +#if defined(IN_RING0) || defined(IN_RING3) +/** + * Schedule a event semaphore for signalling upon critsect exit. + * + * @returns VINF_SUCCESS on success. + * @returns VERR_TOO_MANY_SEMAPHORES if an event was already scheduled. + * @returns VERR_NOT_OWNER if we're not the critsect owner (ring-3 only). + * @returns VERR_SEM_DESTROYED if RTCritSectDelete was called while waiting. + * + * @param pCritSect The critical section. + * @param hEventToSignal The support driver event semaphore that should be + * signalled. + */ +VMMDECL(int) PDMHCCritSectScheduleExitEvent(PPDMCRITSECT pCritSect, SUPSEMEVENT hEventToSignal) +{ + AssertPtr(pCritSect); + Assert(!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP)); + Assert(hEventToSignal != NIL_SUPSEMEVENT); +# ifdef IN_RING3 + if (RT_UNLIKELY(!RTCritSectIsOwner(&pCritSect->s.Core))) + return VERR_NOT_OWNER; +# endif + if (RT_LIKELY( pCritSect->s.hEventToSignal == NIL_RTSEMEVENT + || pCritSect->s.hEventToSignal == hEventToSignal)) + { + pCritSect->s.hEventToSignal = hEventToSignal; + return VINF_SUCCESS; + } + return VERR_TOO_MANY_SEMAPHORES; +} +#endif /* IN_RING0 || IN_RING3 */ + + +/** + * Checks the caller is the owner of the critical section. + * + * @returns true if owner. + * @returns false if not owner. + * @param pVM The cross context VM structure. + * @param pCritSect The critical section. + */ +VMMDECL(bool) PDMCritSectIsOwner(PVMCC pVM, PCPDMCRITSECT pCritSect) +{ +#ifdef IN_RING3 + RT_NOREF(pVM); + return RTCritSectIsOwner(&pCritSect->s.Core); +#else + PVMCPUCC pVCpu = VMMGetCpu(pVM); + if ( !pVCpu + || pCritSect->s.Core.NativeThreadOwner != pVCpu->hNativeThread) + return false; + return (pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK) == 0 + || pCritSect->s.Core.cNestings > 1; +#endif +} + + +/** + * Checks the specified VCPU is the owner of the critical section. + * + * @returns true if owner. + * @returns false if not owner. + * @param pVCpu The cross context virtual CPU structure. + * @param pCritSect The critical section. + */ +VMMDECL(bool) PDMCritSectIsOwnerEx(PVMCPUCC pVCpu, PCPDMCRITSECT pCritSect) +{ +#ifdef IN_RING3 + NOREF(pVCpu); + return RTCritSectIsOwner(&pCritSect->s.Core); +#else + Assert(VMCC_GET_CPU(pVCpu->CTX_SUFF(pVM), pVCpu->idCpu) == pVCpu); + if (pCritSect->s.Core.NativeThreadOwner != pVCpu->hNativeThread) + return false; + return (pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK) == 0 + || pCritSect->s.Core.cNestings > 1; +#endif +} + + +/** + * Checks if anyone is waiting on the critical section we own. + * + * @returns true if someone is waiting. + * @returns false if no one is waiting. + * @param pVM The cross context VM structure. + * @param pCritSect The critical section. + */ +VMMDECL(bool) PDMCritSectHasWaiters(PVMCC pVM, PCPDMCRITSECT pCritSect) +{ + AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, false); + Assert(pCritSect->s.Core.NativeThreadOwner == pdmCritSectGetNativeSelf(pVM, pCritSect)); RT_NOREF(pVM); + return pCritSect->s.Core.cLockers >= pCritSect->s.Core.cNestings; +} + + +/** + * Checks if a critical section is initialized or not. + * + * @returns true if initialized. + * @returns false if not initialized. + * @param pCritSect The critical section. + */ +VMMDECL(bool) PDMCritSectIsInitialized(PCPDMCRITSECT pCritSect) +{ + return RTCritSectIsInitialized(&pCritSect->s.Core); +} + + +/** + * Gets the recursion depth. + * + * @returns The recursion depth. + * @param pCritSect The critical section. + */ +VMMDECL(uint32_t) PDMCritSectGetRecursion(PCPDMCRITSECT pCritSect) +{ + return RTCritSectGetRecursion(&pCritSect->s.Core); +} + diff --git a/src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp b/src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp new file mode 100644 index 00000000..3c9fb021 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp @@ -0,0 +1,118 @@ +/* $Id: PDMAllCritSectBoth.cpp $ */ +/** @file + * PDM - Code Common to Both Critical Section Types, All Contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PDM_CRITSECT +#include "PDMInternal.h" +#include +#include +#include +#include + +#include +#include +#include + + +#if defined(IN_RING3) /*|| defined(IN_RING0) - not called from ring-0 */ +/** + * Process the critical sections (both types) queued for ring-3 'leave'. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(void) PDMCritSectBothFF(PVMCC pVM, PVMCPUCC pVCpu) +{ + uint32_t i; + Assert( pVCpu->pdm.s.cQueuedCritSectLeaves > 0 + || pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves > 0 + || pVCpu->pdm.s.cQueuedCritSectRwExclLeaves > 0); + + /* Shared leaves. */ + i = pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves; + pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves = 0; + while (i-- > 0) + { +# ifdef IN_RING3 + PPDMCRITSECTRW pCritSectRw = pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i]; +# else + PPDMCRITSECTRW pCritSectRw = (PPDMCRITSECTRW)MMHyperR3ToCC(pVCpu->CTX_SUFF(pVM), + pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i]); +# endif + + pdmCritSectRwLeaveSharedQueued(pVM, pCritSectRw); + LogIt(RTLOGGRPFLAGS_FLOW, LOG_GROUP_PDM_CRITSECTRW, ("PDMR3CritSectFF: %p (shared)\n", pCritSectRw)); + } + + /* Last, exclusive leaves. */ + i = pVCpu->pdm.s.cQueuedCritSectRwExclLeaves; + pVCpu->pdm.s.cQueuedCritSectRwExclLeaves = 0; + while (i-- > 0) + { +# ifdef IN_RING3 + PPDMCRITSECTRW pCritSectRw = pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i]; +# else + PPDMCRITSECTRW pCritSectRw = (PPDMCRITSECTRW)MMHyperR3ToCC(pVCpu->CTX_SUFF(pVM), + pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i]); +# endif + + pdmCritSectRwLeaveExclQueued(pVM, pCritSectRw); + LogIt(RTLOGGRPFLAGS_FLOW, LOG_GROUP_PDM_CRITSECTRW, ("PDMR3CritSectFF: %p (exclusive)\n", pCritSectRw)); + } + + /* Normal leaves. */ + i = pVCpu->pdm.s.cQueuedCritSectLeaves; + pVCpu->pdm.s.cQueuedCritSectLeaves = 0; + while (i-- > 0) + { +# ifdef IN_RING3 + PPDMCRITSECT pCritSect = pVCpu->pdm.s.apQueuedCritSectLeaves[i]; +# else + PPDMCRITSECT pCritSect = (PPDMCRITSECT)MMHyperR3ToCC(pVCpu->CTX_SUFF(pVM), pVCpu->pdm.s.apQueuedCritSectLeaves[i]); +# endif + Assert(pCritSect->s.Core.NativeThreadOwner == pVCpu->hNativeThread); + + /* Note! We *must* clear the pending-unlock flag here and not depend on + PDMCritSectLeave to do it, as the EMT might be sitting on + further nestings since it queued the section to be left, and + leaving it set would throw subsequent PDMCritSectIsOwner calls. + + This will happen with the PGM lock if we nip back to ring-3 for + more handy pages or similar where the lock is supposed to be + held while in ring-3. */ + ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK); + PDMCritSectLeave(pVM, pCritSect); + LogFlow(("PDMR3CritSectFF: %p\n", pCritSect)); + } + + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PDM_CRITSECT); +} +#endif /* IN_RING3 || IN_RING0 */ + diff --git a/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp b/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp new file mode 100644 index 00000000..7e66d6ae --- /dev/null +++ b/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp @@ -0,0 +1,2134 @@ +/* $Id: PDMAllCritSectRw.cpp $ */ +/** @file + * IPRT - Read/Write Critical Section, Generic. + */ + +/* + * Copyright (C) 2009-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PDM_CRITSECTRW +#include "PDMInternal.h" +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#ifdef IN_RING3 +# include +#endif +#if defined(IN_RING3) || defined(IN_RING0) +# include +# include +#endif +#ifdef IN_RING0 +# include +#endif +#ifdef RT_ARCH_AMD64 +# include +#endif + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +#if 0 /* unused */ +/** The number loops to spin for shared access in ring-3. */ +#define PDMCRITSECTRW_SHRD_SPIN_COUNT_R3 20 +/** The number loops to spin for shared access in ring-0. */ +#define PDMCRITSECTRW_SHRD_SPIN_COUNT_R0 128 +/** The number loops to spin for shared access in the raw-mode context. */ +#define PDMCRITSECTRW_SHRD_SPIN_COUNT_RC 128 + +/** The number loops to spin for exclusive access in ring-3. */ +#define PDMCRITSECTRW_EXCL_SPIN_COUNT_R3 20 +/** The number loops to spin for exclusive access in ring-0. */ +#define PDMCRITSECTRW_EXCL_SPIN_COUNT_R0 256 +/** The number loops to spin for exclusive access in the raw-mode context. */ +#define PDMCRITSECTRW_EXCL_SPIN_COUNT_RC 256 +#endif + +/** Max number of write or write/read recursions. */ +#define PDM_CRITSECTRW_MAX_RECURSIONS _1M + +/** Skips some of the overly paranoid atomic reads and updates. + * Makes some assumptions about cache coherence, though not brave enough not to + * always end with an atomic update. */ +#define PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF + +/** For reading RTCRITSECTRWSTATE::s::u64State. */ +#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF +# define PDMCRITSECTRW_READ_STATE(a_pu64State) ASMAtomicUoReadU64(a_pu64State) +#else +# define PDMCRITSECTRW_READ_STATE(a_pu64State) ASMAtomicReadU64(a_pu64State) +#endif + + +/* Undefine the automatic VBOX_STRICT API mappings. */ +#undef PDMCritSectRwEnterExcl +#undef PDMCritSectRwTryEnterExcl +#undef PDMCritSectRwEnterShared +#undef PDMCritSectRwTryEnterShared + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +#if defined(RTASM_HAVE_CMP_WRITE_U128) && defined(RT_ARCH_AMD64) +static int32_t g_fCmpWriteSupported = -1; +#endif + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal); + + +#ifdef RTASM_HAVE_CMP_WRITE_U128 + +# ifdef RT_ARCH_AMD64 +/** + * Called once to initialize g_fCmpWriteSupported. + */ +DECL_NO_INLINE(static, bool) pdmCritSectRwIsCmpWriteU128SupportedSlow(void) +{ + bool const fCmpWriteSupported = RT_BOOL(ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_CX16); + ASMAtomicWriteS32(&g_fCmpWriteSupported, fCmpWriteSupported); + return fCmpWriteSupported; +} +# endif + + +/** + * Indicates whether hardware actually supports 128-bit compare & write. + */ +DECL_FORCE_INLINE(bool) pdmCritSectRwIsCmpWriteU128Supported(void) +{ +# ifdef RT_ARCH_AMD64 + int32_t const fCmpWriteSupported = g_fCmpWriteSupported; + if (RT_LIKELY(fCmpWriteSupported >= 0)) + return fCmpWriteSupported != 0; + return pdmCritSectRwIsCmpWriteU128SupportedSlow(); +# else + return true; +# endif +} + +#endif /* RTASM_HAVE_CMP_WRITE_U128 */ + +/** + * Gets the ring-3 native thread handle of the calling thread. + * + * @returns native thread handle (ring-3). + * @param pVM The cross context VM structure. + * @param pThis The read/write critical section. This is only used in + * R0 and RC. + */ +DECL_FORCE_INLINE(RTNATIVETHREAD) pdmCritSectRwGetNativeSelf(PVMCC pVM, PCPDMCRITSECTRW pThis) +{ +#ifdef IN_RING3 + RT_NOREF(pVM, pThis); + RTNATIVETHREAD hNativeSelf = RTThreadNativeSelf(); + +#elif defined(IN_RING0) + AssertMsgReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, ("%RX32\n", pThis->s.Core.u32Magic), + NIL_RTNATIVETHREAD); + RTNATIVETHREAD hNativeSelf = GVMMR0GetRing3ThreadForSelf(pVM); + Assert(hNativeSelf != NIL_RTNATIVETHREAD); + +#else +# error "invalid context" +#endif + return hNativeSelf; +} + + +DECL_NO_INLINE(static, int) pdmCritSectRwCorrupted(PPDMCRITSECTRW pThis, const char *pszMsg) +{ + ASMAtomicWriteU32(&pThis->s.Core.u32Magic, PDMCRITSECTRW_MAGIC_CORRUPT); + LogRel(("PDMCritSect: %s pCritSect=%p\n", pszMsg, pThis)); + return VERR_PDM_CRITSECTRW_IPE; +} + + + +#ifdef IN_RING3 +/** + * Changes the lock validator sub-class of the read/write critical section. + * + * It is recommended to try make sure that nobody is using this critical section + * while changing the value. + * + * @returns The old sub-class. RTLOCKVAL_SUB_CLASS_INVALID is returns if the + * lock validator isn't compiled in or either of the parameters are + * invalid. + * @param pThis Pointer to the read/write critical section. + * @param uSubClass The new sub-class value. + */ +VMMDECL(uint32_t) PDMR3CritSectRwSetSubClass(PPDMCRITSECTRW pThis, uint32_t uSubClass) +{ + AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID); +# if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + AssertReturn(!(pThis->s.Core.fFlags & RTCRITSECT_FLAGS_NOP), RTLOCKVAL_SUB_CLASS_INVALID); + + RTLockValidatorRecSharedSetSubClass(pThis->s.Core.pValidatorRead, uSubClass); + return RTLockValidatorRecExclSetSubClass(pThis->s.Core.pValidatorWrite, uSubClass); +# else + NOREF(uSubClass); + return RTLOCKVAL_SUB_CLASS_INVALID; +# endif +} +#endif /* IN_RING3 */ + + +/** + * Worker for pdmCritSectRwEnterShared returning with read-ownership of the CS. + */ +DECL_FORCE_INLINE(int) pdmCritSectRwEnterSharedGotIt(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos, + bool fNoVal, RTTHREAD hThreadSelf) +{ +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + if (!fNoVal) + RTLockValidatorRecSharedAddOwner(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos); +#else + RT_NOREF(pSrcPos, fNoVal, hThreadSelf); +#endif + + /* got it! */ + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared)); + Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)); + return VINF_SUCCESS; +} + +/** + * Worker for pdmCritSectRwEnterShared and pdmCritSectRwEnterSharedBailOut + * that decrement the wait count and maybe resets the semaphore. + */ +DECLINLINE(int) pdmCritSectRwEnterSharedGotItAfterWaiting(PVMCC pVM, PPDMCRITSECTRW pThis, uint64_t u64State, + PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf) +{ + for (;;) + { + uint64_t const u64OldState = u64State; + uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; + AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count")); + AssertReturn((u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT > 0, + pdmCritSectRwCorrupted(pThis, "Invalid read count")); + cWait--; + u64State &= ~RTCSRW_WAIT_CNT_RD_MASK; + u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT; + + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + { + if (cWait == 0) + { + if (ASMAtomicXchgBool(&pThis->s.Core.fNeedReset, false)) + { + int rc = SUPSemEventMultiReset(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead); + AssertRCReturn(rc, rc); + } + } + return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf); + } + + ASMNopPause(); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); + ASMNopPause(); + + u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + } + /* not reached */ +} + + +#if defined(IN_RING0) || (defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT)) +/** + * Worker for pdmCritSectRwEnterSharedContended that decrements both read counts + * and returns @a rc. + * + * @note May return VINF_SUCCESS if we race the exclusive leave function and + * come out on the bottom. + * + * Ring-3 only calls in a case where it is _not_ acceptable to take the + * lock, so even if we get the lock we'll have to leave. In the ring-0 + * contexts, we can safely return VINF_SUCCESS in case of a race. + */ +DECL_NO_INLINE(static, int) pdmCritSectRwEnterSharedBailOut(PVMCC pVM, PPDMCRITSECTRW pThis, int rc, + PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf) +{ +#ifdef IN_RING0 + uint64_t const tsStart = RTTimeNanoTS(); + uint64_t cNsElapsed = 0; +#endif + for (;;) + { + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + uint64_t u64OldState = u64State; + + uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; + AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count on bailout")); + cWait--; + + uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; + AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid read count on bailout")); + + if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)) + { + c--; + u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK); + u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT); + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + return rc; + } + else + { + /* + * The direction changed, so we can actually get the lock now. + * + * This means that we _have_ to wait on the semaphore to be signalled + * so we can properly reset it. Otherwise the stuff gets out of wack, + * because signalling and resetting will race one another. An + * exception would be if we're not the last reader waiting and don't + * need to worry about the resetting. + * + * An option would be to do the resetting in PDMCritSectRwEnterExcl, + * but that would still leave a racing PDMCritSectRwEnterShared + * spinning hard for a little bit, which isn't great... + */ + if (cWait == 0) + { +# ifdef IN_RING0 + /* Do timeout processing first to avoid redoing the above. */ + uint32_t cMsWait; + if (cNsElapsed <= RT_NS_10SEC) + cMsWait = 32; + else + { + u64State &= ~RTCSRW_WAIT_CNT_RD_MASK; + u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + { + LogFunc(("%p: giving up\n", pThis)); + return rc; + } + cMsWait = 2; + } + + int rcWait = SUPSemEventMultiWait(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, cMsWait); + Log11Func(("%p: rc=%Rrc %'RU64 ns (hNativeWriter=%p u64State=%#RX64)\n", pThis, rcWait, + RTTimeNanoTS() - tsStart, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State)); +# else + RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false); + int rcWait = SUPSemEventMultiWaitNoResume(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, RT_MS_5SEC); + RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); +# endif + if (rcWait == VINF_SUCCESS) + { +# ifdef IN_RING0 + return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf); +# else + /* ring-3: Cannot return VINF_SUCCESS. */ + Assert(RT_FAILURE_NP(rc)); + int rc2 = pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf); + if (RT_SUCCESS(rc2)) + rc2 = pdmCritSectRwLeaveSharedWorker(pVM, pThis, fNoVal); + return rc; +# endif + } + AssertMsgReturn(rcWait == VERR_TIMEOUT || rcWait == VERR_INTERRUPTED, + ("%p: rcWait=%Rrc rc=%Rrc", pThis, rcWait, rc), + RT_FAILURE_NP(rcWait) ? rcWait : -rcWait); + } + else + { + u64State &= ~RTCSRW_WAIT_CNT_RD_MASK; + u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf); + } + +# ifdef IN_RING0 + /* Calculate the elapsed time here to avoid redoing state work. */ + cNsElapsed = RTTimeNanoTS() - tsStart; +# endif + } + + ASMNopPause(); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); + ASMNopPause(); + } +} +#endif /* IN_RING0 || (IN_RING3 && PDMCRITSECTRW_STRICT) */ + + +/** + * Worker for pdmCritSectRwEnterShared that handles waiting for a contended CS. + * Caller has already added us to the read and read-wait counters. + */ +static int pdmCritSectRwEnterSharedContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis, + int rcBusy, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf) +{ + PSUPDRVSESSION const pSession = pVM->pSession; + SUPSEMEVENTMULTI const hEventMulti = (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead; +# ifdef IN_RING0 + uint64_t const tsStart = RTTimeNanoTS(); + uint64_t const cNsMaxTotalDef = RT_NS_5MIN; + uint64_t cNsMaxTotal = cNsMaxTotalDef; + uint32_t cMsMaxOne = RT_MS_5SEC; + bool fNonInterruptible = false; +# endif + + for (uint32_t iLoop = 0; ; iLoop++) + { + /* + * Wait for the direction to switch. + */ + int rc; +# ifdef IN_RING3 +# if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + rc = RTLockValidatorRecSharedCheckBlocking(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, true, + RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_READ, false); + if (RT_FAILURE(rc)) + return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf); +# else + RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false); +# endif +# endif + + for (;;) + { + /* + * We always wait with a timeout so we can re-check the structure sanity + * and not get stuck waiting on a corrupt or deleted section. + */ +# ifdef IN_RING3 + rc = SUPSemEventMultiWaitNoResume(pSession, hEventMulti, RT_MS_5SEC); +# else + rc = !fNonInterruptible + ? SUPSemEventMultiWaitNoResume(pSession, hEventMulti, cMsMaxOne) + : SUPSemEventMultiWait(pSession, hEventMulti, cMsMaxOne); + Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p u64State=%#RX64)\n", pThis, rc, + RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State)); +# endif + if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)) + { /* likely */ } + else + { +# ifdef IN_RING3 + RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); +# endif + return VERR_SEM_DESTROYED; + } + if (RT_LIKELY(rc == VINF_SUCCESS)) + break; + + /* + * Timeout and interrupted waits needs careful handling in ring-0 + * because we're cooperating with ring-3 on this critical section + * and thus need to make absolutely sure we won't get stuck here. + * + * The r0 interrupted case means something is pending (termination, + * signal, APC, debugger, whatever), so we must try our best to + * return to the caller and to ring-3 so it can be dealt with. + */ + if (rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED) + { +# ifdef IN_RING0 + uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart; + int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD); + AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING, + ("rcTerm=%Rrc\n", rcTerm)); + if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef) + cNsMaxTotal = RT_NS_1MIN; + + if (rc == VERR_TIMEOUT) + { + /* Try return get out of here with a non-VINF_SUCCESS status if + the thread is terminating or if the timeout has been exceeded. */ + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrTimeout); + if ( rcTerm == VINF_THREAD_IS_TERMINATING + || cNsElapsed > cNsMaxTotal) + return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc, + pSrcPos, fNoVal, hThreadSelf); + } + else + { + /* For interrupt cases, we must return if we can. If rcBusy is VINF_SUCCESS, + we will try non-interruptible sleep for a while to help resolve the issue + w/o guru'ing. */ + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrInterrupted); + if ( rcTerm != VINF_THREAD_IS_TERMINATING + && rcBusy == VINF_SUCCESS + && pVCpu != NULL + && cNsElapsed <= cNsMaxTotal) + { + if (!fNonInterruptible) + { + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedNonInterruptibleWaits); + fNonInterruptible = true; + cMsMaxOne = 32; + uint64_t cNsLeft = cNsMaxTotal - cNsElapsed; + if (cNsLeft > RT_NS_10SEC) + cNsMaxTotal = cNsElapsed + RT_NS_10SEC; + } + } + else + return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc, + pSrcPos, fNoVal, hThreadSelf); + } +# else /* IN_RING3 */ + RT_NOREF(pVM, pVCpu, rcBusy); +# endif /* IN_RING3 */ + } + /* + * Any other return code is fatal. + */ + else + { +# ifdef IN_RING3 + RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); +# endif + AssertMsgFailed(("rc=%Rrc\n", rc)); + return RT_FAILURE_NP(rc) ? rc : -rc; + } + } + +# ifdef IN_RING3 + RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ); +# endif + + /* + * Check the direction. + */ + Assert(pThis->s.Core.fNeedReset); + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) + { + /* + * Decrement the wait count and maybe reset the semaphore (if we're last). + */ + return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf); + } + + AssertMsg(iLoop < 1, + ("%p: %u u64State=%#RX64 hNativeWriter=%p\n", pThis, iLoop, u64State, pThis->s.Core.u.s.hNativeWriter)); + RTThreadYield(); + } + + /* not reached */ +} + + +/** + * Worker that enters a read/write critical section with shard access. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param rcBusy The busy return code for ring-0 and ring-3. + * @param fTryOnly Only try enter it, don't wait. + * @param pSrcPos The source position. (Can be NULL.) + * @param fNoVal No validation records. + */ +static int pdmCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly, + PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal) +{ + /* + * Validate input. + */ + AssertPtr(pThis); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); + +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt(); + if (!fTryOnly) + { + int rc9; + RTNATIVETHREAD hNativeWriter; + ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); + if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis)) + rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); + else + rc9 = RTLockValidatorRecSharedCheckOrder(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); + if (RT_FAILURE(rc9)) + return rc9; + } +#else + RTTHREAD hThreadSelf = NIL_RTTHREAD; +#endif + + /* + * Work the state. + */ + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + uint64_t u64OldState = u64State; + for (;;) + { + if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) + { + /* It flows in the right direction, try follow it before it changes. */ + uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; + c++; + Assert(c < RTCSRW_CNT_MASK / 4); + AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS); + u64State &= ~RTCSRW_CNT_RD_MASK; + u64State |= c << RTCSRW_CNT_RD_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf); + } + else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0) + { + /* Wrong direction, but we're alone here and can simply try switch the direction. */ + u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK); + u64State |= (UINT64_C(1) << RTCSRW_CNT_RD_SHIFT) | (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT); + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + { + Assert(!pThis->s.Core.fNeedReset); + return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf); + } + } + else + { + /* Is the writer perhaps doing a read recursion? */ + RTNATIVETHREAD hNativeWriter; + ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); + if (hNativeWriter != NIL_RTNATIVETHREAD) + { + RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis); + if (hNativeSelf == hNativeWriter) + { +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + if (!fNoVal) + { + int rc9 = RTLockValidatorRecExclRecursionMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core, pSrcPos); + if (RT_FAILURE(rc9)) + return rc9; + } +#endif + uint32_t const cReads = ASMAtomicIncU32(&pThis->s.Core.cWriterReads); + Assert(cReads < _16K); + AssertReturnStmt(cReads < PDM_CRITSECTRW_MAX_RECURSIONS, ASMAtomicDecU32(&pThis->s.Core.cWriterReads), + VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS); + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared)); + return VINF_SUCCESS; /* don't break! */ + } + } + + /* + * If we're only trying, return already. + */ + if (fTryOnly) + { + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared)); + return VERR_SEM_BUSY; + } + +#if defined(IN_RING3) || defined(IN_RING0) + /* + * Add ourselves to the queue and wait for the direction to change. + */ + uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; + c++; + Assert(c < RTCSRW_CNT_MASK / 2); + AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS); + + uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT; + cWait++; + Assert(cWait <= c); + Assert(cWait < RTCSRW_CNT_MASK / 2); + AssertReturn(cWait < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS); + + u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK); + u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT); + + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + { + /* + * In ring-3 it's straight forward, just optimize the RTThreadSelf() call. + */ +# if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT) + return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf); +# elif defined(IN_RING3) + return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, RTThreadSelf()); +# else /* IN_RING0 */ + /* + * In ring-0 context we have to take the special VT-x/AMD-V HM context into + * account when waiting on contended locks. + */ + PVMCPUCC pVCpu = VMMGetCpu(pVM); + if (pVCpu) + { + VMMR0EMTBLOCKCTX Ctx; + int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx); + if (rc == VINF_SUCCESS) + { + Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + + rc = pdmCritSectRwEnterSharedContended(pVM, pVCpu, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf); + + VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); + } + else + { + //STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy); + rc = pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf); + } + return rc; + } + + /* Non-EMT. */ + Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf); +# endif /* IN_RING0 */ + } + +#else /* !IN_RING3 && !IN_RING0 */ + /* + * We cannot call SUPSemEventMultiWaitNoResume in this context. Go + * back to ring-3 and do it there or return rcBusy. + */ +# error "Unused code." + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared)); + if (rcBusy == VINF_SUCCESS) + { + PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); + /** @todo Should actually do this in via VMMR0.cpp instead of going all the way + * back to ring-3. Goes for both kind of crit sects. */ + return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED, MMHyperCCToR3(pVM, pThis)); + } + return rcBusy; +#endif /* !IN_RING3 && !IN_RING0 */ + } + + ASMNopPause(); + if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)) + { /* likely */ } + else + return VERR_SEM_DESTROYED; + ASMNopPause(); + + u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + u64OldState = u64State; + } + /* not reached */ +} + + +/** + * Enter a critical section with shared (read) access. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval rcBusy if in ring-0 or raw-mode context and it is busy. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param rcBusy The status code to return when we're in RC or R0 and the + * section is busy. Pass VINF_SUCCESS to acquired the + * critical section thru a ring-3 call if necessary. + * @sa PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterShared, + * PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared, + * RTCritSectRwEnterShared. + */ +VMMDECL(int) PDMCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy) +{ +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, NULL, false /*fNoVal*/); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); + return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/); +#endif +} + + +/** + * Enter a critical section with shared (read) access. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval rcBusy if in ring-0 or raw-mode context and it is busy. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param rcBusy The status code to return when we're in RC or R0 and the + * section is busy. Pass VINF_SUCCESS to acquired the + * critical section thru a ring-3 call if necessary. + * @param uId Where we're entering the section. + * @param SRC_POS The source position. + * @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared, + * PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared, + * RTCritSectRwEnterSharedDebug. + */ +VMMDECL(int) PDMCritSectRwEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL) +{ + NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction); +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, NULL, false /*fNoVal*/); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); + return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/); +#endif +} + + +/** + * Try enter a critical section with shared (read) access. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_SEM_BUSY if the critsect was owned. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @sa PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwEnterShared, + * PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared, + * RTCritSectRwTryEnterShared. + */ +VMMDECL(int) PDMCritSectRwTryEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis) +{ +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL, false /*fNoVal*/); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); + return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/); +#endif +} + + +/** + * Try enter a critical section with shared (read) access. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_SEM_BUSY if the critsect was owned. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param uId Where we're entering the section. + * @param SRC_POS The source position. + * @sa PDMCritSectRwTryEnterShared, PDMCritSectRwEnterShared, + * PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared, + * RTCritSectRwTryEnterSharedDebug. + */ +VMMDECL(int) PDMCritSectRwTryEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL) +{ + NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction); +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL, false /*fNoVal*/); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); + return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/); +#endif +} + + +#ifdef IN_RING3 +/** + * Enters a PDM read/write critical section with shared (read) access. + * + * @returns VINF_SUCCESS if entered successfully. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param fCallRing3 Whether this is a VMMRZCallRing3()request. + */ +VMMR3DECL(int) PDMR3CritSectRwEnterSharedEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3) +{ + return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3); +} +#endif + + +/** + * Leave a critical section held with shared access. + * + * @returns VBox status code. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param fNoVal No validation records (i.e. queued release). + * @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared, + * PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug, + * PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared. + */ +static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal) +{ + /* + * Validate handle. + */ + AssertPtr(pThis); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); + +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + NOREF(fNoVal); +#endif + + /* + * Check the direction and take action accordingly. + */ +#ifdef IN_RING0 + PVMCPUCC pVCpu = NULL; +#endif + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + uint64_t u64OldState = u64State; + if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) + { +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + if (fNoVal) + Assert(!RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD)); + else + { + int rc9 = RTLockValidatorRecSharedCheckAndRelease(pThis->s.Core.pValidatorRead, NIL_RTTHREAD); + if (RT_FAILURE(rc9)) + return rc9; + } +#endif + for (;;) + { + uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; + AssertReturn(c > 0, VERR_NOT_OWNER); + c--; + + if ( c > 0 + || (u64State & RTCSRW_CNT_WR_MASK) == 0) + { + /* Don't change the direction. */ + u64State &= ~RTCSRW_CNT_RD_MASK; + u64State |= c << RTCSRW_CNT_RD_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + break; + } + else + { +#if defined(IN_RING3) || defined(IN_RING0) +# ifdef IN_RING0 + Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe()); + if (!pVCpu) + pVCpu = VMMGetCpu(pVM); + if ( pVCpu == NULL /* non-EMT access, if we implement it must be able to block */ + || VMMRZCallRing3IsEnabled(pVCpu) + || RTSemEventIsSignalSafe() + || ( VMMR0ThreadCtxHookIsEnabled(pVCpu) /* Doesn't matter if Signal() blocks if we have hooks, ... */ + && RTThreadPreemptIsEnabled(NIL_RTTHREAD) /* ... and preemption is still enabled, */ + && ASMIntAreEnabled()) /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */ + ) +# endif + { + /* Reverse the direction and signal the writer threads. */ + u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_DIR_MASK); + u64State |= RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + { + int rc; +# ifdef IN_RING0 + STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared); + if (!RTSemEventIsSignalSafe() && pVCpu != NULL) + { + VMMR0EMTBLOCKCTX Ctx; + rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx); + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc); + + rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite); + + VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); + } + else +# endif + rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite); + AssertRC(rc); + return rc; + } + } +#endif /* IN_RING3 || IN_RING0 */ +#ifndef IN_RING3 +# ifdef IN_RING0 + else +# endif + { + /* Queue the exit request (ring-3). */ +# ifndef IN_RING0 + PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); +# endif + uint32_t i = pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves++; + LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3 c=%d (%#llx)\n", i, pThis, c, u64State)); + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves), + ("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE); + pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] = pThis->s.pSelfR3; + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, + RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i]) + && ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] & HOST_PAGE_OFFSET_MASK) + == ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK), + ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i], pThis), + pdmCritSectRwCorrupted(pThis, "Invalid self pointer")); + VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT); + VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3); + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves); + STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared); + break; + } +#endif + } + + ASMNopPause(); + if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)) + { } + else + return VERR_SEM_DESTROYED; + ASMNopPause(); + + u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + u64OldState = u64State; + } + } + else + { + /* + * Write direction. Check that it's the owner calling and that it has reads to undo. + */ + RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis); + AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT); + + RTNATIVETHREAD hNativeWriter; + ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); + AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER); + AssertReturn(pThis->s.Core.cWriterReads > 0, VERR_NOT_OWNER); +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + if (!fNoVal) + { + int rc = RTLockValidatorRecExclUnwindMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core); + if (RT_FAILURE(rc)) + return rc; + } +#endif + uint32_t cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriterReads); + AssertReturn(cDepth < PDM_CRITSECTRW_MAX_RECURSIONS, pdmCritSectRwCorrupted(pThis, "too many writer-read recursions")); + } + + return VINF_SUCCESS; +} + + +/** + * Leave a critical section held with shared access. + * + * @returns VBox status code. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @sa PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared, + * PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug, + * PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared. + */ +VMMDECL(int) PDMCritSectRwLeaveShared(PVMCC pVM, PPDMCRITSECTRW pThis) +{ + return pdmCritSectRwLeaveSharedWorker(pVM, pThis, false /*fNoVal*/); +} + + +#if defined(IN_RING3) || defined(IN_RING0) +/** + * PDMCritSectBothFF interface. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + */ +void pdmCritSectRwLeaveSharedQueued(PVMCC pVM, PPDMCRITSECTRW pThis) +{ + pdmCritSectRwLeaveSharedWorker(pVM, pThis, true /*fNoVal*/); +} +#endif + + +/** + * Worker for pdmCritSectRwEnterExcl that bails out on wait failure. + * + * @returns @a rc unless corrupted. + * @param pThis Pointer to the read/write critical section. + * @param rc The status to return. + */ +DECL_NO_INLINE(static, int) pdmCritSectRwEnterExclBailOut(PPDMCRITSECTRW pThis, int rc) +{ + /* + * Decrement the counts and return the error. + */ + for (;;) + { + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + uint64_t const u64OldState = u64State; + uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; + AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on bailout")); + c--; + u64State &= ~RTCSRW_CNT_WR_MASK; + u64State |= c << RTCSRW_CNT_WR_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + return rc; + + ASMNopPause(); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); + ASMNopPause(); + } +} + + +/** + * Worker for pdmCritSectRwEnterExcl that handles the red tape after we've + * gotten exclusive ownership of the critical section. + */ +DECL_FORCE_INLINE(int) pdmCritSectRwEnterExclFirst(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos, + bool fNoVal, RTTHREAD hThreadSelf) +{ + RT_NOREF(hThreadSelf, fNoVal, pSrcPos); + Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)); + +#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF + pThis->s.Core.cWriteRecursions = 1; +#else + ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 1); +#endif + Assert(pThis->s.Core.cWriterReads == 0); + +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + if (!fNoVal) + { + if (hThreadSelf == NIL_RTTHREAD) + hThreadSelf = RTThreadSelfAutoAdopt(); + RTLockValidatorRecExclSetOwner(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true); + } +#endif + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl)); + STAM_PROFILE_ADV_START(&pThis->s.StatWriteLocked, swl); + return VINF_SUCCESS; +} + + +#if defined(IN_RING3) || defined(IN_RING0) +/** + * Worker for pdmCritSectRwEnterExcl that handles waiting when the section is + * contended. + */ +static int pdmR3R0CritSectRwEnterExclContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis, RTNATIVETHREAD hNativeSelf, + PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, int rcBusy, RTTHREAD hThreadSelf) +{ + RT_NOREF(hThreadSelf, rcBusy, pSrcPos, fNoVal, pVCpu); + + PSUPDRVSESSION const pSession = pVM->pSession; + SUPSEMEVENT const hEvent = (SUPSEMEVENT)pThis->s.Core.hEvtWrite; +# ifdef IN_RING0 + uint64_t const tsStart = RTTimeNanoTS(); + uint64_t const cNsMaxTotalDef = RT_NS_5MIN; + uint64_t cNsMaxTotal = cNsMaxTotalDef; + uint32_t cMsMaxOne = RT_MS_5SEC; + bool fNonInterruptible = false; +# endif + + for (uint32_t iLoop = 0; ; iLoop++) + { + /* + * Wait for our turn. + */ + int rc; +# ifdef IN_RING3 +# ifdef PDMCRITSECTRW_STRICT + rc = RTLockValidatorRecExclCheckBlocking(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true, + RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_WRITE, false); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + return pdmCritSectRwEnterExclBailOut(pThis, rc); +# else + RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false); +# endif +# endif + + for (;;) + { + /* + * We always wait with a timeout so we can re-check the structure sanity + * and not get stuck waiting on a corrupt or deleted section. + */ +# ifdef IN_RING3 + rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_MS_5SEC); +# else + rc = !fNonInterruptible + ? SUPSemEventWaitNoResume(pSession, hEvent, cMsMaxOne) + : SUPSemEventWait(pSession, hEvent, cMsMaxOne); + Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p)\n", + pThis, rc, RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter)); +# endif + if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)) + { /* likely */ } + else + { +# ifdef IN_RING3 + RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); +# endif + return VERR_SEM_DESTROYED; + } + if (RT_LIKELY(rc == VINF_SUCCESS)) + break; + + /* + * Timeout and interrupted waits needs careful handling in ring-0 + * because we're cooperating with ring-3 on this critical section + * and thus need to make absolutely sure we won't get stuck here. + * + * The r0 interrupted case means something is pending (termination, + * signal, APC, debugger, whatever), so we must try our best to + * return to the caller and to ring-3 so it can be dealt with. + */ + if (rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED) + { +# ifdef IN_RING0 + uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart; + int const rcTerm = RTThreadQueryTerminationStatus(NIL_RTTHREAD); + AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING, + ("rcTerm=%Rrc\n", rcTerm)); + if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef) + cNsMaxTotal = RT_NS_1MIN; + + if (rc == VERR_TIMEOUT) + { + /* Try return get out of here with a non-VINF_SUCCESS status if + the thread is terminating or if the timeout has been exceeded. */ + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrTimeout); + if ( rcTerm == VINF_THREAD_IS_TERMINATING + || cNsElapsed > cNsMaxTotal) + return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc); + } + else + { + /* For interrupt cases, we must return if we can. If rcBusy is VINF_SUCCESS, + we will try non-interruptible sleep for a while to help resolve the issue + w/o guru'ing. */ + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrInterrupted); + if ( rcTerm != VINF_THREAD_IS_TERMINATING + && rcBusy == VINF_SUCCESS + && pVCpu != NULL + && cNsElapsed <= cNsMaxTotal) + { + if (!fNonInterruptible) + { + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclNonInterruptibleWaits); + fNonInterruptible = true; + cMsMaxOne = 32; + uint64_t cNsLeft = cNsMaxTotal - cNsElapsed; + if (cNsLeft > RT_NS_10SEC) + cNsMaxTotal = cNsElapsed + RT_NS_10SEC; + } + } + else + return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc); + } +# else /* IN_RING3 */ + RT_NOREF(pVM, pVCpu, rcBusy); +# endif /* IN_RING3 */ + } + /* + * Any other return code is fatal. + */ + else + { +# ifdef IN_RING3 + RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); +# endif + AssertMsgFailed(("rc=%Rrc\n", rc)); + return RT_FAILURE_NP(rc) ? rc : -rc; + } + } + +# ifdef IN_RING3 + RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE); +# endif + + /* + * Try take exclusive write ownership. + */ + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)) + { + bool fDone; + ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone); + if (fDone) + return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf); + } + AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */ + } +} +#endif /* IN_RING3 || IN_RING0 */ + + +/** + * Worker that enters a read/write critical section with exclusive access. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param rcBusy The busy return code for ring-0 and ring-3. + * @param fTryOnly Only try enter it, don't wait. + * @param pSrcPos The source position. (Can be NULL.) + * @param fNoVal No validation records. + */ +static int pdmCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly, + PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal) +{ + /* + * Validate input. + */ + AssertPtr(pThis); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); + + RTTHREAD hThreadSelf = NIL_RTTHREAD; +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + if (!fTryOnly) + { + hThreadSelf = RTThreadSelfAutoAdopt(); + int rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT); + if (RT_FAILURE(rc9)) + return rc9; + } +#endif + + /* + * Check if we're already the owner and just recursing. + */ + RTNATIVETHREAD const hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis); + AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT); + RTNATIVETHREAD hNativeWriter; + ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); + if (hNativeSelf == hNativeWriter) + { + Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)); +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + if (!fNoVal) + { + int rc9 = RTLockValidatorRecExclRecursion(pThis->s.Core.pValidatorWrite, pSrcPos); + if (RT_FAILURE(rc9)) + return rc9; + } +#endif + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl)); +#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF + uint32_t const cDepth = ++pThis->s.Core.cWriteRecursions; +#else + uint32_t const cDepth = ASMAtomicIncU32(&pThis->s.Core.cWriteRecursions); +#endif + AssertReturnStmt(cDepth > 1 && cDepth <= PDM_CRITSECTRW_MAX_RECURSIONS, + ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions), + VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS); + return VINF_SUCCESS; + } + + /* + * First we try grab an idle critical section using 128-bit atomics. + */ + /** @todo This could be moved up before the recursion check. */ + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); +#ifdef RTASM_HAVE_CMP_WRITE_U128 + if ( (u64State & ~RTCSRW_DIR_MASK) == 0 + && pdmCritSectRwIsCmpWriteU128Supported()) + { + RTCRITSECTRWSTATE OldState; + OldState.s.u64State = u64State; + OldState.s.hNativeWriter = NIL_RTNATIVETHREAD; + AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo)); + + RTCRITSECTRWSTATE NewState; + NewState.s.u64State = (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT); + NewState.s.hNativeWriter = hNativeSelf; + + if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128)) + return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf); + + u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + } +#endif + + /* + * Do it step by step. Update the state to reflect our desire. + */ + uint64_t u64OldState = u64State; + + for (;;) + { + if ( (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT) + || (u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) != 0) + { + /* It flows in the right direction, try follow it before it changes. */ + uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; + AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS); + c++; + Assert(c < RTCSRW_CNT_WR_MASK / 4); + u64State &= ~RTCSRW_CNT_WR_MASK; + u64State |= c << RTCSRW_CNT_WR_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + break; + } + else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0) + { + /* Wrong direction, but we're alone here and can simply try switch the direction. */ + u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK); + u64State |= (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT); + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + break; + } + else if (fTryOnly) + { + /* Wrong direction and we're not supposed to wait, just return. */ + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl)); + return VERR_SEM_BUSY; + } + else + { + /* Add ourselves to the write count and break out to do the wait. */ + uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; + AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS); + c++; + Assert(c < RTCSRW_CNT_WR_MASK / 4); + u64State &= ~RTCSRW_CNT_WR_MASK; + u64State |= c << RTCSRW_CNT_WR_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + break; + } + + ASMNopPause(); + + if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC) + { /* likely */ } + else + return VERR_SEM_DESTROYED; + + ASMNopPause(); + u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + u64OldState = u64State; + } + + /* + * If we're in write mode now try grab the ownership. Play fair if there + * are threads already waiting. + */ + bool fDone = (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT) + && ( ((u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT) == 1 + || fTryOnly); + if (fDone) + { + ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone); + if (fDone) + return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf); + } + + /* + * Okay, we have contention and will have to wait unless we're just trying. + */ + if (fTryOnly) + { + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl)); /** @todo different statistics for this */ + return pdmCritSectRwEnterExclBailOut(pThis, VERR_SEM_BUSY); + } + + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl)); + + /* + * Ring-3 is pretty straight forward. + */ +#if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT) + return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, hThreadSelf); +#elif defined(IN_RING3) + return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, RTThreadSelf()); + +#elif defined(IN_RING0) + /* + * In ring-0 context we have to take the special VT-x/AMD-V HM context into + * account when waiting on contended locks. + */ + PVMCPUCC pVCpu = VMMGetCpu(pVM); + if (pVCpu) + { + VMMR0EMTBLOCKCTX Ctx; + int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx); + if (rc == VINF_SUCCESS) + { + Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + + rc = pdmR3R0CritSectRwEnterExclContended(pVM, pVCpu, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD); + + VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); + } + else + { + //STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy); + rc = pdmCritSectRwEnterExclBailOut(pThis, rc); + } + return rc; + } + + /* Non-EMT. */ + Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD); + +#else +# error "Unused." + /* + * Raw-mode: Call host and take it there if rcBusy is VINF_SUCCESS. + */ + rcBusy = pdmCritSectRwEnterExclBailOut(pThis, rcBusy); + if (rcBusy == VINF_SUCCESS) + { + Assert(!fTryOnly); + PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); + /** @todo Should actually do this in via VMMR0.cpp instead of going all the way + * back to ring-3. Goes for both kind of crit sects. */ + return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL, MMHyperCCToR3(pVM, pThis)); + } + return rcBusy; +#endif +} + + +/** + * Try enter a critical section with exclusive (write) access. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval rcBusy if in ring-0 or raw-mode context and it is busy. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param rcBusy The status code to return when we're in RC or R0 and the + * section is busy. Pass VINF_SUCCESS to acquired the + * critical section thru a ring-3 call if necessary. + * @sa PDMCritSectRwEnterExclDebug, PDMCritSectRwTryEnterExcl, + * PDMCritSectRwTryEnterExclDebug, + * PDMCritSectEnterDebug, PDMCritSectEnter, + * RTCritSectRwEnterExcl. + */ +VMMDECL(int) PDMCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy) +{ +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, NULL, false /*fNoVal*/); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); + return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/); +#endif +} + + +/** + * Try enter a critical section with exclusive (write) access. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval rcBusy if in ring-0 or raw-mode context and it is busy. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param rcBusy The status code to return when we're in RC or R0 and the + * section is busy. Pass VINF_SUCCESS to acquired the + * critical section thru a ring-3 call if necessary. + * @param uId Where we're entering the section. + * @param SRC_POS The source position. + * @sa PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExcl, + * PDMCritSectRwTryEnterExclDebug, + * PDMCritSectEnterDebug, PDMCritSectEnter, + * RTCritSectRwEnterExclDebug. + */ +VMMDECL(int) PDMCritSectRwEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL) +{ + NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction); +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, NULL, false /*fNoVal*/); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); + return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/); +#endif +} + + +/** + * Try enter a critical section with exclusive (write) access. + * + * @retval VINF_SUCCESS on success. + * @retval VERR_SEM_BUSY if the critsect was owned. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @sa PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExclDebug, + * PDMCritSectRwEnterExclDebug, + * PDMCritSectTryEnter, PDMCritSectTryEnterDebug, + * RTCritSectRwTryEnterExcl. + */ +VMMDECL(int) PDMCritSectRwTryEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis) +{ +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL, false /*fNoVal*/); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API(); + return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/); +#endif +} + + +/** + * Try enter a critical section with exclusive (write) access. + * + * @retval VINF_SUCCESS on success. + * @retval VERR_SEM_BUSY if the critsect was owned. + * @retval VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.) + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param uId Where we're entering the section. + * @param SRC_POS The source position. + * @sa PDMCritSectRwTryEnterExcl, PDMCritSectRwEnterExcl, + * PDMCritSectRwEnterExclDebug, + * PDMCritSectTryEnterDebug, PDMCritSectTryEnter, + * RTCritSectRwTryEnterExclDebug. + */ +VMMDECL(int) PDMCritSectRwTryEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL) +{ + NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction); +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL, false /*fNoVal*/); +#else + RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API(); + return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/); +#endif +} + + +#ifdef IN_RING3 +/** + * Enters a PDM read/write critical section with exclusive (write) access. + * + * @returns VINF_SUCCESS if entered successfully. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param fCallRing3 Whether this is a VMMRZCallRing3()request. + */ +VMMR3DECL(int) PDMR3CritSectRwEnterExclEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3) +{ + return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3 /*fNoVal*/); +} +#endif /* IN_RING3 */ + + +/** + * Leave a critical section held exclusively. + * + * @returns VBox status code. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param fNoVal No validation records (i.e. queued release). + * @sa PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl. + */ +static int pdmCritSectRwLeaveExclWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal) +{ + /* + * Validate handle. + */ + AssertPtr(pThis); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED); + +#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3) + NOREF(fNoVal); +#endif + + /* + * Check ownership. + */ + RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis); + AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT); + + RTNATIVETHREAD hNativeWriter; + ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); + AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER); + + + /* + * Unwind one recursion. Not the last? + */ + if (pThis->s.Core.cWriteRecursions != 1) + { +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + if (fNoVal) + Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD); + else + { + int rc9 = RTLockValidatorRecExclUnwind(pThis->s.Core.pValidatorWrite); + if (RT_FAILURE(rc9)) + return rc9; + } +#endif +#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF + uint32_t const cDepth = --pThis->s.Core.cWriteRecursions; +#else + uint32_t const cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions); +#endif + AssertReturn(cDepth != 0 && cDepth < UINT32_MAX, pdmCritSectRwCorrupted(pThis, "Invalid write recursion value on leave")); + return VINF_SUCCESS; + } + + + /* + * Final recursion. + */ + AssertReturn(pThis->s.Core.cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */ +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + if (fNoVal) + Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD); + else + { + int rc9 = RTLockValidatorRecExclReleaseOwner(pThis->s.Core.pValidatorWrite, true); + if (RT_FAILURE(rc9)) + return rc9; + } +#endif + + +#ifdef RTASM_HAVE_CMP_WRITE_U128 + /* + * See if we can get out w/o any signalling as this is a common case. + */ + if (pdmCritSectRwIsCmpWriteU128Supported()) + { + RTCRITSECTRWSTATE OldState; + OldState.s.u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + if (OldState.s.u64State == ((UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))) + { + OldState.s.hNativeWriter = hNativeSelf; + AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo)); + + RTCRITSECTRWSTATE NewState; + NewState.s.u64State = RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT; + NewState.s.hNativeWriter = NIL_RTNATIVETHREAD; + +# ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF + pThis->s.Core.cWriteRecursions = 0; +# else + ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0); +# endif + STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl); + + if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128)) + return VINF_SUCCESS; + + /* bail out. */ + pThis->s.Core.cWriteRecursions = 1; + } + } +#endif /* RTASM_HAVE_CMP_WRITE_U128 */ + + +#if defined(IN_RING3) || defined(IN_RING0) + /* + * Ring-3: Straight forward, just update the state and if necessary signal waiters. + * Ring-0: Try leave for real, depends on host and context. + */ +# ifdef IN_RING0 + Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe()); + PVMCPUCC pVCpu = VMMGetCpu(pVM); + if ( pVCpu == NULL /* non-EMT access, if we implement it must be able to block */ + || VMMRZCallRing3IsEnabled(pVCpu) + || RTSemEventIsSignalSafe() + || ( VMMR0ThreadCtxHookIsEnabled(pVCpu) /* Doesn't matter if Signal() blocks if we have hooks, ... */ + && RTThreadPreemptIsEnabled(NIL_RTTHREAD) /* ... and preemption is still enabled, */ + && ASMIntAreEnabled()) /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */ + ) +# endif + { +# ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF + pThis->s.Core.cWriteRecursions = 0; +# else + ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0); +# endif + STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl); + ASMAtomicWriteHandle(&pThis->s.Core.u.s.hNativeWriter, NIL_RTNATIVETHREAD); + + for (;;) + { + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + uint64_t u64OldState = u64State; + + uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT; + AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on leave")); + c--; + + if ( c > 0 + || (u64State & RTCSRW_CNT_RD_MASK) == 0) + { + /* + * Don't change the direction, wake up the next writer if any. + */ + u64State &= ~RTCSRW_CNT_WR_MASK; + u64State |= c << RTCSRW_CNT_WR_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + { + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl)); + int rc; + if (c == 0) + rc = VINF_SUCCESS; +# ifdef IN_RING0 + else if (!RTSemEventIsSignalSafe() && pVCpu != NULL) + { + VMMR0EMTBLOCKCTX Ctx; + rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx); + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc); + + rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite); + + VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); + } +# endif + else + rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite); + AssertRC(rc); + return rc; + } + } + else + { + /* + * Reverse the direction and signal the reader threads. + */ + u64State &= ~(RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK); + u64State |= RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT; + if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState)) + { + Assert(!pThis->s.Core.fNeedReset); + ASMAtomicWriteBool(&pThis->s.Core.fNeedReset, true); + STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl)); + + int rc; +# ifdef IN_RING0 + if (!RTSemEventMultiIsSignalSafe() && pVCpu != NULL) + { + VMMR0EMTBLOCKCTX Ctx; + rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx); + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc); + + rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead); + + VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx); + } + else +# endif + rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead); + AssertRC(rc); + return rc; + } + } + + ASMNopPause(); + if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC) + { /*likely*/ } + else + return VERR_SEM_DESTROYED; + ASMNopPause(); + } + /* not reached! */ + } +#endif /* IN_RING3 || IN_RING0 */ + + +#ifndef IN_RING3 + /* + * Queue the requested exit for ring-3 execution. + */ +# ifndef IN_RING0 + PVMCPUCC pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu); +# endif + uint32_t i = pVCpu->pdm.s.cQueuedCritSectRwExclLeaves++; + LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3\n", i, pThis)); + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves), + ("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE); + pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] = pThis->s.pSelfR3; + VMM_ASSERT_RELEASE_MSG_RETURN(pVM, + RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i]) + && ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] & HOST_PAGE_OFFSET_MASK) + == ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK), + ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i], pThis), + pdmCritSectRwCorrupted(pThis, "Invalid self pointer on queue (excl)")); + VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT); + VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3); + STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves); + STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveExcl); + return VINF_SUCCESS; +#endif +} + + +/** + * Leave a critical section held exclusively. + * + * @returns VBox status code. + * @retval VERR_SEM_DESTROYED if the critical section is delete before or + * during the operation. + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @sa PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl. + */ +VMMDECL(int) PDMCritSectRwLeaveExcl(PVMCC pVM, PPDMCRITSECTRW pThis) +{ + return pdmCritSectRwLeaveExclWorker(pVM, pThis, false /*fNoVal*/); +} + + +#if defined(IN_RING3) || defined(IN_RING0) +/** + * PDMCritSectBothFF interface. + * + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + */ +void pdmCritSectRwLeaveExclQueued(PVMCC pVM, PPDMCRITSECTRW pThis) +{ + pdmCritSectRwLeaveExclWorker(pVM, pThis, true /*fNoVal*/); +} +#endif + + +/** + * Checks the caller is the exclusive (write) owner of the critical section. + * + * @retval true if owner. + * @retval false if not owner. + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @sa PDMCritSectRwIsReadOwner, PDMCritSectIsOwner, + * RTCritSectRwIsWriteOwner. + */ +VMMDECL(bool) PDMCritSectRwIsWriteOwner(PVMCC pVM, PPDMCRITSECTRW pThis) +{ + /* + * Validate handle. + */ + AssertPtr(pThis); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false); + + /* + * Check ownership. + */ + RTNATIVETHREAD hNativeWriter; + ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter); + if (hNativeWriter == NIL_RTNATIVETHREAD) + return false; + return hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis); +} + + +/** + * Checks if the caller is one of the read owners of the critical section. + * + * @note !CAUTION! This API doesn't work reliably if lock validation isn't + * enabled. Meaning, the answer is not trustworhty unless + * RT_LOCK_STRICT or PDMCRITSECTRW_STRICT was defined at build time. + * Also, make sure you do not use RTCRITSECTRW_FLAGS_NO_LOCK_VAL when + * creating the semaphore. And finally, if you used a locking class, + * don't disable deadlock detection by setting cMsMinDeadlock to + * RT_INDEFINITE_WAIT. + * + * In short, only use this for assertions. + * + * @returns @c true if reader, @c false if not. + * @param pVM The cross context VM structure. + * @param pThis Pointer to the read/write critical section. + * @param fWannaHear What you'd like to hear when lock validation is not + * available. (For avoiding asserting all over the place.) + * @sa PDMCritSectRwIsWriteOwner, RTCritSectRwIsReadOwner. + */ +VMMDECL(bool) PDMCritSectRwIsReadOwner(PVMCC pVM, PPDMCRITSECTRW pThis, bool fWannaHear) +{ + /* + * Validate handle. + */ + AssertPtr(pThis); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false); + + /* + * Inspect the state. + */ + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)) + { + /* + * It's in write mode, so we can only be a reader if we're also the + * current writer. + */ + RTNATIVETHREAD hWriter; + ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hWriter); + if (hWriter == NIL_RTNATIVETHREAD) + return false; + return hWriter == pdmCritSectRwGetNativeSelf(pVM, pThis); + } + + /* + * Read mode. If there are no current readers, then we cannot be a reader. + */ + if (!(u64State & RTCSRW_CNT_RD_MASK)) + return false; + +#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3) + /* + * Ask the lock validator. + * Note! It doesn't know everything, let's deal with that if it becomes an issue... + */ + NOREF(fWannaHear); + return RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD); +#else + /* + * Ok, we don't know, just tell the caller what he want to hear. + */ + return fWannaHear; +#endif +} + + +/** + * Gets the write recursion count. + * + * @returns The write recursion count (0 if bad critsect). + * @param pThis Pointer to the read/write critical section. + * @sa PDMCritSectRwGetWriterReadRecursion, PDMCritSectRwGetReadCount, + * RTCritSectRwGetWriteRecursion. + */ +VMMDECL(uint32_t) PDMCritSectRwGetWriteRecursion(PPDMCRITSECTRW pThis) +{ + /* + * Validate handle. + */ + AssertPtr(pThis); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0); + + /* + * Return the requested data. + */ + return pThis->s.Core.cWriteRecursions; +} + + +/** + * Gets the read recursion count of the current writer. + * + * @returns The read recursion count (0 if bad critsect). + * @param pThis Pointer to the read/write critical section. + * @sa PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetReadCount, + * RTCritSectRwGetWriterReadRecursion. + */ +VMMDECL(uint32_t) PDMCritSectRwGetWriterReadRecursion(PPDMCRITSECTRW pThis) +{ + /* + * Validate handle. + */ + AssertPtr(pThis); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0); + + /* + * Return the requested data. + */ + return pThis->s.Core.cWriterReads; +} + + +/** + * Gets the current number of reads. + * + * This includes all read recursions, so it might be higher than the number of + * read owners. It does not include reads done by the current writer. + * + * @returns The read count (0 if bad critsect). + * @param pThis Pointer to the read/write critical section. + * @sa PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetWriterReadRecursion, + * RTCritSectRwGetReadCount. + */ +VMMDECL(uint32_t) PDMCritSectRwGetReadCount(PPDMCRITSECTRW pThis) +{ + /* + * Validate input. + */ + AssertPtr(pThis); + AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0); + + /* + * Return the requested data. + */ + uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State); + if ((u64State & RTCSRW_DIR_MASK) != (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT)) + return 0; + return (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT; +} + + +/** + * Checks if the read/write critical section is initialized or not. + * + * @retval true if initialized. + * @retval false if not initialized. + * @param pThis Pointer to the read/write critical section. + * @sa PDMCritSectIsInitialized, RTCritSectRwIsInitialized. + */ +VMMDECL(bool) PDMCritSectRwIsInitialized(PCPDMCRITSECTRW pThis) +{ + AssertPtr(pThis); + return pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC; +} + diff --git a/src/VBox/VMM/VMMAll/PDMAllIommu.cpp b/src/VBox/VMM/VMMAll/PDMAllIommu.cpp new file mode 100644 index 00000000..185faa65 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PDMAllIommu.cpp @@ -0,0 +1,468 @@ +/* $Id: PDMAllIommu.cpp $ */ +/** @file + * PDM IOMMU - All Contexts. + */ + +/* + * Copyright (C) 2021-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PDM +#define PDMPCIDEV_INCLUDE_PRIVATE /* Hack to get pdmpcidevint.h included at the right point. */ +#include "PDMInternal.h" + +#include +#include +#ifdef IN_RING3 +# include +#endif + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** + * Gets the PDM IOMMU for the current context from the PDM device instance. + */ +#ifdef IN_RING0 +#define PDMDEVINS_TO_IOMMU(a_pDevIns) &(a_pDevIns)->Internal.s.pGVM->pdmr0.s.aIommus[0]; +#else +#define PDMDEVINS_TO_IOMMU(a_pDevIns) &(a_pDevIns)->Internal.s.pVMR3->pdm.s.aIommus[0]; +#endif + + +/** + * Gets the PCI device ID (Bus:Dev:Fn) for the given PCI device. + * + * @returns PCI device ID. + * @param pDevIns The device instance. + * @param pPciDev The PCI device structure. Cannot be NULL. + */ +DECL_FORCE_INLINE(uint16_t) pdmIommuGetPciDeviceId(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev) +{ + uint8_t const idxBus = pPciDev->Int.s.idxPdmBus; +#if defined(IN_RING0) + PGVM pGVM = pDevIns->Internal.s.pGVM; + Assert(idxBus < RT_ELEMENTS(pGVM->pdmr0.s.aPciBuses)); + PCPDMPCIBUSR0 pBus = &pGVM->pdmr0.s.aPciBuses[idxBus]; +#elif defined(IN_RING3) + PVM pVM = pDevIns->Internal.s.pVMR3; + Assert(idxBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses)); + PCPDMPCIBUS pBus = &pVM->pdm.s.aPciBuses[idxBus]; +#endif + return PCIBDF_MAKE(pBus->iBus, pPciDev->uDevFn); +} + + +/** + * Returns whether an IOMMU instance is present. + * + * @returns @c true if an IOMMU is present, @c false otherwise. + * @param pDevIns The device instance. + */ +bool pdmIommuIsPresent(PPDMDEVINS pDevIns) +{ +#ifdef IN_RING0 + PCPDMIOMMUR3 pIommuR3 = &pDevIns->Internal.s.pGVM->pdm.s.aIommus[0]; +#else + PCPDMIOMMUR3 pIommuR3 = &pDevIns->Internal.s.pVMR3->pdm.s.aIommus[0]; +#endif + return pIommuR3->pDevInsR3 != NULL; +} + + +/** @copydoc PDMIOMMUREGR3::pfnMsiRemap */ +int pdmIommuMsiRemap(PPDMDEVINS pDevIns, uint16_t idDevice, PCMSIMSG pMsiIn, PMSIMSG pMsiOut) +{ + PPDMIOMMU pIommu = PDMDEVINS_TO_IOMMU(pDevIns); + PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns); + Assert(pDevInsIommu); + if (pDevInsIommu != pDevIns) + return pIommu->pfnMsiRemap(pDevInsIommu, idDevice, pMsiIn, pMsiOut); + return VERR_IOMMU_CANNOT_CALL_SELF; +} + + +/** + * Bus master physical memory read after translating the physical address using the + * IOMMU. + * + * @returns VBox status code. + * @retval VERR_IOMMU_NOT_PRESENT if an IOMMU is not present. + * + * @param pDevIns The device instance. + * @param pPciDev The PCI device. Cannot be NULL. + * @param GCPhys The guest-physical address to read. + * @param pvBuf Where to put the data read. + * @param cbRead How many bytes to read. + * @param fFlags Combination of PDM_DEVHLP_PHYS_RW_F_XXX. + * + * @thread Any. + */ +int pdmIommuMemAccessRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, uint32_t fFlags) +{ + PPDMIOMMU pIommu = PDMDEVINS_TO_IOMMU(pDevIns); + PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns); + if (pDevInsIommu) + { + if (pDevInsIommu != pDevIns) + { /* likely */ } + else + return VERR_IOMMU_CANNOT_CALL_SELF; + + uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev); + int rc = VINF_SUCCESS; + while (cbRead > 0) + { + RTGCPHYS GCPhysOut; + size_t cbContig; + rc = pIommu->pfnMemAccess(pDevInsIommu, idDevice, GCPhys, cbRead, PDMIOMMU_MEM_F_READ, &GCPhysOut, &cbContig); + if (RT_SUCCESS(rc)) + { + Assert(cbContig > 0 && cbContig <= cbRead); + /** @todo Handle strict return codes from PGMPhysRead. */ + rc = pDevIns->CTX_SUFF(pHlp)->pfnPhysRead(pDevIns, GCPhysOut, pvBuf, cbContig, fFlags); + if (RT_SUCCESS(rc)) + { + cbRead -= cbContig; + pvBuf = (void *)((uintptr_t)pvBuf + cbContig); + GCPhys += cbContig; + } + else + break; + } + else + { + LogFunc(("IOMMU memory read failed. idDevice=%#x GCPhys=%#RGp cb=%zu rc=%Rrc\n", idDevice, GCPhys, cbRead, rc)); + + /* + * We should initialize the read buffer on failure for devices that don't check + * return codes (but would verify the data). But we still want to propagate the + * error code from the IOMMU to the device, see @bugref{9936#c3}. + */ + memset(pvBuf, 0xff, cbRead); + break; + } + } + return rc; + } + return VERR_IOMMU_NOT_PRESENT; +} + + +/** + * Bus master physical memory write after translating the physical address using the + * IOMMU. + * + * @returns VBox status code. + * @retval VERR_IOMMU_NOT_PRESENT if an IOMMU is not present. + * + * @param pDevIns The device instance. + * @param pPciDev The PCI device structure. Cannot be NULL. + * @param GCPhys The guest-physical address to write. + * @param pvBuf The data to write. + * @param cbWrite How many bytes to write. + * @param fFlags Combination of PDM_DEVHLP_PHYS_RW_F_XXX. + * + * @thread Any. + */ +int pdmIommuMemAccessWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, + uint32_t fFlags) +{ + PPDMIOMMU pIommu = PDMDEVINS_TO_IOMMU(pDevIns); + PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns); + if (pDevInsIommu) + { + if (pDevInsIommu != pDevIns) + { /* likely */ } + else + return VERR_IOMMU_CANNOT_CALL_SELF; + + uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev); + int rc = VINF_SUCCESS; + while (cbWrite > 0) + { + RTGCPHYS GCPhysOut; + size_t cbContig; + rc = pIommu->pfnMemAccess(pDevInsIommu, idDevice, GCPhys, cbWrite, PDMIOMMU_MEM_F_WRITE, &GCPhysOut, &cbContig); + if (RT_SUCCESS(rc)) + { + Assert(cbContig > 0 && cbContig <= cbWrite); + /** @todo Handle strict return codes from PGMPhysWrite. */ + rc = pDevIns->CTX_SUFF(pHlp)->pfnPhysWrite(pDevIns, GCPhysOut, pvBuf, cbContig, fFlags); + if (RT_SUCCESS(rc)) + { + cbWrite -= cbContig; + pvBuf = (const void *)((uintptr_t)pvBuf + cbContig); + GCPhys += cbContig; + } + else + break; + } + else + { + LogFunc(("IOMMU memory write failed. idDevice=%#x GCPhys=%#RGp cb=%zu rc=%Rrc\n", idDevice, GCPhys, cbWrite, rc)); + break; + } + } + return rc; + } + return VERR_IOMMU_NOT_PRESENT; +} + + +#ifdef IN_RING3 +/** + * Requests the mapping of a guest page into ring-3 in preparation for a bus master + * physical memory read operation. + * + * Refer pfnPhysGCPhys2CCPtrReadOnly() for further details. + * + * @returns VBox status code. + * @retval VERR_IOMMU_NOT_PRESENT if an IOMMU is not present. + * + * @param pDevIns The device instance. + * @param pPciDev The PCI device structure. Cannot be NULL. + * @param GCPhys The guest physical address of the page that should be + * mapped. + * @param fFlags Flags reserved for future use, MBZ. + * @param ppv Where to store the address corresponding to GCPhys. + * @param pLock Where to store the lock information that + * pfnPhysReleasePageMappingLock needs. + */ +int pdmR3IommuMemAccessReadCCPtr(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, uint32_t fFlags, void const **ppv, + PPGMPAGEMAPLOCK pLock) +{ + PPDMIOMMU pIommu = PDMDEVINS_TO_IOMMU(pDevIns); + PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns); + if (pDevInsIommu) + { + if (pDevInsIommu != pDevIns) + { /* likely */ } + else + return VERR_IOMMU_CANNOT_CALL_SELF; + + uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev); + size_t cbContig = 0; + RTGCPHYS GCPhysOut = NIL_RTGCPHYS; + int rc = pIommu->pfnMemAccess(pDevInsIommu, idDevice, GCPhys & X86_PAGE_BASE_MASK, X86_PAGE_SIZE, PDMIOMMU_MEM_F_READ, + &GCPhysOut, &cbContig); + if (RT_SUCCESS(rc)) + { + Assert(GCPhysOut != NIL_RTGCPHYS); + Assert(cbContig == X86_PAGE_SIZE); + return pDevIns->pHlpR3->pfnPhysGCPhys2CCPtrReadOnly(pDevIns, GCPhysOut, fFlags, ppv, pLock); + } + + LogFunc(("IOMMU memory read for pointer access failed. idDevice=%#x GCPhys=%#RGp rc=%Rrc\n", idDevice, GCPhys, rc)); + return rc; + } + return VERR_IOMMU_NOT_PRESENT; +} + + +/** + * Requests the mapping of a guest page into ring-3 in preparation for a bus master + * physical memory write operation. + * + * Refer pfnPhysGCPhys2CCPtr() for further details. + * + * @returns VBox status code. + * @retval VERR_IOMMU_NOT_PRESENT if an IOMMU is not present. + * + * @param pDevIns The device instance. + * @param pPciDev The PCI device structure. Cannot be NULL. + * @param GCPhys The guest physical address of the page that should be + * mapped. + * @param fFlags Flags reserved for future use, MBZ. + * @param ppv Where to store the address corresponding to GCPhys. + * @param pLock Where to store the lock information that + * pfnPhysReleasePageMappingLock needs. + */ +int pdmR3IommuMemAccessWriteCCPtr(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, uint32_t fFlags, void **ppv, + PPGMPAGEMAPLOCK pLock) +{ + PPDMIOMMU pIommu = PDMDEVINS_TO_IOMMU(pDevIns); + PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns); + if (pDevInsIommu) + { + if (pDevInsIommu != pDevIns) + { /* likely */ } + else + return VERR_IOMMU_CANNOT_CALL_SELF; + + uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev); + size_t cbContig = 0; + RTGCPHYS GCPhysOut = NIL_RTGCPHYS; + int rc = pIommu->pfnMemAccess(pDevInsIommu, idDevice, GCPhys & X86_PAGE_BASE_MASK, X86_PAGE_SIZE, PDMIOMMU_MEM_F_WRITE, + &GCPhysOut, &cbContig); + if (RT_SUCCESS(rc)) + { + Assert(GCPhysOut != NIL_RTGCPHYS); + Assert(cbContig == X86_PAGE_SIZE); + return pDevIns->pHlpR3->pfnPhysGCPhys2CCPtr(pDevIns, GCPhysOut, fFlags, ppv, pLock); + } + + LogFunc(("IOMMU memory write for pointer access failed. idDevice=%#x GCPhys=%#RGp rc=%Rrc\n", idDevice, GCPhys, rc)); + return rc; + } + return VERR_IOMMU_NOT_PRESENT; +} + + +/** + * Requests the mapping of multiple guest pages into ring-3 in prepartion for a bus + * master physical memory read operation. + * + * Refer pfnPhysBulkGCPhys2CCPtrReadOnly() for further details. + * + * @returns VBox status code. + * @retval VERR_IOMMU_NOT_PRESENT if an IOMMU is not present. + * + * @param pDevIns The device instance. + * @param pPciDev The PCI device structure. Cannot be NULL. + * @param cPages Number of pages to lock. + * @param paGCPhysPages The guest physical address of the pages that + * should be mapped (@a cPages entries). + * @param fFlags Flags reserved for future use, MBZ. + * @param papvPages Where to store the ring-3 mapping addresses + * corresponding to @a paGCPhysPages. + * @param paLocks Where to store the locking information that + * pfnPhysBulkReleasePageMappingLock needs (@a cPages + * in length). + */ +int pdmR3IommuMemAccessBulkReadCCPtr(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t cPages, PCRTGCPHYS paGCPhysPages, + uint32_t fFlags, const void **papvPages, PPGMPAGEMAPLOCK paLocks) +{ + PPDMIOMMU pIommu = PDMDEVINS_TO_IOMMU(pDevIns); + PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns); + if (pDevInsIommu) + { + if (pDevInsIommu != pDevIns) + { /* likely */ } + else + return VERR_IOMMU_CANNOT_CALL_SELF; + + /* Allocate space for translated addresses. */ + size_t const cbIovas = cPages * sizeof(uint64_t); + PRTGCPHYS paGCPhysOut = (PRTGCPHYS)RTMemAllocZ(cbIovas); + if (paGCPhysOut) + { /* likely */ } + else + { + LogFunc(("caller='%s'/%d: returns %Rrc - Failed to alloc %zu bytes for IOVA addresses\n", + pDevIns->pReg->szName, pDevIns->iInstance, VERR_NO_MEMORY, cbIovas)); + return VERR_NO_MEMORY; + } + + /* Ask the IOMMU for corresponding translated physical addresses. */ + uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev); + AssertCompile(sizeof(RTGCPHYS) == sizeof(uint64_t)); + int rc = pIommu->pfnMemBulkAccess(pDevInsIommu, idDevice, cPages, (uint64_t const *)paGCPhysPages, PDMIOMMU_MEM_F_READ, + paGCPhysOut); + if (RT_SUCCESS(rc)) + { + /* Perform the bulk mapping but with the translated addresses. */ + rc = pDevIns->pHlpR3->pfnPhysBulkGCPhys2CCPtrReadOnly(pDevIns, cPages, paGCPhysOut, fFlags, papvPages, paLocks); + if (RT_FAILURE(rc)) + LogFunc(("Bulk mapping for read access failed. cPages=%zu fFlags=%#x rc=%Rrc\n", rc, cPages, fFlags)); + } + else + LogFunc(("Bulk translation for read access failed. idDevice=%#x cPages=%zu rc=%Rrc\n", idDevice, cPages, rc)); + + RTMemFree(paGCPhysOut); + return rc; + } + return VERR_IOMMU_NOT_PRESENT; +} + + +/** + * Requests the mapping of multiple guest pages into ring-3 in prepartion for a bus + * master physical memory write operation. + * + * Refer pfnPhysBulkGCPhys2CCPtr() for further details. + * + * @returns VBox status code. + * @retval VERR_IOMMU_NOT_PRESENT if an IOMMU is not present. + * + * @param pDevIns The device instance. + * @param pPciDev The PCI device structure. Cannot be NULL. + * @param cPages Number of pages to lock. + * @param paGCPhysPages The guest physical address of the pages that + * should be mapped (@a cPages entries). + * @param fFlags Flags reserved for future use, MBZ. + * @param papvPages Where to store the ring-3 mapping addresses + * corresponding to @a paGCPhysPages. + * @param paLocks Where to store the locking information that + * pfnPhysBulkReleasePageMappingLock needs (@a cPages + * in length). + */ +int pdmR3IommuMemAccessBulkWriteCCPtr(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t cPages, PCRTGCPHYS paGCPhysPages, + uint32_t fFlags, void **papvPages, PPGMPAGEMAPLOCK paLocks) +{ + PPDMIOMMU pIommu = PDMDEVINS_TO_IOMMU(pDevIns); + PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns); + if (pDevInsIommu) + { + if (pDevInsIommu != pDevIns) + { /* likely */ } + else + return VERR_IOMMU_CANNOT_CALL_SELF; + + /* Allocate space for translated addresses. */ + size_t const cbIovas = cPages * sizeof(uint64_t); + PRTGCPHYS paGCPhysOut = (PRTGCPHYS)RTMemAllocZ(cbIovas); + if (paGCPhysOut) + { /* likely */ } + else + { + LogFunc(("caller='%s'/%d: returns %Rrc - Failed to alloc %zu bytes for IOVA addresses\n", + pDevIns->pReg->szName, pDevIns->iInstance, VERR_NO_MEMORY, cbIovas)); + return VERR_NO_MEMORY; + } + + /* Ask the IOMMU for corresponding translated physical addresses. */ + uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev); + AssertCompile(sizeof(RTGCPHYS) == sizeof(uint64_t)); + int rc = pIommu->pfnMemBulkAccess(pDevInsIommu, idDevice, cPages, (uint64_t const *)paGCPhysPages, PDMIOMMU_MEM_F_WRITE, + paGCPhysOut); + if (RT_SUCCESS(rc)) + { + /* Perform the bulk mapping but with the translated addresses. */ + rc = pDevIns->pHlpR3->pfnPhysBulkGCPhys2CCPtr(pDevIns, cPages, paGCPhysOut, fFlags, papvPages, paLocks); + if (RT_FAILURE(rc)) + LogFunc(("Bulk mapping of addresses failed. cPages=%zu fFlags=%#x rc=%Rrc\n", rc, cPages, fFlags)); + } + else + LogFunc(("IOMMU bulk translation failed. idDevice=%#x cPages=%zu rc=%Rrc\n", idDevice, cPages, rc)); + + RTMemFree(paGCPhysOut); + return rc; + } + return VERR_IOMMU_NOT_PRESENT; +} +#endif /* IN_RING3 */ + diff --git a/src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp b/src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp new file mode 100644 index 00000000..d1b56dd5 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp @@ -0,0 +1,139 @@ +/* $Id: PDMAllNetShaper.cpp $ */ +/** @file + * PDM Network Shaper - Limit network traffic according to bandwidth group settings. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_NET_SHAPER +#include +#include "PDMInternal.h" +#include + +#include +#include +#include + + +/** + * Obtain bandwidth in a bandwidth group. + * + * @returns True if bandwidth was allocated, false if not. + * @param pVM The cross context VM structure. + * @param pFilter Pointer to the filter that allocates bandwidth. + * @param cbTransfer Number of bytes to allocate. + */ +VMM_INT_DECL(bool) PDMNetShaperAllocateBandwidth(PVMCC pVM, PPDMNSFILTER pFilter, size_t cbTransfer) +{ + AssertPtrReturn(pFilter, true); + + /* + * If we haven't got a valid bandwidth group, we always allow the traffic. + */ + bool fAllowed = true; + uint32_t iGroup = ASMAtomicUoReadU32(&pFilter->iGroup); + if (iGroup != 0) + { + if (iGroup <= RT_MIN(pVM->pdm.s.cNsGroups, RT_ELEMENTS(pVM->pdm.s.aNsGroups))) + { + PPDMNSBWGROUP pGroup = &pVM->pdm.s.aNsGroups[iGroup - 1]; + int rc = PDMCritSectEnter(pVM, &pGroup->Lock, VINF_TRY_AGAIN); + if (rc == VINF_SUCCESS) + { + uint64_t const cbPerSecMax = pGroup->cbPerSecMax; + if (cbPerSecMax > 0) + { + /* + * Re-fill the bucket first + * + * Note! We limit the cTokensAdded calculation to 1 second, since it's really + * pointless to calculate much beyond PDM_NETSHAPER_MAX_LATENCY (100ms) + * let alone 1 sec. This makes it possible to use ASMMultU64ByU32DivByU32 + * as the cNsDelta is less than 30 bits wide now, which means we don't get + * into overflow issues when multiplying two 64-bit values. + */ + uint64_t const nsNow = RTTimeSystemNanoTS(); + uint64_t const cNsDelta = nsNow - pGroup->tsUpdatedLast; + uint64_t const cTokensAdded = cNsDelta < RT_NS_1SEC + ? ASMMultU64ByU32DivByU32(cbPerSecMax, (uint32_t)cNsDelta, RT_NS_1SEC) + : cbPerSecMax; + uint32_t const cbBucket = pGroup->cbBucket; + uint32_t const cbTokensLast = pGroup->cbTokensLast; + uint32_t const cTokens = (uint32_t)RT_MIN(cbBucket, cTokensAdded + cbTokensLast); + + /* + * Allowed? + */ + if (cbTransfer <= cTokens) + { + pGroup->cbTokensLast = cTokens - (uint32_t)cbTransfer; + pGroup->tsUpdatedLast = nsNow; + Log2(("pdmNsAllocateBandwidth/%s: allowed - cbTransfer=%#zx cTokens=%#x cTokensAdded=%#x\n", + pGroup->szName, cbTransfer, cTokens, cTokensAdded)); + } + else + { + /* + * No, we're choked. Arm the unchoke timer for the next period. + * Just do this on a simple PDM_NETSHAPER_MAX_LATENCY clock granularity. + * ASSUMES the timer uses millisecond resolution clock. + */ + ASMAtomicWriteBool(&pFilter->fChoked, true); + if (ASMAtomicCmpXchgBool(&pVM->pdm.s.fNsUnchokeTimerArmed, true, false)) + { + Assert(TMTimerGetFreq(pVM, pVM->pdm.s.hNsUnchokeTimer) == RT_MS_1SEC); + uint64_t const msNow = TMTimerGet(pVM, pVM->pdm.s.hNsUnchokeTimer); + uint64_t const msExpire = (msNow / PDM_NETSHAPER_MAX_LATENCY + 1) * PDM_NETSHAPER_MAX_LATENCY; + rc = TMTimerSet(pVM, pVM->pdm.s.hNsUnchokeTimer, msExpire); + AssertRC(rc); + + Log2(("pdmNsAllocateBandwidth/%s: refused - cbTransfer=%#zx cTokens=%#x cTokensAdded=%#x cMsExpire=%u\n", + pGroup->szName, cbTransfer, cTokens, cTokensAdded, msExpire - msNow)); + } + else + Log2(("pdmNsAllocateBandwidth/%s: refused - cbTransfer=%#zx cTokens=%#x cTokensAdded=%#x\n", + pGroup->szName, cbTransfer, cTokens, cTokensAdded)); + ASMAtomicIncU64(&pGroup->cTotalChokings); + fAllowed = false; + } + } + else + Log2(("pdmNsAllocateBandwidth/%s: disabled\n", pGroup->szName)); + + rc = PDMCritSectLeave(pVM, &pGroup->Lock); + AssertRCSuccess(rc); + } + else if (rc == VINF_TRY_AGAIN) /* (accounted for by the critsect stats) */ + Log2(("pdmNsAllocateBandwidth/%s: allowed - lock contention\n", pGroup->szName)); + else + PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pGroup->Lock, rc); + } + else + AssertMsgFailed(("Invalid iGroup=%d\n", iGroup)); + } + return fAllowed; +} diff --git a/src/VBox/VMM/VMMAll/PDMAllQueue.cpp b/src/VBox/VMM/VMMAll/PDMAllQueue.cpp new file mode 100644 index 00000000..f64a198b --- /dev/null +++ b/src/VBox/VMM/VMMAll/PDMAllQueue.cpp @@ -0,0 +1,313 @@ +/* $Id: PDMAllQueue.cpp $ */ +/** @file + * PDM Queue - Transport data and tasks to EMT and R3. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PDM_QUEUE +#include "PDMInternal.h" +#include +#ifndef IN_RC +# include +#endif +#include +#include +#include +#include +#include +#include + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/* + * Macros for thoroughly validating a queue handle and ownership. + */ +#define PDMQUEUE_HANDLE_TO_VARS_RETURN_COMMON(a_cbMax, a_cbTotalMax) \ + AssertReturn(cbItem >= sizeof(PDMQUEUEITEMCORE), pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \ + AssertReturn(cbItem <= (a_cbMax), pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \ + \ + /* paranoia^3: */ \ + AssertReturn(cItems > 0, pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \ + AssertReturn(cItems <= PDMQUEUE_MAX_ITEMS, pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \ + AssertReturn(cbItem * cItems <= (a_cbTotalMax), pQueue->rcOkay = VERR_INTERNAL_ERROR_4) + +#ifdef IN_RING0 +# define PDMQUEUE_HANDLE_TO_VARS_RETURN(a_pVM, a_hQueue, a_pvOwner) \ + AssertPtrReturn((a_pvOwner), VERR_INVALID_PARAMETER); \ + \ + AssertCompile(RT_ELEMENTS((a_pVM)->pdm.s.apRing0Queues) == RT_ELEMENTS((a_pVM)->pdmr0.s.aQueues)); \ + AssertReturn((a_hQueue) < RT_ELEMENTS((a_pVM)->pdmr0.s.aQueues), VERR_INVALID_HANDLE); \ + AssertReturn((a_hQueue) < (a_pVM)->pdmr0.s.cQueues, VERR_INVALID_HANDLE); \ + AssertReturn((a_pVM)->pdmr0.s.aQueues[(a_hQueue)].pvOwner == (a_pvOwner), VERR_INVALID_HANDLE); \ + PPDMQUEUE pQueue = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].pQueue; \ + AssertPtrReturn(pQueue, VERR_INVALID_HANDLE); \ + AssertReturn(pQueue->u32Magic == PDMQUEUE_MAGIC, VERR_INVALID_HANDLE); \ + AssertReturn(pQueue->rcOkay == VINF_SUCCESS, pQueue->rcOkay); \ + \ + uint32_t const cbItem = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].cbItem; \ + uint32_t const cItems = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].cItems; \ + uint32_t const offItems = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].offItems; \ + \ + /* paranoia^2: */ \ + AssertReturn(pQueue->cbItem == cbItem, pQueue->rcOkay = VERR_INTERNAL_ERROR_3); \ + AssertReturn(pQueue->cItems == cItems, pQueue->rcOkay = VERR_INTERNAL_ERROR_3); \ + AssertReturn(pQueue->offItems == offItems, pQueue->rcOkay = VERR_INTERNAL_ERROR_3); \ + \ + PDMQUEUE_HANDLE_TO_VARS_RETURN_COMMON(PDMQUEUE_MAX_ITEM_SIZE, PDMQUEUE_MAX_TOTAL_SIZE_R0) + +#else +# define PDMQUEUE_HANDLE_TO_VARS_RETURN(a_pVM, a_hQueue, a_pvOwner) \ + AssertPtrReturn((a_pvOwner), VERR_INVALID_PARAMETER); \ + \ + PPDMQUEUE pQueue; \ + if ((a_hQueue) < RT_ELEMENTS((a_pVM)->pdm.s.apRing0Queues)) \ + pQueue = (a_pVM)->pdm.s.apRing0Queues[(a_hQueue)]; \ + else \ + { \ + (a_hQueue) -= RT_ELEMENTS((a_pVM)->pdm.s.apRing0Queues); \ + AssertReturn((a_pVM)->pdm.s.cRing3Queues, VERR_INVALID_HANDLE); \ + pQueue = (a_pVM)->pdm.s.papRing3Queues[(a_hQueue)]; \ + } \ + AssertPtrReturn(pQueue, VERR_INVALID_HANDLE); \ + AssertReturn(pQueue->u32Magic == PDMQUEUE_MAGIC, VERR_INVALID_HANDLE); \ + AssertReturn(pQueue->u.Gen.pvOwner == (a_pvOwner), VERR_INVALID_HANDLE); \ + AssertReturn(pQueue->rcOkay == VINF_SUCCESS, pQueue->rcOkay); \ + \ + uint32_t const cbItem = pQueue->cbItem; \ + uint32_t const cItems = pQueue->cItems; \ + uint32_t const offItems = pQueue->offItems; \ + \ + PDMQUEUE_HANDLE_TO_VARS_RETURN_COMMON(PDMQUEUE_MAX_ITEM_SIZE, PDMQUEUE_MAX_TOTAL_SIZE_R3) + +#endif + + +/** + * Commmon function for initializing the shared queue structure. + */ +void pdmQueueInit(PPDMQUEUE pQueue, uint32_t cbBitmap, uint32_t cbItem, uint32_t cItems, + const char *pszName, PDMQUEUETYPE enmType, RTR3PTR pfnCallback, RTR3PTR pvOwner) +{ + Assert(cbBitmap * 8 >= cItems); + + pQueue->u32Magic = PDMQUEUE_MAGIC; + pQueue->cbItem = cbItem; + pQueue->cItems = cItems; + pQueue->offItems = RT_UOFFSETOF(PDMQUEUE, bmAlloc) + cbBitmap; + pQueue->rcOkay = VINF_SUCCESS; + pQueue->u32Padding = 0; + pQueue->hTimer = NIL_TMTIMERHANDLE; + pQueue->cMilliesInterval = 0; + pQueue->enmType = enmType; + pQueue->u.Gen.pfnCallback = pfnCallback; + pQueue->u.Gen.pvOwner = pvOwner; + RTStrCopy(pQueue->szName, sizeof(pQueue->szName), pszName); + pQueue->iPending = UINT32_MAX; + RT_BZERO(pQueue->bmAlloc, cbBitmap); + ASMBitSetRange(pQueue->bmAlloc, 0, cItems); + + uint8_t *pbItem = (uint8_t *)&pQueue->bmAlloc[0] + cbBitmap; + while (cItems-- > 0) + { + ((PPDMQUEUEITEMCORE)pbItem)->u64View = UINT64_C(0xfeedfeedfeedfeed); + + /* next */ + pbItem += cbItem; + } +} + + +/** + * Allocate an item from a queue, extended version. + * + * The allocated item must be handed on to PDMR3QueueInsert() after the + * data have been filled in. + * + * @returns VBox status code. + * @param pVM Pointer to the cross context VM structure w/ ring-0. + * @param hQueue The queue handle. + * @param pvOwner The queue owner. + * @param ppNew Where to return the item pointer on success. + * @thread Any thread. + */ +VMMDECL(int) PDMQueueAllocEx(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner, PPDMQUEUEITEMCORE *ppNew) +{ + /* + * Validate and translate input. + */ + *ppNew = NULL; + PDMQUEUE_HANDLE_TO_VARS_RETURN(pVM, hQueue, pvOwner); + + /* + * Do the allocation. + */ + uint32_t cEmptyScans = 0; + for (;;) + { + int32_t iBit = ASMBitFirstSet(pQueue->bmAlloc, cItems); + if (iBit >= 0) + { + if (ASMAtomicBitTestAndClear(pQueue->bmAlloc, iBit)) + { + PPDMQUEUEITEMCORE pNew = (PPDMQUEUEITEMCORE)&((uint8_t *)pQueue)[offItems + iBit * cbItem]; + pNew->u64View = UINT64_C(0xbeefbeefbeefbeef); + *ppNew = pNew; + return VINF_SUCCESS; + } + cEmptyScans = 0; + } + else if (++cEmptyScans < 16) + ASMNopPause(); + else + { + STAM_REL_COUNTER_INC(&pQueue->StatAllocFailures); + return VERR_OUT_OF_RESOURCES; + } + } +} + + +/** + * Allocate an item from a queue. + * + * The allocated item must be handed on to PDMR3QueueInsert() after the + * data have been filled in. + * + * @returns Pointer to the new item on success, NULL on failure. + * @param pVM Pointer to the cross context VM structure w/ ring-0. + * @param hQueue The queue handle. + * @param pvOwner The queue owner. + * @thread Any thread. + */ +VMMDECL(PPDMQUEUEITEMCORE) PDMQueueAlloc(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner) +{ + PPDMQUEUEITEMCORE pNew = NULL; + int rc = PDMQueueAllocEx(pVM, hQueue, pvOwner, &pNew); + if (RT_SUCCESS(rc)) + return pNew; + return NULL; +} + + +/** + * Sets the FFs and fQueueFlushed. + * + * @param pVM Pointer to the cross context VM structure w/ ring-0. + */ +static void pdmQueueSetFF(PVMCC pVM) +{ + Log2(("PDMQueueInsert: VM_FF_PDM_QUEUES %d -> 1\n", VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES))); + VM_FF_SET(pVM, VM_FF_PDM_QUEUES); + ASMAtomicBitSet(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT); +#ifdef IN_RING3 + VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_DONE_REM); +#endif +} + + +/** + * Queue an item. + * + * The item must have been obtained using PDMQueueAlloc(). Once the item + * have been passed to this function it must not be touched! + * + * @returns VBox status code. + * @param pVM Pointer to the cross context VM structure w/ ring-0. + * @param hQueue The queue handle. + * @param pvOwner The queue owner. + * @param pInsert The item to insert. + * @thread Any thread. + */ +VMMDECL(int) PDMQueueInsert(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner, PPDMQUEUEITEMCORE pInsert) +{ + /* + * Validate and translate input. + */ + PDMQUEUE_HANDLE_TO_VARS_RETURN(pVM, hQueue, pvOwner); + + uint8_t * const pbItems = (uint8_t *)pQueue + offItems; + uintptr_t const offInsert = (uintptr_t)pInsert - (uintptr_t)pbItems; + uintptr_t const iInsert = offInsert / cbItem; + AssertReturn(iInsert < cItems, VERR_INVALID_PARAMETER); + AssertReturn(iInsert * cbItem == offInsert, VERR_INVALID_PARAMETER); + + AssertReturn(ASMBitTest(pQueue->bmAlloc, iInsert) == false, VERR_INVALID_PARAMETER); + + /* + * Append the item to the pending list. + */ + for (;;) + { + uint32_t const iOldPending = ASMAtomicUoReadU32(&pQueue->iPending); + pInsert->iNext = iOldPending; + if (ASMAtomicCmpXchgU32(&pQueue->iPending, iInsert, iOldPending)) + break; + ASMNopPause(); + } + + if (pQueue->hTimer == NIL_TMTIMERHANDLE) + pdmQueueSetFF(pVM); + STAM_REL_COUNTER_INC(&pQueue->StatInsert); + STAM_STATS({ ASMAtomicIncU32(&pQueue->cStatPending); }); + + return VINF_SUCCESS; +} + + +/** + * Schedule the queue for flushing (processing) if necessary. + * + * @returns VBox status code. + * @retval VINF_SUCCESS if a flush was necessary. + * @retval VINF_NO_CHANGE if no flushing needed. + * + * @param pVM The cross context VM structure. + * @param pvOwner The alleged queue owner. + * @param hQueue The queueu to maybe flush. + */ +VMMDECL(int) PDMQueueFlushIfNecessary(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner) +{ + /* + * Validate input. + */ + PDMQUEUE_HANDLE_TO_VARS_RETURN(pVM, hQueue, pvOwner); + RT_NOREF(offItems); + + /* + * Check and maybe flush. + */ + if (ASMAtomicUoReadU32(&pQueue->iPending) != UINT32_MAX) + { + pdmQueueSetFF(pVM); + return VINF_SUCCESS; + } + return VINF_NO_CHANGE; +} + diff --git a/src/VBox/VMM/VMMAll/PDMAllTask.cpp b/src/VBox/VMM/VMMAll/PDMAllTask.cpp new file mode 100644 index 00000000..72bce3d5 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PDMAllTask.cpp @@ -0,0 +1,124 @@ +/* $Id: PDMAllTask.cpp $ */ +/** @file + * PDM Task - Asynchronous user mode tasks, all context code. + */ + +/* + * Copyright (C) 2019-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PDM_TASK +#include "PDMInternal.h" +#include +#include +#include + +#include +#include +#include +#include +#include + + + +/** + * Triggers a task. + * + * @returns VBox status code. + * @retval VINF_ALREADY_POSTED is the task is already pending. + * + * @param pVM The cross context VM structure. + * @param enmType The task owner type. + * @param pvOwner The task owner. + * @param hTask The task to trigger. + * @thread Any + */ +VMM_INT_DECL(int) PDMTaskTrigger(PVMCC pVM, PDMTASKTYPE enmType, RTR3PTR pvOwner, PDMTASKHANDLE hTask) +{ + /* + * Validate input and translate the handle to a task. + */ + AssertReturn(pvOwner, VERR_NOT_OWNER); + AssertReturn(enmType >= PDMTASKTYPE_DEV && enmType <= PDMTASKTYPE_INTERNAL, VERR_NOT_OWNER); + + size_t const iTask = hTask % RT_ELEMENTS(pVM->pdm.s.aTaskSets[0].aTasks); + size_t const iTaskSet = hTask / RT_ELEMENTS(pVM->pdm.s.aTaskSets[0].aTasks); +#ifdef IN_RING3 + AssertReturn(iTaskSet < RT_ELEMENTS(pVM->pdm.s.apTaskSets), VERR_INVALID_HANDLE); + PPDMTASKSET pTaskSet = pVM->pdm.s.apTaskSets[iTaskSet]; + AssertReturn(pTaskSet, VERR_INVALID_HANDLE); +#else + AssertReturn(iTaskSet < RT_ELEMENTS(pVM->pdm.s.aTaskSets), + iTaskSet < RT_ELEMENTS(pVM->pdm.s.apTaskSets) ? VERR_INVALID_CONTEXT : VERR_INVALID_HANDLE); + PPDMTASKSET pTaskSet = &pVM->pdm.s.aTaskSets[iTaskSet]; +#endif + AssertReturn(pTaskSet->u32Magic == PDMTASKSET_MAGIC, VERR_INVALID_MAGIC); + PPDMTASK pTask = &pTaskSet->aTasks[iTask]; + + /* + * Check task ownership. + */ + AssertReturn(pvOwner == pTask->pvOwner, VERR_NOT_OWNER); + AssertReturn(enmType == pTask->enmType, VERR_NOT_OWNER); + + /* + * Trigger the task, wake up the thread if the task isn't triggered yet. + */ + if (!ASMAtomicBitTestAndSet(&pTaskSet->fTriggered, (uint32_t)iTask)) + { + Log(("PDMTaskTrigger: Triggered %RU64 (%s)\n", hTask, R3STRING(pTask->pszName))); +#ifdef IN_RING3 + if (pTaskSet->hEventR3 != NIL_RTSEMEVENT) + { + int rc = RTSemEventSignal(pTaskSet->hEventR3); + AssertLogRelRCReturn(rc, rc); + } + else +#endif + { + int rc = SUPSemEventSignal(pVM->pSession, pTaskSet->hEventR0); + AssertLogRelRCReturn(rc, rc); + } + return VINF_SUCCESS; + } + ASMAtomicIncU32(&pTask->cAlreadyTrigged); + Log(("PDMTaskTrigger: %RU64 (%s) was already triggered\n", hTask, R3STRING(pTask->pszName))); + return VINF_ALREADY_POSTED; +} + + +/** + * Triggers an internal task. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param hTask The task to trigger. + * @thread Any + */ +VMM_INT_DECL(int) PDMTaskTriggerInternal(PVMCC pVM, PDMTASKHANDLE hTask) +{ + return PDMTaskTrigger(pVM, PDMTASKTYPE_INTERNAL, pVM->pVMR3, hTask); +} + diff --git a/src/VBox/VMM/VMMAll/PGMAll.cpp b/src/VBox/VMM/VMMAll/PGMAll.cpp new file mode 100644 index 00000000..0074071a --- /dev/null +++ b/src/VBox/VMM/VMMAll/PGMAll.cpp @@ -0,0 +1,4077 @@ +/* $Id: PGMAll.cpp $ */ +/** @file + * PGM - Page Manager and Monitor - All context code. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PGM +#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "PGMInternal.h" +#include +#include "PGMInline.h" +#include +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +# include +#endif +#include +#include +#include +#include + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPT *ppPdpt, PX86PDPAE *ppPD); +DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde); +DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk, + PPGMPTWALKGST pGstWalk); +static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3); +static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT *ppPdpt, PEPTPD *ppPD, + PPGMPTWALKGST pGstWalkAll); +#endif +static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD); +static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT *ppPdpt, PEPTPD *ppPD); + + +/* + * Second level transation - EPT. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +# define PGM_SLAT_TYPE PGM_SLAT_TYPE_EPT +# include "PGMSlatDefs.h" +# include "PGMAllGstSlatEpt.cpp.h" +# undef PGM_SLAT_TYPE +#endif + + +/* + * Shadow - 32-bit mode + */ +#define PGM_SHW_TYPE PGM_TYPE_32BIT +#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name) +#include "PGMAllShw.h" + +/* Guest - real mode */ +#define PGM_GST_TYPE PGM_TYPE_REAL +#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS +#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS +#include "PGMGstDefs.h" +#include "PGMAllGst.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef BTH_PGMPOOLKIND_ROOT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - protected mode */ +#define PGM_GST_TYPE PGM_TYPE_PROT +#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS +#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS +#include "PGMGstDefs.h" +#include "PGMAllGst.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef BTH_PGMPOOLKIND_ROOT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - 32-bit mode */ +#define PGM_GST_TYPE PGM_TYPE_32BIT +#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT +#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB +#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD +#include "PGMGstDefs.h" +#include "PGMAllGst.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_BIG +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef BTH_PGMPOOLKIND_ROOT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +#undef PGM_SHW_TYPE +#undef PGM_SHW_NAME + + +/* + * Shadow - PAE mode + */ +#define PGM_SHW_TYPE PGM_TYPE_PAE +#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name) +#include "PGMAllShw.h" + +/* Guest - real mode */ +#define PGM_GST_TYPE PGM_TYPE_REAL +#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS +#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef BTH_PGMPOOLKIND_ROOT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - protected mode */ +#define PGM_GST_TYPE PGM_TYPE_PROT +#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS +#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef BTH_PGMPOOLKIND_ROOT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - 32-bit mode */ +#define PGM_GST_TYPE PGM_TYPE_32BIT +#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT +#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB +#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_BIG +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef BTH_PGMPOOLKIND_ROOT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + + +/* Guest - PAE mode */ +#define PGM_GST_TYPE PGM_TYPE_PAE +#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT +#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB +#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT +#include "PGMGstDefs.h" +#include "PGMAllGst.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_BIG +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef BTH_PGMPOOLKIND_ROOT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +#undef PGM_SHW_TYPE +#undef PGM_SHW_NAME + + +/* + * Shadow - AMD64 mode + */ +#define PGM_SHW_TYPE PGM_TYPE_AMD64 +#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name) +#include "PGMAllShw.h" + +/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */ +/** @todo retire this hack. */ +#define PGM_GST_TYPE PGM_TYPE_PROT +#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS +#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef BTH_PGMPOOLKIND_ROOT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +#ifdef VBOX_WITH_64_BITS_GUESTS +/* Guest - AMD64 mode */ +# define PGM_GST_TYPE PGM_TYPE_AMD64 +# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name) +# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name) +# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT +# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB +# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4 +# include "PGMGstDefs.h" +# include "PGMAllGst.h" +# include "PGMAllBth.h" +# undef BTH_PGMPOOLKIND_PT_FOR_BIG +# undef BTH_PGMPOOLKIND_PT_FOR_PT +# undef BTH_PGMPOOLKIND_ROOT +# undef PGM_BTH_NAME +# undef PGM_GST_TYPE +# undef PGM_GST_NAME +#endif /* VBOX_WITH_64_BITS_GUESTS */ + +#undef PGM_SHW_TYPE +#undef PGM_SHW_NAME + + +/* + * Shadow - 32-bit nested paging mode. + */ +#define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT +#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name) +#include "PGMAllShw.h" + +/* Guest - real mode */ +#define PGM_GST_TYPE PGM_TYPE_REAL +#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - protected mode */ +#define PGM_GST_TYPE PGM_TYPE_PROT +#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - 32-bit mode */ +#define PGM_GST_TYPE PGM_TYPE_32BIT +#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - PAE mode */ +#define PGM_GST_TYPE PGM_TYPE_PAE +#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +#ifdef VBOX_WITH_64_BITS_GUESTS +/* Guest - AMD64 mode */ +# define PGM_GST_TYPE PGM_TYPE_AMD64 +# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name) +# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name) +# include "PGMGstDefs.h" +# include "PGMAllBth.h" +# undef PGM_BTH_NAME +# undef PGM_GST_TYPE +# undef PGM_GST_NAME +#endif /* VBOX_WITH_64_BITS_GUESTS */ + +#undef PGM_SHW_TYPE +#undef PGM_SHW_NAME + + +/* + * Shadow - PAE nested paging mode. + */ +#define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE +#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name) +#include "PGMAllShw.h" + +/* Guest - real mode */ +#define PGM_GST_TYPE PGM_TYPE_REAL +#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - protected mode */ +#define PGM_GST_TYPE PGM_TYPE_PROT +#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - 32-bit mode */ +#define PGM_GST_TYPE PGM_TYPE_32BIT +#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - PAE mode */ +#define PGM_GST_TYPE PGM_TYPE_PAE +#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +#ifdef VBOX_WITH_64_BITS_GUESTS +/* Guest - AMD64 mode */ +# define PGM_GST_TYPE PGM_TYPE_AMD64 +# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name) +# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name) +# include "PGMGstDefs.h" +# include "PGMAllBth.h" +# undef PGM_BTH_NAME +# undef PGM_GST_TYPE +# undef PGM_GST_NAME +#endif /* VBOX_WITH_64_BITS_GUESTS */ + +#undef PGM_SHW_TYPE +#undef PGM_SHW_NAME + + +/* + * Shadow - AMD64 nested paging mode. + */ +#define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64 +#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name) +#include "PGMAllShw.h" + +/* Guest - real mode */ +#define PGM_GST_TYPE PGM_TYPE_REAL +#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - protected mode */ +#define PGM_GST_TYPE PGM_TYPE_PROT +#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - 32-bit mode */ +#define PGM_GST_TYPE PGM_TYPE_32BIT +#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - PAE mode */ +#define PGM_GST_TYPE PGM_TYPE_PAE +#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +#ifdef VBOX_WITH_64_BITS_GUESTS +/* Guest - AMD64 mode */ +# define PGM_GST_TYPE PGM_TYPE_AMD64 +# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name) +# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name) +# include "PGMGstDefs.h" +# include "PGMAllBth.h" +# undef PGM_BTH_NAME +# undef PGM_GST_TYPE +# undef PGM_GST_NAME +#endif /* VBOX_WITH_64_BITS_GUESTS */ + +#undef PGM_SHW_TYPE +#undef PGM_SHW_NAME + + +/* + * Shadow - EPT. + */ +#define PGM_SHW_TYPE PGM_TYPE_EPT +#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name) +#include "PGMAllShw.h" + +/* Guest - real mode */ +#define PGM_GST_TYPE PGM_TYPE_REAL +#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - protected mode */ +#define PGM_GST_TYPE PGM_TYPE_PROT +#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - 32-bit mode */ +#define PGM_GST_TYPE PGM_TYPE_32BIT +#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - PAE mode */ +#define PGM_GST_TYPE PGM_TYPE_PAE +#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name) +#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef BTH_PGMPOOLKIND_PT_FOR_PT +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +#ifdef VBOX_WITH_64_BITS_GUESTS +/* Guest - AMD64 mode */ +# define PGM_GST_TYPE PGM_TYPE_AMD64 +# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name) +# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name) +# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS +# include "PGMGstDefs.h" +# include "PGMAllBth.h" +# undef BTH_PGMPOOLKIND_PT_FOR_PT +# undef PGM_BTH_NAME +# undef PGM_GST_TYPE +# undef PGM_GST_NAME +#endif /* VBOX_WITH_64_BITS_GUESTS */ + +#undef PGM_SHW_TYPE +#undef PGM_SHW_NAME + + +/* + * Shadow - NEM / None. + */ +#define PGM_SHW_TYPE PGM_TYPE_NONE +#define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name) +#include "PGMAllShw.h" + +/* Guest - real mode */ +#define PGM_GST_TYPE PGM_TYPE_REAL +#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - protected mode */ +#define PGM_GST_TYPE PGM_TYPE_PROT +#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - 32-bit mode */ +#define PGM_GST_TYPE PGM_TYPE_32BIT +#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +/* Guest - PAE mode */ +#define PGM_GST_TYPE PGM_TYPE_PAE +#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name) +#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name) +#include "PGMGstDefs.h" +#include "PGMAllBth.h" +#undef PGM_BTH_NAME +#undef PGM_GST_TYPE +#undef PGM_GST_NAME + +#ifdef VBOX_WITH_64_BITS_GUESTS +/* Guest - AMD64 mode */ +# define PGM_GST_TYPE PGM_TYPE_AMD64 +# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name) +# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name) +# include "PGMGstDefs.h" +# include "PGMAllBth.h" +# undef PGM_BTH_NAME +# undef PGM_GST_TYPE +# undef PGM_GST_NAME +#endif /* VBOX_WITH_64_BITS_GUESTS */ + +#undef PGM_SHW_TYPE +#undef PGM_SHW_NAME + + + +/** + * Guest mode data array. + */ +PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] = +{ + { UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */ + { + PGM_TYPE_REAL, + PGM_GST_NAME_REAL(GetPage), + PGM_GST_NAME_REAL(ModifyPage), + PGM_GST_NAME_REAL(Enter), + PGM_GST_NAME_REAL(Exit), +#ifdef IN_RING3 + PGM_GST_NAME_REAL(Relocate), +#endif + }, + { + PGM_TYPE_PROT, + PGM_GST_NAME_PROT(GetPage), + PGM_GST_NAME_PROT(ModifyPage), + PGM_GST_NAME_PROT(Enter), + PGM_GST_NAME_PROT(Exit), +#ifdef IN_RING3 + PGM_GST_NAME_PROT(Relocate), +#endif + }, + { + PGM_TYPE_32BIT, + PGM_GST_NAME_32BIT(GetPage), + PGM_GST_NAME_32BIT(ModifyPage), + PGM_GST_NAME_32BIT(Enter), + PGM_GST_NAME_32BIT(Exit), +#ifdef IN_RING3 + PGM_GST_NAME_32BIT(Relocate), +#endif + }, + { + PGM_TYPE_PAE, + PGM_GST_NAME_PAE(GetPage), + PGM_GST_NAME_PAE(ModifyPage), + PGM_GST_NAME_PAE(Enter), + PGM_GST_NAME_PAE(Exit), +#ifdef IN_RING3 + PGM_GST_NAME_PAE(Relocate), +#endif + }, +#ifdef VBOX_WITH_64_BITS_GUESTS + { + PGM_TYPE_AMD64, + PGM_GST_NAME_AMD64(GetPage), + PGM_GST_NAME_AMD64(ModifyPage), + PGM_GST_NAME_AMD64(Enter), + PGM_GST_NAME_AMD64(Exit), +# ifdef IN_RING3 + PGM_GST_NAME_AMD64(Relocate), +# endif + }, +#endif +}; + + +/** + * The shadow mode data array. + */ +PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] = +{ + { UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */ + { UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */ + { UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */ + { + PGM_TYPE_32BIT, + PGM_SHW_NAME_32BIT(GetPage), + PGM_SHW_NAME_32BIT(ModifyPage), + PGM_SHW_NAME_32BIT(Enter), + PGM_SHW_NAME_32BIT(Exit), +#ifdef IN_RING3 + PGM_SHW_NAME_32BIT(Relocate), +#endif + }, + { + PGM_TYPE_PAE, + PGM_SHW_NAME_PAE(GetPage), + PGM_SHW_NAME_PAE(ModifyPage), + PGM_SHW_NAME_PAE(Enter), + PGM_SHW_NAME_PAE(Exit), +#ifdef IN_RING3 + PGM_SHW_NAME_PAE(Relocate), +#endif + }, + { + PGM_TYPE_AMD64, + PGM_SHW_NAME_AMD64(GetPage), + PGM_SHW_NAME_AMD64(ModifyPage), + PGM_SHW_NAME_AMD64(Enter), + PGM_SHW_NAME_AMD64(Exit), +#ifdef IN_RING3 + PGM_SHW_NAME_AMD64(Relocate), +#endif + }, + { + PGM_TYPE_NESTED_32BIT, + PGM_SHW_NAME_NESTED_32BIT(GetPage), + PGM_SHW_NAME_NESTED_32BIT(ModifyPage), + PGM_SHW_NAME_NESTED_32BIT(Enter), + PGM_SHW_NAME_NESTED_32BIT(Exit), +#ifdef IN_RING3 + PGM_SHW_NAME_NESTED_32BIT(Relocate), +#endif + }, + { + PGM_TYPE_NESTED_PAE, + PGM_SHW_NAME_NESTED_PAE(GetPage), + PGM_SHW_NAME_NESTED_PAE(ModifyPage), + PGM_SHW_NAME_NESTED_PAE(Enter), + PGM_SHW_NAME_NESTED_PAE(Exit), +#ifdef IN_RING3 + PGM_SHW_NAME_NESTED_PAE(Relocate), +#endif + }, + { + PGM_TYPE_NESTED_AMD64, + PGM_SHW_NAME_NESTED_AMD64(GetPage), + PGM_SHW_NAME_NESTED_AMD64(ModifyPage), + PGM_SHW_NAME_NESTED_AMD64(Enter), + PGM_SHW_NAME_NESTED_AMD64(Exit), +#ifdef IN_RING3 + PGM_SHW_NAME_NESTED_AMD64(Relocate), +#endif + }, + { + PGM_TYPE_EPT, + PGM_SHW_NAME_EPT(GetPage), + PGM_SHW_NAME_EPT(ModifyPage), + PGM_SHW_NAME_EPT(Enter), + PGM_SHW_NAME_EPT(Exit), +#ifdef IN_RING3 + PGM_SHW_NAME_EPT(Relocate), +#endif + }, + { + PGM_TYPE_NONE, + PGM_SHW_NAME_NONE(GetPage), + PGM_SHW_NAME_NONE(ModifyPage), + PGM_SHW_NAME_NONE(Enter), + PGM_SHW_NAME_NONE(Exit), +#ifdef IN_RING3 + PGM_SHW_NAME_NONE(Relocate), +#endif + }, +}; + + +/** + * The guest+shadow mode data array. + */ +PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] = +{ +#if !defined(IN_RING3) && !defined(VBOX_STRICT) +# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL } +# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \ + { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler) } + +#elif !defined(IN_RING3) && defined(VBOX_STRICT) +# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL } +# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \ + { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler), Nm(AssertCR3) } + +#elif defined(IN_RING3) && !defined(VBOX_STRICT) +# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL } +# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \ + { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), } + +#elif defined(IN_RING3) && defined(VBOX_STRICT) +# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL } +# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \ + { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) } + +#else +# error "Misconfig." +#endif + + /* 32-bit shadow paging mode: */ + PGMMODEDATABTH_NULL_ENTRY(), /* 0 */ + PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL), + PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT), + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */ + + /* PAE shadow paging mode: */ + PGMMODEDATABTH_NULL_ENTRY(), /* 0 */ + PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL), + PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE), + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */ + + /* AMD64 shadow paging mode: */ + PGMMODEDATABTH_NULL_ENTRY(), /* 0 */ + PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL), + PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT), + PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT), + PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE), +#ifdef VBOX_WITH_64_BITS_GUESTS + PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64), +#else + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */ +#endif + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */ + + /* 32-bit nested paging mode: */ + PGMMODEDATABTH_NULL_ENTRY(), /* 0 */ + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE), +#ifdef VBOX_WITH_64_BITS_GUESTS + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64), +#else + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */ +#endif + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */ + + /* PAE nested paging mode: */ + PGMMODEDATABTH_NULL_ENTRY(), /* 0 */ + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE), +#ifdef VBOX_WITH_64_BITS_GUESTS + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64), +#else + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */ +#endif + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */ + + /* AMD64 nested paging mode: */ + PGMMODEDATABTH_NULL_ENTRY(), /* 0 */ + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE), +#ifdef VBOX_WITH_64_BITS_GUESTS + PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64), +#else + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */ +#endif + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */ + + /* EPT nested paging mode: */ + PGMMODEDATABTH_NULL_ENTRY(), /* 0 */ + PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL), + PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE), +#ifdef VBOX_WITH_64_BITS_GUESTS + PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64), +#else + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */ +#endif + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */ + + /* NONE / NEM: */ + PGMMODEDATABTH_NULL_ENTRY(), /* 0 */ + PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT), + PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE), +#ifdef VBOX_WITH_64_BITS_GUESTS + PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64), +#else + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */ +#endif + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */ + PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */ + + +#undef PGMMODEDATABTH_ENTRY +#undef PGMMODEDATABTH_NULL_ENTRY +}; + + +/** Mask array used by pgmGetCr3MaskForMode. + * X86_CR3_AMD64_PAGE_MASK is used for modes that doesn't have a CR3 or EPTP. */ +static uint64_t const g_auCr3MaskForMode[PGMMODE_MAX] = +{ + /* [PGMMODE_INVALID] = */ X86_CR3_AMD64_PAGE_MASK, + /* [PGMMODE_REAL] = */ X86_CR3_AMD64_PAGE_MASK, + /* [PGMMODE_PROTECTED] = */ X86_CR3_AMD64_PAGE_MASK, + /* [PGMMODE_32_BIT] = */ X86_CR3_PAGE_MASK, + /* [PGMMODE_PAE] = */ X86_CR3_PAE_PAGE_MASK, + /* [PGMMODE_PAE_NX] = */ X86_CR3_PAE_PAGE_MASK, + /* [PGMMODE_AMD64] = */ X86_CR3_AMD64_PAGE_MASK, + /* [PGMMODE_AMD64_NX] = */ X86_CR3_AMD64_PAGE_MASK, + /* [PGMMODE_NESTED_32BIT = */ X86_CR3_PAGE_MASK, + /* [PGMMODE_NESTED_PAE] = */ X86_CR3_PAE_PAGE_MASK, + /* [PGMMODE_NESTED_AMD64] = */ X86_CR3_AMD64_PAGE_MASK, + /* [PGMMODE_EPT] = */ X86_CR3_EPT_PAGE_MASK, + /* [PGMMODE_NONE] = */ X86_CR3_AMD64_PAGE_MASK, +}; + + +/** + * Gets the physical address mask for CR3 in the given paging mode. + * + * The mask is for eliminating flags and other stuff in CR3/EPTP when + * extracting the physical address. It is not for validating whether there are + * reserved bits set. PGM ASSUMES that whoever loaded the CR3 value and passed + * it to PGM checked for reserved bits, including reserved physical address + * bits. + * + * @returns The CR3 mask. + * @param enmMode The paging mode. + * @param enmSlatMode The second-level address translation mode. + */ +DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode, PGMSLAT enmSlatMode) +{ + if (enmSlatMode == PGMSLAT_DIRECT) + { + Assert(enmMode != PGMMODE_EPT); + return g_auCr3MaskForMode[(unsigned)enmMode < (unsigned)PGMMODE_MAX ? enmMode : 0]; + } + Assert(enmSlatMode == PGMSLAT_EPT); + return X86_CR3_EPT_PAGE_MASK; +} + + +/** + * Gets the masked CR3 value according to the current guest paging mode. + * + * See disclaimer in pgmGetCr3MaskForMode. + * + * @returns The masked PGM CR3 value. + * @param pVCpu The cross context virtual CPU structure. + * @param uCr3 The raw guest CR3 value. + */ +DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3) +{ + uint64_t const fCr3Mask = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode, pVCpu->pgm.s.enmGuestSlatMode); + RTGCPHYS GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask); + PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3); + return GCPhysCR3; +} + + +#ifdef IN_RING0 +/** + * #PF Handler. + * + * @returns VBox status code (appropriate for trap handling and GC return). + * @param pVCpu The cross context virtual CPU structure. + * @param uErr The trap error code. + * @param pCtx Pointer to the register context for the CPU. + * @param pvFault The fault address. + */ +VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu))); + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } ); + + +# ifdef VBOX_WITH_STATISTICS + /* + * Error code stats. + */ + if (uErr & X86_TRAP_PF_US) + { + if (!(uErr & X86_TRAP_PF_P)) + { + if (uErr & X86_TRAP_PF_RW) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite); + else + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead); + } + else if (uErr & X86_TRAP_PF_RW) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite); + else if (uErr & X86_TRAP_PF_RSVD) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved); + else if (uErr & X86_TRAP_PF_ID) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE); + else + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead); + } + else + { /* Supervisor */ + if (!(uErr & X86_TRAP_PF_P)) + { + if (uErr & X86_TRAP_PF_RW) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite); + else + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead); + } + else if (uErr & X86_TRAP_PF_RW) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite); + else if (uErr & X86_TRAP_PF_ID) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE); + else if (uErr & X86_TRAP_PF_RSVD) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved); + } +# endif /* VBOX_WITH_STATISTICS */ + + /* + * Call the worker. + */ + uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE); + bool fLockTaken = false; + int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pCtx, pvFault, &fLockTaken); + if (fLockTaken) + { + PGM_LOCK_ASSERT_OWNER(pVM); + PGM_UNLOCK(pVM); + } + LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc)); + + /* + * Return code tweaks. + */ + if (rc != VINF_SUCCESS) + { + if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE) + rc = VINF_SUCCESS; + + /* Note: hack alert for difficult to reproduce problem. */ + if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */ + || rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */ + || rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */ + || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */ + { + Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pCtx->rip)); + /* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */ + rc = VINF_SUCCESS; + } + } + + STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); }); + STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0) + pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; }); + STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a); + return rc; +} +#endif /* IN_RING0 */ + + +/** + * Prefetch a page + * + * Typically used to sync commonly used pages before entering raw mode + * after a CR3 reload. + * + * @returns VBox status code suitable for scheduling. + * @retval VINF_SUCCESS on success. + * @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtrPage Page to invalidate. + */ +VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage) +{ + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a); + + uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE); + int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage); + + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a); + AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("rc=%Rrc\n", rc)); + return rc; +} + + +/** + * Emulation of the invlpg instruction (HC only actually). + * + * @returns Strict VBox status code, special care required. + * @retval VINF_PGM_SYNC_CR3 - handled. + * @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only). + * @retval VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtrPage Page to invalidate. + * + * @remark ASSUMES the page table entry or page directory is valid. Fairly + * safe, but there could be edge cases! + * + * @todo Flush page or page directory only if necessary! + * @todo VBOXSTRICTRC + */ +VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + int rc; + Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage)); + + IEMTlbInvalidatePage(pVCpu, GCPtrPage); + + /* + * Call paging mode specific worker. + */ + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a); + PGM_LOCK_VOID(pVM); + + uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE); + AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE); + rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage); + + PGM_UNLOCK(pVM); + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a); + + /* Ignore all irrelevant error codes. */ + if ( rc == VERR_PAGE_NOT_PRESENT + || rc == VERR_PAGE_TABLE_NOT_PRESENT + || rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT + || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) + rc = VINF_SUCCESS; + + return rc; +} + + +/** + * Executes an instruction using the interpreter. + * + * @returns VBox status code (appropriate for trap handling and GC return). + * @param pVCpu The cross context virtual CPU structure. + * @param pvFault Fault address. + */ +VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCPUCC pVCpu, RTGCPTR pvFault) +{ + RT_NOREF(pvFault); + VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu); + if (rc == VERR_EM_INTERPRETER) + rc = VINF_EM_RAW_EMULATE_INSTR; + if (rc != VINF_SUCCESS) + Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault)); + return rc; +} + + +/** + * Gets effective page information (from the VMM page directory). + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Guest Context virtual address of the page. + * @param pfFlags Where to store the flags. These are X86_PTE_*. + * @param pHCPhys Where to store the HC physical address of the page. + * This is page aligned. + * @remark You should use PGMMapGetPage() for pages in a mapping. + */ +VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + + uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData; + AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE); + int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys); + + PGM_UNLOCK(pVM); + return rc; +} + + +/** + * Modify page flags for a range of pages in the shadow context. + * + * The existing flags are ANDed with the fMask and ORed with the fFlags. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Virtual address of the first page in the range. + * @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course. + * @param fMask The AND mask - page flags X86_PTE_*. + * Be very CAREFUL when ~'ing constants which could be 32-bit! + * @param fOpFlags A combination of the PGM_MK_PK_XXX flags. + * @remark You must use PGMMapModifyPage() for pages in a mapping. + */ +DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags) +{ + AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags)); + Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 | PGM_MK_PG_IS_WRITE_FAULT))); + + GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; /** @todo this ain't necessary, right... */ + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + + uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData; + AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE); + int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, GUEST_PAGE_SIZE, fFlags, fMask, fOpFlags); + + PGM_UNLOCK(pVM); + return rc; +} + + +/** + * Changing the page flags for a single page in the shadow page tables so as to + * make it read-only. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Virtual address of the first page in the range. + * @param fOpFlags A combination of the PGM_MK_PK_XXX flags. + */ +VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags) +{ + return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags); +} + + +/** + * Changing the page flags for a single page in the shadow page tables so as to + * make it writable. + * + * The call must know with 101% certainty that the guest page tables maps this + * as writable too. This function will deal shared, zero and write monitored + * pages. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Virtual address of the first page in the range. + * @param fOpFlags A combination of the PGM_MK_PK_XXX flags. + */ +VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags) +{ + if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */ + return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags); + return VINF_SUCCESS; +} + + +/** + * Changing the page flags for a single page in the shadow page tables so as to + * make it not present. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Virtual address of the first page in the range. + * @param fOpFlags A combination of the PGM_MK_PG_XXX flags. + */ +VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags) +{ + return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags); +} + + +/** + * Changing the page flags for a single page in the shadow page tables so as to + * make it supervisor and writable. + * + * This if for dealing with CR0.WP=0 and readonly user pages. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Virtual address of the first page in the range. + * @param fBigPage Whether or not this is a big page. If it is, we have to + * change the shadow PDE as well. If it isn't, the caller + * has checked that the shadow PDE doesn't need changing. + * We ASSUME 4KB pages backing the big page here! + * @param fOpFlags A combination of the PGM_MK_PG_XXX flags. + */ +int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags) +{ + int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags); + if (rc == VINF_SUCCESS && fBigPage) + { + /* this is a bit ugly... */ + switch (pVCpu->pgm.s.enmShadowMode) + { + case PGMMODE_32_BIT: + { + PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr); + AssertReturn(pPde, VERR_INTERNAL_ERROR_3); + Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u)); + pPde->u |= X86_PDE_RW; + Log(("-> PDE=%#llx (32)\n", pPde->u)); + break; + } + case PGMMODE_PAE: + case PGMMODE_PAE_NX: + { + PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr); + AssertReturn(pPde, VERR_INTERNAL_ERROR_3); + Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u)); + pPde->u |= X86_PDE_RW; + Log(("-> PDE=%#llx (PAE)\n", pPde->u)); + break; + } + default: + AssertFailedReturn(VERR_INTERNAL_ERROR_4); + } + } + return rc; +} + + +/** + * Gets the shadow page directory for the specified address, PAE. + * + * @returns Pointer to the shadow PD. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr The address. + * @param uGstPdpe Guest PDPT entry. Valid. + * @param ppPD Receives address of page directory + */ +int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PPGMPOOLPAGE pShwPage; + int rc; + PGM_LOCK_ASSERT_OWNER(pVM); + + + /* Allocate page directory if not present. */ + const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE; + PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu); + PX86PDPE pPdpe = &pPdpt->a[iPdPt]; + X86PGPAEUINT const uPdpe = pPdpe->u; + if (uPdpe & (X86_PDPE_P | X86_PDPE_PG_MASK)) + { + pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK); + AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED); + Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key); + + pgmPoolCacheUsed(pPool, pShwPage); + + /* Update the entry if necessary. */ + X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key | (uGstPdpe & (X86_PDPE_P | X86_PDPE_A)) | (uPdpe & PGM_PDPT_FLAGS); + if (uPdpeNew == uPdpe) + { /* likely */ } + else + ASMAtomicWriteU64(&pPdpe->u, uPdpeNew); + } + else + { + RTGCPTR64 GCPdPt; + PGMPOOLKIND enmKind; + if (pVM->pgm.s.fNestedPaging || !CPUMIsGuestPagingEnabled(pVCpu)) + { + /* AMD-V nested paging or real/protected mode without paging. */ + GCPdPt = GCPtr & ~(RT_BIT_64(X86_PDPT_SHIFT) - 1); + enmKind = PGMPOOLKIND_PAE_PD_PHYS; + } + else if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE) + { + if (uGstPdpe & X86_PDPE_P) + { + GCPdPt = uGstPdpe & X86_PDPE_PG_MASK; + enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD; + } + else + { + /* PD not present; guest must reload CR3 to change it. + * No need to monitor anything in this case. */ + /** @todo r=bird: WTF is hit?!? */ + /*Assert(VM_IS_RAW_MODE_ENABLED(pVM)); - ??? */ + GCPdPt = uGstPdpe & X86_PDPE_PG_MASK; + enmKind = PGMPOOLKIND_PAE_PD_PHYS; + Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */ + } + } + else + { + GCPdPt = CPUMGetGuestCR3(pVCpu); + enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt); + } + + /* Create a reference back to the PDPT by using the index in its shadow page. */ + rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu), + pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /*fLockPage*/, + &pShwPage); + AssertRCReturn(rc, rc); + + /* Hook it up. */ + ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key | (uGstPdpe & (X86_PDPE_P | X86_PDPE_A)) | (uPdpe & PGM_PDPT_FLAGS)); + } + PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe); + + *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + return VINF_SUCCESS; +} + + +/** + * Gets the pointer to the shadow page directory entry for an address, PAE. + * + * @returns Pointer to the PDE. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPtr The address. + * @param ppShwPde Receives the address of the pgm pool page for the shadow page directory + */ +DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_ASSERT_OWNER(pVM); + + PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu); + AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */ + const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE; + X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u; + if (!(uPdpe & X86_PDPE_P)) + { + LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe)); + return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT; + } + AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr)); + + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK); + AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED); + + *ppShwPde = pShwPde; + return VINF_SUCCESS; +} + + +/** + * Syncs the SHADOW page directory pointer for the specified address. + * + * Allocates backing pages in case the PDPT or PML4 entry is missing. + * + * The caller is responsible for making sure the guest has a valid PD before + * calling this function. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr The address. + * @param uGstPml4e Guest PML4 entry (valid). + * @param uGstPdpe Guest PDPT entry (valid). + * @param ppPD Receives address of page directory + */ +static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging || !CPUMIsGuestPagingEnabled(pVCpu); + int rc; + + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * PML4. + */ + PPGMPOOLPAGE pShwPage; + { + const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK; + PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4); + X86PGPAEUINT const uPml4e = pPml4e->u; + + /* Allocate page directory pointer table if not present. */ + if (uPml4e & (X86_PML4E_P | X86_PML4E_PG_MASK)) + { + pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK); + AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED); + + pgmPoolCacheUsed(pPool, pShwPage); + + /* Update the entry if needed. */ + X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key | (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask) + | (uPml4e & PGM_PML4_FLAGS); + if (uPml4e == uPml4eNew) + { /* likely */ } + else + ASMAtomicWriteU64(&pPml4e->u, uPml4eNew); + } + else + { + Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); + + RTGCPTR64 GCPml4; + PGMPOOLKIND enmKind; + if (fNestedPagingOrNoGstPaging) + { + /* AMD-V nested paging or real/protected mode without paging */ + GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT; /** @todo bogus calculation for PML5 */ + enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS; + } + else + { + GCPml4 = uGstPml4e & X86_PML4E_PG_MASK; + enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT; + } + + /* Create a reference back to the PDPT by using the index in its shadow page. */ + rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu), + pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /*fLockPage*/, + &pShwPage); + AssertRCReturn(rc, rc); + + /* Hook it up. */ + ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask) + | (uPml4e & PGM_PML4_FLAGS)); + } + } + + /* + * PDPT. + */ + const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; + PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + PX86PDPE pPdpe = &pPdpt->a[iPdPt]; + X86PGPAEUINT const uPdpe = pPdpe->u; + + /* Allocate page directory if not present. */ + if (uPdpe & (X86_PDPE_P | X86_PDPE_PG_MASK)) + { + pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK); + AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED); + + pgmPoolCacheUsed(pPool, pShwPage); + + /* Update the entry if needed. */ + X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key | (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) + | (uPdpe & PGM_PDPT_FLAGS); + if (uPdpe == uPdpeNew) + { /* likely */ } + else + ASMAtomicWriteU64(&pPdpe->u, uPdpeNew); + } + else + { + RTGCPTR64 GCPdPt; + PGMPOOLKIND enmKind; + if (fNestedPagingOrNoGstPaging) + { + /* AMD-V nested paging or real/protected mode without paging */ + GCPdPt = GCPtr & ~(RT_BIT_64(iPdPt << X86_PDPT_SHIFT) - 1); + enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS; + } + else + { + GCPdPt = uGstPdpe & X86_PDPE_PG_MASK; + enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD; + } + + /* Create a reference back to the PDPT by using the index in its shadow page. */ + rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu), + pShwPage->idx, iPdPt, false /*fLockPage*/, + &pShwPage); + AssertRCReturn(rc, rc); + + /* Hook it up. */ + ASMAtomicWriteU64(&pPdpe->u, + pShwPage->Core.Key | (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) | (uPdpe & PGM_PDPT_FLAGS)); + } + + *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + return VINF_SUCCESS; +} + + +/** + * Gets the SHADOW page directory pointer for the specified address (long mode). + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr The address. + * @param ppPml4e Receives the address of the page map level 4 entry. + * @param ppPdpt Receives the address of the page directory pointer table. + * @param ppPD Receives the address of the page directory. + */ +DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPT *ppPdpt, PX86PDPAE *ppPD) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * PML4 + */ + const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK; + PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4); + AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING); + if (ppPml4e) + *ppPml4e = (PX86PML4E)pPml4e; + X86PGPAEUINT const uPml4e = pPml4e->u; + Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e)); + if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */ + return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT; + + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK); + AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED); + + /* + * PDPT + */ + const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; + PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u; + if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */ + return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT; + + pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK); + AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED); + + *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + Log4(("pgmShwGetLongModePDPtr %RGv -> *ppPD=%p PDE=%p/%RX64\n", GCPtr, *ppPD, &(*ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (*ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u)); + return VINF_SUCCESS; +} + + +/** + * Syncs the SHADOW EPT page directory pointer for the specified address. Allocates + * backing pages in case the PDPT or PML4 entry is missing. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr The address. + * @param ppPdpt Receives address of pdpt + * @param ppPD Receives address of page directory + */ +static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT *ppPdpt, PEPTPD *ppPD) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + int rc; + + Assert(pVM->pgm.s.fNestedPaging); + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * PML4 level. + */ + PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); + Assert(pPml4); + + /* Allocate page directory pointer table if not present. */ + PPGMPOOLPAGE pShwPage; + { + const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK; + PEPTPML4E pPml4e = &pPml4->a[iPml4]; + EPTPML4E Pml4e; + Pml4e.u = pPml4e->u; + if (!(Pml4e.u & (EPT_E_PG_MASK | EPT_E_READ))) + { + RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT; + rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu), + pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /*fLockPage*/, + &pShwPage); + AssertRCReturn(rc, rc); + + /* Hook up the new PDPT now. */ + ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE); + } + else + { + pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK); + AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED); + + pgmPoolCacheUsed(pPool, pShwPage); + + /* Hook up the cached PDPT if needed (probably not given 512*512 PTs to sync). */ + if (Pml4e.u == (pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE)) + { } + else + ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE); + } + } + + /* + * PDPT level. + */ + const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK; + PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + PEPTPDPTE pPdpe = &pPdpt->a[iPdPt]; + + if (ppPdpt) + *ppPdpt = pPdpt; + + /* Allocate page directory if not present. */ + EPTPDPTE Pdpe; + Pdpe.u = pPdpe->u; + if (!(Pdpe.u & (EPT_E_PG_MASK | EPT_E_READ))) + { + RTGCPTR64 const GCPdPt = GCPtr & ~(RT_BIT_64(EPT_PDPT_SHIFT) - 1); + rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu), + pShwPage->idx, iPdPt, false /*fLockPage*/, + &pShwPage); + AssertRCReturn(rc, rc); + + /* Hook up the new PD now. */ + ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE); + } + else + { + pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK); + AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED); + + pgmPoolCacheUsed(pPool, pShwPage); + + /* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */ + if (Pdpe.u == (pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE)) + { } + else + ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE); + } + + *ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + return VINF_SUCCESS; +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** + * Syncs the SHADOW nested-guest page directory pointer for the specified address. + * Allocates backing pages in case the PDPT or PML4 entry is missing. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhysNested The nested-guest physical address. + * @param ppPdpt Where to store the PDPT. Optional, can be NULL. + * @param ppPD Where to store the PD. Optional, can be NULL. + * @param pGstWalkAll The guest walk info. + */ +static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT *ppPdpt, PEPTPD *ppPD, + PPGMPTWALKGST pGstWalkAll) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + int rc; + + PPGMPOOLPAGE pShwPage; + Assert(pVM->pgm.s.fNestedPaging); + Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT); + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * PML4 level. + */ + { + PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); + Assert(pPml4); + + /* Allocate page directory pointer table if not present. */ + { + uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pml4e.u & pVCpu->pgm.s.fGstEptShadowedPml4eMask; + const unsigned iPml4e = (GCPhysNested >> EPT_PML4_SHIFT) & EPT_PML4_MASK; + PEPTPML4E pPml4e = &pPml4->a[iPml4e]; + + if (!(pPml4e->u & (EPT_E_PG_MASK | EPT_PRESENT_MASK))) + { + RTGCPHYS const GCPhysPdpt = pGstWalkAll->u.Ept.Pml4e.u & EPT_PML4E_PG_MASK; + rc = pgmPoolAlloc(pVM, GCPhysPdpt, PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, PGMPOOLACCESS_DONTCARE, + PGM_A20_IS_ENABLED(pVCpu), pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4e, false /*fLockPage*/, + &pShwPage); + AssertRCReturn(rc, rc); + + /* Hook up the new PDPT now. */ + ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | fShwFlags); + } + else + { + pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK); + AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED); + + pgmPoolCacheUsed(pPool, pShwPage); + + /* Hook up the cached PDPT if needed (probably not given 512*512 PTs to sync). */ + if (pPml4e->u != (pShwPage->Core.Key | fShwFlags)) + ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | fShwFlags); + } + Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage)); + Log7Func(("GstPml4e=%RX64 ShwPml4e=%RX64 iPml4e=%u\n", pGstWalkAll->u.Ept.Pml4e.u, pPml4e->u, iPml4e)); + } + } + + /* + * PDPT level. + */ + { + AssertReturn(!(pGstWalkAll->u.Ept.Pdpte.u & EPT_E_LEAF), VERR_NOT_SUPPORTED); /* shadowing 1GB pages not supported yet. */ + + PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + if (ppPdpt) + *ppPdpt = pPdpt; + + uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pdpte.u & pVCpu->pgm.s.fGstEptShadowedPdpteMask; + const unsigned iPdPte = (GCPhysNested >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK; + PEPTPDPTE pPdpte = &pPdpt->a[iPdPte]; + + if (!(pPdpte->u & (EPT_E_PG_MASK | EPT_PRESENT_MASK))) + { + RTGCPHYS const GCPhysPd = pGstWalkAll->u.Ept.Pdpte.u & EPT_PDPTE_PG_MASK; + rc = pgmPoolAlloc(pVM, GCPhysPd, PGMPOOLKIND_EPT_PD_FOR_EPT_PD, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu), + pShwPage->idx, iPdPte, false /*fLockPage*/, &pShwPage); + AssertRCReturn(rc, rc); + + /* Hook up the new PD now. */ + ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key | fShwFlags); + } + else + { + pShwPage = pgmPoolGetPage(pPool, pPdpte->u & EPT_PDPTE_PG_MASK); + AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED); + + pgmPoolCacheUsed(pPool, pShwPage); + + /* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */ + if (pPdpte->u != (pShwPage->Core.Key | fShwFlags)) + ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key | fShwFlags); + } + Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage)); + Log7Func(("GstPdpte=%RX64 ShwPdpte=%RX64 iPdPte=%u \n", pGstWalkAll->u.Ept.Pdpte.u, pPdpte->u, iPdPte)); + + *ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + } + + return VINF_SUCCESS; +} +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ + + +#ifdef IN_RING0 +/** + * Synchronizes a range of nested page table entries. + * + * The caller must own the PGM lock. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPhys Where to start. + * @param cPages How many pages which entries should be synced. + * @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x, + * host paging mode for AMD-V). + */ +int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode) +{ + PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM)); + +/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */ + int rc; + switch (enmShwPagingMode) + { + case PGMMODE_32_BIT: + { + X86PDE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A }; + rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/); + break; + } + + case PGMMODE_PAE: + case PGMMODE_PAE_NX: + { + X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A }; + rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/); + break; + } + + case PGMMODE_AMD64: + case PGMMODE_AMD64_NX: + { + X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A }; + rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/); + break; + } + + case PGMMODE_EPT: + { + X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A }; + rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/); + break; + } + + default: + AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE); + } + return rc; +} +#endif /* IN_RING0 */ + + +/** + * Gets effective Guest OS page information. + * + * When GCPtr is in a big page, the function will return as if it was a normal + * 4KB page. If the need for distinguishing between big and normal page becomes + * necessary at a later point, a PGMGstGetPage() will be created for that + * purpose. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPtr Guest Context virtual address of the page. + * @param pWalk Where to store the page walk information. + */ +VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk) +{ + VMCPU_ASSERT_EMT(pVCpu); + Assert(pWalk); + uintptr_t idx = pVCpu->pgm.s.idxGuestModeData; + AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE); + return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pWalk); +} + + +/** + * Maps the guest CR3. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhysCr3 The guest CR3 value. + * @param pHCPtrGuestCr3 Where to store the mapped memory. + */ +DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3) +{ + /** @todo this needs some reworking wrt. locking? */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + PPGMPAGE pPageCr3 = pgmPhysGetPage(pVM, GCPhysCr3); + AssertReturnStmt(pPageCr3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR); + + RTHCPTR HCPtrGuestCr3; + int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCr3, GCPhysCr3, (void **)&HCPtrGuestCr3); + PGM_UNLOCK(pVM); + + *pHCPtrGuestCr3 = HCPtrGuestCr3; + return rc; +} + + +/** + * Unmaps the guest CR3. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(int) pgmGstUnmapCr3(PVMCPUCC pVCpu) +{ + uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE); + return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu); +} + + +/** + * Performs a guest page table walk. + * + * The guest should be in paged protect mode or long mode when making a call to + * this function. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details. + * @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is + * not valid, except enmType is PGMPTWALKGSTTYPE_INVALID. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPtr The guest virtual address to walk by. + * @param pWalk Where to return the walk result. This is valid for some + * error codes as well. + * @param pGstWalk The guest mode specific page walk information. + */ +int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk) +{ + VMCPU_ASSERT_EMT(pVCpu); + switch (pVCpu->pgm.s.enmGuestMode) + { + case PGMMODE_32_BIT: + pGstWalk->enmType = PGMPTWALKGSTTYPE_32BIT; + return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Legacy); + + case PGMMODE_PAE: + case PGMMODE_PAE_NX: + pGstWalk->enmType = PGMPTWALKGSTTYPE_PAE; + return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Pae); + + case PGMMODE_AMD64: + case PGMMODE_AMD64_NX: + pGstWalk->enmType = PGMPTWALKGSTTYPE_AMD64; + return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Amd64); + + case PGMMODE_REAL: + case PGMMODE_PROTECTED: + pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID; + return VERR_PGM_NOT_USED_IN_MODE; + + case PGMMODE_EPT: + case PGMMODE_NESTED_32BIT: + case PGMMODE_NESTED_PAE: + case PGMMODE_NESTED_AMD64: + default: + AssertFailed(); + pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID; + return VERR_PGM_NOT_USED_IN_MODE; + } +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** + * Performs a guest second-level address translation (SLAT). + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details. + * @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is + * not valid, except enmType is PGMPTWALKGSTTYPE_INVALID. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPhysNested The nested-guest physical address being translated. + * @param fIsLinearAddrValid Whether the linear address in @a GCPtrNested is the + * cause for this translation. + * @param GCPtrNested The nested-guest virtual address that initiated the + * SLAT. If none, pass 0 (and not NIL_RTGCPTR). + * @param pWalk Where to return the walk result. This is updated for + * all error codes other than + * VERR_PGM_NOT_USED_IN_MODE. + * @param pGstWalk Where to store the second-level paging-mode specific + * walk info. + */ +static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, + PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk) +{ + /* SLAT mode must be valid at this point as this should only be used -after- we have determined SLAT mode. */ + Assert( pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_DIRECT + && pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_INVALID); + AssertPtr(pWalk); + AssertPtr(pGstWalk); + switch (pVCpu->pgm.s.enmGuestSlatMode) + { + case PGMSLAT_EPT: + pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT; + return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &pGstWalk->u.Ept); + + default: + AssertFailed(); + pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID; + return VERR_PGM_NOT_USED_IN_MODE; + } +} +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ + + +/** + * Tries to continue the previous walk. + * + * @note Requires the caller to hold the PGM lock from the first + * pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise + * we cannot use the pointers. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details. + * @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is + * not valid, except enmType is PGMPTWALKGSTTYPE_INVALID. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPtr The guest virtual address to walk by. + * @param pWalk Pointer to the previous walk result and where to return + * the result of this walk. This is valid for some error + * codes as well. + * @param pGstWalk The guest-mode specific walk information. + */ +int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk) +{ + /* + * We can only handle successfully walks. + * We also limit ourselves to the next page. + */ + if ( pWalk->fSucceeded + && GCPtr - pWalk->GCPtr == GUEST_PAGE_SIZE) + { + Assert(pWalk->uLevel == 0); + if (pGstWalk->enmType == PGMPTWALKGSTTYPE_AMD64) + { + /* + * AMD64 + */ + if (!pWalk->fGigantPage && !pWalk->fBigPage) + { + /* + * We fall back to full walk if the PDE table changes, if any + * reserved bits are set, or if the effective page access changes. + */ + const uint64_t fPteSame = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_PWT + | X86_PTE_PCD | X86_PTE_A | X86_PTE_PAE_NX; + const uint64_t fPdeSame = X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT + | X86_PDE_PCD | X86_PDE_A | X86_PDE_PAE_NX | X86_PDE_PS; + + if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->GCPtr >> X86_PD_PAE_SHIFT)) + { + if (pGstWalk->u.Amd64.pPte) + { + X86PTEPAE Pte; + Pte.u = pGstWalk->u.Amd64.pPte[1].u; + if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame) + && !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask)) + { + pWalk->GCPtr = GCPtr; + pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK; + pGstWalk->u.Amd64.Pte.u = Pte.u; + pGstWalk->u.Amd64.pPte++; + return VINF_SUCCESS; + } + } + } + else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->GCPtr >> X86_PDPT_SHIFT)) + { + Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */ + if (pGstWalk->u.Amd64.pPde) + { + X86PDEPAE Pde; + Pde.u = pGstWalk->u.Amd64.pPde[1].u; + if ( (Pde.u & fPdeSame) == (pGstWalk->u.Amd64.Pde.u & fPdeSame) + && !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask)) + { + /* Get the new PTE and check out the first entry. */ + int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)), + &pGstWalk->u.Amd64.pPt); + if (RT_SUCCESS(rc)) + { + pGstWalk->u.Amd64.pPte = &pGstWalk->u.Amd64.pPt->a[0]; + X86PTEPAE Pte; + Pte.u = pGstWalk->u.Amd64.pPte->u; + if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame) + && !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask)) + { + pWalk->GCPtr = GCPtr; + pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK; + pGstWalk->u.Amd64.Pte.u = Pte.u; + pGstWalk->u.Amd64.Pde.u = Pde.u; + pGstWalk->u.Amd64.pPde++; + return VINF_SUCCESS; + } + } + } + } + } + } + else if (!pWalk->fGigantPage) + { + if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_2M_BASE_MASK)) + { + pWalk->GCPtr = GCPtr; + pWalk->GCPhys += GUEST_PAGE_SIZE; + return VINF_SUCCESS; + } + } + else + { + if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_1G_BASE_MASK)) + { + pWalk->GCPtr = GCPtr; + pWalk->GCPhys += GUEST_PAGE_SIZE; + return VINF_SUCCESS; + } + } + } + } + /* Case we don't handle. Do full walk. */ + return pgmGstPtWalk(pVCpu, GCPtr, pWalk, pGstWalk); +} + + +/** + * Modify page flags for a range of pages in the guest's tables + * + * The existing flags are ANDed with the fMask and ORed with the fFlags. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Virtual address of the first page in the range. + * @param cb Size (in bytes) of the range to apply the modification to. + * @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course. + * @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course. + * Be very CAREFUL when ~'ing constants which could be 32-bit! + */ +VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask) +{ + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a); + VMCPU_ASSERT_EMT(pVCpu); + + /* + * Validate input. + */ + AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags)); + Assert(cb); + + LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask)); + + /* + * Adjust input. + */ + cb += GCPtr & GUEST_PAGE_OFFSET_MASK; + cb = RT_ALIGN_Z(cb, GUEST_PAGE_SIZE); + GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; + + /* + * Call worker. + */ + uintptr_t idx = pVCpu->pgm.s.idxGuestModeData; + AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE); + int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask); + + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a); + return rc; +} + + +/** + * Checks whether the given PAE PDPEs are potentially valid for the guest. + * + * @returns @c true if the PDPE is valid, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param paPaePdpes The PAE PDPEs to validate. + * + * @remarks This function -only- checks the reserved bits in the PDPE entries. + */ +VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes) +{ + Assert(paPaePdpes); + for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++) + { + X86PDPE const PaePdpe = paPaePdpes[i]; + if ( !(PaePdpe.u & X86_PDPE_P) + || !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask)) + { /* likely */ } + else + return false; + } + return true; +} + + +/** + * Performs the lazy mapping of the 32-bit guest PD. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param ppPd Where to return the pointer to the mapping. This is + * always set. + */ +int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + + Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd)); + + RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3); + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage); + if (RT_SUCCESS(rc)) + { + rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd); + if (RT_SUCCESS(rc)) + { +# ifdef IN_RING3 + pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR; + pVCpu->pgm.s.pGst32BitPdR3 = *ppPd; +# else + pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR; + pVCpu->pgm.s.pGst32BitPdR0 = *ppPd; +# endif + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + AssertRC(rc); + } + PGM_UNLOCK(pVM); + + *ppPd = NULL; + return rc; +} + + +/** + * Performs the lazy mapping of the PAE guest PDPT. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param ppPdpt Where to return the pointer to the mapping. This is + * always set. + */ +int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt) +{ + Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt)); + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + + RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3); + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage); + if (RT_SUCCESS(rc)) + { + rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt); + if (RT_SUCCESS(rc)) + { +# ifdef IN_RING3 + pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR; + pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt; +# else + pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR; + pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt; +# endif + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + AssertRC(rc); + } + + PGM_UNLOCK(pVM); + *ppPdpt = NULL; + return rc; +} + + +/** + * Performs the lazy mapping / updating of a PAE guest PD. + * + * @returns Pointer to the mapping. + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param iPdpt Which PD entry to map (0..3). + * @param ppPd Where to return the pointer to the mapping. This is + * always set. + */ +int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + + PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt); + Assert(pGuestPDPT); + Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P); + RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK; + bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys; + + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage); + if (RT_SUCCESS(rc)) + { + rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd); + AssertRC(rc); + if (RT_SUCCESS(rc)) + { +# ifdef IN_RING3 + pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR; + pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd; +# else + pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR; + pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd; +# endif + if (fChanged) + pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys; + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + } + + /* Invalid page or some failure, invalidate the entry. */ + pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS; + pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR; + pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR; + + PGM_UNLOCK(pVM); + return rc; +} + + +/** + * Performs the lazy mapping of the 32-bit guest PD. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param ppPml4 Where to return the pointer to the mapping. This will + * always be set. + */ +int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4) +{ + Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4)); + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + + RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3); + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage); + if (RT_SUCCESS(rc)) + { + rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4); + if (RT_SUCCESS(rc)) + { +# ifdef IN_RING3 + pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR; + pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4; +# else + pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR; + pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4; +# endif + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + } + + PGM_UNLOCK(pVM); + *ppPml4 = NULL; + return rc; +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + /** + * Performs the lazy mapping of the guest PML4 table when using EPT paging. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param ppEptPml4 Where to return the pointer to the mapping. This will + * always be set. + */ +int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4) +{ + Assert(!pVCpu->pgm.s.CTX_SUFF(pGstEptPml4)); + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + + RTGCPHYS const GCPhysEpt = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK; + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhysEpt, &pPage); + if (RT_SUCCESS(rc)) + { + rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysEpt, (void **)ppEptPml4); + if (RT_SUCCESS(rc)) + { +# ifdef IN_RING3 + pVCpu->pgm.s.pGstEptPml4R0 = NIL_RTR0PTR; + pVCpu->pgm.s.pGstEptPml4R3 = *ppEptPml4; +# else + pVCpu->pgm.s.pGstEptPml4R3 = NIL_RTR3PTR; + pVCpu->pgm.s.pGstEptPml4R0 = *ppEptPml4; +# endif + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + } + + PGM_UNLOCK(pVM); + *ppEptPml4 = NULL; + return rc; +} +#endif + + +/** + * Gets the current CR3 register value for the shadow memory context. + * @returns CR3 value. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu) +{ + PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3); + AssertPtrReturn(pPoolPage, NIL_RTHCPHYS); + return pPoolPage->Core.Key; +} + + +/** + * Forces lazy remapping of the guest's PAE page-directory structures. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void pgmGstFlushPaePdpes(PVMCPU pVCpu) +{ + for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++) + { + pVCpu->pgm.s.apGstPaePDsR3[i] = 0; + pVCpu->pgm.s.apGstPaePDsR0[i] = 0; + pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS; + } +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** + * Performs second-level address translation for the given CR3 and updates the + * nested-guest CR3 when successful. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uCr3 The masked nested-guest CR3 value. + * @param pGCPhysCR3 Where to store the translated CR3. + * + * @warning This updates PGMCPU::GCPhysNstGstCR3 when the translation succeeds. Be + * mindful of this in code that's hyper sensitive to the order of + * operations. + */ +static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3) +{ + if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3) + { + PGMPTWALK Walk; + PGMPTWALKGST GstWalk; + int const rc = pgmGstSlatWalk(pVCpu, uCr3, false /* fIsLinearAddrValid */, 0 /* GCPtrNested */, &Walk, &GstWalk); + if (RT_SUCCESS(rc)) + { + /* Update nested-guest CR3. */ + pVCpu->pgm.s.GCPhysNstGstCR3 = uCr3; + + /* Pass back the translated result. */ + *pGCPhysCr3 = Walk.GCPhys; + return VINF_SUCCESS; + } + + /* Translation failed. */ + *pGCPhysCr3 = NIL_RTGCPHYS; + return rc; + } + + /* + * If the nested-guest CR3 has not changed, then the previously + * translated CR3 result (i.e. GCPhysCR3) is passed back. + */ + *pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3; + return VINF_SUCCESS; +} +#endif + + +/** + * Performs and schedules necessary updates following a CR3 load or reload. + * + * This will normally involve mapping the guest PD or nPDPT + * + * @returns VBox status code. + * @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can + * safely be ignored and overridden since the FF will be set too then. + * @param pVCpu The cross context virtual CPU structure. + * @param cr3 The new cr3. + * @param fGlobal Indicates whether this is a global flush or not. + */ +VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal) +{ + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a); + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + VMCPU_ASSERT_EMT(pVCpu); + + /* + * Always flag the necessary updates; necessary for hardware acceleration + */ + /** @todo optimize this, it shouldn't always be necessary. */ + VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL); + if (fGlobal) + VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); + + /* + * Remap the CR3 content and adjust the monitoring if CR3 was actually changed. + */ + RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3; + RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if ( pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT + && PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode)) + { + RTGCPHYS GCPhysOut; + int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut); + if (RT_SUCCESS(rc)) + GCPhysCR3 = GCPhysOut; + else + { + /* CR3 SLAT translation failed but we try to pretend it + succeeded for the reasons mentioned in PGMHCChangeMode(). */ + AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc)); + int const rc2 = pgmGstUnmapCr3(pVCpu); + pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS; + pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS; + return rc2; + } + } +#endif + + LogFlowFunc(("cr3=%RX64 old=%RX64 fGlobal=%d\n", cr3, GCPhysOldCR3, fGlobal)); + int rc = VINF_SUCCESS; + if (GCPhysOldCR3 != GCPhysCR3) + { + uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE); + + pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3; + rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3); + if (RT_LIKELY(rc == VINF_SUCCESS)) + { } + else + { + AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc)); + Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_PGM_SYNC_CR3)); + pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) = false; + pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS; + pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3; + pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_MAP_CR3; + } + + if (fGlobal) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global)); + else + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3)); + } + else + { +#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + if (pPool->cDirtyPages) + { + PGM_LOCK_VOID(pVM); + pgmPoolResetDirtyPages(pVM); + PGM_UNLOCK(pVM); + } +#endif + if (fGlobal) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global)); + else + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3)); + + /* + * Flush PAE PDPTEs. + */ + if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode)) + pgmGstFlushPaePdpes(pVCpu); + } + + IEMTlbInvalidateAll(pVCpu); + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a); + return rc; +} + + +/** + * Performs and schedules necessary updates following a CR3 load or reload when + * using nested or extended paging. + * + * This API is an alternative to PGMFlushTLB that avoids actually flushing the + * TLB and triggering a SyncCR3. + * + * This will normally involve mapping the guest PD or nPDPT + * + * @returns VBox status code. + * @retval VINF_SUCCESS. + * @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested + * paging modes). This can safely be ignored and overridden since the + * FF will be set too then. + * @param pVCpu The cross context virtual CPU structure. + * @param cr3 The new CR3. + */ +VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3) +{ + VMCPU_ASSERT_EMT(pVCpu); + + /* We assume we're only called in nested paging mode. */ + Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging || pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT); + + /* + * Remap the CR3 content and adjust the monitoring if CR3 was actually changed. + */ + RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3; + RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) + { + RTGCPHYS GCPhysOut; + int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut); + if (RT_SUCCESS(rc)) + GCPhysCR3 = GCPhysOut; + else + { + /* CR3 SLAT translation failed but we try to pretend it + succeeded for the reasons mentioned in PGMHCChangeMode(). */ + Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc)); + int const rc2 = pgmGstUnmapCr3(pVCpu); + pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS; + pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS; + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3); + return rc2; + } + } +#endif + + LogFlowFunc(("cr3=%RX64 old=%RX64\n", cr3, GCPhysOldCR3)); + int rc = VINF_SUCCESS; + if (GCPhysOldCR3 != GCPhysCR3) + { + uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE); + + pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3; + rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3); + + AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. */ /** @todo this isn't true for the mac, but we need hw to test/fix this. */ + } + /* + * Flush PAE PDPTEs. + */ + else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode)) + pgmGstFlushPaePdpes(pVCpu); + + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3); + return rc; +} + + +/** + * Synchronize the paging structures. + * + * This function is called in response to the VM_FF_PGM_SYNC_CR3 and + * VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set + * in several places, most importantly whenever the CR3 is loaded. + * + * @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0. + * @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map + * the VMM into guest context. + * @param pVCpu The cross context virtual CPU structure. + * @param cr0 Guest context CR0 register + * @param cr3 Guest context CR3 register + * @param cr4 Guest context CR4 register + * @param fGlobal Including global page directories or not + */ +VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal) +{ + int rc; + + VMCPU_ASSERT_EMT(pVCpu); + + /* + * The pool may have pending stuff and even require a return to ring-3 to + * clear the whole thing. + */ + rc = pgmPoolSyncCR3(pVCpu); + if (rc != VINF_SUCCESS) + return rc; + + /* + * We might be called when we shouldn't. + * + * The mode switching will ensure that the PD is resynced after every mode + * switch. So, if we find ourselves here when in protected or real mode + * we can safely clear the FF and return immediately. + */ + if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED) + { + Assert((cr0 & (X86_CR0_PG | X86_CR0_PE)) != (X86_CR0_PG | X86_CR0_PE)); + Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3); + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL); + return VINF_SUCCESS; + } + + /* If global pages are not supported, then all flushes are global. */ + if (!(cr4 & X86_CR4_PGE)) + fGlobal = true; + LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal, + VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))); + + /* + * Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB). + * This should be done before SyncCR3. + */ + if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3) + { + pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3; + + RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3; + RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) + { + RTGCPHYS GCPhysOut; + int rc2 = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut); + if (RT_SUCCESS(rc2)) + GCPhysCR3 = GCPhysOut; + else + { + /* CR3 SLAT translation failed but we try to pretend it + succeeded for the reasons mentioned in PGMHCChangeMode(). */ + AssertMsgFailed(("Failed to translate CR3 %#RX64. rc=%Rrc\n", cr3, rc2)); + pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS; + pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS; + return rc2; + } + } +#endif + Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped)); + if (GCPhysOldCR3 != GCPhysCR3) + { + uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE); + pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3; + rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3); + } + + /* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */ + if ( rc == VINF_PGM_SYNC_CR3 + || (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)) + { + Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n")); +#ifdef IN_RING3 + rc = pgmPoolSyncCR3(pVCpu); +#else + if (rc == VINF_PGM_SYNC_CR3) + pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3; + return VINF_PGM_SYNC_CR3; +#endif + } + AssertRCReturn(rc, rc); + AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS); + } + + /* + * Let the 'Bth' function do the work and we'll just keep track of the flags. + */ + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a); + + uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE); + rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal); + + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a); + AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("rc=%Rrc\n", rc)); + if (rc == VINF_SUCCESS) + { + if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) + { + /* Go back to ring 3 if a pgm pool sync is again pending. */ + return VINF_PGM_SYNC_CR3; + } + + if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS)) + { + Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3); + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL); + } + } + + /* + * Now flush the CR3 (guest context). + */ + if (rc == VINF_SUCCESS) + PGM_INVL_VCPU_TLBS(pVCpu); + return rc; +} + + +/** + * Maps all the PAE PDPE entries. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param paPaePdpes The new PAE PDPE values. + * + * @remarks This function may be invoked during the process of changing the guest + * paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not + * reflect PAE paging just yet. + */ +VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes) +{ + Assert(paPaePdpes); + for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++) + { + X86PDPE const PaePdpe = paPaePdpes[i]; + + /* + * In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs + * are deferred.[1] Also, different situations require different handling of invalid + * PDPE entries. Here we assume the caller has already validated or doesn't require + * validation of the PDPEs. + * + * In the case of nested EPT (i.e. for nested-guests), the PAE PDPEs have been + * validated by the VMX transition. + * + * [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode". + */ + if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask | X86_PDPE_P)) == X86_PDPE_P) + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + RTHCPTR HCPtr; + + RTGCPHYS GCPhys; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) + { + PGMPTWALK Walk; + PGMPTWALKGST GstWalk; + RTGCPHYS const GCPhysNested = PaePdpe.u & X86_PDPE_PG_MASK; + int const rc = pgmGstSlatWalk(pVCpu, GCPhysNested, false /* fIsLinearAddrValid */, 0 /* GCPtrNested */, + &Walk, &GstWalk); + if (RT_SUCCESS(rc)) + GCPhys = Walk.GCPhys; + else + { + /* + * Second-level address translation of the PAE PDPE has failed but we must -NOT- + * abort and return a failure now. This is because we're called from a Mov CRx + * instruction (or similar operation). Let's just pretend success but flag that + * we need to map this PDPE lazily later. + * + * See Intel spec. 25.3 "Changes to instruction behavior in VMX non-root operation". + * See Intel spec. 28.3.1 "EPT Overview". + */ + pVCpu->pgm.s.apGstPaePDsR3[i] = 0; + pVCpu->pgm.s.apGstPaePDsR0[i] = 0; + pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS; + continue; + } + } + else +#endif + { + GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK); + } + + PGM_LOCK_VOID(pVM); + PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys); + AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR); + int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr); + PGM_UNLOCK(pVM); + if (RT_SUCCESS(rc)) + { +#ifdef IN_RING3 + pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr; + pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR; +#else + pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR; + pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr; +#endif + pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys; + continue; + } + AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i)); + } + pVCpu->pgm.s.apGstPaePDsR3[i] = 0; + pVCpu->pgm.s.apGstPaePDsR0[i] = 0; + pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS; + } + return VINF_SUCCESS; +} + + +/** + * Validates and maps the PDPT and PAE PDPEs referenced by the given CR3. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cr3 The guest CR3 value. + * + * @remarks This function may be invoked during the process of changing the guest + * paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect + * PAE paging just yet. + */ +VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3) +{ + /* + * Read the page-directory-pointer table (PDPT) at CR3. + */ + RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK); + PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3); + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) + { + RTGCPHYS GCPhysOut; + int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut); + if (RT_SUCCESS(rc)) + GCPhysCR3 = GCPhysOut; + else + { + Log(("Failed to load CR3 at %#RX64. rc=%Rrc\n", GCPhysCR3, rc)); + return rc; + } + } +#endif + + RTHCPTR HCPtrGuestCr3; + int rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCr3); + if (RT_SUCCESS(rc)) + { + /* + * Validate the page-directory-pointer table entries (PDPE). + */ + X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES]; + memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes)); + if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0])) + { + /* + * Map the PDPT. + * We deliberately don't update PGM's GCPhysCR3 here as it's expected + * that PGMFlushTLB will be called soon and only a change to CR3 then + * will cause the shadow page tables to be updated. + */ +#ifdef IN_RING3 + pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3; + pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR; +#else + pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR; + pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3; +#endif + + /* + * Update CPUM and map the 4 PAE PDPEs. + */ + CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]); + rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]); + if (RT_SUCCESS(rc)) + { +#ifdef IN_RING3 + pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true; + pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false; +#else + pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false; + pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true; +#endif + pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3; + } + } + else + rc = VERR_PGM_PAE_PDPE_RSVD; + } + return rc; +} + + +/** + * Called whenever CR0 or CR4 in a way which may affect the paging mode. + * + * @returns VBox status code, with the following informational code for + * VM scheduling. + * @retval VINF_SUCCESS if the was no change, or it was successfully dealt with. + * @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only) + * + * @param pVCpu The cross context virtual CPU structure. + * @param cr0 The new cr0. + * @param cr4 The new cr4. + * @param efer The new extended feature enable register. + * @param fForce Whether to force a mode change. + */ +VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer, bool fForce) +{ + VMCPU_ASSERT_EMT(pVCpu); + + /* + * Calc the new guest mode. + * + * Note! We check PG before PE and without requiring PE because of the + * special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9). + */ + PGMMODE enmGuestMode; + if (cr0 & X86_CR0_PG) + { + if (!(cr4 & X86_CR4_PAE)) + { + bool const fPse = !!(cr4 & X86_CR4_PSE); + if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse) + Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse)); + pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse; + enmGuestMode = PGMMODE_32_BIT; + } + else if (!(efer & MSR_K6_EFER_LME)) + { + if (!(efer & MSR_K6_EFER_NXE)) + enmGuestMode = PGMMODE_PAE; + else + enmGuestMode = PGMMODE_PAE_NX; + } + else + { + if (!(efer & MSR_K6_EFER_NXE)) + enmGuestMode = PGMMODE_AMD64; + else + enmGuestMode = PGMMODE_AMD64_NX; + } + } + else if (!(cr0 & X86_CR0_PE)) + enmGuestMode = PGMMODE_REAL; + else + enmGuestMode = PGMMODE_PROTECTED; + + /* + * Did it change? + */ + if ( !fForce + && pVCpu->pgm.s.enmGuestMode == enmGuestMode) + return VINF_SUCCESS; + + /* Flush the TLB */ + PGM_INVL_VCPU_TLBS(pVCpu); + return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode, fForce); +} + + +/** + * Converts a PGMMODE value to a PGM_TYPE_* \#define. + * + * @returns PGM_TYPE_*. + * @param pgmMode The mode value to convert. + */ +DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode) +{ + switch (pgmMode) + { + case PGMMODE_REAL: return PGM_TYPE_REAL; + case PGMMODE_PROTECTED: return PGM_TYPE_PROT; + case PGMMODE_32_BIT: return PGM_TYPE_32BIT; + case PGMMODE_PAE: + case PGMMODE_PAE_NX: return PGM_TYPE_PAE; + case PGMMODE_AMD64: + case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64; + case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT; + case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE; + case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64; + case PGMMODE_EPT: return PGM_TYPE_EPT; + case PGMMODE_NONE: return PGM_TYPE_NONE; + default: + AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode)); + } +} + + +/** + * Calculates the shadow paging mode. + * + * @returns The shadow paging mode. + * @param pVM The cross context VM structure. + * @param enmGuestMode The guest mode. + * @param enmHostMode The host mode. + * @param enmShadowMode The current shadow mode. + */ +static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode) +{ + switch (enmGuestMode) + { + case PGMMODE_REAL: + case PGMMODE_PROTECTED: + switch (enmHostMode) + { + case SUPPAGINGMODE_32_BIT: + case SUPPAGINGMODE_32_BIT_GLOBAL: + enmShadowMode = PGMMODE_32_BIT; + break; + + case SUPPAGINGMODE_PAE: + case SUPPAGINGMODE_PAE_NX: + case SUPPAGINGMODE_PAE_GLOBAL: + case SUPPAGINGMODE_PAE_GLOBAL_NX: + enmShadowMode = PGMMODE_PAE; + break; + + case SUPPAGINGMODE_AMD64: + case SUPPAGINGMODE_AMD64_GLOBAL: + case SUPPAGINGMODE_AMD64_NX: + case SUPPAGINGMODE_AMD64_GLOBAL_NX: + enmShadowMode = PGMMODE_PAE; + break; + + default: + AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID); + } + break; + + case PGMMODE_32_BIT: + switch (enmHostMode) + { + case SUPPAGINGMODE_32_BIT: + case SUPPAGINGMODE_32_BIT_GLOBAL: + enmShadowMode = PGMMODE_32_BIT; + break; + + case SUPPAGINGMODE_PAE: + case SUPPAGINGMODE_PAE_NX: + case SUPPAGINGMODE_PAE_GLOBAL: + case SUPPAGINGMODE_PAE_GLOBAL_NX: + enmShadowMode = PGMMODE_PAE; + break; + + case SUPPAGINGMODE_AMD64: + case SUPPAGINGMODE_AMD64_GLOBAL: + case SUPPAGINGMODE_AMD64_NX: + case SUPPAGINGMODE_AMD64_GLOBAL_NX: + enmShadowMode = PGMMODE_PAE; + break; + + default: + AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID); + } + break; + + case PGMMODE_PAE: + case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */ + switch (enmHostMode) + { + case SUPPAGINGMODE_32_BIT: + case SUPPAGINGMODE_32_BIT_GLOBAL: + enmShadowMode = PGMMODE_PAE; + break; + + case SUPPAGINGMODE_PAE: + case SUPPAGINGMODE_PAE_NX: + case SUPPAGINGMODE_PAE_GLOBAL: + case SUPPAGINGMODE_PAE_GLOBAL_NX: + enmShadowMode = PGMMODE_PAE; + break; + + case SUPPAGINGMODE_AMD64: + case SUPPAGINGMODE_AMD64_GLOBAL: + case SUPPAGINGMODE_AMD64_NX: + case SUPPAGINGMODE_AMD64_GLOBAL_NX: + enmShadowMode = PGMMODE_PAE; + break; + + default: + AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID); + } + break; + + case PGMMODE_AMD64: + case PGMMODE_AMD64_NX: + switch (enmHostMode) + { + case SUPPAGINGMODE_32_BIT: + case SUPPAGINGMODE_32_BIT_GLOBAL: + enmShadowMode = PGMMODE_AMD64; + break; + + case SUPPAGINGMODE_PAE: + case SUPPAGINGMODE_PAE_NX: + case SUPPAGINGMODE_PAE_GLOBAL: + case SUPPAGINGMODE_PAE_GLOBAL_NX: + enmShadowMode = PGMMODE_AMD64; + break; + + case SUPPAGINGMODE_AMD64: + case SUPPAGINGMODE_AMD64_GLOBAL: + case SUPPAGINGMODE_AMD64_NX: + case SUPPAGINGMODE_AMD64_GLOBAL_NX: + enmShadowMode = PGMMODE_AMD64; + break; + + default: + AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID); + } + break; + + default: + AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID); + } + + /* + * Override the shadow mode when NEM, IEM or nested paging is active. + */ + if (!VM_IS_HM_ENABLED(pVM)) + { + Assert(VM_IS_NEM_ENABLED(pVM) || VM_IS_EXEC_ENGINE_IEM(pVM)); + pVM->pgm.s.fNestedPaging = true; + enmShadowMode = PGMMODE_NONE; + } + else + { + bool fNestedPaging = HMIsNestedPagingActive(pVM); + pVM->pgm.s.fNestedPaging = fNestedPaging; + if (fNestedPaging) + { + if (HMIsVmxActive(pVM)) + enmShadowMode = PGMMODE_EPT; + else + { + /* The nested SVM paging depends on the host one. */ + Assert(HMIsSvmActive(pVM)); + if ( enmGuestMode == PGMMODE_AMD64 + || enmGuestMode == PGMMODE_AMD64_NX) + enmShadowMode = PGMMODE_NESTED_AMD64; + else + switch (pVM->pgm.s.enmHostMode) + { + case SUPPAGINGMODE_32_BIT: + case SUPPAGINGMODE_32_BIT_GLOBAL: + enmShadowMode = PGMMODE_NESTED_32BIT; + break; + + case SUPPAGINGMODE_PAE: + case SUPPAGINGMODE_PAE_GLOBAL: + case SUPPAGINGMODE_PAE_NX: + case SUPPAGINGMODE_PAE_GLOBAL_NX: + enmShadowMode = PGMMODE_NESTED_PAE; + break; + + case SUPPAGINGMODE_AMD64: + case SUPPAGINGMODE_AMD64_GLOBAL: + case SUPPAGINGMODE_AMD64_NX: + case SUPPAGINGMODE_AMD64_GLOBAL_NX: + enmShadowMode = PGMMODE_NESTED_AMD64; + break; + + default: + AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID); + } + } + } +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + else + { + /* Nested paging is a requirement for nested VT-x. */ + AssertLogRelMsgReturn(enmGuestMode != PGMMODE_EPT, ("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID); + } +#endif + } + + return enmShadowMode; +} + + +/** + * Performs the actual mode change. + * This is called by PGMChangeMode and pgmR3InitPaging(). + * + * @returns VBox status code. May suspend or power off the VM on error, but this + * will trigger using FFs and not informational status codes. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param enmGuestMode The new guest mode. This is assumed to be different from + * the current mode. + * @param fForce Whether to force a shadow paging mode change. + */ +VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode, bool fForce) +{ + Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode))); + STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges); + + /* + * Calc the shadow mode and switcher. + */ + PGMMODE const enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode); + bool const fShadowModeChanged = enmShadowMode != pVCpu->pgm.s.enmShadowMode || fForce; + + /* + * Exit old mode(s). + */ + /* shadow */ + if (fShadowModeChanged) + { + LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode))); + uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData; + if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData) + && g_aPgmShadowModeData[idxOldShw].pfnExit) + { + int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu); + AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc); + } + } + else + LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode))); + + /* guest */ + uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData; + if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData) + && g_aPgmGuestModeData[idxOldGst].pfnExit) + { + int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu); + AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc); + } + pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS; + pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS; + pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS; + Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped)); + + /* + * Change the paging mode data indexes. + */ + uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode); + AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE); +#ifdef IN_RING3 + AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE); +#endif + + uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode); + AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE); +#ifdef IN_RING3 + AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE); +#endif + + uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst; + AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE); + AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE); +#ifdef VBOX_STRICT + AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE); +#endif + + /* + * Determine SLAT mode -before- entering the new shadow mode! + */ + pVCpu->pgm.s.enmGuestSlatMode = !CPUMIsGuestVmxEptPagingEnabled(pVCpu) ? PGMSLAT_DIRECT : PGMSLAT_EPT; + + /* + * Enter new shadow mode (if changed). + */ + if (fShadowModeChanged) + { + pVCpu->pgm.s.enmShadowMode = enmShadowMode; + int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu); + AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc), + pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc); + } + + /* + * Always flag the necessary updates + */ + VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); + + /* + * Enter the new guest and shadow+guest modes. + */ + /* Calc the new CR3 value. */ + RTGCPHYS GCPhysCR3; + switch (enmGuestMode) + { + case PGMMODE_REAL: + case PGMMODE_PROTECTED: + GCPhysCR3 = NIL_RTGCPHYS; + break; + + case PGMMODE_32_BIT: + GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK; + break; + + case PGMMODE_PAE_NX: + case PGMMODE_PAE: + if (!pVM->cpum.ro.GuestFeatures.fPae) +#ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */ + return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode", + N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)")); +#else + AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE); + +#endif + GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK; + break; + +#ifdef VBOX_WITH_64_BITS_GUESTS + case PGMMODE_AMD64_NX: + case PGMMODE_AMD64: + GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK; + break; +#endif + default: + AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE); + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + /* + * If a nested-guest is using EPT paging: + * - Update the second-level address translation (SLAT) mode. + * - Indicate that the CR3 is nested-guest physical address. + */ + if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) + { + if (PGMMODE_WITH_PAGING(enmGuestMode)) + { + /* + * Translate CR3 to its guest-physical address. + * We don't use pgmGstSlatTranslateCr3() here as we want to update GCPhysNstGstCR3 -after- + * switching modes to keep it consistent with how GCPhysCR3 is updated. + */ + PGMPTWALK Walk; + PGMPTWALKGST GstWalk; + int const rc = pgmGstSlatWalk(pVCpu, GCPhysCR3, false /* fIsLinearAddrValid */, 0 /* GCPtrNested */, &Walk, + &GstWalk); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + /* + * SLAT failed but we avoid reporting this to the caller because the caller + * is not supposed to fail. The only time the caller needs to indicate a + * failure to software is when PAE paging is used by the nested-guest, but + * we handle the PAE case separately (e.g., see VMX transition in IEM). + * In all other cases, the failure will be indicated when CR3 tries to be + * translated on the next linear-address memory access. + * See Intel spec. 27.2.1 "EPT Overview". + */ + Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc)); + + /* Trying to coax PGM to succeed for the time being... */ + Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS); + pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3; + pVCpu->pgm.s.enmGuestMode = enmGuestMode; + HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode); + return VINF_SUCCESS; + } + pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3; + GCPhysCR3 = Walk.GCPhys & X86_CR3_EPT_PAGE_MASK; + } + } + else + Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS); +#endif + + /* + * Enter the new guest mode. + */ + pVCpu->pgm.s.enmGuestMode = enmGuestMode; + int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3); + int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3); + + /* Set the new guest CR3 (and nested-guest CR3). */ + pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3; + + /* status codes. */ + AssertRC(rc); + AssertRC(rc2); + if (RT_SUCCESS(rc)) + { + rc = rc2; + if (RT_SUCCESS(rc)) /* no informational status codes. */ + rc = VINF_SUCCESS; + } + + /* + * Notify HM. + */ + HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode); + return rc; +} + + +/** + * Called by CPUM or REM when CR0.WP changes to 1. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread EMT + */ +VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu) +{ + /* + * Netware WP0+RO+US hack cleanup when WP0 -> WP1. + * + * Use the counter to judge whether there might be pool pages with active + * hacks in them. If there are, we will be running the risk of messing up + * the guest by allowing it to write to read-only pages. Thus, we have to + * clear the page pool ASAP if there is the slightest chance. + */ + if (pVCpu->pgm.s.cNetwareWp0Hacks > 0) + { + Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1); + + Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks)); + pVCpu->pgm.s.cNetwareWp0Hacks = 0; + pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL; + VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); + } +} + + +/** + * Gets the current guest paging mode. + * + * If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode(). + * + * @returns The current paging mode. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu) +{ + return pVCpu->pgm.s.enmGuestMode; +} + + +/** + * Gets the current shadow paging mode. + * + * @returns The current paging mode. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu) +{ + return pVCpu->pgm.s.enmShadowMode; +} + + +/** + * Gets the current host paging mode. + * + * @returns The current paging mode. + * @param pVM The cross context VM structure. + */ +VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM) +{ + switch (pVM->pgm.s.enmHostMode) + { + case SUPPAGINGMODE_32_BIT: + case SUPPAGINGMODE_32_BIT_GLOBAL: + return PGMMODE_32_BIT; + + case SUPPAGINGMODE_PAE: + case SUPPAGINGMODE_PAE_GLOBAL: + return PGMMODE_PAE; + + case SUPPAGINGMODE_PAE_NX: + case SUPPAGINGMODE_PAE_GLOBAL_NX: + return PGMMODE_PAE_NX; + + case SUPPAGINGMODE_AMD64: + case SUPPAGINGMODE_AMD64_GLOBAL: + return PGMMODE_AMD64; + + case SUPPAGINGMODE_AMD64_NX: + case SUPPAGINGMODE_AMD64_GLOBAL_NX: + return PGMMODE_AMD64_NX; + + default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break; + } + + return PGMMODE_INVALID; +} + + +/** + * Get mode name. + * + * @returns read-only name string. + * @param enmMode The mode which name is desired. + */ +VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode) +{ + switch (enmMode) + { + case PGMMODE_REAL: return "Real"; + case PGMMODE_PROTECTED: return "Protected"; + case PGMMODE_32_BIT: return "32-bit"; + case PGMMODE_PAE: return "PAE"; + case PGMMODE_PAE_NX: return "PAE+NX"; + case PGMMODE_AMD64: return "AMD64"; + case PGMMODE_AMD64_NX: return "AMD64+NX"; + case PGMMODE_NESTED_32BIT: return "Nested-32"; + case PGMMODE_NESTED_PAE: return "Nested-PAE"; + case PGMMODE_NESTED_AMD64: return "Nested-AMD64"; + case PGMMODE_EPT: return "EPT"; + case PGMMODE_NONE: return "None"; + default: return "unknown mode value"; + } +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** + * Gets the SLAT mode name. + * + * @returns The read-only SLAT mode descriptive string. + * @param enmSlatMode The SLAT mode value. + */ +VMM_INT_DECL(const char *) PGMGetSlatModeName(PGMSLAT enmSlatMode) +{ + switch (enmSlatMode) + { + case PGMSLAT_DIRECT: return "Direct"; + case PGMSLAT_EPT: return "EPT"; + case PGMSLAT_32BIT: return "32-bit"; + case PGMSLAT_PAE: return "PAE"; + case PGMSLAT_AMD64: return "AMD64"; + default: return "Unknown"; + } +} +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ + + +/** + * Gets the physical address represented in the guest CR3 as PGM sees it. + * + * This is mainly for logging and debugging. + * + * @returns PGM's guest CR3 value. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu) +{ + return pVCpu->pgm.s.GCPhysCR3; +} + + + +/** + * Notification from CPUM that the EFER.NXE bit has changed. + * + * @param pVCpu The cross context virtual CPU structure of the CPU for + * which EFER changed. + * @param fNxe The new NXE state. + */ +VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe) +{ +/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */ + Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe)); + + pVCpu->pgm.s.fNoExecuteEnabled = fNxe; + if (fNxe) + { + /*pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A */ + pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX; + pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX; + pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX; + /*pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A */ + pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX; + pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX; + pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX; + pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX; + pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX; + pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX; + + pVCpu->pgm.s.fGst64ShadowedPteMask |= X86_PTE_PAE_NX; + pVCpu->pgm.s.fGst64ShadowedPdeMask |= X86_PDE_PAE_NX; + pVCpu->pgm.s.fGst64ShadowedBigPdeMask |= X86_PDE2M_PAE_NX; + pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask |= X86_PDE2M_PAE_NX; + pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask |= X86_PDPE_LM_NX; + pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask |= X86_PML4E_NX; + } + else + { + /*pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A */ + pVCpu->pgm.s.fGstPaeMbzPteMask |= X86_PTE_PAE_NX; + pVCpu->pgm.s.fGstPaeMbzPdeMask |= X86_PDE_PAE_NX; + pVCpu->pgm.s.fGstPaeMbzBigPdeMask |= X86_PDE2M_PAE_NX; + /*pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A */ + pVCpu->pgm.s.fGstAmd64MbzPteMask |= X86_PTE_PAE_NX; + pVCpu->pgm.s.fGstAmd64MbzPdeMask |= X86_PDE_PAE_NX; + pVCpu->pgm.s.fGstAmd64MbzBigPdeMask |= X86_PDE2M_PAE_NX; + pVCpu->pgm.s.fGstAmd64MbzPdpeMask |= X86_PDPE_LM_NX; + pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask |= X86_PDPE_LM_NX; + pVCpu->pgm.s.fGstAmd64MbzPml4eMask |= X86_PML4E_NX; + + pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX; + pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX; + pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX; + pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX; + pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX; + pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX; + } +} + + +/** + * Check if any pgm pool pages are marked dirty (not monitored) + * + * @returns bool locked/not locked + * @param pVM The cross context VM structure. + */ +VMMDECL(bool) PGMHasDirtyPages(PVM pVM) +{ + return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0; +} + + +/** + * Check if this VCPU currently owns the PGM lock. + * + * @returns bool owner/not owner + * @param pVM The cross context VM structure. + */ +VMMDECL(bool) PGMIsLockOwner(PVMCC pVM) +{ + return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX); +} + + +/** + * Enable or disable large page usage + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param fUseLargePages Use/not use large pages + */ +VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages) +{ + VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE); + + pVM->pgm.s.fUseLargePages = fUseLargePages; + return VINF_SUCCESS; +} + + +/** + * Acquire the PGM lock. + * + * @returns VBox status code + * @param pVM The cross context VM structure. + * @param fVoid Set if the caller cannot handle failure returns. + * @param SRC_POS The source position of the caller (RT_SRC_POS). + */ +#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING) +int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL) +#else +int pgmLock(PVMCC pVM, bool fVoid) +#endif +{ +#if defined(VBOX_STRICT) + int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS); +#else + int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS); +#endif + if (RT_SUCCESS(rc)) + return rc; + if (fVoid) + PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc); + else + AssertRC(rc); + return rc; +} + + +/** + * Release the PGM lock. + * + * @param pVM The cross context VM structure. + */ +void pgmUnlock(PVMCC pVM) +{ + uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks; + pVM->pgm.s.cDeprecatedPageLocks = 0; + int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX); + if (rc == VINF_SEM_NESTED) + pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks; +} + + +#if !defined(IN_R0) || defined(LOG_ENABLED) + +/** Format handler for PGMPAGE. + * @copydoc FNRTSTRFORMATTYPE */ +static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput, + const char *pszType, void const *pvValue, + int cchWidth, int cchPrecision, unsigned fFlags, + void *pvUser) +{ + size_t cch; + PCPGMPAGE pPage = (PCPGMPAGE)pvValue; + if (RT_VALID_PTR(pPage)) + { + char szTmp[64+80]; + + cch = 0; + + /* The single char state stuff. */ + static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' }; + szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)]; + +# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) || cchPrecision == (lvl) || cchPrecision >= (lvl)+10 ) + if (IS_PART_INCLUDED(5)) + { + static const char s_achHandlerStates[4*2] = { '-', 't', 'w', 'a' , '_', 'T', 'W', 'A' }; + szTmp[cch++] = s_achHandlerStates[ PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) + | ((uint8_t)PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) << 2)]; + } + + /* The type. */ + if (IS_PART_INCLUDED(4)) + { + szTmp[cch++] = ':'; + static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" }; + szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0]; + szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1]; + szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2]; + } + + /* The numbers. */ + if (IS_PART_INCLUDED(3)) + { + szTmp[cch++] = ':'; + cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD | RTSTR_F_64BIT); + } + + if (IS_PART_INCLUDED(2)) + { + szTmp[cch++] = ':'; + cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD | RTSTR_F_32BIT); + } + + if (IS_PART_INCLUDED(6)) + { + szTmp[cch++] = ':'; + static const char s_achRefs[4] = { '-', 'U', '!', 'L' }; + szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)]; + cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD | RTSTR_F_16BIT); + } +# undef IS_PART_INCLUDED + + cch = pfnOutput(pvArgOutput, szTmp, cch); + } + else + cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("")); + NOREF(pszType); NOREF(cchWidth); NOREF(pvUser); + return cch; +} + + +/** Format handler for PGMRAMRANGE. + * @copydoc FNRTSTRFORMATTYPE */ +static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput, + const char *pszType, void const *pvValue, + int cchWidth, int cchPrecision, unsigned fFlags, + void *pvUser) +{ + size_t cch; + PGMRAMRANGE const *pRam = (PGMRAMRANGE const *)pvValue; + if (RT_VALID_PTR(pRam)) + { + char szTmp[80]; + cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast); + cch = pfnOutput(pvArgOutput, szTmp, cch); + } + else + cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("")); + NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags); + return cch; +} + +/** Format type andlers to be registered/deregistered. */ +static const struct +{ + char szType[24]; + PFNRTSTRFORMATTYPE pfnHandler; +} g_aPgmFormatTypes[] = +{ + { "pgmpage", pgmFormatTypeHandlerPage }, + { "pgmramrange", pgmFormatTypeHandlerRamRange } +}; + +#endif /* !IN_R0 || LOG_ENABLED */ + +/** + * Registers the global string format types. + * + * This should be called at module load time or in some other manner that ensure + * that it's called exactly one time. + * + * @returns IPRT status code on RTStrFormatTypeRegister failure. + */ +VMMDECL(int) PGMRegisterStringFormatTypes(void) +{ +#if !defined(IN_R0) || defined(LOG_ENABLED) + int rc = VINF_SUCCESS; + unsigned i; + for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++) + { + rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL); +# ifdef IN_RING0 + if (rc == VERR_ALREADY_EXISTS) + { + /* in case of cleanup failure in ring-0 */ + RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType); + rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL); + } +# endif + } + if (RT_FAILURE(rc)) + while (i-- > 0) + RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType); + + return rc; +#else + return VINF_SUCCESS; +#endif +} + + +/** + * Deregisters the global string format types. + * + * This should be called at module unload time or in some other manner that + * ensure that it's called exactly one time. + */ +VMMDECL(void) PGMDeregisterStringFormatTypes(void) +{ +#if !defined(IN_R0) || defined(LOG_ENABLED) + for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++) + RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType); +#endif +} + + +#ifdef VBOX_STRICT +/** + * Asserts that everything related to the guest CR3 is correctly shadowed. + * + * This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(), + * and assert the correctness of the guest CR3 mapping before asserting that the + * shadow page tables is in sync with the guest page tables. + * + * @returns Number of conflicts. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param cr3 The current guest CR3 register value. + * @param cr4 The current guest CR4 register value. + */ +VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4) +{ + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a); + + uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE); + + PGM_LOCK_VOID(pVM); + unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0); + PGM_UNLOCK(pVM); + + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a); + return cErrors; +} +#endif /* VBOX_STRICT */ + + +/** + * Updates PGM's copy of the guest's EPT pointer. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uEptPtr The EPT pointer. + * + * @remarks This can be called as part of VM-entry so we might be in the midst of + * switching to VMX non-root mode. + */ +VMM_INT_DECL(void) PGMSetGuestEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + pVCpu->pgm.s.uEptPtr = uEptPtr; + pVCpu->pgm.s.pGstEptPml4R3 = 0; + pVCpu->pgm.s.pGstEptPml4R0 = 0; + PGM_UNLOCK(pVM); +} + diff --git a/src/VBox/VMM/VMMAll/PGMAllBth.h b/src/VBox/VMM/VMMAll/PGMAllBth.h new file mode 100644 index 00000000..50b7a30e --- /dev/null +++ b/src/VBox/VMM/VMMAll/PGMAllBth.h @@ -0,0 +1,5268 @@ +/* $Id: PGMAllBth.h $ */ +/** @file + * VBox - Page Manager, Shadow+Guest Paging Template - All context code. + * + * @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to + * PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT). + * bird: WTF does this mean these days? Looking at PGMAll.cpp it's + * + * @remarks This file is one big \#ifdef-orgy! + * + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + +#ifdef _MSC_VER +/** @todo we're generating unnecessary code in nested/ept shadow mode and for + * real/prot-guest+RC mode. */ +# pragma warning(disable: 4505) +#endif + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +RT_C_DECLS_BEGIN +PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3); +#ifndef IN_RING3 +PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken); +PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault, + bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken); +# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT +static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, + unsigned iPte, PCPGMPTWALKGST pGstWalkAll); +static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages, + uint32_t uErr, PPGMPTWALKGST pGstWalkAll); +static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll); +# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ +#endif +PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage); +static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr); +static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage); +static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage); +#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) +static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst); +#else +static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst); +#endif +PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr); +PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage); +PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal); +#ifdef VBOX_STRICT +PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0); +#endif +PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3); +PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu); + +#ifdef IN_RING3 +PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta); +#endif +RT_C_DECLS_END + + + + +/* + * Filter out some illegal combinations of guest and shadow paging, so we can + * remove redundant checks inside functions. + */ +#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \ + && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE +# error "Invalid combination; PAE guest implies PAE shadow" +#endif + +#if (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \ + && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 \ + || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE) +# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging." +#endif + +#if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \ + && !( PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE \ + || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE) +# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging." +#endif + +#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \ + || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT) +# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa" +#endif + + +/** + * Enters the shadow+guest mode. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhysCR3 The physical address from the CR3 register. + */ +PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3) +{ + /* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches; + * Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables. + */ +#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \ + || PGM_SHW_TYPE == PGM_TYPE_PAE \ + || PGM_SHW_TYPE == PGM_TYPE_AMD64) \ + && ( PGM_GST_TYPE == PGM_TYPE_REAL \ + || PGM_GST_TYPE == PGM_TYPE_PROT)) + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + Assert(!pVM->pgm.s.fNestedPaging); + + PGM_LOCK_VOID(pVM); + /* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes), + * but any calls to GC need a proper shadow page setup as well. + */ + /* Free the previous root mapping if still active. */ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3); + if (pOldShwPageCR3) + { + Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE); + + /* Mark the page as unlocked; allow flushing again. */ + pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); + + pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX); + pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR; + pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR; + } + + /* construct a fake address. */ + GCPhysCR3 = RT_BIT_64(63); + PPGMPOOLPAGE pNewShwPageCR3; + int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu), + NIL_PGMPOOL_IDX, UINT32_MAX, false /*fLockPage*/, + &pNewShwPageCR3); + AssertRCReturn(rc, rc); + + pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3); + pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3); + + /* Mark the page as locked; disallow flushing. */ + pgmPoolLockPage(pPool, pNewShwPageCR3); + + /* Set the current hypervisor CR3. */ + CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu)); + + PGM_UNLOCK(pVM); + return rc; +#else + NOREF(pVCpu); NOREF(GCPhysCR3); + return VINF_SUCCESS; +#endif +} + + +#ifndef IN_RING3 + +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) +/** + * Deal with a guest page fault. + * + * @returns Strict VBox status code. + * @retval VINF_EM_RAW_GUEST_TRAP + * @retval VINF_EM_RAW_EMULATE_INSTR + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pWalk The guest page table walk result. + * @param uErr The error code. + */ +PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, RTGCUINT uErr) +{ + /* + * Calc the error code for the guest trap. + */ + uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu) + ? uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID) + : uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US); + if ( pWalk->fRsvdError + || pWalk->fBadPhysAddr) + { + uNewErr |= X86_TRAP_PF_RSVD | X86_TRAP_PF_P; + Assert(!pWalk->fNotPresent); + } + else if (!pWalk->fNotPresent) + uNewErr |= X86_TRAP_PF_P; + TRPMSetErrorCode(pVCpu, uNewErr); + + LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pWalk->GCPtr, uErr, pWalk->uLevel)); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; }); + return VINF_EM_RAW_GUEST_TRAP; +} +# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */ + + +#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE +/** + * Deal with a guest page fault. + * + * The caller has taken the PGM lock. + * + * @returns Strict VBox status code. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uErr The error code. + * @param pCtx Pointer to the register context for the CPU. + * @param pvFault The fault address. + * @param pPage The guest page at @a pvFault. + * @param pWalk The guest page table walk result. + * @param pGstWalk The guest paging-mode specific walk information. + * @param pfLockTaken PGM lock taken here or not (out). This is true + * when we're called. + */ +static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, + RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING) + , PPGMPTWALK pWalk + , PGSTPTWALK pGstWalk +# endif + ) +{ +# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A }; +# endif + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VBOXSTRICTRC rcStrict; + + if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage)) + { + /* + * Physical page access handler. + */ +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + const RTGCPHYS GCPhysFault = pWalk->GCPhys; +# else + const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault); +# endif + PPGMPHYSHANDLER pCur; + rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysFault, &pCur); + if (RT_SUCCESS(rcStrict)) + { + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur); + +# ifdef PGM_SYNC_N_PAGES + /* + * If the region is write protected and we got a page not present fault, then sync + * the pages. If the fault was caused by a read, then restart the instruction. + * In case of write access continue to the GC write handler. + * + * ASSUMES that there is only one handler per page or that they have similar write properties. + */ + if ( !(uErr & X86_TRAP_PF_P) + && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE) + { +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr); +# else + rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr); +# endif + if ( RT_FAILURE(rcStrict) + || !(uErr & X86_TRAP_PF_RW) + || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE) + { + AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; }); + return rcStrict; + } + } +# endif +# ifdef PGM_WITH_MMIO_OPTIMIZATIONS + /* + * If the access was not thru a #PF(RSVD|...) resync the page. + */ + if ( !(uErr & X86_TRAP_PF_RSVD) + && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + && (pWalk->fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) + == PGM_PTATTRS_W_MASK /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */ +# endif + ) + { +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr); +# else + rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr); +# endif + if ( RT_FAILURE(rcStrict) + || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE) + { + AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; }); + return rcStrict; + } + } +# endif + + AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE + || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)), + ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n", + pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind)); + if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite); + else + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll); + if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt); + } + + if (pCurType->pfnPfHandler) + { + STAM_PROFILE_START(&pCur->Stat, h); + + if (pCurType->fKeepPgmLock) + { + rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault, + !pCurType->fRing0DevInsIdx ? pCur->uUser + : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser)); + + STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */ + } + else + { + uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser + : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser); + PGM_UNLOCK(pVM); + *pfLockTaken = false; + + rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault, uUser); + + STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */ + } + } + else + rcStrict = VINF_EM_RAW_EMULATE_INSTR; + + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; }); + return rcStrict; + } + AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict); + } + + /* + * There is a handled area of the page, but this fault doesn't belong to it. + * We must emulate the instruction. + * + * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler + * we first check if this was a page-not-present fault for a page with only + * write access handlers. Restart the instruction if it wasn't a write access. + */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersUnhandled); + + if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) + && !(uErr & X86_TRAP_PF_P)) + { +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr); +# else + rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr); +# endif + if ( RT_FAILURE(rcStrict) + || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE + || !(uErr & X86_TRAP_PF_RW)) + { + AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; }); + return rcStrict; + } + } + + /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06 + * It's writing to an unhandled part of the LDT page several million times. + */ + rcStrict = PGMInterpretInstruction(pVCpu, pvFault); + LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage)); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndUnhandled; }); + return rcStrict; +} /* if any kind of handler */ +# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/ + + +/** + * \#PF Handler for raw-mode guest execution. + * + * @returns VBox status code (appropriate for trap handling and GC return). + * + * @param pVCpu The cross context virtual CPU structure. + * @param uErr The trap error code. + * @param pCtx Pointer to the register context for the CPU. + * @param pvFault The fault address. + * @param pfLockTaken PGM lock taken here or not (out) + */ +PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM); + + *pfLockTaken = false; + +# if ( PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT \ + || PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64) \ + && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \ + && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \ + && PGM_SHW_TYPE != PGM_TYPE_NONE + int rc; + +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + /* + * Walk the guest page translation tables and check if it's a guest fault. + */ + PGMPTWALK Walk; + GSTPTWALK GstWalk; + rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &Walk, &GstWalk); + if (RT_FAILURE_NP(rc)) + return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr)); + + /* assert some GstWalk sanity. */ +# if PGM_GST_TYPE == PGM_TYPE_AMD64 + /*AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. */ +# endif +# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE + /*AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto */ +# endif + /*AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto */ + /*AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto */ + Assert(Walk.fSucceeded); + Assert(Walk.fEffective & PGM_PTATTRS_R_MASK); + + if (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID)) + { + if ( ( (uErr & X86_TRAP_PF_RW) + && !(Walk.fEffective & PGM_PTATTRS_W_MASK) + && ( (uErr & X86_TRAP_PF_US) + || CPUMIsGuestR0WriteProtEnabled(pVCpu)) ) + || ((uErr & X86_TRAP_PF_US) && !(Walk.fEffective & PGM_PTATTRS_US_MASK)) + || ((uErr & X86_TRAP_PF_ID) && (Walk.fEffective & PGM_PTATTRS_NX_MASK)) + ) + return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr)); + } + + /* Take the big lock now before we update flags. */ + *pfLockTaken = true; + PGM_LOCK_VOID(pVM); + + /* + * Set the accessed and dirty flags. + */ + /** @todo Should probably use cmpxchg logic here as we're potentially racing + * other CPUs in SMP configs. (the lock isn't enough, since we take it + * after walking and the page tables could be stale already) */ +# if PGM_GST_TYPE == PGM_TYPE_AMD64 + if (!(GstWalk.Pml4e.u & X86_PML4E_A)) + { + GstWalk.Pml4e.u |= X86_PML4E_A; + GST_ATOMIC_OR(&GstWalk.pPml4e->u, X86_PML4E_A); + } + if (!(GstWalk.Pdpe.u & X86_PDPE_A)) + { + GstWalk.Pdpe.u |= X86_PDPE_A; + GST_ATOMIC_OR(&GstWalk.pPdpe->u, X86_PDPE_A); + } +# endif + if (Walk.fBigPage) + { + Assert(GstWalk.Pde.u & X86_PDE_PS); + if (uErr & X86_TRAP_PF_RW) + { + if ((GstWalk.Pde.u & (X86_PDE4M_A | X86_PDE4M_D)) != (X86_PDE4M_A | X86_PDE4M_D)) + { + GstWalk.Pde.u |= X86_PDE4M_A | X86_PDE4M_D; + GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A | X86_PDE4M_D); + } + } + else + { + if (!(GstWalk.Pde.u & X86_PDE4M_A)) + { + GstWalk.Pde.u |= X86_PDE4M_A; + GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A); + } + } + } + else + { + Assert(!(GstWalk.Pde.u & X86_PDE_PS)); + if (!(GstWalk.Pde.u & X86_PDE_A)) + { + GstWalk.Pde.u |= X86_PDE_A; + GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE_A); + } + + if (uErr & X86_TRAP_PF_RW) + { +# ifdef VBOX_WITH_STATISTICS + if (GstWalk.Pte.u & X86_PTE_D) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageAlreadyDirty)); + else + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtiedPage)); +# endif + if ((GstWalk.Pte.u & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D)) + { + GstWalk.Pte.u |= X86_PTE_A | X86_PTE_D; + GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A | X86_PTE_D); + } + } + else + { + if (!(GstWalk.Pte.u & X86_PTE_A)) + { + GstWalk.Pte.u |= X86_PTE_A; + GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A); + } + } + Assert(GstWalk.Pte.u == GstWalk.pPte->u); + } +#if 0 + /* Disabling this since it's not reliable for SMP, see @bugref{10092#c22}. */ + AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u, + ("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u)); +#endif + +# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */ + GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A}; /** @todo eliminate this */ + + /* Take the big lock now. */ + *pfLockTaken = true; + PGM_LOCK_VOID(pVM); +# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */ + +# ifdef PGM_WITH_MMIO_OPTIMIZATIONS + /* + * If it is a reserved bit fault we know that it is an MMIO (access + * handler) related fault and can skip some 200 lines of code. + */ + if (uErr & X86_TRAP_PF_RSVD) + { + Assert(uErr & X86_TRAP_PF_P); + PPGMPAGE pPage; +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + rc = pgmPhysGetPageEx(pVM, Walk.GCPhys, &pPage); + if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) + return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, + pfLockTaken, &Walk, &GstWalk)); + rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr); +# else + rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage); + if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) + return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken)); + rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr); +# endif + AssertRC(rc); + PGM_INVL_PG(pVCpu, pvFault); + return rc; /* Restart with the corrected entry. */ + } +# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */ + + /* + * Fetch the guest PDE, PDPE and PML4E. + */ +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + const unsigned iPDDst = pvFault >> SHW_PD_SHIFT; + PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu); + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE + const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */ + PX86PDPAE pPDDst; +# if PGM_GST_TYPE == PGM_TYPE_PAE + rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst); +# else + rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */ +# endif + AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS); + +# elif PGM_SHW_TYPE == PGM_TYPE_AMD64 + const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK); + PX86PDPAE pPDDst; +# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */ + rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A, + X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A, &pPDDst); +# else + rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst); +# endif + AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS); + +# elif PGM_SHW_TYPE == PGM_TYPE_EPT + const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK); + PEPTPD pPDDst; + rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst); + AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS); +# endif + Assert(pPDDst); + +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + /* + * Dirty page handling. + * + * If we successfully correct the write protection fault due to dirty bit + * tracking, then return immediately. + */ + if (uErr & X86_TRAP_PF_RW) /* write fault? */ + { + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a); + rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault); + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a); + if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT) + { + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 + = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT + ? &pVCpu->pgm.s.Stats.StatRZTrap0eTime2DirtyAndAccessed + : &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; }); + Log8(("Trap0eHandler: returns VINF_SUCCESS\n")); + return VINF_SUCCESS; + } +#ifdef DEBUG_bird + AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests. + AssertMsg(Walk.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto. +#endif + } + +# if 0 /* rarely useful; leave for debugging. */ + STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]); +# endif +# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */ + + /* + * A common case is the not-present error caused by lazy page table syncing. + * + * It is IMPORTANT that we weed out any access to non-present shadow PDEs + * here so we can safely assume that the shadow PT is present when calling + * SyncPage later. + * + * On failure, we ASSUME that SyncPT is out of memory or detected some kind + * of mapping conflict and defer to SyncCR3 in R3. + * (Again, we do NOT support access handlers for non-present guest pages.) + * + */ +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + Assert(GstWalk.Pde.u & X86_PDE_P); +# endif + if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */ + && !SHW_PDE_IS_P(pPDDst->a[iPDDst])) + { + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; }); +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u)); + rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault); +# else + LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault)); + rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault); +# endif + if (RT_SUCCESS(rc)) + return rc; + Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc)); + VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */ + return VINF_PGM_SYNC_CR3; + } + + /* + * Check if this fault address is flagged for special treatment, + * which means we'll have to figure out the physical address and + * check flags associated with it. + * + * ASSUME that we can limit any special access handling to pages + * in page tables which the guest believes to be present. + */ +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + RTGCPHYS GCPhys = Walk.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; +# else + RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK); +# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */ + PPGMPAGE pPage; + rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage); + if (RT_FAILURE(rc)) + { + /* + * When the guest accesses invalid physical memory (e.g. probing + * of RAM or accessing a remapped MMIO range), then we'll fall + * back to the recompiler to emulate the instruction. + */ + LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc)); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; }); + return VINF_EM_RAW_EMULATE_INSTR; + } + + /* + * Any handlers for this page? + */ + if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage)) +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken, + &Walk, &GstWalk)); +# else + return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken)); +# endif + + /* + * We are here only if page is present in Guest page tables and + * trap is not handled by our handlers. + * + * Check it for page out-of-sync situation. + */ + if (!(uErr & X86_TRAP_PF_P)) + { + /* + * Page is not present in our page tables. Try to sync it! + */ + if (uErr & X86_TRAP_PF_US) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser)); + else /* supervisor */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor)); + + if (PGM_PAGE_IS_BALLOONED(pPage)) + { + /* Emulate reads from ballooned pages as they are not present in + our shadow page tables. (Required for e.g. Solaris guests; soft + ecc, random nr generator.) */ + rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault)); + LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage)); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncBallloon)); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Ballooned; }); + return rc; + } + +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr); +# else + rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr); +# endif + if (RT_SUCCESS(rc)) + { + /* The page was successfully synced, return to the guest. */ + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; }); + return VINF_SUCCESS; + } + } + else /* uErr & X86_TRAP_PF_P: */ + { + /* + * Write protected pages are made writable when the guest makes the + * first write to it. This happens for pages that are shared, write + * monitored or not yet allocated. + * + * We may also end up here when CR0.WP=0 in the guest. + * + * Also, a side effect of not flushing global PDEs are out of sync + * pages due to physical monitored regions, that are no longer valid. + * Assume for now it only applies to the read/write flag. + */ + if (uErr & X86_TRAP_PF_RW) + { + /* + * Check if it is a read-only page. + */ + if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED) + { + Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr)); + Assert(!PGM_PAGE_IS_ZERO(pPage)); + AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys)); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; }); + + rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys); + if (rc != VINF_SUCCESS) + { + AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc)); + return rc; + } + if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))) + return VINF_EM_NO_MEMORY; + } + +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + /* + * Check to see if we need to emulate the instruction if CR0.WP=0. + */ + if ( !(Walk.fEffective & PGM_PTATTRS_W_MASK) + && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG + && CPUMGetGuestCPL(pVCpu) < 3) + { + Assert((uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P)); + + /* + * The Netware WP0+RO+US hack. + * + * Netware sometimes(/always?) runs with WP0. It has been observed doing + * excessive write accesses to pages which are mapped with US=1 and RW=0 + * while WP=0. This causes a lot of exits and extremely slow execution. + * To avoid trapping and emulating every write here, we change the shadow + * page table entry to map it as US=0 and RW=1 until user mode tries to + * access it again (see further below). We count these shadow page table + * changes so we can avoid having to clear the page pool every time the WP + * bit changes to 1 (see PGMCr0WpEnabled()). + */ +# if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) && 1 + if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK + && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW)) + && pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */) + { + Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, Walk.fBigPage)); + rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, Walk.fBigPage, PGM_MK_PG_IS_WRITE_FAULT); + if (rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3) + { + PGM_INVL_PG(pVCpu, pvFault); + pVCpu->pgm.s.cNetwareWp0Hacks++; + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsHack; }); + return rc; + } + AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc)); + Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc)); + } +# endif + + /* Interpret the access. */ + rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault)); + Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), Walk.fBigPage, !!(Walk.fEffective & PGM_PTATTRS_US_MASK))); + if (RT_SUCCESS(rc)) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulInRZ); + else + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulToR3); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2WPEmulation; }); + return rc; + } +# endif + /// @todo count the above case; else + if (uErr & X86_TRAP_PF_US) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUserWrite)); + else /* supervisor */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite)); + + /* + * Sync the page. + * + * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the + * page is not present, which is not true in this case. + */ +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr); +# else + rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr); +# endif + if (RT_SUCCESS(rc)) + { + /* + * Page was successfully synced, return to guest but invalidate + * the TLB first as the page is very likely to be in it. + */ +# if PGM_SHW_TYPE == PGM_TYPE_EPT + HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault); +# else + PGM_INVL_PG(pVCpu, pvFault); +# endif +# ifdef VBOX_STRICT + PGMPTWALK GstPageWalk; + GstPageWalk.GCPhys = RTGCPHYS_MAX; + if (!pVM->pgm.s.fNestedPaging) + { + rc = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk); + AssertMsg(RT_SUCCESS(rc) && ((GstPageWalk.fEffective & X86_PTE_RW) || ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, GstPageWalk.fEffective)); + LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GstPageWalk.GCPhys, GstPageWalk.fEffective)); + } +# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */ + uint64_t fPageShw = 0; + rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL); + AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) || pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */, + ("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GstPageWalk.GCPhys, fPageGst, pvFault)); +# endif +# endif /* VBOX_STRICT */ + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; }); + return VINF_SUCCESS; + } + } +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + /* + * Check for Netware WP0+RO+US hack from above and undo it when user + * mode accesses the page again. + */ + else if ( (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK + && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW)) + && pVCpu->pgm.s.cNetwareWp0Hacks > 0 + && (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG + && CPUMGetGuestCPL(pVCpu) == 3 + && pVM->cCpus == 1 + ) + { + Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr)); + rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr); + if (RT_SUCCESS(rc)) + { + PGM_INVL_PG(pVCpu, pvFault); + pVCpu->pgm.s.cNetwareWp0Hacks--; + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsUnhack; }); + return VINF_SUCCESS; + } + } +# endif /* PGM_WITH_PAGING */ + + /** @todo else: why are we here? */ + +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT) + /* + * Check for VMM page flags vs. Guest page flags consistency. + * Currently only for debug purposes. + */ + if (RT_SUCCESS(rc)) + { + /* Get guest page flags. */ + PGMPTWALK GstPageWalk; + int rc2 = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk); + if (RT_SUCCESS(rc2)) + { + uint64_t fPageShw = 0; + rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL); + +#if 0 + /* + * Compare page flags. + * Note: we have AVL, A, D bits desynced. + */ + AssertMsg( (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) + == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) + || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 + && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) + == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) + && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW + && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US), + ("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n", + pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc)); +01:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US) +01:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0 + +01:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US) +01:01:15.625516 00:08:43.268051 Location : +e:\vbox\svn\trunk\srcPage flags mismatch! +pvFault=fffff801b0d7b000 + uErr=11 X86_TRAP_PF_ID | X86_TRAP_PF_P +GCPhys=0000000019b52000 +fPageShw=0 +fPageGst=77b0000000000121 +rc=0 +#endif + + } + else + AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc)); + } + else + AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc)); +# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */ + } + + + /* + * If we get here it is because something failed above, i.e. most like guru + * meditiation time. + */ + LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n", + __PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pCtx->cs.Sel, pCtx->rip)); + return rc; + +# else /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE. */ + NOREF(uErr); NOREF(pCtx); NOREF(pvFault); + AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE)); + return VERR_PGM_NOT_USED_IN_MODE; +# endif +} + + +# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) +/** + * Deals with a nested-guest \#PF fault for a guest-physical page with a handler. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uErr The error code. + * @param pCtx Pointer to the register context for the CPU. + * @param GCPhysNestedFault The nested-guest physical address of the fault. + * @param pPage The guest page at @a GCPhysNestedFault. + * @param GCPhysFault The guest-physical address of the fault. + * @param pGstWalkAll The guest page walk result. + * @param pfLockTaken Where to store whether the PGM is still held when + * this function completes. + * + * @note The caller has taken the PGM lock. + */ +static VBOXSTRICTRC PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, + RTGCPHYS GCPhysNestedFault, PPGMPAGE pPage, + RTGCPHYS GCPhysFault, PPGMPTWALKGST pGstWalkAll, + bool *pfLockTaken) +{ +# if PGM_GST_TYPE == PGM_TYPE_PROT \ + && PGM_SHW_TYPE == PGM_TYPE_EPT + + /** @todo Assert uErr isn't X86_TRAP_PF_RSVD and remove release checks. */ + PGM_A20_ASSERT_MASKED(pVCpu, GCPhysFault); + AssertMsgReturn(PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage), ("%RGp %RGp uErr=%u\n", GCPhysNestedFault, GCPhysFault, uErr), + VERR_PGM_HANDLER_IPE_1); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + RTGCPHYS const GCPhysPage = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + + /* + * Physical page access handler. + */ + PPGMPHYSHANDLER pCur; + VBOXSTRICTRC rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysPage, &pCur); + AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict); + + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur); + Assert(pCurType); + + /* + * If the region is write protected and we got a page not present fault, then sync + * the pages. If the fault was caused by a read, then restart the instruction. + * In case of write access continue to the GC write handler. + */ + if ( !(uErr & X86_TRAP_PF_P) + && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE) + { + Log7Func(("Syncing Monitored: GCPhysNestedPage=%RGp GCPhysPage=%RGp uErr=%#x\n", GCPhysNestedPage, GCPhysPage, uErr)); + rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll); + Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE); + if ( RT_FAILURE(rcStrict) + || !(uErr & X86_TRAP_PF_RW)) + { + AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; }); + return rcStrict; + } + } + else if ( !(uErr & X86_TRAP_PF_RSVD) + && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE) + { + /* + * If the access was NOT through an EPT misconfig (i.e. RSVD), sync the page. + * This can happen for the VMX APIC-access page. + */ + Log7Func(("Syncing MMIO: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage)); + rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll); + Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE); + if (RT_FAILURE(rcStrict)) + { + AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; }); + return rcStrict; + } + } + + AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE + || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)), + ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n", + GCPhysNestedFault, GCPhysFault, pPage, uErr, pCurType->enmKind)); + if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite); + else + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll); + if (uErr & X86_TRAP_PF_RSVD) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt); + } + + if (pCurType->pfnPfHandler) + { + STAM_PROFILE_START(&pCur->Stat, h); + uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser + : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser); + + if (pCurType->fKeepPgmLock) + { + rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser); + STAM_PROFILE_STOP(&pCur->Stat, h); + } + else + { + PGM_UNLOCK(pVM); + *pfLockTaken = false; + rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser); + STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */ + } + } + else + { + AssertMsgFailed(("What's going on here!? Fault falls outside handler range!?\n")); + rcStrict = VINF_EM_RAW_EMULATE_INSTR; + } + + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; }); + return rcStrict; + +# else + RT_NOREF8(pVCpu, uErr, pCtx, GCPhysNestedFault, pPage, GCPhysFault, pGstWalkAll, pfLockTaken); + AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE)); + return VERR_PGM_NOT_USED_IN_MODE; +# endif +} +# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ + + +/** + * Nested \#PF handler for nested-guest hardware-assisted execution using nested + * paging. + * + * @returns VBox status code (appropriate for trap handling and GC return). + * @param pVCpu The cross context virtual CPU structure. + * @param uErr The fault error (X86_TRAP_PF_*). + * @param pCtx Pointer to the register context for the CPU. + * @param GCPhysNestedFault The nested-guest physical address of the fault. + * @param fIsLinearAddrValid Whether translation of a nested-guest linear address + * caused this fault. If @c false, GCPtrNestedFault + * must be 0. + * @param GCPtrNestedFault The nested-guest linear address of this fault. + * @param pWalk The guest page table walk result. + * @param pfLockTaken Where to store whether the PGM lock is still held + * when this function completes. + */ +PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault, + bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken) +{ + *pfLockTaken = false; +# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \ + && PGM_GST_TYPE == PGM_TYPE_PROT \ + && PGM_SHW_TYPE == PGM_TYPE_EPT + + Assert(CPUMIsGuestVmxEptPagingEnabled(pVCpu)); + Assert(PGM_A20_IS_ENABLED(pVCpu)); + + /* We don't support mode-based execute control for EPT yet. */ + Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt); + Assert(!(uErr & X86_TRAP_PF_US)); + + /* Take the big lock now. */ + *pfLockTaken = true; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + + /* + * Walk the guest EPT tables and check if it's an EPT violation or misconfiguration. + */ + if (fIsLinearAddrValid) + Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x GCPtrNestedFault=%RGv\n", + pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr, GCPtrNestedFault)); + else + Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x\n", + pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr)); + PGMPTWALKGST GstWalkAll; + int rc = pgmGstSlatWalk(pVCpu, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk, &GstWalkAll); + if (RT_FAILURE(rc)) + return rc; + + Assert(GstWalkAll.enmType == PGMPTWALKGSTTYPE_EPT); + Assert(pWalk->fSucceeded); + Assert(pWalk->fEffective & (PGM_PTATTRS_EPT_R_MASK | PGM_PTATTRS_EPT_W_MASK | PGM_PTATTRS_EPT_X_SUPER_MASK)); + Assert(pWalk->fIsSlat); + +# ifdef DEBUG_ramshankar + /* Paranoia. */ + Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_R_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_R_MASK)); + Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_W_MASK) == RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_W_MASK)); + Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_NX_MASK) == !RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_X_SUPER_MASK)); +# endif + + /* + * Check page-access permissions. + */ + if ( ((uErr & X86_TRAP_PF_RW) && !(pWalk->fEffective & PGM_PTATTRS_W_MASK)) + || ((uErr & X86_TRAP_PF_ID) && (pWalk->fEffective & PGM_PTATTRS_NX_MASK))) + { + Log7Func(("Permission failed! GCPtrNested=%RGv GCPhysNested=%RGp uErr=%#x fEffective=%#RX64\n", GCPtrNestedFault, + GCPhysNestedFault, uErr, pWalk->fEffective)); + pWalk->fFailed = PGM_WALKFAIL_EPT_VIOLATION; + return VERR_ACCESS_DENIED; + } + + PGM_A20_ASSERT_MASKED(pVCpu, pWalk->GCPhys); + RTGCPHYS const GCPhysPage = pWalk->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + + /* + * If we were called via an EPT misconfig, it should've already resulted in a nested-guest VM-exit. + */ + AssertMsgReturn(!(uErr & X86_TRAP_PF_RSVD), + ("Unexpected EPT misconfig VM-exit. GCPhysPage=%RGp GCPhysNestedPage=%RGp\n", GCPhysPage, GCPhysNestedPage), + VERR_PGM_MAPPING_IPE); + + /* + * Fetch and sync the nested-guest EPT page directory pointer. + */ + PEPTPD pEptPd; + rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL /*ppPdpt*/, &pEptPd, &GstWalkAll); + AssertRCReturn(rc, rc); + Assert(pEptPd); + + /* + * A common case is the not-present error caused by lazy page table syncing. + * + * It is IMPORTANT that we weed out any access to non-present shadow PDEs + * here so we can safely assume that the shadow PT is present when calling + * NestedSyncPage later. + * + * NOTE: It's possible we will be syncing the VMX APIC-access page here. + * In that case, we would sync the page but will NOT go ahead with emulating + * the APIC-access VM-exit through IEM. However, once the page is mapped in + * the shadow tables, subsequent APIC-access VM-exits for the nested-guest + * will be triggered by hardware. Maybe calling the IEM #PF handler can be + * considered as an optimization later. + */ + unsigned const iPde = (GCPhysNestedPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + if ( !(uErr & X86_TRAP_PF_P) + && !(pEptPd->a[iPde].u & EPT_PRESENT_MASK)) + { + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; }); + Log7Func(("NestedSyncPT: Lazy. GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage)); + rc = PGM_BTH_NAME(NestedSyncPT)(pVCpu, GCPhysNestedPage, GCPhysPage, &GstWalkAll); + if (RT_SUCCESS(rc)) + return rc; + AssertMsgFailedReturn(("NestedSyncPT: %RGv failed! rc=%Rrc\n", GCPhysNestedPage, rc), VERR_PGM_MAPPING_IPE); + } + + /* + * Check if this fault address is flagged for special treatment. + * This handles faults on an MMIO or write-monitored page. + * + * If this happens to be the VMX APIC-access page, we don't treat is as MMIO + * but rather sync it further below (as a regular guest page) which lets + * hardware-assisted execution trigger the APIC-access VM-exits of the + * nested-guest directly. + */ + PPGMPAGE pPage; + rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); + AssertRCReturn(rc, rc); + if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage)) + { + Log7Func(("MMIO: Calling NestedTrap0eHandlerDoAccessHandlers for GCPhys %RGp\n", GCPhysPage)); + return VBOXSTRICTRC_TODO(PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, GCPhysNestedFault, + pPage, pWalk->GCPhys, &GstWalkAll, + pfLockTaken)); + } + + /* + * We are here only if page is present in nested-guest page tables but the + * trap is not handled by our handlers. Check for page out-of-sync situation. + */ + if (!(uErr & X86_TRAP_PF_P)) + { + Assert(!PGM_PAGE_IS_BALLOONED(pPage)); + Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor)); + + Log7Func(("SyncPage: Not-Present: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedFault, GCPhysPage)); + rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, PGM_SYNC_NR_PAGES, uErr, &GstWalkAll); + if (RT_SUCCESS(rc)) + { + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; }); + return VINF_SUCCESS; + } + } + else if (uErr & X86_TRAP_PF_RW) + { + /* + * Write protected pages are made writable when the guest makes the + * first write to it. This happens for pages that are shared, write + * monitored or not yet allocated. + * + * We may also end up here when CR0.WP=0 in the guest. + * + * Also, a side effect of not flushing global PDEs are out of sync + * pages due to physical monitored regions, that are no longer valid. + * Assume for now it only applies to the read/write flag. + */ + if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED) + { + /* This is a read-only page. */ + AssertMsgFailed(("Failed\n")); + + Assert(!PGM_PAGE_IS_ZERO(pPage)); + AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhysPage)); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; }); + + Log7Func(("Calling pgmPhysPageMakeWritable for GCPhysPage=%RGp\n", GCPhysPage)); + rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage); + if (rc != VINF_SUCCESS) + { + AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc)); + return rc; + } + if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))) + return VINF_EM_NO_MEMORY; + } + + Assert(!(uErr & X86_TRAP_PF_US)); /* Mode-based execute not supported yet. */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite)); + + /* + * Sync the write-protected page. + * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the + * page is not present, which is not true in this case. + */ + Log7Func(("SyncPage: RW: cs:rip=%04x:%#RX64 GCPhysNestedPage=%RGp uErr=%#RX32 GCPhysPage=%RGp WalkGCPhys=%RGp\n", + pCtx->cs.Sel, pCtx->rip, GCPhysNestedPage, (uint32_t)uErr, GCPhysPage, pWalk->GCPhys)); + rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /* cPages */, uErr, &GstWalkAll); + if (RT_SUCCESS(rc)) + { + HMInvalidatePhysPage(pVM, GCPhysPage); + STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; }); + return VINF_SUCCESS; + } + } + + /* + * If we get here it is because something failed above => guru meditation time? + */ + LogRelMaxFunc(32, ("rc=%Rrc GCPhysNestedFault=%#RGp (%#RGp) uErr=%#RX32 cs:rip=%04x:%08RX64\n", + rc, GCPhysNestedFault, GCPhysPage, (uint32_t)uErr, pCtx->cs.Sel, pCtx->rip)); + return VERR_PGM_MAPPING_IPE; + +# else /* !VBOX_WITH_NESTED_HWVIRT_VMX_EPT || PGM_GST_TYPE != PGM_TYPE_PROT || PGM_SHW_TYPE != PGM_TYPE_EPT */ + RT_NOREF7(pVCpu, uErr, pCtx, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk); + AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE)); + return VERR_PGM_NOT_USED_IN_MODE; +# endif +} + +#endif /* !IN_RING3 */ + + +/** + * Emulation of the invlpg instruction. + * + * + * @returns VBox status code. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtrPage Page to invalidate. + * + * @remark ASSUMES that the guest is updating before invalidating. This order + * isn't required by the CPU, so this is speculative and could cause + * trouble. + * @remark No TLB shootdown is done on any other VCPU as we assume that + * invlpg emulation is the *only* reason for calling this function. + * (The guest has to shoot down TLB entries on other CPUs itself) + * Currently true, but keep in mind! + * + * @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses. + * Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now)) + */ +PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage) +{ +#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \ + && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \ + && PGM_SHW_TYPE != PGM_TYPE_NONE + int rc; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + + PGM_LOCK_ASSERT_OWNER(pVM); + + LogFlow(("InvalidatePage %RGv\n", GCPtrPage)); + + /* + * Get the shadow PD entry and skip out if this PD isn't present. + * (Guessing that it is frequent for a shadow PDE to not be present, do this first.) + */ +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT; + PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage); + + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3); +# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */ + if (!pShwPde) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped)); + return VINF_SUCCESS; + } +# else + Assert(pShwPde); +# endif + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE + const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT; + PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu); + + /* If the shadow PDPE isn't present, then skip the invalidate. */ +# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */ + if (!pPdptDst || !(pPdptDst->a[iPdpt].u & X86_PDPE_P)) +# else + if (!(pPdptDst->a[iPdpt].u & X86_PDPE_P)) +# endif + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped)); + PGM_INVL_PG(pVCpu, GCPtrPage); + return VINF_SUCCESS; + } + + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK); + AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED); + + PX86PDPAE pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde); + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst]; + +# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */ + /* PML4 */ + /*const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;*/ + const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + PX86PDPAE pPDDst; + PX86PDPT pPdptDst; + PX86PML4E pPml4eDst; + rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst); + if (rc != VINF_SUCCESS) + { + AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc)); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped)); + PGM_INVL_PG(pVCpu, GCPtrPage); + return VINF_SUCCESS; + } + PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst]; + Assert(pPDDst); + Assert(pPdptDst->a[iPdpt].u & X86_PDPE_P); + + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK); + Assert(pShwPde); + +# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */ + + const SHWPDE PdeDst = *pPdeDst; + if (!(PdeDst.u & X86_PDE_P)) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped)); + PGM_INVL_PG(pVCpu, GCPtrPage); + return VINF_SUCCESS; + } + + /* + * Get the guest PD entry and calc big page. + */ +# if PGM_GST_TYPE == PGM_TYPE_32BIT + PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu); + const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT; + GSTPDE PdeSrc = pPDSrc->a[iPDSrc]; +# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */ + unsigned iPDSrc = 0; +# if PGM_GST_TYPE == PGM_TYPE_PAE + X86PDPE PdpeSrcIgn; + PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn); +# else /* AMD64 */ + PX86PML4E pPml4eSrcIgn; + X86PDPE PdpeSrcIgn; + PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc); +# endif + GSTPDE PdeSrc; + + if (pPDSrc) + PdeSrc = pPDSrc->a[iPDSrc]; + else + PdeSrc.u = 0; +# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */ + const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE); + const bool fIsBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu); + if (fWasBigPage != fIsBigPage) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped)); + +# ifdef IN_RING3 + /* + * If a CR3 Sync is pending we may ignore the invalidate page operation + * depending on the kind of sync and if it's a global page or not. + * This doesn't make sense in GC/R0 so we'll skip it entirely there. + */ +# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH + if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3) + || ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) + && fIsBigPage + && (PdeSrc.u & X86_PDE4M_G) + ) + ) +# else + if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) ) +# endif + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped)); + return VINF_SUCCESS; + } +# endif /* IN_RING3 */ + + /* + * Deal with the Guest PDE. + */ + rc = VINF_SUCCESS; + if (PdeSrc.u & X86_PDE_P) + { + Assert( (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US) + && ((PdeSrc.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW) || pVCpu->pgm.s.cNetwareWp0Hacks > 0)); + if (!fIsBigPage) + { + /* + * 4KB - page. + */ + PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK); + RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc); + +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */ + GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2))); +# endif + if (pShwPage->GCPhys == GCPhys) + { + /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */ + PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + + PGSTPT pPTSrc; + rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc); + if (RT_SUCCESS(rc)) + { + const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK; + GSTPTE PteSrc = pPTSrc->a[iPTSrc]; + const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst); + Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n", + GCPtrPage, PteSrc.u & X86_PTE_P, + (PteSrc.u & PdeSrc.u & X86_PTE_RW), + (PteSrc.u & PdeSrc.u & X86_PTE_US), + (uint64_t)PteSrc.u, + SHW_PTE_LOG64(pPTDst->a[iPTDst]), + SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "")); + } + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4KBPages)); + PGM_INVL_PG(pVCpu, GCPtrPage); + } + else + { + /* + * The page table address changed. + */ + LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n", + GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst)); + pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst); + SHW_PDE_ATOMIC_SET(*pPdeDst, 0); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync)); + PGM_INVL_VCPU_TLBS(pVCpu); + } + } + else + { + /* + * 2/4MB - page. + */ + /* Before freeing the page, check if anything really changed. */ + PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK); + RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc); +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/ + GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT))); +# endif + if ( pShwPage->GCPhys == GCPhys + && pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG) + { + /* ASSUMES a the given bits are identical for 4M and normal PDEs */ + /** @todo This test is wrong as it cannot check the G bit! + * FIXME */ + if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US)) + == (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US)) + && ( (PdeSrc.u & X86_PDE4M_D) /** @todo rainy day: What about read-only 4M pages? not very common, but still... */ + || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))) + { + LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u)); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip)); + return VINF_SUCCESS; + } + } + + /* + * Ok, the page table is present and it's been changed in the guest. + * If we're in host context, we'll just mark it as not present taking the lazy approach. + * We could do this for some flushes in GC too, but we need an algorithm for + * deciding which 4MB pages containing code likely to be executed very soon. + */ + LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n", + GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u)); + pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst); + SHW_PDE_ATOMIC_SET(*pPdeDst, 0); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPages)); + PGM_INVL_BIG_PG(pVCpu, GCPtrPage); + } + } + else + { + /* + * Page directory is not present, mark shadow PDE not present. + */ + pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst); + SHW_PDE_ATOMIC_SET(*pPdeDst, 0); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDNPs)); + PGM_INVL_PG(pVCpu, GCPtrPage); + } + return rc; + +#else /* guest real and protected mode, nested + ept, none. */ + /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */ + NOREF(pVCpu); NOREF(GCPtrPage); + return VINF_SUCCESS; +#endif +} + +#if PGM_SHW_TYPE != PGM_TYPE_NONE + +/** + * Update the tracking of shadowed pages. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pShwPage The shadow page. + * @param HCPhys The physical page we is being dereferenced. + * @param iPte Shadow PTE index + * @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set) + */ +DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte, + RTGCPHYS GCPhysPage) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + +# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \ + && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \ + && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */) + + /* Use the hint we retrieved from the cached guest PT. */ + if (pShwPage->fDirty) + { + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + + Assert(pShwPage->cPresent); + Assert(pPool->cPresent); + pShwPage->cPresent--; + pPool->cPresent--; + + PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage); + AssertRelease(pPhysPage); + pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte); + return; + } +# else + NOREF(GCPhysPage); +# endif + + STAM_PROFILE_START(&pVM->pgm.s.Stats.StatTrackDeref, a); + LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx)); + + /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done: + * 1. have a medium sized HCPhys -> GCPhys TLB (hash?) + * 2. write protect all shadowed pages. I.e. implement caching. + */ + /** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */ + + /* + * Find the guest address. + */ + for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX); + pRam; + pRam = pRam->CTX_SUFF(pNext)) + { + unsigned iPage = pRam->cb >> GUEST_PAGE_SHIFT; + while (iPage-- > 0) + { + if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys) + { + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + + Assert(pShwPage->cPresent); + Assert(pPool->cPresent); + pShwPage->cPresent--; + pPool->cPresent--; + + pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte); + STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatTrackDeref, a); + return; + } + } + } + + for (;;) + AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys)); +} + + +/** + * Update the tracking of shadowed pages. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pShwPage The shadow page. + * @param u16 The top 16-bit of the pPage->HCPhys. + * @param pPage Pointer to the guest page. this will be modified. + * @param iPTDst The index into the shadow table. + */ +DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, + PPGMPAGE pPage, const unsigned iPTDst) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Just deal with the simple first time here. + */ + if (!u16) + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackVirgin); + u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx); + /* Save the page table index. */ + PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst); + } + else + u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst); + + /* write back */ + Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst)); + PGM_PAGE_SET_TRACKING(pVM, pPage, u16); + + /* update statistics. */ + pVM->pgm.s.CTX_SUFF(pPool)->cPresent++; + pShwPage->cPresent++; + if (pShwPage->iFirstPresent > iPTDst) + pShwPage->iFirstPresent = iPTDst; +} + + +/** + * Modifies a shadow PTE to account for access handlers. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pPage The page in question. + * @param GCPhysPage The guest-physical address of the page. + * @param fPteSrc The shadowed flags of the source PTE. Must include the + * A (accessed) bit so it can be emulated correctly. + * @param pPteDst The shadow PTE (output). This is temporary storage and + * does not need to be set atomically. + */ +DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PVMCPUCC pVCpu, PCPGMPAGE pPage, RTGCPHYS GCPhysPage, uint64_t fPteSrc, + PSHWPTE pPteDst) +{ + RT_NOREF_PV(pVM); RT_NOREF_PV(fPteSrc); RT_NOREF_PV(pVCpu); RT_NOREF_PV(GCPhysPage); + + /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No. + * Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */ + if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) + { + LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage)); +# if PGM_SHW_TYPE == PGM_TYPE_EPT + pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage) | EPT_E_READ | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT; +# else + if (fPteSrc & X86_PTE_A) + { + SHW_PTE_SET(*pPteDst, fPteSrc | PGM_PAGE_GET_HCPHYS(pPage)); + SHW_PTE_SET_RO(*pPteDst); + } + else + SHW_PTE_SET(*pPteDst, 0); +# endif + } +# ifdef PGM_WITH_MMIO_OPTIMIZATIONS +# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 + else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) + && ( BTH_IS_NP_ACTIVE(pVM) + || (fPteSrc & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */ +# if PGM_SHW_TYPE == PGM_TYPE_AMD64 + && pVM->pgm.s.fLessThan52PhysicalAddressBits +# endif + ) + { + LogFlow(("SyncHandlerPte: MMIO page -> invalid \n")); +# if PGM_SHW_TYPE == PGM_TYPE_EPT + /* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */ + pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg + /* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */ + | EPT_E_WRITE + /* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */ + | EPT_E_MEMTYPE_INVALID_3; +# else + /* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */ + SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX | X86_PTE_P); +# endif + } +# endif +# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */ + else + { + LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage)); + SHW_PTE_SET(*pPteDst, 0); + } + /** @todo count these kinds of entries. */ +} + + +/** + * Creates a 4K shadow page for a guest page. + * + * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the + * physical address. The PdeSrc argument only the flags are used. No page + * structured will be mapped in this function. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pPteDst Destination page table entry. + * @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry). + * Can safely assume that only the flags are being used. + * @param PteSrc Source page table entry (i.e. Guest OS page table entry). + * @param pShwPage Pointer to the shadow page. + * @param iPTDst The index into the shadow table. + * + * @remark Not used for 2/4MB pages! + */ +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING) +static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, + PPGMPOOLPAGE pShwPage, unsigned iPTDst) +# else +static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, + PPGMPOOLPAGE pShwPage, unsigned iPTDst) +# endif +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS; + +# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \ + && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \ + && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */) + + if (pShwPage->fDirty) + { + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PGSTPT pGstPT; + + /* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */ + pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0]; + GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]); + pGstPT->a[iPTDst].u = PteSrc.u; + } +# else + Assert(!pShwPage->fDirty); +# endif + +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + if ( (PteSrc.u & X86_PTE_P) + && GST_IS_PTE_VALID(pVCpu, PteSrc)) +# endif + { +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc); +# endif + PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage); + + /* + * Find the ram range. + */ + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); + if (RT_SUCCESS(rc)) + { + /* Ignore ballooned pages. + Don't return errors or use a fatal assert here as part of a + shadow sync range might included ballooned pages. */ + if (PGM_PAGE_IS_BALLOONED(pPage)) + { + Assert(!SHW_PTE_IS_P(*pPteDst)); /** @todo user tracking needs updating if this triggers. */ + return; + } + +# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC + /* Make the page writable if necessary. */ + if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM + && ( PGM_PAGE_IS_ZERO(pPage) +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + || ( (PteSrc.u & X86_PTE_RW) +# else + || ( 1 +# endif + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED +# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED +# endif +# ifdef VBOX_WITH_PAGE_SHARING + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED +# endif + ) + ) + ) + { + rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage); + AssertRC(rc); + } +# endif + + /* + * Make page table entry. + */ + SHWPTE PteDst; +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc); +# else + uint64_t fGstShwPteFlags = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D; +# endif + if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage)) + { +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + /* + * If the page or page directory entry is not marked accessed, + * we mark the page not present. + */ + if (!(PteSrc.u & X86_PTE_A) || !(PdeSrc.u & X86_PDE_A)) + { + LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n")); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,AccessedPage)); + SHW_PTE_SET(PteDst, 0); + } + /* + * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit + * when the page is modified. + */ + else if (!(PteSrc.u & X86_PTE_D) && (PdeSrc.u & PteSrc.u & X86_PTE_RW)) + { + AssertCompile(X86_PTE_RW == X86_PDE_RW); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPage)); + SHW_PTE_SET(PteDst, + fGstShwPteFlags + | PGM_PAGE_GET_HCPHYS(pPage) + | PGM_PTFLAGS_TRACK_DIRTY); + SHW_PTE_SET_RO(PteDst); + } + else +# endif + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageSkipped)); +# if PGM_SHW_TYPE == PGM_TYPE_EPT + PteDst.u = PGM_PAGE_GET_HCPHYS(pPage) + | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT; +# else + SHW_PTE_SET(PteDst, fGstShwPteFlags | PGM_PAGE_GET_HCPHYS(pPage)); +# endif + } + + /* + * Make sure only allocated pages are mapped writable. + */ + if ( SHW_PTE_IS_P_RW(PteDst) + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED) + { + /* Still applies to shared pages. */ + Assert(!PGM_PAGE_IS_ZERO(pPage)); + SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */ + Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst)); + } + } + else + PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhysPage, fGstShwPteFlags, &PteDst); + + /* + * Keep user track up to date. + */ + if (SHW_PTE_IS_P(PteDst)) + { + if (!SHW_PTE_IS_P(*pPteDst)) + PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst); + else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst)) + { + Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst), SHW_PTE_LOG64(PteDst))); + PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage); + PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst); + } + } + else if (SHW_PTE_IS_P(*pPteDst)) + { + Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst))); + PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage); + } + + /* + * Update statistics and commit the entry. + */ +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + if (!(PteSrc.u & X86_PTE_G)) + pShwPage->fSeenNonGlobal = true; +# endif + SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst); + return; + } + +/** @todo count these three different kinds. */ + Log2(("SyncPageWorker: invalid address in Pte\n")); + } +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + else if (!(PteSrc.u & X86_PTE_P)) + Log2(("SyncPageWorker: page not present in Pte\n")); + else + Log2(("SyncPageWorker: invalid Pte\n")); +# endif + + /* + * The page is not present or the PTE is bad. Replace the shadow PTE by + * an empty entry, making sure to keep the user tracking up to date. + */ + if (SHW_PTE_IS_P(*pPteDst)) + { + Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst))); + PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage); + } + SHW_PTE_ATOMIC_SET(*pPteDst, 0); +} + + +/** + * Syncs a guest OS page. + * + * There are no conflicts at this point, neither is there any need for + * page table allocations. + * + * When called in PAE or AMD64 guest mode, the guest PDPE shall be valid. + * When called in AMD64 guest mode, the guest PML4E shall be valid. + * + * @returns VBox status code. + * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way. + * @param pVCpu The cross context virtual CPU structure. + * @param PdeSrc Page directory entry of the guest. + * @param GCPtrPage Guest context page address. + * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1). + * @param uErr Fault error (X86_TRAP_PF_*). + */ +static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool); + LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr)); + RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage); + + PGM_LOCK_ASSERT_OWNER(pVM); + +# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64) \ + && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) + + /* + * Assert preconditions. + */ + Assert(PdeSrc.u & X86_PDE_P); + Assert(cPages); +# if 0 /* rarely useful; leave for debugging. */ + STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]); +# endif + + /* + * Get the shadow PDE, find the shadow page table in the pool. + */ +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage); + + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3); + Assert(pShwPde); + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + PPGMPOOLPAGE pShwPde = NULL; + PX86PDPAE pPDDst; + + /* Fetch the pgm pool shadow descriptor. */ + int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde); + AssertRCSuccessReturn(rc2, rc2); + Assert(pShwPde); + + pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde); + PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst]; + +# elif PGM_SHW_TYPE == PGM_TYPE_AMD64 + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; + PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */ + PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */ + + int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst); + AssertRCSuccessReturn(rc2, rc2); + Assert(pPDDst && pPdptDst); + PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst]; +# endif + SHWPDE PdeDst = *pPdeDst; + + /* + * - In the guest SMP case we could have blocked while another VCPU reused + * this page table. + * - With W7-64 we may also take this path when the A bit is cleared on + * higher level tables (PDPE/PML4E). The guest does not invalidate the + * relevant TLB entries. If we're write monitoring any page mapped by + * the modified entry, we may end up here with a "stale" TLB entry. + */ + if (!(PdeDst.u & X86_PDE_P)) + { + Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr)); + AssertMsg(pVM->cCpus > 1 || (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW), + ("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr)); + if (uErr & X86_TRAP_PF_P) + PGM_INVL_PG(pVCpu, GCPtrPage); + return VINF_SUCCESS; /* force the instruction to be executed again. */ + } + + PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK); + Assert(pShwPage); + +# if PGM_GST_TYPE == PGM_TYPE_AMD64 + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK); + Assert(pShwPde); +# endif + + /* + * Check that the page is present and that the shadow PDE isn't out of sync. + */ + const bool fBigPage = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu); + const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc); + RTGCPHYS GCPhys; + if (!fBigPage) + { + GCPhys = GST_GET_PDE_GCPHYS(PdeSrc); +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */ + GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2))); +# endif + } + else + { + GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc); +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/ + GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT))); +# endif + } + /** @todo This doesn't check the G bit of 2/4MB pages. FIXME */ + if ( fPdeValid + && pShwPage->GCPhys == GCPhys + && (PdeSrc.u & X86_PDE_P) + && (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US) + && ((PdeSrc.u & X86_PDE_RW) == (PdeDst.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW)) +# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE) + && ((PdeSrc.u & X86_PDE_PAE_NX) == (PdeDst.u & X86_PDE_PAE_NX) || !GST_IS_NX_ACTIVE(pVCpu)) +# endif + ) + { + /* + * Check that the PDE is marked accessed already. + * Since we set the accessed bit *before* getting here on a #PF, this + * check is only meant for dealing with non-#PF'ing paths. + */ + if (PdeSrc.u & X86_PDE_A) + { + PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + if (!fBigPage) + { + /* + * 4KB Page - Map the guest page table. + */ + PGSTPT pPTSrc; + int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc); + if (RT_SUCCESS(rc)) + { +# ifdef PGM_SYNC_N_PAGES + Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P)); + if ( cPages > 1 + && !(uErr & X86_TRAP_PF_P) + && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) + { + /* + * This code path is currently only taken when the caller is PGMTrap0eHandler + * for non-present pages! + * + * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and + * deal with locality. + */ + unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK; +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */ + const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512; +# else + const unsigned offPTSrc = 0; +# endif + const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a)); + if (iPTDst < PGM_SYNC_NR_PAGES / 2) + iPTDst = 0; + else + iPTDst -= PGM_SYNC_NR_PAGES / 2; + + for (; iPTDst < iPTDstEnd; iPTDst++) + { + const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst]; + + if ( (pPteSrc->u & X86_PTE_P) + && !SHW_PTE_IS_P(pPTDst->a[iPTDst])) + { + RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) + | ((offPTSrc + iPTDst) << GUEST_PAGE_SHIFT); + NOREF(GCPtrCurPage); + PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst); + Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n", + GCPtrCurPage, pPteSrc->u & X86_PTE_P, + !!(pPteSrc->u & PdeSrc.u & X86_PTE_RW), + !!(pPteSrc->u & PdeSrc.u & X86_PTE_US), + (uint64_t)pPteSrc->u, + SHW_PTE_LOG64(pPTDst->a[iPTDst]), + SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "")); + } + } + } + else +# endif /* PGM_SYNC_N_PAGES */ + { + const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK; + GSTPTE PteSrc = pPTSrc->a[iPTSrc]; + const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst); + Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n", + GCPtrPage, PteSrc.u & X86_PTE_P, + !!(PteSrc.u & PdeSrc.u & X86_PTE_RW), + !!(PteSrc.u & PdeSrc.u & X86_PTE_US), + (uint64_t)PteSrc.u, + SHW_PTE_LOG64(pPTDst->a[iPTDst]), + SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "")); + } + } + else /* MMIO or invalid page: emulated in #PF handler. */ + { + LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc)); + Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK])); + } + } + else + { + /* + * 4/2MB page - lazy syncing shadow 4K pages. + * (There are many causes of getting here, it's no longer only CSAM.) + */ + /* Calculate the GC physical address of this 4KB shadow page. */ + GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) | (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK)); + /* Find ram range. */ + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage); + if (RT_SUCCESS(rc)) + { + AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys)); + +# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC + /* Try to make the page writable if necessary. */ + if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM + && ( PGM_PAGE_IS_ZERO(pPage) + || ( (PdeSrc.u & X86_PDE_RW) + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED +# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED +# endif +# ifdef VBOX_WITH_PAGE_SHARING + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED +# endif + ) + ) + ) + { + rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys); + AssertRC(rc); + } +# endif + + /* + * Make shadow PTE entry. + */ + SHWPTE PteDst; + if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage)) + SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) | PGM_PAGE_GET_HCPHYS(pPage)); + else + PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst); + + const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + if ( SHW_PTE_IS_P(PteDst) + && !SHW_PTE_IS_P(pPTDst->a[iPTDst])) + PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst); + + /* Make sure only allocated pages are mapped writable. */ + if ( SHW_PTE_IS_P_RW(PteDst) + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED) + { + /* Still applies to shared pages. */ + Assert(!PGM_PAGE_IS_ZERO(pPage)); + SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */ + Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage)); + } + + SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst); + + /* + * If the page is not flagged as dirty and is writable, then make it read-only + * at PD level, so we can set the dirty bit when the page is modified. + * + * ASSUMES that page access handlers are implemented on page table entry level. + * Thus we will first catch the dirty access and set PDE.D and restart. If + * there is an access handler, we'll trap again and let it work on the problem. + */ + /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already. + * As for invlpg, it simply frees the whole shadow PT. + * ...It's possibly because the guest clears it and the guest doesn't really tell us... */ + if ((PdeSrc.u & (X86_PDE4M_D | X86_PDE_RW)) == X86_PDE_RW) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig)); + PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY; + PdeDst.u &= ~(SHWUINT)X86_PDE_RW; + } + else + { + PdeDst.u &= ~(SHWUINT)(PGM_PDFLAGS_TRACK_DIRTY | X86_PDE_RW); + PdeDst.u |= PdeSrc.u & X86_PDE_RW; + } + SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst); + Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n", + GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US), + (uint64_t)PdeSrc.u, GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : "")); + } + else + { + LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc)); + /** @todo must wipe the shadow page table entry in this + * case. */ + } + } + PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst); + return VINF_SUCCESS; + } + + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDNAs)); + } + else if (fPdeValid) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDOutOfSync)); + Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n", + GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys)); + } + else + { +/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid)); + Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n", + GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys)); + } + + /* + * Mark the PDE not present. Restart the instruction and let #PF call SyncPT. + * Yea, I'm lazy. + */ + pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst); + SHW_PDE_ATOMIC_SET(*pPdeDst, 0); + + PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst); + PGM_INVL_VCPU_TLBS(pVCpu); + return VINF_PGM_SYNCPAGE_MODIFIED_PDE; + + +# elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \ + && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \ + && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) + NOREF(PdeSrc); + +# ifdef PGM_SYNC_N_PAGES + /* + * Get the shadow PDE, find the shadow page table in the pool. + */ +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage); + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE + X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage); + +# elif PGM_SHW_TYPE == PGM_TYPE_AMD64 + const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK); + const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt); + PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */ + X86PDEPAE PdeDst; + PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */ + + int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst); + AssertRCSuccessReturn(rc, rc); + Assert(pPDDst && pPdptDst); + PdeDst = pPDDst->a[iPDDst]; + +# elif PGM_SHW_TYPE == PGM_TYPE_EPT + const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK); + PEPTPD pPDDst; + EPTPDE PdeDst; + + int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst); + if (rc != VINF_SUCCESS) + { + AssertRC(rc); + return rc; + } + Assert(pPDDst); + PdeDst = pPDDst->a[iPDDst]; +# endif + /* In the guest SMP case we could have blocked while another VCPU reused this page table. */ + if (!SHW_PDE_IS_P(PdeDst)) + { + AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u)); + Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage)); + return VINF_SUCCESS; /* force the instruction to be executed again. */ + } + + /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */ + if (SHW_PDE_IS_BIG(PdeDst)) + { + Assert(pVM->pgm.s.fNestedPaging); + Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage)); + return VINF_SUCCESS; + } + + /* Mask away the page offset. */ + GCPtrPage &= ~((RTGCPTR)0xfff); + + PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK); + PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + + Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P)); + if ( cPages > 1 + && !(uErr & X86_TRAP_PF_P) + && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) + { + /* + * This code path is currently only taken when the caller is PGMTrap0eHandler + * for non-present pages! + * + * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and + * deal with locality. + */ + unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a)); + if (iPTDst < PGM_SYNC_NR_PAGES / 2) + iPTDst = 0; + else + iPTDst -= PGM_SYNC_NR_PAGES / 2; + for (; iPTDst < iPTDstEnd; iPTDst++) + { + if (!SHW_PTE_IS_P(pPTDst->a[iPTDst])) + { + RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) + | (iPTDst << GUEST_PAGE_SHIFT)); + + PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst); + Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n", + GCPtrCurPage, + SHW_PTE_LOG64(pPTDst->a[iPTDst]), + SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "")); + + if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))) + break; + } + else + Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", + (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << GUEST_PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) )); + } + } + else +# endif /* PGM_SYNC_N_PAGES */ + { + const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) + | (iPTDst << GUEST_PAGE_SHIFT)); + + PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst); + + Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n", + GCPtrPage, + SHW_PTE_LOG64(pPTDst->a[iPTDst]), + SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "")); + } + return VINF_SUCCESS; + +# else + NOREF(PdeSrc); + AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE)); + return VERR_PGM_NOT_USED_IN_MODE; +# endif +} + +#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */ + +#if !defined(IN_RING3) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT + +/** + * Sync a shadow page for a nested-guest page. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pPte The shadow page table entry. + * @param GCPhysPage The guest-physical address of the page. + * @param pShwPage The shadow page of the page table. + * @param iPte The index of the page table entry. + * @param pGstWalkAll The guest page table walk result. + * + * @note Not to be used for 2/4MB pages! + */ +static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, + unsigned iPte, PCPGMPTWALKGST pGstWalkAll) +{ + /* + * Do not make assumptions about anything other than the final PTE entry in the + * guest page table walk result. For instance, while mapping 2M PDEs as 4K pages, + * the PDE might still be having its leaf bit set. + * + * In the future, we could consider introducing a generic SLAT macro like PSLATPTE + * and using that instead of passing the full SLAT translation result. + */ + PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage); + Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage)); + Assert(!pShwPage->fDirty); + Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT); + AssertMsg((pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK) == GCPhysPage, + ("PTE address mismatch. GCPhysPage=%RGp Pte=%RX64\n", GCPhysPage, pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK)); + + /* + * Find the ram range. + */ + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVCpu->CTX_SUFF(pVM), GCPhysPage, &pPage); + AssertRCReturnVoid(rc); + + Assert(!PGM_PAGE_IS_BALLOONED(pPage)); + +# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC + /* Make the page writable if necessary. */ + /** @todo This needs to be applied to the regular case below, not here. And, + * no we should *NOT* make the page writble, instead we need to write + * protect them if necessary. */ + if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM + && ( PGM_PAGE_IS_ZERO(pPage) + || ( (pGstWalkAll->u.Ept.Pte.u & EPT_E_WRITE) + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED +# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED +# endif +# ifdef VBOX_WITH_PAGE_SHARING + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED +# endif + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_BALLOONED + ) + ) + ) + { + AssertMsgFailed(("GCPhysPage=%RGp\n", GCPhysPage)); /** @todo Shouldn't happen but if it does deal with it later. */ + } +# endif + + /* + * Make page table entry. + */ + SHWPTE Pte; + uint64_t const fGstShwPteFlags = pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptShadowedPteMask; + if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage)) + { + /** @todo access bit. */ + Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | fGstShwPteFlags; + Log7Func(("regular page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u)); + } + else if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) + { + /** @todo access bit. */ + Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | (fGstShwPteFlags & ~EPT_E_WRITE); + Log7Func(("monitored page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u)); + } + else + { + /** @todo Do MMIO optimizations here too? */ + Log7Func(("mmio/all page (%R[pgmpage]) at %RGp -> 0\n", pPage, GCPhysPage)); + Pte.u = 0; + } + + /* Make sure only allocated pages are mapped writable. */ + Assert(!SHW_PTE_IS_P_RW(Pte) || PGM_PAGE_IS_ALLOCATED(pPage)); + + /* + * Keep user track up to date. + */ + if (SHW_PTE_IS_P(Pte)) + { + if (!SHW_PTE_IS_P(*pPte)) + PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte); + else if (SHW_PTE_GET_HCPHYS(*pPte) != SHW_PTE_GET_HCPHYS(Pte)) + { + Log2(("SyncPageWorker: deref! *pPte=%RX64 Pte=%RX64\n", SHW_PTE_LOG64(*pPte), SHW_PTE_LOG64(Pte))); + PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS); + PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte); + } + } + else if (SHW_PTE_IS_P(*pPte)) + { + Log2(("SyncPageWorker: deref! *pPte=%RX64\n", SHW_PTE_LOG64(*pPte))); + PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS); + } + + /* + * Commit the entry. + */ + SHW_PTE_ATOMIC_SET2(*pPte, Pte); + return; +} + + +/** + * Syncs a nested-guest page. + * + * There are no conflicts at this point, neither is there any need for + * page table allocations. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhysNestedPage The nested-guest physical address of the page being + * synced. + * @param GCPhysPage The guest-physical address of the page being synced. + * @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1). + * @param uErr The page fault error (X86_TRAP_PF_XXX). + * @param pGstWalkAll The guest page table walk result. + */ +static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages, + uint32_t uErr, PPGMPTWALKGST pGstWalkAll) +{ + PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage); + Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK)); + Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK)); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool); + Log7Func(("GCPhysNestedPage=%RGv GCPhysPage=%RGp cPages=%u uErr=%#x\n", GCPhysNestedPage, GCPhysPage, cPages, uErr)); + RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); + + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * Get the shadow PDE, find the shadow page table in the pool. + */ + unsigned const iPde = ((GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK); + PEPTPD pPd; + int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL, &pPd, pGstWalkAll); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + Log(("Failed to fetch EPT PD for %RGp (%RGp) rc=%Rrc\n", GCPhysNestedPage, GCPhysPage, rc)); + return rc; + } + Assert(pPd); + EPTPDE Pde = pPd->a[iPde]; + + /* In the guest SMP case we could have blocked while another VCPU reused this page table. */ + if (!SHW_PDE_IS_P(Pde)) + { + AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)Pde.u)); + Log7Func(("CPU%d: SyncPage: Pde at %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage)); + return VINF_SUCCESS; /* force the instruction to be executed again. */ + } + + /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */ + if (SHW_PDE_IS_BIG(Pde)) + { + Log7Func(("CPU%d: SyncPage: %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage)); + return VINF_SUCCESS; + } + + PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, Pde.u & EPT_PDE_PG_MASK); + PEPTPT pPt = (PEPTPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + + /* + * If we've shadowed a guest EPT PDE that maps a 2M page using a 4K table, + * then sync the 4K sub-page in the 2M range. + */ + if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF) + { + Assert(!SHW_PDE_IS_BIG(Pde)); + + Assert(pGstWalkAll->u.Ept.Pte.u == 0); + Assert((Pde.u & EPT_PRESENT_MASK) == (pGstWalkAll->u.Ept.Pde.u & EPT_PRESENT_MASK)); + Assert(pShwPage->GCPhys == (pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK)); + +#if defined(VBOX_STRICT) && defined(DEBUG_ramshankar) + PPGMPAGE pPage; + rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); AssertRC(rc); + Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE); + Assert(pShwPage->enmKind == PGMPOOLKIND_EPT_PT_FOR_EPT_2MB); +#endif + uint64_t const fGstPteFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask & ~EPT_E_LEAF; + pGstWalkAll->u.Ept.Pte.u = GCPhysPage | fGstPteFlags; + + unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll); + Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte]))); + + /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */ + pGstWalkAll->u.Ept.Pte.u = 0; + return VINF_SUCCESS; + } + + Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P)); +# ifdef PGM_SYNC_N_PAGES + if ( cPages > 1 + && !(uErr & X86_TRAP_PF_P) + && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) + { + /* + * This code path is currently only taken for non-present pages! + * + * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and + * deal with locality. + */ + unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + unsigned const iPteEnd = RT_MIN(iPte + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPt->a)); + if (iPte < PGM_SYNC_NR_PAGES / 2) + iPte = 0; + else + iPte -= PGM_SYNC_NR_PAGES / 2; + for (; iPte < iPteEnd; iPte++) + { + if (!SHW_PTE_IS_P(pPt->a[iPte])) + { + PGMPTWALKGST GstWalkPt; + PGMPTWALK WalkPt; + GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT); + GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT); + rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt, + &GstWalkPt); + if (RT_SUCCESS(rc)) + PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], WalkPt.GCPhys, pShwPage, iPte, &GstWalkPt); + else + { + /* + * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages. + * Ensure the shadow tables entry is not-present. + */ + /** @todo Potential room for optimization (explained in NestedSyncPT). */ + AssertMsg(!pPt->a[iPte].u, ("%RX64\n", pPt->a[iPte].u)); + } + Log7Func(("Many: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte]))); + if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))) + break; + } + else + { +# ifdef VBOX_STRICT + /* Paranoia - Verify address of the page is what it should be. */ + PGMPTWALKGST GstWalkPt; + PGMPTWALK WalkPt; + GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT); + GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT); + rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt, &GstWalkPt); + AssertRC(rc); + PPGMPAGE pPage; + rc = pgmPhysGetPageEx(pVM, WalkPt.GCPhys, &pPage); + AssertRC(rc); + AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]), + ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp HCPhys=%RHp Shw=%RHp\n", + GCPhysNestedPage, WalkPt.GCPhys, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte]))); +# endif + Log7Func(("Many3: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte]))); + } + } + } + else +# endif /* PGM_SYNC_N_PAGES */ + { + unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll); + Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte]))); + } + + return VINF_SUCCESS; +} + + +/** + * Sync a shadow page table for a nested-guest page table. + * + * The shadow page table is not present in the shadow PDE. + * + * Handles mapping conflicts. + * + * A precondition for this method is that the shadow PDE is not present. The + * caller must take the PGM lock before checking this and continue to hold it + * when calling this method. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhysNestedPage The nested-guest physical page address of the page + * being synced. + * @param GCPhysPage The guest-physical address of the page being synced. + * @param pGstWalkAll The guest page table walk result. + */ +static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll) +{ + PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage); + Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK)); + Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK)); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + + Log7Func(("GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage)); + + PGM_LOCK_ASSERT_OWNER(pVM); + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + + PEPTPD pPd; + PEPTPDPT pPdpt; + unsigned const iPde = (GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK; + int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, &pPdpt, &pPd, pGstWalkAll); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + { + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + AssertRC(rc); + return rc; + } + Assert(pPd); + PSHWPDE pPde = &pPd->a[iPde]; + + unsigned const iPdpt = (GCPhysNestedPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK; + PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdpt->a[iPdpt].u & EPT_PDPTE_PG_MASK); + Assert(pShwPde->enmKind == PGMPOOLKIND_EPT_PD_FOR_EPT_PD); + + SHWPDE Pde = *pPde; + Assert(!SHW_PDE_IS_P(Pde)); /* We're only supposed to call SyncPT on PDE!P and conflicts. */ + +# ifdef PGM_WITH_LARGE_PAGES + if (BTH_IS_NP_ACTIVE(pVM)) + { + /* + * Check if the guest is mapping a 2M page here. + */ + PPGMPAGE pPage; + rc = pgmPhysGetPageEx(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK, &pPage); + AssertRCReturn(rc, rc); + if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF) + { + /* A20 is always enabled in VMX root and non-root operation. */ + Assert(PGM_A20_IS_ENABLED(pVCpu)); + + RTHCPHYS HCPhys = NIL_RTHCPHYS; + if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE) + { + STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused); + AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED); + HCPhys = PGM_PAGE_GET_HCPHYS(pPage); + } + else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED) + { + /* Recheck the entire 2 MB range to see if we can use it again as a large page. */ + rc = pgmPhysRecheckLargePage(pVM, GCPhysPage, pPage); + if (RT_SUCCESS(rc)) + { + Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED); + Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE); + HCPhys = PGM_PAGE_GET_HCPHYS(pPage); + } + } + else if (PGMIsUsingLargePages(pVM)) + { + rc = pgmPhysAllocLargePage(pVM, GCPhysPage); + if (RT_SUCCESS(rc)) + { + Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED); + Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE); + HCPhys = PGM_PAGE_GET_HCPHYS(pPage); + } + } + + /* + * If we have a 2M large page, we can map the guest's 2M large page right away. + */ + uint64_t const fShwBigPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask; + if (HCPhys != NIL_RTHCPHYS) + { + Pde.u = HCPhys | fShwBigPdeFlags; + Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzBigPdeMask)); + Assert(Pde.u & EPT_E_LEAF); + SHW_PDE_ATOMIC_SET2(*pPde, Pde); + + /* Add a reference to the first page only. */ + PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPde); + + Assert(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED); + + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + Log7Func(("GstPde=%RGp ShwPde=%RX64 [2M]\n", pGstWalkAll->u.Ept.Pde.u, Pde.u)); + return VINF_SUCCESS; + } + + /* + * We didn't get a perfect 2M fit. Split the 2M page into 4K pages. + * The page ought not to be marked as a big (2M) page at this point. + */ + Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE); + + /* Determine the right kind of large page to avoid incorrect cached entry reuse. */ + PGMPOOLACCESS enmAccess; + { + Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_USER_EXECUTE)); /* Mode-based execute control for EPT not supported. */ + bool const fNoExecute = !(pGstWalkAll->u.Ept.Pde.u & EPT_E_EXECUTE); + if (pGstWalkAll->u.Ept.Pde.u & EPT_E_WRITE) + enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW; + else + enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R; + } + + /* + * Allocate & map a 4K shadow table to cover the 2M guest page. + */ + PPGMPOOLPAGE pShwPage; + RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK; + rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_2MB, enmAccess, PGM_A20_IS_ENABLED(pVCpu), + pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage); + if ( rc == VINF_SUCCESS + || rc == VINF_PGM_CACHED_PAGE) + { /* likely */ } + else + { + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS); + } + + PSHWPT pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + Assert(pPt); + Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage)); + if (rc == VINF_SUCCESS) + { + /* The 4K PTEs shall inherit the flags of the 2M PDE page sans the leaf bit. */ + uint64_t const fShwPteFlags = fShwBigPdeFlags & ~EPT_E_LEAF; + + /* Sync each 4K pages in the 2M range. */ + for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++) + { + RTGCPHYS const GCPhysSubPage = GCPhysPt | (iPte << GUEST_PAGE_SHIFT); + pGstWalkAll->u.Ept.Pte.u = GCPhysSubPage | fShwPteFlags; + Assert(!(pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptMbzPteMask)); + PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysSubPage, pShwPage, iPte, pGstWalkAll); + Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [2M->4K]\n", pGstWalkAll->u.Ept.Pte, pPt->a[iPte].u, iPte)); + if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))) + break; + } + + /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */ + pGstWalkAll->u.Ept.Pte.u = 0; + } + else + { + Assert(rc == VINF_PGM_CACHED_PAGE); +# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar) + /* Paranoia - Verify address of each of the subpages are what they should be. */ + RTGCPHYS GCPhysSubPage = GCPhysPt; + for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++, GCPhysSubPage += GUEST_PAGE_SIZE) + { + PPGMPAGE pSubPage; + rc = pgmPhysGetPageEx(pVM, GCPhysSubPage, &pSubPage); + AssertRC(rc); + AssertMsg( PGM_PAGE_GET_HCPHYS(pSubPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]) + || !SHW_PTE_IS_P(pPt->a[iPte]), + ("PGM 2M page and shadow PTE conflict. GCPhysSubPage=%RGp Page=%RHp Shw=%RHp\n", + GCPhysSubPage, PGM_PAGE_GET_HCPHYS(pSubPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte]))); + } +# endif + rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */ + } + + /* Save the new PDE. */ + uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask; + Pde.u = pShwPage->Core.Key | fShwPdeFlags; + Assert(!(Pde.u & EPT_E_LEAF)); + Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask)); + SHW_PDE_ATOMIC_SET2(*pPde, Pde); + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde)); + return rc; + } + } +# endif /* PGM_WITH_LARGE_PAGES */ + + /* + * Allocate & map the shadow page table. + */ + PSHWPT pPt; + PPGMPOOLPAGE pShwPage; + + RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE_PG_MASK; + rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_PT, PGMPOOLACCESS_DONTCARE, + PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage); + if ( rc == VINF_SUCCESS + || rc == VINF_PGM_CACHED_PAGE) + { /* likely */ } + else + { + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS); + } + + pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + Assert(pPt); + Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage)); + + if (rc == VINF_SUCCESS) + { + /* Sync the page we've already translated through SLAT. */ + const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll); + Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte)); + + /* Sync the rest of page table (expensive but might be cheaper than nested-guest VM-exits in hardware). */ + for (unsigned iPteCur = 0; iPteCur < RT_ELEMENTS(pPt->a); iPteCur++) + { + if (iPteCur != iPte) + { + PGMPTWALKGST GstWalkPt; + PGMPTWALK WalkPt; + GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT); + GCPhysNestedPage |= (iPteCur << GUEST_PAGE_SHIFT); + int const rc2 = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, + &WalkPt, &GstWalkPt); + if (RT_SUCCESS(rc2)) + { + PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPteCur], WalkPt.GCPhys, pShwPage, iPteCur, &GstWalkPt); + Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", GstWalkPt.u.Ept.Pte.u, pPt->a[iPteCur].u, iPteCur)); + } + else + { + /* + * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages. + * Ensure the shadow tables entry is not-present. + */ + /** @todo We currently don't configure these to cause EPT misconfigs but rather trap + * them using EPT violations and walk the guest EPT tables to determine + * whether they are EPT misconfigs VM-exits for the nested-hypervisor. We + * could optimize this by using a specific combination of reserved bits + * which we could immediately identify as EPT misconfigs of the + * nested-hypervisor without having to walk its EPT tables. However, tracking + * non-present entries might be tricky... + */ + AssertMsg(!pPt->a[iPteCur].u, ("%RX64\n", pPt->a[iPteCur].u)); + } + if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))) + break; + } + } + } + else + { + Assert(rc == VINF_PGM_CACHED_PAGE); +# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar) + /* Paranoia - Verify address of the page is what it should be. */ + PPGMPAGE pPage; + rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); + AssertRC(rc); + const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]) || !SHW_PTE_IS_P(pPt->a[iPte]), + ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp Page=%RHp Shw=%RHp\n", + GCPhysNestedPage, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte]))); + Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [cache]\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte)); +# endif + rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */ + } + + /* Save the new PDE. */ + uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask; + Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)); + Assert(!(pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask)); + Pde.u = pShwPage->Core.Key | fShwPdeFlags; + SHW_PDE_ATOMIC_SET2(*pPde, Pde); + Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde)); + + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + return rc; +} + +#endif /* !IN_RING3 && VBOX_WITH_NESTED_HWVIRT_VMX_EPT && PGM_SHW_TYPE == PGM_TYPE_EPT*/ +#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE + +/** + * Handle dirty bit tracking faults. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uErr Page fault error code. + * @param pPdeSrc Guest page directory entry. + * @param pPdeDst Shadow page directory entry. + * @param GCPtrPage Guest context page address. + */ +static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, + RTGCPTR GCPtrPage) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + NOREF(uErr); + + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * Handle big page. + */ + if ((pPdeSrc->u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu)) + { + if ((pPdeDst->u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap)); + Assert(pPdeSrc->u & X86_PDE_RW); + + /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply + * fault again and take this path to only invalidate the entry (see below). */ + SHWPDE PdeDst = *pPdeDst; + PdeDst.u &= ~(SHWUINT)PGM_PDFLAGS_TRACK_DIRTY; + PdeDst.u |= X86_PDE_RW | X86_PDE_A; + SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst); + PGM_INVL_BIG_PG(pVCpu, GCPtrPage); + return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */ + } + +# ifdef IN_RING0 + /* Check for stale TLB entry; only applies to the SMP guest case. */ + if ( pVM->cCpus > 1 + && (pPdeDst->u & (X86_PDE_P | X86_PDE_RW | X86_PDE_A)) == (X86_PDE_P | X86_PDE_RW | X86_PDE_A)) + { + PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK); + if (pShwPage) + { + PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]; + if (SHW_PTE_IS_P_RW(*pPteDst)) + { + /* Stale TLB entry. */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale)); + PGM_INVL_PG(pVCpu, GCPtrPage); + return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */ + } + } + } +# endif /* IN_RING0 */ + return VINF_PGM_NO_DIRTY_BIT_TRACKING; + } + + /* + * Map the guest page table. + */ + PGSTPT pPTSrc; + int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc); + AssertRCReturn(rc, rc); + + if (SHW_PDE_IS_P(*pPdeDst)) + { + GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK]; + const GSTPTE PteSrc = *pPteSrc; + + /* + * Map shadow page table. + */ + PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK); + if (pShwPage) + { + PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]; + if (SHW_PTE_IS_P(*pPteDst)) /** @todo Optimize accessed bit emulation? */ + { + if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst)) + { + PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc)); + SHWPTE PteDst = *pPteDst; + + LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage)); + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap)); + + Assert(PteSrc.u & X86_PTE_RW); + + /* Note: No need to invalidate this entry on other VCPUs as a stale TLB + * entry will not harm; write access will simply fault again and + * take this path to only invalidate the entry. + */ + if (RT_LIKELY(pPage)) + { + if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage)) + { + //AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage)); + Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)); + /* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */ + SHW_PTE_SET_RO(PteDst); + } + else + { + if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED + && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM) + { + rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc)); + AssertRC(rc); + } + if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED) + SHW_PTE_SET_RW(PteDst); + else + { + /* Still applies to shared pages. */ + Assert(!PGM_PAGE_IS_ZERO(pPage)); + SHW_PTE_SET_RO(PteDst); + } + } + } + else + SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */ + + SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) | X86_PTE_D | X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY); + SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst); + PGM_INVL_PG(pVCpu, GCPtrPage); + return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */ + } + +# ifdef IN_RING0 + /* Check for stale TLB entry; only applies to the SMP guest case. */ + if ( pVM->cCpus > 1 + && SHW_PTE_IS_RW(*pPteDst) + && SHW_PTE_IS_A(*pPteDst)) + { + /* Stale TLB entry. */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale)); + PGM_INVL_PG(pVCpu, GCPtrPage); + return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */ + } +# endif + } + } + else + AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK)); + } + + return VINF_PGM_NO_DIRTY_BIT_TRACKING; +} + +#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */ + +/** + * Sync a shadow page table. + * + * The shadow page table is not present in the shadow PDE. + * + * Handles mapping conflicts. + * + * This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on + * conflict), and Trap0eHandler. + * + * A precondition for this method is that the shadow PDE is not present. The + * caller must take the PGM lock before checking this and continue to hold it + * when calling this method. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param iPDSrc Page directory index. + * @param pPDSrc Source page directory (i.e. Guest OS page directory). + * Assume this is a temporary mapping. + * @param GCPtrPage GC Pointer of the page that caused the fault + */ +static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool); + +#if 0 /* rarely useful; leave for debugging. */ + STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]); +#endif + LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage); + + PGM_LOCK_ASSERT_OWNER(pVM); + +#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64) \ + && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \ + && PGM_SHW_TYPE != PGM_TYPE_NONE + int rc = VINF_SUCCESS; + + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + + /* + * Some input validation first. + */ + AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage)); + + /* + * Get the relevant shadow PDE entry. + */ +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT; + PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage); + + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3); + Assert(pShwPde); + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + PPGMPOOLPAGE pShwPde = NULL; + PX86PDPAE pPDDst; + PSHWPDE pPdeDst; + + /* Fetch the pgm pool shadow descriptor. */ + rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde); + AssertRCSuccessReturn(rc, rc); + Assert(pShwPde); + + pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde); + pPdeDst = &pPDDst->a[iPDDst]; + +# elif PGM_SHW_TYPE == PGM_TYPE_AMD64 + const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */ + PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */ + rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst); + AssertRCSuccessReturn(rc, rc); + Assert(pPDDst); + PSHWPDE pPdeDst = &pPDDst->a[iPDDst]; + +# endif + SHWPDE PdeDst = *pPdeDst; + +# if PGM_GST_TYPE == PGM_TYPE_AMD64 + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK); + Assert(pShwPde); +# endif + + Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P.*/ + + /* + * Sync the page directory entry. + */ + GSTPDE PdeSrc = pPDSrc->a[iPDSrc]; + const bool fPageTable = !(PdeSrc.u & X86_PDE_PS) || !GST_IS_PSE_ACTIVE(pVCpu); + if ( (PdeSrc.u & X86_PDE_P) + && (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) ) + { + /* + * Allocate & map the page table. + */ + PSHWPT pPTDst; + PPGMPOOLPAGE pShwPage; + RTGCPHYS GCPhys; + if (fPageTable) + { + GCPhys = GST_GET_PDE_GCPHYS(PdeSrc); +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */ + GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2))); +# endif + rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu), + pShwPde->idx, iPDDst, false /*fLockPage*/, + &pShwPage); + } + else + { + PGMPOOLACCESS enmAccess; +# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE) + const bool fNoExecute = (PdeSrc.u & X86_PDE_PAE_NX) && GST_IS_NX_ACTIVE(pVCpu); +# else + const bool fNoExecute = false; +# endif + + GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc); +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/ + GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT))); +# endif + /* Determine the right kind of large page to avoid incorrect cached entry reuse. */ + if (PdeSrc.u & X86_PDE_US) + { + if (PdeSrc.u & X86_PDE_RW) + enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW; + else + enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R; + } + else + { + if (PdeSrc.u & X86_PDE_RW) + enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW; + else + enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R; + } + rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu), + pShwPde->idx, iPDDst, false /*fLockPage*/, + &pShwPage); + } + if (rc == VINF_SUCCESS) + pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + else if (rc == VINF_PGM_CACHED_PAGE) + { + /* + * The PT was cached, just hook it up. + */ + if (fPageTable) + PdeDst.u = pShwPage->Core.Key | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc); + else + { + PdeDst.u = pShwPage->Core.Key | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc); + /* (see explanation and assumptions further down.) */ + if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig)); + PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY; + PdeDst.u &= ~(SHWUINT)X86_PDE_RW; + } + } + SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst); + PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst); + return VINF_SUCCESS; + } + else + AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS); + /** @todo Why do we bother preserving X86_PDE_AVL_MASK here? + * Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be + * irrelevant at this point. */ + PdeDst.u &= X86_PDE_AVL_MASK; + PdeDst.u |= pShwPage->Core.Key; + + /* + * Page directory has been accessed (this is a fault situation, remember). + */ + /** @todo + * Well, when the caller is PrefetchPage or InvalidatePage is isn't a + * fault situation. What's more, the Trap0eHandler has already set the + * accessed bit. So, it's actually just VerifyAccessSyncPage which + * might need setting the accessed flag. + * + * The best idea is to leave this change to the caller and add an + * assertion that it's set already. */ + pPDSrc->a[iPDSrc].u |= X86_PDE_A; + if (fPageTable) + { + /* + * Page table - 4KB. + * + * Sync all or just a few entries depending on PGM_SYNC_N_PAGES. + */ + Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n", + GCPtrPage, PdeSrc.u & X86_PTE_P, !!(PdeSrc.u & X86_PTE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u)); + PGSTPT pPTSrc; + rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc); + if (RT_SUCCESS(rc)) + { + /* + * Start by syncing the page directory entry so CSAM's TLB trick works. + */ + PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK)) + | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc); + SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst); + PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst); + + /* + * Directory/page user or supervisor privilege: (same goes for read/write) + * + * Directory Page Combined + * U/S U/S U/S + * 0 0 0 + * 0 1 0 + * 1 0 0 + * 1 1 1 + * + * Simple AND operation. Table listed for completeness. + * + */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4K)); +# ifdef PGM_SYNC_N_PAGES + unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK; + unsigned iPTDst = iPTBase; + const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a)); + if (iPTDst <= PGM_SYNC_NR_PAGES / 2) + iPTDst = 0; + else + iPTDst -= PGM_SYNC_NR_PAGES / 2; +# else /* !PGM_SYNC_N_PAGES */ + unsigned iPTDst = 0; + const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a); +# endif /* !PGM_SYNC_N_PAGES */ + RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1)) + | ((RTGCPTR)iPTDst << GUEST_PAGE_SHIFT); +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */ + const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512; +# else + const unsigned offPTSrc = 0; +# endif + for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += GUEST_PAGE_SIZE) + { + const unsigned iPTSrc = iPTDst + offPTSrc; + const GSTPTE PteSrc = pPTSrc->a[iPTSrc]; + if (PteSrc.u & X86_PTE_P) + { + PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst); + Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n", + GCPtrCur, + PteSrc.u & X86_PTE_P, + !!(PteSrc.u & PdeSrc.u & X86_PTE_RW), + !!(PteSrc.u & PdeSrc.u & X86_PTE_US), + (uint64_t)PteSrc.u, + SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0], + (RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) )); + } + /* else: the page table was cleared by the pool */ + } /* for PTEs */ + } + } + else + { + /* + * Big page - 2/4MB. + * + * We'll walk the ram range list in parallel and optimize lookups. + * We will only sync one shadow page table at a time. + */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4M)); + + /** + * @todo It might be more efficient to sync only a part of the 4MB + * page (similar to what we do for 4KB PDs). + */ + + /* + * Start by syncing the page directory entry. + */ + PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY))) + | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc); + + /* + * If the page is not flagged as dirty and is writable, then make it read-only + * at PD level, so we can set the dirty bit when the page is modified. + * + * ASSUMES that page access handlers are implemented on page table entry level. + * Thus we will first catch the dirty access and set PDE.D and restart. If + * there is an access handler, we'll trap again and let it work on the problem. + */ + /** @todo move the above stuff to a section in the PGM documentation. */ + Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)); + if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW) + { + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig)); + PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY; + PdeDst.u &= ~(SHWUINT)X86_PDE_RW; + } + SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst); + PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst); + + /* + * Fill the shadow page table. + */ + /* Get address and flags from the source PDE. */ + SHWPTE PteDstBase; + SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc)); + + /* Loop thru the entries in the shadow PT. */ + const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr); + Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n", + GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u, GCPtr, + GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : "")); + PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys); + unsigned iPTDst = 0; + while ( iPTDst < RT_ELEMENTS(pPTDst->a) + && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) + { + if (pRam && GCPhys >= pRam->GCPhys) + { +# ifndef PGM_WITH_A20 + unsigned iHCPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT; +# endif + do + { + /* Make shadow PTE. */ +# ifdef PGM_WITH_A20 + PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT]; +# else + PPGMPAGE pPage = &pRam->aPages[iHCPage]; +# endif + SHWPTE PteDst; + +# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC + /* Try to make the page writable if necessary. */ + if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM + && ( PGM_PAGE_IS_ZERO(pPage) + || ( SHW_PTE_IS_RW(PteDstBase) + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED +# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED +# endif +# ifdef VBOX_WITH_PAGE_SHARING + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED +# endif + && !PGM_PAGE_IS_BALLOONED(pPage)) + ) + ) + { + rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys); + AssertRCReturn(rc, rc); + if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) + break; + } +# endif + + if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage)) + PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, SHW_PTE_GET_U(PteDstBase), &PteDst); + else if (PGM_PAGE_IS_BALLOONED(pPage)) + SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */ + else + SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) | SHW_PTE_GET_U(PteDstBase)); + + /* Only map writable pages writable. */ + if ( SHW_PTE_IS_P_RW(PteDst) + && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED) + { + /* Still applies to shared pages. */ + Assert(!PGM_PAGE_IS_ZERO(pPage)); + SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */ + Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)))); + } + + if (SHW_PTE_IS_P(PteDst)) + PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst); + + /* commit it (not atomic, new table) */ + pPTDst->a[iPTDst] = PteDst; + Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n", + (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst), + SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : "")); + + /* advance */ + GCPhys += GUEST_PAGE_SIZE; + PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys); +# ifndef PGM_WITH_A20 + iHCPage++; +# endif + iPTDst++; + } while ( iPTDst < RT_ELEMENTS(pPTDst->a) + && GCPhys <= pRam->GCPhysLast); + + /* Advance ram range list. */ + while (pRam && GCPhys > pRam->GCPhysLast) + pRam = pRam->CTX_SUFF(pNext); + } + else if (pRam) + { + Log(("Invalid pages at %RGp\n", GCPhys)); + do + { + SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */ + GCPhys += GUEST_PAGE_SIZE; + iPTDst++; + } while ( iPTDst < RT_ELEMENTS(pPTDst->a) + && GCPhys < pRam->GCPhys); + PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys); + } + else + { + Log(("Invalid pages at %RGp (2)\n", GCPhys)); + for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++) + SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */ + } + } /* while more PTEs */ + } /* 4KB / 4MB */ + } + else + AssertRelease(!SHW_PDE_IS_P(PdeDst)); + + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + if (RT_FAILURE(rc)) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed)); + return rc; + +#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \ + && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \ + && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \ + && PGM_SHW_TYPE != PGM_TYPE_NONE + NOREF(iPDSrc); NOREF(pPDSrc); + + STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + + /* + * Validate input a little bit. + */ + int rc = VINF_SUCCESS; +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage); + + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3); + Assert(pShwPde); + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */ + PX86PDPAE pPDDst; + PSHWPDE pPdeDst; + + /* Fetch the pgm pool shadow descriptor. */ + rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde); + AssertRCSuccessReturn(rc, rc); + Assert(pShwPde); + + pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde); + pPdeDst = &pPDDst->a[iPDDst]; + +# elif PGM_SHW_TYPE == PGM_TYPE_AMD64 + const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; + const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK; + PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */ + PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */ + rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst); + AssertRCSuccessReturn(rc, rc); + Assert(pPDDst); + PSHWPDE pPdeDst = &pPDDst->a[iPDDst]; + + /* Fetch the pgm pool shadow descriptor. */ + PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK); + Assert(pShwPde); + +# elif PGM_SHW_TYPE == PGM_TYPE_EPT + const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK; + const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK); + PEPTPD pPDDst; + PEPTPDPT pPdptDst; + + rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst); + if (rc != VINF_SUCCESS) + { + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + AssertRC(rc); + return rc; + } + Assert(pPDDst); + PSHWPDE pPdeDst = &pPDDst->a[iPDDst]; + + /* Fetch the pgm pool shadow descriptor. */ + /** @todo r=bird: didn't pgmShwGetEPTPDPtr just do this lookup already? */ + PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK); + Assert(pShwPde); +# endif + SHWPDE PdeDst = *pPdeDst; + + Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/ + +# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE + if (BTH_IS_NP_ACTIVE(pVM)) + { + Assert(!VM_IS_NEM_ENABLED(pVM)); + + /* Check if we allocated a big page before for this 2 MB range. */ + PPGMPAGE pPage; + rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage); + if (RT_SUCCESS(rc)) + { + RTHCPHYS HCPhys = NIL_RTHCPHYS; + if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE) + { + if (PGM_A20_IS_ENABLED(pVCpu)) + { + STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused); + AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED); + HCPhys = PGM_PAGE_GET_HCPHYS(pPage); + } + else + { + PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED); + pVM->pgm.s.cLargePagesDisabled++; + } + } + else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED + && PGM_A20_IS_ENABLED(pVCpu)) + { + /* Recheck the entire 2 MB range to see if we can use it again as a large page. */ + rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage); + if (RT_SUCCESS(rc)) + { + Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED); + Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE); + HCPhys = PGM_PAGE_GET_HCPHYS(pPage); + } + } + else if ( PGMIsUsingLargePages(pVM) + && PGM_A20_IS_ENABLED(pVCpu)) + { + rc = pgmPhysAllocLargePage(pVM, GCPtrPage); + if (RT_SUCCESS(rc)) + { + Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED); + Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE); + HCPhys = PGM_PAGE_GET_HCPHYS(pPage); + } + else + LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc)); + } + + if (HCPhys != NIL_RTHCPHYS) + { +# if PGM_SHW_TYPE == PGM_TYPE_EPT + PdeDst.u = HCPhys | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_LEAF | EPT_E_IGNORE_PAT | EPT_E_MEMTYPE_WB + | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo do we need this? */; +# else + PdeDst.u = HCPhys | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PS + | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo PGM_PD_FLAGS? */; +# endif + SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst); + + Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u)); + /* Add a reference to the first page only. */ + PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst); + + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + return VINF_SUCCESS; + } + } + } +# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */ + + /* + * Allocate & map the page table. + */ + PSHWPT pPTDst; + PPGMPOOLPAGE pShwPage; + RTGCPHYS GCPhys; + + /* Virtual address = physical address */ + GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK); + rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, + PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /*fLockPage*/, + &pShwPage); + if ( rc == VINF_SUCCESS + || rc == VINF_PGM_CACHED_PAGE) + pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage); + else + { + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS); + } + + if (rc == VINF_SUCCESS) + { + /* New page table; fully set it up. */ + Assert(pPTDst); + + /* Mask away the page offset. */ + GCPtrPage &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; + + for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++) + { + RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) + | (iPTDst << GUEST_PAGE_SHIFT)); + + PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst); + Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n", + GCPtrCurPage, + SHW_PTE_LOG64(pPTDst->a[iPTDst]), + SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "")); + + if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))) + break; + } + } + else + rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */ + + /* Save the new PDE. */ +# if PGM_SHW_TYPE == PGM_TYPE_EPT + PdeDst.u = pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE + | (PdeDst.u & X86_PDE_AVL_MASK /** @todo do we really need this? */); +# else + PdeDst.u = pShwPage->Core.Key | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A + | (PdeDst.u & X86_PDE_AVL_MASK /** @todo use a PGM_PD_FLAGS define */); +# endif + SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst); + + STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a); + if (RT_FAILURE(rc)) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed)); + return rc; + +#else + NOREF(iPDSrc); NOREF(pPDSrc); + AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE)); + return VERR_PGM_NOT_USED_IN_MODE; +#endif +} + + + +/** + * Prefetch a page/set of pages. + * + * Typically used to sync commonly used pages before entering raw mode + * after a CR3 reload. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtrPage Page to invalidate. + */ +PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage) +{ +#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_REAL \ + || PGM_GST_TYPE == PGM_TYPE_PROT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \ + && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \ + && PGM_SHW_TYPE != PGM_TYPE_NONE + /* + * Check that all Guest levels thru the PDE are present, getting the + * PD and PDE in the processes. + */ + int rc = VINF_SUCCESS; +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) +# if PGM_GST_TYPE == PGM_TYPE_32BIT + const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT; + PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu); +# elif PGM_GST_TYPE == PGM_TYPE_PAE + unsigned iPDSrc; + X86PDPE PdpeSrc; + PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc); + if (!pPDSrc) + return VINF_SUCCESS; /* not present */ +# elif PGM_GST_TYPE == PGM_TYPE_AMD64 + unsigned iPDSrc; + PX86PML4E pPml4eSrc; + X86PDPE PdpeSrc; + PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc); + if (!pPDSrc) + return VINF_SUCCESS; /* not present */ +# endif + const GSTPDE PdeSrc = pPDSrc->a[iPDSrc]; +# else + PGSTPD pPDSrc = NULL; + const unsigned iPDSrc = 0; + GSTPDE const PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */ +# endif + + if ((PdeSrc.u & (X86_PDE_P | X86_PDE_A)) == (X86_PDE_P | X86_PDE_A)) + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_VOID(pVM); + +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage); +# elif PGM_SHW_TYPE == PGM_TYPE_PAE + const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK); + PX86PDPAE pPDDst; + X86PDEPAE PdeDst; +# if PGM_GST_TYPE != PGM_TYPE_PAE + X86PDPE PdpeSrc; + + /* Fake PDPT entry; access control handled on the page table level, so allow everything. */ + PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */ +# endif + rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst); + if (rc != VINF_SUCCESS) + { + PGM_UNLOCK(pVM); + AssertRC(rc); + return rc; + } + Assert(pPDDst); + PdeDst = pPDDst->a[iPDDst]; + +# elif PGM_SHW_TYPE == PGM_TYPE_AMD64 + const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK); + PX86PDPAE pPDDst; + X86PDEPAE PdeDst; + +# if PGM_GST_TYPE == PGM_TYPE_PROT + /* AMD-V nested paging */ + X86PML4E Pml4eSrc; + X86PDPE PdpeSrc; + PX86PML4E pPml4eSrc = &Pml4eSrc; + + /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */ + Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A; + PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A; +# endif + + rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst); + if (rc != VINF_SUCCESS) + { + PGM_UNLOCK(pVM); + AssertRC(rc); + return rc; + } + Assert(pPDDst); + PdeDst = pPDDst->a[iPDDst]; +# endif + if (!(PdeDst.u & X86_PDE_P)) + { + /** @todo r=bird: This guy will set the A bit on the PDE, + * probably harmless. */ + rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage); + } + else + { + /* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because + * R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it + * makes no sense to prefetch more than one page. + */ + rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0); + if (RT_SUCCESS(rc)) + rc = VINF_SUCCESS; + } + PGM_UNLOCK(pVM); + } + return rc; + +#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE + NOREF(pVCpu); NOREF(GCPtrPage); + return VINF_SUCCESS; /* ignore */ +#else + AssertCompile(0); +#endif +} + + + + +/** + * Syncs a page during a PGMVerifyAccess() call. + * + * @returns VBox status code (informational included). + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtrPage The address of the page to sync. + * @param fPage The effective guest page flags. + * @param uErr The trap error code. + * @remarks This will normally never be called on invalid guest page + * translation entries. + */ +PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM); + + LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr)); + RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr); + + Assert(!pVM->pgm.s.fNestedPaging); +#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_REAL \ + || PGM_GST_TYPE == PGM_TYPE_PROT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \ + && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \ + && PGM_SHW_TYPE != PGM_TYPE_NONE + + /* + * Get guest PD and index. + */ + /** @todo Performance: We've done all this a jiffy ago in the + * PGMGstGetPage call. */ +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) +# if PGM_GST_TYPE == PGM_TYPE_32BIT + const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT; + PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu); + +# elif PGM_GST_TYPE == PGM_TYPE_PAE + unsigned iPDSrc = 0; + X86PDPE PdpeSrc; + PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc); + if (RT_UNLIKELY(!pPDSrc)) + { + Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage)); + return VINF_EM_RAW_GUEST_TRAP; + } + +# elif PGM_GST_TYPE == PGM_TYPE_AMD64 + unsigned iPDSrc = 0; /* shut up gcc */ + PX86PML4E pPml4eSrc = NULL; /* ditto */ + X86PDPE PdpeSrc; + PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc); + if (RT_UNLIKELY(!pPDSrc)) + { + Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage)); + return VINF_EM_RAW_GUEST_TRAP; + } +# endif + +# else /* !PGM_WITH_PAGING */ + PGSTPD pPDSrc = NULL; + const unsigned iPDSrc = 0; +# endif /* !PGM_WITH_PAGING */ + int rc = VINF_SUCCESS; + + PGM_LOCK_VOID(pVM); + + /* + * First check if the shadow pd is present. + */ +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage); + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE + PX86PDEPAE pPdeDst; + const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK); + PX86PDPAE pPDDst; +# if PGM_GST_TYPE != PGM_TYPE_PAE + /* Fake PDPT entry; access control handled on the page table level, so allow everything. */ + X86PDPE PdpeSrc; + PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */ +# endif + rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst); + if (rc != VINF_SUCCESS) + { + PGM_UNLOCK(pVM); + AssertRC(rc); + return rc; + } + Assert(pPDDst); + pPdeDst = &pPDDst->a[iPDDst]; + +# elif PGM_SHW_TYPE == PGM_TYPE_AMD64 + const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK); + PX86PDPAE pPDDst; + PX86PDEPAE pPdeDst; + +# if PGM_GST_TYPE == PGM_TYPE_PROT + /* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */ + X86PML4E Pml4eSrc; + X86PDPE PdpeSrc; + PX86PML4E pPml4eSrc = &Pml4eSrc; + Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A; + PdpeSrc.u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A; +# endif + + rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst); + if (rc != VINF_SUCCESS) + { + PGM_UNLOCK(pVM); + AssertRC(rc); + return rc; + } + Assert(pPDDst); + pPdeDst = &pPDDst->a[iPDDst]; +# endif + + if (!(pPdeDst->u & X86_PDE_P)) + { + rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage); + if (rc != VINF_SUCCESS) + { + PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst); + PGM_UNLOCK(pVM); + AssertRC(rc); + return rc; + } + } + +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + /* Check for dirty bit fault */ + rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage); + if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT) + Log(("PGMVerifyAccess: success (dirty)\n")); + else +# endif + { +# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) + GSTPDE PdeSrc = pPDSrc->a[iPDSrc]; +# else + GSTPDE const PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */ +# endif + + Assert(rc != VINF_EM_RAW_GUEST_TRAP); + if (uErr & X86_TRAP_PF_US) + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser)); + else /* supervisor */ + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor)); + + rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0); + if (RT_SUCCESS(rc)) + { + /* Page was successfully synced */ + Log2(("PGMVerifyAccess: success (sync)\n")); + rc = VINF_SUCCESS; + } + else + { + Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc)); + rc = VINF_EM_RAW_GUEST_TRAP; + } + } + PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst); + PGM_UNLOCK(pVM); + return rc; + +#else /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */ + + AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE)); + return VERR_PGM_NOT_USED_IN_MODE; +#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */ +} + + +/** + * Syncs the paging hierarchy starting at CR3. + * + * @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other + * informational status codes. + * @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map + * the VMM into guest context. + * @param pVCpu The cross context virtual CPU structure. + * @param cr0 Guest context CR0 register. + * @param cr3 Guest context CR3 register. Not subjected to the A20 + * mask. + * @param cr4 Guest context CR4 register. + * @param fGlobal Including global page directories or not + */ +PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM); + NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal); + + LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal)); + +#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE +# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + PGM_LOCK_VOID(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + if (pPool->cDirtyPages) + pgmPoolResetDirtyPages(pVM); + PGM_UNLOCK(pVM); +# endif +#endif /* !NESTED && !EPT */ + +#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE + /* + * Nested / EPT / None - No work. + */ + return VINF_SUCCESS; + +#elif PGM_SHW_TYPE == PGM_TYPE_AMD64 + /* + * AMD64 (Shw & Gst) - No need to check all paging levels; we zero + * out the shadow parts when the guest modifies its tables. + */ + return VINF_SUCCESS; + +#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */ + + return VINF_SUCCESS; +#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */ +} + + + + +#ifdef VBOX_STRICT + +/** + * Checks that the shadow page table is in sync with the guest one. + * + * @returns The number of errors. + * @param pVCpu The cross context virtual CPU structure. + * @param cr3 Guest context CR3 register. + * @param cr4 Guest context CR4 register. + * @param GCPtr Where to start. Defaults to 0. + * @param cb How much to check. Defaults to everything. + */ +PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb) +{ + NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb); +#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE + return 0; +#else + unsigned cErrors = 0; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool); + +# if PGM_GST_TYPE == PGM_TYPE_PAE + /** @todo currently broken; crashes below somewhere */ + AssertFailed(); +# endif + +# if PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64 + + bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu); + PPGMCPU pPGM = &pVCpu->pgm.s; + RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */ + RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */ +# ifndef IN_RING0 + RTHCPHYS HCPhys; /* general usage. */ +# endif + int rc; + + /* + * Check that the Guest CR3 and all its mappings are correct. + */ + AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), + ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3), + false); +# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64 +# if 0 +# if PGM_GST_TYPE == PGM_TYPE_32BIT + rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw); +# else + rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw); +# endif + AssertRCReturn(rc, 1); + HCPhys = NIL_RTHCPHYS; + rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys); + AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false); +# endif +# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3) + pgmGstGet32bitPDPtr(pVCpu); + RTGCPHYS GCPhys; + rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys); + AssertRCReturn(rc, 1); + AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false); +# endif +# endif /* !IN_RING0 */ + + /* + * Get and check the Shadow CR3. + */ +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + unsigned cPDEs = X86_PG_ENTRIES; + unsigned cIncrement = X86_PG_ENTRIES * GUEST_PAGE_SIZE; +# elif PGM_SHW_TYPE == PGM_TYPE_PAE +# if PGM_GST_TYPE == PGM_TYPE_32BIT + unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */ +# else + unsigned cPDEs = X86_PG_PAE_ENTRIES; +# endif + unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE; +# elif PGM_SHW_TYPE == PGM_TYPE_AMD64 + unsigned cPDEs = X86_PG_PAE_ENTRIES; + unsigned cIncrement = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE; +# endif + if (cb != ~(RTGCPTR)0) + cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1); + +/** @todo call the other two PGMAssert*() functions. */ + +# if PGM_GST_TYPE == PGM_TYPE_AMD64 + unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK; + + for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++) + { + PPGMPOOLPAGE pShwPdpt = NULL; + PX86PML4E pPml4eSrc; + PX86PML4E pPml4eDst; + RTGCPHYS GCPhysPdptSrc; + + pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4); + pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4); + + /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */ + if (!(pPml4eDst->u & X86_PML4E_P)) + { + GCPtr += _2M * UINT64_C(512) * UINT64_C(512); + continue; + } + + pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK); + GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK); + + if ((pPml4eSrc->u & X86_PML4E_P) != (pPml4eDst->u & X86_PML4E_P)) + { + AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u)); + GCPtr += _2M * UINT64_C(512) * UINT64_C(512); + cErrors++; + continue; + } + + if (GCPhysPdptSrc != pShwPdpt->GCPhys) + { + AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc)); + GCPtr += _2M * UINT64_C(512) * UINT64_C(512); + cErrors++; + continue; + } + + if ( (pPml4eDst->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX)) + != (pPml4eSrc->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX))) + { + AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u)); + GCPtr += _2M * UINT64_C(512) * UINT64_C(512); + cErrors++; + continue; + } +# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */ + { +# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */ + +# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE + /* + * Check the PDPTEs too. + */ + unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK; + + for (;iPdpt <= SHW_PDPT_MASK; iPdpt++) + { + unsigned iPDSrc = 0; /* initialized to shut up gcc */ + PPGMPOOLPAGE pShwPde = NULL; + PX86PDPE pPdpeDst; + RTGCPHYS GCPhysPdeSrc; + X86PDPE PdpeSrc; + PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */ +# if PGM_GST_TYPE == PGM_TYPE_PAE + PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc); + PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu); +# else + PX86PML4E pPml4eSrcIgn; + PX86PDPT pPdptDst; + PX86PDPAE pPDDst; + PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc); + + rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst); + if (rc != VINF_SUCCESS) + { + AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc)); + GCPtr += 512 * _2M; + continue; /* next PDPTE */ + } + Assert(pPDDst); +# endif + Assert(iPDSrc == 0); + + pPdpeDst = &pPdptDst->a[iPdpt]; + + if (!(pPdpeDst->u & X86_PDPE_P)) + { + GCPtr += 512 * _2M; + continue; /* next PDPTE */ + } + + pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK); + GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK); + + if ((pPdpeDst->u & X86_PDPE_P) != (PdpeSrc.u & X86_PDPE_P)) + { + AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u)); + GCPtr += 512 * _2M; + cErrors++; + continue; + } + + if (GCPhysPdeSrc != pShwPde->GCPhys) + { +# if PGM_GST_TYPE == PGM_TYPE_AMD64 + AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc)); +# else + AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc)); +# endif + GCPtr += 512 * _2M; + cErrors++; + continue; + } + +# if PGM_GST_TYPE == PGM_TYPE_AMD64 + if ( (pPdpeDst->u & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX)) + != (PdpeSrc.u & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX))) + { + AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u)); + GCPtr += 512 * _2M; + cErrors++; + continue; + } +# endif + +# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */ + { +# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */ +# if PGM_GST_TYPE == PGM_TYPE_32BIT + GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu); +# if PGM_SHW_TYPE == PGM_TYPE_32BIT + PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu); +# endif +# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */ + /* + * Iterate the shadow page directory. + */ + GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT; + unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK; + + for (; + iPDDst < cPDEs; + iPDDst++, GCPtr += cIncrement) + { +# if PGM_SHW_TYPE == PGM_TYPE_PAE + const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr); +# else + const SHWPDE PdeDst = pPDDst->a[iPDDst]; +# endif + if ( (PdeDst.u & X86_PDE_P) + || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) ) + { + HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK; + PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw); + if (!pPoolPage) + { + AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n", + HCPhysShw, GCPtr, (uint64_t)PdeDst.u)); + cErrors++; + continue; + } + const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage); + + if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD)) + { + AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n", + GCPtr, (uint64_t)PdeDst.u)); + cErrors++; + } + + if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D)) + { + AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n", + GCPtr, (uint64_t)PdeDst.u)); + cErrors++; + } + + const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK]; + if (!(PdeSrc.u & X86_PDE_P)) + { + AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n", + GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u)); + cErrors++; + continue; + } + + if ( !(PdeSrc.u & X86_PDE_PS) + || !fBigPagesSupported) + { + GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc); +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2))); +# endif + } + else + { +# if PGM_GST_TYPE == PGM_TYPE_32BIT + if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK) + { + AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n", + GCPtr, (uint64_t)PdeSrc.u)); + cErrors++; + continue; + } +# endif + GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc); +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT))); +# endif + } + + if ( pPoolPage->enmKind + != (!(PdeSrc.u & X86_PDE_PS) || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG)) + { + AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n", + pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u)); + cErrors++; + } + + PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst); + if (!pPhysPage) + { + AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n", + GCPhysGst, GCPtr, (uint64_t)PdeSrc.u)); + cErrors++; + continue; + } + + if (GCPhysGst != pPoolPage->GCPhys) + { + AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n", + GCPhysGst, pPoolPage->GCPhys, GCPtr)); + cErrors++; + continue; + } + + if ( !(PdeSrc.u & X86_PDE_PS) + || !fBigPagesSupported) + { + /* + * Page Table. + */ + const GSTPT *pPTSrc; + rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(GUEST_PAGE_SIZE - 1)), + &pPTSrc); + if (RT_FAILURE(rc)) + { + AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n", + GCPhysGst, GCPtr, (uint64_t)PdeSrc.u)); + cErrors++; + continue; + } + if ( (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)) + != (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))) + { + /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here! + // (This problem will go away when/if we shadow multiple CR3s.) + AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n", + GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u)); + cErrors++; + continue; + } + if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY) + { + AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n", + GCPtr, (uint64_t)PdeDst.u)); + cErrors++; + continue; + } + + /* iterate the page table. */ +# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT + /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */ + const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512; +# else + const unsigned offPTSrc = 0; +# endif + for (unsigned iPT = 0, off = 0; + iPT < RT_ELEMENTS(pPTDst->a); + iPT++, off += GUEST_PAGE_SIZE) + { + const SHWPTE PteDst = pPTDst->a[iPT]; + + /* skip not-present and dirty tracked entries. */ + if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */ + continue; + Assert(SHW_PTE_IS_P(PteDst)); + + const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc]; + if (!(PteSrc.u & X86_PTE_P)) + { +# ifdef IN_RING3 + PGMAssertHandlerAndFlagsInSync(pVM); + DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE + | DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3, + 0, 0, UINT64_MAX, 99, NULL); +# endif + AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n", + GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0], + (uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc))); + cErrors++; + continue; + } + + uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT; +# if 1 /** @todo sync accessed bit properly... */ + fIgnoreFlags |= X86_PTE_A; +# endif + + /* match the physical addresses */ + HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst); + GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc); + +# ifdef IN_RING3 + rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys); + if (RT_FAILURE(rc)) + { +# if 0 + if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */ + { + AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } +# endif + } + else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK)) + { + AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } +# endif + + pPhysPage = pgmPhysGetPage(pVM, GCPhysGst); + if (!pPhysPage) + { +# if 0 + if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */ + { + AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } +# endif + if (SHW_PTE_IS_RW(PteDst)) + { + AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + } + fIgnoreFlags |= X86_PTE_RW; + } + else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage)) + { + AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + + /* flags */ + if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage)) + { + if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage)) + { + if (SHW_PTE_IS_RW(PteDst)) + { + AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + fIgnoreFlags |= X86_PTE_RW; + } + else + { + if ( SHW_PTE_IS_P(PteDst) +# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 + && !PGM_PAGE_IS_MMIO(pPhysPage) +# endif + ) + { + AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + fIgnoreFlags |= X86_PTE_P; + } + } + else + { + if ((PteSrc.u & (X86_PTE_RW | X86_PTE_D)) == X86_PTE_RW) + { + if (SHW_PTE_IS_RW(PteDst)) + { + AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + if (!SHW_PTE_IS_TRACK_DIRTY(PteDst)) + { + AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + if (SHW_PTE_IS_D(PteDst)) + { + AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + } +# if 0 /** @todo sync access bit properly... */ + if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed) + { + AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + } + fIgnoreFlags |= X86_PTE_RW; +# else + fIgnoreFlags |= X86_PTE_RW | X86_PTE_A; +# endif + } + else if (SHW_PTE_IS_TRACK_DIRTY(PteDst)) + { + /* access bit emulation (not implemented). */ + if ((PteSrc.u & X86_PTE_A) || SHW_PTE_IS_P(PteDst)) + { + AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + if (!SHW_PTE_IS_A(PteDst)) + { + AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + } + fIgnoreFlags |= X86_PTE_P; + } +# ifdef DEBUG_sandervl + fIgnoreFlags |= X86_PTE_D | X86_PTE_A; +# endif + } + + if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) + && (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) + ) + { + AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags, + fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + } /* foreach PTE */ + } + else + { + /* + * Big Page. + */ + uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD; + if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW) + { + if (PdeDst.u & X86_PDE_RW) + { + AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n", + GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u)); + cErrors++; + continue; + } + if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)) + { + AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u)); + cErrors++; + continue; + } +# if 0 /** @todo sync access bit properly... */ + if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed) + { + AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n", + GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u)); + cErrors++; + } + fIgnoreFlags |= X86_PTE_RW; +# else + fIgnoreFlags |= X86_PTE_RW | X86_PTE_A; +# endif + } + else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY) + { + /* access bit emulation (not implemented). */ + if ((PdeSrc.u & X86_PDE_A) || SHW_PDE_IS_P(PdeDst)) + { + AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n", + GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u)); + cErrors++; + continue; + } + if (!SHW_PDE_IS_A(PdeDst)) + { + AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n", + GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u)); + cErrors++; + } + fIgnoreFlags |= X86_PTE_P; + } + + if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags)) + { + AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n", + GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags, + fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u)); + cErrors++; + } + + /* iterate the page table. */ + for (unsigned iPT = 0, off = 0; + iPT < RT_ELEMENTS(pPTDst->a); + iPT++, off += GUEST_PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + GUEST_PAGE_SIZE)) + { + const SHWPTE PteDst = pPTDst->a[iPT]; + + if (SHW_PTE_IS_TRACK_DIRTY(PteDst)) + { + AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + } + + /* skip not-present entries. */ + if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */ + continue; + + fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX; + + /* match the physical addresses */ + HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst); + +# ifdef IN_RING3 + rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys); + if (RT_FAILURE(rc)) + { +# if 0 + if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */ + { + AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n", + GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + } +# endif + } + else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK)) + { + AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } +# endif + pPhysPage = pgmPhysGetPage(pVM, GCPhysGst); + if (!pPhysPage) + { +# if 0 /** @todo make MMR3PageDummyHCPhys an 'All' function! */ + if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */ + { + AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n", + GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } +# endif + if (SHW_PTE_IS_RW(PteDst)) + { + AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + } + fIgnoreFlags |= X86_PTE_RW; + } + else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage)) + { + AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + + /* flags */ + if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage)) + { + if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage)) + { + if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED) + { + if ( SHW_PTE_IS_RW(PteDst) + && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage)) + { + AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + fIgnoreFlags |= X86_PTE_RW; + } + } + else + { + if ( SHW_PTE_IS_P(PteDst) + && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage) +# if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 + && !PGM_PAGE_IS_MMIO(pPhysPage) +# endif + ) + { + AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + fIgnoreFlags |= X86_PTE_P; + } + } + + if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) + && (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */ + ) + { + AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n", + GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags, + fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst))); + cErrors++; + continue; + } + } /* for each PTE */ + } + } + /* not present */ + + } /* for each PDE */ + + } /* for each PDPTE */ + + } /* for each PML4E */ + +# ifdef DEBUG + if (cErrors) + LogFlow(("AssertCR3: cErrors=%d\n", cErrors)); +# endif +# endif /* GST is in {32BIT, PAE, AMD64} */ + return cErrors; +#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */ +} +#endif /* VBOX_STRICT */ + + +/** + * Sets up the CR3 for shadow paging + * + * @returns Strict VBox status code. + * @retval VINF_SUCCESS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhysCR3 The physical address in the CR3 register. (A20 mask + * already applied.) + */ +PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM); + int rc = VINF_SUCCESS; + + /* Update guest paging info. */ +#if PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64 + + LogFlow(("MapCR3: %RGp\n", GCPhysCR3)); + PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3); + +# if PGM_GST_TYPE == PGM_TYPE_PAE + if ( !pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) + || pVCpu->pgm.s.GCPhysPaeCR3 != GCPhysCR3) +# endif + { + /* + * Map the page CR3 points at. + */ + RTHCPTR HCPtrGuestCR3; + rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCR3); + if (RT_SUCCESS(rc)) + { +# if PGM_GST_TYPE == PGM_TYPE_32BIT +# ifdef IN_RING3 + pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3; + pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR; +# else + pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR; + pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3; +# endif + +# elif PGM_GST_TYPE == PGM_TYPE_PAE +# ifdef IN_RING3 + pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3; + pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR; +# else + pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR; + pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3; +# endif + + X86PDPE aGstPaePdpes[X86_PG_PAE_PDPE_ENTRIES]; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + /* + * When EPT is enabled by the nested-hypervisor and the nested-guest is in PAE mode, + * the guest-CPU context would've already been updated with the 4 PAE PDPEs specified + * in the virtual VMCS. The PDPEs can differ from those in guest memory referenced by + * the translated nested-guest CR3. We -MUST- use the PDPEs provided in the virtual VMCS + * rather than those in guest memory. + * + * See Intel spec. 26.3.2.4 "Loading Page-Directory-Pointer-Table Entries". + */ + if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) + CPUMGetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]); + else +#endif + { + /* Update CPUM with the PAE PDPEs referenced by CR3. */ + memcpy(&aGstPaePdpes, HCPtrGuestCR3, sizeof(aGstPaePdpes)); + CPUMSetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]); + } + + /* + * Map the 4 PAE PDPEs. + */ + rc = PGMGstMapPaePdpes(pVCpu, &aGstPaePdpes[0]); + if (RT_SUCCESS(rc)) + { +# ifdef IN_RING3 + pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true; + pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false; +# else + pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false; + pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true; +# endif + pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3; + } + +# elif PGM_GST_TYPE == PGM_TYPE_AMD64 +# ifdef IN_RING3 + pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3; + pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR; +# else + pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR; + pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3; +# endif +# endif + } + else + AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3)); + } +#endif + + /* + * Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64). + */ +# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \ + || PGM_SHW_TYPE == PGM_TYPE_PAE \ + || PGM_SHW_TYPE == PGM_TYPE_AMD64) \ + && ( PGM_GST_TYPE != PGM_TYPE_REAL \ + && PGM_GST_TYPE != PGM_TYPE_PROT)) + + Assert(!pVM->pgm.s.fNestedPaging); + PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3); + + /* + * Update the shadow root page as well since that's not fixed. + */ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3); + PPGMPOOLPAGE pNewShwPageCR3; + + PGM_LOCK_VOID(pVM); + +# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + if (pPool->cDirtyPages) + pgmPoolResetDirtyPages(pVM); +# endif + + Assert(!(GCPhysCR3 >> (GUEST_PAGE_SHIFT + 32))); /** @todo what is this for? */ + int const rc2 = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, + PGM_A20_IS_ENABLED(pVCpu), NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/, &pNewShwPageCR3); + AssertFatalRC(rc2); + + pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3); + pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3); + + /* Set the current hypervisor CR3. */ + CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu)); + + /* Clean up the old CR3 root. */ + if ( pOldShwPageCR3 + && pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */) + { + Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE); + + /* Mark the page as unlocked; allow flushing again. */ + pgmPoolUnlockPage(pPool, pOldShwPageCR3); + + pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX); + } + PGM_UNLOCK(pVM); +# else + NOREF(GCPhysCR3); +# endif + + return rc; +} + +/** + * Unmaps the shadow CR3. + * + * @returns VBox status, no specials. + * @param pVCpu The cross context virtual CPU structure. + */ +PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu) +{ + LogFlow(("UnmapCR3\n")); + + int rc = VINF_SUCCESS; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM); + + /* + * Update guest paging info. + */ +#if PGM_GST_TYPE == PGM_TYPE_32BIT + pVCpu->pgm.s.pGst32BitPdR3 = 0; + pVCpu->pgm.s.pGst32BitPdR0 = 0; + +#elif PGM_GST_TYPE == PGM_TYPE_PAE + pVCpu->pgm.s.pGstPaePdptR3 = 0; + pVCpu->pgm.s.pGstPaePdptR0 = 0; + for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++) + { + pVCpu->pgm.s.apGstPaePDsR3[i] = 0; + pVCpu->pgm.s.apGstPaePDsR0[i] = 0; + pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS; + } + +#elif PGM_GST_TYPE == PGM_TYPE_AMD64 + pVCpu->pgm.s.pGstAmd64Pml4R3 = 0; + pVCpu->pgm.s.pGstAmd64Pml4R0 = 0; + +#else /* prot/real mode stub */ + /* nothing to do */ +#endif + + /* + * PAE PDPEs (and CR3) might have been mapped via PGMGstMapPaePdpesAtCr3() + * prior to switching to PAE in pfnMapCr3(), so we need to clear them here. + */ + pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false; + pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false; + pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS; + + /* + * Update shadow paging info. + */ +#if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \ + || PGM_SHW_TYPE == PGM_TYPE_PAE \ + || PGM_SHW_TYPE == PGM_TYPE_AMD64)) +# if PGM_GST_TYPE != PGM_TYPE_REAL + Assert(!pVM->pgm.s.fNestedPaging); +# endif + PGM_LOCK_VOID(pVM); + + if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)) + { + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + +# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + if (pPool->cDirtyPages) + pgmPoolResetDirtyPages(pVM); +# endif + + /* Mark the page as unlocked; allow flushing again. */ + pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); + + pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX); + pVCpu->pgm.s.pShwPageCR3R3 = 0; + pVCpu->pgm.s.pShwPageCR3R0 = 0; + } + + PGM_UNLOCK(pVM); +#endif + + return rc; +} + diff --git a/src/VBox/VMM/VMMAll/PGMAllGst.h b/src/VBox/VMM/VMMAll/PGMAllGst.h new file mode 100644 index 00000000..9bf00255 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PGMAllGst.h @@ -0,0 +1,545 @@ +/* $Id: PGMAllGst.h $ */ +/** @file + * VBox - Page Manager, Guest Paging Template - All context code. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +RT_C_DECLS_BEGIN +/** @todo Do we really need any of these forward declarations? */ +#if PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64 +DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PGSTPTWALK pGstWalk); +#endif +PGM_GST_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3); +PGM_GST_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk); +PGM_GST_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask); +PGM_GST_DECL(int, Exit)(PVMCPUCC pVCpu); + +#ifdef IN_RING3 /* r3 only for now. */ +PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta); +#endif +RT_C_DECLS_END + + +/** + * Enters the guest mode. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPhysCR3 The physical address from the CR3 register. + */ +PGM_GST_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3) +{ + /* + * Map and monitor CR3 + */ + uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE); + return g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3); +} + + +/** + * Exits the guest mode. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + */ +PGM_GST_DECL(int, Exit)(PVMCPUCC pVCpu) +{ + uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData; + AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE); + AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE); + return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu); +} + + +#if PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64 + + +DECLINLINE(int) PGM_GST_NAME(WalkReturnNotPresent)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, int iLevel) +{ + NOREF(iLevel); NOREF(pVCpu); + pWalk->fNotPresent = true; + pWalk->uLevel = (uint8_t)iLevel; + return VERR_PAGE_TABLE_NOT_PRESENT; +} + +DECLINLINE(int) PGM_GST_NAME(WalkReturnBadPhysAddr)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, int iLevel, int rc) +{ + AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); NOREF(pVCpu); + pWalk->fBadPhysAddr = true; + pWalk->uLevel = (uint8_t)iLevel; + return VERR_PAGE_TABLE_NOT_PRESENT; +} + +DECLINLINE(int) PGM_GST_NAME(WalkReturnRsvdError)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, int iLevel) +{ + NOREF(pVCpu); + pWalk->fRsvdError = true; + pWalk->uLevel = (uint8_t)iLevel; + return VERR_PAGE_TABLE_NOT_PRESENT; +} + + +/** + * Performs a guest page table walk. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPtr The guest virtual address to walk by. + * @param pWalk The page walk info. + * @param pGstWalk The guest mode specific page walk info. + */ +DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PGSTPTWALK pGstWalk) +{ + int rc; + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** @def PGM_GST_SLAT_WALK + * Macro to perform guest second-level address translation (EPT or Nested). + * + * @param a_pVCpu The cross context virtual CPU structure of the calling + * EMT. + * @param a_GCPtrNested The nested-guest linear address that caused the + * second-level translation. + * @param a_GCPhysNested The nested-guest physical address to translate. + * @param a_GCPhysOut Where to store the guest-physical address (result). + */ +# define PGM_GST_SLAT_WALK(a_pVCpu, a_GCPtrNested, a_GCPhysNested, a_GCPhysOut, a_pWalk) \ + do { \ + if ((a_pVCpu)->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) \ + { \ + PGMPTWALK WalkSlat; \ + PGMPTWALKGST WalkGstSlat; \ + int const rcX = pgmGstSlatWalk(a_pVCpu, a_GCPhysNested, true /* fIsLinearAddrValid */, a_GCPtrNested, &WalkSlat, \ + &WalkGstSlat); \ + if (RT_SUCCESS(rcX)) \ + (a_GCPhysOut) = WalkSlat.GCPhys; \ + else \ + { \ + *(a_pWalk) = WalkSlat; \ + return rcX; \ + } \ + } \ + } while (0) +#endif + + /* + * Init the walking structures. + */ + RT_ZERO(*pWalk); + RT_ZERO(*pGstWalk); + pWalk->GCPtr = GCPtr; + +# if PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_PAE + /* + * Boundary check for PAE and 32-bit (prevents trouble further down). + */ + if (RT_UNLIKELY(GCPtr >= _4G)) + return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 8); +# endif + + uint64_t fEffective; + { +# if PGM_GST_TYPE == PGM_TYPE_AMD64 + /* + * The PML4 table. + */ + rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGstWalk->pPml4); + if (RT_SUCCESS(rc)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 4, rc); + + PX86PML4E pPml4e; + pGstWalk->pPml4e = pPml4e = &pGstWalk->pPml4->a[(GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK]; + X86PML4E Pml4e; + pGstWalk->Pml4e.u = Pml4e.u = pPml4e->u; + + if (GST_IS_PGENTRY_PRESENT(pVCpu, Pml4e)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 4); + + if (RT_LIKELY(GST_IS_PML4E_VALID(pVCpu, Pml4e))) { /* likely */ } + else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 4); + + fEffective = Pml4e.u & ( X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_PWT | X86_PML4E_PCD | X86_PML4E_A + | X86_PML4E_NX); + pWalk->fEffective = fEffective; + + /* + * The PDPT. + */ + RTGCPHYS GCPhysPdpt = Pml4e.u & X86_PML4E_PG_MASK; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPdpt, GCPhysPdpt, pWalk); +#endif + rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPdpt, &pGstWalk->pPdpt); + if (RT_SUCCESS(rc)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc); + +# elif PGM_GST_TYPE == PGM_TYPE_PAE + rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGstWalk->pPdpt); + if (RT_SUCCESS(rc)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc); +#endif + } + { +# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE + PX86PDPE pPdpe; + pGstWalk->pPdpe = pPdpe = &pGstWalk->pPdpt->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK]; + X86PDPE Pdpe; + pGstWalk->Pdpe.u = Pdpe.u = pPdpe->u; + + if (GST_IS_PGENTRY_PRESENT(pVCpu, Pdpe)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 3); + + if (RT_LIKELY(GST_IS_PDPE_VALID(pVCpu, Pdpe))) { /* likely */ } + else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 3); + +# if PGM_GST_TYPE == PGM_TYPE_AMD64 + fEffective &= (Pdpe.u & ( X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US + | X86_PDPE_PWT | X86_PDPE_PCD | X86_PDPE_A)); + fEffective |= Pdpe.u & X86_PDPE_LM_NX; +# else + /* + * NX in the legacy-mode PAE PDPE is reserved. The valid check above ensures the NX bit is not set. + * The RW, US, A bits MBZ in PAE PDPTE entries but must be 1 the way we compute cumulative (effective) access rights. + */ + Assert(!(Pdpe.u & X86_PDPE_LM_NX)); + fEffective = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A + | (Pdpe.u & (X86_PDPE_PWT | X86_PDPE_PCD)); +# endif + pWalk->fEffective = fEffective; + + /* + * The PD. + */ + RTGCPHYS GCPhysPd = Pdpe.u & X86_PDPE_PG_MASK; +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPd, GCPhysPd, pWalk); +# endif + rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPd, &pGstWalk->pPd); + if (RT_SUCCESS(rc)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 2, rc); + +# elif PGM_GST_TYPE == PGM_TYPE_32BIT + rc = pgmGstGet32bitPDPtrEx(pVCpu, &pGstWalk->pPd); + if (RT_SUCCESS(rc)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc); +# endif + } + { + PGSTPDE pPde; + pGstWalk->pPde = pPde = &pGstWalk->pPd->a[(GCPtr >> GST_PD_SHIFT) & GST_PD_MASK]; + GSTPDE Pde; + pGstWalk->Pde.u = Pde.u = pPde->u; + if (GST_IS_PGENTRY_PRESENT(pVCpu, Pde)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 2); + if ((Pde.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu)) + { + if (RT_LIKELY(GST_IS_BIG_PDE_VALID(pVCpu, Pde))) { /* likely */ } + else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2); + + /* + * We're done. + */ +# if PGM_GST_TYPE == PGM_TYPE_32BIT + fEffective = Pde.u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PWT | X86_PDE4M_PCD | X86_PDE4M_A); +# else + fEffective &= Pde.u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PWT | X86_PDE4M_PCD | X86_PDE4M_A); + fEffective |= Pde.u & X86_PDE2M_PAE_NX; +# endif + fEffective |= Pde.u & (X86_PDE4M_D | X86_PDE4M_G); + fEffective |= (Pde.u & X86_PDE4M_PAT) >> X86_PDE4M_PAT_SHIFT; + pWalk->fEffective = fEffective; + Assert(GST_IS_NX_ACTIVE(pVCpu) || !(fEffective & PGM_PTATTRS_NX_MASK)); + Assert(fEffective & PGM_PTATTRS_R_MASK); + + pWalk->fBigPage = true; + pWalk->fSucceeded = true; + RTGCPHYS GCPhysPde = GST_GET_BIG_PDE_GCPHYS(pVCpu->CTX_SUFF(pVM), Pde) + | (GCPtr & GST_BIG_PAGE_OFFSET_MASK); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPde, GCPhysPde, pWalk); +# endif + pWalk->GCPhys = GCPhysPde; + PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys); + return VINF_SUCCESS; + } + + if (RT_UNLIKELY(!GST_IS_PDE_VALID(pVCpu, Pde))) + return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2); +# if PGM_GST_TYPE == PGM_TYPE_32BIT + fEffective = Pde.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD | X86_PDE_A); +# else + fEffective &= Pde.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD | X86_PDE_A); + fEffective |= Pde.u & X86_PDE_PAE_NX; +# endif + pWalk->fEffective = fEffective; + + /* + * The PT. + */ + RTGCPHYS GCPhysPt = GST_GET_PDE_GCPHYS(Pde); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPt, GCPhysPt, pWalk); +# endif + rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPt, &pGstWalk->pPt); + if (RT_SUCCESS(rc)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 1, rc); + } + { + PGSTPTE pPte; + pGstWalk->pPte = pPte = &pGstWalk->pPt->a[(GCPtr >> GST_PT_SHIFT) & GST_PT_MASK]; + GSTPTE Pte; + pGstWalk->Pte.u = Pte.u = pPte->u; + + if (GST_IS_PGENTRY_PRESENT(pVCpu, Pte)) { /* probable */ } + else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 1); + + if (RT_LIKELY(GST_IS_PTE_VALID(pVCpu, Pte))) { /* likely */ } + else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 1); + + /* + * We're done. + */ + fEffective &= Pte.u & (X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A); + fEffective |= Pte.u & (X86_PTE_D | X86_PTE_PAT | X86_PTE_G); +# if PGM_GST_TYPE != PGM_TYPE_32BIT + fEffective |= Pte.u & X86_PTE_PAE_NX; +# endif + pWalk->fEffective = fEffective; + Assert(GST_IS_NX_ACTIVE(pVCpu) || !(fEffective & PGM_PTATTRS_NX_MASK)); + Assert(fEffective & PGM_PTATTRS_R_MASK); + + pWalk->fSucceeded = true; + RTGCPHYS GCPhysPte = GST_GET_PTE_GCPHYS(Pte) + | (GCPtr & GUEST_PAGE_OFFSET_MASK); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPte, GCPhysPte, pWalk); +# endif + pWalk->GCPhys = GCPhysPte; + return VINF_SUCCESS; + } +} + +#endif /* 32BIT, PAE, AMD64 */ + +/** + * Gets effective Guest OS page information. + * + * When GCPtr is in a big page, the function will return as if it was a normal + * 4KB page. If the need for distinguishing between big and normal page becomes + * necessary at a later point, a PGMGstGetPage Ex() will be created for that + * purpose. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Guest Context virtual address of the page. + * @param pWalk Where to store the page walk info. + */ +PGM_GST_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk) +{ +#if PGM_GST_TYPE == PGM_TYPE_REAL \ + || PGM_GST_TYPE == PGM_TYPE_PROT + + RT_ZERO(*pWalk); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) + { + PGMPTWALK WalkSlat; + PGMPTWALKGST WalkGstSlat; + int const rc = pgmGstSlatWalk(pVCpu, GCPtr, true /* fIsLinearAddrValid */, GCPtr, &WalkSlat, &WalkGstSlat); + if (RT_SUCCESS(rc)) + { + pWalk->fSucceeded = true; + pWalk->GCPtr = GCPtr; + pWalk->GCPhys = WalkSlat.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + pWalk->fEffective = X86_PTE_P | X86_PTE_RW | X86_PTE_US; + } + else + *pWalk = WalkSlat; + return rc; + } +# endif + + /* + * Fake it. + */ + pWalk->fSucceeded = true; + pWalk->GCPtr = GCPtr; + pWalk->GCPhys = GCPtr & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + pWalk->fEffective = X86_PTE_P | X86_PTE_RW | X86_PTE_US; + NOREF(pVCpu); + return VINF_SUCCESS; + +#elif PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64 + + GSTPTWALK GstWalk; + int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, pWalk, &GstWalk); + if (RT_FAILURE(rc)) + return rc; + + Assert(pWalk->fSucceeded); + Assert(pWalk->GCPtr == GCPtr); + + PGMPTATTRS fFlags; + if (!pWalk->fBigPage) + fFlags = (GstWalk.Pte.u & ~(GST_PTE_PG_MASK | X86_PTE_RW | X86_PTE_US)) /* NX not needed */ + | (pWalk->fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) +# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE) + | (pWalk->fEffective & PGM_PTATTRS_NX_MASK) +# endif + ; + else + { + fFlags = (GstWalk.Pde.u & ~(GST_PTE_PG_MASK | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PS)) /* NX not needed */ + | (pWalk->fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK | PGM_PTATTRS_PAT_MASK)) +# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE) + | (pWalk->fEffective & PGM_PTATTRS_NX_MASK) +# endif + ; + } + + pWalk->GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + pWalk->fEffective = fFlags; + return VINF_SUCCESS; + +#else +# error "shouldn't be here!" + /* something else... */ + return VERR_NOT_SUPPORTED; +#endif +} + + +/** + * Modify page flags for a range of pages in the guest's tables + * + * The existing flags are ANDed with the fMask and ORed with the fFlags. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Virtual address of the first page in the range. Page aligned! + * @param cb Size (in bytes) of the page range to apply the modification to. Page aligned! + * @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course. + * @param fMask The AND mask - page flags X86_PTE_*. + */ +PGM_GST_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask) +{ + Assert((cb & GUEST_PAGE_OFFSET_MASK) == 0); RT_NOREF_PV(cb); + +#if PGM_GST_TYPE == PGM_TYPE_32BIT \ + || PGM_GST_TYPE == PGM_TYPE_PAE \ + || PGM_GST_TYPE == PGM_TYPE_AMD64 + for (;;) + { + PGMPTWALK Walk; + GSTPTWALK GstWalk; + int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, &Walk, &GstWalk); + if (RT_FAILURE(rc)) + return rc; + + if (!Walk.fBigPage) + { + /* + * 4KB Page table, process + * + * Walk pages till we're done. + */ + unsigned iPTE = (GCPtr >> GST_PT_SHIFT) & GST_PT_MASK; + while (iPTE < RT_ELEMENTS(GstWalk.pPt->a)) + { + GSTPTE Pte = GstWalk.pPt->a[iPTE]; + Pte.u = (Pte.u & (fMask | X86_PTE_PAE_PG_MASK)) + | (fFlags & ~GST_PTE_PG_MASK); + GstWalk.pPt->a[iPTE] = Pte; + + /* next page */ + cb -= GUEST_PAGE_SIZE; + if (!cb) + return VINF_SUCCESS; + GCPtr += GUEST_PAGE_SIZE; + iPTE++; + } + } + else + { + /* + * 2/4MB Page table + */ + GSTPDE PdeNew; +# if PGM_GST_TYPE == PGM_TYPE_32BIT + PdeNew.u = (GstWalk.Pde.u & (fMask | ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) | GST_PDE_BIG_PG_MASK | X86_PDE4M_PG_HIGH_MASK | X86_PDE4M_PS)) +# else + PdeNew.u = (GstWalk.Pde.u & (fMask | ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) | GST_PDE_BIG_PG_MASK | X86_PDE4M_PS)) +# endif + | (fFlags & ~GST_PTE_PG_MASK) + | ((fFlags & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT); + *GstWalk.pPde = PdeNew; + + /* advance */ + const unsigned cbDone = GST_BIG_PAGE_SIZE - (GCPtr & GST_BIG_PAGE_OFFSET_MASK); + if (cbDone >= cb) + return VINF_SUCCESS; + cb -= cbDone; + GCPtr += cbDone; + } + } + +#else + /* real / protected mode: ignore. */ + NOREF(pVCpu); NOREF(GCPtr); NOREF(fFlags); NOREF(fMask); + return VINF_SUCCESS; +#endif +} + + +#ifdef IN_RING3 +/** + * Relocate any GC pointers related to guest mode paging. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param offDelta The relocation offset. + */ +PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta) +{ + RT_NOREF(pVCpu, offDelta); + return VINF_SUCCESS; +} +#endif diff --git a/src/VBox/VMM/VMMAll/PGMAllGstSlatEpt.cpp.h b/src/VBox/VMM/VMMAll/PGMAllGstSlatEpt.cpp.h new file mode 100644 index 00000000..410abae1 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PGMAllGstSlatEpt.cpp.h @@ -0,0 +1,408 @@ +/* $Id: PGMAllGstSlatEpt.cpp.h $ */ +/** @file + * VBox - Page Manager, Guest EPT SLAT - All context code. + */ + +/* + * Copyright (C) 2021-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + +#if PGM_SLAT_TYPE != PGM_SLAT_TYPE_EPT +# error "Unsupported SLAT type." +#endif + +/** + * Checks if the EPT PTE permissions are valid. + * + * @returns @c true if valid, @c false otherwise. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param uEntry The EPT page table entry to check. + */ +DECLINLINE(bool) PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(PCVMCPUCC pVCpu, uint64_t uEntry) +{ + if (!(uEntry & EPT_E_READ)) + { + Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt); + Assert(!RT_BF_GET(pVCpu->pgm.s.uEptVpidCapMsr, VMX_BF_EPT_VPID_CAP_EXEC_ONLY)); + NOREF(pVCpu); + if (uEntry & (EPT_E_WRITE | EPT_E_EXECUTE)) + return false; + } + return true; +} + + +/** + * Checks if the EPT memory type is valid. + * + * @returns @c true if valid, @c false otherwise. + * @param uEntry The EPT page table entry to check. + * @param uLevel The page table walk level. + */ +DECLINLINE(bool) PGM_GST_SLAT_NAME_EPT(WalkIsMemTypeValid)(uint64_t uEntry, uint8_t uLevel) +{ + Assert(uLevel <= 3 && uLevel >= 1); NOREF(uLevel); + uint8_t const fEptMemTypeMask = uEntry & VMX_BF_EPT_PT_MEMTYPE_MASK; + switch (fEptMemTypeMask) + { + case EPT_E_MEMTYPE_WB: + case EPT_E_MEMTYPE_UC: + case EPT_E_MEMTYPE_WP: + case EPT_E_MEMTYPE_WT: + case EPT_E_MEMTYPE_WC: + return true; + } + return false; +} + + +/** + * Updates page walk result info when a not-present page is encountered. + * + * @returns VERR_PAGE_TABLE_NOT_PRESENT. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pWalk The page walk info to update. + * @param uEntry The EPT PTE that is not present. + * @param uLevel The page table walk level. + */ +DECLINLINE(int) PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(PCVMCPUCC pVCpu, PPGMPTWALK pWalk, uint64_t uEntry, uint8_t uLevel) +{ + static PGMWALKFAIL const s_afEptViolations[] = { PGM_WALKFAIL_EPT_VIOLATION, PGM_WALKFAIL_EPT_VIOLATION_CONVERTIBLE }; + uint8_t const fEptVeSupported = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxEptXcptVe; + uint8_t const fConvertible = RT_BOOL(uLevel == 1 || (uEntry & EPT_E_BIT_LEAF)); + uint8_t const idxViolationType = fEptVeSupported & fConvertible & !RT_BF_GET(uEntry, VMX_BF_EPT_PT_SUPPRESS_VE); + + pWalk->fNotPresent = true; + pWalk->uLevel = uLevel; + pWalk->fFailed = s_afEptViolations[idxViolationType]; + return VERR_PAGE_TABLE_NOT_PRESENT; +} + + +/** + * Updates page walk result info when a bad physical address is encountered. + * + * @returns VERR_PAGE_TABLE_NOT_PRESENT . + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pWalk The page walk info to update. + * @param uLevel The page table walk level. + * @param rc The error code that caused this bad physical address situation. + */ +DECLINLINE(int) PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(PCVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel, int rc) +{ + AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); NOREF(pVCpu); + pWalk->fBadPhysAddr = true; + pWalk->uLevel = uLevel; + pWalk->fFailed = PGM_WALKFAIL_EPT_VIOLATION; + return VERR_PAGE_TABLE_NOT_PRESENT; +} + + +/** + * Updates page walk result info when reserved bits are encountered. + * + * @returns VERR_PAGE_TABLE_NOT_PRESENT. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pWalk The page walk info to update. + * @param uLevel The page table walk level. + */ +DECLINLINE(int) PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel) +{ + NOREF(pVCpu); + pWalk->fRsvdError = true; + pWalk->uLevel = uLevel; + pWalk->fFailed = PGM_WALKFAIL_EPT_MISCONFIG; + return VERR_PAGE_TABLE_NOT_PRESENT; +} + + +/** + * Walks the guest's EPT page table (second-level address translation). + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPhysNested The nested-guest physical address to walk. + * @param fIsLinearAddrValid Whether the linear-address in @c GCPtrNested caused + * this page walk. + * @param GCPtrNested The nested-guest linear address that caused this + * translation. If @c fIsLinearAddrValid is false, pass + * 0. + * @param pWalk The page walk info. + * @param pSlatWalk The SLAT mode specific page walk info. + */ +DECLINLINE(int) PGM_GST_SLAT_NAME_EPT(Walk)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, + PPGMPTWALK pWalk, PSLATPTWALK pSlatWalk) +{ + Assert(fIsLinearAddrValid || GCPtrNested == 0); + + /* + * Init walk structures. + */ + RT_ZERO(*pWalk); + RT_ZERO(*pSlatWalk); + + pWalk->GCPtr = GCPtrNested; + pWalk->GCPhysNested = GCPhysNested; + pWalk->fIsLinearAddrValid = fIsLinearAddrValid; + pWalk->fIsSlat = true; + + /* + * Figure out EPT attributes that are cumulative (logical-AND) across page walks. + * - R, W, X_SUPER are unconditionally cumulative. + * See Intel spec. Table 26-7 "Exit Qualification for EPT Violations". + * + * - X_USER is cumulative but relevant only when mode-based execute control for EPT + * which we currently don't support it (asserted below). + * + * - MEMTYPE is not cumulative and only applicable to the final paging entry. + * + * - A, D EPT bits map to the regular page-table bit positions. Thus, they're not + * included in the mask below and handled separately. Accessed bits are + * cumulative but dirty bits are not cumulative as they're only applicable to + * the final paging entry. + */ + Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt); + uint64_t const fEptAndMask = ( PGM_PTATTRS_EPT_R_MASK + | PGM_PTATTRS_EPT_W_MASK + | PGM_PTATTRS_EPT_X_SUPER_MASK) & PGM_PTATTRS_EPT_MASK; + + /* + * Do the walk. + */ + uint64_t fEffective; + { + /* + * EPTP. + * + * We currently only support 4-level EPT paging. + * EPT 5-level paging was documented at some point (bit 7 of MSR_IA32_VMX_EPT_VPID_CAP) + * but for some reason seems to have been removed from subsequent specs. + */ + int const rc = pgmGstGetEptPML4PtrEx(pVCpu, &pSlatWalk->pPml4); + if (RT_SUCCESS(rc)) + { /* likely */ } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(pVCpu, pWalk, 4, rc); + } + { + /* + * PML4E. + */ + PEPTPML4E pPml4e; + pSlatWalk->pPml4e = pPml4e = &pSlatWalk->pPml4->a[(GCPhysNested >> SLAT_PML4_SHIFT) & SLAT_PML4_MASK]; + EPTPML4E Pml4e; + pSlatWalk->Pml4e.u = Pml4e.u = pPml4e->u; + + if (SLAT_IS_PGENTRY_PRESENT(pVCpu, Pml4e)) { /* probable */ } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(pVCpu, pWalk, Pml4e.u, 4); + + if (RT_LIKELY( SLAT_IS_PML4E_VALID(pVCpu, Pml4e) + && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pml4e.u))) + { /* likely */ } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(pVCpu, pWalk, 4); + + uint64_t const fEptAttrs = Pml4e.u & EPT_PML4E_ATTR_MASK; + uint8_t const fRead = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ); + uint8_t const fWrite = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE); + uint8_t const fExecute = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE); + uint8_t const fAccessed = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED); + uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask; + fEffective = RT_BF_MAKE(PGM_PTATTRS_R, fRead) + | RT_BF_MAKE(PGM_PTATTRS_W, fWrite) + | RT_BF_MAKE(PGM_PTATTRS_NX, !fExecute) + | RT_BF_MAKE(PGM_PTATTRS_A, fAccessed) + | fEptAndBits; + pWalk->fEffective = fEffective; + + int const rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pml4e.u & EPT_PML4E_PG_MASK, &pSlatWalk->pPdpt); + if (RT_SUCCESS(rc)) { /* probable */ } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc); + } + { + /* + * PDPTE. + */ + PEPTPDPTE pPdpte; + pSlatWalk->pPdpte = pPdpte = &pSlatWalk->pPdpt->a[(GCPhysNested >> SLAT_PDPT_SHIFT) & SLAT_PDPT_MASK]; + EPTPDPTE Pdpte; + pSlatWalk->Pdpte.u = Pdpte.u = pPdpte->u; + + if (SLAT_IS_PGENTRY_PRESENT(pVCpu, Pdpte)) { /* probable */ } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(pVCpu, pWalk, Pdpte.u, 3); + + /* The order of the following "if" and "else if" statements matter. */ + if ( SLAT_IS_PDPE_VALID(pVCpu, Pdpte) + && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pdpte.u)) + { + uint64_t const fEptAttrs = Pdpte.u & EPT_PDPTE_ATTR_MASK; + uint8_t const fRead = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ); + uint8_t const fWrite = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE); + uint8_t const fExecute = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE); + uint8_t const fAccessed = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED); + uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask; + fEffective &= RT_BF_MAKE(PGM_PTATTRS_R, fRead) + | RT_BF_MAKE(PGM_PTATTRS_W, fWrite) + | RT_BF_MAKE(PGM_PTATTRS_A, fAccessed) + | fEptAndBits; + fEffective |= RT_BF_MAKE(PGM_PTATTRS_NX, !fExecute); + pWalk->fEffective = fEffective; + } + else if ( SLAT_IS_BIG_PDPE_VALID(pVCpu, Pdpte) + && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pdpte.u) + && PGM_GST_SLAT_NAME_EPT(WalkIsMemTypeValid)(Pdpte.u, 3)) + { + uint64_t const fEptAttrs = Pdpte.u & EPT_PDPTE1G_ATTR_MASK; + uint8_t const fRead = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ); + uint8_t const fWrite = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE); + uint8_t const fExecute = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE); + uint8_t const fAccessed = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED); + uint8_t const fDirty = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_DIRTY); + uint8_t const fMemType = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_MEMTYPE); + uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask; + fEffective &= RT_BF_MAKE(PGM_PTATTRS_R, fRead) + | RT_BF_MAKE(PGM_PTATTRS_W, fWrite) + | RT_BF_MAKE(PGM_PTATTRS_A, fAccessed) + | fEptAndBits; + fEffective |= RT_BF_MAKE(PGM_PTATTRS_D, fDirty) + | RT_BF_MAKE(PGM_PTATTRS_EPT_MEMTYPE, fMemType) + | RT_BF_MAKE(PGM_PTATTRS_NX, !fExecute); + pWalk->fEffective = fEffective; + + pWalk->fGigantPage = true; + pWalk->fSucceeded = true; + pWalk->GCPhys = SLAT_GET_PDPE1G_GCPHYS(pVCpu, Pdpte) + | (GCPhysNested & SLAT_PAGE_1G_OFFSET_MASK); + PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys); + return VINF_SUCCESS; + } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(pVCpu, pWalk, 3); + + int const rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pdpte.u & EPT_PDPTE_PG_MASK, &pSlatWalk->pPd); + if (RT_SUCCESS(rc)) { /* probable */ } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc); + } + { + /* + * PDE. + */ + PSLATPDE pPde; + pSlatWalk->pPde = pPde = &pSlatWalk->pPd->a[(GCPhysNested >> SLAT_PD_SHIFT) & SLAT_PD_MASK]; + SLATPDE Pde; + pSlatWalk->Pde.u = Pde.u = pPde->u; + + if (SLAT_IS_PGENTRY_PRESENT(pVCpu, Pde)) { /* probable */ } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(pVCpu, pWalk, Pde.u, 2); + + /* The order of the following "if" and "else if" statements matter. */ + if ( SLAT_IS_PDE_VALID(pVCpu, Pde) + && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pde.u)) + { + uint64_t const fEptAttrs = Pde.u & EPT_PDE_ATTR_MASK; + uint8_t const fRead = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ); + uint8_t const fWrite = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE); + uint8_t const fExecute = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE); + uint8_t const fAccessed = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED); + uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask; + fEffective &= RT_BF_MAKE(PGM_PTATTRS_R, fRead) + | RT_BF_MAKE(PGM_PTATTRS_W, fWrite) + | RT_BF_MAKE(PGM_PTATTRS_A, fAccessed) + | fEptAndBits; + fEffective |= RT_BF_MAKE(PGM_PTATTRS_NX, !fExecute); + pWalk->fEffective = fEffective; + } + else if ( SLAT_IS_BIG_PDE_VALID(pVCpu, Pde) + && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pde.u) + && PGM_GST_SLAT_NAME_EPT(WalkIsMemTypeValid)(Pde.u, 2)) + { + uint64_t const fEptAttrs = Pde.u & EPT_PDE2M_ATTR_MASK; + uint8_t const fRead = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ); + uint8_t const fWrite = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE); + uint8_t const fExecute = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE); + uint8_t const fAccessed = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED); + uint8_t const fDirty = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_DIRTY); + uint8_t const fMemType = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_MEMTYPE); + uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask; + fEffective &= RT_BF_MAKE(PGM_PTATTRS_R, fRead) + | RT_BF_MAKE(PGM_PTATTRS_W, fWrite) + | RT_BF_MAKE(PGM_PTATTRS_A, fAccessed) + | fEptAndBits; + fEffective |= RT_BF_MAKE(PGM_PTATTRS_D, fDirty) + | RT_BF_MAKE(PGM_PTATTRS_EPT_MEMTYPE, fMemType) + | RT_BF_MAKE(PGM_PTATTRS_NX, !fExecute); + pWalk->fEffective = fEffective; + + pWalk->fBigPage = true; + pWalk->fSucceeded = true; + pWalk->GCPhys = SLAT_GET_PDE2M_GCPHYS(pVCpu, Pde) + | (GCPhysNested & SLAT_PAGE_2M_OFFSET_MASK); + PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys); + return VINF_SUCCESS; + } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(pVCpu, pWalk, 2); + + int const rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pde.u & EPT_PDE_PG_MASK, &pSlatWalk->pPt); + if (RT_SUCCESS(rc)) { /* probable */ } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(pVCpu, pWalk, 1, rc); + } + { + /* + * PTE. + */ + PSLATPTE pPte; + pSlatWalk->pPte = pPte = &pSlatWalk->pPt->a[(GCPhysNested >> SLAT_PT_SHIFT) & SLAT_PT_MASK]; + SLATPTE Pte; + pSlatWalk->Pte.u = Pte.u = pPte->u; + + if (SLAT_IS_PGENTRY_PRESENT(pVCpu, Pte)) { /* probable */ } + else return PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(pVCpu, pWalk, Pte.u, 1); + + if ( SLAT_IS_PTE_VALID(pVCpu, Pte) + && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pte.u) + && PGM_GST_SLAT_NAME_EPT(WalkIsMemTypeValid)(Pte.u, 1)) + { /* likely*/ } + else + return PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(pVCpu, pWalk, 1); + + uint64_t const fEptAttrs = Pte.u & EPT_PTE_ATTR_MASK; + uint8_t const fRead = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ); + uint8_t const fWrite = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE); + uint8_t const fExecute = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE); + uint8_t const fAccessed = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED); + uint8_t const fDirty = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_DIRTY); + uint8_t const fMemType = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_MEMTYPE); + uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask; + fEffective &= RT_BF_MAKE(PGM_PTATTRS_R, fRead) + | RT_BF_MAKE(PGM_PTATTRS_W, fWrite) + | RT_BF_MAKE(PGM_PTATTRS_A, fAccessed) + | fEptAndBits; + fEffective |= RT_BF_MAKE(PGM_PTATTRS_D, fDirty) + | RT_BF_MAKE(PGM_PTATTRS_EPT_MEMTYPE, fMemType) + | RT_BF_MAKE(PGM_PTATTRS_NX, !fExecute); + pWalk->fEffective = fEffective; + + pWalk->fSucceeded = true; + pWalk->GCPhys = SLAT_GET_PTE_GCPHYS(pVCpu, Pte) | (GCPhysNested & GUEST_PAGE_OFFSET_MASK); + return VINF_SUCCESS; + } +} + diff --git a/src/VBox/VMM/VMMAll/PGMAllHandler.cpp b/src/VBox/VMM/VMMAll/PGMAllHandler.cpp new file mode 100644 index 00000000..c7d8a433 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PGMAllHandler.cpp @@ -0,0 +1,2001 @@ +/* $Id: PGMAllHandler.cpp $ */ +/** @file + * PGM - Page Manager / Monitor, Access Handlers. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PGM +#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef IN_RING0 +# include +#endif +#include "PGMInternal.h" +#include +#include "PGMInline.h" + +#include +#include +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +# include +#endif +#include +#include +#include +#include + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +/** Dummy physical access handler type record. */ +CTX_SUFF(PGMPHYSHANDLERTYPEINT) const g_pgmHandlerPhysicalDummyType = +{ + /* .hType = */ UINT64_C(0x93b7557e1937aaff), + /* .enmKind = */ PGMPHYSHANDLERKIND_INVALID, + /* .uState = */ PGM_PAGE_HNDL_PHYS_STATE_ALL, + /* .fKeepPgmLock = */ true, + /* .fRing0DevInsIdx = */ false, +#ifdef IN_RING0 + /* .fNotInHm = */ false, + /* .pfnHandler = */ pgmR0HandlerPhysicalHandlerToRing3, + /* .pfnPfHandler = */ pgmR0HandlerPhysicalPfHandlerToRing3, +#elif defined(IN_RING3) + /* .fRing0Enabled = */ false, + /* .fNotInHm = */ false, + /* .pfnHandler = */ pgmR3HandlerPhysicalHandlerInvalid, +#else +# error "unsupported context" +#endif + /* .pszDesc = */ "dummy" +}; + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +static int pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(PVMCC pVM, PPGMPHYSHANDLER pCur, PPGMRAMRANGE pRam, + void *pvBitmap, uint32_t offBitmap); +static void pgmHandlerPhysicalDeregisterNotifyNEM(PVMCC pVM, PPGMPHYSHANDLER pCur); +static void pgmHandlerPhysicalResetRamFlags(PVMCC pVM, PPGMPHYSHANDLER pCur); + + +#ifndef IN_RING3 + +/** + * @callback_method_impl{FNPGMPHYSHANDLER, + * Dummy for forcing ring-3 handling of the access.} + */ +DECLCALLBACK(VBOXSTRICTRC) +pgmR0HandlerPhysicalHandlerToRing3(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, + PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser) +{ + RT_NOREF(pVM, pVCpu, GCPhys, pvPhys, pvBuf, cbBuf, enmAccessType, enmOrigin, uUser); + return VINF_EM_RAW_EMULATE_INSTR; +} + + +/** + * @callback_method_impl{FNPGMRZPHYSPFHANDLER, + * Dummy for forcing ring-3 handling of the access.} + */ +DECLCALLBACK(VBOXSTRICTRC) +pgmR0HandlerPhysicalPfHandlerToRing3(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx, + RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser) +{ + RT_NOREF(pVM, pVCpu, uErrorCode, pCtx, pvFault, GCPhysFault, uUser); + return VINF_EM_RAW_EMULATE_INSTR; +} + +#endif /* !IN_RING3 */ + + +/** + * Creates a physical access handler, allocation part. + * + * @returns VBox status code. + * @retval VERR_OUT_OF_RESOURCES if no more handlers available. + * + * @param pVM The cross context VM structure. + * @param hType The handler type registration handle. + * @param uUser User argument to the handlers (not pointer). + * @param pszDesc Description of this handler. If NULL, the type + * description will be used instead. + * @param ppPhysHandler Where to return the access handler structure on + * success. + */ +int pgmHandlerPhysicalExCreate(PVMCC pVM, PGMPHYSHANDLERTYPE hType, uint64_t uUser, + R3PTRTYPE(const char *) pszDesc, PPGMPHYSHANDLER *ppPhysHandler) +{ + /* + * Validate input. + */ + PCPGMPHYSHANDLERTYPEINT const pType = pgmHandlerPhysicalTypeHandleToPtr(pVM, hType); + AssertReturn(pType, VERR_INVALID_HANDLE); + AssertReturn(pType->enmKind > PGMPHYSHANDLERKIND_INVALID && pType->enmKind < PGMPHYSHANDLERKIND_END, VERR_INVALID_HANDLE); + AssertPtr(ppPhysHandler); + + Log(("pgmHandlerPhysicalExCreate: uUser=%#RX64 hType=%#x (%d, %s) pszDesc=%RHv:%s\n", + uUser, hType, pType->enmKind, pType->pszDesc, pszDesc, R3STRING(pszDesc))); + + /* + * Allocate and initialize the new entry. + */ + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + PPGMPHYSHANDLER pNew = pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator.allocateNode(); + if (pNew) + { + pNew->Key = NIL_RTGCPHYS; + pNew->KeyLast = NIL_RTGCPHYS; + pNew->cPages = 0; + pNew->cAliasedPages = 0; + pNew->cTmpOffPages = 0; + pNew->uUser = uUser; + pNew->hType = hType; + pNew->pszDesc = pszDesc != NIL_RTR3PTR ? pszDesc +#ifdef IN_RING3 + : pType->pszDesc; +#else + : pVM->pgm.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK].pszDesc; +#endif + + PGM_UNLOCK(pVM); + *ppPhysHandler = pNew; + return VINF_SUCCESS; + } + + PGM_UNLOCK(pVM); + return VERR_OUT_OF_RESOURCES; +} + + +/** + * Duplicates a physical access handler. + * + * @returns VBox status code. + * @retval VINF_SUCCESS when successfully installed. + * + * @param pVM The cross context VM structure. + * @param pPhysHandlerSrc The source handler to duplicate + * @param ppPhysHandler Where to return the access handler structure on + * success. + */ +int pgmHandlerPhysicalExDup(PVMCC pVM, PPGMPHYSHANDLER pPhysHandlerSrc, PPGMPHYSHANDLER *ppPhysHandler) +{ + return pgmHandlerPhysicalExCreate(pVM, pPhysHandlerSrc->hType, pPhysHandlerSrc->uUser, + pPhysHandlerSrc->pszDesc, ppPhysHandler); +} + + +/** + * Register a access handler for a physical range. + * + * @returns VBox status code. + * @retval VINF_SUCCESS when successfully installed. + * @retval VINF_PGM_GCPHYS_ALIASED could be returned. + * + * @param pVM The cross context VM structure. + * @param pPhysHandler The physical handler. + * @param GCPhys Start physical address. + * @param GCPhysLast Last physical address. (inclusive) + */ +int pgmHandlerPhysicalExRegister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast) +{ + /* + * Validate input. + */ + AssertReturn(pPhysHandler, VERR_INVALID_POINTER); + PGMPHYSHANDLERTYPE const hType = pPhysHandler->hType; + PCPGMPHYSHANDLERTYPEINT const pType = pgmHandlerPhysicalTypeHandleToPtr(pVM, hType); + AssertReturn(pType, VERR_INVALID_HANDLE); + AssertReturn(pType->enmKind > PGMPHYSHANDLERKIND_INVALID && pType->enmKind < PGMPHYSHANDLERKIND_END, VERR_INVALID_HANDLE); + + AssertPtr(pPhysHandler); + + Log(("pgmHandlerPhysicalExRegister: GCPhys=%RGp GCPhysLast=%RGp hType=%#x (%d, %s) pszDesc=%RHv:%s\n", GCPhys, GCPhysLast, + hType, pType->enmKind, pType->pszDesc, pPhysHandler->pszDesc, R3STRING(pPhysHandler->pszDesc))); + AssertReturn(pPhysHandler->Key == NIL_RTGCPHYS, VERR_WRONG_ORDER); + + AssertMsgReturn(GCPhys < GCPhysLast, ("GCPhys >= GCPhysLast (%#x >= %#x)\n", GCPhys, GCPhysLast), VERR_INVALID_PARAMETER); + Assert(GCPhysLast - GCPhys < _4G); /* ASSUMPTION in PGMAllPhys.cpp */ + + switch (pType->enmKind) + { + case PGMPHYSHANDLERKIND_WRITE: + if (!pType->fNotInHm) + break; + RT_FALL_THRU(); /* Simplification: fNotInHm can only be used with full pages */ + case PGMPHYSHANDLERKIND_MMIO: + case PGMPHYSHANDLERKIND_ALL: + /* Simplification for PGMPhysRead, PGMR0Trap0eHandlerNPMisconfig and others: Full pages. */ + AssertMsgReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), ("%RGp\n", GCPhys), VERR_INVALID_PARAMETER); + AssertMsgReturn((GCPhysLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK, ("%RGp\n", GCPhysLast), VERR_INVALID_PARAMETER); + break; + default: + AssertMsgFailed(("Invalid input enmKind=%d!\n", pType->enmKind)); + return VERR_INVALID_PARAMETER; + } + + /* + * We require the range to be within registered ram. + * There is no apparent need to support ranges which cover more than one ram range. + */ + PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys); + if ( !pRam + || GCPhysLast > pRam->GCPhysLast) + { +#ifdef IN_RING3 + DBGFR3Info(pVM->pUVM, "phys", NULL, NULL); +#endif + AssertMsgFailed(("No RAM range for %RGp-%RGp\n", GCPhys, GCPhysLast)); + return VERR_PGM_HANDLER_PHYSICAL_NO_RAM_RANGE; + } + Assert(GCPhys >= pRam->GCPhys && GCPhys < pRam->GCPhysLast); + Assert(GCPhysLast <= pRam->GCPhysLast && GCPhysLast >= pRam->GCPhys); + + /* + * Try insert into list. + */ + pPhysHandler->Key = GCPhys; + pPhysHandler->KeyLast = GCPhysLast; + pPhysHandler->cPages = (GCPhysLast - (GCPhys & X86_PTE_PAE_PG_MASK) + GUEST_PAGE_SIZE) >> GUEST_PAGE_SHIFT; + + int rc = PGM_LOCK(pVM); + if (RT_SUCCESS(rc)) + { + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->insert(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, pPhysHandler); + if (RT_SUCCESS(rc)) + { + rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pPhysHandler, pRam, NULL /*pvBitmap*/, 0 /*offBitmap*/); + if (rc == VINF_PGM_SYNC_CR3) + rc = VINF_PGM_GCPHYS_ALIASED; + +#if defined(IN_RING3) || defined(IN_RING0) + NEMHCNotifyHandlerPhysicalRegister(pVM, pType->enmKind, GCPhys, GCPhysLast - GCPhys + 1); +#endif + PGM_UNLOCK(pVM); + + if (rc != VINF_SUCCESS) + Log(("PGMHandlerPhysicalRegisterEx: returns %Rrc (%RGp-%RGp)\n", rc, GCPhys, GCPhysLast)); + return rc; + } + PGM_UNLOCK(pVM); + } + + pPhysHandler->Key = NIL_RTGCPHYS; + pPhysHandler->KeyLast = NIL_RTGCPHYS; + + AssertMsgReturn(rc == VERR_ALREADY_EXISTS, ("%Rrc GCPhys=%RGp GCPhysLast=%RGp\n", rc, GCPhys, GCPhysLast), rc); + +#if defined(IN_RING3) && defined(VBOX_STRICT) + DBGFR3Info(pVM->pUVM, "handlers", "phys nostats", NULL); +#endif + AssertMsgFailed(("Conflict! GCPhys=%RGp GCPhysLast=%RGp pszDesc=%s/%s\n", + GCPhys, GCPhysLast, R3STRING(pPhysHandler->pszDesc), R3STRING(pType->pszDesc))); + return VERR_PGM_HANDLER_PHYSICAL_CONFLICT; +} + + +/** + * Register a access handler for a physical range. + * + * @returns VBox status code. + * @retval VINF_SUCCESS when successfully installed. + * @retval VINF_PGM_GCPHYS_ALIASED when the shadow PTs could be updated because + * the guest page aliased or/and mapped by multiple PTs. A CR3 sync has been + * flagged together with a pool clearing. + * @retval VERR_PGM_HANDLER_PHYSICAL_CONFLICT if the range conflicts with an existing + * one. A debug assertion is raised. + * + * @param pVM The cross context VM structure. + * @param GCPhys Start physical address. + * @param GCPhysLast Last physical address. (inclusive) + * @param hType The handler type registration handle. + * @param uUser User argument to the handler. + * @param pszDesc Description of this handler. If NULL, the type + * description will be used instead. + */ +VMMDECL(int) PGMHandlerPhysicalRegister(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, PGMPHYSHANDLERTYPE hType, + uint64_t uUser, R3PTRTYPE(const char *) pszDesc) +{ +#ifdef LOG_ENABLED + PCPGMPHYSHANDLERTYPEINT pType = pgmHandlerPhysicalTypeHandleToPtr(pVM, hType); + Log(("PGMHandlerPhysicalRegister: GCPhys=%RGp GCPhysLast=%RGp uUser=%#RX64 hType=%#x (%d, %s) pszDesc=%RHv:%s\n", + GCPhys, GCPhysLast, uUser, hType, pType->enmKind, R3STRING(pType->pszDesc), pszDesc, R3STRING(pszDesc))); +#endif + + PPGMPHYSHANDLER pNew; + int rc = pgmHandlerPhysicalExCreate(pVM, hType, uUser, pszDesc, &pNew); + if (RT_SUCCESS(rc)) + { + rc = pgmHandlerPhysicalExRegister(pVM, pNew, GCPhys, GCPhysLast); + if (RT_SUCCESS(rc)) + return rc; + pgmHandlerPhysicalExDestroy(pVM, pNew); + } + return rc; +} + + +/** + * Sets ram range flags and attempts updating shadow PTs. + * + * @returns VBox status code. + * @retval VINF_SUCCESS when shadow PTs was successfully updated. + * @retval VINF_PGM_SYNC_CR3 when the shadow PTs could be updated because + * the guest page aliased or/and mapped by multiple PTs. FFs set. + * @param pVM The cross context VM structure. + * @param pCur The physical handler. + * @param pRam The RAM range. + * @param pvBitmap Dirty bitmap. Optional. + * @param offBitmap Dirty bitmap offset. + */ +static int pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(PVMCC pVM, PPGMPHYSHANDLER pCur, PPGMRAMRANGE pRam, + void *pvBitmap, uint32_t offBitmap) +{ + /* + * Iterate the guest ram pages updating the flags and flushing PT entries + * mapping the page. + */ + bool fFlushTLBs = false; + int rc = VINF_SUCCESS; + PCPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + const unsigned uState = pCurType->uState; + uint32_t cPages = pCur->cPages; + uint32_t i = (pCur->Key - pRam->GCPhys) >> GUEST_PAGE_SHIFT; + for (;;) + { + PPGMPAGE pPage = &pRam->aPages[i]; + AssertMsg(pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO || PGM_PAGE_IS_MMIO(pPage), + ("%RGp %R[pgmpage]\n", pRam->GCPhys + (i << GUEST_PAGE_SHIFT), pPage)); + + /* Only do upgrades. */ + if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) < uState) + { + PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, uState, pCurType->fNotInHm); + + const RTGCPHYS GCPhysPage = pRam->GCPhys + (i << GUEST_PAGE_SHIFT); + int rc2 = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pPage, + false /* allow updates of PTEs (instead of flushing) */, &fFlushTLBs); + if (rc2 != VINF_SUCCESS && rc == VINF_SUCCESS) + rc = rc2; + +#ifdef VBOX_WITH_NATIVE_NEM + /* Tell NEM about the protection update. */ + if (VM_IS_NEM_ENABLED(pVM)) + { + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); + PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage); + NEMHCNotifyPhysPageProtChanged(pVM, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), + PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage), + pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State); + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + } +#endif + if (pvBitmap) + ASMBitSet(pvBitmap, offBitmap); + } + + /* next */ + if (--cPages == 0) + break; + i++; + offBitmap++; + } + + if (fFlushTLBs) + { + PGM_INVL_ALL_VCPU_TLBS(pVM); + Log(("pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs: flushing guest TLBs; rc=%d\n", rc)); + } + else + Log(("pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs: doesn't flush guest TLBs. rc=%Rrc; sync flags=%x VMCPU_FF_PGM_SYNC_CR3=%d\n", rc, VMMGetCpu(pVM)->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(VMMGetCpu(pVM), VMCPU_FF_PGM_SYNC_CR3))); + + return rc; +} + + +/** + * Deregister a physical page access handler. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pPhysHandler The handler to deregister (but not free). + */ +int pgmHandlerPhysicalExDeregister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler) +{ + LogFlow(("pgmHandlerPhysicalExDeregister: Removing Range %RGp-%RGp %s\n", + pPhysHandler->Key, pPhysHandler->KeyLast, R3STRING(pPhysHandler->pszDesc))); + + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + RTGCPHYS const GCPhys = pPhysHandler->Key; + AssertReturnStmt(GCPhys != NIL_RTGCPHYS, PGM_UNLOCK(pVM), VERR_PGM_HANDLER_NOT_FOUND); + + /* + * Remove the handler from the tree. + */ + + PPGMPHYSHANDLER pRemoved; + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->remove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pRemoved); + if (RT_SUCCESS(rc)) + { + if (pRemoved == pPhysHandler) + { + /* + * Clear the page bits, notify the REM about this change and clear + * the cache. + */ + pgmHandlerPhysicalResetRamFlags(pVM, pPhysHandler); + if (VM_IS_NEM_ENABLED(pVM)) + pgmHandlerPhysicalDeregisterNotifyNEM(pVM, pPhysHandler); + pVM->pgm.s.idxLastPhysHandler = 0; + + pPhysHandler->Key = NIL_RTGCPHYS; + pPhysHandler->KeyLast = NIL_RTGCPHYS; + + PGM_UNLOCK(pVM); + + return VINF_SUCCESS; + } + + /* + * Both of the failure conditions here are considered internal processing + * errors because they can only be caused by race conditions or corruption. + * If we ever need to handle concurrent deregistration, we have to move + * the NIL_RTGCPHYS check inside the PGM lock. + */ + pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->insert(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, pRemoved); + } + + PGM_UNLOCK(pVM); + + if (RT_FAILURE(rc)) + AssertMsgFailed(("Didn't find range starting at %RGp in the tree! %Rrc=rc\n", GCPhys, rc)); + else + AssertMsgFailed(("Found different handle at %RGp in the tree: got %p insteaded of %p\n", + GCPhys, pRemoved, pPhysHandler)); + return VERR_PGM_HANDLER_IPE_1; +} + + +/** + * Destroys (frees) a physical handler. + * + * The caller must deregister it before destroying it! + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pHandler The handler to free. NULL if ignored. + */ +int pgmHandlerPhysicalExDestroy(PVMCC pVM, PPGMPHYSHANDLER pHandler) +{ + if (pHandler) + { + AssertPtr(pHandler); + AssertReturn(pHandler->Key == NIL_RTGCPHYS, VERR_WRONG_ORDER); + + int rc = PGM_LOCK(pVM); + if (RT_SUCCESS(rc)) + { + rc = pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator.freeNode(pHandler); + PGM_UNLOCK(pVM); + } + return rc; + } + return VINF_SUCCESS; +} + + +/** + * Deregister a physical page access handler. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys Start physical address. + */ +VMMDECL(int) PGMHandlerPhysicalDeregister(PVMCC pVM, RTGCPHYS GCPhys) +{ + AssertReturn(pVM->VMCC_CTX(pgm).s.pPhysHandlerTree, VERR_PGM_HANDLER_IPE_1); + + /* + * Find the handler. + */ + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + PPGMPHYSHANDLER pRemoved; + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->remove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pRemoved); + if (RT_SUCCESS(rc)) + { + Assert(pRemoved->Key == GCPhys); + LogFlow(("PGMHandlerPhysicalDeregister: Removing Range %RGp-%RGp %s\n", + pRemoved->Key, pRemoved->KeyLast, R3STRING(pRemoved->pszDesc))); + + /* + * Clear the page bits, notify the REM about this change and clear + * the cache. + */ + pgmHandlerPhysicalResetRamFlags(pVM, pRemoved); + if (VM_IS_NEM_ENABLED(pVM)) + pgmHandlerPhysicalDeregisterNotifyNEM(pVM, pRemoved); + pVM->pgm.s.idxLastPhysHandler = 0; + + pRemoved->Key = NIL_RTGCPHYS; + rc = pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator.freeNode(pRemoved); + + PGM_UNLOCK(pVM); + return rc; + } + + PGM_UNLOCK(pVM); + + if (rc == VERR_NOT_FOUND) + { + AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys)); + rc = VERR_PGM_HANDLER_NOT_FOUND; + } + return rc; +} + + +/** + * Shared code with modify. + */ +static void pgmHandlerPhysicalDeregisterNotifyNEM(PVMCC pVM, PPGMPHYSHANDLER pCur) +{ +#ifdef VBOX_WITH_NATIVE_NEM + PCPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + RTGCPHYS GCPhysStart = pCur->Key; + RTGCPHYS GCPhysLast = pCur->KeyLast; + + /* + * Page align the range. + * + * Since we've reset (recalculated) the physical handler state of all pages + * we can make use of the page states to figure out whether a page should be + * included in the REM notification or not. + */ + if ( (pCur->Key & GUEST_PAGE_OFFSET_MASK) + || ((pCur->KeyLast + 1) & GUEST_PAGE_OFFSET_MASK)) + { + Assert(pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO); + + if (GCPhysStart & GUEST_PAGE_OFFSET_MASK) + { + PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhysStart); + if ( pPage + && PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE) + { + RTGCPHYS GCPhys = (GCPhysStart + (GUEST_PAGE_SIZE - 1)) & X86_PTE_PAE_PG_MASK; + if ( GCPhys > GCPhysLast + || GCPhys < GCPhysStart) + return; + GCPhysStart = GCPhys; + } + else + GCPhysStart &= X86_PTE_PAE_PG_MASK; + Assert(!pPage || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO); /* these are page aligned atm! */ + } + + if (GCPhysLast & GUEST_PAGE_OFFSET_MASK) + { + PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhysLast); + if ( pPage + && PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE) + { + RTGCPHYS GCPhys = (GCPhysLast & X86_PTE_PAE_PG_MASK) - 1; + if ( GCPhys < GCPhysStart + || GCPhys > GCPhysLast) + return; + GCPhysLast = GCPhys; + } + else + GCPhysLast |= GUEST_PAGE_OFFSET_MASK; + Assert(!pPage || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO); /* these are page aligned atm! */ + } + } + + /* + * Tell NEM. + */ + PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhysStart); + RTGCPHYS const cb = GCPhysLast - GCPhysStart + 1; + uint8_t u2State = UINT8_MAX; + NEMHCNotifyHandlerPhysicalDeregister(pVM, pCurType->enmKind, GCPhysStart, cb, + pRam ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysStart) : NULL, &u2State); + if (u2State != UINT8_MAX && pRam) + pgmPhysSetNemStateForPages(&pRam->aPages[(GCPhysStart - pRam->GCPhys) >> GUEST_PAGE_SHIFT], + cb >> GUEST_PAGE_SHIFT, u2State); +#else + RT_NOREF(pVM, pCur); +#endif +} + + +/** + * pgmHandlerPhysicalResetRamFlags helper that checks for other handlers on + * edge pages. + */ +DECLINLINE(void) pgmHandlerPhysicalRecalcPageState(PVMCC pVM, RTGCPHYS GCPhys, bool fAbove, PPGMRAMRANGE *ppRamHint) +{ + /* + * Look for other handlers. + */ + unsigned uState = PGM_PAGE_HNDL_PHYS_STATE_NONE; + for (;;) + { + PPGMPHYSHANDLER pCur; + int rc; + if (fAbove) + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, + GCPhys, &pCur); + else + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookupMatchingOrBelow(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, + GCPhys, &pCur); + if (rc == VERR_NOT_FOUND) + break; + AssertRCBreak(rc); + if (((fAbove ? pCur->Key : pCur->KeyLast) >> GUEST_PAGE_SHIFT) != (GCPhys >> GUEST_PAGE_SHIFT)) + break; + PCPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + uState = RT_MAX(uState, pCurType->uState); + + /* next? */ + RTGCPHYS GCPhysNext = fAbove + ? pCur->KeyLast + 1 + : pCur->Key - 1; + if ((GCPhysNext >> GUEST_PAGE_SHIFT) != (GCPhys >> GUEST_PAGE_SHIFT)) + break; + GCPhys = GCPhysNext; + } + + /* + * Update if we found something that is a higher priority state than the current. + * Note! The PGMPHYSHANDLER_F_NOT_IN_HM can be ignored here as it requires whole pages. + */ + if (uState != PGM_PAGE_HNDL_PHYS_STATE_NONE) + { + PPGMPAGE pPage; + int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, ppRamHint); + if ( RT_SUCCESS(rc) + && PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) < uState) + { + /* This should normally not be necessary. */ + PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, uState); + bool fFlushTLBs; + rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, false /*fFlushPTEs*/, &fFlushTLBs); + if (RT_SUCCESS(rc) && fFlushTLBs) + PGM_INVL_ALL_VCPU_TLBS(pVM); + else + AssertRC(rc); + +#ifdef VBOX_WITH_NATIVE_NEM + /* Tell NEM about the protection update. */ + if (VM_IS_NEM_ENABLED(pVM)) + { + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); + PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage); + NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage), + PGM_RAMRANGE_CALC_PAGE_R3PTR(*ppRamHint, GCPhys), + pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State); + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + } +#endif + } + else + AssertRC(rc); + } +} + + +/** + * Resets an aliased page. + * + * @param pVM The cross context VM structure. + * @param pPage The page. + * @param GCPhysPage The page address in case it comes in handy. + * @param pRam The RAM range the page is associated with (for NEM + * notifications). + * @param fDoAccounting Whether to perform accounting. (Only set during + * reset where pgmR3PhysRamReset doesn't have the + * handler structure handy.) + * @param fFlushIemTlbs Whether to perform IEM TLB flushing or not. This + * can be cleared only if the caller does the flushing + * after calling this function. + */ +void pgmHandlerPhysicalResetAliasedPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, PPGMRAMRANGE pRam, + bool fDoAccounting, bool fFlushIemTlbs) +{ + Assert( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO + || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO); + Assert(PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_DISABLED); +#ifdef VBOX_WITH_NATIVE_NEM + RTHCPHYS const HCPhysPrev = PGM_PAGE_GET_HCPHYS(pPage); +#endif + + /* + * Flush any shadow page table references *first*. + */ + bool fFlushTLBs = false; + int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pPage, true /*fFlushPTEs*/, &fFlushTLBs); + AssertLogRelRCReturnVoid(rc); + HMFlushTlbOnAllVCpus(pVM); + + /* + * Make it an MMIO/Zero page. + */ + PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg); + PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_MMIO); + PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO); + PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID); + PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, PGM_PAGE_HNDL_PHYS_STATE_ALL); + + /* + * Flush its TLB entry. + */ + pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhysPage); + if (fFlushIemTlbs) + IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID); + + /* + * Do accounting for pgmR3PhysRamReset. + */ + if (fDoAccounting) + { + PPGMPHYSHANDLER pHandler; + rc = pgmHandlerPhysicalLookup(pVM, GCPhysPage, &pHandler); + if (RT_SUCCESS(rc)) + { + Assert(pHandler->cAliasedPages > 0); + pHandler->cAliasedPages--; + } + else + AssertMsgFailed(("rc=%Rrc GCPhysPage=%RGp\n", rc, GCPhysPage)); + } + +#ifdef VBOX_WITH_NATIVE_NEM + /* + * Tell NEM about the protection change. + */ + if (VM_IS_NEM_ENABLED(pVM)) + { + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); + NEMHCNotifyPhysPageChanged(pVM, GCPhysPage, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg, + PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage), + NEM_PAGE_PROT_NONE, PGMPAGETYPE_MMIO, &u2State); + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + } +#else + RT_NOREF(pRam); +#endif +} + + +/** + * Resets ram range flags. + * + * @param pVM The cross context VM structure. + * @param pCur The physical handler. + * + * @remark We don't start messing with the shadow page tables, as we've + * already got code in Trap0e which deals with out of sync handler + * flags (originally conceived for global pages). + */ +static void pgmHandlerPhysicalResetRamFlags(PVMCC pVM, PPGMPHYSHANDLER pCur) +{ + /* + * Iterate the guest ram pages updating the state. + */ + RTUINT cPages = pCur->cPages; + RTGCPHYS GCPhys = pCur->Key; + PPGMRAMRANGE pRamHint = NULL; + for (;;) + { + PPGMPAGE pPage; + int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint); + if (RT_SUCCESS(rc)) + { + /* Reset aliased MMIO pages to MMIO, since this aliasing is our business. + (We don't flip MMIO to RAM though, that's PGMPhys.cpp's job.) */ + bool fNemNotifiedAlready = false; + if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO + || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO) + { + Assert(pCur->cAliasedPages > 0); + pgmHandlerPhysicalResetAliasedPage(pVM, pPage, GCPhys, pRamHint, false /*fDoAccounting*/, true /*fFlushIemTlbs*/); + pCur->cAliasedPages--; + fNemNotifiedAlready = true; + } +#ifdef VBOX_STRICT + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur); + AssertMsg(pCurType && (pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO || PGM_PAGE_IS_MMIO(pPage)), + ("%RGp %R[pgmpage]\n", GCPhys, pPage)); +#endif + PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_NONE, false); + +#ifdef VBOX_WITH_NATIVE_NEM + /* Tell NEM about the protection change. */ + if (VM_IS_NEM_ENABLED(pVM) && !fNemNotifiedAlready) + { + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); + PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage); + NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage), + PGM_RAMRANGE_CALC_PAGE_R3PTR(pRamHint, GCPhys), + pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State); + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + } +#endif + RT_NOREF(fNemNotifiedAlready); + } + else + AssertRC(rc); + + /* next */ + if (--cPages == 0) + break; + GCPhys += GUEST_PAGE_SIZE; + } + + pCur->cAliasedPages = 0; + pCur->cTmpOffPages = 0; + + /* + * Check for partial start and end pages. + */ + if (pCur->Key & GUEST_PAGE_OFFSET_MASK) + pgmHandlerPhysicalRecalcPageState(pVM, pCur->Key - 1, false /* fAbove */, &pRamHint); + if ((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) != GUEST_PAGE_OFFSET_MASK) + pgmHandlerPhysicalRecalcPageState(pVM, pCur->KeyLast + 1, true /* fAbove */, &pRamHint); +} + + +#if 0 /* unused */ +/** + * Modify a physical page access handler. + * + * Modification can only be done to the range it self, not the type or anything else. + * + * @returns VBox status code. + * For all return codes other than VERR_PGM_HANDLER_NOT_FOUND and VINF_SUCCESS the range is deregistered + * and a new registration must be performed! + * @param pVM The cross context VM structure. + * @param GCPhysCurrent Current location. + * @param GCPhys New location. + * @param GCPhysLast New last location. + */ +VMMDECL(int) PGMHandlerPhysicalModify(PVMCC pVM, RTGCPHYS GCPhysCurrent, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast) +{ + /* + * Remove it. + */ + int rc; + PGM_LOCK_VOID(pVM); + PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysCurrent); + if (pCur) + { + /* + * Clear the ram flags. (We're gonna move or free it!) + */ + pgmHandlerPhysicalResetRamFlags(pVM, pCur); + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + @todo pCurType validation + bool const fRestoreAsRAM = pCurType->pfnHandlerR3 /** @todo this isn't entirely correct. */ + && pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO; + + /* + * Validate the new range, modify and reinsert. + */ + if (GCPhysLast >= GCPhys) + { + /* + * We require the range to be within registered ram. + * There is no apparent need to support ranges which cover more than one ram range. + */ + PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys); + if ( pRam + && GCPhys <= pRam->GCPhysLast + && GCPhysLast >= pRam->GCPhys) + { + pCur->Core.Key = GCPhys; + pCur->Core.KeyLast = GCPhysLast; + pCur->cPages = (GCPhysLast - (GCPhys & X86_PTE_PAE_PG_MASK) + 1) >> GUEST_PAGE_SHIFT; + + if (RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, &pCur->Core)) + { + RTGCPHYS const cb = GCPhysLast - GCPhys + 1; + PGMPHYSHANDLERKIND const enmKind = pCurType->enmKind; + + /* + * Set ram flags, flush shadow PT entries and finally tell REM about this. + */ + rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pCur, pRam, NULL, 0); + + /** @todo NEM: not sure we need this notification... */ + NEMHCNotifyHandlerPhysicalModify(pVM, enmKind, GCPhysCurrent, GCPhys, cb, fRestoreAsRAM); + + PGM_UNLOCK(pVM); + + PGM_INVL_ALL_VCPU_TLBS(pVM); + Log(("PGMHandlerPhysicalModify: GCPhysCurrent=%RGp -> GCPhys=%RGp GCPhysLast=%RGp\n", + GCPhysCurrent, GCPhys, GCPhysLast)); + return VINF_SUCCESS; + } + + AssertMsgFailed(("Conflict! GCPhys=%RGp GCPhysLast=%RGp\n", GCPhys, GCPhysLast)); + rc = VERR_PGM_HANDLER_PHYSICAL_CONFLICT; + } + else + { + AssertMsgFailed(("No RAM range for %RGp-%RGp\n", GCPhys, GCPhysLast)); + rc = VERR_PGM_HANDLER_PHYSICAL_NO_RAM_RANGE; + } + } + else + { + AssertMsgFailed(("Invalid range %RGp-%RGp\n", GCPhys, GCPhysLast)); + rc = VERR_INVALID_PARAMETER; + } + + /* + * Invalid new location, flush the cache and free it. + * We've only gotta notify REM and free the memory. + */ + if (VM_IS_NEM_ENABLED(pVM)) + pgmHandlerPhysicalDeregisterNotifyNEM(pVM, pCur); + pVM->pgm.s.pLastPhysHandlerR0 = 0; + pVM->pgm.s.pLastPhysHandlerR3 = 0; + PGMHandlerPhysicalTypeRelease(pVM, pCur->hType); + MMHyperFree(pVM, pCur); + } + else + { + AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhysCurrent)); + rc = VERR_PGM_HANDLER_NOT_FOUND; + } + + PGM_UNLOCK(pVM); + return rc; +} +#endif /* unused */ + + +/** + * Changes the user callback arguments associated with a physical access handler. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys Start physical address of the handler. + * @param uUser User argument to the handlers. + */ +VMMDECL(int) PGMHandlerPhysicalChangeUserArg(PVMCC pVM, RTGCPHYS GCPhys, uint64_t uUser) +{ + /* + * Find the handler and make the change. + */ + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + PPGMPHYSHANDLER pCur; + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur); + if (RT_SUCCESS(rc)) + { + Assert(pCur->Key == GCPhys); + pCur->uUser = uUser; + } + else if (rc == VERR_NOT_FOUND) + { + AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys)); + rc = VERR_PGM_HANDLER_NOT_FOUND; + } + + PGM_UNLOCK(pVM); + return rc; +} + +#if 0 /* unused */ + +/** + * Splits a physical access handler in two. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys Start physical address of the handler. + * @param GCPhysSplit The split address. + */ +VMMDECL(int) PGMHandlerPhysicalSplit(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysSplit) +{ + AssertReturn(GCPhys < GCPhysSplit, VERR_INVALID_PARAMETER); + + /* + * Do the allocation without owning the lock. + */ + PPGMPHYSHANDLER pNew; + int rc = MMHyperAlloc(pVM, sizeof(*pNew), 0, MM_TAG_PGM_HANDLERS, (void **)&pNew); + if (RT_FAILURE(rc)) + return rc; + + /* + * Get the handler. + */ + PGM_LOCK_VOID(pVM); + PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys); + if (RT_LIKELY(pCur)) + { + if (RT_LIKELY(GCPhysSplit <= pCur->Core.KeyLast)) + { + /* + * Create new handler node for the 2nd half. + */ + *pNew = *pCur; + pNew->Core.Key = GCPhysSplit; + pNew->cPages = (pNew->Core.KeyLast - (pNew->Core.Key & X86_PTE_PAE_PG_MASK) + GUEST_PAGE_SIZE) >> GUEST_PAGE_SHIFT; + + pCur->Core.KeyLast = GCPhysSplit - 1; + pCur->cPages = (pCur->Core.KeyLast - (pCur->Core.Key & X86_PTE_PAE_PG_MASK) + GUEST_PAGE_SIZE) >> GUEST_PAGE_SHIFT; + + if (RT_LIKELY(RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, &pNew->Core))) + { + LogFlow(("PGMHandlerPhysicalSplit: %RGp-%RGp and %RGp-%RGp\n", + pCur->Core.Key, pCur->Core.KeyLast, pNew->Core.Key, pNew->Core.KeyLast)); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + AssertMsgFailed(("whu?\n")); + rc = VERR_PGM_PHYS_HANDLER_IPE; + } + else + { + AssertMsgFailed(("outside range: %RGp-%RGp split %RGp\n", pCur->Core.Key, pCur->Core.KeyLast, GCPhysSplit)); + rc = VERR_INVALID_PARAMETER; + } + } + else + { + AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys)); + rc = VERR_PGM_HANDLER_NOT_FOUND; + } + PGM_UNLOCK(pVM); + MMHyperFree(pVM, pNew); + return rc; +} + + +/** + * Joins up two adjacent physical access handlers which has the same callbacks. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys1 Start physical address of the first handler. + * @param GCPhys2 Start physical address of the second handler. + */ +VMMDECL(int) PGMHandlerPhysicalJoin(PVMCC pVM, RTGCPHYS GCPhys1, RTGCPHYS GCPhys2) +{ + /* + * Get the handlers. + */ + int rc; + PGM_LOCK_VOID(pVM); + PPGMPHYSHANDLER pCur1 = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys1); + if (RT_LIKELY(pCur1)) + { + PPGMPHYSHANDLER pCur2 = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys2); + if (RT_LIKELY(pCur2)) + { + /* + * Make sure that they are adjacent, and that they've got the same callbacks. + */ + if (RT_LIKELY(pCur1->Core.KeyLast + 1 == pCur2->Core.Key)) + { + if (RT_LIKELY(pCur1->hType == pCur2->hType)) + { + PPGMPHYSHANDLER pCur3 = (PPGMPHYSHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys2); + if (RT_LIKELY(pCur3 == pCur2)) + { + pCur1->Core.KeyLast = pCur2->Core.KeyLast; + pCur1->cPages = (pCur1->Core.KeyLast - (pCur1->Core.Key & X86_PTE_PAE_PG_MASK) + GUEST_PAGE_SIZE) >> GUEST_PAGE_SHIFT; + LogFlow(("PGMHandlerPhysicalJoin: %RGp-%RGp %RGp-%RGp\n", + pCur1->Core.Key, pCur1->Core.KeyLast, pCur2->Core.Key, pCur2->Core.KeyLast)); + pVM->pgm.s.pLastPhysHandlerR0 = 0; + pVM->pgm.s.pLastPhysHandlerR3 = 0; + PGMHandlerPhysicalTypeRelease(pVM, pCur2->hType); + MMHyperFree(pVM, pCur2); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + + Assert(pCur3 == pCur2); + rc = VERR_PGM_PHYS_HANDLER_IPE; + } + else + { + AssertMsgFailed(("mismatching handlers\n")); + rc = VERR_ACCESS_DENIED; + } + } + else + { + AssertMsgFailed(("not adjacent: %RGp-%RGp %RGp-%RGp\n", + pCur1->Core.Key, pCur1->Core.KeyLast, pCur2->Core.Key, pCur2->Core.KeyLast)); + rc = VERR_INVALID_PARAMETER; + } + } + else + { + AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys2)); + rc = VERR_PGM_HANDLER_NOT_FOUND; + } + } + else + { + AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys1)); + rc = VERR_PGM_HANDLER_NOT_FOUND; + } + PGM_UNLOCK(pVM); + return rc; + +} + +#endif /* unused */ + +/** + * Resets any modifications to individual pages in a physical page access + * handler region. + * + * This is used in pair with PGMHandlerPhysicalPageTempOff(), + * PGMHandlerPhysicalPageAliasMmio2() or PGMHandlerPhysicalPageAliasHC(). + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys The start address of the handler regions, i.e. what you + * passed to PGMR3HandlerPhysicalRegister(), + * PGMHandlerPhysicalRegisterEx() or + * PGMHandlerPhysicalModify(). + */ +VMMDECL(int) PGMHandlerPhysicalReset(PVMCC pVM, RTGCPHYS GCPhys) +{ + LogFlow(("PGMHandlerPhysicalReset GCPhys=%RGp\n", GCPhys)); + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + /* + * Find the handler. + */ + PPGMPHYSHANDLER pCur; + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur); + if (RT_SUCCESS(rc)) + { + Assert(pCur->Key == GCPhys); + + /* + * Validate kind. + */ + PCPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + switch (pCurType->enmKind) + { + case PGMPHYSHANDLERKIND_WRITE: + case PGMPHYSHANDLERKIND_ALL: + case PGMPHYSHANDLERKIND_MMIO: /* NOTE: Only use when clearing MMIO ranges with aliased MMIO2 pages! */ + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysHandlerReset)); /** @todo move out of switch */ + PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys); + Assert(pRam); + Assert(pRam->GCPhys <= pCur->Key); + Assert(pRam->GCPhysLast >= pCur->KeyLast); + + if (pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO) + { + /* + * Reset all the PGMPAGETYPE_MMIO2_ALIAS_MMIO pages first and that's it. + * This could probably be optimized a bit wrt to flushing, but I'm too lazy + * to do that now... + */ + if (pCur->cAliasedPages) + { + PPGMPAGE pPage = &pRam->aPages[(pCur->Key - pRam->GCPhys) >> GUEST_PAGE_SHIFT]; + RTGCPHYS GCPhysPage = pCur->Key; + uint32_t cLeft = pCur->cPages; + bool fFlushIemTlb = false; + while (cLeft-- > 0) + { + if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO + || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO) + { + fFlushIemTlb |= PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO; + Assert(pCur->cAliasedPages > 0); + pgmHandlerPhysicalResetAliasedPage(pVM, pPage, GCPhysPage, pRam, + false /*fDoAccounting*/, false /*fFlushIemTlbs*/); + --pCur->cAliasedPages; +#ifndef VBOX_STRICT + if (pCur->cAliasedPages == 0) + break; +#endif + } + Assert(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO); + GCPhysPage += GUEST_PAGE_SIZE; + pPage++; + } + Assert(pCur->cAliasedPages == 0); + + /* + * Flush IEM TLBs in case they contain any references to aliased pages. + * This is only necessary for MMIO2 aliases. + */ + if (fFlushIemTlb) + IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID); + } + } + else if (pCur->cTmpOffPages > 0) + { + /* + * Set the flags and flush shadow PT entries. + */ + rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pCur, pRam, NULL /*pvBitmap*/, 0 /*offBitmap*/); + } + + pCur->cAliasedPages = 0; + pCur->cTmpOffPages = 0; + + rc = VINF_SUCCESS; + break; + } + + /* + * Invalid. + */ + default: + AssertMsgFailed(("Invalid type %d/%#x! Corruption!\n", pCurType->enmKind, pCur->hType)); + rc = VERR_PGM_PHYS_HANDLER_IPE; + break; + } + } + else if (rc == VERR_NOT_FOUND) + { + AssertMsgFailed(("Didn't find MMIO Range starting at %#x\n", GCPhys)); + rc = VERR_PGM_HANDLER_NOT_FOUND; + } + + PGM_UNLOCK(pVM); + return rc; +} + + +/** + * Special version of PGMHandlerPhysicalReset used by MMIO2 w/ dirty page + * tracking. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys The start address of the handler region. + * @param pvBitmap Dirty bitmap. Caller has cleared this already, only + * dirty bits will be set. Caller also made sure it's big + * enough. + * @param offBitmap Dirty bitmap offset. + * @remarks Caller must own the PGM critical section. + */ +DECLHIDDEN(int) pgmHandlerPhysicalResetMmio2WithBitmap(PVMCC pVM, RTGCPHYS GCPhys, void *pvBitmap, uint32_t offBitmap) +{ + LogFlow(("pgmHandlerPhysicalResetMmio2WithBitmap GCPhys=%RGp\n", GCPhys)); + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * Find the handler. + */ + PPGMPHYSHANDLER pCur; + int rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur); + if (RT_SUCCESS(rc)) + { + Assert(pCur->Key == GCPhys); + + /* + * Validate kind. + */ + PCPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur); + if ( pCurType + && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE) + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysHandlerReset)); + + PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys); + Assert(pRam); + Assert(pRam->GCPhys <= pCur->Key); + Assert(pRam->GCPhysLast >= pCur->KeyLast); + + /* + * Set the flags and flush shadow PT entries. + */ + if (pCur->cTmpOffPages > 0) + { + rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pCur, pRam, pvBitmap, offBitmap); + pCur->cTmpOffPages = 0; + } + else + rc = VINF_SUCCESS; + } + else + { + AssertFailed(); + rc = VERR_WRONG_TYPE; + } + } + else if (rc == VERR_NOT_FOUND) + { + AssertMsgFailed(("Didn't find MMIO Range starting at %#x\n", GCPhys)); + rc = VERR_PGM_HANDLER_NOT_FOUND; + } + + return rc; +} + + +/** + * Temporarily turns off the access monitoring of a page within a monitored + * physical write/all page access handler region. + * + * Use this when no further \#PFs are required for that page. Be aware that + * a page directory sync might reset the flags, and turn on access monitoring + * for the page. + * + * The caller must do required page table modifications. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys The start address of the access handler. This + * must be a fully page aligned range or we risk + * messing up other handlers installed for the + * start and end pages. + * @param GCPhysPage The physical address of the page to turn off + * access monitoring for. + */ +VMMDECL(int) PGMHandlerPhysicalPageTempOff(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysPage) +{ + LogFlow(("PGMHandlerPhysicalPageTempOff GCPhysPage=%RGp\n", GCPhysPage)); + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + /* + * Validate the range. + */ + PPGMPHYSHANDLER pCur; + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur); + if (RT_SUCCESS(rc)) + { + Assert(pCur->Key == GCPhys); + if (RT_LIKELY( GCPhysPage >= pCur->Key + && GCPhysPage <= pCur->KeyLast)) + { + Assert(!(pCur->Key & GUEST_PAGE_OFFSET_MASK)); + Assert((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK); + + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur); + AssertReturnStmt( pCurType + && ( pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE + || pCurType->enmKind == PGMPHYSHANDLERKIND_ALL), + PGM_UNLOCK(pVM), VERR_ACCESS_DENIED); + + /* + * Change the page status. + */ + PPGMPAGE pPage; + PPGMRAMRANGE pRam; + rc = pgmPhysGetPageAndRangeEx(pVM, GCPhysPage, &pPage, &pRam); + AssertReturnStmt(RT_SUCCESS_NP(rc), PGM_UNLOCK(pVM), rc); + if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED) + { + PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, PGM_PAGE_HNDL_PHYS_STATE_DISABLED); + pCur->cTmpOffPages++; + +#ifdef VBOX_WITH_NATIVE_NEM + /* Tell NEM about the protection change (VGA is using this to track dirty pages). */ + if (VM_IS_NEM_ENABLED(pVM)) + { + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); + PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage); + NEMHCNotifyPhysPageProtChanged(pVM, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), + PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage), + pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State); + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + } +#endif + } + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + PGM_UNLOCK(pVM); + AssertMsgFailed(("The page %#x is outside the range %#x-%#x\n", GCPhysPage, pCur->Key, pCur->KeyLast)); + return VERR_INVALID_PARAMETER; + } + PGM_UNLOCK(pVM); + + if (rc == VERR_NOT_FOUND) + { + AssertMsgFailed(("Specified physical handler start address %#x is invalid.\n", GCPhys)); + return VERR_PGM_HANDLER_NOT_FOUND; + } + return rc; +} + + +/** + * Resolves an MMIO2 page. + * + * Caller as taken the PGM lock. + * + * @returns Pointer to the page if valid, NULL otherwise + * @param pVM The cross context VM structure. + * @param pDevIns The device owning it. + * @param hMmio2 The MMIO2 region. + * @param offMmio2Page The offset into the region. + */ +static PPGMPAGE pgmPhysResolveMmio2PageLocked(PVMCC pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS offMmio2Page) +{ + /* Only works if the handle is in the handle table! */ + AssertReturn(hMmio2 != 0, NULL); + hMmio2--; + + /* Must check the first one for PGMREGMMIO2RANGE_F_FIRST_CHUNK. */ + AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3), NULL); + PPGMREGMMIO2RANGE pCur = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[hMmio2]; + AssertReturn(pCur, NULL); + AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, NULL); + + /* Loop thru the sub-ranges till we find the one covering offMmio2. */ + for (;;) + { +#ifdef IN_RING3 + AssertReturn(pCur->pDevInsR3 == pDevIns, NULL); +#else + AssertReturn(pCur->pDevInsR3 == pDevIns->pDevInsForR3, NULL); +#endif + + /* Does it match the offset? */ + if (offMmio2Page < pCur->cbReal) + return &pCur->RamRange.aPages[offMmio2Page >> GUEST_PAGE_SHIFT]; + + /* Advance if we can. */ + AssertReturn(!(pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK), NULL); + offMmio2Page -= pCur->cbReal; + hMmio2++; + AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3), NULL); + pCur = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[hMmio2]; + AssertReturn(pCur, NULL); + } +} + + +/** + * Replaces an MMIO page with an MMIO2 page. + * + * This is a worker for IOMMMIOMapMMIO2Page that works in a similar way to + * PGMHandlerPhysicalPageTempOff but for an MMIO page. Since an MMIO page has no + * backing, the caller must provide a replacement page. For various reasons the + * replacement page must be an MMIO2 page. + * + * The caller must do required page table modifications. You can get away + * without making any modifications since it's an MMIO page, the cost is an extra + * \#PF which will the resync the page. + * + * Call PGMHandlerPhysicalReset() to restore the MMIO page. + * + * The caller may still get handler callback even after this call and must be + * able to deal correctly with such calls. The reason for these callbacks are + * either that we're executing in the recompiler (which doesn't know about this + * arrangement) or that we've been restored from saved state (where we won't + * save the change). + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys The start address of the access handler. This + * must be a fully page aligned range or we risk + * messing up other handlers installed for the + * start and end pages. + * @param GCPhysPage The physical address of the page to turn off + * access monitoring for and replace with the MMIO2 + * page. + * @param pDevIns The device instance owning @a hMmio2. + * @param hMmio2 Handle to the MMIO2 region containing the page + * to remap in the the MMIO page at @a GCPhys. + * @param offMmio2PageRemap The offset into @a hMmio2 of the MMIO2 page that + * should serve as backing memory. + * + * @remark May cause a page pool flush if used on a page that is already + * aliased. + * + * @note This trick does only work reliably if the two pages are never ever + * mapped in the same page table. If they are the page pool code will + * be confused should either of them be flushed. See the special case + * of zero page aliasing mentioned in #3170. + * + */ +VMMDECL(int) PGMHandlerPhysicalPageAliasMmio2(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysPage, + PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS offMmio2PageRemap) +{ +#ifdef VBOX_WITH_PGM_NEM_MODE + AssertReturn(!VM_IS_NEM_ENABLED(pVM) || !pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE); +#endif + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + /* + * Resolve the MMIO2 reference. + */ + PPGMPAGE pPageRemap = pgmPhysResolveMmio2PageLocked(pVM, pDevIns, hMmio2, offMmio2PageRemap); + if (RT_LIKELY(pPageRemap)) + AssertMsgReturnStmt(PGM_PAGE_GET_TYPE(pPageRemap) == PGMPAGETYPE_MMIO2, + ("hMmio2=%RU64 offMmio2PageRemap=%RGp %R[pgmpage]\n", hMmio2, offMmio2PageRemap, pPageRemap), + PGM_UNLOCK(pVM), VERR_PGM_PHYS_NOT_MMIO2); + else + { + PGM_UNLOCK(pVM); + return VERR_OUT_OF_RANGE; + } + + /* + * Lookup and validate the range. + */ + PPGMPHYSHANDLER pCur; + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur); + if (RT_SUCCESS(rc)) + { + Assert(pCur->Key == GCPhys); + if (RT_LIKELY( GCPhysPage >= pCur->Key + && GCPhysPage <= pCur->KeyLast)) + { + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + AssertReturnStmt(pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO, PGM_UNLOCK(pVM), VERR_ACCESS_DENIED); + AssertReturnStmt(!(pCur->Key & GUEST_PAGE_OFFSET_MASK), PGM_UNLOCK(pVM), VERR_INVALID_PARAMETER); + AssertReturnStmt((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK, + PGM_UNLOCK(pVM), VERR_INVALID_PARAMETER); + + /* + * Validate the page. + */ + PPGMPAGE pPage; + PPGMRAMRANGE pRam; + rc = pgmPhysGetPageAndRangeEx(pVM, GCPhysPage, &pPage, &pRam); + AssertReturnStmt(RT_SUCCESS_NP(rc), PGM_UNLOCK(pVM), rc); + if (PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO) + { + AssertMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO, + ("GCPhysPage=%RGp %R[pgmpage]\n", GCPhysPage, pPage), + VERR_PGM_PHYS_NOT_MMIO2); + if (PGM_PAGE_GET_HCPHYS(pPage) == PGM_PAGE_GET_HCPHYS(pPageRemap)) + { + PGM_UNLOCK(pVM); + return VINF_PGM_HANDLER_ALREADY_ALIASED; + } + + /* + * The page is already mapped as some other page, reset it + * to an MMIO/ZERO page before doing the new mapping. + */ + Log(("PGMHandlerPhysicalPageAliasMmio2: GCPhysPage=%RGp (%R[pgmpage]; %RHp -> %RHp\n", + GCPhysPage, pPage, PGM_PAGE_GET_HCPHYS(pPage), PGM_PAGE_GET_HCPHYS(pPageRemap))); + pgmHandlerPhysicalResetAliasedPage(pVM, pPage, GCPhysPage, pRam, + false /*fDoAccounting*/, false /*fFlushIemTlbs*/); + pCur->cAliasedPages--; + + /* Since this may be present in the TLB and now be wrong, invalid + the guest physical address part of the IEM TLBs. Note, we do + this here as we will not invalid */ + IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID); + } + Assert(PGM_PAGE_IS_ZERO(pPage)); + + /* + * Do the actual remapping here. + * This page now serves as an alias for the backing memory specified. + */ + LogFlow(("PGMHandlerPhysicalPageAliasMmio2: %RGp (%R[pgmpage]) alias for %RU64/%RGp (%R[pgmpage])\n", + GCPhysPage, pPage, hMmio2, offMmio2PageRemap, pPageRemap )); + PGM_PAGE_SET_HCPHYS(pVM, pPage, PGM_PAGE_GET_HCPHYS(pPageRemap)); + PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_MMIO2_ALIAS_MMIO); + PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED); + PGM_PAGE_SET_PAGEID(pVM, pPage, PGM_PAGE_GET_PAGEID(pPageRemap)); + PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, PGM_PAGE_HNDL_PHYS_STATE_DISABLED); + pCur->cAliasedPages++; + Assert(pCur->cAliasedPages <= pCur->cPages); + + /* + * Flush its TLB entry. + * + * Not calling IEMTlbInvalidateAllPhysicalAllCpus here to conserve + * all the other IEM TLB entires. When this one is kicked out and + * reloaded, it will be using the MMIO2 alias, but till then we'll + * continue doing MMIO. + */ + pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhysPage); + /** @todo Do some preformance checks of calling + * IEMTlbInvalidateAllPhysicalAllCpus when in IEM mode, to see if it + * actually makes sense or not. Screen updates are typically massive + * and important when this kind of aliasing is used, so it may pay of... */ + +#ifdef VBOX_WITH_NATIVE_NEM + /* Tell NEM about the backing and protection change. */ + if (VM_IS_NEM_ENABLED(pVM)) + { + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); + NEMHCNotifyPhysPageChanged(pVM, GCPhysPage, pVM->pgm.s.HCPhysZeroPg, PGM_PAGE_GET_HCPHYS(pPage), + PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage), + pgmPhysPageCalcNemProtection(pPage, PGMPAGETYPE_MMIO2_ALIAS_MMIO), + PGMPAGETYPE_MMIO2_ALIAS_MMIO, &u2State); + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + } +#endif + LogFlow(("PGMHandlerPhysicalPageAliasMmio2: => %R[pgmpage]\n", pPage)); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + + PGM_UNLOCK(pVM); + AssertMsgFailed(("The page %#x is outside the range %#x-%#x\n", GCPhysPage, pCur->Key, pCur->KeyLast)); + return VERR_INVALID_PARAMETER; + } + + PGM_UNLOCK(pVM); + if (rc == VERR_NOT_FOUND) + { + AssertMsgFailed(("Specified physical handler start address %#x is invalid.\n", GCPhys)); + return VERR_PGM_HANDLER_NOT_FOUND; + } + return rc; +} + + +/** + * Replaces an MMIO page with an arbitrary HC page in the shadow page tables. + * + * This differs from PGMHandlerPhysicalPageAliasMmio2 in that the page doesn't + * need to be a known MMIO2 page and that only shadow paging may access the + * page. The latter distinction is important because the only use for this + * feature is for mapping the special APIC access page that VT-x uses to detect + * APIC MMIO operations, the page is shared between all guest CPUs and actually + * not written to. At least at the moment. + * + * The caller must do required page table modifications. You can get away + * without making any modifications since it's an MMIO page, the cost is an extra + * \#PF which will the resync the page. + * + * Call PGMHandlerPhysicalReset() to restore the MMIO page. + * + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhys The start address of the access handler. This + * must be a fully page aligned range or we risk + * messing up other handlers installed for the + * start and end pages. + * @param GCPhysPage The physical address of the page to turn off + * access monitoring for. + * @param HCPhysPageRemap The physical address of the HC page that + * serves as backing memory. + * + * @remark May cause a page pool flush if used on a page that is already + * aliased. + */ +VMMDECL(int) PGMHandlerPhysicalPageAliasHC(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysPage, RTHCPHYS HCPhysPageRemap) +{ +/// Assert(!IOMIsLockOwner(pVM)); /* We mustn't own any other locks when calling this */ +#ifdef VBOX_WITH_PGM_NEM_MODE + AssertReturn(!VM_IS_NEM_ENABLED(pVM) || !pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE); +#endif + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + /* + * Lookup and validate the range. + */ + PPGMPHYSHANDLER pCur; + rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur); + if (RT_SUCCESS(rc)) + { + Assert(pCur->Key == GCPhys); + if (RT_LIKELY( GCPhysPage >= pCur->Key + && GCPhysPage <= pCur->KeyLast)) + { + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + AssertReturnStmt(pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO, PGM_UNLOCK(pVM), VERR_ACCESS_DENIED); + AssertReturnStmt(!(pCur->Key & GUEST_PAGE_OFFSET_MASK), PGM_UNLOCK(pVM), VERR_INVALID_PARAMETER); + AssertReturnStmt((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK, + PGM_UNLOCK(pVM), VERR_INVALID_PARAMETER); + + /* + * Get and validate the pages. + */ + PPGMPAGE pPage; + rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); + AssertReturnStmt(RT_SUCCESS_NP(rc), PGM_UNLOCK(pVM), rc); + if (PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO) + { + PGM_UNLOCK(pVM); + AssertMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO, + ("GCPhysPage=%RGp %R[pgmpage]\n", GCPhysPage, pPage), + VERR_PGM_PHYS_NOT_MMIO2); + return VINF_PGM_HANDLER_ALREADY_ALIASED; + } + Assert(PGM_PAGE_IS_ZERO(pPage)); + + /* + * Do the actual remapping here. + * This page now serves as an alias for the backing memory + * specified as far as shadow paging is concerned. + */ + LogFlow(("PGMHandlerPhysicalPageAliasHC: %RGp (%R[pgmpage]) alias for %RHp\n", + GCPhysPage, pPage, HCPhysPageRemap)); + PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhysPageRemap); + PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_SPECIAL_ALIAS_MMIO); + PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED); + PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID); + PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, PGM_PAGE_HNDL_PHYS_STATE_DISABLED); + pCur->cAliasedPages++; + Assert(pCur->cAliasedPages <= pCur->cPages); + + /* + * Flush its TLB entry. + * + * Not calling IEMTlbInvalidateAllPhysicalAllCpus here as special + * aliased MMIO pages are handled like MMIO by the IEM TLB. + */ + pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhysPage); + +#ifdef VBOX_WITH_NATIVE_NEM + /* Tell NEM about the backing and protection change. */ + if (VM_IS_NEM_ENABLED(pVM)) + { + PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhysPage); + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); + NEMHCNotifyPhysPageChanged(pVM, GCPhysPage, pVM->pgm.s.HCPhysZeroPg, PGM_PAGE_GET_HCPHYS(pPage), + PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage), + pgmPhysPageCalcNemProtection(pPage, PGMPAGETYPE_SPECIAL_ALIAS_MMIO), + PGMPAGETYPE_SPECIAL_ALIAS_MMIO, &u2State); + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + } +#endif + LogFlow(("PGMHandlerPhysicalPageAliasHC: => %R[pgmpage]\n", pPage)); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + PGM_UNLOCK(pVM); + AssertMsgFailed(("The page %#x is outside the range %#x-%#x\n", GCPhysPage, pCur->Key, pCur->KeyLast)); + return VERR_INVALID_PARAMETER; + } + PGM_UNLOCK(pVM); + + if (rc == VERR_NOT_FOUND) + { + AssertMsgFailed(("Specified physical handler start address %#x is invalid.\n", GCPhys)); + return VERR_PGM_HANDLER_NOT_FOUND; + } + return rc; +} + + +/** + * Checks if a physical range is handled + * + * @returns boolean + * @param pVM The cross context VM structure. + * @param GCPhys Start physical address earlier passed to PGMR3HandlerPhysicalRegister(). + * @remarks Caller must take the PGM lock... + * @thread EMT. + */ +VMMDECL(bool) PGMHandlerPhysicalIsRegistered(PVMCC pVM, RTGCPHYS GCPhys) +{ + /* + * Find the handler. + */ + PGM_LOCK_VOID(pVM); + PPGMPHYSHANDLER pCur; + int rc = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur); + if (RT_SUCCESS(rc)) + { +#ifdef VBOX_STRICT + Assert(GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast); + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + Assert( pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE + || pCurType->enmKind == PGMPHYSHANDLERKIND_ALL + || pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO); +#endif + PGM_UNLOCK(pVM); + return true; + } + PGM_UNLOCK(pVM); + return false; +} + + +/** + * Checks if it's an disabled all access handler or write access handler at the + * given address. + * + * @returns true if it's an all access handler, false if it's a write access + * handler. + * @param pVM The cross context VM structure. + * @param GCPhys The address of the page with a disabled handler. + * + * @remarks The caller, PGMR3PhysTlbGCPhys2Ptr, must hold the PGM lock. + */ +bool pgmHandlerPhysicalIsAll(PVMCC pVM, RTGCPHYS GCPhys) +{ + PGM_LOCK_VOID(pVM); + PPGMPHYSHANDLER pCur; + int rc = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur); + AssertRCReturnStmt(rc, PGM_UNLOCK(pVM), true); + + /* Only whole pages can be disabled. */ + Assert( pCur->Key <= (GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK) + && pCur->KeyLast >= (GCPhys | GUEST_PAGE_OFFSET_MASK)); + + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + Assert( pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE + || pCurType->enmKind == PGMPHYSHANDLERKIND_ALL + || pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO); /* sanity */ + bool const fRet = pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE; + PGM_UNLOCK(pVM); + return fRet; +} + +#ifdef VBOX_STRICT + +/** + * State structure used by the PGMAssertHandlerAndFlagsInSync() function + * and its AVL enumerators. + */ +typedef struct PGMAHAFIS +{ + /** The current physical address. */ + RTGCPHYS GCPhys; + /** Number of errors. */ + unsigned cErrors; + /** Pointer to the VM. */ + PVM pVM; +} PGMAHAFIS, *PPGMAHAFIS; + + +/** + * Asserts that the handlers+guest-page-tables == ramrange-flags and + * that the physical addresses associated with virtual handlers are correct. + * + * @returns Number of mismatches. + * @param pVM The cross context VM structure. + */ +VMMDECL(unsigned) PGMAssertHandlerAndFlagsInSync(PVMCC pVM) +{ + PPGM pPGM = &pVM->pgm.s; + PGMAHAFIS State; + State.GCPhys = 0; + State.cErrors = 0; + State.pVM = pVM; + + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * Check the RAM flags against the handlers. + */ + PPGMPHYSHANDLERTREE const pPhysHandlerTree = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree; + for (PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRangesX); pRam; pRam = pRam->CTX_SUFF(pNext)) + { + const uint32_t cPages = pRam->cb >> GUEST_PAGE_SHIFT; + for (uint32_t iPage = 0; iPage < cPages; iPage++) + { + PGMPAGE const *pPage = &pRam->aPages[iPage]; + if (PGM_PAGE_HAS_ANY_HANDLERS(pPage)) + { + State.GCPhys = pRam->GCPhys + (iPage << GUEST_PAGE_SHIFT); + + /* + * Physical first - calculate the state based on the handlers + * active on the page, then compare. + */ + if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage)) + { + /* the first */ + PPGMPHYSHANDLER pPhys; + int rc = pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, State.GCPhys, &pPhys); + if (rc == VERR_NOT_FOUND) + { + rc = pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, + State.GCPhys, &pPhys); + if (RT_SUCCESS(rc)) + { + Assert(pPhys->Key >= State.GCPhys); + if (pPhys->Key > (State.GCPhys + GUEST_PAGE_SIZE - 1)) + pPhys = NULL; + } + else + AssertLogRelMsgReturn(rc == VERR_NOT_FOUND, ("rc=%Rrc GCPhys=%RGp\n", rc, State.GCPhys), 999); + } + else + AssertLogRelMsgReturn(RT_SUCCESS(rc), ("rc=%Rrc GCPhys=%RGp\n", rc, State.GCPhys), 999); + + if (pPhys) + { + PCPGMPHYSHANDLERTYPEINT pPhysType = pgmHandlerPhysicalTypeHandleToPtr(pVM, pPhys->hType); + unsigned uState = pPhysType->uState; + bool const fNotInHm = pPhysType->fNotInHm; /* whole pages, so no need to accumulate sub-page configs. */ + + /* more? */ + while (pPhys->KeyLast < (State.GCPhys | GUEST_PAGE_OFFSET_MASK)) + { + PPGMPHYSHANDLER pPhys2; + rc = pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, + pPhys->KeyLast + 1, &pPhys2); + if (rc == VERR_NOT_FOUND) + break; + AssertLogRelMsgReturn(RT_SUCCESS(rc), ("rc=%Rrc KeyLast+1=%RGp\n", rc, pPhys->KeyLast + 1), 999); + if (pPhys2->Key > (State.GCPhys | GUEST_PAGE_OFFSET_MASK)) + break; + PCPGMPHYSHANDLERTYPEINT pPhysType2 = pgmHandlerPhysicalTypeHandleToPtr(pVM, pPhys2->hType); + uState = RT_MAX(uState, pPhysType2->uState); + pPhys = pPhys2; + } + + /* compare.*/ + if ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != uState + && PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED) + { + AssertMsgFailed(("ram range vs phys handler flags mismatch. GCPhys=%RGp state=%d expected=%d %s\n", + State.GCPhys, PGM_PAGE_GET_HNDL_PHYS_STATE(pPage), uState, pPhysType->pszDesc)); + State.cErrors++; + } + AssertMsgStmt(PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) == fNotInHm, + ("ram range vs phys handler flags mismatch. GCPhys=%RGp fNotInHm=%d, %d %s\n", + State.GCPhys, PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage), fNotInHm, pPhysType->pszDesc), + State.cErrors++); + } + else + { + AssertMsgFailed(("ram range vs phys handler mismatch. no handler for GCPhys=%RGp\n", State.GCPhys)); + State.cErrors++; + } + } + } + } /* foreach page in ram range. */ + } /* foreach ram range. */ + + /* + * Do the reverse check for physical handlers. + */ + /** @todo */ + + return State.cErrors; +} + +#endif /* VBOX_STRICT */ + diff --git a/src/VBox/VMM/VMMAll/PGMAllPhys.cpp b/src/VBox/VMM/VMMAll/PGMAllPhys.cpp new file mode 100644 index 00000000..457f7de0 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PGMAllPhys.cpp @@ -0,0 +1,4151 @@ +/* $Id: PGMAllPhys.cpp $ */ +/** @file + * PGM - Page Manager and Monitor, Physical Memory Addressing. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PGM_PHYS +#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */ +#include +#include +#include +#include +#include +#include +#include +#include "PGMInternal.h" +#include +#include "PGMInline.h" +#include +#include +#include +#include +#include +#ifdef IN_RING3 +# include +#endif + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +/** Enable the physical TLB. */ +#define PGM_WITH_PHYS_TLB + +/** @def PGM_HANDLER_PHYS_IS_VALID_STATUS + * Checks if valid physical access handler return code (normal handler, not PF). + * + * Checks if the given strict status code is one of the expected ones for a + * physical access handler in the current context. + * + * @returns true or false. + * @param a_rcStrict The status code. + * @param a_fWrite Whether it is a write or read being serviced. + * + * @remarks We wish to keep the list of statuses here as short as possible. + * When changing, please make sure to update the PGMPhysRead, + * PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too. + */ +#ifdef IN_RING3 +# define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \ + ( (a_rcStrict) == VINF_SUCCESS \ + || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT) +#elif defined(IN_RING0) +#define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \ + ( (a_rcStrict) == VINF_SUCCESS \ + || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \ + \ + || (a_rcStrict) == ((a_fWrite) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ) \ + || (a_rcStrict) == VINF_IOM_R3_MMIO_READ_WRITE \ + || ((a_rcStrict) == VINF_IOM_R3_MMIO_COMMIT_WRITE && (a_fWrite)) \ + \ + || (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR \ + || (a_rcStrict) == VINF_EM_DBG_STOP \ + || (a_rcStrict) == VINF_EM_DBG_EVENT \ + || (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \ + || (a_rcStrict) == VINF_EM_OFF \ + || (a_rcStrict) == VINF_EM_SUSPEND \ + || (a_rcStrict) == VINF_EM_RESET \ + ) +#else +# error "Context?" +#endif + +/** @def PGM_HANDLER_VIRT_IS_VALID_STATUS + * Checks if valid virtual access handler return code (normal handler, not PF). + * + * Checks if the given strict status code is one of the expected ones for a + * virtual access handler in the current context. + * + * @returns true or false. + * @param a_rcStrict The status code. + * @param a_fWrite Whether it is a write or read being serviced. + * + * @remarks We wish to keep the list of statuses here as short as possible. + * When changing, please make sure to update the PGMPhysRead, + * PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too. + */ +#ifdef IN_RING3 +# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \ + ( (a_rcStrict) == VINF_SUCCESS \ + || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT) +#elif defined(IN_RING0) +# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \ + (false /* no virtual handlers in ring-0! */ ) +#else +# error "Context?" +#endif + + + +/** + * Calculate the actual table size. + * + * The memory is layed out like this: + * - PGMPHYSHANDLERTREE (8 bytes) + * - Allocation bitmap (8-byte size align) + * - Slab of PGMPHYSHANDLER. Start is 64 byte aligned. + */ +uint32_t pgmHandlerPhysicalCalcTableSizes(uint32_t *pcEntries, uint32_t *pcbTreeAndBitmap) +{ + /* + * A minimum of 64 entries and a maximum of ~64K. + */ + uint32_t cEntries = *pcEntries; + if (cEntries <= 64) + cEntries = 64; + else if (cEntries >= _64K) + cEntries = _64K; + else + cEntries = RT_ALIGN_32(cEntries, 16); + + /* + * Do the initial calculation. + */ + uint32_t cbBitmap = RT_ALIGN_32(cEntries, 64) / 8; + uint32_t cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64); + uint32_t cbTable = cEntries * sizeof(PGMPHYSHANDLER); + uint32_t cbTotal = cbTreeAndBitmap + cbTable; + + /* + * Align the total and try use up extra space from that. + */ + uint32_t cbTotalAligned = RT_ALIGN_32(cbTotal, RT_MAX(HOST_PAGE_SIZE, _16K)); + uint32_t cAvail = cbTotalAligned - cbTotal; + cAvail /= sizeof(PGMPHYSHANDLER); + if (cAvail >= 1) + for (;;) + { + cbBitmap = RT_ALIGN_32(cEntries, 64) / 8; + cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64); + cbTable = cEntries * sizeof(PGMPHYSHANDLER); + cbTotal = cbTreeAndBitmap + cbTable; + if (cbTotal <= cbTotalAligned) + break; + cEntries--; + Assert(cEntries >= 16); + } + + /* + * Return the result. + */ + *pcbTreeAndBitmap = cbTreeAndBitmap; + *pcEntries = cEntries; + return cbTotalAligned; +} + + +/** + * Looks up a ROM range by its PGMROMRANGE::GCPhys value. + */ +DECLINLINE(PPGMROMRANGE) pgmPhysRomLookupByBase(PVMCC pVM, RTGCPHYS GCPhys) +{ + for (PPGMROMRANGE pRom = pVM->pgm.s.CTX_SUFF(pRomRanges); pRom; pRom = pRom->CTX_SUFF(pNext)) + if (pRom->GCPhys == GCPhys) + return pRom; + return NULL; +} + +#ifndef IN_RING3 + +/** + * @callback_method_impl{FNPGMRZPHYSPFHANDLER, + * \#PF access handler callback for guest ROM range write access.} + * + * @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value. + */ +DECLCALLBACK(VBOXSTRICTRC) pgmPhysRomWritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx, + RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser) + +{ + PPGMROMRANGE const pRom = pgmPhysRomLookupByBase(pVM, uUser); + AssertReturn(pRom, VINF_EM_RAW_EMULATE_INSTR); + uint32_t const iPage = (GCPhysFault - pRom->GCPhys) >> GUEST_PAGE_SHIFT; + int rc; + RT_NOREF(uErrorCode, pvFault); + + Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */ + + Assert(iPage < (pRom->cb >> GUEST_PAGE_SHIFT)); + switch (pRom->aPages[iPage].enmProt) + { + case PGMROMPROT_READ_ROM_WRITE_IGNORE: + case PGMROMPROT_READ_RAM_WRITE_IGNORE: + { + /* + * If it's a simple instruction which doesn't change the cpu state + * we will simply skip it. Otherwise we'll have to defer it to REM. + */ + uint32_t cbOp; + PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState; + rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbOp); + if ( RT_SUCCESS(rc) + && pDis->uCpuMode == DISCPUMODE_32BIT /** @todo why does this matter? */ + && !(pDis->fPrefix & (DISPREFIX_REPNE | DISPREFIX_REP | DISPREFIX_SEG))) + { + switch (pDis->bOpCode) + { + /** @todo Find other instructions we can safely skip, possibly + * adding this kind of detection to DIS or EM. */ + case OP_MOV: + pCtx->rip += cbOp; + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteHandled); + return VINF_SUCCESS; + } + } + break; + } + + case PGMROMPROT_READ_RAM_WRITE_RAM: + pRom->aPages[iPage].LiveSave.fWrittenTo = true; + rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK); + AssertRC(rc); + break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */ + + case PGMROMPROT_READ_ROM_WRITE_RAM: + /* Handle it in ring-3 because it's *way* easier there. */ + pRom->aPages[iPage].LiveSave.fWrittenTo = true; + break; + + default: + AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n", + pRom->aPages[iPage].enmProt, iPage, GCPhysFault), + VERR_IPE_NOT_REACHED_DEFAULT_CASE); + } + + STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteUnhandled); + return VINF_EM_RAW_EMULATE_INSTR; +} + +#endif /* !IN_RING3 */ + + +/** + * @callback_method_impl{FNPGMPHYSHANDLER, + * Access handler callback for ROM write accesses.} + * + * @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value. + */ +DECLCALLBACK(VBOXSTRICTRC) +pgmPhysRomWriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, + PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser) +{ + PPGMROMRANGE const pRom = pgmPhysRomLookupByBase(pVM, uUser); + AssertReturn(pRom, VERR_INTERNAL_ERROR_3); + uint32_t const iPage = (GCPhys - pRom->GCPhys) >> GUEST_PAGE_SHIFT; + Assert(iPage < (pRom->cb >> GUEST_PAGE_SHIFT)); + PPGMROMPAGE const pRomPage = &pRom->aPages[iPage]; + + Log5(("pgmPhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf)); + RT_NOREF(pVCpu, pvPhys, enmOrigin); + + if (enmAccessType == PGMACCESSTYPE_READ) + { + switch (pRomPage->enmProt) + { + /* + * Take the default action. + */ + case PGMROMPROT_READ_ROM_WRITE_IGNORE: + case PGMROMPROT_READ_RAM_WRITE_IGNORE: + case PGMROMPROT_READ_ROM_WRITE_RAM: + case PGMROMPROT_READ_RAM_WRITE_RAM: + return VINF_PGM_HANDLER_DO_DEFAULT; + + default: + AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n", + pRom->aPages[iPage].enmProt, iPage, GCPhys), + VERR_IPE_NOT_REACHED_DEFAULT_CASE); + } + } + else + { + Assert(enmAccessType == PGMACCESSTYPE_WRITE); + switch (pRomPage->enmProt) + { + /* + * Ignore writes. + */ + case PGMROMPROT_READ_ROM_WRITE_IGNORE: + case PGMROMPROT_READ_RAM_WRITE_IGNORE: + return VINF_SUCCESS; + + /* + * Write to the RAM page. + */ + case PGMROMPROT_READ_ROM_WRITE_RAM: + case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's *way* simpler that way. */ + { + /* This should be impossible now, pvPhys doesn't work cross page anylonger. */ + Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> GUEST_PAGE_SHIFT) == iPage); + + /* + * Take the lock, do lazy allocation, map the page and copy the data. + * + * Note that we have to bypass the mapping TLB since it works on + * guest physical addresses and entering the shadow page would + * kind of screw things up... + */ + PGM_LOCK_VOID(pVM); + + PPGMPAGE pShadowPage = &pRomPage->Shadow; + if (!PGMROMPROT_IS_ROM(pRomPage->enmProt)) + { + pShadowPage = pgmPhysGetPage(pVM, GCPhys); + AssertLogRelMsgReturnStmt(pShadowPage, ("%RGp\n", GCPhys), PGM_UNLOCK(pVM), VERR_PGM_PHYS_PAGE_GET_IPE); + } + + void *pvDstPage; + int rc; +#if defined(VBOX_WITH_PGM_NEM_MODE) && defined(IN_RING3) + if (PGM_IS_IN_NEM_MODE(pVM) && PGMROMPROT_IS_ROM(pRomPage->enmProt)) + { + pvDstPage = &pRom->pbR3Alternate[GCPhys - pRom->GCPhys]; + rc = VINF_SUCCESS; + } + else +#endif + { + rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage); + if (RT_SUCCESS(rc)) + pvDstPage = (uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK); + } + if (RT_SUCCESS(rc)) + { + memcpy((uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK), pvBuf, cbBuf); + pRomPage->LiveSave.fWrittenTo = true; + + AssertMsg( rc == VINF_SUCCESS + || ( rc == VINF_PGM_SYNC_CR3 + && VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)) + , ("%Rrc\n", rc)); + rc = VINF_SUCCESS; + } + + PGM_UNLOCK(pVM); + return rc; + } + + default: + AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n", + pRom->aPages[iPage].enmProt, iPage, GCPhys), + VERR_IPE_NOT_REACHED_DEFAULT_CASE); + } + } +} + + +/** + * Common worker for pgmPhysMmio2WriteHandler and pgmPhysMmio2WritePfHandler. + */ +static VBOXSTRICTRC pgmPhysMmio2WriteHandlerCommon(PVMCC pVM, PVMCPUCC pVCpu, uint64_t hMmio2, RTGCPHYS GCPhys, RTGCPTR GCPtr) +{ + /* + * Get the MMIO2 range. + */ + AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3), VERR_INTERNAL_ERROR_3); + AssertReturn(hMmio2 != 0, VERR_INTERNAL_ERROR_3); + PPGMREGMMIO2RANGE pMmio2 = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[hMmio2 - 1]; + Assert(pMmio2->idMmio2 == hMmio2); + AssertReturn((pMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES) == PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES, + VERR_INTERNAL_ERROR_4); + + /* + * Get the page and make sure it's an MMIO2 page. + */ + PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys); + AssertReturn(pPage, VINF_EM_RAW_EMULATE_INSTR); + AssertReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2, VINF_EM_RAW_EMULATE_INSTR); + + /* + * Set the dirty flag so we can avoid scanning all the pages when it isn't dirty. + * (The PGM_PAGE_HNDL_PHYS_STATE_DISABLED handler state indicates that a single + * page is dirty, saving the need for additional storage (bitmap).) + */ + pMmio2->fFlags |= PGMREGMMIO2RANGE_F_IS_DIRTY; + + /* + * Disable the handler for this page. + */ + int rc = PGMHandlerPhysicalPageTempOff(pVM, pMmio2->RamRange.GCPhys, GCPhys & X86_PTE_PG_MASK); + AssertRC(rc); +#ifndef IN_RING3 + if (RT_SUCCESS(rc) && GCPtr != ~(RTGCPTR)0) + { + rc = PGMShwMakePageWritable(pVCpu, GCPtr, PGM_MK_PG_IS_MMIO2 | PGM_MK_PG_IS_WRITE_FAULT); + AssertMsgReturn(rc == VINF_SUCCESS || rc == VERR_PAGE_TABLE_NOT_PRESENT, + ("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", GCPtr, rc), rc); + } +#else + RT_NOREF(pVCpu, GCPtr); +#endif + return VINF_SUCCESS; +} + + +#ifndef IN_RING3 +/** + * @callback_method_impl{FNPGMRZPHYSPFHANDLER, + * \#PF access handler callback for guest MMIO2 dirty page tracing.} + * + * @remarks The @a uUser is the MMIO2 index. + */ +DECLCALLBACK(VBOXSTRICTRC) pgmPhysMmio2WritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx, + RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser) +{ + RT_NOREF(pVCpu, uErrorCode, pCtx); + VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */ + if (RT_SUCCESS(rcStrict)) + { + rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhysFault, pvFault); + PGM_UNLOCK(pVM); + } + return rcStrict; +} +#endif /* !IN_RING3 */ + + +/** + * @callback_method_impl{FNPGMPHYSHANDLER, + * Access handler callback for MMIO2 dirty page tracing.} + * + * @remarks The @a uUser is the MMIO2 index. + */ +DECLCALLBACK(VBOXSTRICTRC) +pgmPhysMmio2WriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, + PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser) +{ + VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */ + if (RT_SUCCESS(rcStrict)) + { + rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhys, ~(RTGCPTR)0); + PGM_UNLOCK(pVM); + if (rcStrict == VINF_SUCCESS) + rcStrict = VINF_PGM_HANDLER_DO_DEFAULT; + } + RT_NOREF(pvPhys, pvBuf, cbBuf, enmAccessType, enmOrigin); + return rcStrict; +} + + +/** + * Invalidates the RAM range TLBs. + * + * @param pVM The cross context VM structure. + */ +void pgmPhysInvalidRamRangeTlbs(PVMCC pVM) +{ + PGM_LOCK_VOID(pVM); + RT_ZERO(pVM->pgm.s.apRamRangesTlbR3); + RT_ZERO(pVM->pgm.s.apRamRangesTlbR0); + PGM_UNLOCK(pVM); +} + + +/** + * Tests if a value of type RTGCPHYS is negative if the type had been signed + * instead of unsigned. + * + * @returns @c true if negative, @c false if positive or zero. + * @param a_GCPhys The value to test. + * @todo Move me to iprt/types.h. + */ +#define RTGCPHYS_IS_NEGATIVE(a_GCPhys) ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1))) + + +/** + * Slow worker for pgmPhysGetRange. + * + * @copydoc pgmPhysGetRange + */ +PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys) +{ + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses)); + + PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree); + while (pRam) + { + RTGCPHYS off = GCPhys - pRam->GCPhys; + if (off < pRam->cb) + { + pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam; + return pRam; + } + if (RTGCPHYS_IS_NEGATIVE(off)) + pRam = pRam->CTX_SUFF(pLeft); + else + pRam = pRam->CTX_SUFF(pRight); + } + return NULL; +} + + +/** + * Slow worker for pgmPhysGetRangeAtOrAbove. + * + * @copydoc pgmPhysGetRangeAtOrAbove + */ +PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys) +{ + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses)); + + PPGMRAMRANGE pLastLeft = NULL; + PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree); + while (pRam) + { + RTGCPHYS off = GCPhys - pRam->GCPhys; + if (off < pRam->cb) + { + pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam; + return pRam; + } + if (RTGCPHYS_IS_NEGATIVE(off)) + { + pLastLeft = pRam; + pRam = pRam->CTX_SUFF(pLeft); + } + else + pRam = pRam->CTX_SUFF(pRight); + } + return pLastLeft; +} + + +/** + * Slow worker for pgmPhysGetPage. + * + * @copydoc pgmPhysGetPage + */ +PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys) +{ + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses)); + + PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree); + while (pRam) + { + RTGCPHYS off = GCPhys - pRam->GCPhys; + if (off < pRam->cb) + { + pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam; + return &pRam->aPages[off >> GUEST_PAGE_SHIFT]; + } + + if (RTGCPHYS_IS_NEGATIVE(off)) + pRam = pRam->CTX_SUFF(pLeft); + else + pRam = pRam->CTX_SUFF(pRight); + } + return NULL; +} + + +/** + * Slow worker for pgmPhysGetPageEx. + * + * @copydoc pgmPhysGetPageEx + */ +int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage) +{ + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses)); + + PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree); + while (pRam) + { + RTGCPHYS off = GCPhys - pRam->GCPhys; + if (off < pRam->cb) + { + pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam; + *ppPage = &pRam->aPages[off >> GUEST_PAGE_SHIFT]; + return VINF_SUCCESS; + } + + if (RTGCPHYS_IS_NEGATIVE(off)) + pRam = pRam->CTX_SUFF(pLeft); + else + pRam = pRam->CTX_SUFF(pRight); + } + + *ppPage = NULL; + return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS; +} + + +/** + * Slow worker for pgmPhysGetPageAndRangeEx. + * + * @copydoc pgmPhysGetPageAndRangeEx + */ +int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam) +{ + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses)); + + PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree); + while (pRam) + { + RTGCPHYS off = GCPhys - pRam->GCPhys; + if (off < pRam->cb) + { + pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam; + *ppRam = pRam; + *ppPage = &pRam->aPages[off >> GUEST_PAGE_SHIFT]; + return VINF_SUCCESS; + } + + if (RTGCPHYS_IS_NEGATIVE(off)) + pRam = pRam->CTX_SUFF(pLeft); + else + pRam = pRam->CTX_SUFF(pRight); + } + + *ppRam = NULL; + *ppPage = NULL; + return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS; +} + + +/** + * Checks if Address Gate 20 is enabled or not. + * + * @returns true if enabled. + * @returns false if disabled. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu) +{ + LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled)); + return pVCpu->pgm.s.fA20Enabled; +} + + +/** + * Validates a GC physical address. + * + * @returns true if valid. + * @returns false if invalid. + * @param pVM The cross context VM structure. + * @param GCPhys The physical address to validate. + */ +VMMDECL(bool) PGMPhysIsGCPhysValid(PVMCC pVM, RTGCPHYS GCPhys) +{ + PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys); + return pPage != NULL; +} + + +/** + * Checks if a GC physical address is a normal page, + * i.e. not ROM, MMIO or reserved. + * + * @returns true if normal. + * @returns false if invalid, ROM, MMIO or reserved page. + * @param pVM The cross context VM structure. + * @param GCPhys The physical address to check. + */ +VMMDECL(bool) PGMPhysIsGCPhysNormal(PVMCC pVM, RTGCPHYS GCPhys) +{ + PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys); + return pPage + && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM; +} + + +/** + * Converts a GC physical address to a HC physical address. + * + * @returns VINF_SUCCESS on success. + * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical + * page but has no physical backing. + * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid + * GC physical address. + * + * @param pVM The cross context VM structure. + * @param GCPhys The GC physical address to convert. + * @param pHCPhys Where to store the HC physical address on success. + */ +VMM_INT_DECL(int) PGMPhysGCPhys2HCPhys(PVMCC pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys) +{ + PGM_LOCK_VOID(pVM); + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage); + if (RT_SUCCESS(rc)) + *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & GUEST_PAGE_OFFSET_MASK); + PGM_UNLOCK(pVM); + return rc; +} + + +/** + * Invalidates all page mapping TLBs. + * + * @param pVM The cross context VM structure. + */ +void pgmPhysInvalidatePageMapTLB(PVMCC pVM) +{ + PGM_LOCK_VOID(pVM); + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushes); + + /* Clear the R3 & R0 TLBs completely. */ + for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR0.aEntries); i++) + { + pVM->pgm.s.PhysTlbR0.aEntries[i].GCPhys = NIL_RTGCPHYS; + pVM->pgm.s.PhysTlbR0.aEntries[i].pPage = 0; + pVM->pgm.s.PhysTlbR0.aEntries[i].pv = 0; + } + + for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++) + { + pVM->pgm.s.PhysTlbR3.aEntries[i].GCPhys = NIL_RTGCPHYS; + pVM->pgm.s.PhysTlbR3.aEntries[i].pPage = 0; + pVM->pgm.s.PhysTlbR3.aEntries[i].pMap = 0; + pVM->pgm.s.PhysTlbR3.aEntries[i].pv = 0; + } + + IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID); + PGM_UNLOCK(pVM); +} + + +/** + * Invalidates a page mapping TLB entry + * + * @param pVM The cross context VM structure. + * @param GCPhys GCPhys entry to flush + * + * @note Caller is responsible for calling IEMTlbInvalidateAllPhysicalAllCpus + * when needed. + */ +void pgmPhysInvalidatePageMapTLBEntry(PVMCC pVM, RTGCPHYS GCPhys) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushEntry); + + unsigned const idx = PGM_PAGER3MAPTLB_IDX(GCPhys); + + pVM->pgm.s.PhysTlbR0.aEntries[idx].GCPhys = NIL_RTGCPHYS; + pVM->pgm.s.PhysTlbR0.aEntries[idx].pPage = 0; + pVM->pgm.s.PhysTlbR0.aEntries[idx].pv = 0; + + pVM->pgm.s.PhysTlbR3.aEntries[idx].GCPhys = NIL_RTGCPHYS; + pVM->pgm.s.PhysTlbR3.aEntries[idx].pPage = 0; + pVM->pgm.s.PhysTlbR3.aEntries[idx].pMap = 0; + pVM->pgm.s.PhysTlbR3.aEntries[idx].pv = 0; +} + + +/** + * Makes sure that there is at least one handy page ready for use. + * + * This will also take the appropriate actions when reaching water-marks. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_EM_NO_MEMORY if we're really out of memory. + * + * @param pVM The cross context VM structure. + * + * @remarks Must be called from within the PGM critical section. It may + * nip back to ring-3/0 in some cases. + */ +static int pgmPhysEnsureHandyPage(PVMCC pVM) +{ + AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages)); + + /* + * Do we need to do anything special? + */ +#ifdef IN_RING3 + if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC)) +#else + if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3)) +#endif + { + /* + * Allocate pages only if we're out of them, or in ring-3, almost out. + */ +#ifdef IN_RING3 + if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC) +#else + if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC) +#endif + { + Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n", + pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) )); +#ifdef IN_RING3 + int rc = PGMR3PhysAllocateHandyPages(pVM); +#else + int rc = pgmR0PhysAllocateHandyPages(pVM, VMMGetCpuId(pVM), false /*fRing3*/); +#endif + if (RT_UNLIKELY(rc != VINF_SUCCESS)) + { + if (RT_FAILURE(rc)) + return rc; + AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS); + if (!pVM->pgm.s.cHandyPages) + { + LogRel(("PGM: no more handy pages!\n")); + return VERR_EM_NO_MEMORY; + } + Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES)); + Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)); +#ifndef IN_RING3 + VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */ +#endif + } + AssertMsgReturn( pVM->pgm.s.cHandyPages > 0 + && pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), + ("%u\n", pVM->pgm.s.cHandyPages), + VERR_PGM_HANDY_PAGE_IPE); + } + else + { + if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF) + VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES); +#ifndef IN_RING3 + if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3) + { + Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages))); + VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); + } +#endif + } + } + + return VINF_SUCCESS; +} + + +/** + * Replace a zero or shared page with new page that we can write to. + * + * @returns The following VBox status codes. + * @retval VINF_SUCCESS on success, pPage is modified. + * @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending. + * @retval VERR_EM_NO_MEMORY if we're totally out of memory. + * + * @todo Propagate VERR_EM_NO_MEMORY up the call tree. + * + * @param pVM The cross context VM structure. + * @param pPage The physical page tracking structure. This will + * be modified on success. + * @param GCPhys The address of the page. + * + * @remarks Must be called from within the PGM critical section. It may + * nip back to ring-3/0 in some cases. + * + * @remarks This function shouldn't really fail, however if it does + * it probably means we've screwed up the size of handy pages and/or + * the low-water mark. Or, that some device I/O is causing a lot of + * pages to be allocated while while the host is in a low-memory + * condition. This latter should be handled elsewhere and in a more + * controlled manner, it's on the @bugref{3170} todo list... + */ +int pgmPhysAllocPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys) +{ + LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys)); + + /* + * Prereqs. + */ + PGM_LOCK_ASSERT_OWNER(pVM); + AssertMsg(PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys)); + Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); + +# ifdef PGM_WITH_LARGE_PAGES + /* + * Try allocate a large page if applicable. + */ + if ( PGMIsUsingLargePages(pVM) + && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM + && !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Implement large pages support. */ + { + RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK; + PPGMPAGE pBasePage; + + int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage); + AssertRCReturn(rc, rc); /* paranoia; can't happen. */ + if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE) + { + rc = pgmPhysAllocLargePage(pVM, GCPhys); + if (rc == VINF_SUCCESS) + return rc; + } + /* Mark the base as type page table, so we don't check over and over again. */ + PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT); + + /* fall back to 4KB pages. */ + } +# endif + + /* + * Flush any shadow page table mappings of the page. + * When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any. + */ + bool fFlushTLBs = false; + int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /*fFlushTLBs*/, &fFlushTLBs); + AssertMsgReturn(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS); + + /* + * Ensure that we've got a page handy, take it and use it. + */ + int rc2 = pgmPhysEnsureHandyPage(pVM); + if (RT_FAILURE(rc2)) + { + if (fFlushTLBs) + PGM_INVL_ALL_VCPU_TLBS(pVM); + Assert(rc2 == VERR_EM_NO_MEMORY); + return rc2; + } + /* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */ + PGM_LOCK_ASSERT_OWNER(pVM); + AssertMsg(PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys)); + Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); + + uint32_t iHandyPage = --pVM->pgm.s.cHandyPages; + AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage)); + Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_GMMPAGEDESC_PHYS); + Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK)); + Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID); + Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID); + + /* + * There are one or two action to be taken the next time we allocate handy pages: + * - Tell the GMM (global memory manager) what the page is being used for. + * (Speeds up replacement operations - sharing and defragmenting.) + * - If the current backing is shared, it must be freed. + */ + const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys; + pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + + void const *pvSharedPage = NULL; + if (PGM_PAGE_IS_SHARED(pPage)) + { + /* Mark this shared page for freeing/dereferencing. */ + pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage); + Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID); + + Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage), + GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys)); + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageReplaceShared)); + pVM->pgm.s.cSharedPages--; + + /* Grab the address of the page so we can make a copy later on. (safe) */ + rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage); + AssertRC(rc); + } + else + { + Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys)); + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatRZPageReplaceZero); + pVM->pgm.s.cZeroPages--; + } + + /* + * Do the PGMPAGE modifications. + */ + pVM->pgm.s.cPrivatePages++; + PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys); + PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage); + PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED); + PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT); + pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys); + IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID); + + /* Copy the shared page contents to the replacement page. */ + if (pvSharedPage) + { + /* Get the virtual address of the new page. */ + PGMPAGEMAPLOCK PgMpLck; + void *pvNewPage; + rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc); + if (RT_SUCCESS(rc)) + { + memcpy(pvNewPage, pvSharedPage, GUEST_PAGE_SIZE); /** @todo todo write ASMMemCopyPage */ + pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck); + } + } + + if ( fFlushTLBs + && rc != VINF_PGM_GCPHYS_ALIASED) + PGM_INVL_ALL_VCPU_TLBS(pVM); + + /* + * Notify NEM about the mapping change for this page. + * + * Note! Shadow ROM pages are complicated as they can definitely be + * allocated while not visible, so play safe. + */ + if (VM_IS_NEM_ENABLED(pVM)) + { + PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage); + if ( enmType != PGMPAGETYPE_ROM_SHADOW + || pgmPhysGetPage(pVM, GCPhys) == pPage) + { + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); + rc2 = NEMHCNotifyPhysPageAllocated(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, HCPhys, + pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State); + if (RT_SUCCESS(rc)) + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + else + rc = rc2; + } + } + + return rc; +} + +#ifdef PGM_WITH_LARGE_PAGES + +/** + * Replace a 2 MB range of zero pages with new pages that we can write to. + * + * @returns The following VBox status codes. + * @retval VINF_SUCCESS on success, pPage is modified. + * @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending. + * @retval VERR_EM_NO_MEMORY if we're totally out of memory. + * + * @todo Propagate VERR_EM_NO_MEMORY up the call tree. + * + * @param pVM The cross context VM structure. + * @param GCPhys The address of the page. + * + * @remarks Must be called from within the PGM critical section. It may block + * on GMM and host mutexes/locks, leaving HM context. + */ +int pgmPhysAllocLargePage(PVMCC pVM, RTGCPHYS GCPhys) +{ + RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK; + LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase)); + Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */ + + /* + * Check Prereqs. + */ + PGM_LOCK_ASSERT_OWNER(pVM); + Assert(PGMIsUsingLargePages(pVM)); + + /* + * All the pages must be unallocated RAM pages, i.e. mapping the ZERO page. + */ + PPGMPAGE pFirstPage; + int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage); + if ( RT_SUCCESS(rc) + && PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM + && PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO) + { + /* + * Further they should have PDE type set to PGM_PAGE_PDE_TYPE_DONTCARE, + * since they are unallocated. + */ + unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage); + Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE); + if (uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE) + { + /* + * Now, make sure all the other pages in the 2 MB is in the same state. + */ + GCPhys = GCPhysBase; + unsigned cLeft = _2M / GUEST_PAGE_SIZE; + while (cLeft-- > 0) + { + PPGMPAGE pSubPage = pgmPhysGetPage(pVM, GCPhys); + if ( pSubPage + && PGM_PAGE_GET_TYPE(pSubPage) == PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */ + && PGM_PAGE_GET_STATE(pSubPage) == PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */ + { + Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE); + GCPhys += GUEST_PAGE_SIZE; + } + else + { + LogFlow(("pgmPhysAllocLargePage: Found page %RGp with wrong attributes (type=%d; state=%d); cancel check.\n", + GCPhys, pSubPage ? PGM_PAGE_GET_TYPE(pSubPage) : -1, pSubPage ? PGM_PAGE_GET_STATE(pSubPage) : -1)); + + /* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */ + STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused); + PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT); + return VERR_PGM_INVALID_LARGE_PAGE_RANGE; + } + } + + /* + * Do the allocation. + */ +# ifdef IN_RING3 + rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_PAGE, GCPhysBase, NULL); +# elif defined(IN_RING0) + rc = pgmR0PhysAllocateLargePage(pVM, VMMGetCpuId(pVM), GCPhysBase); +# else +# error "Port me" +# endif + if (RT_SUCCESS(rc)) + { + Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED); + pVM->pgm.s.cLargePages++; + return VINF_SUCCESS; + } + + /* If we fail once, it most likely means the host's memory is too + fragmented; don't bother trying again. */ + LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc)); + return rc; + } + } + return VERR_PGM_INVALID_LARGE_PAGE_RANGE; +} + + +/** + * Recheck the entire 2 MB range to see if we can use it again as a large page. + * + * @returns The following VBox status codes. + * @retval VINF_SUCCESS on success, the large page can be used again + * @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused + * + * @param pVM The cross context VM structure. + * @param GCPhys The address of the page. + * @param pLargePage Page structure of the base page + */ +int pgmPhysRecheckLargePage(PVMCC pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage) +{ + STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck); + + Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */ + + AssertCompile(X86_PDE2M_PAE_PG_MASK == EPT_PDE2M_PG_MASK); /* Paranoia: Caller uses this for guest EPT tables as well. */ + GCPhys &= X86_PDE2M_PAE_PG_MASK; + + /* Check the base page. */ + Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED); + if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED + || PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM + || PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE) + { + LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage))); + return VERR_PGM_INVALID_LARGE_PAGE_RANGE; + } + + STAM_PROFILE_START(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a); + /* Check all remaining pages in the 2 MB range. */ + unsigned i; + GCPhys += GUEST_PAGE_SIZE; + for (i = 1; i < _2M / GUEST_PAGE_SIZE; i++) + { + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage); + AssertRCBreak(rc); + + if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED + || PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE + || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM + || PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE) + { + LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage))); + break; + } + + GCPhys += GUEST_PAGE_SIZE; + } + STAM_PROFILE_STOP(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a); + + if (i == _2M / GUEST_PAGE_SIZE) + { + PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE); + pVM->pgm.s.cLargePagesDisabled--; + Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M)); + return VINF_SUCCESS; + } + + return VERR_PGM_INVALID_LARGE_PAGE_RANGE; +} + +#endif /* PGM_WITH_LARGE_PAGES */ + + +/** + * Deal with a write monitored page. + * + * @param pVM The cross context VM structure. + * @param pPage The physical page tracking structure. + * @param GCPhys The guest physical address of the page. + * PGMPhysReleasePageMappingLock() passes NIL_RTGCPHYS in a + * very unlikely situation where it is okay that we let NEM + * fix the page access in a lazy fasion. + * + * @remarks Called from within the PGM critical section. + */ +void pgmPhysPageMakeWriteMonitoredWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys) +{ + Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED); + PGM_PAGE_SET_WRITTEN_TO(pVM, pPage); + PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED); + Assert(pVM->pgm.s.cMonitoredPages > 0); + pVM->pgm.s.cMonitoredPages--; + pVM->pgm.s.cWrittenToPages++; + +#ifdef VBOX_WITH_NATIVE_NEM + /* + * Notify NEM about the protection change so we won't spin forever. + * + * Note! NEM need to be handle to lazily correct page protection as we cannot + * really get it 100% right here it seems. The page pool does this too. + */ + if (VM_IS_NEM_ENABLED(pVM) && GCPhys != NIL_RTGCPHYS) + { + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage); + PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage); + PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys); + NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage), + pRam ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhys) : NULL, + pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State); + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + } +#else + RT_NOREF(GCPhys); +#endif +} + + +/** + * Deal with pages that are not writable, i.e. not in the ALLOCATED state. + * + * @returns VBox strict status code. + * @retval VINF_SUCCESS on success. + * @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * + * @param pVM The cross context VM structure. + * @param pPage The physical page tracking structure. + * @param GCPhys The address of the page. + * + * @remarks Called from within the PGM critical section. + */ +int pgmPhysPageMakeWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + switch (PGM_PAGE_GET_STATE(pPage)) + { + case PGM_PAGE_STATE_WRITE_MONITORED: + pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys); + RT_FALL_THRU(); + default: /* to shut up GCC */ + case PGM_PAGE_STATE_ALLOCATED: + return VINF_SUCCESS; + + /* + * Zero pages can be dummy pages for MMIO or reserved memory, + * so we need to check the flags before joining cause with + * shared page replacement. + */ + case PGM_PAGE_STATE_ZERO: + if (PGM_PAGE_IS_MMIO(pPage)) + return VERR_PGM_PHYS_PAGE_RESERVED; + RT_FALL_THRU(); + case PGM_PAGE_STATE_SHARED: + return pgmPhysAllocPage(pVM, pPage, GCPhys); + + /* Not allowed to write to ballooned pages. */ + case PGM_PAGE_STATE_BALLOONED: + return VERR_PGM_PHYS_PAGE_BALLOONED; + } +} + + +/** + * Internal usage: Map the page specified by its GMM ID. + * + * This is similar to pgmPhysPageMap + * + * @returns VBox status code. + * + * @param pVM The cross context VM structure. + * @param idPage The Page ID. + * @param HCPhys The physical address (for SUPR0HCPhysToVirt). + * @param ppv Where to store the mapping address. + * + * @remarks Called from within the PGM critical section. The mapping is only + * valid while you are inside this section. + */ +int pgmPhysPageMapByPageID(PVMCC pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv) +{ + /* + * Validation. + */ + PGM_LOCK_ASSERT_OWNER(pVM); + AssertReturn(HCPhys && !(HCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER); + const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT; + AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER); + +#ifdef IN_RING0 +# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM + return SUPR0HCPhysToVirt(HCPhys & ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK, ppv); +# else + return GMMR0PageIdToVirt(pVM, idPage, ppv); +# endif + +#else + /* + * Find/make Chunk TLB entry for the mapping chunk. + */ + PPGMCHUNKR3MAP pMap; + PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)]; + if (pTlbe->idChunk == idChunk) + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits)); + pMap = pTlbe->pChunk; + } + else + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses)); + + /* + * Find the chunk, map it if necessary. + */ + pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk); + if (pMap) + pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow; + else + { + int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap); + if (RT_FAILURE(rc)) + return rc; + } + + /* + * Enter it into the Chunk TLB. + */ + pTlbe->idChunk = idChunk; + pTlbe->pChunk = pMap; + } + + *ppv = (uint8_t *)pMap->pv + ((idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT); + return VINF_SUCCESS; +#endif +} + + +/** + * Maps a page into the current virtual address space so it can be accessed. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * + * @param pVM The cross context VM structure. + * @param pPage The physical page tracking structure. + * @param GCPhys The address of the page. + * @param ppMap Where to store the address of the mapping tracking structure. + * @param ppv Where to store the mapping address of the page. The page + * offset is masked off! + * + * @remarks Called from within the PGM critical section. + */ +static int pgmPhysPageMapCommon(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + NOREF(GCPhys); + + /* + * Special cases: MMIO2, ZERO and specially aliased MMIO pages. + */ + if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2 + || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO) + { + /* Decode the page id to a page in a MMIO2 ram range. */ + uint8_t idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage)); + uint32_t iPage = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage)); + AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)), + ("idMmio2=%u size=%u type=%u GCPHys=%#RGp Id=%u State=%u", idMmio2, + RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)), PGM_PAGE_GET_TYPE(pPage), GCPhys, + pPage->s.idPage, pPage->s.uStateY), + VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE); + PPGMREGMMIO2RANGE pMmio2Range = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[idMmio2 - 1]; + AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE); + AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE); + AssertLogRelReturn(iPage < (pMmio2Range->RamRange.cb >> GUEST_PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE); + *ppMap = NULL; +# if defined(IN_RING0) && defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM) + return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv); +# elif defined(IN_RING0) + *ppv = (uint8_t *)pMmio2Range->pvR0 + ((uintptr_t)iPage << GUEST_PAGE_SHIFT); + return VINF_SUCCESS; +# else + *ppv = (uint8_t *)pMmio2Range->RamRange.pvR3 + ((uintptr_t)iPage << GUEST_PAGE_SHIFT); + return VINF_SUCCESS; +# endif + } + +# ifdef VBOX_WITH_PGM_NEM_MODE + if (pVM->pgm.s.fNemMode) + { +# ifdef IN_RING3 + /* + * Find the corresponding RAM range and use that to locate the mapping address. + */ + /** @todo Use the page ID for some kind of indexing as we do with MMIO2 above. */ + PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhys); + AssertLogRelMsgReturn(pRam, ("%RTGp\n", GCPhys), VERR_INTERNAL_ERROR_3); + size_t const idxPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT; + Assert(pPage == &pRam->aPages[idxPage]); + *ppMap = NULL; + *ppv = (uint8_t *)pRam->pvR3 + (idxPage << GUEST_PAGE_SHIFT); + return VINF_SUCCESS; +# else + AssertFailedReturn(VERR_INTERNAL_ERROR_2); +# endif + } +# endif + + const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage); + if (idChunk == NIL_GMM_CHUNKID) + { + AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage), + VERR_PGM_PHYS_PAGE_MAP_IPE_1); + if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage)) + { + AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage), + VERR_PGM_PHYS_PAGE_MAP_IPE_3); + AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage), + VERR_PGM_PHYS_PAGE_MAP_IPE_4); + *ppv = pVM->pgm.s.abZeroPg; + } + else + *ppv = pVM->pgm.s.abZeroPg; + *ppMap = NULL; + return VINF_SUCCESS; + } + +# if defined(IN_RING0) && defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM) + /* + * Just use the physical address. + */ + *ppMap = NULL; + return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv); + +# elif defined(IN_RING0) + /* + * Go by page ID thru GMMR0. + */ + *ppMap = NULL; + return GMMR0PageIdToVirt(pVM, PGM_PAGE_GET_PAGEID(pPage), ppv); + +# else + /* + * Find/make Chunk TLB entry for the mapping chunk. + */ + PPGMCHUNKR3MAP pMap; + PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)]; + if (pTlbe->idChunk == idChunk) + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits)); + pMap = pTlbe->pChunk; + AssertPtr(pMap->pv); + } + else + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses)); + + /* + * Find the chunk, map it if necessary. + */ + pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk); + if (pMap) + { + AssertPtr(pMap->pv); + pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow; + } + else + { + int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap); + if (RT_FAILURE(rc)) + return rc; + AssertPtr(pMap->pv); + } + + /* + * Enter it into the Chunk TLB. + */ + pTlbe->idChunk = idChunk; + pTlbe->pChunk = pMap; + } + + *ppv = (uint8_t *)pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << GUEST_PAGE_SHIFT); + *ppMap = pMap; + return VINF_SUCCESS; +# endif /* !IN_RING0 */ +} + + +/** + * Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable. + * + * This is typically used is paths where we cannot use the TLB methods (like ROM + * pages) or where there is no point in using them since we won't get many hits. + * + * @returns VBox strict status code. + * @retval VINF_SUCCESS on success. + * @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * + * @param pVM The cross context VM structure. + * @param pPage The physical page tracking structure. + * @param GCPhys The address of the page. + * @param ppv Where to store the mapping address of the page. The page + * offset is masked off! + * + * @remarks Called from within the PGM critical section. The mapping is only + * valid while you are inside section. + */ +int pgmPhysPageMakeWritableAndMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv) +{ + int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys); + if (RT_SUCCESS(rc)) + { + AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc)); + PPGMPAGEMAP pMapIgnore; + int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv); + if (RT_FAILURE(rc2)) /* preserve rc */ + rc = rc2; + } + return rc; +} + + +/** + * Maps a page into the current virtual address space so it can be accessed for + * both writing and reading. + * + * This is typically used is paths where we cannot use the TLB methods (like ROM + * pages) or where there is no point in using them since we won't get many hits. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * + * @param pVM The cross context VM structure. + * @param pPage The physical page tracking structure. Must be in the + * allocated state. + * @param GCPhys The address of the page. + * @param ppv Where to store the mapping address of the page. The page + * offset is masked off! + * + * @remarks Called from within the PGM critical section. The mapping is only + * valid while you are inside section. + */ +int pgmPhysPageMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv) +{ + Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED); + PPGMPAGEMAP pMapIgnore; + return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv); +} + + +/** + * Maps a page into the current virtual address space so it can be accessed for + * reading. + * + * This is typically used is paths where we cannot use the TLB methods (like ROM + * pages) or where there is no point in using them since we won't get many hits. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * + * @param pVM The cross context VM structure. + * @param pPage The physical page tracking structure. + * @param GCPhys The address of the page. + * @param ppv Where to store the mapping address of the page. The page + * offset is masked off! + * + * @remarks Called from within the PGM critical section. The mapping is only + * valid while you are inside this section. + */ +int pgmPhysPageMapReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv) +{ + PPGMPAGEMAP pMapIgnore; + return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv); +} + + +/** + * Load a guest page into the ring-3 physical TLB. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success + * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. + * @param pVM The cross context VM structure. + * @param GCPhys The guest physical address in question. + */ +int pgmPhysPageLoadIntoTlb(PVMCC pVM, RTGCPHYS GCPhys) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * Find the ram range and page and hand it over to the with-page function. + * 99.8% of requests are expected to be in the first range. + */ + PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys); + if (!pPage) + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses)); + return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS; + } + + return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys); +} + + +/** + * Load a guest page into the ring-3 physical TLB. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success + * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. + * + * @param pVM The cross context VM structure. + * @param pPage Pointer to the PGMPAGE structure corresponding to + * GCPhys. + * @param GCPhys The guest physical address in question. + */ +int pgmPhysPageLoadIntoTlbWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses)); + + /* + * Map the page. + * Make a special case for the zero page as it is kind of special. + */ + PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)]; + if ( !PGM_PAGE_IS_ZERO(pPage) + && !PGM_PAGE_IS_BALLOONED(pPage)) + { + void *pv; + PPGMPAGEMAP pMap; + int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv); + if (RT_FAILURE(rc)) + return rc; +# ifndef IN_RING0 + pTlbe->pMap = pMap; +# endif + pTlbe->pv = pv; + Assert(!((uintptr_t)pTlbe->pv & GUEST_PAGE_OFFSET_MASK)); + } + else + { + AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage)); +# ifndef IN_RING0 + pTlbe->pMap = NULL; +# endif + pTlbe->pv = pVM->pgm.s.abZeroPg; + } +# ifdef PGM_WITH_PHYS_TLB + if ( PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW + || PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM) + pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK; + else + pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */ +# else + pTlbe->GCPhys = NIL_RTGCPHYS; +# endif + pTlbe->pPage = pPage; + return VINF_SUCCESS; +} + + +/** + * Internal version of PGMPhysGCPhys2CCPtr that expects the caller to + * own the PGM lock and therefore not need to lock the mapped page. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. + * + * @param pVM The cross context VM structure. + * @param GCPhys The guest physical address of the page that should be mapped. + * @param pPage Pointer to the PGMPAGE structure for the page. + * @param ppv Where to store the address corresponding to GCPhys. + * + * @internal + * @deprecated Use pgmPhysGCPhys2CCPtrInternalEx. + */ +int pgmPhysGCPhys2CCPtrInternalDepr(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv) +{ + int rc; + AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM); + PGM_LOCK_ASSERT_OWNER(pVM); + pVM->pgm.s.cDeprecatedPageLocks++; + + /* + * Make sure the page is writable. + */ + if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)) + { + rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys); + if (RT_FAILURE(rc)) + return rc; + AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc)); + } + Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0); + + /* + * Get the mapping address. + */ + PPGMPAGEMAPTLBE pTlbe; + rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe); + if (RT_FAILURE(rc)) + return rc; + *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK)); + return VINF_SUCCESS; +} + + +/** + * Locks a page mapping for writing. + * + * @param pVM The cross context VM structure. + * @param pPage The page. + * @param pTlbe The mapping TLB entry for the page. + * @param pLock The lock structure (output). + */ +DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock) +{ +# ifndef IN_RING0 + PPGMPAGEMAP pMap = pTlbe->pMap; + if (pMap) + pMap->cRefs++; +# else + RT_NOREF(pTlbe); +# endif + + unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage); + if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1)) + { + if (cLocks == 0) + pVM->pgm.s.cWriteLockedPages++; + PGM_PAGE_INC_WRITE_LOCKS(pPage); + } + else if (cLocks != PGM_PAGE_MAX_LOCKS) + { + PGM_PAGE_INC_WRITE_LOCKS(pPage); + AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage)); +# ifndef IN_RING0 + if (pMap) + pMap->cRefs++; /* Extra ref to prevent it from going away. */ +# endif + } + + pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE; +# ifndef IN_RING0 + pLock->pvMap = pMap; +# else + pLock->pvMap = NULL; +# endif +} + +/** + * Locks a page mapping for reading. + * + * @param pVM The cross context VM structure. + * @param pPage The page. + * @param pTlbe The mapping TLB entry for the page. + * @param pLock The lock structure (output). + */ +DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock) +{ +# ifndef IN_RING0 + PPGMPAGEMAP pMap = pTlbe->pMap; + if (pMap) + pMap->cRefs++; +# else + RT_NOREF(pTlbe); +# endif + + unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage); + if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1)) + { + if (cLocks == 0) + pVM->pgm.s.cReadLockedPages++; + PGM_PAGE_INC_READ_LOCKS(pPage); + } + else if (cLocks != PGM_PAGE_MAX_LOCKS) + { + PGM_PAGE_INC_READ_LOCKS(pPage); + AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage)); +# ifndef IN_RING0 + if (pMap) + pMap->cRefs++; /* Extra ref to prevent it from going away. */ +# endif + } + + pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ; +# ifndef IN_RING0 + pLock->pvMap = pMap; +# else + pLock->pvMap = NULL; +# endif +} + + +/** + * Internal version of PGMPhysGCPhys2CCPtr that expects the caller to + * own the PGM lock and have access to the page structure. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. + * + * @param pVM The cross context VM structure. + * @param GCPhys The guest physical address of the page that should be mapped. + * @param pPage Pointer to the PGMPAGE structure for the page. + * @param ppv Where to store the address corresponding to GCPhys. + * @param pLock Where to store the lock information that + * pgmPhysReleaseInternalPageMappingLock needs. + * + * @internal + */ +int pgmPhysGCPhys2CCPtrInternal(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock) +{ + int rc; + AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM); + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * Make sure the page is writable. + */ + if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)) + { + rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys); + if (RT_FAILURE(rc)) + return rc; + AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc)); + } + Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0); + + /* + * Do the job. + */ + PPGMPAGEMAPTLBE pTlbe; + rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe); + if (RT_FAILURE(rc)) + return rc; + pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock); + *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK)); + return VINF_SUCCESS; +} + + +/** + * Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to + * own the PGM lock and have access to the page structure. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. + * + * @param pVM The cross context VM structure. + * @param GCPhys The guest physical address of the page that should be mapped. + * @param pPage Pointer to the PGMPAGE structure for the page. + * @param ppv Where to store the address corresponding to GCPhys. + * @param pLock Where to store the lock information that + * pgmPhysReleaseInternalPageMappingLock needs. + * + * @internal + */ +int pgmPhysGCPhys2CCPtrInternalReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock) +{ + AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM); + PGM_LOCK_ASSERT_OWNER(pVM); + Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0); + + /* + * Do the job. + */ + PPGMPAGEMAPTLBE pTlbe; + int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe); + if (RT_FAILURE(rc)) + return rc; + pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock); + *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK)); + return VINF_SUCCESS; +} + + +/** + * Requests the mapping of a guest page into the current context. + * + * This API should only be used for very short term, as it will consume scarse + * resources (R0 and GC) in the mapping cache. When you're done with the page, + * call PGMPhysReleasePageMappingLock() ASAP to release it. + * + * This API will assume your intention is to write to the page, and will + * therefore replace shared and zero pages. If you do not intend to modify + * the page, use the PGMPhysGCPhys2CCPtrReadOnly() API. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. + * + * @param pVM The cross context VM structure. + * @param GCPhys The guest physical address of the page that should be + * mapped. + * @param ppv Where to store the address corresponding to GCPhys. + * @param pLock Where to store the lock information that + * PGMPhysReleasePageMappingLock needs. + * + * @remarks The caller is responsible for dealing with access handlers. + * @todo Add an informational return code for pages with access handlers? + * + * @remark Avoid calling this API from within critical sections (other than + * the PGM one) because of the deadlock risk. External threads may + * need to delegate jobs to the EMTs. + * @remarks Only one page is mapped! Make no assumption about what's after or + * before the returned page! + * @thread Any thread. + */ +VMM_INT_DECL(int) PGMPhysGCPhys2CCPtr(PVMCC pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock) +{ + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + /* + * Query the Physical TLB entry for the page (may fail). + */ + PPGMPAGEMAPTLBE pTlbe; + rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe); + if (RT_SUCCESS(rc)) + { + /* + * If the page is shared, the zero page, or being write monitored + * it must be converted to a page that's writable if possible. + */ + PPGMPAGE pPage = pTlbe->pPage; + if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)) + { + rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys); + if (RT_SUCCESS(rc)) + { + AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc)); + rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe); + } + } + if (RT_SUCCESS(rc)) + { + /* + * Now, just perform the locking and calculate the return address. + */ + pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock); + *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK)); + } + } + + PGM_UNLOCK(pVM); + return rc; +} + + +/** + * Requests the mapping of a guest page into the current context. + * + * This API should only be used for very short term, as it will consume scarse + * resources (R0 and GC) in the mapping cache. When you're done with the page, + * call PGMPhysReleasePageMappingLock() ASAP to release it. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. + * + * @param pVM The cross context VM structure. + * @param GCPhys The guest physical address of the page that should be + * mapped. + * @param ppv Where to store the address corresponding to GCPhys. + * @param pLock Where to store the lock information that + * PGMPhysReleasePageMappingLock needs. + * + * @remarks The caller is responsible for dealing with access handlers. + * @todo Add an informational return code for pages with access handlers? + * + * @remarks Avoid calling this API from within critical sections (other than + * the PGM one) because of the deadlock risk. + * @remarks Only one page is mapped! Make no assumption about what's after or + * before the returned page! + * @thread Any thread. + */ +VMM_INT_DECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVMCC pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock) +{ + int rc = PGM_LOCK(pVM); + AssertRCReturn(rc, rc); + + /* + * Query the Physical TLB entry for the page (may fail). + */ + PPGMPAGEMAPTLBE pTlbe; + rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe); + if (RT_SUCCESS(rc)) + { + /* MMIO pages doesn't have any readable backing. */ + PPGMPAGE pPage = pTlbe->pPage; + if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))) + rc = VERR_PGM_PHYS_PAGE_RESERVED; + else + { + /* + * Now, just perform the locking and calculate the return address. + */ + pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock); + *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK)); + } + } + + PGM_UNLOCK(pVM); + return rc; +} + + +/** + * Requests the mapping of a guest page given by virtual address into the current context. + * + * This API should only be used for very short term, as it will consume + * scarse resources (R0 and GC) in the mapping cache. When you're done + * with the page, call PGMPhysReleasePageMappingLock() ASAP to release it. + * + * This API will assume your intention is to write to the page, and will + * therefore replace shared and zero pages. If you do not intend to modify + * the page, use the PGMPhysGCPtr2CCPtrReadOnly() API. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present. + * @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr The guest physical address of the page that should be + * mapped. + * @param ppv Where to store the address corresponding to GCPhys. + * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs. + * + * @remark Avoid calling this API from within critical sections (other than + * the PGM one) because of the deadlock risk. + * @thread EMT + */ +VMM_INT_DECL(int) PGMPhysGCPtr2CCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock) +{ + VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM)); + RTGCPHYS GCPhys; + int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys); + if (RT_SUCCESS(rc)) + rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock); + return rc; +} + + +/** + * Requests the mapping of a guest page given by virtual address into the current context. + * + * This API should only be used for very short term, as it will consume + * scarse resources (R0 and GC) in the mapping cache. When you're done + * with the page, call PGMPhysReleasePageMappingLock() ASAP to release it. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present. + * @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present. + * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing. + * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address. + * + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr The guest physical address of the page that should be + * mapped. + * @param ppv Where to store the address corresponding to GCPtr. + * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs. + * + * @remark Avoid calling this API from within critical sections (other than + * the PGM one) because of the deadlock risk. + * @thread EMT + */ +VMM_INT_DECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPUCC pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock) +{ + VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM)); + RTGCPHYS GCPhys; + int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys); + if (RT_SUCCESS(rc)) + rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock); + return rc; +} + + +/** + * Release the mapping of a guest page. + * + * This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly + * PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly. + * + * @param pVM The cross context VM structure. + * @param pLock The lock structure initialized by the mapping function. + */ +VMMDECL(void) PGMPhysReleasePageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock) +{ +# ifndef IN_RING0 + PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap; +# endif + PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK); + bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE; + + pLock->uPageAndType = 0; + pLock->pvMap = NULL; + + PGM_LOCK_VOID(pVM); + if (fWriteLock) + { + unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage); + Assert(cLocks > 0); + if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS)) + { + if (cLocks == 1) + { + Assert(pVM->pgm.s.cWriteLockedPages > 0); + pVM->pgm.s.cWriteLockedPages--; + } + PGM_PAGE_DEC_WRITE_LOCKS(pPage); + } + + if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED) + { /* probably extremely likely */ } + else + pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS); + } + else + { + unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage); + Assert(cLocks > 0); + if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS)) + { + if (cLocks == 1) + { + Assert(pVM->pgm.s.cReadLockedPages > 0); + pVM->pgm.s.cReadLockedPages--; + } + PGM_PAGE_DEC_READ_LOCKS(pPage); + } + } + +# ifndef IN_RING0 + if (pMap) + { + Assert(pMap->cRefs >= 1); + pMap->cRefs--; + } +# endif + PGM_UNLOCK(pVM); +} + + +#ifdef IN_RING3 +/** + * Release the mapping of multiple guest pages. + * + * This is the counter part to PGMR3PhysBulkGCPhys2CCPtrExternal() and + * PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal(). + * + * @param pVM The cross context VM structure. + * @param cPages Number of pages to unlock. + * @param paLocks Array of locks lock structure initialized by the mapping + * function. + */ +VMMDECL(void) PGMPhysBulkReleasePageMappingLocks(PVMCC pVM, uint32_t cPages, PPGMPAGEMAPLOCK paLocks) +{ + Assert(cPages > 0); + bool const fWriteLock = (paLocks[0].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE; +#ifdef VBOX_STRICT + for (uint32_t i = 1; i < cPages; i++) + { + Assert(fWriteLock == ((paLocks[i].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE)); + AssertPtr(paLocks[i].uPageAndType); + } +#endif + + PGM_LOCK_VOID(pVM); + if (fWriteLock) + { + /* + * Write locks: + */ + for (uint32_t i = 0; i < cPages; i++) + { + PPGMPAGE pPage = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK); + unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage); + Assert(cLocks > 0); + if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS)) + { + if (cLocks == 1) + { + Assert(pVM->pgm.s.cWriteLockedPages > 0); + pVM->pgm.s.cWriteLockedPages--; + } + PGM_PAGE_DEC_WRITE_LOCKS(pPage); + } + + if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED) + { /* probably extremely likely */ } + else + pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS); + + PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap; + if (pMap) + { + Assert(pMap->cRefs >= 1); + pMap->cRefs--; + } + + /* Yield the lock: */ + if ((i & 1023) == 1023 && i + 1 < cPages) + { + PGM_UNLOCK(pVM); + PGM_LOCK_VOID(pVM); + } + } + } + else + { + /* + * Read locks: + */ + for (uint32_t i = 0; i < cPages; i++) + { + PPGMPAGE pPage = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK); + unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage); + Assert(cLocks > 0); + if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS)) + { + if (cLocks == 1) + { + Assert(pVM->pgm.s.cReadLockedPages > 0); + pVM->pgm.s.cReadLockedPages--; + } + PGM_PAGE_DEC_READ_LOCKS(pPage); + } + + PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap; + if (pMap) + { + Assert(pMap->cRefs >= 1); + pMap->cRefs--; + } + + /* Yield the lock: */ + if ((i & 1023) == 1023 && i + 1 < cPages) + { + PGM_UNLOCK(pVM); + PGM_LOCK_VOID(pVM); + } + } + } + PGM_UNLOCK(pVM); + + RT_BZERO(paLocks, sizeof(paLocks[0]) * cPages); +} +#endif /* IN_RING3 */ + + +/** + * Release the internal mapping of a guest page. + * + * This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and + * pgmPhysGCPhys2CCPtrInternalReadOnly. + * + * @param pVM The cross context VM structure. + * @param pLock The lock structure initialized by the mapping function. + * + * @remarks Caller must hold the PGM lock. + */ +void pgmPhysReleaseInternalPageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */ +} + + +/** + * Converts a GC physical address to a HC ring-3 pointer. + * + * @returns VINF_SUCCESS on success. + * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical + * page but has no physical backing. + * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid + * GC physical address. + * @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses + * a dynamic ram chunk boundary + * + * @param pVM The cross context VM structure. + * @param GCPhys The GC physical address to convert. + * @param pR3Ptr Where to store the R3 pointer on success. + * + * @deprecated Avoid when possible! + */ +int pgmPhysGCPhys2R3Ptr(PVMCC pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr) +{ +/** @todo this is kind of hacky and needs some more work. */ +#ifndef DEBUG_sandervl + VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */ +#endif + + Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */ + PGM_LOCK_VOID(pVM); + + PPGMRAMRANGE pRam; + PPGMPAGE pPage; + int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam); + if (RT_SUCCESS(rc)) + rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr); + + PGM_UNLOCK(pVM); + Assert(rc <= VINF_SUCCESS); + return rc; +} + + +/** + * Converts a guest pointer to a GC physical address. + * + * This uses the current CR3/CR0/CR4 of the guest. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr The guest pointer to convert. + * @param pGCPhys Where to store the GC physical address. + */ +VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys) +{ + PGMPTWALK Walk; + int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk); + if (pGCPhys && RT_SUCCESS(rc)) + *pGCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK); + return rc; +} + + +/** + * Converts a guest pointer to a HC physical address. + * + * This uses the current CR3/CR0/CR4 of the guest. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr The guest pointer to convert. + * @param pHCPhys Where to store the HC physical address. + */ +VMM_INT_DECL(int) PGMPhysGCPtr2HCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGMPTWALK Walk; + int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk); + if (RT_SUCCESS(rc)) + rc = PGMPhysGCPhys2HCPhys(pVM, Walk.GCPhys | ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK), pHCPhys); + return rc; +} + + + +#undef LOG_GROUP +#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS + + +#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION) +/** + * Cache PGMPhys memory access + * + * @param pVM The cross context VM structure. + * @param pCache Cache structure pointer + * @param GCPhys GC physical address + * @param pbR3 HC pointer corresponding to physical page + * + * @thread EMT. + */ +static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE *pCache, RTGCPHYS GCPhys, uint8_t *pbR3) +{ + uint32_t iCacheIndex; + + Assert(VM_IS_EMT(pVM)); + + GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + pbR3 = (uint8_t *)((uintptr_t)pbR3 & ~(uintptr_t)GUEST_PAGE_OFFSET_MASK); + + iCacheIndex = ((GCPhys >> GUEST_PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK); + + ASMBitSet(&pCache->aEntries, iCacheIndex); + + pCache->Entry[iCacheIndex].GCPhys = GCPhys; + pCache->Entry[iCacheIndex].pbR3 = pbR3; +} +#endif /* IN_RING3 */ + + +/** + * Deals with reading from a page with one or more ALL access handlers. + * + * @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3. + * See PGM_HANDLER_PHYS_IS_VALID_STATUS and + * PGM_HANDLER_VIRT_IS_VALID_STATUS for details. + * + * @param pVM The cross context VM structure. + * @param pPage The page descriptor. + * @param GCPhys The physical address to start reading at. + * @param pvBuf Where to put the bits we read. + * @param cb How much to read - less or equal to a page. + * @param enmOrigin The origin of this call. + */ +static VBOXSTRICTRC pgmPhysReadHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb, + PGMACCESSORIGIN enmOrigin) +{ + /* + * The most frequent access here is MMIO and shadowed ROM. + * The current code ASSUMES all these access handlers covers full pages! + */ + + /* + * Whatever we do we need the source page, map it first. + */ + PGMPAGEMAPLOCK PgMpLck; + const void *pvSrc = NULL; + int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck); +/** @todo Check how this can work for MMIO pages? */ + if (RT_FAILURE(rc)) + { + AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n", + GCPhys, pPage, rc)); + memset(pvBuf, 0xff, cb); + return VINF_SUCCESS; + } + + VBOXSTRICTRC rcStrict = VINF_PGM_HANDLER_DO_DEFAULT; + + /* + * Deal with any physical handlers. + */ + PVMCPUCC pVCpu = VMMGetCpu(pVM); + if ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL + || PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)) + { + PPGMPHYSHANDLER pCur; + rc = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur); + if (RT_SUCCESS(rc)) + { + Assert(pCur && GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast); + Assert((pCur->Key & GUEST_PAGE_OFFSET_MASK) == 0); + Assert((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK); +#ifndef IN_RING3 + if (enmOrigin != PGMACCESSORIGIN_IEM) + { + /* Cannot reliably handle informational status codes in this context */ + pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck); + return VERR_PGM_PHYS_WR_HIT_HANDLER; + } +#endif + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler; Assert(pfnHandler); + uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser + : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser); + + Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pCur->pszDesc) )); + STAM_PROFILE_START(&pCur->Stat, h); + PGM_LOCK_ASSERT_OWNER(pVM); + + /* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */ + PGM_UNLOCK(pVM); + rcStrict = pfnHandler(pVM, pVCpu, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, enmOrigin, uUser); + PGM_LOCK_VOID(pVM); + + STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */ + pCur = NULL; /* might not be valid anymore. */ + AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, false), + ("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys)); + if ( rcStrict != VINF_PGM_HANDLER_DO_DEFAULT + && !PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + { + pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck); + return rcStrict; + } + } + else if (rc == VERR_NOT_FOUND) + AssertLogRelMsgFailed(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb)); + else + AssertLogRelMsgFailedReturn(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb), rc); + } + + /* + * Take the default action. + */ + if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT) + { + memcpy(pvBuf, pvSrc, cb); + rcStrict = VINF_SUCCESS; + } + pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck); + return rcStrict; +} + + +/** + * Read physical memory. + * + * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you + * want to ignore those. + * + * @returns Strict VBox status code in raw-mode and ring-0, normal VBox status + * code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check. + * @retval VINF_SUCCESS in all context - read completed. + * + * @retval VINF_EM_OFF in RC and R0 - read completed. + * @retval VINF_EM_SUSPEND in RC and R0 - read completed. + * @retval VINF_EM_RESET in RC and R0 - read completed. + * @retval VINF_EM_HALT in RC and R0 - read completed. + * @retval VINF_SELM_SYNC_GDT in RC only - read completed. + * + * @retval VINF_EM_DBG_STOP in RC and R0 - read completed. + * @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - read completed. + * @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only. + * + * @retval VINF_IOM_R3_MMIO_READ in RC and R0. + * @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0. + * + * @retval VINF_PATM_CHECK_PATCH_PAGE in RC only. + * + * @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that + * haven't been cleared for strict status codes yet. + * + * @param pVM The cross context VM structure. + * @param GCPhys Physical address start reading from. + * @param pvBuf Where to put the read bits. + * @param cbRead How many bytes to read. + * @param enmOrigin The origin of this call. + */ +VMMDECL(VBOXSTRICTRC) PGMPhysRead(PVMCC pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin) +{ + AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS); + LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead)); + + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysRead)); + STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysReadBytes), cbRead); + + PGM_LOCK_VOID(pVM); + + /* + * Copy loop on ram ranges. + */ + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys); + for (;;) + { + /* Inside range or not? */ + if (pRam && GCPhys >= pRam->GCPhys) + { + /* + * Must work our way thru this page by page. + */ + RTGCPHYS off = GCPhys - pRam->GCPhys; + while (off < pRam->cb) + { + unsigned iPage = off >> GUEST_PAGE_SHIFT; + PPGMPAGE pPage = &pRam->aPages[iPage]; + size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK); + if (cb > cbRead) + cb = cbRead; + + /* + * Normal page? Get the pointer to it. + */ + if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) + && !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage)) + { + /* + * Get the pointer to the page. + */ + PGMPAGEMAPLOCK PgMpLck; + const void *pvSrc; + int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck); + if (RT_SUCCESS(rc)) + { + memcpy(pvBuf, pvSrc, cb); + pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck); + } + else + { + AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n", + pRam->GCPhys + off, pPage, rc)); + memset(pvBuf, 0xff, cb); + } + } + /* + * Have ALL/MMIO access handlers. + */ + else + { + VBOXSTRICTRC rcStrict2 = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin); + if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2)) + PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2); + else + { + /* Set the remaining buffer to a known value. */ + memset(pvBuf, 0xff, cbRead); + PGM_UNLOCK(pVM); + return rcStrict2; + } + } + + /* next page */ + if (cb >= cbRead) + { + PGM_UNLOCK(pVM); + return rcStrict; + } + cbRead -= cb; + off += cb; + pvBuf = (char *)pvBuf + cb; + } /* walk pages in ram range. */ + + GCPhys = pRam->GCPhysLast + 1; + } + else + { + LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead)); + + /* + * Unassigned address space. + */ + size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0; + if (cb >= cbRead) + { + memset(pvBuf, 0xff, cbRead); + break; + } + memset(pvBuf, 0xff, cb); + + cbRead -= cb; + pvBuf = (char *)pvBuf + cb; + GCPhys += cb; + } + + /* Advance range if necessary. */ + while (pRam && GCPhys > pRam->GCPhysLast) + pRam = pRam->CTX_SUFF(pNext); + } /* Ram range walk */ + + PGM_UNLOCK(pVM); + return rcStrict; +} + + +/** + * Deals with writing to a page with one or more WRITE or ALL access handlers. + * + * @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3. + * See PGM_HANDLER_PHYS_IS_VALID_STATUS and + * PGM_HANDLER_VIRT_IS_VALID_STATUS for details. + * + * @param pVM The cross context VM structure. + * @param pPage The page descriptor. + * @param GCPhys The physical address to start writing at. + * @param pvBuf What to write. + * @param cbWrite How much to write - less or equal to a page. + * @param enmOrigin The origin of this call. + */ +static VBOXSTRICTRC pgmPhysWriteHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite, + PGMACCESSORIGIN enmOrigin) +{ + PGMPAGEMAPLOCK PgMpLck; + void *pvDst = NULL; + VBOXSTRICTRC rcStrict; + + /* + * Give priority to physical handlers (like #PF does). + * + * Hope for a lonely physical handler first that covers the whole write + * area. This should be a pretty frequent case with MMIO and the heavy + * usage of full page handlers in the page pool. + */ + PVMCPUCC pVCpu = VMMGetCpu(pVM); + PPGMPHYSHANDLER pCur; + rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur); + if (RT_SUCCESS(rcStrict)) + { + Assert(GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast); +#ifndef IN_RING3 + if (enmOrigin != PGMACCESSORIGIN_IEM) + /* Cannot reliably handle informational status codes in this context */ + return VERR_PGM_PHYS_WR_HIT_HANDLER; +#endif + size_t cbRange = pCur->KeyLast - GCPhys + 1; + if (cbRange > cbWrite) + cbRange = cbWrite; + + Assert(PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->pfnHandler); + Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", + GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) )); + if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)) + rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck); + else + rcStrict = VINF_SUCCESS; + if (RT_SUCCESS(rcStrict)) + { + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur); + PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler; + uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser + : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser); + STAM_PROFILE_START(&pCur->Stat, h); + + /* Most handlers will want to release the PGM lock for deadlock prevention + (esp. MMIO), though some PGM internal ones like the page pool and MMIO2 + dirty page trackers will want to keep it for performance reasons. */ + PGM_LOCK_ASSERT_OWNER(pVM); + if (pCurType->fKeepPgmLock) + rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser); + else + { + PGM_UNLOCK(pVM); + rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser); + PGM_LOCK_VOID(pVM); + } + + STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */ + pCur = NULL; /* might not be valid anymore. */ + if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT) + { + if (pvDst) + memcpy(pvDst, pvBuf, cbRange); + rcStrict = VINF_SUCCESS; + } + else + AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, true), + ("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", + VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pCur ? R3STRING(pCur->pszDesc) : "")); + } + else + AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", + GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict); + if (RT_LIKELY(cbRange == cbWrite) || !PGM_PHYS_RW_IS_SUCCESS(rcStrict)) + { + if (pvDst) + pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck); + return rcStrict; + } + + /* more fun to be had below */ + cbWrite -= cbRange; + GCPhys += cbRange; + pvBuf = (uint8_t *)pvBuf + cbRange; + pvDst = (uint8_t *)pvDst + cbRange; + } + else if (rcStrict == VERR_NOT_FOUND) /* The handler is somewhere else in the page, deal with it below. */ + rcStrict = VINF_SUCCESS; + else + AssertMsgFailedReturn(("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict); + Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all GUEST_PAGE_SIZEed! */ + + /* + * Deal with all the odd ends (used to be deal with virt+phys). + */ + Assert(rcStrict != VINF_PGM_HANDLER_DO_DEFAULT); + + /* We need a writable destination page. */ + if (!pvDst) + { + int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck); + AssertLogRelMsgReturn(RT_SUCCESS(rc2), + ("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", GCPhys, pPage, rc2), + rc2); + } + + /** @todo clean up this code some more now there are no virtual handlers any + * more. */ + /* The loop state (big + ugly). */ + PPGMPHYSHANDLER pPhys = NULL; + uint32_t offPhys = GUEST_PAGE_SIZE; + uint32_t offPhysLast = GUEST_PAGE_SIZE; + bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage); + + /* The loop. */ + for (;;) + { + if (fMorePhys && !pPhys) + { + rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pPhys); + if (RT_SUCCESS_NP(rcStrict)) + { + offPhys = 0; + offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */ + } + else + { + AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict); + + rcStrict = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, + GCPhys, &pPhys); + AssertMsgReturn(RT_SUCCESS(rcStrict) || rcStrict == VERR_NOT_FOUND, + ("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict); + + if ( RT_SUCCESS(rcStrict) + && pPhys->Key <= GCPhys + (cbWrite - 1)) + { + offPhys = pPhys->Key - GCPhys; + offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */ + Assert(pPhys->KeyLast - pPhys->Key < _4G); + } + else + { + pPhys = NULL; + fMorePhys = false; + offPhys = offPhysLast = GUEST_PAGE_SIZE; + } + } + } + + /* + * Handle access to space without handlers (that's easy). + */ + VBOXSTRICTRC rcStrict2 = VINF_PGM_HANDLER_DO_DEFAULT; + uint32_t cbRange = (uint32_t)cbWrite; + Assert(cbRange == cbWrite); + + /* + * Physical handler. + */ + if (!offPhys) + { +#ifndef IN_RING3 + if (enmOrigin != PGMACCESSORIGIN_IEM) + /* Cannot reliably handle informational status codes in this context */ + return VERR_PGM_PHYS_WR_HIT_HANDLER; +#endif + if (cbRange > offPhysLast + 1) + cbRange = offPhysLast + 1; + + PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pPhys); + PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler; + uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pPhys->uUser + : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pPhys->uUser); + + Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) )); + STAM_PROFILE_START(&pPhys->Stat, h); + + /* Most handlers will want to release the PGM lock for deadlock prevention + (esp. MMIO), though some PGM internal ones like the page pool and MMIO2 + dirty page trackers will want to keep it for performance reasons. */ + PGM_LOCK_ASSERT_OWNER(pVM); + if (pCurType->fKeepPgmLock) + rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser); + else + { + PGM_UNLOCK(pVM); + rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser); + PGM_LOCK_VOID(pVM); + } + + STAM_PROFILE_STOP(&pPhys->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */ + pPhys = NULL; /* might not be valid anymore. */ + AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true), + ("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2), + VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : "")); + } + + /* + * Execute the default action and merge the status codes. + */ + if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT) + { + memcpy(pvDst, pvBuf, cbRange); + rcStrict2 = VINF_SUCCESS; + } + else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2)) + { + pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck); + return rcStrict2; + } + else + PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2); + + /* + * Advance if we've got more stuff to do. + */ + if (cbRange >= cbWrite) + { + pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck); + return rcStrict; + } + + + cbWrite -= cbRange; + GCPhys += cbRange; + pvBuf = (uint8_t *)pvBuf + cbRange; + pvDst = (uint8_t *)pvDst + cbRange; + + offPhys -= cbRange; + offPhysLast -= cbRange; + } +} + + +/** + * Write to physical memory. + * + * This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you + * want to ignore those. + * + * @returns Strict VBox status code in raw-mode and ring-0, normal VBox status + * code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check. + * @retval VINF_SUCCESS in all context - write completed. + * + * @retval VINF_EM_OFF in RC and R0 - write completed. + * @retval VINF_EM_SUSPEND in RC and R0 - write completed. + * @retval VINF_EM_RESET in RC and R0 - write completed. + * @retval VINF_EM_HALT in RC and R0 - write completed. + * @retval VINF_SELM_SYNC_GDT in RC only - write completed. + * + * @retval VINF_EM_DBG_STOP in RC and R0 - write completed. + * @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - write completed. + * @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only. + * + * @retval VINF_IOM_R3_MMIO_WRITE in RC and R0. + * @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0. + * @retval VINF_IOM_R3_MMIO_COMMIT_WRITE in RC and R0. + * + * @retval VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT in RC only - write completed. + * @retval VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT in RC only. + * @retval VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT in RC only. + * @retval VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT in RC only. + * @retval VINF_CSAM_PENDING_ACTION in RC only. + * @retval VINF_PATM_CHECK_PATCH_PAGE in RC only. + * + * @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that + * haven't been cleared for strict status codes yet. + * + * + * @param pVM The cross context VM structure. + * @param GCPhys Physical address to write to. + * @param pvBuf What to write. + * @param cbWrite How many bytes to write. + * @param enmOrigin Who is calling. + */ +VMMDECL(VBOXSTRICTRC) PGMPhysWrite(PVMCC pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin) +{ + AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()! enmOrigin=%d\n", enmOrigin)); + AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS); + LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite)); + + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWrite)); + STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWriteBytes), cbWrite); + + PGM_LOCK_VOID(pVM); + + /* + * Copy loop on ram ranges. + */ + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys); + for (;;) + { + /* Inside range or not? */ + if (pRam && GCPhys >= pRam->GCPhys) + { + /* + * Must work our way thru this page by page. + */ + RTGCPTR off = GCPhys - pRam->GCPhys; + while (off < pRam->cb) + { + RTGCPTR iPage = off >> GUEST_PAGE_SHIFT; + PPGMPAGE pPage = &pRam->aPages[iPage]; + size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK); + if (cb > cbWrite) + cb = cbWrite; + + /* + * Normal page? Get the pointer to it. + */ + if ( !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) + && !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage)) + { + PGMPAGEMAPLOCK PgMpLck; + void *pvDst; + int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck); + if (RT_SUCCESS(rc)) + { + Assert(!PGM_PAGE_IS_BALLOONED(pPage)); + memcpy(pvDst, pvBuf, cb); + pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck); + } + /* Ignore writes to ballooned pages. */ + else if (!PGM_PAGE_IS_BALLOONED(pPage)) + AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", + pRam->GCPhys + off, pPage, rc)); + } + /* + * Active WRITE or ALL access handlers. + */ + else + { + VBOXSTRICTRC rcStrict2 = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin); + if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2)) + PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2); + else + { + PGM_UNLOCK(pVM); + return rcStrict2; + } + } + + /* next page */ + if (cb >= cbWrite) + { + PGM_UNLOCK(pVM); + return rcStrict; + } + + cbWrite -= cb; + off += cb; + pvBuf = (const char *)pvBuf + cb; + } /* walk pages in ram range */ + + GCPhys = pRam->GCPhysLast + 1; + } + else + { + /* + * Unassigned address space, skip it. + */ + if (!pRam) + break; + size_t cb = pRam->GCPhys - GCPhys; + if (cb >= cbWrite) + break; + cbWrite -= cb; + pvBuf = (const char *)pvBuf + cb; + GCPhys += cb; + } + + /* Advance range if necessary. */ + while (pRam && GCPhys > pRam->GCPhysLast) + pRam = pRam->CTX_SUFF(pNext); + } /* Ram range walk */ + + PGM_UNLOCK(pVM); + return rcStrict; +} + + +/** + * Read from guest physical memory by GC physical address, bypassing + * MMIO and access handlers. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pvDst The destination address. + * @param GCPhysSrc The source address (GC physical address). + * @param cb The number of bytes to read. + */ +VMMDECL(int) PGMPhysSimpleReadGCPhys(PVMCC pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb) +{ + /* + * Treat the first page as a special case. + */ + if (!cb) + return VINF_SUCCESS; + + /* map the 1st page */ + void const *pvSrc; + PGMPAGEMAPLOCK Lock; + int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock); + if (RT_FAILURE(rc)) + return rc; + + /* optimize for the case where access is completely within the first page. */ + size_t cbPage = GUEST_PAGE_SIZE - (GCPhysSrc & GUEST_PAGE_OFFSET_MASK); + if (RT_LIKELY(cb <= cbPage)) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + return VINF_SUCCESS; + } + + /* copy to the end of the page. */ + memcpy(pvDst, pvSrc, cbPage); + PGMPhysReleasePageMappingLock(pVM, &Lock); + GCPhysSrc += cbPage; + pvDst = (uint8_t *)pvDst + cbPage; + cb -= cbPage; + + /* + * Page by page. + */ + for (;;) + { + /* map the page */ + rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock); + if (RT_FAILURE(rc)) + return rc; + + /* last page? */ + if (cb <= GUEST_PAGE_SIZE) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + return VINF_SUCCESS; + } + + /* copy the entire page and advance */ + memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE); + PGMPhysReleasePageMappingLock(pVM, &Lock); + GCPhysSrc += GUEST_PAGE_SIZE; + pvDst = (uint8_t *)pvDst + GUEST_PAGE_SIZE; + cb -= GUEST_PAGE_SIZE; + } + /* won't ever get here. */ +} + + +/** + * Write to guest physical memory referenced by GC pointer. + * Write memory to GC physical address in guest physical memory. + * + * This will bypass MMIO and access handlers. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param GCPhysDst The GC physical address of the destination. + * @param pvSrc The source buffer. + * @param cb The number of bytes to write. + */ +VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVMCC pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb) +{ + LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb)); + + /* + * Treat the first page as a special case. + */ + if (!cb) + return VINF_SUCCESS; + + /* map the 1st page */ + void *pvDst; + PGMPAGEMAPLOCK Lock; + int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock); + if (RT_FAILURE(rc)) + return rc; + + /* optimize for the case where access is completely within the first page. */ + size_t cbPage = GUEST_PAGE_SIZE - (GCPhysDst & GUEST_PAGE_OFFSET_MASK); + if (RT_LIKELY(cb <= cbPage)) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + return VINF_SUCCESS; + } + + /* copy to the end of the page. */ + memcpy(pvDst, pvSrc, cbPage); + PGMPhysReleasePageMappingLock(pVM, &Lock); + GCPhysDst += cbPage; + pvSrc = (const uint8_t *)pvSrc + cbPage; + cb -= cbPage; + + /* + * Page by page. + */ + for (;;) + { + /* map the page */ + rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock); + if (RT_FAILURE(rc)) + return rc; + + /* last page? */ + if (cb <= GUEST_PAGE_SIZE) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + return VINF_SUCCESS; + } + + /* copy the entire page and advance */ + memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE); + PGMPhysReleasePageMappingLock(pVM, &Lock); + GCPhysDst += GUEST_PAGE_SIZE; + pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE; + cb -= GUEST_PAGE_SIZE; + } + /* won't ever get here. */ +} + + +/** + * Read from guest physical memory referenced by GC pointer. + * + * This function uses the current CR3/CR0/CR4 of the guest and will + * bypass access handlers and not set any accessed bits. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pvDst The destination address. + * @param GCPtrSrc The source address (GC pointer). + * @param cb The number of bytes to read. + */ +VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); +/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */ + + /* + * Treat the first page as a special case. + */ + if (!cb) + return VINF_SUCCESS; + + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleRead)); + STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleReadBytes), cb); + + /* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive + * when many VCPUs are fighting for the lock. + */ + PGM_LOCK_VOID(pVM); + + /* map the 1st page */ + void const *pvSrc; + PGMPAGEMAPLOCK Lock; + int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock); + if (RT_FAILURE(rc)) + { + PGM_UNLOCK(pVM); + return rc; + } + + /* optimize for the case where access is completely within the first page. */ + size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK); + if (RT_LIKELY(cb <= cbPage)) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + + /* copy to the end of the page. */ + memcpy(pvDst, pvSrc, cbPage); + PGMPhysReleasePageMappingLock(pVM, &Lock); + GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage); + pvDst = (uint8_t *)pvDst + cbPage; + cb -= cbPage; + + /* + * Page by page. + */ + for (;;) + { + /* map the page */ + rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock); + if (RT_FAILURE(rc)) + { + PGM_UNLOCK(pVM); + return rc; + } + + /* last page? */ + if (cb <= GUEST_PAGE_SIZE) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + + /* copy the entire page and advance */ + memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE); + PGMPhysReleasePageMappingLock(pVM, &Lock); + GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + GUEST_PAGE_SIZE); + pvDst = (uint8_t *)pvDst + GUEST_PAGE_SIZE; + cb -= GUEST_PAGE_SIZE; + } + /* won't ever get here. */ +} + + +/** + * Write to guest physical memory referenced by GC pointer. + * + * This function uses the current CR3/CR0/CR4 of the guest and will + * bypass access handlers and not set dirty or accessed bits. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPtrDst The destination address (GC pointer). + * @param pvSrc The source address. + * @param cb The number of bytes to write. + */ +VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPU_ASSERT_EMT(pVCpu); + + /* + * Treat the first page as a special case. + */ + if (!cb) + return VINF_SUCCESS; + + STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWrite)); + STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb); + + /* map the 1st page */ + void *pvDst; + PGMPAGEMAPLOCK Lock; + int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock); + if (RT_FAILURE(rc)) + return rc; + + /* optimize for the case where access is completely within the first page. */ + size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK); + if (RT_LIKELY(cb <= cbPage)) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + return VINF_SUCCESS; + } + + /* copy to the end of the page. */ + memcpy(pvDst, pvSrc, cbPage); + PGMPhysReleasePageMappingLock(pVM, &Lock); + GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage); + pvSrc = (const uint8_t *)pvSrc + cbPage; + cb -= cbPage; + + /* + * Page by page. + */ + for (;;) + { + /* map the page */ + rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock); + if (RT_FAILURE(rc)) + return rc; + + /* last page? */ + if (cb <= GUEST_PAGE_SIZE) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + return VINF_SUCCESS; + } + + /* copy the entire page and advance */ + memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE); + PGMPhysReleasePageMappingLock(pVM, &Lock); + GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE); + pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE; + cb -= GUEST_PAGE_SIZE; + } + /* won't ever get here. */ +} + + +/** + * Write to guest physical memory referenced by GC pointer and update the PTE. + * + * This function uses the current CR3/CR0/CR4 of the guest and will + * bypass access handlers but will set any dirty and accessed bits in the PTE. + * + * If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr(). + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPtrDst The destination address (GC pointer). + * @param pvSrc The source address. + * @param cb The number of bytes to write. + */ +VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPU_ASSERT_EMT(pVCpu); + + /* + * Treat the first page as a special case. + * Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage. + */ + if (!cb) + return VINF_SUCCESS; + + /* map the 1st page */ + void *pvDst; + PGMPAGEMAPLOCK Lock; + int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock); + if (RT_FAILURE(rc)) + return rc; + + /* optimize for the case where access is completely within the first page. */ + size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK); + if (RT_LIKELY(cb <= cbPage)) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc); + return VINF_SUCCESS; + } + + /* copy to the end of the page. */ + memcpy(pvDst, pvSrc, cbPage); + PGMPhysReleasePageMappingLock(pVM, &Lock); + rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc); + GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage); + pvSrc = (const uint8_t *)pvSrc + cbPage; + cb -= cbPage; + + /* + * Page by page. + */ + for (;;) + { + /* map the page */ + rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock); + if (RT_FAILURE(rc)) + return rc; + + /* last page? */ + if (cb <= GUEST_PAGE_SIZE) + { + memcpy(pvDst, pvSrc, cb); + PGMPhysReleasePageMappingLock(pVM, &Lock); + rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc); + return VINF_SUCCESS; + } + + /* copy the entire page and advance */ + memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE); + PGMPhysReleasePageMappingLock(pVM, &Lock); + rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc); + GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE); + pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE; + cb -= GUEST_PAGE_SIZE; + } + /* won't ever get here. */ +} + + +/** + * Read from guest physical memory referenced by GC pointer. + * + * This function uses the current CR3/CR0/CR4 of the guest and will + * respect access handlers and set accessed bits. + * + * @returns Strict VBox status, see PGMPhysRead for details. + * @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the + * specified virtual address. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pvDst The destination address. + * @param GCPtrSrc The source address (GC pointer). + * @param cb The number of bytes to read. + * @param enmOrigin Who is calling. + * @thread EMT(pVCpu) + */ +VMMDECL(VBOXSTRICTRC) PGMPhysReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, PGMACCESSORIGIN enmOrigin) +{ + int rc; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPU_ASSERT_EMT(pVCpu); + + /* + * Anything to do? + */ + if (!cb) + return VINF_SUCCESS; + + LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb)); + + /* + * Optimize reads within a single page. + */ + if (((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE) + { + /* Convert virtual to physical address + flags */ + PGMPTWALK Walk; + rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk); + AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc); + RTGCPHYS const GCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK); + + /* mark the guest page as accessed. */ + if (!(Walk.fEffective & X86_PTE_A)) + { + rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A)); + AssertRC(rc); + } + + return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin); + } + + /* + * Page by page. + */ + for (;;) + { + /* Convert virtual to physical address + flags */ + PGMPTWALK Walk; + rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk); + AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc); + RTGCPHYS const GCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK); + + /* mark the guest page as accessed. */ + if (!(Walk.fEffective & X86_PTE_A)) + { + rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A)); + AssertRC(rc); + } + + /* copy */ + size_t cbRead = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK); + if (cbRead < cb) + { + VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, pvDst, cbRead, enmOrigin); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else + return rcStrict; + } + else /* Last page (cbRead is GUEST_PAGE_SIZE, we only need cb!) */ + return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin); + + /* next */ + Assert(cb > cbRead); + cb -= cbRead; + pvDst = (uint8_t *)pvDst + cbRead; + GCPtrSrc += cbRead; + } +} + + +/** + * Write to guest physical memory referenced by GC pointer. + * + * This function uses the current CR3/CR0/CR4 of the guest and will + * respect access handlers and set dirty and accessed bits. + * + * @returns Strict VBox status, see PGMPhysWrite for details. + * @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the + * specified virtual address. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param GCPtrDst The destination address (GC pointer). + * @param pvSrc The source address. + * @param cb The number of bytes to write. + * @param enmOrigin Who is calling. + */ +VMMDECL(VBOXSTRICTRC) PGMPhysWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, PGMACCESSORIGIN enmOrigin) +{ + int rc; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + VMCPU_ASSERT_EMT(pVCpu); + + /* + * Anything to do? + */ + if (!cb) + return VINF_SUCCESS; + + LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb)); + + /* + * Optimize writes within a single page. + */ + if (((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE) + { + /* Convert virtual to physical address + flags */ + PGMPTWALK Walk; + rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk); + AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc); + RTGCPHYS const GCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK); + + /* Mention when we ignore X86_PTE_RW... */ + if (!(Walk.fEffective & X86_PTE_RW)) + Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb)); + + /* Mark the guest page as accessed and dirty if necessary. */ + if ((Walk.fEffective & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D)) + { + rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); + AssertRC(rc); + } + + return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin); + } + + /* + * Page by page. + */ + for (;;) + { + /* Convert virtual to physical address + flags */ + PGMPTWALK Walk; + rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk); + AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc); + RTGCPHYS const GCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK); + + /* Mention when we ignore X86_PTE_RW... */ + if (!(Walk.fEffective & X86_PTE_RW)) + Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb)); + + /* Mark the guest page as accessed and dirty if necessary. */ + if ((Walk.fEffective & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D)) + { + rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); + AssertRC(rc); + } + + /* copy */ + size_t cbWrite = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK); + if (cbWrite < cb) + { + VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite, enmOrigin); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else + return rcStrict; + } + else /* Last page (cbWrite is GUEST_PAGE_SIZE, we only need cb!) */ + return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin); + + /* next */ + Assert(cb > cbWrite); + cb -= cbWrite; + pvSrc = (uint8_t *)pvSrc + cbWrite; + GCPtrDst += cbWrite; + } +} + + +/** + * Return the page type of the specified physical address. + * + * @returns The page type. + * @param pVM The cross context VM structure. + * @param GCPhys Guest physical address + */ +VMM_INT_DECL(PGMPAGETYPE) PGMPhysGetPageType(PVMCC pVM, RTGCPHYS GCPhys) +{ + PGM_LOCK_VOID(pVM); + PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys); + PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID; + PGM_UNLOCK(pVM); + + return enmPgType; +} + + +/** + * Converts a GC physical address to a HC ring-3 pointer, with some + * additional checks. + * + * @returns VBox status code (no informational statuses). + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the + * calling EMT. + * @param GCPhys The GC physical address to convert. This API mask + * the A20 line when necessary. + * @param puTlbPhysRev Where to read the physical TLB revision. Needs to + * be done while holding the PGM lock. + * @param ppb Where to store the pointer corresponding to GCPhys + * on success. + * @param pfTlb The TLB flags and revision. We only add stuff. + * + * @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr and + * PGMPhysIemGCPhys2Ptr. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(int) PGMPhysIemGCPhys2PtrNoLock(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint64_t const volatile *puTlbPhysRev, + R3R0PTRTYPE(uint8_t *) *ppb, + uint64_t *pfTlb) +{ + PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys); + Assert(!(GCPhys & X86_PAGE_OFFSET_MASK)); + + PGM_LOCK_VOID(pVM); + + PPGMRAMRANGE pRam; + PPGMPAGE pPage; + int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam); + if (RT_SUCCESS(rc)) + { + if (!PGM_PAGE_IS_BALLOONED(pPage)) + { + if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage)) + { + if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage)) + { + /* + * No access handler. + */ + switch (PGM_PAGE_GET_STATE(pPage)) + { + case PGM_PAGE_STATE_ALLOCATED: + *pfTlb |= *puTlbPhysRev; + break; + case PGM_PAGE_STATE_BALLOONED: + AssertFailed(); + RT_FALL_THRU(); + case PGM_PAGE_STATE_ZERO: + case PGM_PAGE_STATE_SHARED: + case PGM_PAGE_STATE_WRITE_MONITORED: + *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE; + break; + } + + PPGMPAGEMAPTLBE pTlbe; + rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe); + AssertLogRelRCReturn(rc, rc); + *ppb = (uint8_t *)pTlbe->pv; + } + else if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) + { + /* + * MMIO or similar all access handler: Catch all access. + */ + *pfTlb |= *puTlbPhysRev + | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3; + *ppb = NULL; + } + else + { + /* + * Write access handler: Catch write accesses if active. + */ + if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)) + *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE; + else + switch (PGM_PAGE_GET_STATE(pPage)) + { + case PGM_PAGE_STATE_ALLOCATED: + *pfTlb |= *puTlbPhysRev; + break; + case PGM_PAGE_STATE_BALLOONED: + AssertFailed(); + RT_FALL_THRU(); + case PGM_PAGE_STATE_ZERO: + case PGM_PAGE_STATE_SHARED: + case PGM_PAGE_STATE_WRITE_MONITORED: + *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE; + break; + } + + PPGMPAGEMAPTLBE pTlbe; + rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe); + AssertLogRelRCReturn(rc, rc); + *ppb = (uint8_t *)pTlbe->pv; + } + } + else + { + /* Alias MMIO: For now, we catch all access. */ + *pfTlb |= *puTlbPhysRev + | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3; + *ppb = NULL; + } + } + else + { + /* Ballooned: Shouldn't get here, but we read zero page via PGMPhysRead and writes goes to /dev/null. */ + *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3; + *ppb = NULL; + } + Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp *ppb=%p *pfTlb=%#RX64 pPage=%R[pgmpage]\n", GCPhys, *ppb, *pfTlb, pPage)); + } + else + { + *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ + | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 | PGMIEMGCPHYS2PTR_F_UNASSIGNED; + *ppb = NULL; + Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp *ppb=%p *pfTlb=%#RX64 (rc=%Rrc)\n", GCPhys, *ppb, *pfTlb, rc)); + } + + PGM_UNLOCK(pVM); + return VINF_SUCCESS; +} + + +/** + * Converts a GC physical address to a HC ring-3 pointer, with some + * additional checks. + * + * @returns VBox status code (no informational statuses). + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write + * access handler of some kind. + * @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all + * accesses or is odd in any way. + * @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the + * calling EMT. + * @param GCPhys The GC physical address to convert. This API mask + * the A20 line when necessary. + * @param fWritable Whether write access is required. + * @param fByPassHandlers Whether to bypass access handlers. + * @param ppv Where to store the pointer corresponding to GCPhys + * on success. + * @param pLock + * + * @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr. + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers, + void **ppv, PPGMPAGEMAPLOCK pLock) +{ + PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys); + + PGM_LOCK_VOID(pVM); + + PPGMRAMRANGE pRam; + PPGMPAGE pPage; + int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam); + if (RT_SUCCESS(rc)) + { + if (PGM_PAGE_IS_BALLOONED(pPage)) + rc = VERR_PGM_PHYS_TLB_CATCH_WRITE; + else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage)) + rc = VERR_PGM_PHYS_TLB_CATCH_ALL; + else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage) + || (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) ) + rc = VINF_SUCCESS; + else + { + if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */ + { + Assert(!fByPassHandlers || PGM_PAGE_IS_MMIO(pPage)); + rc = VERR_PGM_PHYS_TLB_CATCH_ALL; + } + else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable) + { + Assert(!fByPassHandlers); + rc = VERR_PGM_PHYS_TLB_CATCH_WRITE; + } + } + if (RT_SUCCESS(rc)) + { + int rc2; + + /* Make sure what we return is writable. */ + if (fWritable) + switch (PGM_PAGE_GET_STATE(pPage)) + { + case PGM_PAGE_STATE_ALLOCATED: + break; + case PGM_PAGE_STATE_BALLOONED: + AssertFailed(); + break; + case PGM_PAGE_STATE_ZERO: + case PGM_PAGE_STATE_SHARED: + case PGM_PAGE_STATE_WRITE_MONITORED: + rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK); + AssertLogRelRCReturn(rc2, rc2); + break; + } + + /* Get a ring-3 mapping of the address. */ + PPGMPAGEMAPTLBE pTlbe; + rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe); + AssertLogRelRCReturn(rc2, rc2); + + /* Lock it and calculate the address. */ + if (fWritable) + pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock); + else + pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock); + *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK)); + + Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv)); + } + else + Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage)); + + /* else: handler catching all access, no pointer returned. */ + } + else + rc = VERR_PGM_PHYS_TLB_UNASSIGNED; + + PGM_UNLOCK(pVM); + return rc; +} + + +/** + * Checks if the give GCPhys page requires special handling for the given access + * because it's MMIO or otherwise monitored. + * + * @returns VBox status code (no informational statuses). + * @retval VINF_SUCCESS on success. + * @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write + * access handler of some kind. + * @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all + * accesses or is odd in any way. + * @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist. + * + * @param pVM The cross context VM structure. + * @param GCPhys The GC physical address to convert. Since this is + * only used for filling the REM TLB, the A20 mask must + * be applied before calling this API. + * @param fWritable Whether write access is required. + * @param fByPassHandlers Whether to bypass access handlers. + * + * @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just + * a stop gap thing that should be removed once there is a better TLB + * for virtual address accesses. + */ +VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVMCC pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers) +{ + PGM_LOCK_VOID(pVM); + PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys); + + PPGMRAMRANGE pRam; + PPGMPAGE pPage; + int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam); + if (RT_SUCCESS(rc)) + { + if (PGM_PAGE_IS_BALLOONED(pPage)) + rc = VERR_PGM_PHYS_TLB_CATCH_WRITE; + else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage)) + rc = VERR_PGM_PHYS_TLB_CATCH_ALL; + else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage) + || (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) ) + rc = VINF_SUCCESS; + else + { + if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */ + { + Assert(!fByPassHandlers || PGM_PAGE_IS_MMIO(pPage)); + rc = VERR_PGM_PHYS_TLB_CATCH_ALL; + } + else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable) + { + Assert(!fByPassHandlers); + rc = VERR_PGM_PHYS_TLB_CATCH_WRITE; + } + } + } + + PGM_UNLOCK(pVM); + return rc; +} + +#ifdef VBOX_WITH_NATIVE_NEM + +/** + * Interface used by NEM to check what to do on a memory access exit. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per virtual CPU structure. + * Optional. + * @param GCPhys The guest physical address. + * @param fMakeWritable Whether to try make the page writable or not. If it + * cannot be made writable, NEM_PAGE_PROT_WRITE won't + * be returned and the return code will be unaffected + * @param pInfo Where to return the page information. This is + * initialized even on failure. + * @param pfnChecker Page in-sync checker callback. Optional. + * @param pvUser User argument to pass to pfnChecker. + */ +VMM_INT_DECL(int) PGMPhysNemPageInfoChecker(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fMakeWritable, PPGMPHYSNEMPAGEINFO pInfo, + PFNPGMPHYSNEMCHECKPAGE pfnChecker, void *pvUser) +{ + PGM_LOCK_VOID(pVM); + + PPGMPAGE pPage; + int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage); + if (RT_SUCCESS(rc)) + { + /* Try make it writable if requested. */ + pInfo->u2OldNemState = PGM_PAGE_GET_NEM_STATE(pPage); + if (fMakeWritable) + switch (PGM_PAGE_GET_STATE(pPage)) + { + case PGM_PAGE_STATE_SHARED: + case PGM_PAGE_STATE_WRITE_MONITORED: + case PGM_PAGE_STATE_ZERO: + rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys); + if (rc == VERR_PGM_PHYS_PAGE_RESERVED) + rc = VINF_SUCCESS; + break; + } + + /* Fill in the info. */ + pInfo->HCPhys = PGM_PAGE_GET_HCPHYS(pPage); + pInfo->u2NemState = PGM_PAGE_GET_NEM_STATE(pPage); + pInfo->fHasHandlers = PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) ? 1 : 0; + PGMPAGETYPE const enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage); + pInfo->enmType = enmType; + pInfo->fNemProt = pgmPhysPageCalcNemProtection(pPage, enmType); + switch (PGM_PAGE_GET_STATE(pPage)) + { + case PGM_PAGE_STATE_ALLOCATED: + pInfo->fZeroPage = 0; + break; + + case PGM_PAGE_STATE_ZERO: + pInfo->fZeroPage = 1; + break; + + case PGM_PAGE_STATE_WRITE_MONITORED: + pInfo->fZeroPage = 0; + break; + + case PGM_PAGE_STATE_SHARED: + pInfo->fZeroPage = 0; + break; + + case PGM_PAGE_STATE_BALLOONED: + pInfo->fZeroPage = 1; + break; + + default: + pInfo->fZeroPage = 1; + AssertFailedStmt(rc = VERR_PGM_PHYS_PAGE_GET_IPE); + } + + /* Call the checker and update NEM state. */ + if (pfnChecker) + { + rc = pfnChecker(pVM, pVCpu, GCPhys, pInfo, pvUser); + PGM_PAGE_SET_NEM_STATE(pPage, pInfo->u2NemState); + } + + /* Done. */ + PGM_UNLOCK(pVM); + } + else + { + PGM_UNLOCK(pVM); + + pInfo->HCPhys = NIL_RTHCPHYS; + pInfo->fNemProt = NEM_PAGE_PROT_NONE; + pInfo->u2NemState = 0; + pInfo->fHasHandlers = 0; + pInfo->fZeroPage = 0; + pInfo->enmType = PGMPAGETYPE_INVALID; + } + + return rc; +} + + +/** + * NEM helper that performs @a pfnCallback on pages with NEM state @a uMinState + * or higher. + * + * @returns VBox status code from callback. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context per CPU structure. This is + * optional as its only for passing to callback. + * @param uMinState The minimum NEM state value to call on. + * @param pfnCallback The callback function. + * @param pvUser User argument for the callback. + */ +VMM_INT_DECL(int) PGMPhysNemEnumPagesByState(PVMCC pVM, PVMCPUCC pVCpu, uint8_t uMinState, + PFNPGMPHYSNEMENUMCALLBACK pfnCallback, void *pvUser) +{ + /* + * Just brute force this problem. + */ + PGM_LOCK_VOID(pVM); + int rc = VINF_SUCCESS; + for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX); pRam; pRam = pRam->CTX_SUFF(pNext)) + { + uint32_t const cPages = pRam->cb >> X86_PAGE_SHIFT; + for (uint32_t iPage = 0; iPage < cPages; iPage++) + { + uint8_t u2State = PGM_PAGE_GET_NEM_STATE(&pRam->aPages[iPage]); + if (u2State < uMinState) + { /* likely */ } + else + { + rc = pfnCallback(pVM, pVCpu, pRam->GCPhys + ((RTGCPHYS)iPage << X86_PAGE_SHIFT), &u2State, pvUser); + if (RT_SUCCESS(rc)) + PGM_PAGE_SET_NEM_STATE(&pRam->aPages[iPage], u2State); + else + break; + } + } + } + PGM_UNLOCK(pVM); + + return rc; +} + + +/** + * Helper for setting the NEM state for a range of pages. + * + * @param paPages Array of pages to modify. + * @param cPages How many pages to modify. + * @param u2State The new state value. + */ +void pgmPhysSetNemStateForPages(PPGMPAGE paPages, RTGCPHYS cPages, uint8_t u2State) +{ + PPGMPAGE pPage = paPages; + while (cPages-- > 0) + { + PGM_PAGE_SET_NEM_STATE(pPage, u2State); + pPage++; + } +} + +#endif /* VBOX_WITH_NATIVE_NEM */ + diff --git a/src/VBox/VMM/VMMAll/PGMAllPool.cpp b/src/VBox/VMM/VMMAll/PGMAllPool.cpp new file mode 100644 index 00000000..59f946e5 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PGMAllPool.cpp @@ -0,0 +1,5898 @@ +/* $Id: PGMAllPool.cpp $ */ +/** @file + * PGM Shadow Page Pool. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_PGM_POOL +#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */ +#include +#include +#include +#include +#include "PGMInternal.h" +#include +#include "PGMInline.h" +#include +#include + +#include +#include +#include +#include + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +RT_C_DECLS_BEGIN +#if 0 /* unused */ +DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind); +DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind); +#endif /* unused */ +static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage); +static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage); +static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable); +static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage); +#if defined(LOG_ENABLED) || defined(VBOX_STRICT) +static const char *pgmPoolPoolKindToStr(uint8_t enmKind); +#endif +#if 0 /*defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT)*/ +static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT); +#endif + +int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage); +PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt); +void pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt); +void pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt); + +RT_C_DECLS_END + + +#if 0 /* unused */ +/** + * Checks if the specified page pool kind is for a 4MB or 2MB guest page. + * + * @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false. + * @param enmKind The page kind. + */ +DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind) +{ + switch (enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + return true; + default: + return false; + } +} +#endif /* unused */ + + +/** + * Flushes a chain of pages sharing the same access monitor. + * + * @param pPool The pool. + * @param pPage A page in the chain. + */ +void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind)); + + /* + * Find the list head. + */ + uint16_t idx = pPage->idx; + if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX) + { + while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX) + { + idx = pPage->iMonitoredPrev; + Assert(idx != pPage->idx); + pPage = &pPool->aPages[idx]; + } + } + + /* + * Iterate the list flushing each shadow page. + */ + for (;;) + { + idx = pPage->iMonitoredNext; + Assert(idx != pPage->idx); + if (pPage->idx >= PGMPOOL_IDX_FIRST) + { + int rc2 = pgmPoolFlushPage(pPool, pPage); + AssertRC(rc2); + } + /* next */ + if (idx == NIL_PGMPOOL_IDX) + break; + pPage = &pPool->aPages[idx]; + } +} + + +/** + * Wrapper for getting the current context pointer to the entry being modified. + * + * @returns VBox status code suitable for scheduling. + * @param pVM The cross context VM structure. + * @param pvDst Destination address + * @param pvSrc Pointer to the mapping of @a GCPhysSrc or NULL depending + * on the context (e.g. \#PF in R0 & RC). + * @param GCPhysSrc The source guest physical address. + * @param cb Size of data to read + */ +DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVMCC pVM, void *pvDst, void const *pvSrc, RTGCPHYS GCPhysSrc, size_t cb) +{ +#if defined(IN_RING3) + NOREF(pVM); NOREF(GCPhysSrc); + memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb); + return VINF_SUCCESS; +#else + /** @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */ + NOREF(pvSrc); + return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb); +#endif +} + + +/** + * Process shadow entries before they are changed by the guest. + * + * For PT entries we will clear them. For PD entries, we'll simply check + * for mapping conflicts and set the SyncCR3 FF if found. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pPool The pool. + * @param pPage The head page. + * @param GCPhysFault The guest physical fault address. + * @param pvAddress Pointer to the mapping of @a GCPhysFault or NULL + * depending on the context (e.g. \#PF in R0 & RC). + * @param cbWrite Write size; might be zero if the caller knows we're not crossing entry boundaries + */ +static void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, + void const *pvAddress, unsigned cbWrite) +{ + AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%u (idx=%u)\n", pPage->iMonitoredPrev, pPage->idx)); + const unsigned off = GCPhysFault & GUEST_PAGE_OFFSET_MASK; + PVMCC pVM = pPool->CTX_SUFF(pVM); + NOREF(pVCpu); + + LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n", + (RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite)); + + if (PGMPOOL_PAGE_IS_NESTED(pPage)) + Log7Func(("%RGv phys=%RGp cbWrite=%d\n", (RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite)); + + for (;;) + { + union + { + void *pv; + PX86PT pPT; + PPGMSHWPTPAE pPTPae; + PX86PD pPD; + PX86PDPAE pPDPae; + PX86PDPT pPDPT; + PX86PML4 pPML4; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + PEPTPDPT pEptPdpt; + PEPTPD pEptPd; + PEPTPT pEptPt; +#endif + } uShw; + + LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n", + pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite)); + + uShw.pv = NULL; + switch (pPage->enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + { + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT)); + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(X86PTE); + LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw)); + X86PGUINT const uPde = uShw.pPT->a[iShw].u; + if (uPde & X86_PTE_P) + { + X86PTE GstPte; + int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte)); + AssertRC(rc); + Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK, iShw); + ASMAtomicWriteU32(&uShw.pPT->a[iShw].u, 0); + } + break; + } + + /* page/2 sized */ + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + { + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT)); + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2))) + { + const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1); + LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw)); + if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw])) + { + X86PTE GstPte; + int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte)); + AssertRC(rc); + + Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, + PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), + GstPte.u & X86_PTE_PG_MASK, + iShw); + PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0); + } + } + break; + } + + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + { + unsigned iGst = off / sizeof(X86PDE); + unsigned iShwPdpt = iGst / 256; + unsigned iShw = (iGst % 256) * 2; + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + + LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)); + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD)); + if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD) + { + for (unsigned i = 0; i < 2; i++) + { + X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw + i].u; + if (uPde & X86_PDE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw + i, uPde)); + pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw + i); + ASMAtomicWriteU64(&uShw.pPDPae->a[iShw + i].u, 0); + } + + /* paranoia / a bit assumptive. */ + if ( (off & 3) + && (off & 3) + cbWrite > 4) + { + const unsigned iShw2 = iShw + 2 + i; + if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a)) + { + X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u; + if (uPde2 & X86_PDE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2)); + pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0); + } + } + } + } + } + break; + } + + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + { + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(X86PTEPAE); + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT)); + if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw])) + { + X86PTEPAE GstPte; + int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte)); + AssertRC(rc); + + Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), GstPte.u & X86_PTE_PAE_PG_MASK)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, + PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), + GstPte.u & X86_PTE_PAE_PG_MASK, + iShw); + PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0); + } + + /* paranoia / a bit assumptive. */ + if ( (off & 7) + && (off & 7) + cbWrite > sizeof(X86PTEPAE)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE); + AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a)); + + if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw2])) + { + X86PTEPAE GstPte; + int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, + pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL, + GCPhysFault + sizeof(GstPte), sizeof(GstPte)); + AssertRC(rc); + Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), GstPte.u & X86_PTE_PAE_PG_MASK)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, + PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), + GstPte.u & X86_PTE_PAE_PG_MASK, + iShw2); + PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw2], 0); + } + } + break; + } + + case PGMPOOLKIND_32BIT_PD: + { + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging! + + LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw)); + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD)); + X86PGUINT const uPde = uShw.pPD->a[iShw].u; + if (uPde & X86_PDE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde)); + pgmPoolFree(pVM, uPde & X86_PDE_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0); + } + + /* paranoia / a bit assumptive. */ + if ( (off & 3) + && (off & 3) + cbWrite > sizeof(X86PTE)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE); + if ( iShw2 != iShw + && iShw2 < RT_ELEMENTS(uShw.pPD->a)) + { + X86PGUINT const uPde2 = uShw.pPD->a[iShw2].u; + if (uPde2 & X86_PDE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2)); + pgmPoolFree(pVM, uPde2 & X86_PDE_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU32(&uShw.pPD->a[iShw2].u, 0); + } + } + } +#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). - not working any longer... */ + if ( uShw.pPD->a[iShw].n.u1Present + && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)) + { + LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u)); + pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0); + } +#endif + break; + } + + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + { + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(X86PDEPAE); + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD)); + + /* + * Causes trouble when the guest uses a PDE to refer to the whole page table level + * structure. (Invalidate here; faults later on when it tries to change the page + * table entries -> recheck; probably only applies to the RC case.) + */ + X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u; + if (uPde & X86_PDE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde)); + pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0); + } + + /* paranoia / a bit assumptive. */ + if ( (off & 7) + && (off & 7) + cbWrite > sizeof(X86PDEPAE)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE); + AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a)); + + X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u; + if (uPde2 & X86_PDE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2)); + pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0); + } + } + break; + } + + case PGMPOOLKIND_PAE_PDPT: + { + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT)); + /* + * Hopefully this doesn't happen very often: + * - touching unused parts of the page + * - messing with the bits of pd pointers without changing the physical address + */ + /* PDPT roots are not page aligned; 32 byte only! */ + const unsigned offPdpt = GCPhysFault - pPage->GCPhys; + + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = offPdpt / sizeof(X86PDPE); + if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */ + { + X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u; + if (uPdpe & X86_PDPE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u)); + pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0); + } + + /* paranoia / a bit assumptive. */ + if ( (offPdpt & 7) + && (offPdpt & 7) + cbWrite > sizeof(X86PDPE)) + { + const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE); + if ( iShw2 != iShw + && iShw2 < X86_PG_PAE_PDPE_ENTRIES) + { + X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u; + if (uPdpe2 & X86_PDPE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u)); + pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0); + } + } + } + } + break; + } + + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + { + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD)); + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(X86PDEPAE); + X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u; + if (uPde & X86_PDE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde)); + pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0); + } + + /* paranoia / a bit assumptive. */ + if ( (off & 7) + && (off & 7) + cbWrite > sizeof(X86PDEPAE)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE); + AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a)); + X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u; + if (uPde2 & X86_PDE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2)); + pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0); + } + } + break; + } + + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + { + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT)); + /* + * Hopefully this doesn't happen very often: + * - messing with the bits of pd pointers without changing the physical address + */ + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(X86PDPE); + X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u; + if (uPdpe & X86_PDPE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uPdpe)); + pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0); + } + /* paranoia / a bit assumptive. */ + if ( (off & 7) + && (off & 7) + cbWrite > sizeof(X86PDPE)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE); + X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u; + if (uPdpe2 & X86_PDPE_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpe2)); + pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0); + } + } + break; + } + + case PGMPOOLKIND_64BIT_PML4: + { + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPML4)); + /* + * Hopefully this doesn't happen very often: + * - messing with the bits of pd pointers without changing the physical address + */ + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(X86PDPE); + X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u; + if (uPml4e & X86_PML4E_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uPml4e)); + pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0); + } + /* paranoia / a bit assumptive. */ + if ( (off & 7) + && (off & 7) + cbWrite > sizeof(X86PDPE)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E); + X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u; + if (uPml4e2 & X86_PML4E_P) + { + LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2)); + pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0); + } + } + break; + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4: + { + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(EPTPML4E); + X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u; + if (uPml4e & EPT_PRESENT_MASK) + { + Log7Func(("PML4 iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPml4e, pPage->GCPhys)); + pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0); + } + + /* paranoia / a bit assumptive. */ + if ( (off & 7) + && (off & 7) + cbWrite > sizeof(X86PML4E)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E); + X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u; + if (uPml4e2 & EPT_PRESENT_MASK) + { + Log7Func(("PML4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2)); + pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0); + } + } + break; + } + + case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT: + { + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(EPTPDPTE); + X86PGPAEUINT const uPdpte = uShw.pEptPdpt->a[iShw].u; + if (uPdpte & EPT_PRESENT_MASK) + { + Log7Func(("EPT PDPT iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPdpte, pPage->GCPhys)); + pgmPoolFree(pVM, uPdpte & EPT_PDPTE_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw].u, 0); + } + + /* paranoia / a bit assumptive. */ + if ( (off & 7) + && (off & 7) + cbWrite > sizeof(EPTPDPTE)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDPTE); + X86PGPAEUINT const uPdpte2 = uShw.pEptPdpt->a[iShw2].u; + if (uPdpte2 & EPT_PRESENT_MASK) + { + Log7Func(("EPT PDPT iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpte2)); + pgmPoolFree(pVM, uPdpte2 & EPT_PDPTE_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw2].u, 0); + } + } + break; + } + + case PGMPOOLKIND_EPT_PD_FOR_EPT_PD: + { + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(EPTPDE); + X86PGPAEUINT const uPde = uShw.pEptPd->a[iShw].u; + if (uPde & EPT_PRESENT_MASK) + { + Log7Func(("EPT PD iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPde, pPage->GCPhys)); + pgmPoolFree(pVM, uPde & EPT_PDE_PG_MASK, pPage->idx, iShw); + ASMAtomicWriteU64(&uShw.pEptPd->a[iShw].u, 0); + } + + /* paranoia / a bit assumptive. */ + if ( (off & 7) + && (off & 7) + cbWrite > sizeof(EPTPDE)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDE); + AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPd->a)); + X86PGPAEUINT const uPde2 = uShw.pEptPd->a[iShw2].u; + if (uPde2 & EPT_PRESENT_MASK) + { + Log7Func(("EPT PD (2): iShw2=%#x: %RX64 (%RGp) -> freeing it!\n", iShw2, uPde2, pPage->GCPhys)); + pgmPoolFree(pVM, uPde2 & EPT_PDE_PG_MASK, pPage->idx, iShw2); + ASMAtomicWriteU64(&uShw.pEptPd->a[iShw2].u, 0); + } + } + break; + } + + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: + { + uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + const unsigned iShw = off / sizeof(EPTPTE); + X86PGPAEUINT const uPte = uShw.pEptPt->a[iShw].u; + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT)); + if (uPte & EPT_PRESENT_MASK) + { + EPTPTE GstPte; + int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte)); + AssertRC(rc); + + Log7Func(("EPT PT: iShw=%#x %RX64 (%RGp)\n", iShw, uPte, pPage->GCPhys)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, + uShw.pEptPt->a[iShw].u & EPT_PTE_PG_MASK, + GstPte.u & EPT_PTE_PG_MASK, + iShw); + ASMAtomicWriteU64(&uShw.pEptPt->a[iShw].u, 0); + } + + /* paranoia / a bit assumptive. */ + if ( (off & 7) + && (off & 7) + cbWrite > sizeof(EPTPTE)) + { + const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPTE); + AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPt->a)); + X86PGPAEUINT const uPte2 = uShw.pEptPt->a[iShw2].u; + if (uPte2 & EPT_PRESENT_MASK) + { + EPTPTE GstPte; + int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, + pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL, + GCPhysFault + sizeof(GstPte), sizeof(GstPte)); + AssertRC(rc); + Log7Func(("EPT PT (2): iShw=%#x %RX64 (%RGp)\n", iShw2, uPte2, pPage->GCPhys)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, + uShw.pEptPt->a[iShw2].u & EPT_PTE_PG_MASK, + GstPte.u & EPT_PTE_PG_MASK, + iShw2); + ASMAtomicWriteU64(&uShw.pEptPt->a[iShw2].u, 0); + } + } + break; + } +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ + + default: + AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind)); + } + PGM_DYNMAP_UNUSED_HINT_VM(pVM, uShw.pv); + + /* next */ + if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX) + return; + pPage = &pPool->aPages[pPage->iMonitoredNext]; + } +} + +#ifndef IN_RING3 + +/** + * Checks if a access could be a fork operation in progress. + * + * Meaning, that the guest is setting up the parent process for Copy-On-Write. + * + * @returns true if it's likely that we're forking, otherwise false. + * @param pPool The pool. + * @param pDis The disassembled instruction. + * @param offFault The access offset. + */ +DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault) +{ + /* + * i386 linux is using btr to clear X86_PTE_RW. + * The functions involved are (2.6.16 source inspection): + * clear_bit + * ptep_set_wrprotect + * copy_one_pte + * copy_pte_range + * copy_pmd_range + * copy_pud_range + * copy_page_range + * dup_mmap + * dup_mm + * copy_mm + * copy_process + * do_fork + */ + if ( pDis->pCurInstr->uOpcode == OP_BTR + && !(offFault & 4) + /** @todo Validate that the bit index is X86_PTE_RW. */ + ) + { + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool); + return true; + } + return false; +} + + +/** + * Determine whether the page is likely to have been reused. + * + * @returns true if we consider the page as being reused for a different purpose. + * @returns false if we consider it to still be a paging page. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pCtx Pointer to the register context for the CPU. + * @param pDis The disassembly info for the faulting instruction. + * @param pvFault The fault address. + * @param pPage The pool page being accessed. + * + * @remark The REP prefix check is left to the caller because of STOSD/W. + */ +DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, RTGCPTR pvFault, + PPGMPOOLPAGE pPage) +{ + /* Locked (CR3, PDPTR*4) should not be reusable. Considering them as + such may cause loops booting tst-ubuntu-15_10-64-efi, ++. */ + if (pPage->cLocked) + { + Log2(("pgmRZPoolMonitorIsReused: %RGv (%p) can't have been resued, because it's locked!\n", pvFault, pPage)); + return false; + } + + /** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */ + if ( HMHasPendingIrq(pVM) + && pCtx->rsp - pvFault < 32) + { + /* Fault caused by stack writes while trying to inject an interrupt event. */ + Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pCtx->rsp)); + return true; + } + + LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pCtx->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse, pDis->Param1.Base.idxGenReg)); + + /* Non-supervisor mode write means it's used for something else. */ + if (CPUMGetGuestCPL(pVCpu) == 3) + return true; + + switch (pDis->pCurInstr->uOpcode) + { + /* call implies the actual push of the return address faulted */ + case OP_CALL: + Log4(("pgmRZPoolMonitorIsReused: CALL\n")); + return true; + case OP_PUSH: + Log4(("pgmRZPoolMonitorIsReused: PUSH\n")); + return true; + case OP_PUSHF: + Log4(("pgmRZPoolMonitorIsReused: PUSHF\n")); + return true; + case OP_PUSHA: + Log4(("pgmRZPoolMonitorIsReused: PUSHA\n")); + return true; + case OP_FXSAVE: + Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n")); + return true; + case OP_MOVNTI: /* solaris - block_zero_no_xmm */ + Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n")); + return true; + case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */ + Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n")); + return true; + case OP_MOVSWD: + case OP_STOSWD: + if ( pDis->fPrefix == (DISPREFIX_REP|DISPREFIX_REX) + && pCtx->rcx >= 0x40 + ) + { + Assert(pDis->uCpuMode == DISCPUMODE_64BIT); + + Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n")); + return true; + } + break; + + default: + /* + * Anything having ESP on the left side means stack writes. + */ + if ( ( (pDis->Param1.fUse & DISUSE_REG_GEN32) + || (pDis->Param1.fUse & DISUSE_REG_GEN64)) + && (pDis->Param1.Base.idxGenReg == DISGREG_ESP)) + { + Log4(("pgmRZPoolMonitorIsReused: ESP\n")); + return true; + } + break; + } + + /* + * Page table updates are very very unlikely to be crossing page boundraries, + * and we don't want to deal with that in pgmPoolMonitorChainChanging and such. + */ + uint32_t const cbWrite = DISGetParamSize(pDis, &pDis->Param1); + if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) ) + { + Log4(("pgmRZPoolMonitorIsReused: cross page write\n")); + return true; + } + + /* + * Nobody does an unaligned 8 byte write to a page table, right. + */ + if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0) + { + Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n")); + return true; + } + + return false; +} + + +/** + * Flushes the page being accessed. + * + * @returns VBox status code suitable for scheduling. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pPool The pool. + * @param pPage The pool page (head). + * @param pDis The disassembly of the write instruction. + * @param pCtx Pointer to the register context for the CPU. + * @param GCPhysFault The fault address as guest physical address. + * @todo VBOXSTRICTRC + */ +static int pgmRZPoolAccessPfHandlerFlush(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis, + PCPUMCTX pCtx, RTGCPHYS GCPhysFault) +{ + NOREF(pVM); NOREF(GCPhysFault); + + /* + * First, do the flushing. + */ + pgmPoolMonitorChainFlush(pPool, pPage); + + /* + * Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection). + * Must do this in raw mode (!); XP boot will fail otherwise. + */ + int rc = VINF_SUCCESS; + VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, pCtx->rip); + if (rc2 == VINF_SUCCESS) + { /* do nothing */ } + else if (rc2 == VINF_EM_RESCHEDULE) + { + rc = VBOXSTRICTRC_VAL(rc2); +# ifndef IN_RING3 + VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3); +# endif + } + else if (rc2 == VERR_EM_INTERPRETER) + { + rc = VINF_EM_RAW_EMULATE_INSTR; + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr)); + } + else if (RT_FAILURE_NP(rc2)) + rc = VBOXSTRICTRC_VAL(rc2); + else + AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */ + + LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc)); + return rc; +} + + +/** + * Handles the STOSD write accesses. + * + * @returns VBox status code suitable for scheduling. + * @param pVM The cross context VM structure. + * @param pPool The pool. + * @param pPage The pool page (head). + * @param pDis The disassembly of the write instruction. + * @param pCtx Pointer to the register context for the CPU. + * @param GCPhysFault The fault address as guest physical address. + * @param pvFault The fault address. + */ +DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis, + PCPUMCTX pCtx, RTGCPHYS GCPhysFault, RTGCPTR pvFault) +{ + unsigned uIncrement = pDis->Param1.cb; + NOREF(pVM); + + Assert(pDis->uCpuMode == DISCPUMODE_32BIT || pDis->uCpuMode == DISCPUMODE_64BIT); + Assert(pCtx->rcx <= 0x20); + +# ifdef VBOX_STRICT + if (pDis->uOpMode == DISCPUMODE_32BIT) + Assert(uIncrement == 4); + else + Assert(uIncrement == 8); +# endif + + Log3(("pgmRZPoolAccessPfHandlerSTOSD\n")); + + /* + * Increment the modification counter and insert it into the list + * of modified pages the first time. + */ + if (!pPage->cModifications++) + pgmPoolMonitorModifiedInsert(pPool, pPage); + + /* + * Execute REP STOSD. + * + * This ASSUMES that we're not invoked by Trap0e on in a out-of-sync + * write situation, meaning that it's safe to write here. + */ + PVMCPUCC pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM)); + RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault; + while (pCtx->rcx) + { + pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement); + PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pCtx->rax, uIncrement); + pu32 += uIncrement; + GCPhysFault += uIncrement; + pCtx->rdi += uIncrement; + pCtx->rcx--; + } + pCtx->rip += pDis->cbInstr; + + LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n")); + return VINF_SUCCESS; +} + + +/** + * Handles the simple write accesses. + * + * @returns VBox status code suitable for scheduling. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pPool The pool. + * @param pPage The pool page (head). + * @param pDis The disassembly of the write instruction. + * @param pCtx Pointer to the register context for the CPU. + * @param GCPhysFault The fault address as guest physical address. + * @param pfReused Reused state (in/out) + */ +DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis, + PCPUMCTX pCtx, RTGCPHYS GCPhysFault, bool *pfReused) +{ + Log3(("pgmRZPoolAccessPfHandlerSimple\n")); + NOREF(pVM); + NOREF(pfReused); /* initialized by caller */ + + /* + * Increment the modification counter and insert it into the list + * of modified pages the first time. + */ + if (!pPage->cModifications++) + pgmPoolMonitorModifiedInsert(pPool, pPage); + + /* + * Clear all the pages. + */ + uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1); + if (cbWrite <= 8) + pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, cbWrite); + else if (cbWrite <= 16) + { + pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, 8); + pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + 8, NULL, cbWrite - 8); + } + else + { + Assert(cbWrite <= 32); + for (uint32_t off = 0; off < cbWrite; off += 8) + pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + off, NULL, RT_MIN(8, cbWrite - off)); + } + + /* + * Interpret the instruction. + */ + VBOXSTRICTRC rc = EMInterpretInstructionDisasState(pVCpu, pDis, pCtx->rip); + if (RT_SUCCESS(rc)) + AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rc))); /* ASSUMES no complicated stuff here. */ + else if (rc == VERR_EM_INTERPRETER) + { + LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n", + pCtx->cs.Sel, (RTGCPTR)pCtx->rip, pDis->pCurInstr->uOpcode)); + rc = VINF_EM_RAW_EMULATE_INSTR; + STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr)); + } + +# if 0 /* experimental code */ + if (rc == VINF_SUCCESS) + { + switch (pPage->enmKind) + { + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + { + X86PTEPAE GstPte; + int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte)); + AssertRC(rc); + + /* Check the new value written by the guest. If present and with a bogus physical address, then + * it's fairly safe to assume the guest is reusing the PT. + */ + if (GstPte.n.u1Present) + { + RTHCPHYS HCPhys = -1; + int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys); + if (rc != VINF_SUCCESS) + { + *pfReused = true; + STAM_COUNTER_INC(&pPool->StatForceFlushReused); + } + } + break; + } + } + } +# endif + + LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc))); + return VBOXSTRICTRC_VAL(rc); +} + + +/** + * @callback_method_impl{FNPGMRZPHYSPFHANDLER, + * \#PF access handler callback for page table pages.} + * + * @remarks The @a uUser argument is the index of the PGMPOOLPAGE. + */ +DECLCALLBACK(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx, + RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser) +{ + STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a); + PPGMPOOL const pPool = pVM->pgm.s.CTX_SUFF(pPool); + AssertReturn(uUser < pPool->cCurPages, VERR_PGM_POOL_IPE); + PPGMPOOLPAGE const pPage = &pPool->aPages[uUser]; + unsigned cMaxModifications; + bool fForcedFlush = false; + RT_NOREF_PV(uErrorCode); + +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + AssertMsg(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT, + ("pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault)); +# endif + LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault)); + + PGM_LOCK_VOID(pVM); + if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys)) + { + /* Pool page changed while we were waiting for the lock; ignore. */ + Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys))); + STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } +# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + if (pPage->fDirty) + { +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); +# endif + Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH)); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */ + } +# endif + +# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */ + if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT) + { + void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage); + void *pvGst; + int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc); + pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw); + } +# endif + +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + if (PGMPOOL_PAGE_IS_NESTED(pPage)) + { + Assert(!CPUMIsGuestInVmxNonRootMode(CPUMQueryGuestCtxPtr(pVCpu))); + Log7Func(("Flushing pvFault=%RGv GCPhysFault=%RGp\n", pvFault, GCPhysFault)); + pgmPoolMonitorChainFlush(pPool, pPage); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } +# endif + + /* + * Disassemble the faulting instruction. + */ + PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState; + int rc = EMInterpretDisasCurrent(pVCpu, pDis, NULL); + if (RT_UNLIKELY(rc != VINF_SUCCESS)) + { + AssertMsg(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT, ("Unexpected rc %d\n", rc)); + PGM_UNLOCK(pVM); + return rc; + } + + Assert(pPage->enmKind != PGMPOOLKIND_FREE); + + /* + * We should ALWAYS have the list head as user parameter. This + * is because we use that page to record the changes. + */ + Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX); + +# ifdef IN_RING0 + /* Maximum nr of modifications depends on the page type. */ + if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT + || pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT) + cMaxModifications = 4; + else + cMaxModifications = 24; +# else + cMaxModifications = 48; +# endif + + /* + * Incremental page table updates should weigh more than random ones. + * (Only applies when started from offset 0) + */ + pVCpu->pgm.s.cPoolAccessHandler++; + if ( pPage->GCPtrLastAccessHandlerRip >= pCtx->rip - 0x40 /* observed loops in Windows 7 x64 */ + && pPage->GCPtrLastAccessHandlerRip < pCtx->rip + 0x40 + && pvFault == (pPage->GCPtrLastAccessHandlerFault + pDis->Param1.cb) + && pVCpu->pgm.s.cPoolAccessHandler == pPage->cLastAccessHandler + 1) + { + Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind))); + Assert(pPage->cModifications < 32000); + pPage->cModifications = pPage->cModifications * 2; + pPage->GCPtrLastAccessHandlerFault = pvFault; + pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler; + if (pPage->cModifications >= cMaxModifications) + { + STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit); + fForcedFlush = true; + } + } + + if (pPage->cModifications >= cMaxModifications) + Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind))); + + /* + * Check if it's worth dealing with. + */ + bool fReused = false; + bool fNotReusedNotForking = false; + if ( ( pPage->cModifications < cMaxModifications /** @todo \#define */ /** @todo need to check that it's not mapping EIP. */ /** @todo adjust this! */ + || pgmPoolIsPageLocked(pPage) + ) + && !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pCtx, pDis, pvFault, pPage)) + && !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK)) + { + /* + * Simple instructions, no REP prefix. + */ + if (!(pDis->fPrefix & (DISPREFIX_REP | DISPREFIX_REPNE))) + { + rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pCtx, GCPhysFault, &fReused); + if (fReused) + goto flushPage; + + /* A mov instruction to change the first page table entry will be remembered so we can detect + * full page table changes early on. This will reduce the amount of unnecessary traps we'll take. + */ + if ( rc == VINF_SUCCESS + && !pPage->cLocked /* only applies to unlocked pages as we can't free locked ones (e.g. cr3 root). */ + && pDis->pCurInstr->uOpcode == OP_MOV + && (pvFault & PAGE_OFFSET_MASK) == 0) + { + pPage->GCPtrLastAccessHandlerFault = pvFault; + pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler; + pPage->GCPtrLastAccessHandlerRip = pCtx->rip; + /* Make sure we don't kick out a page too quickly. */ + if (pPage->cModifications > 8) + pPage->cModifications = 2; + } + else if (pPage->GCPtrLastAccessHandlerFault == pvFault) + { + /* ignore the 2nd write to this page table entry. */ + pPage->cLastAccessHandler = pVCpu->pgm.s.cPoolAccessHandler; + } + else + { + pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR; + pPage->GCPtrLastAccessHandlerRip = 0; + } + + STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a); + PGM_UNLOCK(pVM); + return rc; + } + + /* + * Windows is frequently doing small memset() operations (netio test 4k+). + * We have to deal with these or we'll kill the cache and performance. + */ + if ( pDis->pCurInstr->uOpcode == OP_STOSWD + && !pCtx->eflags.Bits.u1DF + && pDis->uOpMode == pDis->uCpuMode + && pDis->uAddrMode == pDis->uCpuMode) + { + bool fValidStosd = false; + + if ( pDis->uCpuMode == DISCPUMODE_32BIT + && pDis->fPrefix == DISPREFIX_REP + && pCtx->ecx <= 0x20 + && pCtx->ecx * 4 <= GUEST_PAGE_SIZE - ((uintptr_t)pvFault & GUEST_PAGE_OFFSET_MASK) + && !((uintptr_t)pvFault & 3) + && (pCtx->eax == 0 || pCtx->eax == 0x80) /* the two values observed. */ + ) + { + fValidStosd = true; + pCtx->rcx &= 0xffffffff; /* paranoia */ + } + else + if ( pDis->uCpuMode == DISCPUMODE_64BIT + && pDis->fPrefix == (DISPREFIX_REP | DISPREFIX_REX) + && pCtx->rcx <= 0x20 + && pCtx->rcx * 8 <= GUEST_PAGE_SIZE - ((uintptr_t)pvFault & GUEST_PAGE_OFFSET_MASK) + && !((uintptr_t)pvFault & 7) + && (pCtx->rax == 0 || pCtx->rax == 0x80) /* the two values observed. */ + ) + { + fValidStosd = true; + } + + if (fValidStosd) + { + rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pCtx, GCPhysFault, pvFault); + STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a); + PGM_UNLOCK(pVM); + return rc; + } + } + + /* REP prefix, don't bother. */ + STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix); + Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n", + pCtx->eax, pCtx->ecx, pCtx->edi, pCtx->esi, (RTGCPTR)pCtx->rip, pDis->pCurInstr->uOpcode, pDis->fPrefix)); + fNotReusedNotForking = true; + } + +# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0) + /* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This + * leads to pgm pool trashing and an excessive amount of write faults due to page monitoring. + */ + if ( pPage->cModifications >= cMaxModifications + && !fForcedFlush + && (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT || pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT) + && ( fNotReusedNotForking + || ( !pgmRZPoolMonitorIsReused(pVM, pVCpu, pCtx, pDis, pvFault, pPage) + && !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK)) + ) + ) + { + Assert(!pgmPoolIsPageLocked(pPage)); + Assert(pPage->fDirty == false); + + /* Flush any monitored duplicates as we will disable write protection. */ + if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX + || pPage->iMonitoredPrev != NIL_PGMPOOL_IDX) + { + PPGMPOOLPAGE pPageHead = pPage; + + /* Find the monitor head. */ + while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX) + pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev]; + + while (pPageHead) + { + unsigned idxNext = pPageHead->iMonitoredNext; + + if (pPageHead != pPage) + { + STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush); + Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind))); + int rc2 = pgmPoolFlushPage(pPool, pPageHead); + AssertRC(rc2); + } + + if (idxNext == NIL_PGMPOOL_IDX) + break; + + pPageHead = &pPool->aPages[idxNext]; + } + } + + /* The flushing above might fail for locked pages, so double check. */ + if ( pPage->iMonitoredNext == NIL_PGMPOOL_IDX + && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX) + { + pgmPoolAddDirtyPage(pVM, pPool, pPage); + + /* Temporarily allow write access to the page table again. */ + rc = PGMHandlerPhysicalPageTempOff(pVM, + pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK, + pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK); + if (rc == VINF_SUCCESS) + { + rc = PGMShwMakePageWritable(pVCpu, pvFault, PGM_MK_PG_IS_WRITE_FAULT); + AssertMsg(rc == VINF_SUCCESS + /* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */ + || rc == VERR_PAGE_TABLE_NOT_PRESENT + || rc == VERR_PAGE_NOT_PRESENT, + ("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc)); +# ifdef VBOX_STRICT + pPage->GCPtrDirtyFault = pvFault; +# endif + + STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a); + PGM_UNLOCK(pVM); + return rc; + } + } + } +# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT && IN_RING0 */ + + STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow); +flushPage: + /* + * Not worth it, so flush it. + * + * If we considered it to be reused, don't go back to ring-3 + * to emulate failed instructions since we usually cannot + * interpret then. This may be a bit risky, in which case + * the reuse detection must be fixed. + */ + rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pCtx, GCPhysFault); + if ( rc == VINF_EM_RAW_EMULATE_INSTR + && fReused) + { + Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */ + /* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */ + if (PGMShwGetPage(pVCpu, pCtx->rip, NULL, NULL) == VINF_SUCCESS) + rc = VINF_SUCCESS; /* safe to restart the instruction. */ + } + STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a); + PGM_UNLOCK(pVM); + return rc; +} + +#endif /* !IN_RING3 */ + +/** + * @callback_method_impl{FNPGMPHYSHANDLER, + * Access handler for shadowed page table pages.} + * + * @remarks Only uses the VINF_PGM_HANDLER_DO_DEFAULT status. + * @note The @a uUser argument is the index of the PGMPOOLPAGE. + */ +DECLCALLBACK(VBOXSTRICTRC) +pgmPoolAccessHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, + PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser) +{ + PPGMPOOL const pPool = pVM->pgm.s.CTX_SUFF(pPool); + STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a); + AssertReturn(uUser < pPool->cCurPages, VERR_PGM_POOL_IPE); + PPGMPOOLPAGE const pPage = &pPool->aPages[uUser]; + LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n", + GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind)); + + NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType); + + PGM_LOCK_VOID(pVM); + +#ifdef VBOX_WITH_STATISTICS + /* + * Collect stats on the access. + */ + AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19); + if (cbBuf <= 16 && cbBuf > 0) + STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]); + else if (cbBuf >= 17 && cbBuf < 32) + STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]); + else if (cbBuf >= 32 && cbBuf < 64) + STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]); + else if (cbBuf >= 64) + STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]); + + uint8_t cbAlign; + switch (pPage->enmKind) + { + default: + cbAlign = 7; + break; + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_32BIT_PD_PHYS: + cbAlign = 3; + break; + } + AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7); + if ((uint8_t)GCPhys & cbAlign) + STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]); +#endif + + /* + * Make sure the pool page wasn't modified by a different CPU. + */ + if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys)) + { + Assert(pPage->enmKind != PGMPOOLKIND_FREE); + + /* The max modification count before flushing depends on the context and page type. */ +#ifdef IN_RING3 + uint16_t const cMaxModifications = 96; /* it's cheaper here, right? */ +#else + uint16_t cMaxModifications; + if ( pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT + || pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT) + cMaxModifications = 4; + else + cMaxModifications = 24; +#endif + + /* + * We don't have to be very sophisticated about this since there are relativly few calls here. + * However, we must try our best to detect any non-cpu accesses (disk / networking). + */ + if ( ( pPage->cModifications < cMaxModifications + || pgmPoolIsPageLocked(pPage) ) + && enmOrigin != PGMACCESSORIGIN_DEVICE + && cbBuf <= 16) + { + /* Clear the shadow entry. */ + if (!pPage->cModifications++) + pgmPoolMonitorModifiedInsert(pPool, pPage); + + if (cbBuf <= 8) + pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, (uint32_t)cbBuf); + else + { + pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, 8); + pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys + 8, (uint8_t *)pvBuf + 8, (uint32_t)cbBuf - 8); + } + } + else + pgmPoolMonitorChainFlush(pPool, pPage); + + STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a); + } + else + Log(("CPU%d: PGM_ALL_CB_DECL pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys))); + PGM_UNLOCK(pVM); + return VINF_PGM_HANDLER_DO_DEFAULT; +} + + +#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + +# if defined(VBOX_STRICT) && !defined(IN_RING3) + +/** + * Check references to guest physical memory in a PAE / PAE page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + * @param pGstPT The guest page table. + */ +static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT) +{ + unsigned cErrors = 0; + int LastRc = -1; /* initialized to shut up gcc */ + unsigned LastPTE = ~0U; /* initialized to shut up gcc */ + RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */ + PVMCC pVM = pPool->CTX_SUFF(pVM); + +# ifdef VBOX_STRICT + for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++) + AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent)); +# endif + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++) + { + if (PGMSHWPTEPAE_IS_P(pShwPT->a[i])) + { + RTHCPHYS HCPhys = NIL_RTHCPHYS; + int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys); + if ( rc != VINF_SUCCESS + || PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys) + { + Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys)); + LastPTE = i; + LastRc = rc; + LastHCPhys = HCPhys; + cErrors++; + + RTHCPHYS HCPhysPT = NIL_RTHCPHYS; + rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT); + AssertRC(rc); + + for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++) + { + PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage]; + + if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT) + { + PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage); + + for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++) + { + if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j]) + && PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT) + { + Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j]))); + } + } + + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2); + } + } + } + } + } + AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys)); +} + + +/** + * Check references to guest physical memory in a PAE / 32-bit page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + * @param pGstPT The guest page table. + */ +static void pgmPoolTrackCheckPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT) +{ + unsigned cErrors = 0; + int LastRc = -1; /* initialized to shut up gcc */ + unsigned LastPTE = ~0U; /* initialized to shut up gcc */ + RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */ + PVMCC pVM = pPool->CTX_SUFF(pVM); + +# ifdef VBOX_STRICT + for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++) + AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent)); +# endif + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++) + { + if (PGMSHWPTEPAE_IS_P(pShwPT->a[i])) + { + RTHCPHYS HCPhys = NIL_RTHCPHYS; + int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys); + if ( rc != VINF_SUCCESS + || PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys) + { + Log(("rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys)); + LastPTE = i; + LastRc = rc; + LastHCPhys = HCPhys; + cErrors++; + + RTHCPHYS HCPhysPT = NIL_RTHCPHYS; + rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT); + AssertRC(rc); + + for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++) + { + PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage]; + + if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT) + { + PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage); + + for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++) + { + if ( PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j]) + && PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT) + { + Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j]))); + } + } + + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2); + } + } + } + } + } + AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys)); +} + +# endif /* VBOX_STRICT && !IN_RING3 */ + +/** + * Clear references to guest physical memory in a PAE / PAE page table. + * + * @returns nr of changed PTEs + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + * @param pGstPT The guest page table. + * @param pOldGstPT The old cached guest page table. + * @param fAllowRemoval Bail out as soon as we encounter an invalid PTE + * @param pfFlush Flush reused page table (out) + */ +DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT, + PCX86PTPAE pOldGstPT, bool fAllowRemoval, bool *pfFlush) +{ + unsigned cChanged = 0; + +# ifdef VBOX_STRICT + for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++) + AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent)); +# endif + *pfFlush = false; + + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++) + { + /* Check the new value written by the guest. If present and with a bogus physical address, then + * it's fairly safe to assume the guest is reusing the PT. + */ + if ( fAllowRemoval + && (pGstPT->a[i].u & X86_PTE_P)) + { + if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK)) + { + *pfFlush = true; + return ++cChanged; + } + } + if (PGMSHWPTEPAE_IS_P(pShwPT->a[i])) + { + /* If the old cached PTE is identical, then there's no need to flush the shadow copy. */ + if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK)) + { +# ifdef VBOX_STRICT + RTHCPHYS HCPhys = NIL_RTGCPHYS; + int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys); + AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys)); +# endif + uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX); + bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW); + uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX); + bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW); + + if ( uHostAttr == uGuestAttr + && fHostRW <= fGuestRW) + continue; + } + cChanged++; + /* Something was changed, so flush it. */ + Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n", + i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK, i); + PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0); + } + } + return cChanged; +} + + +/** + * Clear references to guest physical memory in a PAE / PAE page table. + * + * @returns nr of changed PTEs + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + * @param pGstPT The guest page table. + * @param pOldGstPT The old cached guest page table. + * @param fAllowRemoval Bail out as soon as we encounter an invalid PTE + * @param pfFlush Flush reused page table (out) + */ +DECLINLINE(unsigned) pgmPoolTrackFlushPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT, + PCX86PT pOldGstPT, bool fAllowRemoval, bool *pfFlush) +{ + unsigned cChanged = 0; + +# ifdef VBOX_STRICT + for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++) + AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), pPage->iFirstPresent)); +# endif + *pfFlush = false; + + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++) + { + /* Check the new value written by the guest. If present and with a bogus physical address, then + * it's fairly safe to assume the guest is reusing the PT. */ + if (fAllowRemoval) + { + X86PGUINT const uPte = pGstPT->a[i].u; + if ( (uPte & X86_PTE_P) + && !PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), uPte & X86_PTE_PG_MASK)) + { + *pfFlush = true; + return ++cChanged; + } + } + if (PGMSHWPTEPAE_IS_P(pShwPT->a[i])) + { + /* If the old cached PTE is identical, then there's no need to flush the shadow copy. */ + if ((pGstPT->a[i].u & X86_PTE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PG_MASK)) + { +# ifdef VBOX_STRICT + RTHCPHYS HCPhys = NIL_RTGCPHYS; + int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys); + AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %x old %x shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys)); +# endif + uint64_t uHostAttr = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G); + bool fHostRW = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW); + uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G); + bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW); + + if ( uHostAttr == uGuestAttr + && fHostRW <= fGuestRW) + continue; + } + cChanged++; + /* Something was changed, so flush it. */ + Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%x\n", + i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK, i); + PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0); + } + } + return cChanged; +} + + +/** + * Flush a dirty page + * + * @param pVM The cross context VM structure. + * @param pPool The pool. + * @param idxSlot Dirty array slot index + * @param fAllowRemoval Allow a reused page table to be removed + */ +static void pgmPoolFlushDirtyPage(PVMCC pVM, PPGMPOOL pPool, unsigned idxSlot, bool fAllowRemoval = false) +{ + AssertCompile(RT_ELEMENTS(pPool->aidxDirtyPages) == RT_ELEMENTS(pPool->aDirtyPages)); + + Assert(idxSlot < RT_ELEMENTS(pPool->aDirtyPages)); + unsigned idxPage = pPool->aidxDirtyPages[idxSlot]; + if (idxPage == NIL_PGMPOOL_IDX) + return; + + PPGMPOOLPAGE pPage = &pPool->aPages[idxPage]; + Assert(pPage->idx == idxPage); + Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX); + + AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot)); + Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications)); + + /* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */ + int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK); + Assert(rc == VINF_SUCCESS); + pPage->fDirty = false; + +# ifdef VBOX_STRICT + uint64_t fFlags = 0; + RTHCPHYS HCPhys; + rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->GCPtrDirtyFault, &fFlags, &HCPhys); + AssertMsg( ( rc == VINF_SUCCESS + && (!(fFlags & X86_PTE_RW) || HCPhys != pPage->Core.Key)) + /* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */ + || rc == VERR_PAGE_TABLE_NOT_PRESENT + || rc == VERR_PAGE_NOT_PRESENT, + ("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->GCPtrDirtyFault, rc, fFlags)); +# endif + + /* Flush those PTEs that have changed. */ + STAM_PROFILE_START(&pPool->StatTrackDeref,a); + void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage); + void *pvGst; + rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc); + bool fFlush; + unsigned cChanges; + + if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT) + cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst, + (PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush); + else + { + Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */ + cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst, + (PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush); + } + + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw); + STAM_PROFILE_STOP(&pPool->StatTrackDeref,a); + /* Note: we might want to consider keeping the dirty page active in case there were many changes. */ + + /* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */ + Assert(pPage->cModifications); + if (cChanges < 4) + pPage->cModifications = 1; /* must use > 0 here */ + else + pPage->cModifications = RT_MAX(1, pPage->cModifications / 2); + + STAM_COUNTER_INC(&pPool->StatResetDirtyPages); + if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages)) + pPool->idxFreeDirtyPage = idxSlot; + + pPool->cDirtyPages--; + pPool->aidxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX; + Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages)); + if (fFlush) + { + Assert(fAllowRemoval); + Log(("Flush reused page table!\n")); + pgmPoolFlushPage(pPool, pPage); + STAM_COUNTER_INC(&pPool->StatForceFlushReused); + } + else + Log(("Removed dirty page %RGp cMods=%d cChanges=%d\n", pPage->GCPhys, pPage->cModifications, cChanges)); +} + + +# ifndef IN_RING3 +/** + * Add a new dirty page + * + * @param pVM The cross context VM structure. + * @param pPool The pool. + * @param pPage The page. + */ +void pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 || RT_ELEMENTS(pPool->aDirtyPages) == 16); + Assert(!pPage->fDirty); + Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); + + unsigned idxFree = pPool->idxFreeDirtyPage; + Assert(idxFree < RT_ELEMENTS(pPool->aDirtyPages)); + Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX); + + if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aDirtyPages)) + { + STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush); + pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* allow removal of reused page tables*/); + } + Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages)); + AssertMsg(pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages)); + + Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree)); + + /* + * Make a copy of the guest page table as we require valid GCPhys addresses + * when removing references to physical pages. + * (The HCPhys linear lookup is *extremely* expensive!) + */ + void *pvGst; + int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc); + memcpy(&pPool->aDirtyPages[idxFree].aPage[0], pvGst, + pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT ? PAGE_SIZE : PAGE_SIZE / 2); +# ifdef VBOX_STRICT + void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage); + if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT) + pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst); + else + pgmPoolTrackCheckPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw); +# endif + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst); + + STAM_COUNTER_INC(&pPool->StatDirtyPage); + pPage->fDirty = true; + pPage->idxDirtyEntry = (uint8_t)idxFree; Assert(pPage->idxDirtyEntry == idxFree); + pPool->aidxDirtyPages[idxFree] = pPage->idx; + pPool->cDirtyPages++; + + pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aDirtyPages) - 1); + if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages) + && pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX) + { + unsigned i; + for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++) + { + idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aDirtyPages) - 1); + if (pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX) + { + pPool->idxFreeDirtyPage = idxFree; + break; + } + } + Assert(i != RT_ELEMENTS(pPool->aDirtyPages)); + } + + Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages) || pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX); + + /* + * Clear all references to this shadow table. See @bugref{7298}. + */ + pgmPoolTrackClearPageUsers(pPool, pPage); +} +# endif /* !IN_RING3 */ + + +/** + * Check if the specified page is dirty (not write monitored) + * + * @return dirty or not + * @param pVM The cross context VM structure. + * @param GCPhys Guest physical address + */ +bool pgmPoolIsDirtyPageSlow(PVMCC pVM, RTGCPHYS GCPhys) +{ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PGM_LOCK_ASSERT_OWNER(pVM); + if (!pPool->cDirtyPages) + return false; + + GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK; + + for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++) + { + unsigned idxPage = pPool->aidxDirtyPages[i]; + if (idxPage != NIL_PGMPOOL_IDX) + { + PPGMPOOLPAGE pPage = &pPool->aPages[idxPage]; + if (pPage->GCPhys == GCPhys) + return true; + } + } + return false; +} + + +/** + * Reset all dirty pages by reinstating page monitoring. + * + * @param pVM The cross context VM structure. + */ +void pgmPoolResetDirtyPages(PVMCC pVM) +{ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PGM_LOCK_ASSERT_OWNER(pVM); + Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages)); + + if (!pPool->cDirtyPages) + return; + + Log(("pgmPoolResetDirtyPages\n")); + for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++) + pgmPoolFlushDirtyPage(pVM, pPool, i, true /* allow removal of reused page tables*/); + + pPool->idxFreeDirtyPage = 0; + if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages) + && pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX) + { + unsigned i; + for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++) + { + if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX) + { + pPool->idxFreeDirtyPage = i; + break; + } + } + AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages)); + } + + Assert(pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX || pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages)); + return; +} + + +/** + * Invalidate the PT entry for the specified page + * + * @param pVM The cross context VM structure. + * @param GCPtrPage Guest page to invalidate + */ +void pgmPoolResetDirtyPage(PVMCC pVM, RTGCPTR GCPtrPage) +{ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PGM_LOCK_ASSERT_OWNER(pVM); + Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages)); + + if (!pPool->cDirtyPages) + return; + + Log(("pgmPoolResetDirtyPage %RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage); + for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++) + { + /** @todo What was intended here??? This looks incomplete... */ + } +} + + +/** + * Reset all dirty pages by reinstating page monitoring. + * + * @param pVM The cross context VM structure. + * @param GCPhysPT Physical address of the page table + */ +void pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT) +{ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PGM_LOCK_ASSERT_OWNER(pVM); + Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages)); + unsigned idxDirtyPage = RT_ELEMENTS(pPool->aDirtyPages); + + if (!pPool->cDirtyPages) + return; + + GCPhysPT = GCPhysPT & ~(RTGCPHYS)PAGE_OFFSET_MASK; + + for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++) + { + unsigned idxPage = pPool->aidxDirtyPages[i]; + if (idxPage != NIL_PGMPOOL_IDX) + { + PPGMPOOLPAGE pPage = &pPool->aPages[idxPage]; + if (pPage->GCPhys == GCPhysPT) + { + idxDirtyPage = i; + break; + } + } + } + + if (idxDirtyPage != RT_ELEMENTS(pPool->aDirtyPages)) + { + pgmPoolFlushDirtyPage(pVM, pPool, idxDirtyPage, true /* allow removal of reused page tables*/); + if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages) + && pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX) + { + unsigned i; + for (i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++) + { + if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX) + { + pPool->idxFreeDirtyPage = i; + break; + } + } + AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages)); + } + } +} + +#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */ + +/** + * Inserts a page into the GCPhys hash table. + * + * @param pPool The pool. + * @param pPage The page. + */ +DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys)); + Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX); + uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys); + pPage->iNext = pPool->aiHash[iHash]; + pPool->aiHash[iHash] = pPage->idx; +} + + +/** + * Removes a page from the GCPhys hash table. + * + * @param pPool The pool. + * @param pPage The page. + */ +DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys)); + uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys); + if (pPool->aiHash[iHash] == pPage->idx) + pPool->aiHash[iHash] = pPage->iNext; + else + { + uint16_t iPrev = pPool->aiHash[iHash]; + for (;;) + { + const int16_t i = pPool->aPages[iPrev].iNext; + if (i == pPage->idx) + { + pPool->aPages[iPrev].iNext = pPage->iNext; + break; + } + if (i == NIL_PGMPOOL_IDX) + { + AssertReleaseMsgFailed(("GCPhys=%RGp idx=%d\n", pPage->GCPhys, pPage->idx)); + break; + } + iPrev = i; + } + } + pPage->iNext = NIL_PGMPOOL_IDX; +} + + +/** + * Frees up one cache page. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @param pPool The pool. + * @param iUser The user index. + */ +static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser) +{ + const PVMCC pVM = pPool->CTX_SUFF(pVM); + Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */ + STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne); + + /* + * Select one page from the tail of the age list. + */ + PPGMPOOLPAGE pPage; + for (unsigned iLoop = 0; ; iLoop++) + { + uint16_t iToFree = pPool->iAgeTail; + if (iToFree == iUser && iUser != NIL_PGMPOOL_IDX) + iToFree = pPool->aPages[iToFree].iAgePrev; +/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls. + if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX) + { + uint16_t i = pPool->aPages[iToFree].iAgePrev; + for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev) + { + if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX) + continue; + iToFree = i; + break; + } + } +*/ + Assert(iToFree != iUser); + AssertReleaseMsg(iToFree != NIL_PGMPOOL_IDX, + ("iToFree=%#x (iAgeTail=%#x) iUser=%#x iLoop=%u - pPool=%p LB %#zx\n", + iToFree, pPool->iAgeTail, iUser, iLoop, pPool, + RT_UOFFSETOF_DYN(PGMPOOL, aPages[pPool->cMaxPages]) + + pPool->cMaxUsers * sizeof(PGMPOOLUSER) + + pPool->cMaxPhysExts * sizeof(PGMPOOLPHYSEXT) )); + + pPage = &pPool->aPages[iToFree]; + + /* + * Reject any attempts at flushing the currently active shadow CR3 mapping. + * Call pgmPoolCacheUsed to move the page to the head of the age list. + */ + if ( !pgmPoolIsPageLocked(pPage) + && pPage->idx >= PGMPOOL_IDX_FIRST /* paranoia (#6349) */) + break; + LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n")); + pgmPoolCacheUsed(pPool, pPage); + AssertLogRelReturn(iLoop < 8192, VERR_PGM_POOL_TOO_MANY_LOOPS); + } + + /* + * Found a usable page, flush it and return. + */ + int rc = pgmPoolFlushPage(pPool, pPage); + /* This flush was initiated by us and not the guest, so explicitly flush the TLB. */ + /** @todo find out why this is necessary; pgmPoolFlushPage should trigger a flush if one is really needed. */ + if (rc == VINF_SUCCESS) + PGM_INVL_ALL_VCPU_TLBS(pVM); + return rc; +} + + +/** + * Checks if a kind mismatch is really a page being reused + * or if it's just normal remappings. + * + * @returns true if reused and the cached page (enmKind1) should be flushed + * @returns false if not reused. + * @param enmKind1 The kind of the cached page. + * @param enmKind2 The kind of the requested page. + */ +static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2) +{ + switch (enmKind1) + { + /* + * Never reuse them. There is no remapping in non-paging mode. + */ + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_32BIT_PD_PHYS: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_PHYS: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */ + return false; + + /* + * It's perfectly fine to reuse these, except for PAE and non-paging stuff. + */ + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_PAE_PDPT: + Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2)); + switch (enmKind2) + { + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + case PGMPOOLKIND_64BIT_PML4: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + return true; + default: + return false; + } + + /* + * It's perfectly fine to reuse these, except for PAE and non-paging stuff. + */ + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + case PGMPOOLKIND_64BIT_PML4: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2)); + switch (enmKind2) + { + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + return true; + default: + return false; + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: + case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB: + case PGMPOOLKIND_EPT_PD_FOR_EPT_PD: + case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT: + return PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2); + + case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4: + return false; +#endif + + /* + * These cannot be flushed, and it's common to reuse the PDs as PTs. + */ + case PGMPOOLKIND_ROOT_NESTED: + return false; + + default: + AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1)); + } +} + + +/** + * Attempts to satisfy a pgmPoolAlloc request from the cache. + * + * @returns VBox status code. + * @retval VINF_PGM_CACHED_PAGE on success. + * @retval VERR_FILE_NOT_FOUND if not found. + * @param pPool The pool. + * @param GCPhys The GC physical address of the page we're gonna shadow. + * @param enmKind The kind of mapping. + * @param enmAccess Access type for the mapping (only relevant for big pages) + * @param fA20Enabled Whether the CPU has the A20 gate enabled. + * @param iUser The shadow page pool index of the user table. This is + * NIL_PGMPOOL_IDX for root pages. + * @param iUserTable The index into the user table (shadowed). Ignored if + * root page + * @param ppPage Where to store the pointer to the page. + */ +static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled, + uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage) +{ + /* + * Look up the GCPhys in the hash. + */ + unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)]; + Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i)); + if (i != NIL_PGMPOOL_IDX) + { + do + { + PPGMPOOLPAGE pPage = &pPool->aPages[i]; + Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys)); + if (pPage->GCPhys == GCPhys) + { + if ( (PGMPOOLKIND)pPage->enmKind == enmKind + && (PGMPOOLACCESS)pPage->enmAccess == enmAccess + && pPage->fA20Enabled == fA20Enabled) + { + /* Put it at the start of the use list to make sure pgmPoolTrackAddUser + * doesn't flush it in case there are no more free use records. + */ + pgmPoolCacheUsed(pPool, pPage); + + int rc = VINF_SUCCESS; + if (iUser != NIL_PGMPOOL_IDX) + rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable); + if (RT_SUCCESS(rc)) + { + Assert((PGMPOOLKIND)pPage->enmKind == enmKind); + *ppPage = pPage; + if (pPage->cModifications) + pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */ + STAM_COUNTER_INC(&pPool->StatCacheHits); + return VINF_PGM_CACHED_PAGE; + } + return rc; + } + + if ((PGMPOOLKIND)pPage->enmKind != enmKind) + { + /* + * The kind is different. In some cases we should now flush the page + * as it has been reused, but in most cases this is normal remapping + * of PDs as PT or big pages using the GCPhys field in a slightly + * different way than the other kinds. + */ + if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind)) + { + STAM_COUNTER_INC(&pPool->StatCacheKindMismatches); + pgmPoolFlushPage(pPool, pPage); + break; + } + } + } + + /* next */ + i = pPage->iNext; + } while (i != NIL_PGMPOOL_IDX); + } + + Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind))); + STAM_COUNTER_INC(&pPool->StatCacheMisses); + return VERR_FILE_NOT_FOUND; +} + + +/** + * Inserts a page into the cache. + * + * @param pPool The pool. + * @param pPage The cached page. + * @param fCanBeCached Set if the page is fit for caching from the caller's point of view. + */ +static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached) +{ + /* + * Insert into the GCPhys hash if the page is fit for that. + */ + Assert(!pPage->fCached); + if (fCanBeCached) + { + pPage->fCached = true; + pgmPoolHashInsert(pPool, pPage); + Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n", + pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys)); + STAM_COUNTER_INC(&pPool->StatCacheCacheable); + } + else + { + Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n", + pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys)); + STAM_COUNTER_INC(&pPool->StatCacheUncacheable); + } + + /* + * Insert at the head of the age list. + */ + pPage->iAgePrev = NIL_PGMPOOL_IDX; + pPage->iAgeNext = pPool->iAgeHead; + if (pPool->iAgeHead != NIL_PGMPOOL_IDX) + pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx; + else + pPool->iAgeTail = pPage->idx; + pPool->iAgeHead = pPage->idx; +} + + +/** + * Flushes a cached page. + * + * @param pPool The pool. + * @param pPage The cached page. + */ +static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys)); + + /* + * Remove the page from the hash. + */ + if (pPage->fCached) + { + pPage->fCached = false; + pgmPoolHashRemove(pPool, pPage); + } + else + Assert(pPage->iNext == NIL_PGMPOOL_IDX); + + /* + * Remove it from the age list. + */ + if (pPage->iAgeNext != NIL_PGMPOOL_IDX) + pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev; + else + pPool->iAgeTail = pPage->iAgePrev; + if (pPage->iAgePrev != NIL_PGMPOOL_IDX) + pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext; + else + pPool->iAgeHead = pPage->iAgeNext; + pPage->iAgeNext = NIL_PGMPOOL_IDX; + pPage->iAgePrev = NIL_PGMPOOL_IDX; +} + + +/** + * Looks for pages sharing the monitor. + * + * @returns Pointer to the head page. + * @returns NULL if not found. + * @param pPool The Pool + * @param pNewPage The page which is going to be monitored. + */ +static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage) +{ + /* + * Look up the GCPhys in the hash. + */ + RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK; + unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)]; + if (i == NIL_PGMPOOL_IDX) + return NULL; + do + { + PPGMPOOLPAGE pPage = &pPool->aPages[i]; + if ( pPage->GCPhys - GCPhys < PAGE_SIZE + && pPage != pNewPage) + { + switch (pPage->enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + case PGMPOOLKIND_64BIT_PML4: + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_PAE_PDPT: +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: + case PGMPOOLKIND_EPT_PD_FOR_EPT_PD: + case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT: +#endif + { + /* find the head */ + while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX) + { + Assert(pPage->iMonitoredPrev != pPage->idx); + pPage = &pPool->aPages[pPage->iMonitoredPrev]; + } + return pPage; + } + + /* ignore, no monitoring. */ + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + case PGMPOOLKIND_ROOT_NESTED: + case PGMPOOLKIND_PAE_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_PHYS: + case PGMPOOLKIND_32BIT_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB: + case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4: +#endif + break; + default: + AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx)); + } + } + + /* next */ + i = pPage->iNext; + } while (i != NIL_PGMPOOL_IDX); + return NULL; +} + + +/** + * Enabled write monitoring of a guest page. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @param pPool The pool. + * @param pPage The cached page. + */ +static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK)); + + /* + * Filter out the relevant kinds. + */ + switch (pPage->enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + case PGMPOOLKIND_64BIT_PML4: + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_PAE_PDPT: + break; + + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + case PGMPOOLKIND_ROOT_NESTED: + /* Nothing to monitor here. */ + return VINF_SUCCESS; + + case PGMPOOLKIND_32BIT_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_PHYS: + case PGMPOOLKIND_PAE_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: + /* Nothing to monitor here. */ + return VINF_SUCCESS; + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: + case PGMPOOLKIND_EPT_PD_FOR_EPT_PD: + case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT: + break; + + case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB: + case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4: + /* Nothing to monitor here. */ + return VINF_SUCCESS; +#endif + + default: + AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind)); + } + + /* + * Install handler. + */ + int rc; + PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage); + if (pPageHead) + { + Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx); + Assert(pPageHead->iMonitoredPrev != pPage->idx); + +#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + if (pPageHead->fDirty) + pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirtyEntry, false /* do not remove */); +#endif + + pPage->iMonitoredPrev = pPageHead->idx; + pPage->iMonitoredNext = pPageHead->iMonitoredNext; + if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX) + pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx; + pPageHead->iMonitoredNext = pPage->idx; + rc = VINF_SUCCESS; + if (PGMPOOL_PAGE_IS_NESTED(pPage)) + Log7Func(("Adding to monitoring list GCPhysPage=%RGp\n", pPage->GCPhys)); + } + else + { + if (PGMPOOL_PAGE_IS_NESTED(pPage)) + Log7Func(("Started monitoring GCPhysPage=%RGp HCPhys=%RHp enmKind=%s\n", pPage->GCPhys, pPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind))); + + Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX); + PVMCC pVM = pPool->CTX_SUFF(pVM); + const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK; + rc = PGMHandlerPhysicalRegister(pVM, GCPhysPage, GCPhysPage + PAGE_OFFSET_MASK, pPool->hAccessHandlerType, + pPage - &pPool->aPages[0], NIL_RTR3PTR /*pszDesc*/); + /** @todo we should probably deal with out-of-memory conditions here, but for now increasing + * the heap size should suffice. */ + AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc)); + PVMCPU pVCpu = VMMGetCpu(pVM); + AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), ("fSyncFlags=%x syncff=%d\n", pVCpu->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))); + } + pPage->fMonitored = true; + return rc; +} + + +/** + * Disables write monitoring of a guest page. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @param pPool The pool. + * @param pPage The cached page. + */ +static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + /* + * Filter out the relevant kinds. + */ + switch (pPage->enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + case PGMPOOLKIND_64BIT_PML4: + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_PAE_PDPT: + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + break; + + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + case PGMPOOLKIND_ROOT_NESTED: + case PGMPOOLKIND_PAE_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_PHYS: + case PGMPOOLKIND_32BIT_PD_PHYS: + /* Nothing to monitor here. */ + Assert(!pPage->fMonitored); + return VINF_SUCCESS; + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: + case PGMPOOLKIND_EPT_PD_FOR_EPT_PD: + case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT: + break; + + case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB: + case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4: + /* Nothing to monitor here. */ + Assert(!pPage->fMonitored); + return VINF_SUCCESS; +#endif + + default: + AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind)); + } + Assert(pPage->fMonitored); + + /* + * Remove the page from the monitored list or uninstall it if last. + */ + const PVMCC pVM = pPool->CTX_SUFF(pVM); + int rc; + if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX + || pPage->iMonitoredPrev != NIL_PGMPOOL_IDX) + { + if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX) + { + PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext]; + pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX; + rc = PGMHandlerPhysicalChangeUserArg(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK, pPage->iMonitoredNext); + + AssertFatalRCSuccess(rc); + pPage->iMonitoredNext = NIL_PGMPOOL_IDX; + } + else + { + pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext; + if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX) + { + pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev; + pPage->iMonitoredNext = NIL_PGMPOOL_IDX; + } + pPage->iMonitoredPrev = NIL_PGMPOOL_IDX; + rc = VINF_SUCCESS; + } + } + else + { + rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK); + AssertFatalRC(rc); + PVMCPU pVCpu = VMMGetCpu(pVM); + AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), + ("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions)); + } + pPage->fMonitored = false; + + /* + * Remove it from the list of modified pages (if in it). + */ + pgmPoolMonitorModifiedRemove(pPool, pPage); + + if (PGMPOOL_PAGE_IS_NESTED(pPage)) + Log7Func(("Stopped monitoring %RGp\n", pPage->GCPhys)); + + return rc; +} + + +/** + * Inserts the page into the list of modified pages. + * + * @param pPool The pool. + * @param pPage The page. + */ +void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx)); + AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX + && pPage->iModifiedPrev == NIL_PGMPOOL_IDX + && pPool->iModifiedHead != pPage->idx, + ("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n", + pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications, + pPool->iModifiedHead, pPool->cModifiedPages)); + + pPage->iModifiedNext = pPool->iModifiedHead; + if (pPool->iModifiedHead != NIL_PGMPOOL_IDX) + pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx; + pPool->iModifiedHead = pPage->idx; + pPool->cModifiedPages++; +#ifdef VBOX_WITH_STATISTICS + if (pPool->cModifiedPages > pPool->cModifiedPagesHigh) + pPool->cModifiedPagesHigh = pPool->cModifiedPages; +#endif +} + + +/** + * Removes the page from the list of modified pages and resets the + * modification counter. + * + * @param pPool The pool. + * @param pPage The page which is believed to be in the list of modified pages. + */ +static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications)); + if (pPool->iModifiedHead == pPage->idx) + { + Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX); + pPool->iModifiedHead = pPage->iModifiedNext; + if (pPage->iModifiedNext != NIL_PGMPOOL_IDX) + { + pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX; + pPage->iModifiedNext = NIL_PGMPOOL_IDX; + } + pPool->cModifiedPages--; + } + else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX) + { + pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext; + if (pPage->iModifiedNext != NIL_PGMPOOL_IDX) + { + pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev; + pPage->iModifiedNext = NIL_PGMPOOL_IDX; + } + pPage->iModifiedPrev = NIL_PGMPOOL_IDX; + pPool->cModifiedPages--; + } + else + Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX); + pPage->cModifications = 0; +} + + +/** + * Zaps the list of modified pages, resetting their modification counters in the process. + * + * @param pVM The cross context VM structure. + */ +static void pgmPoolMonitorModifiedClearAll(PVMCC pVM) +{ + PGM_LOCK_VOID(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages)); + + unsigned cPages = 0; NOREF(cPages); + +#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + pgmPoolResetDirtyPages(pVM); +#endif + + uint16_t idx = pPool->iModifiedHead; + pPool->iModifiedHead = NIL_PGMPOOL_IDX; + while (idx != NIL_PGMPOOL_IDX) + { + PPGMPOOLPAGE pPage = &pPool->aPages[idx]; + idx = pPage->iModifiedNext; + pPage->iModifiedNext = NIL_PGMPOOL_IDX; + pPage->iModifiedPrev = NIL_PGMPOOL_IDX; + pPage->cModifications = 0; + Assert(++cPages); + } + AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages)); + pPool->cModifiedPages = 0; + PGM_UNLOCK(pVM); +} + + +/** + * Handle SyncCR3 pool tasks + * + * @returns VBox status code. + * @retval VINF_SUCCESS if successfully added. + * @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only) + * @param pVCpu The cross context virtual CPU structure. + * @remark Should only be used when monitoring is available, thus placed in + * the PGMPOOL_WITH_MONITORING \#ifdef. + */ +int pgmPoolSyncCR3(PVMCPUCC pVCpu) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + LogFlow(("pgmPoolSyncCR3 fSyncFlags=%x\n", pVCpu->pgm.s.fSyncFlags)); + + /* + * When monitoring shadowed pages, we reset the modification counters on CR3 sync. + * Occasionally we will have to clear all the shadow page tables because we wanted + * to monitor a page which was mapped by too many shadowed page tables. This operation + * sometimes referred to as a 'lightweight flush'. + */ +# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */ + if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) + pgmR3PoolClearAll(pVM, false /*fFlushRemTlb*/); +# else /* !IN_RING3 */ + if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) + { + Log(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n")); + VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */ + + /* Make sure all other VCPUs return to ring 3. */ + if (pVM->cCpus > 1) + { + VM_FF_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING); + PGM_INVL_ALL_VCPU_TLBS(pVM); + } + return VINF_PGM_SYNC_CR3; + } +# endif /* !IN_RING3 */ + else + { + pgmPoolMonitorModifiedClearAll(pVM); + + /* pgmPoolMonitorModifiedClearAll can cause a pgm pool flush (dirty page clearing), so make sure we handle this! */ + if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) + { + Log(("pgmPoolMonitorModifiedClearAll caused a pgm flush -> call pgmPoolSyncCR3 again!\n")); + return pgmPoolSyncCR3(pVCpu); + } + } + return VINF_SUCCESS; +} + + +/** + * Frees up at least one user entry. + * + * @returns VBox status code. + * @retval VINF_SUCCESS if successfully added. + * + * @param pPool The pool. + * @param iUser The user index. + */ +static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser) +{ + STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser); + /* + * Just free cached pages in a braindead fashion. + */ + /** @todo walk the age list backwards and free the first with usage. */ + int rc = VINF_SUCCESS; + do + { + int rc2 = pgmPoolCacheFreeOne(pPool, iUser); + if (RT_FAILURE(rc2) && rc == VINF_SUCCESS) + rc = rc2; + } while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX); + return rc; +} + + +/** + * Inserts a page into the cache. + * + * This will create user node for the page, insert it into the GCPhys + * hash, and insert it into the age list. + * + * @returns VBox status code. + * @retval VINF_SUCCESS if successfully added. + * + * @param pPool The pool. + * @param pPage The cached page. + * @param GCPhys The GC physical address of the page we're gonna shadow. + * @param iUser The user index. + * @param iUserTable The user table index. + */ +DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable) +{ + int rc = VINF_SUCCESS; + PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers); + + LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser=%d iUserTable=%x\n", GCPhys, iUser, iUserTable)); RT_NOREF_PV(GCPhys); + + if (iUser != NIL_PGMPOOL_IDX) + { +#ifdef VBOX_STRICT + /* + * Check that the entry doesn't already exists. + */ + if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX) + { + uint16_t i = pPage->iUserHead; + do + { + Assert(i < pPool->cMaxUsers); + AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable)); + i = paUsers[i].iNext; + } while (i != NIL_PGMPOOL_USER_INDEX); + } +#endif + + /* + * Find free a user node. + */ + uint16_t i = pPool->iUserFreeHead; + if (i == NIL_PGMPOOL_USER_INDEX) + { + rc = pgmPoolTrackFreeOneUser(pPool, iUser); + if (RT_FAILURE(rc)) + return rc; + i = pPool->iUserFreeHead; + } + + /* + * Unlink the user node from the free list, + * initialize and insert it into the user list. + */ + pPool->iUserFreeHead = paUsers[i].iNext; + paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX; + paUsers[i].iUser = iUser; + paUsers[i].iUserTable = iUserTable; + pPage->iUserHead = i; + } + else + pPage->iUserHead = NIL_PGMPOOL_USER_INDEX; + + + /* + * Insert into cache and enable monitoring of the guest page if enabled. + * + * Until we implement caching of all levels, including the CR3 one, we'll + * have to make sure we don't try monitor & cache any recursive reuse of + * a monitored CR3 page. Because all windows versions are doing this we'll + * have to be able to do combined access monitoring, CR3 + PT and + * PD + PT (guest PAE). + * + * Update: + * We're now cooperating with the CR3 monitor if an uncachable page is found. + */ + const bool fCanBeMonitored = true; + pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */ + if (fCanBeMonitored) + { + rc = pgmPoolMonitorInsert(pPool, pPage); + AssertRC(rc); + } + return rc; +} + + +/** + * Adds a user reference to a page. + * + * This will move the page to the head of the + * + * @returns VBox status code. + * @retval VINF_SUCCESS if successfully added. + * + * @param pPool The pool. + * @param pPage The cached page. + * @param iUser The user index. + * @param iUserTable The user table. + */ +static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable) +{ + Log3(("pgmPoolTrackAddUser: GCPhys=%RGp iUser=%x iUserTable=%x\n", pPage->GCPhys, iUser, iUserTable)); + PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers); + Assert(iUser != NIL_PGMPOOL_IDX); + +# ifdef VBOX_STRICT + /* + * Check that the entry doesn't already exists. We only allow multiple + * users of top-level paging structures (SHW_POOL_ROOT_IDX). + */ + if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX) + { + uint16_t i = pPage->iUserHead; + do + { + Assert(i < pPool->cMaxUsers); + /** @todo this assertion looks odd... Shouldn't it be && here? */ + AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable)); + i = paUsers[i].iNext; + } while (i != NIL_PGMPOOL_USER_INDEX); + } +# endif + + /* + * Allocate a user node. + */ + uint16_t i = pPool->iUserFreeHead; + if (i == NIL_PGMPOOL_USER_INDEX) + { + int rc = pgmPoolTrackFreeOneUser(pPool, iUser); + if (RT_FAILURE(rc)) + return rc; + i = pPool->iUserFreeHead; + } + pPool->iUserFreeHead = paUsers[i].iNext; + + /* + * Initialize the user node and insert it. + */ + paUsers[i].iNext = pPage->iUserHead; + paUsers[i].iUser = iUser; + paUsers[i].iUserTable = iUserTable; + pPage->iUserHead = i; + +# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + if (pPage->fDirty) + pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirtyEntry, false /* do not remove */); +# endif + + /* + * Tell the cache to update its replacement stats for this page. + */ + pgmPoolCacheUsed(pPool, pPage); + return VINF_SUCCESS; +} + + +/** + * Frees a user record associated with a page. + * + * This does not clear the entry in the user table, it simply replaces the + * user record to the chain of free records. + * + * @param pPool The pool. + * @param pPage The shadow page. + * @param iUser The shadow page pool index of the user table. + * @param iUserTable The index into the user table (shadowed). + * + * @remarks Don't call this for root pages. + */ +static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable) +{ + Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable)); + PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers); + Assert(iUser != NIL_PGMPOOL_IDX); + + /* + * Unlink and free the specified user entry. + */ + + /* Special: For PAE and 32-bit paging, there is usually no more than one user. */ + uint16_t i = pPage->iUserHead; + if ( i != NIL_PGMPOOL_USER_INDEX + && paUsers[i].iUser == iUser + && paUsers[i].iUserTable == iUserTable) + { + pPage->iUserHead = paUsers[i].iNext; + + paUsers[i].iUser = NIL_PGMPOOL_IDX; + paUsers[i].iNext = pPool->iUserFreeHead; + pPool->iUserFreeHead = i; + return; + } + + /* General: Linear search. */ + uint16_t iPrev = NIL_PGMPOOL_USER_INDEX; + while (i != NIL_PGMPOOL_USER_INDEX) + { + if ( paUsers[i].iUser == iUser + && paUsers[i].iUserTable == iUserTable) + { + if (iPrev != NIL_PGMPOOL_USER_INDEX) + paUsers[iPrev].iNext = paUsers[i].iNext; + else + pPage->iUserHead = paUsers[i].iNext; + + paUsers[i].iUser = NIL_PGMPOOL_IDX; + paUsers[i].iNext = pPool->iUserFreeHead; + pPool->iUserFreeHead = i; + return; + } + iPrev = i; + i = paUsers[i].iNext; + } + + /* Fatal: didn't find it */ + AssertFatalMsgFailed(("Didn't find the user entry! iUser=%d iUserTable=%#x GCPhys=%RGp\n", + iUser, iUserTable, pPage->GCPhys)); +} + + +#if 0 /* unused */ +/** + * Gets the entry size of a shadow table. + * + * @param enmKind The kind of page. + * + * @returns The size of the entry in bytes. That is, 4 or 8. + * @returns If the kind is not for a table, an assertion is raised and 0 is + * returned. + */ +DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind) +{ + switch (enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_32BIT_PD_PHYS: + return 4; + + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + case PGMPOOLKIND_64BIT_PML4: + case PGMPOOLKIND_PAE_PDPT: + case PGMPOOLKIND_ROOT_NESTED: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_PHYS: + return 8; + + default: + AssertFatalMsgFailed(("enmKind=%d\n", enmKind)); + } +} +#endif /* unused */ + +#if 0 /* unused */ +/** + * Gets the entry size of a guest table. + * + * @param enmKind The kind of page. + * + * @returns The size of the entry in bytes. That is, 0, 4 or 8. + * @returns If the kind is not for a table, an assertion is raised and 0 is + * returned. + */ +DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind) +{ + switch (enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + return 4; + + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + case PGMPOOLKIND_64BIT_PML4: + case PGMPOOLKIND_PAE_PDPT: + return 8; + + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + case PGMPOOLKIND_ROOT_NESTED: + case PGMPOOLKIND_PAE_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_PHYS: + case PGMPOOLKIND_32BIT_PD_PHYS: + /** @todo can we return 0? (nobody is calling this...) */ + AssertFailed(); + return 0; + + default: + AssertFatalMsgFailed(("enmKind=%d\n", enmKind)); + } +} +#endif /* unused */ + + +/** + * Checks one shadow page table entry for a mapping of a physical page. + * + * @returns true / false indicating removal of all relevant PTEs + * + * @param pVM The cross context VM structure. + * @param pPhysPage The guest page in question. + * @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change) + * @param iShw The shadow page table. + * @param iPte Page table entry or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown + */ +static bool pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw, uint16_t iPte) +{ + LogFlow(("pgmPoolTrackFlushGCPhysPTInt: pPhysPage=%RHp iShw=%d iPte=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw, iPte)); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + bool fRet = false; + + /* + * Assert sanity. + */ + Assert(iPte != NIL_PGMPOOL_PHYSEXT_IDX_PTE); + AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw)); + PPGMPOOLPAGE pPage = &pPool->aPages[iShw]; + + /* + * Then, clear the actual mappings to the page in the shadow PT. + */ + switch (pPage->enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + { + const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P; + PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage); + uint32_t u32AndMask = 0; + uint32_t u32OrMask = 0; + + if (!fFlushPTEs) + { + /* Note! Disregarding the PGMPHYSHANDLER_F_NOT_IN_HM bit here. Should be harmless. */ + switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage)) + { + case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */ + case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */ + u32OrMask = X86_PTE_RW; + u32AndMask = UINT32_MAX; + fRet = true; + STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep); + break; + + case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */ + u32OrMask = 0; + u32AndMask = ~X86_PTE_RW; + fRet = true; + STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep); + break; + default: + /* We will end up here when called with an "ALL" access handler. */ + STAM_COUNTER_INC(&pPool->StatTrackFlushEntry); + break; + } + } + else + STAM_COUNTER_INC(&pPool->StatTrackFlushEntry); + + /* Update the counter if we're removing references. */ + if (!u32AndMask) + { + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + } + + if ((pPT->a[iPte].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32) + { + Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32\n", iPte, pPT->a[iPte])); + X86PTE Pte; + Pte.u = (pPT->a[iPte].u & u32AndMask) | u32OrMask; + if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY) + Pte.u &= ~(X86PGUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */ + + ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT); + return fRet; + } +#ifdef LOG_ENABLED + Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent)); + for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++) + if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32) + { + Log(("i=%d cFound=%d\n", i, ++cFound)); + } +#endif + AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u32=%RX32 poolkind=%x\n", pPage->iFirstPresent, pPage->cPresent, u32, pPage->enmKind)); + /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);*/ + break; + } + + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: /* physical mask the same as PAE; RW bit as well; be careful! */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: +#endif + { + const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P; + PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage); + uint64_t u64OrMask = 0; + uint64_t u64AndMask = 0; + + if (!fFlushPTEs) + { + /* Note! Disregarding the PGMPHYSHANDLER_F_NOT_IN_HM bit here. Should be harmless. */ + switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage)) + { + case PGM_PAGE_HNDL_PHYS_STATE_NONE: /* No handler installed. */ + case PGM_PAGE_HNDL_PHYS_STATE_DISABLED: /* Monitoring is temporarily disabled. */ + u64OrMask = X86_PTE_RW; + u64AndMask = UINT64_MAX; + fRet = true; + STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep); + break; + + case PGM_PAGE_HNDL_PHYS_STATE_WRITE: /* Write access is monitored. */ + u64OrMask = 0; + u64AndMask = ~(uint64_t)X86_PTE_RW; + fRet = true; + STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep); + break; + + default: + /* We will end up here when called with an "ALL" access handler. */ + STAM_COUNTER_INC(&pPool->StatTrackFlushEntry); + break; + } + } + else + STAM_COUNTER_INC(&pPool->StatTrackFlushEntry); + + /* Update the counter if we're removing references. */ + if (!u64AndMask) + { + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + } + + if ((PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == u64) + { + Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64\n", iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte]))); + X86PTEPAE Pte; + Pte.u = (PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & u64AndMask) | u64OrMask; + if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY) + Pte.u &= ~(X86PGPAEUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */ + + PGMSHWPTEPAE_ATOMIC_SET(pPT->a[iPte], Pte.u); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT); + return fRet; + } +#ifdef LOG_ENABLED + Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent)); + Log(("Found %RX64 expected %RX64\n", PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX), u64)); + for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++) + if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == u64) + Log(("i=%d cFound=%d\n", i, ++cFound)); +#endif + AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u64=%RX64 poolkind=%x iPte=%d PT=%RX64\n", pPage->iFirstPresent, pPage->cPresent, u64, pPage->enmKind, iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte]))); + /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);*/ + break; + } + +#ifdef PGM_WITH_LARGE_PAGES + /* Large page case only. */ + case PGMPOOLKIND_EPT_PD_FOR_PHYS: +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB: /* X86_PDE4M_PS is same as leaf bit in EPT; be careful! */ +#endif + { + Assert(pVM->pgm.s.fNestedPaging); + + const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PDE4M_P | X86_PDE4M_PS; + PEPTPD pPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pPage); + + if ((pPD->a[iPte].u & (EPT_PDE2M_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64) + { + Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte])); + STAM_COUNTER_INC(&pPool->StatTrackFlushEntry); + pPD->a[iPte].u = 0; + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD); + + /* Update the counter as we're removing references. */ + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + + return fRet; + } +# ifdef LOG_ENABLED + Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent)); + for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++) + if ((pPD->a[i].u & (EPT_PDE2M_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64) + Log(("i=%d cFound=%d\n", i, ++cFound)); +# endif + AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent)); + /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);*/ + break; + } + + /* AMD-V nested paging */ /** @todo merge with EPT as we only check the parts that are identical. */ + case PGMPOOLKIND_PAE_PD_PHYS: + { + Assert(pVM->pgm.s.fNestedPaging); + + const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PDE4M_P | X86_PDE4M_PS; + PX86PDPAE pPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage); + + if ((pPD->a[iPte].u & (X86_PDE2M_PAE_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64) + { + Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte])); + STAM_COUNTER_INC(&pPool->StatTrackFlushEntry); + pPD->a[iPte].u = 0; + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD); + + /* Update the counter as we're removing references. */ + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + return fRet; + } +# ifdef LOG_ENABLED + Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent)); + for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++) + if ((pPD->a[i].u & (X86_PDE2M_PAE_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64) + Log(("i=%d cFound=%d\n", i, ++cFound)); +# endif + AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent)); + /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);*/ + break; + } +#endif /* PGM_WITH_LARGE_PAGES */ + + default: + AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw)); + } + + /* not reached. */ +#ifndef _MSC_VER + return fRet; +#endif +} + + +/** + * Scans one shadow page table for mappings of a physical page. + * + * @param pVM The cross context VM structure. + * @param pPhysPage The guest page in question. + * @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change) + * @param iShw The shadow page table. + */ +static void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw) +{ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool); + + /* We should only come here with when there's only one reference to this physical page. */ + Assert(PGMPOOL_TD_GET_CREFS(PGM_PAGE_GET_TRACKING(pPhysPage)) == 1); + + Log2(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%RHp iShw=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw)); + STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f); + bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, iShw, PGM_PAGE_GET_PTE_INDEX(pPhysPage)); + if (!fKeptPTEs) + PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0); + STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f); +} + + +/** + * Flushes a list of shadow page tables mapping the same physical page. + * + * @param pVM The cross context VM structure. + * @param pPhysPage The guest page in question. + * @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change) + * @param iPhysExt The physical cross reference extent list to flush. + */ +static void pgmPoolTrackFlushGCPhysPTs(PVMCC pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iPhysExt) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + bool fKeepList = false; + + STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f); + Log2(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%RHp iPhysExt=%u\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iPhysExt)); + + const uint16_t iPhysExtStart = iPhysExt; + PPGMPOOLPHYSEXT pPhysExt; + do + { + Assert(iPhysExt < pPool->cMaxPhysExts); + pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt]; + for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++) + { + if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX) + { + bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, pPhysExt->aidx[i], pPhysExt->apte[i]); + if (!fKeptPTEs) + { + pPhysExt->aidx[i] = NIL_PGMPOOL_IDX; + pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE; + } + else + fKeepList = true; + } + } + /* next */ + iPhysExt = pPhysExt->iNext; + } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX); + + if (!fKeepList) + { + /* insert the list into the free list and clear the ram range entry. */ + pPhysExt->iNext = pPool->iPhysExtFreeHead; + pPool->iPhysExtFreeHead = iPhysExtStart; + /* Invalidate the tracking data. */ + PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0); + } + + STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f); +} + + +/** + * Flushes all shadow page table mappings of the given guest page. + * + * This is typically called when the host page backing the guest one has been + * replaced or when the page protection was changed due to a guest access + * caught by the monitoring. + * + * @returns VBox status code. + * @retval VINF_SUCCESS if all references has been successfully cleared. + * @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page + * pool cleaning. FF and sync flags are set. + * + * @param pVM The cross context VM structure. + * @param GCPhysPage GC physical address of the page in question + * @param pPhysPage The guest page in question. + * @param fFlushPTEs Flush PTEs or allow them to be updated (e.g. in case of an RW bit change) + * @param pfFlushTLBs This is set to @a true if the shadow TLBs should be + * flushed, it is NOT touched if this isn't necessary. + * The caller MUST initialized this to @a false. + */ +int pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs) +{ + PVMCPUCC pVCpu = VMMGetCpu(pVM); + PGM_LOCK_VOID(pVM); + int rc = VINF_SUCCESS; + +#ifdef PGM_WITH_LARGE_PAGES + /* Is this page part of a large page? */ + if (PGM_PAGE_GET_PDE_TYPE(pPhysPage) == PGM_PAGE_PDE_TYPE_PDE) + { + RTGCPHYS GCPhysBase = GCPhysPage & X86_PDE2M_PAE_PG_MASK; + GCPhysPage &= X86_PDE_PAE_PG_MASK; + + /* Fetch the large page base. */ + PPGMPAGE pLargePage; + if (GCPhysBase != GCPhysPage) + { + pLargePage = pgmPhysGetPage(pVM, GCPhysBase); + AssertFatal(pLargePage); + } + else + pLargePage = pPhysPage; + + Log(("pgmPoolTrackUpdateGCPhys: update large page PDE for %RGp (%RGp)\n", GCPhysBase, GCPhysPage)); + + if (PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE) + { + /* Mark the large page as disabled as we need to break it up to change a single page in the 2 MB range. */ + PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE_DISABLED); + pVM->pgm.s.cLargePagesDisabled++; + + /* Update the base as that *only* that one has a reference and there's only one PDE to clear. */ + rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysBase, pLargePage, fFlushPTEs, pfFlushTLBs); + + *pfFlushTLBs = true; + PGM_UNLOCK(pVM); + return rc; + } + } +#else + NOREF(GCPhysPage); +#endif /* PGM_WITH_LARGE_PAGES */ + + const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage); + if (u16) + { + /* + * The zero page is currently screwing up the tracking and we'll + * have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC + * is defined, zero pages won't normally be mapped. Some kind of solution + * will be needed for this problem of course, but it will have to wait... + */ + if ( PGM_PAGE_IS_ZERO(pPhysPage) + || PGM_PAGE_IS_BALLOONED(pPhysPage)) + rc = VINF_PGM_GCPHYS_ALIASED; + else + { + if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT) + { + Assert(PGMPOOL_TD_GET_CREFS(u16) == 1); + pgmPoolTrackFlushGCPhysPT(pVM, + pPhysPage, + fFlushPTEs, + PGMPOOL_TD_GET_IDX(u16)); + } + else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED)) + pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, fFlushPTEs, PGMPOOL_TD_GET_IDX(u16)); + else + rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage); + *pfFlushTLBs = true; + } + } + + if (rc == VINF_PGM_GCPHYS_ALIASED) + { + pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL; + VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); + rc = VINF_PGM_SYNC_CR3; + } + PGM_UNLOCK(pVM); + return rc; +} + + +/** + * Scans all shadow page tables for mappings of a physical page. + * + * This may be slow, but it's most likely more efficient than cleaning + * out the entire page pool / cache. + * + * @returns VBox status code. + * @retval VINF_SUCCESS if all references has been successfully cleared. + * @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and + * a page pool cleaning. + * + * @param pVM The cross context VM structure. + * @param pPhysPage The guest page in question. + */ +int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage) +{ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s); + LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n", + pPool->cUsedPages, pPool->cPresent, pPhysPage)); + + /* + * There is a limit to what makes sense. + */ + if ( pPool->cPresent > 1024 + && pVM->cCpus == 1) + { + LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent)); + STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s); + return VINF_PGM_GCPHYS_ALIASED; + } + + /* + * Iterate all the pages until we've encountered all that in use. + * This is simple but not quite optimal solution. + */ + const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage); + unsigned cLeft = pPool->cUsedPages; + unsigned iPage = pPool->cCurPages; + while (--iPage >= PGMPOOL_IDX_FIRST) + { + PPGMPOOLPAGE pPage = &pPool->aPages[iPage]; + if ( pPage->GCPhys != NIL_RTGCPHYS + && pPage->cPresent) + { + Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* see if it hits */ + switch (pPage->enmKind) + { + /* + * We only care about shadow page tables. + */ + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + { + const uint32_t u32 = (uint32_t)u64; + unsigned cPresent = pPage->cPresent; + PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage); + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++) + { + const X86PGUINT uPte = pPT->a[i].u; + if (uPte & X86_PTE_P) + { + if ((uPte & X86_PTE_PG_MASK) == u32) + { + //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i])); + ASMAtomicWriteU32(&pPT->a[i].u, 0); + + /* Update the counter as we're removing references. */ + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + } + if (!--cPresent) + break; + } + } + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT); + break; + } + + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + { + unsigned cPresent = pPage->cPresent; + PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage); + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++) + if (PGMSHWPTEPAE_IS_P(pPT->a[i])) + { + if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & X86_PTE_PAE_PG_MASK) == u64) + { + //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i])); + PGMSHWPTEPAE_ATOMIC_SET(pPT->a[i], 0); /// @todo why not atomic? + + /* Update the counter as we're removing references. */ + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + } + if (!--cPresent) + break; + } + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT); + break; + } + + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + { + unsigned cPresent = pPage->cPresent; + PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage); + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++) + { + X86PGPAEUINT const uPte = pPT->a[i].u; + if (uPte & EPT_E_READ) + { + if ((uPte & EPT_PTE_PG_MASK) == u64) + { + //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i])); + ASMAtomicWriteU64(&pPT->a[i].u, 0); + + /* Update the counter as we're removing references. */ + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + } + if (!--cPresent) + break; + } + } + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT); + break; + } + } + + if (!--cLeft) + break; + } + } + + PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0); + STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s); + + /* + * There is a limit to what makes sense. The above search is very expensive, so force a pgm pool flush. + */ + if (pPool->cPresent > 1024) + { + LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent)); + return VINF_PGM_GCPHYS_ALIASED; + } + + return VINF_SUCCESS; +} + + +/** + * Clears the user entry in a user table. + * + * This is used to remove all references to a page when flushing it. + */ +static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser) +{ + Assert(pUser->iUser != NIL_PGMPOOL_IDX); + Assert(pUser->iUser < pPool->cCurPages); + uint32_t iUserTable = pUser->iUserTable; + + /* + * Map the user page. Ignore references made by fictitious pages. + */ + PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser]; + LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind))); + union + { + uint64_t *pau64; + uint32_t *pau32; + } u; + if (pUserPage->idx < PGMPOOL_IDX_FIRST) + { + Assert(!pUserPage->pvPageR3); + return; + } + u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage); + + + /* Safety precaution in case we change the paging for other modes too in the future. */ + Assert(!pgmPoolIsPageLocked(pPage)); RT_NOREF_PV(pPage); + +#ifdef VBOX_STRICT + /* + * Some sanity checks. + */ + switch (pUserPage->enmKind) + { + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_32BIT_PD_PHYS: + Assert(iUserTable < X86_PG_ENTRIES); + break; + case PGMPOOLKIND_PAE_PDPT: + case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: + case PGMPOOLKIND_PAE_PDPT_PHYS: + Assert(iUserTable < 4); + Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT)); + break; + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_PAE_PD_PHYS: + Assert(iUserTable < X86_PG_PAE_ENTRIES); + break; + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + Assert(iUserTable < X86_PG_PAE_ENTRIES); + break; + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + Assert(iUserTable < X86_PG_PAE_ENTRIES); + Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT)); + break; + case PGMPOOLKIND_64BIT_PML4: + Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT)); + /* GCPhys >> PAGE_SHIFT is the index here */ + break; + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + Assert(iUserTable < X86_PG_PAE_ENTRIES); + break; + + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + Assert(iUserTable < X86_PG_PAE_ENTRIES); + break; + + case PGMPOOLKIND_ROOT_NESTED: + Assert(iUserTable < X86_PG_PAE_ENTRIES); + break; + +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: + case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB: + case PGMPOOLKIND_EPT_PD_FOR_EPT_PD: + case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT: + case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4: + Assert(iUserTable < EPT_PG_ENTRIES); + break; +# endif + + default: + AssertMsgFailed(("enmKind=%d GCPhys=%RGp\n", pUserPage->enmKind, pPage->GCPhys)); + break; + } +#endif /* VBOX_STRICT */ + + /* + * Clear the entry in the user page. + */ + switch (pUserPage->enmKind) + { + /* 32-bit entries */ + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_32BIT_PD_PHYS: + ASMAtomicWriteU32(&u.pau32[iUserTable], 0); + break; + + /* 64-bit entries */ + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_PAE_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + case PGMPOOLKIND_64BIT_PML4: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_PAE_PDPT: + case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: + case PGMPOOLKIND_ROOT_NESTED: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: + case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB: + case PGMPOOLKIND_EPT_PD_FOR_EPT_PD: + case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT: + case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4: +#endif + ASMAtomicWriteU64(&u.pau64[iUserTable], 0); + break; + + default: + AssertFatalMsgFailed(("enmKind=%d iUser=%d iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable)); + } + PGM_DYNMAP_UNUSED_HINT_VM(pPool->CTX_SUFF(pVM), u.pau64); +} + + +/** + * Clears all users of a page. + */ +static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + /* + * Free all the user records. + */ + LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys)); + + PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers); + uint16_t i = pPage->iUserHead; + while (i != NIL_PGMPOOL_USER_INDEX) + { + /* Clear enter in user table. */ + pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]); + + /* Free it. */ + const uint16_t iNext = paUsers[i].iNext; + paUsers[i].iUser = NIL_PGMPOOL_IDX; + paUsers[i].iNext = pPool->iUserFreeHead; + pPool->iUserFreeHead = i; + + /* Next. */ + i = iNext; + } + pPage->iUserHead = NIL_PGMPOOL_USER_INDEX; +} + + +/** + * Allocates a new physical cross reference extent. + * + * @returns Pointer to the allocated extent on success. NULL if we're out of them. + * @param pVM The cross context VM structure. + * @param piPhysExt Where to store the phys ext index. + */ +PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + uint16_t iPhysExt = pPool->iPhysExtFreeHead; + if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX) + { + STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures); + return NULL; + } + PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt]; + pPool->iPhysExtFreeHead = pPhysExt->iNext; + pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX; + *piPhysExt = iPhysExt; + return pPhysExt; +} + + +/** + * Frees a physical cross reference extent. + * + * @param pVM The cross context VM structure. + * @param iPhysExt The extent to free. + */ +void pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + Assert(iPhysExt < pPool->cMaxPhysExts); + PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt]; + for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++) + { + pPhysExt->aidx[i] = NIL_PGMPOOL_IDX; + pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE; + } + pPhysExt->iNext = pPool->iPhysExtFreeHead; + pPool->iPhysExtFreeHead = iPhysExt; +} + + +/** + * Frees a physical cross reference extent. + * + * @param pVM The cross context VM structure. + * @param iPhysExt The extent to free. + */ +void pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + + const uint16_t iPhysExtStart = iPhysExt; + PPGMPOOLPHYSEXT pPhysExt; + do + { + Assert(iPhysExt < pPool->cMaxPhysExts); + pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt]; + for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++) + { + pPhysExt->aidx[i] = NIL_PGMPOOL_IDX; + pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE; + } + + /* next */ + iPhysExt = pPhysExt->iNext; + } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX); + + pPhysExt->iNext = pPool->iPhysExtFreeHead; + pPool->iPhysExtFreeHead = iPhysExtStart; +} + + +/** + * Insert a reference into a list of physical cross reference extents. + * + * @returns The new tracking data for PGMPAGE. + * + * @param pVM The cross context VM structure. + * @param iPhysExt The physical extent index of the list head. + * @param iShwPT The shadow page table index. + * @param iPte Page table entry + * + */ +static uint16_t pgmPoolTrackPhysExtInsert(PVMCC pVM, uint16_t iPhysExt, uint16_t iShwPT, uint16_t iPte) +{ + PGM_LOCK_ASSERT_OWNER(pVM); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts); + + /* + * Special common cases. + */ + if (paPhysExts[iPhysExt].aidx[1] == NIL_PGMPOOL_IDX) + { + paPhysExts[iPhysExt].aidx[1] = iShwPT; + paPhysExts[iPhysExt].apte[1] = iPte; + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany); + LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,%d pte %d,}\n", iPhysExt, iShwPT, iPte)); + return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt); + } + if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX) + { + paPhysExts[iPhysExt].aidx[2] = iShwPT; + paPhysExts[iPhysExt].apte[2] = iPte; + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany); + LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d pte %d}\n", iPhysExt, iShwPT, iPte)); + return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt); + } + AssertCompile(RT_ELEMENTS(paPhysExts[iPhysExt].aidx) == 3); + + /* + * General treatment. + */ + const uint16_t iPhysExtStart = iPhysExt; + unsigned cMax = 15; + for (;;) + { + Assert(iPhysExt < pPool->cMaxPhysExts); + for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++) + if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX) + { + paPhysExts[iPhysExt].aidx[i] = iShwPT; + paPhysExts[iPhysExt].apte[i] = iPte; + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany); + LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d pte %d} i=%d cMax=%d\n", iPhysExt, iShwPT, iPte, i, cMax)); + return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart); + } + if (!--cMax) + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackOverflows); + pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart); + LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT)); + return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED); + } + + /* advance */ + iPhysExt = paPhysExts[iPhysExt].iNext; + if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX) + break; + } + + /* + * Add another extent to the list. + */ + PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt); + if (!pNew) + { + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackNoExtentsLeft); + pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart); + LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT)); + return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED); + } + pNew->iNext = iPhysExtStart; + pNew->aidx[0] = iShwPT; + pNew->apte[0] = iPte; + LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d pte %d}->%d\n", iPhysExt, iShwPT, iPte, iPhysExtStart)); + return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt); +} + + +/** + * Add a reference to guest physical page where extents are in use. + * + * @returns The new tracking data for PGMPAGE. + * + * @param pVM The cross context VM structure. + * @param pPhysPage Pointer to the aPages entry in the ram range. + * @param u16 The ram range flags (top 16-bits). + * @param iShwPT The shadow page table index. + * @param iPte Page table entry + */ +uint16_t pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte) +{ + PGM_LOCK_VOID(pVM); + if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT) + { + /* + * Convert to extent list. + */ + Assert(PGMPOOL_TD_GET_CREFS(u16) == 1); + uint16_t iPhysExt; + PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt); + if (pPhysExt) + { + LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT)); + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliased); + pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16); + pPhysExt->apte[0] = PGM_PAGE_GET_PTE_INDEX(pPhysPage); + pPhysExt->aidx[1] = iShwPT; + pPhysExt->apte[1] = iPte; + u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt); + } + else + u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED); + } + else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED)) + { + /* + * Insert into the extent list. + */ + u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT, iPte); + } + else + STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedLots); + PGM_UNLOCK(pVM); + return u16; +} + + +/** + * Clear references to guest physical memory. + * + * @param pPool The pool. + * @param pPage The page. + * @param pPhysPage Pointer to the aPages entry in the ram range. + * @param iPte Shadow PTE index + */ +void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage, uint16_t iPte) +{ + PVMCC pVM = pPool->CTX_SUFF(pVM); + const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage); + AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx)); + + uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage); + if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED) + { + PGM_LOCK_VOID(pVM); + + uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX; + PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts); + do + { + Assert(iPhysExt < pPool->cMaxPhysExts); + + /* + * Look for the shadow page and check if it's all freed. + */ + for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++) + { + if ( paPhysExts[iPhysExt].aidx[i] == pPage->idx + && paPhysExts[iPhysExt].apte[i] == iPte) + { + paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX; + paPhysExts[iPhysExt].apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE; + + for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++) + if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX) + { + Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx)); + PGM_UNLOCK(pVM); + return; + } + + /* we can free the node. */ + const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext; + if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX + && iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX) + { + /* lonely node */ + pgmPoolTrackPhysExtFree(pVM, iPhysExt); + Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx)); + PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0); + } + else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX) + { + /* head */ + Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx)); + PGM_PAGE_SET_TRACKING(pVM, pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext)); + pgmPoolTrackPhysExtFree(pVM, iPhysExt); + } + else + { + /* in list */ + Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d in list\n", pPhysPage, pPage->idx)); + paPhysExts[iPhysExtPrev].iNext = iPhysExtNext; + pgmPoolTrackPhysExtFree(pVM, iPhysExt); + } + iPhysExt = iPhysExtNext; + PGM_UNLOCK(pVM); + return; + } + } + + /* next */ + iPhysExtPrev = iPhysExt; + iPhysExt = paPhysExts[iPhysExt].iNext; + } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX); + + PGM_UNLOCK(pVM); + AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx)); + } + else /* nothing to do */ + Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage)); +} + +/** + * Clear references to guest physical memory. + * + * This is the same as pgmPoolTracDerefGCPhysHint except that the guest + * physical address is assumed to be correct, so the linear search can be + * skipped and we can assert at an earlier point. + * + * @param pPool The pool. + * @param pPage The page. + * @param HCPhys The host physical address corresponding to the guest page. + * @param GCPhys The guest physical address corresponding to HCPhys. + * @param iPte Shadow PTE index + */ +static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys, uint16_t iPte) +{ + /* + * Lookup the page and check if it checks out before derefing it. + */ + PVMCC pVM = pPool->CTX_SUFF(pVM); + PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhys); + if (pPhysPage) + { + Assert(PGM_PAGE_GET_HCPHYS(pPhysPage)); +#ifdef LOG_ENABLED + RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage); + Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys)); +#endif + if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys) + { + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte); + return; + } + + AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp iPte=%u fIsNested=%RTbool\n", + HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage), iPte, PGMPOOL_PAGE_IS_NESTED(pPage))); + } + AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys)); +} + + +/** + * Clear references to guest physical memory. + * + * @param pPool The pool. + * @param pPage The page. + * @param HCPhys The host physical address corresponding to the guest page. + * @param GCPhysHint The guest physical address which may corresponding to HCPhys. + * @param iPte Shadow pte index + */ +void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte) +{ + Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint)); + + /* + * Try the hint first. + */ + RTHCPHYS HCPhysHinted; + PVMCC pVM = pPool->CTX_SUFF(pVM); + PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysHint); + if (pPhysPage) + { + HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage); + Assert(HCPhysHinted); + if (HCPhysHinted == HCPhys) + { + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte); + return; + } + } + else + HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef); + + /* + * Damn, the hint didn't work. We'll have to do an expensive linear search. + */ + STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches); + PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX); + while (pRam) + { + unsigned iPage = pRam->cb >> PAGE_SHIFT; + while (iPage-- > 0) + { + if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys) + { + Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n", + HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT))); + Assert(pPage->cPresent); + Assert(pPool->cPresent); + pPage->cPresent--; + pPool->cPresent--; + pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte); + return; + } + } + pRam = pRam->CTX_SUFF(pNext); + } + + AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted)); +} + + +/** + * Clear references to guest physical memory in a 32-bit / 32-bit page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + * @param pGstPT The guest page table. + */ +DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT) +{ + RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20); + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++) + { + const X86PGUINT uPte = pShwPT->a[i].u; + Assert(!(uPte & RT_BIT_32(10))); + if (uPte & X86_PTE_P) + { + Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n", + i, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & fPgMask, i); + if (!pPage->cPresent) + break; + } + } +} + + +/** + * Clear references to guest physical memory in a PAE / 32-bit page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + * @param pGstPT The guest page table (just a half one). + */ +DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT) +{ + RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20); + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++) + { + Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0 + || (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000)); + if (PGMSHWPTEPAE_IS_P(pShwPT->a[i])) + { + Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n", + i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PG_MASK)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i); + if (!pPage->cPresent) + break; + } + } +} + + +/** + * Clear references to guest physical memory in a PAE / PAE page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + * @param pGstPT The guest page table. + */ +DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT) +{ + RTGCPHYS const fPgMask = pPage->fA20Enabled ? X86_PTE_PAE_PG_MASK : X86_PTE_PAE_PG_MASK & ~RT_BIT_64(20); + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++) + { + Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0 + || (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000)); + if (PGMSHWPTEPAE_IS_P(pShwPT->a[i])) + { + Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n", + i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK)); + pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i); + if (!pPage->cPresent) + break; + } + } +} + + +/** + * Clear references to guest physical memory in a 32-bit / 4MB page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT) +{ + RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20); + RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent; + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE) + { + const X86PGUINT uPte = pShwPT->a[i].u; + Assert(!(uPte & RT_BIT_32(10))); + if (uPte & X86_PTE_P) + { + Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n", + i, uPte & X86_PTE_PG_MASK, GCPhys)); + pgmPoolTracDerefGCPhys(pPool, pPage, uPte & X86_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i); + if (!pPage->cPresent) + break; + } + } +} + + +/** + * Clear references to guest physical memory in a PAE / 2/4MB page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT) +{ + RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20); + RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent; + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE) + { + Assert( (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0 + || (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000)); + if (PGMSHWPTEPAE_IS_P(pShwPT->a[i])) + { + Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n", + i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys)); + pgmPoolTracDerefGCPhys(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys & GCPhysA20Mask, i); + if (!pPage->cPresent) + break; + } + } +} + + +/** + * Clear references to shadowed pages in an EPT page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page directory pointer table (mapping of the + * page). + */ +DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT) +{ + RTGCPHYS const GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20); + RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent; + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE) + { + X86PGPAEUINT const uPte = pShwPT->a[i].u; + Assert((uPte & UINT64_C(0xfff0000000000f80)) == 0); + if (uPte & EPT_E_READ) + { + Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n", + i, uPte & EPT_PTE_PG_MASK, pPage->GCPhys)); + pgmPoolTracDerefGCPhys(pPool, pPage, uPte & EPT_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i); + if (!pPage->cPresent) + break; + } + } +} + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + +/** + * Clears references to shadowed pages in a SLAT EPT page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + * @param pGstPT The guest page table. + */ +DECLINLINE(void) pgmPoolTrackDerefNestedPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT, PCEPTPT pGstPT) +{ + Assert(PGMPOOL_PAGE_IS_NESTED(pPage)); + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++) + { + X86PGPAEUINT const uShwPte = pShwPT->a[i].u; + Assert((uShwPte & UINT64_C(0xfff0000000000f80)) == 0); /* Access, Dirty, UserX (not supported) and ignored bits 7, 11. */ + if (uShwPte & EPT_PRESENT_MASK) + { + Log7Func(("Shw=%RX64 GstPte=%RX64\n", uShwPte, pGstPT->a[i].u)); + pgmPoolTracDerefGCPhys(pPool, pPage, uShwPte & EPT_PTE_PG_MASK, pGstPT->a[i].u & EPT_PTE_PG_MASK, i); + if (!pPage->cPresent) + break; + } + } +} + + +/** + * Clear references to guest physical memory in a SLAT 2MB EPT page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPT The shadow page table (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefNestedPTEPT2MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT) +{ + Assert(pPage->fA20Enabled); + RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent; + for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE) + { + X86PGPAEUINT const uShwPte = pShwPT->a[i].u; + Assert((uShwPte & UINT64_C(0xfff0000000000f80)) == 0); /* Access, Dirty, UserX (not supported) and ignored bits 7, 11. */ + if (uShwPte & EPT_PRESENT_MASK) + { + Log7Func(("Shw=%RX64 GstPte=%RX64\n", uShwPte, GCPhys)); + pgmPoolTracDerefGCPhys(pPool, pPage, uShwPte & EPT_PTE_PG_MASK, GCPhys, i); + if (!pPage->cPresent) + break; + } + } +} + + +/** + * Clear references to shadowed pages in a SLAT EPT page directory. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPD The shadow page directory (mapping of the page). + * @param pGstPD The guest page directory. + */ +DECLINLINE(void) pgmPoolTrackDerefNestedPDEpt(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD, PCEPTPD pGstPD) +{ + for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++) + { + X86PGPAEUINT const uPde = pShwPD->a[i].u; +#ifdef PGM_WITH_LARGE_PAGES + AssertMsg((uPde & UINT64_C(0xfff0000000000f00)) == 0, ("uPde=%RX64\n", uPde)); +#else + AssertMsg((uPde & UINT64_C(0xfff0000000000f80)) == 0, ("uPde=%RX64\n", uPde)); +#endif + if (uPde & EPT_PRESENT_MASK) + { +#ifdef PGM_WITH_LARGE_PAGES + if (uPde & EPT_E_LEAF) + { + Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n", i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys)); + pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK, pGstPD->a[i].u & EPT_PDE2M_PG_MASK, i); + } + else +#endif + { + PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & EPT_PDE_PG_MASK); + if (pSubPage) + pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i); + else + AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK)); + } + } + } +} + +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ + + +/** + * Clear references to shadowed pages in a 32 bits page directory. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPD The shadow page directory (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD) +{ + for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++) + { + X86PGUINT const uPde = pShwPD->a[i].u; + if (uPde & X86_PDE_P) + { + PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK); + if (pSubPage) + pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i); + else + AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK)); + } + } +} + + +/** + * Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPD The shadow page directory (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD) +{ + for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++) + { + X86PGPAEUINT const uPde = pShwPD->a[i].u; + if (uPde & X86_PDE_P) + { +#ifdef PGM_WITH_LARGE_PAGES + if (uPde & X86_PDE_PS) + { + Log4(("pgmPoolTrackDerefPDPae: i=%d pde=%RX64 GCPhys=%RX64\n", + i, uPde & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys)); + pgmPoolTracDerefGCPhys(pPool, pPage, uPde & X86_PDE2M_PAE_PG_MASK, + pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */, + i); + } + else +#endif + { + Assert((uPde & (X86_PDE_PAE_MBZ_MASK_NX | UINT64_C(0x7ff0000000000000))) == 0); + PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & X86_PDE_PAE_PG_MASK); + if (pSubPage) + pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i); + else + AssertFatalMsgFailed(("%RX64\n", uPde & X86_PDE_PAE_PG_MASK)); + /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */ + } + } + } +} + + +/** + * Clear references to shadowed pages in a PAE page directory pointer table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPDPT The shadow page directory pointer table (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT) +{ + for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++) + { + X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u; + Assert((uPdpe & (X86_PDPE_PAE_MBZ_MASK | UINT64_C(0x7ff0000000000200))) == 0); + if (uPdpe & X86_PDPE_P) + { + PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK); + if (pSubPage) + pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i); + else + AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK)); + } + } +} + + +/** + * Clear references to shadowed pages in a 64-bit page directory pointer table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPDPT The shadow page directory pointer table (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT) +{ + for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++) + { + X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u; + Assert((uPdpe & (X86_PDPE_LM_MBZ_MASK_NX | UINT64_C(0x7ff0000000000200))) == 0); + if (uPdpe & X86_PDPE_P) + { + PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK); + if (pSubPage) + pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i); + else + AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK)); + /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */ + } + } +} + + +/** + * Clear references to shadowed pages in a 64-bit level 4 page table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPML4 The shadow page directory pointer table (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4) +{ + for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++) + { + X86PGPAEUINT const uPml4e = pShwPML4->a[i].u; + Assert((uPml4e & (X86_PML4E_MBZ_MASK_NX | UINT64_C(0x7ff0000000000200))) == 0); + if (uPml4e & X86_PML4E_P) + { + PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPml4e & X86_PDPE_PG_MASK); + if (pSubPage) + pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i); + else + AssertFatalMsgFailed(("%RX64\n", uPml4e & X86_PML4E_PG_MASK)); + /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */ + } + } +} + + +/** + * Clear references to shadowed pages in an EPT page directory. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPD The shadow page directory (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD) +{ + for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++) + { + X86PGPAEUINT const uPde = pShwPD->a[i].u; +#ifdef PGM_WITH_LARGE_PAGES + AssertMsg((uPde & UINT64_C(0xfff0000000000f00)) == 0, ("uPde=%RX64\n", uPde)); +#else + AssertMsg((uPde & UINT64_C(0xfff0000000000f80)) == 0, ("uPde=%RX64\n", uPde)); +#endif + if (uPde & EPT_E_READ) + { +#ifdef PGM_WITH_LARGE_PAGES + if (uPde & EPT_E_LEAF) + { + Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n", + i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys)); + pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK, + pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */, + i); + } + else +#endif + { + PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & EPT_PDE_PG_MASK); + if (pSubPage) + pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i); + else + AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK)); + } + /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */ + } + } +} + + +/** + * Clear references to shadowed pages in an EPT page directory pointer table. + * + * @param pPool The pool. + * @param pPage The page. + * @param pShwPDPT The shadow page directory pointer table (mapping of the page). + */ +DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT) +{ + for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++) + { + X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u; + Assert((uPdpe & UINT64_C(0xfff0000000000f80)) == 0); + if (uPdpe & EPT_E_READ) + { + PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & EPT_PDPTE_PG_MASK); + if (pSubPage) + pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i); + else + AssertFatalMsgFailed(("%RX64\n", uPdpe & EPT_PDPTE_PG_MASK)); + /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */ + } + } +} + + +/** + * Clears all references made by this page. + * + * This includes other shadow pages and GC physical addresses. + * + * @param pPool The pool. + * @param pPage The page. + */ +static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage) +{ + /* + * Map the shadow page and take action according to the page kind. + */ + PVMCC pVM = pPool->CTX_SUFF(pVM); + void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage); + switch (pPage->enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + { + STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g); + void *pvGst; + int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc); + pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst); + STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g); + break; + } + + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + { + STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g); + void *pvGst; + int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc); + pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst); + STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g); + break; + } + + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + { + STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g); + void *pvGst; + int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc); + pgmPoolTrackDerefPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst); + STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g); + break; + } + + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */ + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + { + STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g); + pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw); + STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g); + break; + } + + case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */ + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + { + STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g); + pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PPGMSHWPTPAE)pvShw); + STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g); + break; + } + + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_PAE_PD_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw); + break; + + case PGMPOOLKIND_32BIT_PD_PHYS: + case PGMPOOLKIND_32BIT_PD: + pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw); + break; + + case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: + case PGMPOOLKIND_PAE_PDPT: + case PGMPOOLKIND_PAE_PDPT_PHYS: + pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw); + break; + + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw); + break; + + case PGMPOOLKIND_64BIT_PML4: + pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw); + break; + + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw); + break; + + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw); + break; + + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw); + break; + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: + { + void *pvGst; + int const rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc); + pgmPoolTrackDerefNestedPTEPT(pPool, pPage, (PEPTPT)pvShw, (PCEPTPT)pvGst); + break; + } + + case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB: + pgmPoolTrackDerefNestedPTEPT2MB(pPool, pPage, (PEPTPT)pvShw); + break; + + case PGMPOOLKIND_EPT_PD_FOR_EPT_PD: + { + void *pvGst; + int const rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc); + pgmPoolTrackDerefNestedPDEpt(pPool, pPage, (PEPTPD)pvShw, (PCEPTPD)pvGst); + break; + } + + case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT: + pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw); + break; +#endif + + default: + AssertFatalMsgFailed(("enmKind=%d GCPhys=%RGp\n", pPage->enmKind, pPage->GCPhys)); + } + + /* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */ + STAM_PROFILE_START(&pPool->StatZeroPage, z); + ASMMemZeroPage(pvShw); + STAM_PROFILE_STOP(&pPool->StatZeroPage, z); + pPage->fZeroed = true; + Assert(!pPage->cPresent); + PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw); +} + + +/** + * Flushes a pool page. + * + * This moves the page to the free list after removing all user references to it. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * @param pPool The pool. + * @param pPage The shadow page. + * @param fFlush Flush the TLBS when required (should only be false in very specific use cases!!) + */ +int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush) +{ + PVMCC pVM = pPool->CTX_SUFF(pVM); + bool fFlushRequired = false; + + int rc = VINF_SUCCESS; + STAM_PROFILE_START(&pPool->StatFlushPage, f); + LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n", + pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys)); + + if (PGMPOOL_PAGE_IS_NESTED(pPage)) + Log7Func(("pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n", + pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys)); + + /* + * Reject any attempts at flushing any of the special root pages (shall + * not happen). + */ + AssertMsgReturn(pPage->idx >= PGMPOOL_IDX_FIRST, + ("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n", + pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx), + VINF_SUCCESS); + + PGM_LOCK_VOID(pVM); + + /* + * Quietly reject any attempts at flushing the currently active shadow CR3 mapping + */ + if (pgmPoolIsPageLocked(pPage)) + { + AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4 + || pPage->enmKind == PGMPOOLKIND_PAE_PDPT + || pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT + || pPage->enmKind == PGMPOOLKIND_32BIT_PD + || pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD + || pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + || pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD + || pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD + || pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD + || pPage->enmKind == PGMPOOLKIND_ROOT_NESTED, + ("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind)); + Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx)); + PGM_UNLOCK(pVM); + return VINF_SUCCESS; + } + + /* + * Mark the page as being in need of an ASMMemZeroPage(). + */ + pPage->fZeroed = false; + +#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + if (pPage->fDirty) + pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirtyEntry, false /* do not remove */); +#endif + + /* If there are any users of this table, then we *must* issue a tlb flush on all VCPUs. */ + if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX) + fFlushRequired = true; + + /* + * Clear the page. + */ + pgmPoolTrackClearPageUsers(pPool, pPage); + STAM_PROFILE_START(&pPool->StatTrackDeref,a); + pgmPoolTrackDeref(pPool, pPage); + STAM_PROFILE_STOP(&pPool->StatTrackDeref,a); + + /* + * Flush it from the cache. + */ + pgmPoolCacheFlushPage(pPool, pPage); + + /* + * Deregistering the monitoring. + */ + if (pPage->fMonitored) + rc = pgmPoolMonitorFlush(pPool, pPage); + + /* + * Free the page. + */ + Assert(pPage->iNext == NIL_PGMPOOL_IDX); + pPage->iNext = pPool->iFreeHead; + pPool->iFreeHead = pPage->idx; + pPage->enmKind = PGMPOOLKIND_FREE; + pPage->enmAccess = PGMPOOLACCESS_DONTCARE; + pPage->GCPhys = NIL_RTGCPHYS; + pPage->fReusedFlushPending = false; + + pPool->cUsedPages--; + + /* Flush the TLBs of all VCPUs if required. */ + if ( fFlushRequired + && fFlush) + { + PGM_INVL_ALL_VCPU_TLBS(pVM); + } + + PGM_UNLOCK(pVM); + STAM_PROFILE_STOP(&pPool->StatFlushPage, f); + return rc; +} + + +/** + * Frees a usage of a pool page. + * + * The caller is responsible to updating the user table so that it no longer + * references the shadow page. + * + * @param pPool The pool. + * @param pPage The shadow page. + * @param iUser The shadow page pool index of the user table. + * NIL_PGMPOOL_IDX for root pages. + * @param iUserTable The index into the user table (shadowed). Ignored if + * root page. + */ +void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable) +{ + PVMCC pVM = pPool->CTX_SUFF(pVM); + + STAM_PROFILE_START(&pPool->StatFree, a); + LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%d iUserTable=%#x\n", + pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable)); + AssertReturnVoid(pPage->idx >= PGMPOOL_IDX_FIRST); /* paranoia (#6349) */ + + PGM_LOCK_VOID(pVM); + if (iUser != NIL_PGMPOOL_IDX) + pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable); + if (!pPage->fCached) + pgmPoolFlushPage(pPool, pPage); + PGM_UNLOCK(pVM); + STAM_PROFILE_STOP(&pPool->StatFree, a); +} + + +/** + * Makes one or more free page free. + * + * @returns VBox status code. + * @retval VINF_SUCCESS on success. + * + * @param pPool The pool. + * @param enmKind Page table kind + * @param iUser The user of the page. + */ +static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser) +{ + PVMCC pVM = pPool->CTX_SUFF(pVM); + LogFlow(("pgmPoolMakeMoreFreePages: enmKind=%d iUser=%d\n", enmKind, iUser)); + NOREF(enmKind); + + /* + * If the pool isn't full grown yet, expand it. + */ + if (pPool->cCurPages < pPool->cMaxPages) + { + STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a); +#ifdef IN_RING3 + int rc = PGMR3PoolGrow(pVM, VMMGetCpu(pVM)); +#else + int rc = PGMR0PoolGrow(pVM, VMMGetCpuId(pVM)); +#endif + if (RT_FAILURE(rc)) + return rc; + STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a); + if (pPool->iFreeHead != NIL_PGMPOOL_IDX) + return VINF_SUCCESS; + } + + /* + * Free one cached page. + */ + return pgmPoolCacheFreeOne(pPool, iUser); +} + + +/** + * Allocates a page from the pool. + * + * This page may actually be a cached page and not in need of any processing + * on the callers part. + * + * @returns VBox status code. + * @retval VINF_SUCCESS if a NEW page was allocated. + * @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned. + * + * @param pVM The cross context VM structure. + * @param GCPhys The GC physical address of the page we're gonna shadow. + * For 4MB and 2MB PD entries, it's the first address the + * shadow PT is covering. + * @param enmKind The kind of mapping. + * @param enmAccess Access type for the mapping (only relevant for big pages) + * @param fA20Enabled Whether the A20 gate is enabled or not. + * @param iUser The shadow page pool index of the user table. Root + * pages should pass NIL_PGMPOOL_IDX. + * @param iUserTable The index into the user table (shadowed). Ignored for + * root pages (iUser == NIL_PGMPOOL_IDX). + * @param fLockPage Lock the page + * @param ppPage Where to store the pointer to the page. NULL is stored here on failure. + */ +int pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled, + uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage) +{ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + STAM_PROFILE_ADV_START(&pPool->StatAlloc, a); + LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%d iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable)); + *ppPage = NULL; + /** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates + * (TRPMR3SyncIDT) because of FF priority. Try fix that? + * Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */ + +#if defined(VBOX_STRICT) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) + PVMCPUCC pVCpu = VMMGetCpu(pVM); + Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT || PGMPOOL_PAGE_IS_KIND_NESTED(enmKind)); +#endif + + PGM_LOCK_VOID(pVM); + + if (pPool->fCacheEnabled) + { + int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, fA20Enabled, iUser, iUserTable, ppPage); + if (RT_SUCCESS(rc2)) + { + if (fLockPage) + pgmPoolLockPage(pPool, *ppPage); + PGM_UNLOCK(pVM); + STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a); + LogFlow(("pgmPoolAlloc: cached returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, *ppPage, (*ppPage)->Core.Key, (*ppPage)->idx)); + return rc2; + } + } + + /* + * Allocate a new one. + */ + int rc = VINF_SUCCESS; + uint16_t iNew = pPool->iFreeHead; + if (iNew == NIL_PGMPOOL_IDX) + { + rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser); + if (RT_FAILURE(rc)) + { + PGM_UNLOCK(pVM); + Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc)); + STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a); + return rc; + } + iNew = pPool->iFreeHead; + AssertReleaseMsgReturn(iNew != NIL_PGMPOOL_IDX, ("iNew=%#x\n", iNew), VERR_PGM_POOL_IPE); + } + + /* unlink the free head */ + PPGMPOOLPAGE pPage = &pPool->aPages[iNew]; + pPool->iFreeHead = pPage->iNext; + pPage->iNext = NIL_PGMPOOL_IDX; + + /* + * Initialize it. + */ + pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */ + pPage->enmKind = enmKind; + pPage->enmAccess = enmAccess; + pPage->GCPhys = GCPhys; + pPage->fA20Enabled = fA20Enabled; + pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */ + pPage->fMonitored = false; + pPage->fCached = false; + pPage->fDirty = false; + pPage->fReusedFlushPending = false; + pPage->cModifications = 0; + pPage->iModifiedNext = NIL_PGMPOOL_IDX; + pPage->iModifiedPrev = NIL_PGMPOOL_IDX; + pPage->cPresent = 0; + pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX; + pPage->idxDirtyEntry = 0; + pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR; + pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR; + pPage->cLastAccessHandler = 0; + pPage->cLocked = 0; +# ifdef VBOX_STRICT + pPage->GCPtrDirtyFault = NIL_RTGCPTR; +# endif + + /* + * Insert into the tracking and cache. If this fails, free the page. + */ + int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable); + if (RT_FAILURE(rc3)) + { + pPool->cUsedPages--; + pPage->enmKind = PGMPOOLKIND_FREE; + pPage->enmAccess = PGMPOOLACCESS_DONTCARE; + pPage->GCPhys = NIL_RTGCPHYS; + pPage->iNext = pPool->iFreeHead; + pPool->iFreeHead = pPage->idx; + PGM_UNLOCK(pVM); + STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a); + Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3)); + return rc3; + } + + /* + * Commit the allocation, clear the page and return. + */ +#ifdef VBOX_WITH_STATISTICS + if (pPool->cUsedPages > pPool->cUsedPagesHigh) + pPool->cUsedPagesHigh = pPool->cUsedPages; +#endif + + if (!pPage->fZeroed) + { + STAM_PROFILE_START(&pPool->StatZeroPage, z); + void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage); + ASMMemZeroPage(pv); + STAM_PROFILE_STOP(&pPool->StatZeroPage, z); + } + + *ppPage = pPage; + if (fLockPage) + pgmPoolLockPage(pPool, pPage); + PGM_UNLOCK(pVM); + LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n", + rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored)); + STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a); + return rc; +} + + +/** + * Frees a usage of a pool page. + * + * @param pVM The cross context VM structure. + * @param HCPhys The HC physical address of the shadow page. + * @param iUser The shadow page pool index of the user table. + * NIL_PGMPOOL_IDX if root page. + * @param iUserTable The index into the user table (shadowed). Ignored if + * root page. + */ +void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable) +{ + LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%d iUserTable=%#x\n", HCPhys, iUser, iUserTable)); + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable); +} + + +/** + * Internal worker for finding a 'in-use' shadow page give by it's physical address. + * + * @returns Pointer to the shadow page structure. + * @param pPool The pool. + * @param HCPhys The HC physical address of the shadow page. + */ +PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys) +{ + PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM)); + + /* + * Look up the page. + */ + PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK); + + AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0)); + return pPage; +} + + +/** + * Internal worker for finding a page for debugging purposes, no assertions. + * + * @returns Pointer to the shadow page structure. NULL on if not found. + * @param pPool The pool. + * @param HCPhys The HC physical address of the shadow page. + */ +PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys) +{ + PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM)); + return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK); +} + + +/** + * Internal worker for PGM_HCPHYS_2_PTR. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param HCPhys The HC physical address of the shadow page. + * @param ppv Where to return the address. + */ +int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv) +{ + PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pVM->pgm.s.CTX_SUFF(pPool)->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK); + AssertMsgReturn(pPage && pPage->enmKind != PGMPOOLKIND_FREE, + ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0), + VERR_PGM_POOL_GET_PAGE_FAILED); + *ppv = (uint8_t *)pPage->CTX_SUFF(pvPage) + (HCPhys & PAGE_OFFSET_MASK); + return VINF_SUCCESS; +} + +#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */ + +/** + * Flush the specified page if present + * + * @param pVM The cross context VM structure. + * @param GCPhys Guest physical address of the page to flush + */ +void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys) +{ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + + VM_ASSERT_EMT(pVM); + + /* + * Look up the GCPhys in the hash. + */ + GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK; + unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)]; + if (i == NIL_PGMPOOL_IDX) + return; + + do + { + PPGMPOOLPAGE pPage = &pPool->aPages[i]; + if (pPage->GCPhys - GCPhys < PAGE_SIZE) + { + Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* Temporary to see if it hits. Remove later. */ + switch (pPage->enmKind) + { + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + case PGMPOOLKIND_64BIT_PML4: + case PGMPOOLKIND_32BIT_PD: + case PGMPOOLKIND_PAE_PDPT: + { + Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys)); +# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + if (pPage->fDirty) + STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage); + else +# endif + STAM_COUNTER_INC(&pPool->StatForceFlushPage); + Assert(!pgmPoolIsPageLocked(pPage)); + pgmPoolMonitorChainFlush(pPool, pPage); + return; + } + + /* ignore, no monitoring. */ + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + case PGMPOOLKIND_ROOT_NESTED: + case PGMPOOLKIND_PAE_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_PHYS: + case PGMPOOLKIND_32BIT_PD_PHYS: + case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: + break; + + default: + AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx)); + } + } + + /* next */ + i = pPage->iNext; + } while (i != NIL_PGMPOOL_IDX); + return; +} + + +/** + * Reset CPU on hot plugging. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + */ +void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu) +{ + pgmR3ExitShadowModeBeforePoolFlush(pVCpu); + + pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu); + VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); + VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH); +} + + +/** + * Flushes the entire cache. + * + * It will assert a global CR3 flush (FF) and assumes the caller is aware of + * this and execute this CR3 flush. + * + * @param pVM The cross context VM structure. + */ +void pgmR3PoolReset(PVM pVM) +{ + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + + PGM_LOCK_ASSERT_OWNER(pVM); + STAM_PROFILE_START(&pPool->StatR3Reset, a); + LogFlow(("pgmR3PoolReset:\n")); + + /* + * If there are no pages in the pool, there is nothing to do. + */ + if (pPool->cCurPages <= PGMPOOL_IDX_FIRST) + { + STAM_PROFILE_STOP(&pPool->StatR3Reset, a); + return; + } + + /* + * Exit the shadow mode since we're going to clear everything, + * including the root page. + */ + VMCC_FOR_EACH_VMCPU(pVM) + pgmR3ExitShadowModeBeforePoolFlush(pVCpu); + VMCC_FOR_EACH_VMCPU_END(pVM); + + + /* + * Nuke the free list and reinsert all pages into it. + */ + for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--) + { + PPGMPOOLPAGE pPage = &pPool->aPages[i]; + + if (pPage->fMonitored) + pgmPoolMonitorFlush(pPool, pPage); + pPage->iModifiedNext = NIL_PGMPOOL_IDX; + pPage->iModifiedPrev = NIL_PGMPOOL_IDX; + pPage->iMonitoredNext = NIL_PGMPOOL_IDX; + pPage->iMonitoredPrev = NIL_PGMPOOL_IDX; + pPage->GCPhys = NIL_RTGCPHYS; + pPage->enmKind = PGMPOOLKIND_FREE; + pPage->enmAccess = PGMPOOLACCESS_DONTCARE; + Assert(pPage->idx == i); + pPage->iNext = i + 1; + pPage->fA20Enabled = true; + pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */ + pPage->fSeenNonGlobal = false; + pPage->fMonitored = false; + pPage->fDirty = false; + pPage->fCached = false; + pPage->fReusedFlushPending = false; + pPage->iUserHead = NIL_PGMPOOL_USER_INDEX; + pPage->cPresent = 0; + pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX; + pPage->cModifications = 0; + pPage->iAgeNext = NIL_PGMPOOL_IDX; + pPage->iAgePrev = NIL_PGMPOOL_IDX; + pPage->idxDirtyEntry = 0; + pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR; + pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR; + pPage->cLastAccessHandler = 0; + pPage->cLocked = 0; +# ifdef VBOX_STRICT + pPage->GCPtrDirtyFault = NIL_RTGCPTR; +# endif + } + pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX; + pPool->iFreeHead = PGMPOOL_IDX_FIRST; + pPool->cUsedPages = 0; + + /* + * Zap and reinitialize the user records. + */ + pPool->cPresent = 0; + pPool->iUserFreeHead = 0; + PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers); + const unsigned cMaxUsers = pPool->cMaxUsers; + for (unsigned i = 0; i < cMaxUsers; i++) + { + paUsers[i].iNext = i + 1; + paUsers[i].iUser = NIL_PGMPOOL_IDX; + paUsers[i].iUserTable = 0xfffffffe; + } + paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX; + + /* + * Clear all the GCPhys links and rebuild the phys ext free list. + */ + for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX); + pRam; + pRam = pRam->CTX_SUFF(pNext)) + { + unsigned iPage = pRam->cb >> PAGE_SHIFT; + while (iPage-- > 0) + PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0); + } + + pPool->iPhysExtFreeHead = 0; + PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts); + const unsigned cMaxPhysExts = pPool->cMaxPhysExts; + for (unsigned i = 0; i < cMaxPhysExts; i++) + { + paPhysExts[i].iNext = i + 1; + paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX; + paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE; + paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX; + paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE; + paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX; + paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE; + } + paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX; + + /* + * Just zap the modified list. + */ + pPool->cModifiedPages = 0; + pPool->iModifiedHead = NIL_PGMPOOL_IDX; + + /* + * Clear the GCPhys hash and the age list. + */ + for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++) + pPool->aiHash[i] = NIL_PGMPOOL_IDX; + pPool->iAgeHead = NIL_PGMPOOL_IDX; + pPool->iAgeTail = NIL_PGMPOOL_IDX; + +# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT + /* Clear all dirty pages. */ + pPool->idxFreeDirtyPage = 0; + pPool->cDirtyPages = 0; + for (unsigned i = 0; i < RT_ELEMENTS(pPool->aidxDirtyPages); i++) + pPool->aidxDirtyPages[i] = NIL_PGMPOOL_IDX; +# endif + + /* + * Reinsert active pages into the hash and ensure monitoring chains are correct. + */ + VMCC_FOR_EACH_VMCPU(pVM) + { + /* + * Re-enter the shadowing mode and assert Sync CR3 FF. + */ + pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu); + VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); + VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH); + } + VMCC_FOR_EACH_VMCPU_END(pVM); + + STAM_PROFILE_STOP(&pPool->StatR3Reset, a); +} + +#endif /* IN_RING3 */ + +#if defined(LOG_ENABLED) || defined(VBOX_STRICT) +/** + * Stringifies a PGMPOOLKIND value. + */ +static const char *pgmPoolPoolKindToStr(uint8_t enmKind) +{ + switch ((PGMPOOLKIND)enmKind) + { + case PGMPOOLKIND_INVALID: + return "PGMPOOLKIND_INVALID"; + case PGMPOOLKIND_FREE: + return "PGMPOOLKIND_FREE"; + case PGMPOOLKIND_32BIT_PT_FOR_PHYS: + return "PGMPOOLKIND_32BIT_PT_FOR_PHYS"; + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT: + return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT"; + case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB: + return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB"; + case PGMPOOLKIND_PAE_PT_FOR_PHYS: + return "PGMPOOLKIND_PAE_PT_FOR_PHYS"; + case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT: + return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT"; + case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB: + return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB"; + case PGMPOOLKIND_PAE_PT_FOR_PAE_PT: + return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT"; + case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB: + return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB"; + case PGMPOOLKIND_32BIT_PD: + return "PGMPOOLKIND_32BIT_PD"; + case PGMPOOLKIND_32BIT_PD_PHYS: + return "PGMPOOLKIND_32BIT_PD_PHYS"; + case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD: + return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD"; + case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD: + return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD"; + case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD: + return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD"; + case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD: + return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD"; + case PGMPOOLKIND_PAE_PD_FOR_PAE_PD: + return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD"; + case PGMPOOLKIND_PAE_PD_PHYS: + return "PGMPOOLKIND_PAE_PD_PHYS"; + case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: + return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT"; + case PGMPOOLKIND_PAE_PDPT: + return "PGMPOOLKIND_PAE_PDPT"; + case PGMPOOLKIND_PAE_PDPT_PHYS: + return "PGMPOOLKIND_PAE_PDPT_PHYS"; + case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT: + return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT"; + case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS: + return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS"; + case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD: + return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD"; + case PGMPOOLKIND_64BIT_PD_FOR_PHYS: + return "PGMPOOLKIND_64BIT_PD_FOR_PHYS"; + case PGMPOOLKIND_64BIT_PML4: + return "PGMPOOLKIND_64BIT_PML4"; + case PGMPOOLKIND_EPT_PDPT_FOR_PHYS: + return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS"; + case PGMPOOLKIND_EPT_PD_FOR_PHYS: + return "PGMPOOLKIND_EPT_PD_FOR_PHYS"; + case PGMPOOLKIND_EPT_PT_FOR_PHYS: + return "PGMPOOLKIND_EPT_PT_FOR_PHYS"; + case PGMPOOLKIND_ROOT_NESTED: + return "PGMPOOLKIND_ROOT_NESTED"; + case PGMPOOLKIND_EPT_PT_FOR_EPT_PT: + return "PGMPOOLKIND_EPT_PT_FOR_EPT_PT"; + case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB: + return "PGMPOOLKIND_EPT_PT_FOR_EPT_2MB"; + case PGMPOOLKIND_EPT_PD_FOR_EPT_PD: + return "PGMPOOLKIND_EPT_PD_FOR_EPT_PD"; + case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT: + return "PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT"; + case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4: + return "PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4"; + } + return "Unknown kind!"; +} +#endif /* LOG_ENABLED || VBOX_STRICT */ + diff --git a/src/VBox/VMM/VMMAll/PGMAllShw.h b/src/VBox/VMM/VMMAll/PGMAllShw.h new file mode 100644 index 00000000..53c09f80 --- /dev/null +++ b/src/VBox/VMM/VMMAll/PGMAllShw.h @@ -0,0 +1,647 @@ +/* $Id: PGMAllShw.h $ */ +/** @file + * VBox - Page Manager, Shadow Paging Template - All context code. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +#undef SHWUINT +#undef SHWPT +#undef PSHWPT +#undef SHWPTE +#undef PSHWPTE +#undef SHWPD +#undef PSHWPD +#undef SHWPDE +#undef PSHWPDE +#undef SHW_PDE_PG_MASK +#undef SHW_PD_SHIFT +#undef SHW_PD_MASK +#undef SHW_PDE_ATOMIC_SET +#undef SHW_PDE_ATOMIC_SET2 +#undef SHW_PDE_IS_P +#undef SHW_PDE_IS_A +#undef SHW_PDE_IS_BIG +#undef SHW_PTE_PG_MASK +#undef SHW_PTE_IS_P +#undef SHW_PTE_IS_RW +#undef SHW_PTE_IS_US +#undef SHW_PTE_IS_A +#undef SHW_PTE_IS_D +#undef SHW_PTE_IS_P_RW +#undef SHW_PTE_IS_TRACK_DIRTY +#undef SHW_PTE_GET_HCPHYS +#undef SHW_PTE_GET_U +#undef SHW_PTE_LOG64 +#undef SHW_PTE_SET +#undef SHW_PTE_ATOMIC_SET +#undef SHW_PTE_ATOMIC_SET2 +#undef SHW_PTE_SET_RO +#undef SHW_PTE_SET_RW +#undef SHW_PT_SHIFT +#undef SHW_PT_MASK +#undef SHW_TOTAL_PD_ENTRIES +#undef SHW_PDPT_SHIFT +#undef SHW_PDPT_MASK +#undef SHW_PDPE_PG_MASK + +#if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT +# define SHWUINT uint32_t +# define SHWPT X86PT +# define PSHWPT PX86PT +# define SHWPTE X86PTE +# define PSHWPTE PX86PTE +# define SHWPD X86PD +# define PSHWPD PX86PD +# define SHWPDE X86PDE +# define PSHWPDE PX86PDE +# define SHW_PDE_PG_MASK X86_PDE_PG_MASK +# define SHW_PD_SHIFT X86_PD_SHIFT +# define SHW_PD_MASK X86_PD_MASK +# define SHW_TOTAL_PD_ENTRIES X86_PG_ENTRIES +# define SHW_PDE_IS_P(Pde) ( (Pde).u & X86_PDE_P ) +# define SHW_PDE_IS_A(Pde) ( (Pde).u & X86_PDE_A ) +# define SHW_PDE_IS_BIG(Pde) ( (Pde).u & X86_PDE_PS ) +# define SHW_PDE_ATOMIC_SET(Pde, uNew) do { ASMAtomicWriteU32(&(Pde).u, (uNew)); } while (0) +# define SHW_PDE_ATOMIC_SET2(Pde, Pde2) do { ASMAtomicWriteU32(&(Pde).u, (Pde2).u); } while (0) +# define SHW_PTE_PG_MASK X86_PTE_PG_MASK +# define SHW_PTE_IS_P(Pte) ( (Pte).u & X86_PTE_P ) +# define SHW_PTE_IS_RW(Pte) ( (Pte).u & X86_PTE_RW ) +# define SHW_PTE_IS_US(Pte) ( (Pte).u & X86_PTE_US ) +# define SHW_PTE_IS_A(Pte) ( (Pte).u & X86_PTE_A ) +# define SHW_PTE_IS_D(Pte) ( (Pte).u & X86_PTE_D ) +# define SHW_PTE_IS_P_RW(Pte) ( ((Pte).u & (X86_PTE_P | X86_PTE_RW)) == (X86_PTE_P | X86_PTE_RW) ) +# define SHW_PTE_IS_TRACK_DIRTY(Pte) ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) ) +# define SHW_PTE_GET_HCPHYS(Pte) ( (Pte).u & X86_PTE_PG_MASK ) +# define SHW_PTE_LOG64(Pte) ( (uint64_t)(Pte).u ) +# define SHW_PTE_GET_U(Pte) ( (Pte).u ) /**< Use with care. */ +# define SHW_PTE_SET(Pte, uNew) do { (Pte).u = (uNew); } while (0) +# define SHW_PTE_ATOMIC_SET(Pte, uNew) do { ASMAtomicWriteU32(&(Pte).u, (uNew)); } while (0) +# define SHW_PTE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU32(&(Pte).u, (Pte2).u); } while (0) +# define SHW_PTE_SET_RO(Pte) do { (Pte).u &= ~(X86PGUINT)X86_PTE_RW; } while (0) +# define SHW_PTE_SET_RW(Pte) do { (Pte).u |= X86_PTE_RW; } while (0) +# define SHW_PT_SHIFT X86_PT_SHIFT +# define SHW_PT_MASK X86_PT_MASK + +#elif PGM_SHW_TYPE == PGM_TYPE_EPT +# define SHWUINT uint64_t +# define SHWPT EPTPT +# define PSHWPT PEPTPT +# define SHWPTE EPTPTE +# define PSHWPTE PEPTPTE +# define SHWPD EPTPD +# define PSHWPD PEPTPD +# define SHWPDE EPTPDE +# define PSHWPDE PEPTPDE +# define SHW_PDE_PG_MASK EPT_PDE_PG_MASK +# define SHW_PD_SHIFT EPT_PD_SHIFT +# define SHW_PD_MASK EPT_PD_MASK +# define SHW_PDE_IS_P(Pde) ( (Pde).u & EPT_E_READ /* always set*/ ) +# define SHW_PDE_IS_A(Pde) ( 1 ) /* We don't use EPT_E_ACCESSED, use with care! */ +# define SHW_PDE_IS_BIG(Pde) ( (Pde).u & EPT_E_LEAF ) +# define SHW_PDE_ATOMIC_SET(Pde, uNew) do { ASMAtomicWriteU64(&(Pde).u, (uNew)); } while (0) +# define SHW_PDE_ATOMIC_SET2(Pde, Pde2) do { ASMAtomicWriteU64(&(Pde).u, (Pde2).u); } while (0) +# define SHW_PTE_PG_MASK EPT_PTE_PG_MASK +# define SHW_PTE_IS_P(Pte) ( (Pte).u & EPT_E_READ ) /* Approximation, works for us. */ +# define SHW_PTE_IS_RW(Pte) ( (Pte).u & EPT_E_WRITE ) +# define SHW_PTE_IS_US(Pte) ( true ) +# define SHW_PTE_IS_A(Pte) ( true ) +# define SHW_PTE_IS_D(Pte) ( true ) +# define SHW_PTE_IS_P_RW(Pte) ( ((Pte).u & (EPT_E_READ | EPT_E_WRITE)) == (EPT_E_READ | EPT_E_WRITE) ) +# define SHW_PTE_IS_TRACK_DIRTY(Pte) ( false ) +# define SHW_PTE_GET_HCPHYS(Pte) ( (Pte).u & EPT_PTE_PG_MASK ) +# define SHW_PTE_LOG64(Pte) ( (Pte).u ) +# define SHW_PTE_GET_U(Pte) ( (Pte).u ) /**< Use with care. */ +# define SHW_PTE_SET(Pte, uNew) do { (Pte).u = (uNew); } while (0) +# define SHW_PTE_ATOMIC_SET(Pte, uNew) do { ASMAtomicWriteU64(&(Pte).u, (uNew)); } while (0) +# define SHW_PTE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0) +# define SHW_PTE_SET_RO(Pte) do { (Pte).u &= ~(uint64_t)EPT_E_WRITE; } while (0) +# define SHW_PTE_SET_RW(Pte) do { (Pte).u |= EPT_E_WRITE; } while (0) +# define SHW_PT_SHIFT EPT_PT_SHIFT +# define SHW_PT_MASK EPT_PT_MASK +# define SHW_PDPT_SHIFT EPT_PDPT_SHIFT +# define SHW_PDPT_MASK EPT_PDPT_MASK +# define SHW_PDPE_PG_MASK EPT_PDPE_PG_MASK +# define SHW_TOTAL_PD_ENTRIES (EPT_PG_AMD64_ENTRIES * EPT_PG_AMD64_PDPE_ENTRIES) + +#else +# define SHWUINT uint64_t +# define SHWPT PGMSHWPTPAE +# define PSHWPT PPGMSHWPTPAE +# define SHWPTE PGMSHWPTEPAE +# define PSHWPTE PPGMSHWPTEPAE +# define SHWPD X86PDPAE +# define PSHWPD PX86PDPAE +# define SHWPDE X86PDEPAE +# define PSHWPDE PX86PDEPAE +# define SHW_PDE_PG_MASK X86_PDE_PAE_PG_MASK +# define SHW_PD_SHIFT X86_PD_PAE_SHIFT +# define SHW_PD_MASK X86_PD_PAE_MASK +# define SHW_PDE_IS_P(Pde) ( (Pde).u & X86_PDE_P ) +# define SHW_PDE_IS_A(Pde) ( (Pde).u & X86_PDE_A ) +# define SHW_PDE_IS_BIG(Pde) ( (Pde).u & X86_PDE_PS ) +# define SHW_PDE_ATOMIC_SET(Pde, uNew) do { ASMAtomicWriteU64(&(Pde).u, (uNew)); } while (0) +# define SHW_PDE_ATOMIC_SET2(Pde, Pde2) do { ASMAtomicWriteU64(&(Pde).u, (Pde2).u); } while (0) +# define SHW_PTE_PG_MASK X86_PTE_PAE_PG_MASK +# define SHW_PTE_IS_P(Pte) PGMSHWPTEPAE_IS_P(Pte) +# define SHW_PTE_IS_RW(Pte) PGMSHWPTEPAE_IS_RW(Pte) +# define SHW_PTE_IS_US(Pte) PGMSHWPTEPAE_IS_US(Pte) +# define SHW_PTE_IS_A(Pte) PGMSHWPTEPAE_IS_A(Pte) +# define SHW_PTE_IS_D(Pte) PGMSHWPTEPAE_IS_D(Pte) +# define SHW_PTE_IS_P_RW(Pte) PGMSHWPTEPAE_IS_P_RW(Pte) +# define SHW_PTE_IS_TRACK_DIRTY(Pte) PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte) +# define SHW_PTE_GET_HCPHYS(Pte) PGMSHWPTEPAE_GET_HCPHYS(Pte) +# define SHW_PTE_LOG64(Pte) PGMSHWPTEPAE_GET_LOG(Pte) +# define SHW_PTE_GET_U(Pte) PGMSHWPTEPAE_GET_U(Pte) /**< Use with care. */ +# define SHW_PTE_SET(Pte, uNew) PGMSHWPTEPAE_SET(Pte, uNew) +# define SHW_PTE_ATOMIC_SET(Pte, uNew) PGMSHWPTEPAE_ATOMIC_SET(Pte, uNew) +# define SHW_PTE_ATOMIC_SET2(Pte, Pte2) PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2) +# define SHW_PTE_SET_RO(Pte) PGMSHWPTEPAE_SET_RO(Pte) +# define SHW_PTE_SET_RW(Pte) PGMSHWPTEPAE_SET_RW(Pte) +# define SHW_PT_SHIFT X86_PT_PAE_SHIFT +# define SHW_PT_MASK X86_PT_PAE_MASK + +# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64 || /* whatever: */ PGM_SHW_TYPE == PGM_TYPE_NONE +# define SHW_PDPT_SHIFT X86_PDPT_SHIFT +# define SHW_PDPT_MASK X86_PDPT_MASK_AMD64 +# define SHW_PDPE_PG_MASK X86_PDPE_PG_MASK +# define SHW_TOTAL_PD_ENTRIES (X86_PG_AMD64_ENTRIES * X86_PG_AMD64_PDPE_ENTRIES) + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE +# define SHW_PDPT_SHIFT X86_PDPT_SHIFT +# define SHW_PDPT_MASK X86_PDPT_MASK_PAE +# define SHW_PDPE_PG_MASK X86_PDPE_PG_MASK +# define SHW_TOTAL_PD_ENTRIES (X86_PG_PAE_ENTRIES * X86_PG_PAE_PDPE_ENTRIES) + +# else +# error "Misconfigured PGM_SHW_TYPE or something..." +# endif +#endif + +#if PGM_SHW_TYPE == PGM_TYPE_NONE && PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) +# error "PGM_TYPE_IS_NESTED_OR_EPT is true for PGM_TYPE_NONE!" +#endif + + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +RT_C_DECLS_BEGIN +PGM_SHW_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys); +PGM_SHW_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags); +PGM_SHW_DECL(int, Exit)(PVMCPUCC pVCpu); +#ifdef IN_RING3 +PGM_SHW_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta); +#endif +RT_C_DECLS_END + + +/** + * Enters the shadow mode. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + */ +PGM_SHW_DECL(int, Enter)(PVMCPUCC pVCpu) +{ +#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) + +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + RTGCPHYS GCPhysCR3; + PGMPOOLKIND enmKind; + if (pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_EPT) + { + GCPhysCR3 = RT_BIT_64(63); + enmKind = PGMPOOLKIND_ROOT_NESTED; + } + else + { + GCPhysCR3 = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK; + enmKind = PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4; + } +# else + RTGCPHYS const GCPhysCR3 = RT_BIT_64(63); + PGMPOOLKIND const enmKind = PGMPOOLKIND_ROOT_NESTED; +# endif + PVMCC const pVM = pVCpu->CTX_SUFF(pVM); + + Assert(HMIsNestedPagingActive(pVM)); + Assert(pVM->pgm.s.fNestedPaging); + Assert(!pVCpu->pgm.s.pShwPageCR3R3); + + PGM_LOCK_VOID(pVM); + + PPGMPOOLPAGE pNewShwPageCR3; + int rc = pgmPoolAlloc(pVM, GCPhysCR3, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu), + NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/, + &pNewShwPageCR3); + AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc); + + pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pVM->pgm.s.CTX_SUFF(pPool), pNewShwPageCR3); + pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pVM->pgm.s.CTX_SUFF(pPool), pNewShwPageCR3); + + PGM_UNLOCK(pVM); + + Log(("Enter nested shadow paging mode: root %RHv phys %RHp\n", pVCpu->pgm.s.pShwPageCR3R3, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->Core.Key)); +#else + NOREF(pVCpu); +#endif + return VINF_SUCCESS; +} + + +/** + * Exits the shadow mode. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + */ +PGM_SHW_DECL(int, Exit)(PVMCPUCC pVCpu) +{ +#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)) + { + PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); + + PGM_LOCK_VOID(pVM); + +# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT + if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) + pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); +# endif + + /* Do *not* unlock this page as we have two of them floating around in the 32-bit host & 64-bit guest case. + * We currently assert when you try to free one of them; don't bother to really allow this. + * + * Note that this is two nested paging root pages max. This isn't a leak. They are reused. + */ + /* pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); */ + + pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX); + pVCpu->pgm.s.pShwPageCR3R3 = 0; + pVCpu->pgm.s.pShwPageCR3R0 = 0; + + PGM_UNLOCK(pVM); + + Log(("Leave nested shadow paging mode\n")); + } +#else + RT_NOREF_PV(pVCpu); +#endif + return VINF_SUCCESS; +} + + +/** + * Gets effective page information (from the VMM page directory). + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Guest Context virtual address of the page. + * @param pfFlags Where to store the flags. These are X86_PTE_*. + * @param pHCPhys Where to store the HC physical address of the page. + * This is page aligned. + * @remark You should use PGMMapGetPage() for pages in a mapping. + */ +PGM_SHW_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys) +{ +#if PGM_SHW_TYPE == PGM_TYPE_NONE + RT_NOREF(pVCpu, GCPtr); + AssertFailed(); + *pfFlags = 0; + *pHCPhys = NIL_RTHCPHYS; + return VERR_PGM_SHW_NONE_IPE; + +#else /* PGM_SHW_TYPE != PGM_TYPE_NONE */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * Get the PDE. + */ +# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64 + X86PDEPAE Pde; + + /* PML4 */ + X86PML4E Pml4e = pgmShwGetLongModePML4E(pVCpu, GCPtr); + if (!(Pml4e.u & X86_PML4E_P)) + return VERR_PAGE_TABLE_NOT_PRESENT; + + /* PDPT */ + PX86PDPT pPDPT; + int rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pml4e.u & X86_PML4E_PG_MASK, &pPDPT); + if (RT_FAILURE(rc)) + return rc; + const unsigned iPDPT = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK; + X86PDPE Pdpe = pPDPT->a[iPDPT]; + if (!(Pdpe.u & X86_PDPE_P)) + return VERR_PAGE_TABLE_NOT_PRESENT; + + /* PD */ + PX86PDPAE pPd; + rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pdpe.u & X86_PDPE_PG_MASK, &pPd); + if (RT_FAILURE(rc)) + return rc; + const unsigned iPd = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK; + Pde = pPd->a[iPd]; + + /* Merge accessed, write, user and no-execute bits into the PDE. */ + AssertCompile(X86_PML4E_A == X86_PDPE_A && X86_PML4E_A == X86_PDE_A); + AssertCompile(X86_PML4E_RW == X86_PDPE_RW && X86_PML4E_RW == X86_PDE_RW); + AssertCompile(X86_PML4E_US == X86_PDPE_US && X86_PML4E_US == X86_PDE_US); + AssertCompile(X86_PML4E_NX == X86_PDPE_LM_NX && X86_PML4E_NX == X86_PDE_PAE_NX); + Pde.u &= (Pml4e.u & Pdpe.u) | ~(X86PGPAEUINT)(X86_PML4E_A | X86_PML4E_RW | X86_PML4E_US); + Pde.u |= (Pml4e.u | Pdpe.u) & X86_PML4E_NX; + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE + X86PDEPAE Pde = pgmShwGetPaePDE(pVCpu, GCPtr); + +# elif PGM_SHW_TYPE == PGM_TYPE_EPT + Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT); + PEPTPD pPDDst; + int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtr, NULL, &pPDDst); + if (rc == VINF_SUCCESS) /** @todo this function isn't expected to return informational status codes. Check callers / fix. */ + { /* likely */ } + else + { + AssertRC(rc); + return rc; + } + Assert(pPDDst); + + const unsigned iPd = ((GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK); + EPTPDE Pde = pPDDst->a[iPd]; + +# elif PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT + X86PDE Pde = pgmShwGet32BitPDE(pVCpu, GCPtr); + +# else +# error "Misconfigured PGM_SHW_TYPE or something..." +# endif + if (!SHW_PDE_IS_P(Pde)) + return VERR_PAGE_TABLE_NOT_PRESENT; + + /* Deal with large pages. */ + if (SHW_PDE_IS_BIG(Pde)) + { + /* + * Store the results. + * RW and US flags depend on the entire page translation hierarchy - except for + * legacy PAE which has a simplified PDPE. + */ + if (pfFlags) + { + *pfFlags = (Pde.u & ~SHW_PDE_PG_MASK); +# if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64 + if ( (Pde.u & X86_PTE_PAE_NX) +# if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE) + && CPUMIsGuestNXEnabled(pVCpu) /** @todo why do we have to check the guest state here? */ +# endif + ) + *pfFlags |= X86_PTE_PAE_NX; +# endif + } + + if (pHCPhys) + *pHCPhys = (Pde.u & SHW_PDE_PG_MASK) + (GCPtr & (RT_BIT(SHW_PD_SHIFT) - 1) & X86_PAGE_4K_BASE_MASK); + + return VINF_SUCCESS; + } + + /* + * Get PT entry. + */ + PSHWPT pPT; + int rc2 = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pde.u & SHW_PDE_PG_MASK, &pPT); + if (RT_FAILURE(rc2)) + return rc2; + const unsigned iPt = (GCPtr >> SHW_PT_SHIFT) & SHW_PT_MASK; + SHWPTE Pte = pPT->a[iPt]; + if (!SHW_PTE_IS_P(Pte)) + return VERR_PAGE_NOT_PRESENT; + + /* + * Store the results. + * RW and US flags depend on the entire page translation hierarchy - except for + * legacy PAE which has a simplified PDPE. + */ + if (pfFlags) + { + *pfFlags = (SHW_PTE_GET_U(Pte) & ~SHW_PTE_PG_MASK) + & ((Pde.u & (X86_PTE_RW | X86_PTE_US)) | ~(uint64_t)(X86_PTE_RW | X86_PTE_US)); + +# if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64 + /* The NX bit is determined by a bitwise OR between the PT and PD */ + if ( ((SHW_PTE_GET_U(Pte) | Pde.u) & X86_PTE_PAE_NX) +# if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE) + && CPUMIsGuestNXEnabled(pVCpu) /** @todo why do we have to check the guest state here? */ +# endif + ) + *pfFlags |= X86_PTE_PAE_NX; +# endif + } + + if (pHCPhys) + *pHCPhys = SHW_PTE_GET_HCPHYS(Pte); + + return VINF_SUCCESS; +#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */ +} + + +/** + * Modify page flags for a range of pages in the shadow context. + * + * The existing flags are ANDed with the fMask and ORed with the fFlags. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param GCPtr Virtual address of the first page in the range. Page aligned! + * @param cb Size (in bytes) of the range to apply the modification to. Page aligned! + * @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course. + * @param fMask The AND mask - page flags X86_PTE_*. + * Be extremely CAREFUL with ~'ing values because they can be 32-bit! + * @param fOpFlags A combination of the PGM_MK_PK_XXX flags. + * @remark You must use PGMMapModifyPage() for pages in a mapping. + */ +PGM_SHW_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags) +{ +#if PGM_SHW_TYPE == PGM_TYPE_NONE + RT_NOREF(pVCpu, GCPtr, cb, fFlags, fMask, fOpFlags); + AssertFailed(); + return VERR_PGM_SHW_NONE_IPE; + +#else /* PGM_SHW_TYPE != PGM_TYPE_NONE */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PGM_LOCK_ASSERT_OWNER(pVM); + + /* + * Walk page tables and pages till we're done. + */ + int rc; + for (;;) + { + /* + * Get the PDE. + */ +# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64 + X86PDEPAE Pde; + /* PML4 */ + X86PML4E Pml4e = pgmShwGetLongModePML4E(pVCpu, GCPtr); + if (!(Pml4e.u & X86_PML4E_P)) + return VERR_PAGE_TABLE_NOT_PRESENT; + + /* PDPT */ + PX86PDPT pPDPT; + rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pml4e.u & X86_PML4E_PG_MASK, &pPDPT); + if (RT_FAILURE(rc)) + return rc; + const unsigned iPDPT = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK; + X86PDPE Pdpe = pPDPT->a[iPDPT]; + if (!(Pdpe.u & X86_PDPE_P)) + return VERR_PAGE_TABLE_NOT_PRESENT; + + /* PD */ + PX86PDPAE pPd; + rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pdpe.u & X86_PDPE_PG_MASK, &pPd); + if (RT_FAILURE(rc)) + return rc; + const unsigned iPd = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK; + Pde = pPd->a[iPd]; + +# elif PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE + X86PDEPAE Pde = pgmShwGetPaePDE(pVCpu, GCPtr); + +# elif PGM_SHW_TYPE == PGM_TYPE_EPT + Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT); + const unsigned iPd = ((GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK); + PEPTPD pPDDst; + EPTPDE Pde; + + rc = pgmShwGetEPTPDPtr(pVCpu, GCPtr, NULL, &pPDDst); + if (rc != VINF_SUCCESS) + { + AssertRC(rc); + return rc; + } + Assert(pPDDst); + Pde = pPDDst->a[iPd]; + +# else /* PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT */ + X86PDE Pde = pgmShwGet32BitPDE(pVCpu, GCPtr); +# endif + if (!SHW_PDE_IS_P(Pde)) + return VERR_PAGE_TABLE_NOT_PRESENT; + + AssertFatalMsg(!SHW_PDE_IS_BIG(Pde), ("Pde=%#RX64\n", (uint64_t)Pde.u)); + + /* + * Map the page table. + */ + PSHWPT pPT; + rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pde.u & SHW_PDE_PG_MASK, &pPT); + if (RT_FAILURE(rc)) + return rc; + + unsigned iPTE = (GCPtr >> SHW_PT_SHIFT) & SHW_PT_MASK; + while (iPTE < RT_ELEMENTS(pPT->a)) + { + if (SHW_PTE_IS_P(pPT->a[iPTE])) + { + SHWPTE const OrgPte = pPT->a[iPTE]; + SHWPTE NewPte; + + SHW_PTE_SET(NewPte, (SHW_PTE_GET_U(OrgPte) & (fMask | SHW_PTE_PG_MASK)) | (fFlags & ~SHW_PTE_PG_MASK)); + if (!SHW_PTE_IS_P(NewPte)) + { + /** @todo Some CSAM code path might end up here and upset + * the page pool. */ + AssertMsgFailed(("NewPte=%#RX64 OrgPte=%#RX64 GCPtr=%#RGv\n", SHW_PTE_LOG64(NewPte), SHW_PTE_LOG64(OrgPte), GCPtr)); + } + else if ( SHW_PTE_IS_RW(NewPte) + && !SHW_PTE_IS_RW(OrgPte) + && !(fOpFlags & PGM_MK_PG_IS_MMIO2) ) + { + /** @todo Optimize \#PF handling by caching data. We can + * then use this when PGM_MK_PG_IS_WRITE_FAULT is + * set instead of resolving the guest physical + * address yet again. */ + PGMPTWALK GstWalk; + rc = PGMGstGetPage(pVCpu, GCPtr, &GstWalk); + AssertRC(rc); + if (RT_SUCCESS(rc)) + { + Assert((GstWalk.fEffective & X86_PTE_RW) || !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP /* allow netware hack */)); + PPGMPAGE pPage = pgmPhysGetPage(pVM, GstWalk.GCPhys); + Assert(pPage); + if (pPage) + { + rc = pgmPhysPageMakeWritable(pVM, pPage, GstWalk.GCPhys); + AssertRCReturn(rc, rc); + Log(("%s: pgmPhysPageMakeWritable on %RGv / %RGp %R[pgmpage]\n", __PRETTY_FUNCTION__, GCPtr, GstWalk.GCPhys, pPage)); + } + } + } + + SHW_PTE_ATOMIC_SET2(pPT->a[iPTE], NewPte); +# if PGM_SHW_TYPE == PGM_TYPE_EPT + HMInvalidatePhysPage(pVM, (RTGCPHYS)GCPtr); +# else + PGM_INVL_PG_ALL_VCPU(pVM, GCPtr); +# endif + } + + /* next page */ + cb -= HOST_PAGE_SIZE; + if (!cb) + return VINF_SUCCESS; + GCPtr += HOST_PAGE_SIZE; + iPTE++; + } + } +#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */ +} + + +#ifdef IN_RING3 +/** + * Relocate any GC pointers related to shadow mode paging. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param offDelta The relocation offset. + */ +PGM_SHW_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta) +{ + RT_NOREF(pVCpu, offDelta); + return VINF_SUCCESS; +} +#endif + diff --git a/src/VBox/VMM/VMMAll/SELMAll.cpp b/src/VBox/VMM/VMMAll/SELMAll.cpp new file mode 100644 index 00000000..c4320022 --- /dev/null +++ b/src/VBox/VMM/VMMAll/SELMAll.cpp @@ -0,0 +1,392 @@ +/* $Id: SELMAll.cpp $ */ +/** @file + * SELM All contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_SELM +#include +#include +#include +#include +#include +#include +#include +#include "SELMInternal.h" +#include +#include +#include +#include +#include +#include +#include + + + +/** + * Converts a GC selector based address to a flat address. + * + * No limit checks are done. Use the SELMToFlat*() or SELMValidate*() functions + * for that. + * + * @returns Flat address. + * @param pVCpu The cross context virtual CPU structure. + * @param idxSeg The selector register to use (X86_SREG_XXX). + * @param pCtx Pointer to the register context for the CPU. + * @param Addr Address part. + */ +VMMDECL(RTGCPTR) SELMToFlat(PVMCPUCC pVCpu, unsigned idxSeg, PCPUMCTX pCtx, RTGCPTR Addr) +{ + Assert(idxSeg < RT_ELEMENTS(pCtx->aSRegs)); + PCPUMSELREG pSReg = &pCtx->aSRegs[idxSeg]; + + /* + * Deal with real & v86 mode first. + */ + if ( pCtx->eflags.Bits.u1VM + || CPUMIsGuestInRealMode(pVCpu)) + { + uint32_t uFlat = (uint32_t)Addr & 0xffff; + if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)) + uFlat += (uint32_t)pSReg->u64Base; + else + uFlat += (uint32_t)pSReg->Sel << 4; + return (RTGCPTR)uFlat; + } + + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->cs)); + + /* 64 bits mode: CS, DS, ES and SS are treated as if each segment base is 0 + (Intel(r) 64 and IA-32 Architectures Software Developer's Manual: 3.4.2.1). */ + if ( pCtx->cs.Attr.n.u1Long + && CPUMIsGuestInLongMode(pVCpu)) + { + switch (idxSeg) + { + case X86_SREG_FS: + case X86_SREG_GS: + return (RTGCPTR)(pSReg->u64Base + Addr); + + default: + return Addr; /* base 0 */ + } + } + + /* AMD64 manual: compatibility mode ignores the high 32 bits when calculating an effective address. */ + Assert(pSReg->u64Base <= 0xffffffff); + return (uint32_t)pSReg->u64Base + (uint32_t)Addr; +} + + +/** + * Converts a GC selector based address to a flat address. + * + * Some basic checking is done, but not all kinds yet. + * + * @returns VBox status + * @param pVCpu The cross context virtual CPU structure. + * @param idxSeg The selector register to use (X86_SREG_XXX). + * @param pCtx Pointer to the register context for the CPU. + * @param Addr Address part. + * @param fFlags SELMTOFLAT_FLAGS_* + * GDT entires are valid. + * @param ppvGC Where to store the GC flat address. + */ +VMMDECL(int) SELMToFlatEx(PVMCPU pVCpu, unsigned idxSeg, PCPUMCTX pCtx, RTGCPTR Addr, uint32_t fFlags, PRTGCPTR ppvGC) +{ + AssertReturn(idxSeg < RT_ELEMENTS(pCtx->aSRegs), VERR_INVALID_PARAMETER); + PCPUMSELREG pSReg = &pCtx->aSRegs[idxSeg]; + + /* + * Deal with real & v86 mode first. + */ + if ( pCtx->eflags.Bits.u1VM + || CPUMIsGuestInRealMode(pVCpu)) + { + if (ppvGC) + { + uint32_t uFlat = (uint32_t)Addr & 0xffff; + if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)) + *ppvGC = (uint32_t)pSReg->u64Base + uFlat; + else + *ppvGC = ((uint32_t)pSReg->Sel << 4) + uFlat; + } + return VINF_SUCCESS; + } + + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->cs)); + + /* 64 bits mode: CS, DS, ES and SS are treated as if each segment base is 0 + (Intel(r) 64 and IA-32 Architectures Software Developer's Manual: 3.4.2.1). */ + RTGCPTR pvFlat; + bool fCheckLimit = true; + if ( pCtx->cs.Attr.n.u1Long + && CPUMIsGuestInLongMode(pVCpu)) + { + fCheckLimit = false; + switch (idxSeg) + { + case X86_SREG_FS: + case X86_SREG_GS: + pvFlat = pSReg->u64Base + Addr; + break; + + default: + pvFlat = Addr; + break; + } + } + else + { + /* AMD64 manual: compatibility mode ignores the high 32 bits when calculating an effective address. */ + Assert(pSReg->u64Base <= UINT32_C(0xffffffff)); + pvFlat = (uint32_t)pSReg->u64Base + (uint32_t)Addr; + Assert(pvFlat <= UINT32_MAX); + } + + /* + * Check type if present. + */ + if (pSReg->Attr.n.u1Present) + { + switch (pSReg->Attr.n.u4Type) + { + /* Read only selector type. */ + case X86_SEL_TYPE_RO: + case X86_SEL_TYPE_RO_ACC: + case X86_SEL_TYPE_RW: + case X86_SEL_TYPE_RW_ACC: + case X86_SEL_TYPE_EO: + case X86_SEL_TYPE_EO_ACC: + case X86_SEL_TYPE_ER: + case X86_SEL_TYPE_ER_ACC: + if (!(fFlags & SELMTOFLAT_FLAGS_NO_PL)) + { + /** @todo fix this mess */ + } + /* check limit. */ + if (fCheckLimit && Addr > pSReg->u32Limit) + return VERR_OUT_OF_SELECTOR_BOUNDS; + /* ok */ + if (ppvGC) + *ppvGC = pvFlat; + return VINF_SUCCESS; + + case X86_SEL_TYPE_EO_CONF: + case X86_SEL_TYPE_EO_CONF_ACC: + case X86_SEL_TYPE_ER_CONF: + case X86_SEL_TYPE_ER_CONF_ACC: + if (!(fFlags & SELMTOFLAT_FLAGS_NO_PL)) + { + /** @todo fix this mess */ + } + /* check limit. */ + if (fCheckLimit && Addr > pSReg->u32Limit) + return VERR_OUT_OF_SELECTOR_BOUNDS; + /* ok */ + if (ppvGC) + *ppvGC = pvFlat; + return VINF_SUCCESS; + + case X86_SEL_TYPE_RO_DOWN: + case X86_SEL_TYPE_RO_DOWN_ACC: + case X86_SEL_TYPE_RW_DOWN: + case X86_SEL_TYPE_RW_DOWN_ACC: + if (!(fFlags & SELMTOFLAT_FLAGS_NO_PL)) + { + /** @todo fix this mess */ + } + /* check limit. */ + if (fCheckLimit) + { + if (!pSReg->Attr.n.u1Granularity && Addr > UINT32_C(0xffff)) + return VERR_OUT_OF_SELECTOR_BOUNDS; + if (Addr <= pSReg->u32Limit) + return VERR_OUT_OF_SELECTOR_BOUNDS; + } + /* ok */ + if (ppvGC) + *ppvGC = pvFlat; + return VINF_SUCCESS; + + default: + return VERR_INVALID_SELECTOR; + + } + } + return VERR_SELECTOR_NOT_PRESENT; +} + + + +/** + * Validates and converts a GC selector based code address to a flat + * address when in real or v8086 mode. + * + * @returns VINF_SUCCESS. + * @param pVCpu The cross context virtual CPU structure. + * @param SelCS Selector part. + * @param pSReg The hidden CS register part. Optional. + * @param Addr Address part. + * @param ppvFlat Where to store the flat address. + */ +DECLINLINE(int) selmValidateAndConvertCSAddrRealMode(PVMCPU pVCpu, RTSEL SelCS, PCCPUMSELREGHID pSReg, RTGCPTR Addr, + PRTGCPTR ppvFlat) +{ + NOREF(pVCpu); + uint32_t uFlat = Addr & 0xffff; + if (!pSReg || !CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)) + uFlat += (uint32_t)SelCS << 4; + else + uFlat += (uint32_t)pSReg->u64Base; + *ppvFlat = uFlat; + return VINF_SUCCESS; +} + + +/** + * Validates and converts a GC selector based code address to a flat address + * when in protected/long mode using the standard hidden selector registers + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param SelCPL Current privilege level. Get this from SS - CS might be + * conforming! A full selector can be passed, we'll only + * use the RPL part. + * @param SelCS Selector part. + * @param pSRegCS The full CS selector register. + * @param Addr The address (think IP/EIP/RIP). + * @param ppvFlat Where to store the flat address upon successful return. + */ +DECLINLINE(int) selmValidateAndConvertCSAddrHidden(PVMCPU pVCpu, RTSEL SelCPL, RTSEL SelCS, PCCPUMSELREGHID pSRegCS, + RTGCPTR Addr, PRTGCPTR ppvFlat) +{ + NOREF(SelCPL); NOREF(SelCS); + + /* + * Check if present. + */ + if (pSRegCS->Attr.n.u1Present) + { + /* + * Type check. + */ + if ( pSRegCS->Attr.n.u1DescType == 1 + && (pSRegCS->Attr.n.u4Type & X86_SEL_TYPE_CODE)) + { + /* 64 bits mode: CS, DS, ES and SS are treated as if each segment base is 0 + (Intel(r) 64 and IA-32 Architectures Software Developer's Manual: 3.4.2.1). */ + if ( pSRegCS->Attr.n.u1Long + && CPUMIsGuestInLongMode(pVCpu)) + { + *ppvFlat = Addr; + return VINF_SUCCESS; + } + + /* + * Limit check. Note that the limit in the hidden register is the + * final value. The granularity bit was included in its calculation. + */ + uint32_t u32Limit = pSRegCS->u32Limit; + if ((uint32_t)Addr <= u32Limit) + { + *ppvFlat = (uint32_t)Addr + (uint32_t)pSRegCS->u64Base; + return VINF_SUCCESS; + } + + return VERR_OUT_OF_SELECTOR_BOUNDS; + } + return VERR_NOT_CODE_SELECTOR; + } + return VERR_SELECTOR_NOT_PRESENT; +} + + +/** + * Validates and converts a GC selector based code address to a flat address. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param fEFlags Current EFLAGS. + * @param SelCPL Current privilege level. Get this from SS - CS might be + * conforming! A full selector can be passed, we'll only + * use the RPL part. + * @param SelCS Selector part. + * @param pSRegCS The full CS selector register. + * @param Addr The address (think IP/EIP/RIP). + * @param ppvFlat Where to store the flat address upon successful return. + */ +VMMDECL(int) SELMValidateAndConvertCSAddr(PVMCPU pVCpu, uint32_t fEFlags, RTSEL SelCPL, RTSEL SelCS, PCPUMSELREG pSRegCS, + RTGCPTR Addr, PRTGCPTR ppvFlat) +{ + if ( (fEFlags & X86_EFL_VM) + || CPUMIsGuestInRealMode(pVCpu)) + return selmValidateAndConvertCSAddrRealMode(pVCpu, SelCS, pSRegCS, Addr, ppvFlat); + + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSRegCS)); + Assert(pSRegCS->Sel == SelCS); + + return selmValidateAndConvertCSAddrHidden(pVCpu, SelCPL, SelCS, pSRegCS, Addr, ppvFlat); +} + + +/** + * Gets info about the current TSS. + * + * @returns VBox status code. + * @retval VINF_SUCCESS if we've got a TSS loaded. + * @retval VERR_SELM_NO_TSS if we haven't got a TSS (rather unlikely). + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param pGCPtrTss Where to store the TSS address. + * @param pcbTss Where to store the TSS size limit. + * @param pfCanHaveIOBitmap Where to store the can-have-I/O-bitmap indicator. (optional) + */ +VMMDECL(int) SELMGetTSSInfo(PVM pVM, PVMCPU pVCpu, PRTGCUINTPTR pGCPtrTss, PRTGCUINTPTR pcbTss, bool *pfCanHaveIOBitmap) +{ + NOREF(pVM); + + /* + * The TR hidden register is always valid. + */ + CPUMSELREGHID trHid; + RTSEL tr = CPUMGetGuestTR(pVCpu, &trHid); + if (!(tr & X86_SEL_MASK_OFF_RPL)) + return VERR_SELM_NO_TSS; + + *pGCPtrTss = trHid.u64Base; + *pcbTss = trHid.u32Limit + (trHid.u32Limit != UINT32_MAX); /* be careful. */ + if (pfCanHaveIOBitmap) + *pfCanHaveIOBitmap = trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL + || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY; + return VINF_SUCCESS; +} + diff --git a/src/VBox/VMM/VMMAll/TMAll.cpp b/src/VBox/VMM/VMMAll/TMAll.cpp new file mode 100644 index 00000000..677decdd --- /dev/null +++ b/src/VBox/VMM/VMMAll/TMAll.cpp @@ -0,0 +1,2850 @@ +/* $Id: TMAll.cpp $ */ +/** @file + * TM - Timeout Manager, all contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_TM +#ifdef DEBUG_bird +# define DBGFTRACE_DISABLED /* annoying */ +#endif +#include +#include +#include +#ifdef IN_RING3 +#endif +#include /* (for TMTIMER_GET_CRITSECT implementation) */ +#include "TMInternal.h" +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef IN_RING3 +# include +#endif + +#include "TMInline.h" + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +#ifdef VBOX_STRICT +/** @def TMTIMER_GET_CRITSECT + * Helper for safely resolving the critical section for a timer belonging to a + * device instance. + * @todo needs reworking later as it uses PDMDEVINSR0::pDevInsR0RemoveMe. */ +# ifdef IN_RING3 +# define TMTIMER_GET_CRITSECT(a_pVM, a_pTimer) ((a_pTimer)->pCritSect) +# else +# define TMTIMER_GET_CRITSECT(a_pVM, a_pTimer) tmRZTimerGetCritSect(a_pVM, a_pTimer) +# endif +#endif + +/** @def TMTIMER_ASSERT_CRITSECT + * Checks that the caller owns the critical section if one is associated with + * the timer. */ +#ifdef VBOX_STRICT +# define TMTIMER_ASSERT_CRITSECT(a_pVM, a_pTimer) \ + do { \ + if ((a_pTimer)->pCritSect) \ + { \ + VMSTATE enmState; \ + PPDMCRITSECT pCritSect = TMTIMER_GET_CRITSECT(a_pVM, a_pTimer); \ + AssertMsg( pCritSect \ + && ( PDMCritSectIsOwner((a_pVM), pCritSect) \ + || (enmState = (a_pVM)->enmVMState) == VMSTATE_CREATING \ + || enmState == VMSTATE_RESETTING \ + || enmState == VMSTATE_RESETTING_LS ),\ + ("pTimer=%p (%s) pCritSect=%p (%s)\n", a_pTimer, (a_pTimer)->szName, \ + (a_pTimer)->pCritSect, R3STRING(PDMR3CritSectName((a_pTimer)->pCritSect)) )); \ + } \ + } while (0) +#else +# define TMTIMER_ASSERT_CRITSECT(pVM, pTimer) do { } while (0) +#endif + +/** @def TMTIMER_ASSERT_SYNC_CRITSECT_ORDER + * Checks for lock order trouble between the timer critsect and the critical + * section critsect. The virtual sync critsect must always be entered before + * the one associated with the timer (see TMR3TimerQueuesDo). It is OK if there + * isn't any critical section associated with the timer or if the calling thread + * doesn't own it, ASSUMING of course that the thread using this macro is going + * to enter the virtual sync critical section anyway. + * + * @remarks This is a sligtly relaxed timer locking attitude compared to + * TMTIMER_ASSERT_CRITSECT, however, the calling device/whatever code + * should know what it's doing if it's stopping or starting a timer + * without taking the device lock. + */ +#ifdef VBOX_STRICT +# define TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer) \ + do { \ + if ((pTimer)->pCritSect) \ + { \ + VMSTATE enmState; \ + PPDMCRITSECT pCritSect = TMTIMER_GET_CRITSECT(pVM, pTimer); \ + AssertMsg( pCritSect \ + && ( !PDMCritSectIsOwner((pVM), pCritSect) \ + || PDMCritSectIsOwner((pVM), &(pVM)->tm.s.VirtualSyncLock) \ + || (enmState = (pVM)->enmVMState) == VMSTATE_CREATING \ + || enmState == VMSTATE_RESETTING \ + || enmState == VMSTATE_RESETTING_LS ),\ + ("pTimer=%p (%s) pCritSect=%p (%s)\n", pTimer, pTimer->szName, \ + (pTimer)->pCritSect, R3STRING(PDMR3CritSectName((pTimer)->pCritSect)) )); \ + } \ + } while (0) +#else +# define TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer) do { } while (0) +#endif + + +#if defined(VBOX_STRICT) && defined(IN_RING0) +/** + * Helper for TMTIMER_GET_CRITSECT + * @todo This needs a redo! + */ +DECLINLINE(PPDMCRITSECT) tmRZTimerGetCritSect(PVMCC pVM, PTMTIMER pTimer) +{ + if (pTimer->enmType == TMTIMERTYPE_DEV) + { + RTCCUINTREG fSavedFlags = ASMAddFlags(X86_EFL_AC); /** @todo fix ring-3 pointer use */ + PPDMDEVINSR0 pDevInsR0 = ((struct PDMDEVINSR3 *)pTimer->u.Dev.pDevIns)->pDevInsR0RemoveMe; /* !ring-3 read! */ + ASMSetFlags(fSavedFlags); + struct PDMDEVINSR3 *pDevInsR3 = pDevInsR0->pDevInsForR3R0; + if (pTimer->pCritSect == pDevInsR3->pCritSectRoR3) + return pDevInsR0->pCritSectRoR0; + uintptr_t offCritSect = (uintptr_t)pTimer->pCritSect - (uintptr_t)pDevInsR3->pvInstanceDataR3; + if (offCritSect < pDevInsR0->pReg->cbInstanceShared) + return (PPDMCRITSECT)((uintptr_t)pDevInsR0->pvInstanceDataR0 + offCritSect); + } + RT_NOREF(pVM); + Assert(pTimer->pCritSect == NULL); + return NULL; +} +#endif /* VBOX_STRICT && IN_RING0 */ + + +/** + * Notification that execution is about to start. + * + * This call must always be paired with a TMNotifyEndOfExecution call. + * + * The function may, depending on the configuration, resume the TSC and future + * clocks that only ticks when we're executing guest code. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(void) TMNotifyStartOfExecution(PVMCC pVM, PVMCPUCC pVCpu) +{ +#ifndef VBOX_WITHOUT_NS_ACCOUNTING + pVCpu->tm.s.uTscStartExecuting = SUPReadTsc(); + pVCpu->tm.s.fExecuting = true; +#endif + if (pVM->tm.s.fTSCTiedToExecution) + tmCpuTickResume(pVM, pVCpu); +} + + +/** + * Notification that execution has ended. + * + * This call must always be paired with a TMNotifyStartOfExecution call. + * + * The function may, depending on the configuration, suspend the TSC and future + * clocks that only ticks when we're executing guest code. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param uTsc TSC value when exiting guest context. + */ +VMMDECL(void) TMNotifyEndOfExecution(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uTsc) +{ + if (pVM->tm.s.fTSCTiedToExecution) + tmCpuTickPause(pVCpu); /** @todo use uTsc here if we can. */ + +#ifndef VBOX_WITHOUT_NS_ACCOUNTING + /* + * Calculate the elapsed tick count and convert it to nanoseconds. + */ +# ifdef IN_RING3 + PSUPGLOBALINFOPAGE const pGip = g_pSUPGlobalInfoPage; + uint64_t cTicks = uTsc - pVCpu->tm.s.uTscStartExecuting - SUPGetTscDelta(pGip); + uint64_t const uCpuHz = pGip ? SUPGetCpuHzFromGip(pGip) : pVM->tm.s.cTSCTicksPerSecondHost; +# else + uint64_t cTicks = uTsc - pVCpu->tm.s.uTscStartExecuting - SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet); + uint64_t const uCpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet); +# endif + AssertStmt(cTicks <= uCpuHz << 2, cTicks = uCpuHz << 2); /* max 4 sec */ + + uint64_t cNsExecutingDelta; + if (uCpuHz < _4G) + cNsExecutingDelta = ASMMultU64ByU32DivByU32(cTicks, RT_NS_1SEC, uCpuHz); + else if (uCpuHz < 16*_1G64) + cNsExecutingDelta = ASMMultU64ByU32DivByU32(cTicks >> 2, RT_NS_1SEC, uCpuHz >> 2); + else + { + Assert(uCpuHz < 64 * _1G64); + cNsExecutingDelta = ASMMultU64ByU32DivByU32(cTicks >> 4, RT_NS_1SEC, uCpuHz >> 4); + } + + /* + * Update the data. + * + * Note! We're not using strict memory ordering here to speed things us. + * The data is in a single cache line and this thread is the only + * one writing to that line, so I cannot quite imagine why we would + * need any strict ordering here. + */ + uint64_t const cNsExecutingNew = pVCpu->tm.s.cNsExecuting + cNsExecutingDelta; + uint32_t uGen = ASMAtomicUoIncU32(&pVCpu->tm.s.uTimesGen); Assert(uGen & 1); + ASMCompilerBarrier(); + pVCpu->tm.s.fExecuting = false; + pVCpu->tm.s.cNsExecuting = cNsExecutingNew; + pVCpu->tm.s.cPeriodsExecuting++; + ASMCompilerBarrier(); + ASMAtomicUoWriteU32(&pVCpu->tm.s.uTimesGen, (uGen | 1) + 1); + + /* + * Update stats. + */ +# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS) + STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecuting, cNsExecutingDelta); + if (cNsExecutingDelta < 5000) + STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecTiny, cNsExecutingDelta); + else if (cNsExecutingDelta < 50000) + STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecShort, cNsExecutingDelta); + else + STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecLong, cNsExecutingDelta); +# endif + + /* The timer triggers occational updating of the others and total stats: */ + if (RT_LIKELY(!pVCpu->tm.s.fUpdateStats)) + { /*likely*/ } + else + { + pVCpu->tm.s.fUpdateStats = false; + + uint64_t const cNsTotalNew = RTTimeNanoTS() - pVCpu->tm.s.nsStartTotal; + uint64_t const cNsOtherNew = cNsTotalNew - cNsExecutingNew - pVCpu->tm.s.cNsHalted; + +# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS) + STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsTotal, cNsTotalNew - pVCpu->tm.s.cNsTotalStat); + int64_t const cNsOtherNewDelta = cNsOtherNew - pVCpu->tm.s.cNsOtherStat; + if (cNsOtherNewDelta > 0) + STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsOther, (uint64_t)cNsOtherNewDelta); +# endif + + pVCpu->tm.s.cNsTotalStat = cNsTotalNew; + pVCpu->tm.s.cNsOtherStat = cNsOtherNew; + } + +#endif +} + + +/** + * Notification that the cpu is entering the halt state + * + * This call must always be paired with a TMNotifyEndOfExecution call. + * + * The function may, depending on the configuration, resume the TSC and future + * clocks that only ticks when we're halted. + * + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(void) TMNotifyStartOfHalt(PVMCPUCC pVCpu) +{ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + +#ifndef VBOX_WITHOUT_NS_ACCOUNTING + pVCpu->tm.s.nsStartHalting = RTTimeNanoTS(); + pVCpu->tm.s.fHalting = true; +#endif + + if ( pVM->tm.s.fTSCTiedToExecution + && !pVM->tm.s.fTSCNotTiedToHalt) + tmCpuTickResume(pVM, pVCpu); +} + + +/** + * Notification that the cpu is leaving the halt state + * + * This call must always be paired with a TMNotifyStartOfHalt call. + * + * The function may, depending on the configuration, suspend the TSC and future + * clocks that only ticks when we're halted. + * + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(void) TMNotifyEndOfHalt(PVMCPUCC pVCpu) +{ + PVM pVM = pVCpu->CTX_SUFF(pVM); + + if ( pVM->tm.s.fTSCTiedToExecution + && !pVM->tm.s.fTSCNotTiedToHalt) + tmCpuTickPause(pVCpu); + +#ifndef VBOX_WITHOUT_NS_ACCOUNTING + uint64_t const u64NsTs = RTTimeNanoTS(); + uint64_t const cNsTotalNew = u64NsTs - pVCpu->tm.s.nsStartTotal; + uint64_t const cNsHaltedDelta = u64NsTs - pVCpu->tm.s.nsStartHalting; + uint64_t const cNsHaltedNew = pVCpu->tm.s.cNsHalted + cNsHaltedDelta; + uint64_t const cNsOtherNew = cNsTotalNew - pVCpu->tm.s.cNsExecuting - cNsHaltedNew; + + uint32_t uGen = ASMAtomicUoIncU32(&pVCpu->tm.s.uTimesGen); Assert(uGen & 1); + ASMCompilerBarrier(); + pVCpu->tm.s.fHalting = false; + pVCpu->tm.s.fUpdateStats = false; + pVCpu->tm.s.cNsHalted = cNsHaltedNew; + pVCpu->tm.s.cPeriodsHalted++; + ASMCompilerBarrier(); + ASMAtomicUoWriteU32(&pVCpu->tm.s.uTimesGen, (uGen | 1) + 1); + +# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS) + STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsHalted, cNsHaltedDelta); + STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsTotal, cNsTotalNew - pVCpu->tm.s.cNsTotalStat); + int64_t const cNsOtherNewDelta = cNsOtherNew - pVCpu->tm.s.cNsOtherStat; + if (cNsOtherNewDelta > 0) + STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsOther, (uint64_t)cNsOtherNewDelta); +# endif + pVCpu->tm.s.cNsTotalStat = cNsTotalNew; + pVCpu->tm.s.cNsOtherStat = cNsOtherNew; +#endif +} + + +/** + * Raise the timer force action flag and notify the dedicated timer EMT. + * + * @param pVM The cross context VM structure. + */ +DECLINLINE(void) tmScheduleNotify(PVMCC pVM) +{ + VMCPUID idCpu = pVM->tm.s.idTimerCpu; + AssertReturnVoid(idCpu < pVM->cCpus); + PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, idCpu); + + if (!VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)) + { + Log5(("TMAll(%u): FF: 0 -> 1\n", __LINE__)); + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER); +#ifdef IN_RING3 + VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM); +#endif + STAM_COUNTER_INC(&pVM->tm.s.StatScheduleSetFF); + } +} + + +/** + * Schedule the queue which was changed. + */ +DECLINLINE(void) tmSchedule(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer) +{ + int rc = PDMCritSectTryEnter(pVM, &pQueue->TimerLock); + if (RT_SUCCESS_NP(rc)) + { + STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a); + Log3(("tmSchedule: tmTimerQueueSchedule\n")); + tmTimerQueueSchedule(pVM, pQueueCC, pQueue); +#ifdef VBOX_STRICT + tmTimerQueuesSanityChecks(pVM, "tmSchedule"); +#endif + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a); + PDMCritSectLeave(pVM, &pQueue->TimerLock); + return; + } + + TMTIMERSTATE enmState = pTimer->enmState; + if (TMTIMERSTATE_IS_PENDING_SCHEDULING(enmState)) + tmScheduleNotify(pVM); +} + + +/** + * Try change the state to enmStateNew from enmStateOld + * and link the timer into the scheduling queue. + * + * @returns Success indicator. + * @param pTimer Timer in question. + * @param enmStateNew The new timer state. + * @param enmStateOld The old timer state. + */ +DECLINLINE(bool) tmTimerTry(PTMTIMER pTimer, TMTIMERSTATE enmStateNew, TMTIMERSTATE enmStateOld) +{ + /* + * Attempt state change. + */ + bool fRc; + TM_TRY_SET_STATE(pTimer, enmStateNew, enmStateOld, fRc); + return fRc; +} + + +/** + * Links the timer onto the scheduling queue. + * + * @param pQueueCC The current context queue (same as @a pQueue for + * ring-3). + * @param pQueue The shared queue data. + * @param pTimer The timer. + * + * @todo FIXME: Look into potential race with the thread running the queues + * and stuff. + */ +DECLINLINE(void) tmTimerLinkSchedule(PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer) +{ + Assert(pTimer->idxScheduleNext == UINT32_MAX); + const uint32_t idxHeadNew = pTimer - &pQueueCC->paTimers[0]; + AssertReturnVoid(idxHeadNew < pQueueCC->cTimersAlloc); + + uint32_t idxHead; + do + { + idxHead = pQueue->idxSchedule; + Assert(idxHead == UINT32_MAX || idxHead < pQueueCC->cTimersAlloc); + pTimer->idxScheduleNext = idxHead; + } while (!ASMAtomicCmpXchgU32(&pQueue->idxSchedule, idxHeadNew, idxHead)); +} + + +/** + * Try change the state to enmStateNew from enmStateOld + * and link the timer into the scheduling queue. + * + * @returns Success indicator. + * @param pQueueCC The current context queue (same as @a pQueue for + * ring-3). + * @param pQueue The shared queue data. + * @param pTimer Timer in question. + * @param enmStateNew The new timer state. + * @param enmStateOld The old timer state. + */ +DECLINLINE(bool) tmTimerTryWithLink(PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer, + TMTIMERSTATE enmStateNew, TMTIMERSTATE enmStateOld) +{ + if (tmTimerTry(pTimer, enmStateNew, enmStateOld)) + { + tmTimerLinkSchedule(pQueueCC, pQueue, pTimer); + return true; + } + return false; +} + + +/** + * Links a timer into the active list of a timer queue. + * + * @param pVM The cross context VM structure. + * @param pQueueCC The current context queue (same as @a pQueue for + * ring-3). + * @param pQueue The shared queue data. + * @param pTimer The timer. + * @param u64Expire The timer expiration time. + * + * @remarks Called while owning the relevant queue lock. + */ +DECL_FORCE_INLINE(void) tmTimerQueueLinkActive(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, + PTMTIMER pTimer, uint64_t u64Expire) +{ + Assert(pTimer->idxNext == UINT32_MAX); + Assert(pTimer->idxPrev == UINT32_MAX); + Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE || pQueue->enmClock != TMCLOCK_VIRTUAL_SYNC); /* (active is not a stable state) */ + RT_NOREF(pVM); + + PTMTIMER pCur = tmTimerQueueGetHead(pQueueCC, pQueue); + if (pCur) + { + for (;; pCur = tmTimerGetNext(pQueueCC, pCur)) + { + if (pCur->u64Expire > u64Expire) + { + const PTMTIMER pPrev = tmTimerGetPrev(pQueueCC, pCur); + tmTimerSetNext(pQueueCC, pTimer, pCur); + tmTimerSetPrev(pQueueCC, pTimer, pPrev); + if (pPrev) + tmTimerSetNext(pQueueCC, pPrev, pTimer); + else + { + tmTimerQueueSetHead(pQueueCC, pQueue, pTimer); + ASMAtomicWriteU64(&pQueue->u64Expire, u64Expire); + DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerQueueLinkActive head", pTimer->szName); + } + tmTimerSetPrev(pQueueCC, pCur, pTimer); + return; + } + if (pCur->idxNext == UINT32_MAX) + { + tmTimerSetNext(pQueueCC, pCur, pTimer); + tmTimerSetPrev(pQueueCC, pTimer, pCur); + DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerQueueLinkActive tail", pTimer->szName); + return; + } + } + } + else + { + tmTimerQueueSetHead(pQueueCC, pQueue, pTimer); + ASMAtomicWriteU64(&pQueue->u64Expire, u64Expire); + DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerQueueLinkActive empty", pTimer->szName); + } +} + + + +/** + * Schedules the given timer on the given queue. + * + * @param pVM The cross context VM structure. + * @param pQueueCC The current context queue (same as @a pQueue for + * ring-3). + * @param pQueue The shared queue data. + * @param pTimer The timer that needs scheduling. + * + * @remarks Called while owning the lock. + */ +DECLINLINE(void) tmTimerQueueScheduleOne(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer) +{ + Assert(pQueue->enmClock != TMCLOCK_VIRTUAL_SYNC); + RT_NOREF(pVM); + + /* + * Processing. + */ + unsigned cRetries = 2; + do + { + TMTIMERSTATE enmState = pTimer->enmState; + switch (enmState) + { + /* + * Reschedule timer (in the active list). + */ + case TMTIMERSTATE_PENDING_RESCHEDULE: + if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE, TMTIMERSTATE_PENDING_RESCHEDULE))) + break; /* retry */ + tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer); + RT_FALL_THRU(); + + /* + * Schedule timer (insert into the active list). + */ + case TMTIMERSTATE_PENDING_SCHEDULE: + Assert(pTimer->idxNext == UINT32_MAX); Assert(pTimer->idxPrev == UINT32_MAX); + if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, TMTIMERSTATE_PENDING_SCHEDULE))) + break; /* retry */ + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, pTimer->u64Expire); + return; + + /* + * Stop the timer in active list. + */ + case TMTIMERSTATE_PENDING_STOP: + if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP_SCHEDULE, TMTIMERSTATE_PENDING_STOP))) + break; /* retry */ + tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer); + RT_FALL_THRU(); + + /* + * Stop the timer (not on the active list). + */ + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + Assert(pTimer->idxNext == UINT32_MAX); Assert(pTimer->idxPrev == UINT32_MAX); + if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_PENDING_STOP_SCHEDULE))) + break; + return; + + /* + * The timer is pending destruction by TMR3TimerDestroy, our caller. + * Nothing to do here. + */ + case TMTIMERSTATE_DESTROY: + break; + + /* + * Postpone these until they get into the right state. + */ + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + tmTimerLinkSchedule(pQueueCC, pQueue, pTimer); + STAM_COUNTER_INC(&pVM->tm.s.CTX_SUFF_Z(StatPostponed)); + return; + + /* + * None of these can be in the schedule. + */ + case TMTIMERSTATE_FREE: + case TMTIMERSTATE_STOPPED: + case TMTIMERSTATE_ACTIVE: + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_EXPIRED_DELIVER: + default: + AssertMsgFailed(("Timer (%p) in the scheduling list has an invalid state %s (%d)!", + pTimer, tmTimerState(pTimer->enmState), pTimer->enmState)); + return; + } + } while (cRetries-- > 0); +} + + +/** + * Schedules the specified timer queue. + * + * @param pVM The cross context VM structure. + * @param pQueueCC The current context queue (same as @a pQueue for + * ring-3) data of the queue to schedule. + * @param pQueue The shared queue data of the queue to schedule. + * + * @remarks Called while owning the lock. + */ +void tmTimerQueueSchedule(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue) +{ + Assert(PDMCritSectIsOwner(pVM, &pQueue->TimerLock)); + + /* + * Dequeue the scheduling list and iterate it. + */ + uint32_t idxNext = ASMAtomicXchgU32(&pQueue->idxSchedule, UINT32_MAX); + Log2(("tmTimerQueueSchedule: pQueue=%p:{.enmClock=%d, idxNext=%RI32, .u64Expired=%'RU64}\n", pQueue, pQueue->enmClock, idxNext, pQueue->u64Expire)); + while (idxNext != UINT32_MAX) + { + AssertBreak(idxNext < pQueueCC->cTimersAlloc); + + /* + * Unlink the head timer and take down the index of the next one. + */ + PTMTIMER pTimer = &pQueueCC->paTimers[idxNext]; + idxNext = pTimer->idxScheduleNext; + pTimer->idxScheduleNext = UINT32_MAX; + + /* + * Do the scheduling. + */ + Log2(("tmTimerQueueSchedule: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, .szName=%s}\n", + pTimer, tmTimerState(pTimer->enmState), pQueue->enmClock, pTimer->enmType, pTimer->szName)); + tmTimerQueueScheduleOne(pVM, pQueueCC, pQueue, pTimer); + Log2(("tmTimerQueueSchedule: %p: new %s\n", pTimer, tmTimerState(pTimer->enmState))); + } + Log2(("tmTimerQueueSchedule: u64Expired=%'RU64\n", pQueue->u64Expire)); +} + + +#ifdef VBOX_STRICT +/** + * Checks that the timer queues are sane. + * + * @param pVM The cross context VM structure. + * @param pszWhere Caller location clue. + */ +void tmTimerQueuesSanityChecks(PVMCC pVM, const char *pszWhere) +{ + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) + { + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[idxQueue]; + PTMTIMERQUEUECC const pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, idxQueue, pQueue); + Assert(pQueue->enmClock == (TMCLOCK)idxQueue); + + int rc = PDMCritSectTryEnter(pVM, &pQueue->TimerLock); + if (RT_SUCCESS(rc)) + { + if ( pQueue->enmClock != TMCLOCK_VIRTUAL_SYNC + || PDMCritSectTryEnter(pVM, &pVM->tm.s.VirtualSyncLock) == VINF_SUCCESS) + { + /* Check the linking of the active lists. */ + PTMTIMER pPrev = NULL; + for (PTMTIMER pCur = tmTimerQueueGetHead(pQueueCC, pQueue); + pCur; + pPrev = pCur, pCur = tmTimerGetNext(pQueueCC, pCur)) + { + AssertMsg(tmTimerGetPrev(pQueueCC, pCur) == pPrev, ("%s: %p != %p\n", pszWhere, tmTimerGetPrev(pQueueCC, pCur), pPrev)); + TMTIMERSTATE enmState = pCur->enmState; + switch (enmState) + { + case TMTIMERSTATE_ACTIVE: + AssertMsg( pCur->idxScheduleNext == UINT32_MAX + || pCur->enmState != TMTIMERSTATE_ACTIVE, + ("%s: %RI32\n", pszWhere, pCur->idxScheduleNext)); + break; + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + break; + default: + AssertMsgFailed(("%s: Invalid state enmState=%d %s\n", pszWhere, enmState, tmTimerState(enmState))); + break; + } + } + +# ifdef IN_RING3 + /* Go thru all the timers and check that the active ones all are in the active lists. */ + uint32_t idxTimer = pQueue->cTimersAlloc; + uint32_t cFree = 0; + while (idxTimer-- > 0) + { + PTMTIMER const pTimer = &pQueue->paTimers[idxTimer]; + TMTIMERSTATE const enmState = pTimer->enmState; + switch (enmState) + { + case TMTIMERSTATE_FREE: + cFree++; + break; + + case TMTIMERSTATE_ACTIVE: + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + { + PTMTIMERR3 pCurAct = tmTimerQueueGetHead(pQueueCC, pQueue); + Assert(pTimer->idxPrev != UINT32_MAX || pTimer == pCurAct); + while (pCurAct && pCurAct != pTimer) + pCurAct = tmTimerGetNext(pQueueCC, pCurAct); + Assert(pCurAct == pTimer); + break; + } + + case TMTIMERSTATE_PENDING_SCHEDULE: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + case TMTIMERSTATE_STOPPED: + case TMTIMERSTATE_EXPIRED_DELIVER: + { + Assert(pTimer->idxNext == UINT32_MAX); + Assert(pTimer->idxPrev == UINT32_MAX); + for (PTMTIMERR3 pCurAct = tmTimerQueueGetHead(pQueueCC, pQueue); + pCurAct; + pCurAct = tmTimerGetNext(pQueueCC, pCurAct)) + { + Assert(pCurAct != pTimer); + Assert(tmTimerGetNext(pQueueCC, pCurAct) != pTimer); + Assert(tmTimerGetPrev(pQueueCC, pCurAct) != pTimer); + } + break; + } + + /* ignore */ + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + break; + + case TMTIMERSTATE_INVALID: + Assert(idxTimer == 0); + break; + + /* shouldn't get here! */ + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_DESTROY: + default: + AssertMsgFailed(("Invalid state enmState=%d %s\n", enmState, tmTimerState(enmState))); + break; + } + + /* Check the handle value. */ + if (enmState > TMTIMERSTATE_INVALID && enmState < TMTIMERSTATE_DESTROY) + { + Assert((pTimer->hSelf & TMTIMERHANDLE_TIMER_IDX_MASK) == idxTimer); + Assert(((pTimer->hSelf >> TMTIMERHANDLE_QUEUE_IDX_SHIFT) & TMTIMERHANDLE_QUEUE_IDX_SMASK) == idxQueue); + } + } + Assert(cFree == pQueue->cTimersFree); +# endif /* IN_RING3 */ + + if (pQueue->enmClock == TMCLOCK_VIRTUAL_SYNC) + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); + } + PDMCritSectLeave(pVM, &pQueue->TimerLock); + } + } +} +#endif /* !VBOX_STRICT */ + +#ifdef VBOX_HIGH_RES_TIMERS_HACK + +/** + * Worker for tmTimerPollInternal that handles misses when the dedicated timer + * EMT is polling. + * + * @returns See tmTimerPollInternal. + * @param pVM The cross context VM structure. + * @param u64Now Current virtual clock timestamp. + * @param u64Delta The delta to the next even in ticks of the + * virtual clock. + * @param pu64Delta Where to return the delta. + */ +DECLINLINE(uint64_t) tmTimerPollReturnMiss(PVM pVM, uint64_t u64Now, uint64_t u64Delta, uint64_t *pu64Delta) +{ + Assert(!(u64Delta & RT_BIT_64(63))); + + if (!pVM->tm.s.fVirtualWarpDrive) + { + *pu64Delta = u64Delta; + return u64Delta + u64Now + pVM->tm.s.u64VirtualOffset; + } + + /* + * Warp drive adjustments - this is the reverse of what tmVirtualGetRaw is doing. + */ + uint64_t const u64Start = pVM->tm.s.u64VirtualWarpDriveStart; + uint32_t const u32Pct = pVM->tm.s.u32VirtualWarpDrivePercentage; + + uint64_t u64GipTime = u64Delta + u64Now + pVM->tm.s.u64VirtualOffset; + u64GipTime -= u64Start; /* the start is GIP time. */ + if (u64GipTime >= u64Delta) + { + ASMMultU64ByU32DivByU32(u64GipTime, 100, u32Pct); + ASMMultU64ByU32DivByU32(u64Delta, 100, u32Pct); + } + else + { + u64Delta -= u64GipTime; + ASMMultU64ByU32DivByU32(u64GipTime, 100, u32Pct); + u64Delta += u64GipTime; + } + *pu64Delta = u64Delta; + u64GipTime += u64Start; + return u64GipTime; +} + + +/** + * Worker for tmTimerPollInternal dealing with returns on virtual CPUs other + * than the one dedicated to timer work. + * + * @returns See tmTimerPollInternal. + * @param pVM The cross context VM structure. + * @param u64Now Current virtual clock timestamp. + * @param pu64Delta Where to return the delta. + */ +DECL_FORCE_INLINE(uint64_t) tmTimerPollReturnOtherCpu(PVM pVM, uint64_t u64Now, uint64_t *pu64Delta) +{ + static const uint64_t s_u64OtherRet = 500000000; /* 500 ms for non-timer EMTs. */ + *pu64Delta = s_u64OtherRet; + return u64Now + pVM->tm.s.u64VirtualOffset + s_u64OtherRet; +} + + +/** + * Worker for tmTimerPollInternal. + * + * @returns See tmTimerPollInternal. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pVCpuDst The cross context virtual CPU structure of the dedicated + * timer EMT. + * @param u64Now Current virtual clock timestamp. + * @param pu64Delta Where to return the delta. + * @param pCounter The statistics counter to update. + */ +DECL_FORCE_INLINE(uint64_t) tmTimerPollReturnHit(PVM pVM, PVMCPU pVCpu, PVMCPU pVCpuDst, uint64_t u64Now, + uint64_t *pu64Delta, PSTAMCOUNTER pCounter) +{ + STAM_COUNTER_INC(pCounter); NOREF(pCounter); + if (pVCpuDst != pVCpu) + return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta); + *pu64Delta = 0; + return 0; +} + + +/** + * Common worker for TMTimerPollGIP and TMTimerPoll. + * + * This function is called before FFs are checked in the inner execution EM loops. + * + * @returns The GIP timestamp of the next event. + * 0 if the next event has already expired. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pu64Delta Where to store the delta. + * + * @thread The emulation thread. + * + * @remarks GIP uses ns ticks. + */ +DECL_FORCE_INLINE(uint64_t) tmTimerPollInternal(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta) +{ + VMCPUID idCpu = pVM->tm.s.idTimerCpu; + AssertReturn(idCpu < pVM->cCpus, 0); + PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, idCpu); + + const uint64_t u64Now = TMVirtualGetNoCheck(pVM); + STAM_COUNTER_INC(&pVM->tm.s.StatPoll); + + /* + * Return straight away if the timer FF is already set ... + */ + if (VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)) + return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollAlreadySet); + + /* + * ... or if timers are being run. + */ + if (ASMAtomicReadBool(&pVM->tm.s.fRunningQueues)) + { + STAM_COUNTER_INC(&pVM->tm.s.StatPollRunning); + return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta); + } + + /* + * Check for TMCLOCK_VIRTUAL expiration. + */ + const uint64_t u64Expire1 = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire); + const int64_t i64Delta1 = u64Expire1 - u64Now; + if (i64Delta1 <= 0) + { + if (!VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)) + { + Log5(("TMAll(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))); + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER); + } + LogFlow(("TMTimerPoll: expire1=%'RU64 <= now=%'RU64\n", u64Expire1, u64Now)); + return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtual); + } + + /* + * Check for TMCLOCK_VIRTUAL_SYNC expiration. + * This isn't quite as straight forward if in a catch-up, not only do + * we have to adjust the 'now' but when have to adjust the delta as well. + */ + + /* + * Optimistic lockless approach. + */ + uint64_t u64VirtualSyncNow; + uint64_t u64Expire2 = ASMAtomicUoReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); + if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)) + { + if (!ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + { + u64VirtualSyncNow = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync); + if (RT_LIKELY( ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking) + && !ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp) + && u64VirtualSyncNow == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) + && u64Expire2 == ASMAtomicUoReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire))) + { + u64VirtualSyncNow = u64Now - u64VirtualSyncNow; + int64_t i64Delta2 = u64Expire2 - u64VirtualSyncNow; + if (i64Delta2 > 0) + { + STAM_COUNTER_INC(&pVM->tm.s.StatPollSimple); + STAM_COUNTER_INC(&pVM->tm.s.StatPollMiss); + + if (pVCpu == pVCpuDst) + return tmTimerPollReturnMiss(pVM, u64Now, RT_MIN(i64Delta1, i64Delta2), pu64Delta); + return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta); + } + + if ( !pVM->tm.s.fRunningQueues + && !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)) + { + Log5(("TMAll(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))); + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER); + } + + STAM_COUNTER_INC(&pVM->tm.s.StatPollSimple); + LogFlow(("TMTimerPoll: expire2=%'RU64 <= now=%'RU64\n", u64Expire2, u64Now)); + return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync); + } + } + } + else + { + STAM_COUNTER_INC(&pVM->tm.s.StatPollSimple); + LogFlow(("TMTimerPoll: stopped\n")); + return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync); + } + + /* + * Complicated lockless approach. + */ + uint64_t off; + uint32_t u32Pct = 0; + bool fCatchUp; + int cOuterTries = 42; + for (;; cOuterTries--) + { + fCatchUp = ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp); + off = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync); + u64Expire2 = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); + if (fCatchUp) + { + /* No changes allowed, try get a consistent set of parameters. */ + uint64_t const u64Prev = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev); + uint64_t const offGivenUp = ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp); + u32Pct = ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage); + if ( ( u64Prev == ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev) + && offGivenUp == ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp) + && u32Pct == ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage) + && off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) + && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire) + && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp) + && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking)) + || cOuterTries <= 0) + { + uint64_t u64Delta = u64Now - u64Prev; + if (RT_LIKELY(!(u64Delta >> 32))) + { + uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, u32Pct, 100); + if (off > u64Sub + offGivenUp) + off -= u64Sub; + else /* we've completely caught up. */ + off = offGivenUp; + } + else + /* More than 4 seconds since last time (or negative), ignore it. */ + Log(("TMVirtualGetSync: u64Delta=%RX64 (NoLock)\n", u64Delta)); + + /* Check that we're still running and in catch up. */ + if ( ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking) + && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + break; + } + } + else if ( off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) + && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire) + && !ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp) + && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking)) + break; /* Got an consistent offset */ + + /* Repeat the initial checks before iterating. */ + if (VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)) + return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollAlreadySet); + if (ASMAtomicUoReadBool(&pVM->tm.s.fRunningQueues)) + { + STAM_COUNTER_INC(&pVM->tm.s.StatPollRunning); + return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta); + } + if (!ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)) + { + LogFlow(("TMTimerPoll: stopped\n")); + return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync); + } + if (cOuterTries <= 0) + break; /* that's enough */ + } + if (cOuterTries <= 0) + STAM_COUNTER_INC(&pVM->tm.s.StatPollELoop); + u64VirtualSyncNow = u64Now - off; + + /* Calc delta and see if we've got a virtual sync hit. */ + int64_t i64Delta2 = u64Expire2 - u64VirtualSyncNow; + if (i64Delta2 <= 0) + { + if ( !pVM->tm.s.fRunningQueues + && !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)) + { + Log5(("TMAll(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))); + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER); + } + STAM_COUNTER_INC(&pVM->tm.s.StatPollVirtualSync); + LogFlow(("TMTimerPoll: expire2=%'RU64 <= now=%'RU64\n", u64Expire2, u64Now)); + return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync); + } + + /* + * Return the time left to the next event. + */ + STAM_COUNTER_INC(&pVM->tm.s.StatPollMiss); + if (pVCpu == pVCpuDst) + { + if (fCatchUp) + i64Delta2 = ASMMultU64ByU32DivByU32(i64Delta2, 100, u32Pct + 100); + return tmTimerPollReturnMiss(pVM, u64Now, RT_MIN(i64Delta1, i64Delta2), pu64Delta); + } + return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta); +} + + +/** + * Set FF if we've passed the next virtual event. + * + * This function is called before FFs are checked in the inner execution EM loops. + * + * @returns true if timers are pending, false if not. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread The emulation thread. + */ +VMMDECL(bool) TMTimerPollBool(PVMCC pVM, PVMCPUCC pVCpu) +{ + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + uint64_t off = 0; + tmTimerPollInternal(pVM, pVCpu, &off); + return off == 0; +} + + +/** + * Set FF if we've passed the next virtual event. + * + * This function is called before FFs are checked in the inner execution EM loops. + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @thread The emulation thread. + */ +VMM_INT_DECL(void) TMTimerPollVoid(PVMCC pVM, PVMCPUCC pVCpu) +{ + uint64_t off; + tmTimerPollInternal(pVM, pVCpu, &off); +} + + +/** + * Set FF if we've passed the next virtual event. + * + * This function is called before FFs are checked in the inner execution EM loops. + * + * @returns The GIP timestamp of the next event. + * 0 if the next event has already expired. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pu64Delta Where to store the delta. + * @thread The emulation thread. + */ +VMM_INT_DECL(uint64_t) TMTimerPollGIP(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta) +{ + return tmTimerPollInternal(pVM, pVCpu, pu64Delta); +} + +#endif /* VBOX_HIGH_RES_TIMERS_HACK */ + +/** + * Locks the timer clock. + * + * @returns VINF_SUCCESS on success, @a rcBusy if busy, and VERR_NOT_SUPPORTED + * if the clock does not have a lock. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param rcBusy What to return in ring-0 and raw-mode context if the + * lock is busy. Pass VINF_SUCCESS to acquired the + * critical section thru a ring-3 call if necessary. + * + * @remarks Currently only supported on timers using the virtual sync clock. + */ +VMMDECL(int) TMTimerLock(PVMCC pVM, TMTIMERHANDLE hTimer, int rcBusy) +{ + TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + AssertReturn(idxQueue == TMCLOCK_VIRTUAL_SYNC, VERR_NOT_SUPPORTED); + return PDMCritSectEnter(pVM, &pVM->tm.s.VirtualSyncLock, rcBusy); +} + + +/** + * Unlocks a timer clock locked by TMTimerLock. + * + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(void) TMTimerUnlock(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_VOID(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + AssertReturnVoid(idxQueue == TMCLOCK_VIRTUAL_SYNC); + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); +} + + +/** + * Checks if the current thread owns the timer clock lock. + * + * @returns @c true if its the owner, @c false if not. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(bool) TMTimerIsLockOwner(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, false); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + AssertReturn(idxQueue == TMCLOCK_VIRTUAL_SYNC, false); + return PDMCritSectIsOwner(pVM, &pVM->tm.s.VirtualSyncLock); +} + + +/** + * Optimized TMTimerSet code path for starting an inactive timer. + * + * @returns VBox status code. + * + * @param pVM The cross context VM structure. + * @param pTimer The timer handle. + * @param u64Expire The new expire time. + * @param pQueue Pointer to the shared timer queue data. + * @param idxQueue The queue index. + */ +static int tmTimerSetOptimizedStart(PVMCC pVM, PTMTIMER pTimer, uint64_t u64Expire, PTMTIMERQUEUE pQueue, uint32_t idxQueue) +{ + Assert(pTimer->idxPrev == UINT32_MAX); + Assert(pTimer->idxNext == UINT32_MAX); + Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE); + + /* + * Calculate and set the expiration time. + */ + if (idxQueue == TMCLOCK_VIRTUAL_SYNC) + { + uint64_t u64Last = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSync); + AssertMsgStmt(u64Expire >= u64Last, + ("exp=%#llx last=%#llx\n", u64Expire, u64Last), + u64Expire = u64Last); + } + ASMAtomicWriteU64(&pTimer->u64Expire, u64Expire); + Log2(("tmTimerSetOptimizedStart: %p:{.pszDesc='%s', .u64Expire=%'RU64}\n", pTimer, pTimer->szName, u64Expire)); + + /* + * Link the timer into the active list. + */ + tmTimerQueueLinkActive(pVM, TM_GET_TIMER_QUEUE_CC(pVM, idxQueue, pQueue), pQueue, pTimer, u64Expire); + + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetOpt); + return VINF_SUCCESS; +} + + +/** + * TMTimerSet for the virtual sync timer queue. + * + * This employs a greatly simplified state machine by always acquiring the + * queue lock and bypassing the scheduling list. + * + * @returns VBox status code + * @param pVM The cross context VM structure. + * @param pTimer The timer handle. + * @param u64Expire The expiration time. + */ +static int tmTimerVirtualSyncSet(PVMCC pVM, PTMTIMER pTimer, uint64_t u64Expire) +{ + STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetVs), a); + VM_ASSERT_EMT(pVM); + TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer); + int rc = PDMCritSectEnter(pVM, &pVM->tm.s.VirtualSyncLock, VINF_SUCCESS); + AssertRCReturn(rc, rc); + + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC]; + PTMTIMERQUEUECC const pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, TMCLOCK_VIRTUAL_SYNC, pQueue); + TMTIMERSTATE const enmState = pTimer->enmState; + switch (enmState) + { + case TMTIMERSTATE_EXPIRED_DELIVER: + case TMTIMERSTATE_STOPPED: + if (enmState == TMTIMERSTATE_EXPIRED_DELIVER) + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStExpDeliver); + else + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStStopped); + + AssertMsg(u64Expire >= pVM->tm.s.u64VirtualSync, + ("%'RU64 < %'RU64 %s\n", u64Expire, pVM->tm.s.u64VirtualSync, pTimer->szName)); + pTimer->u64Expire = u64Expire; + TM_SET_STATE(pTimer, TMTIMERSTATE_ACTIVE); + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire); + rc = VINF_SUCCESS; + break; + + case TMTIMERSTATE_ACTIVE: + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStActive); + tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer); + pTimer->u64Expire = u64Expire; + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire); + rc = VINF_SUCCESS; + break; + + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_SCHEDULE: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_DESTROY: + case TMTIMERSTATE_FREE: + AssertLogRelMsgFailed(("Invalid timer state %s: %s\n", tmTimerState(enmState), pTimer->szName)); + rc = VERR_TM_INVALID_STATE; + break; + + default: + AssertMsgFailed(("Unknown timer state %d: %s\n", enmState, pTimer->szName)); + rc = VERR_TM_UNKNOWN_STATE; + break; + } + + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetVs), a); + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); + return rc; +} + + +/** + * Arm a timer with a (new) expire time. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param u64Expire New expire time. + */ +VMMDECL(int) TMTimerSet(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t u64Expire) +{ + TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + STAM_COUNTER_INC(&pTimer->StatSetAbsolute); + + /* Treat virtual sync timers specially. */ + if (idxQueue == TMCLOCK_VIRTUAL_SYNC) + return tmTimerVirtualSyncSet(pVM, pTimer, u64Expire); + + STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a); + TMTIMER_ASSERT_CRITSECT(pVM, pTimer); + + DBGFTRACE_U64_TAG2(pVM, u64Expire, "TMTimerSet", pTimer->szName); + +#ifdef VBOX_WITH_STATISTICS + /* + * Gather optimization info. + */ + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSet); + TMTIMERSTATE enmOrgState = pTimer->enmState; + switch (enmOrgState) + { + case TMTIMERSTATE_STOPPED: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStStopped); break; + case TMTIMERSTATE_EXPIRED_DELIVER: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStExpDeliver); break; + case TMTIMERSTATE_ACTIVE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStActive); break; + case TMTIMERSTATE_PENDING_STOP: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendStop); break; + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendStopSched); break; + case TMTIMERSTATE_PENDING_SCHEDULE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendSched); break; + case TMTIMERSTATE_PENDING_RESCHEDULE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendResched); break; + default: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStOther); break; + } +#endif + +#if 1 + /* + * The most common case is setting the timer again during the callback. + * The second most common case is starting a timer at some other time. + */ + TMTIMERSTATE enmState1 = pTimer->enmState; + if ( enmState1 == TMTIMERSTATE_EXPIRED_DELIVER + || ( enmState1 == TMTIMERSTATE_STOPPED + && pTimer->pCritSect)) + { + /* Try take the TM lock and check the state again. */ + int rc = PDMCritSectTryEnter(pVM, &pQueue->TimerLock); + if (RT_SUCCESS_NP(rc)) + { + if (RT_LIKELY(tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, enmState1))) + { + tmTimerSetOptimizedStart(pVM, pTimer, u64Expire, pQueue, idxQueue); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a); + PDMCritSectLeave(pVM, &pQueue->TimerLock); + return VINF_SUCCESS; + } + PDMCritSectLeave(pVM, &pQueue->TimerLock); + } + } +#endif + + /* + * Unoptimized code path. + */ + int cRetries = 1000; + do + { + /* + * Change to any of the SET_EXPIRE states if valid and then to SCHEDULE or RESCHEDULE. + */ + TMTIMERSTATE enmState = pTimer->enmState; + Log2(("TMTimerSet: %p:{.enmState=%s, .pszDesc='%s'} cRetries=%d u64Expire=%'RU64\n", + pTimer, tmTimerState(enmState), pTimer->szName, cRetries, u64Expire)); + switch (enmState) + { + case TMTIMERSTATE_EXPIRED_DELIVER: + case TMTIMERSTATE_STOPPED: + if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState)) + { + Assert(pTimer->idxPrev == UINT32_MAX); + Assert(pTimer->idxNext == UINT32_MAX); + pTimer->u64Expire = u64Expire; + TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE); + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a); + return VINF_SUCCESS; + } + break; + + case TMTIMERSTATE_PENDING_SCHEDULE: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState)) + { + pTimer->u64Expire = u64Expire; + TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE); + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a); + return VINF_SUCCESS; + } + break; + + + case TMTIMERSTATE_ACTIVE: + if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState)) + { + pTimer->u64Expire = u64Expire; + TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE); + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a); + return VINF_SUCCESS; + } + break; + + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_STOP: + if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState)) + { + pTimer->u64Expire = u64Expire; + TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE); + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a); + return VINF_SUCCESS; + } + break; + + + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: +#ifdef IN_RING3 + if (!RTThreadYield()) + RTThreadSleep(1); +#else +/** @todo call host context and yield after a couple of iterations */ +#endif + break; + + /* + * Invalid states. + */ + case TMTIMERSTATE_DESTROY: + case TMTIMERSTATE_FREE: + AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, pTimer->szName)); + return VERR_TM_INVALID_STATE; + default: + AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName)); + return VERR_TM_UNKNOWN_STATE; + } + } while (cRetries-- > 0); + + AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName)); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a); + return VERR_TM_TIMER_UNSTABLE_STATE; +} + + +/** + * Return the current time for the specified clock, setting pu64Now if not NULL. + * + * @returns Current time. + * @param pVM The cross context VM structure. + * @param enmClock The clock to query. + * @param pu64Now Optional pointer where to store the return time + */ +DECL_FORCE_INLINE(uint64_t) tmTimerSetRelativeNowWorker(PVMCC pVM, TMCLOCK enmClock, uint64_t *pu64Now) +{ + uint64_t u64Now; + switch (enmClock) + { + case TMCLOCK_VIRTUAL_SYNC: + u64Now = TMVirtualSyncGet(pVM); + break; + case TMCLOCK_VIRTUAL: + u64Now = TMVirtualGet(pVM); + break; + case TMCLOCK_REAL: + u64Now = TMRealGet(pVM); + break; + default: + AssertFatalMsgFailed(("%d\n", enmClock)); + } + + if (pu64Now) + *pu64Now = u64Now; + return u64Now; +} + + +/** + * Optimized TMTimerSetRelative code path. + * + * @returns VBox status code. + * + * @param pVM The cross context VM structure. + * @param pTimer The timer handle. + * @param cTicksToNext Clock ticks until the next time expiration. + * @param pu64Now Where to return the current time stamp used. + * Optional. + * @param pQueueCC The context specific queue data (same as @a pQueue + * for ring-3). + * @param pQueue The shared queue data. + */ +static int tmTimerSetRelativeOptimizedStart(PVMCC pVM, PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now, + PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue) +{ + Assert(pTimer->idxPrev == UINT32_MAX); + Assert(pTimer->idxNext == UINT32_MAX); + Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE); + + /* + * Calculate and set the expiration time. + */ + uint64_t const u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now); + pTimer->u64Expire = u64Expire; + Log2(("tmTimerSetRelativeOptimizedStart: %p:{.pszDesc='%s', .u64Expire=%'RU64} cTicksToNext=%'RU64\n", pTimer, pTimer->szName, u64Expire, cTicksToNext)); + + /* + * Link the timer into the active list. + */ + DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerSetRelativeOptimizedStart", pTimer->szName); + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire); + + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeOpt); + return VINF_SUCCESS; +} + + +/** + * TMTimerSetRelative for the virtual sync timer queue. + * + * This employs a greatly simplified state machine by always acquiring the + * queue lock and bypassing the scheduling list. + * + * @returns VBox status code + * @param pVM The cross context VM structure. + * @param pTimer The timer to (re-)arm. + * @param cTicksToNext Clock ticks until the next time expiration. + * @param pu64Now Where to return the current time stamp used. + * Optional. + */ +static int tmTimerVirtualSyncSetRelative(PVMCC pVM, PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now) +{ + STAM_PROFILE_START(pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelativeVs), a); + VM_ASSERT_EMT(pVM); + TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer); + int rc = PDMCritSectEnter(pVM, &pVM->tm.s.VirtualSyncLock, VINF_SUCCESS); + AssertRCReturn(rc, rc); + + /* Calculate the expiration tick. */ + uint64_t u64Expire = TMVirtualSyncGetNoCheck(pVM); + if (pu64Now) + *pu64Now = u64Expire; + u64Expire += cTicksToNext; + + /* Update the timer. */ + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC]; + PTMTIMERQUEUECC const pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, TMCLOCK_VIRTUAL_SYNC, pQueue); + TMTIMERSTATE const enmState = pTimer->enmState; + switch (enmState) + { + case TMTIMERSTATE_EXPIRED_DELIVER: + case TMTIMERSTATE_STOPPED: + if (enmState == TMTIMERSTATE_EXPIRED_DELIVER) + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStExpDeliver); + else + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStStopped); + pTimer->u64Expire = u64Expire; + TM_SET_STATE(pTimer, TMTIMERSTATE_ACTIVE); + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire); + rc = VINF_SUCCESS; + break; + + case TMTIMERSTATE_ACTIVE: + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStActive); + tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer); + pTimer->u64Expire = u64Expire; + tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire); + rc = VINF_SUCCESS; + break; + + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_SCHEDULE: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_DESTROY: + case TMTIMERSTATE_FREE: + AssertLogRelMsgFailed(("Invalid timer state %s: %s\n", tmTimerState(enmState), pTimer->szName)); + rc = VERR_TM_INVALID_STATE; + break; + + default: + AssertMsgFailed(("Unknown timer state %d: %s\n", enmState, pTimer->szName)); + rc = VERR_TM_UNKNOWN_STATE; + break; + } + + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelativeVs), a); + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); + return rc; +} + + +/** + * Arm a timer with a expire time relative to the current time. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pTimer The timer to arm. + * @param cTicksToNext Clock ticks until the next time expiration. + * @param pu64Now Where to return the current time stamp used. + * Optional. + * @param pQueueCC The context specific queue data (same as @a pQueue + * for ring-3). + * @param pQueue The shared queue data. + */ +static int tmTimerSetRelative(PVMCC pVM, PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now, + PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue) +{ + STAM_COUNTER_INC(&pTimer->StatSetRelative); + + /* Treat virtual sync timers specially. */ + if (pQueue->enmClock == TMCLOCK_VIRTUAL_SYNC) + return tmTimerVirtualSyncSetRelative(pVM, pTimer, cTicksToNext, pu64Now); + + STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelative), a); + TMTIMER_ASSERT_CRITSECT(pVM, pTimer); + + DBGFTRACE_U64_TAG2(pVM, cTicksToNext, "TMTimerSetRelative", pTimer->szName); + +#ifdef VBOX_WITH_STATISTICS + /* + * Gather optimization info. + */ + STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelative); + TMTIMERSTATE enmOrgState = pTimer->enmState; + switch (enmOrgState) + { + case TMTIMERSTATE_STOPPED: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStStopped); break; + case TMTIMERSTATE_EXPIRED_DELIVER: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStExpDeliver); break; + case TMTIMERSTATE_ACTIVE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStActive); break; + case TMTIMERSTATE_PENDING_STOP: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendStop); break; + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendStopSched); break; + case TMTIMERSTATE_PENDING_SCHEDULE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendSched); break; + case TMTIMERSTATE_PENDING_RESCHEDULE: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendResched); break; + default: STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStOther); break; + } +#endif + + /* + * Try to take the TM lock and optimize the common cases. + * + * With the TM lock we can safely make optimizations like immediate + * scheduling and we can also be 100% sure that we're not racing the + * running of the timer queues. As an additional restraint we require the + * timer to have a critical section associated with to be 100% there aren't + * concurrent operations on the timer. (This latter isn't necessary any + * longer as this isn't supported for any timers, critsect or not.) + * + * Note! Lock ordering doesn't apply when we only _try_ to + * get the innermost locks. + */ + bool fOwnTMLock = RT_SUCCESS_NP(PDMCritSectTryEnter(pVM, &pQueue->TimerLock)); +#if 1 + if ( fOwnTMLock + && pTimer->pCritSect) + { + TMTIMERSTATE enmState = pTimer->enmState; + if (RT_LIKELY( ( enmState == TMTIMERSTATE_EXPIRED_DELIVER + || enmState == TMTIMERSTATE_STOPPED) + && tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, enmState))) + { + tmTimerSetRelativeOptimizedStart(pVM, pTimer, cTicksToNext, pu64Now, pQueueCC, pQueue); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelative), a); + PDMCritSectLeave(pVM, &pQueue->TimerLock); + return VINF_SUCCESS; + } + + /* Optimize other states when it becomes necessary. */ + } +#endif + + /* + * Unoptimized path. + */ + int rc; + for (int cRetries = 1000; ; cRetries--) + { + /* + * Change to any of the SET_EXPIRE states if valid and then to SCHEDULE or RESCHEDULE. + */ + TMTIMERSTATE enmState = pTimer->enmState; + switch (enmState) + { + case TMTIMERSTATE_STOPPED: + if (pQueue->enmClock == TMCLOCK_VIRTUAL_SYNC) + { + /** @todo To fix assertion in tmR3TimerQueueRunVirtualSync: + * Figure a safe way of activating this timer while the queue is + * being run. + * (99.9% sure this that the assertion is caused by DevAPIC.cpp + * re-starting the timer in response to a initial_count write.) */ + } + RT_FALL_THRU(); + case TMTIMERSTATE_EXPIRED_DELIVER: + if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState)) + { + Assert(pTimer->idxPrev == UINT32_MAX); + Assert(pTimer->idxNext == UINT32_MAX); + pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now); + Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [EXP/STOP]\n", + pTimer, tmTimerState(enmState), pTimer->szName, pTimer->u64Expire, cRetries)); + TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE); + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + rc = VINF_SUCCESS; + break; + } + rc = VERR_TRY_AGAIN; + break; + + case TMTIMERSTATE_PENDING_SCHEDULE: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState)) + { + pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now); + Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [PEND_SCHED]\n", + pTimer, tmTimerState(enmState), pTimer->szName, pTimer->u64Expire, cRetries)); + TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE); + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + rc = VINF_SUCCESS; + break; + } + rc = VERR_TRY_AGAIN; + break; + + + case TMTIMERSTATE_ACTIVE: + if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState)) + { + pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now); + Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [ACTIVE]\n", + pTimer, tmTimerState(enmState), pTimer->szName, pTimer->u64Expire, cRetries)); + TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE); + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + rc = VINF_SUCCESS; + break; + } + rc = VERR_TRY_AGAIN; + break; + + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_STOP: + if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState)) + { + pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now); + Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [PEND_RESCH/STOP]\n", + pTimer, tmTimerState(enmState), pTimer->szName, pTimer->u64Expire, cRetries)); + TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE); + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + rc = VINF_SUCCESS; + break; + } + rc = VERR_TRY_AGAIN; + break; + + + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: +#ifdef IN_RING3 + if (!RTThreadYield()) + RTThreadSleep(1); +#else +/** @todo call host context and yield after a couple of iterations */ +#endif + rc = VERR_TRY_AGAIN; + break; + + /* + * Invalid states. + */ + case TMTIMERSTATE_DESTROY: + case TMTIMERSTATE_FREE: + AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, pTimer->szName)); + rc = VERR_TM_INVALID_STATE; + break; + + default: + AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName)); + rc = VERR_TM_UNKNOWN_STATE; + break; + } + + /* switch + loop is tedious to break out of. */ + if (rc == VINF_SUCCESS) + break; + + if (rc != VERR_TRY_AGAIN) + { + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now); + break; + } + if (cRetries <= 0) + { + AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName)); + rc = VERR_TM_TIMER_UNSTABLE_STATE; + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now); + break; + } + + /* + * Retry to gain locks. + */ + if (!fOwnTMLock) + fOwnTMLock = RT_SUCCESS_NP(PDMCritSectTryEnter(pVM, &pQueue->TimerLock)); + + } /* for (;;) */ + + /* + * Clean up and return. + */ + if (fOwnTMLock) + PDMCritSectLeave(pVM, &pQueue->TimerLock); + + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelative), a); + return rc; +} + + +/** + * Arm a timer with a expire time relative to the current time. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cTicksToNext Clock ticks until the next time expiration. + * @param pu64Now Where to return the current time stamp used. + * Optional. + */ +VMMDECL(int) TMTimerSetRelative(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cTicksToNext, uint64_t *pu64Now) +{ + TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + return tmTimerSetRelative(pVM, pTimer, cTicksToNext, pu64Now, pQueueCC, pQueue); +} + + +/** + * Drops a hint about the frequency of the timer. + * + * This is used by TM and the VMM to calculate how often guest execution needs + * to be interrupted. The hint is automatically cleared by TMTimerStop. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param uHzHint The frequency hint. Pass 0 to clear the hint. + * + * @remarks We're using an integer hertz value here since anything above 1 HZ + * is not going to be any trouble satisfying scheduling wise. The + * range where it makes sense is >= 100 HZ. + */ +VMMDECL(int) TMTimerSetFrequencyHint(PVMCC pVM, TMTIMERHANDLE hTimer, uint32_t uHzHint) +{ + TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + TMTIMER_ASSERT_CRITSECT(pVM, pTimer); + + uint32_t const uHzOldHint = pTimer->uHzHint; + pTimer->uHzHint = uHzHint; + + uint32_t const uMaxHzHint = pQueue->uMaxHzHint; + if ( uHzHint > uMaxHzHint + || uHzOldHint >= uMaxHzHint) + ASMAtomicOrU64(&pVM->tm.s.HzHint.u64Combined, RT_BIT_32(idxQueue) | RT_BIT_32(idxQueue + 16)); + + return VINF_SUCCESS; +} + + +/** + * TMTimerStop for the virtual sync timer queue. + * + * This employs a greatly simplified state machine by always acquiring the + * queue lock and bypassing the scheduling list. + * + * @returns VBox status code + * @param pVM The cross context VM structure. + * @param pTimer The timer handle. + */ +static int tmTimerVirtualSyncStop(PVMCC pVM, PTMTIMER pTimer) +{ + STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerStopVs), a); + VM_ASSERT_EMT(pVM); + TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer); + int rc = PDMCritSectEnter(pVM, &pVM->tm.s.VirtualSyncLock, VINF_SUCCESS); + AssertRCReturn(rc, rc); + + /* Reset the HZ hint. */ + uint32_t uOldHzHint = pTimer->uHzHint; + if (uOldHzHint) + { + if (uOldHzHint >= pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].uMaxHzHint) + ASMAtomicOrU64(&pVM->tm.s.HzHint.u64Combined, RT_BIT_32(TMCLOCK_VIRTUAL_SYNC) | RT_BIT_32(TMCLOCK_VIRTUAL_SYNC + 16)); + pTimer->uHzHint = 0; + } + + /* Update the timer state. */ + TMTIMERSTATE const enmState = pTimer->enmState; + switch (enmState) + { + case TMTIMERSTATE_ACTIVE: + { + PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC]; + tmTimerQueueUnlinkActive(pVM, TM_GET_TIMER_QUEUE_CC(pVM, TMCLOCK_VIRTUAL_SYNC, pQueue), pQueue, pTimer); + TM_SET_STATE(pTimer, TMTIMERSTATE_STOPPED); + rc = VINF_SUCCESS; + break; + } + + case TMTIMERSTATE_EXPIRED_DELIVER: + TM_SET_STATE(pTimer, TMTIMERSTATE_STOPPED); + rc = VINF_SUCCESS; + break; + + case TMTIMERSTATE_STOPPED: + rc = VINF_SUCCESS; + break; + + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_SCHEDULE: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_DESTROY: + case TMTIMERSTATE_FREE: + AssertLogRelMsgFailed(("Invalid timer state %s: %s\n", tmTimerState(enmState), pTimer->szName)); + rc = VERR_TM_INVALID_STATE; + break; + + default: + AssertMsgFailed(("Unknown timer state %d: %s\n", enmState, pTimer->szName)); + rc = VERR_TM_UNKNOWN_STATE; + break; + } + + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStopVs), a); + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); + return rc; +} + + +/** + * Stop the timer. + * Use TMR3TimerArm() to "un-stop" the timer. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(int) TMTimerStop(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + STAM_COUNTER_INC(&pTimer->StatStop); + + /* Treat virtual sync timers specially. */ + if (idxQueue == TMCLOCK_VIRTUAL_SYNC) + return tmTimerVirtualSyncStop(pVM, pTimer); + + STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a); + TMTIMER_ASSERT_CRITSECT(pVM, pTimer); + + /* + * Reset the HZ hint. + */ + uint32_t const uOldHzHint = pTimer->uHzHint; + if (uOldHzHint) + { + if (uOldHzHint >= pQueue->uMaxHzHint) + ASMAtomicOrU64(&pVM->tm.s.HzHint.u64Combined, RT_BIT_32(idxQueue) | RT_BIT_32(idxQueue + 16)); + pTimer->uHzHint = 0; + } + + /** @todo see if this function needs optimizing. */ + int cRetries = 1000; + do + { + /* + * Change to any of the SET_EXPIRE states if valid and then to SCHEDULE or RESCHEDULE. + */ + TMTIMERSTATE enmState = pTimer->enmState; + Log2(("TMTimerStop: %p:{.enmState=%s, .pszDesc='%s'} cRetries=%d\n", + pTimer, tmTimerState(enmState), pTimer->szName, cRetries)); + switch (enmState) + { + case TMTIMERSTATE_EXPIRED_DELIVER: + //AssertMsgFailed(("You don't stop an expired timer dude!\n")); + return VERR_INVALID_PARAMETER; + + case TMTIMERSTATE_STOPPED: + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a); + return VINF_SUCCESS; + + case TMTIMERSTATE_PENDING_SCHEDULE: + if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP_SCHEDULE, enmState)) + { + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a); + return VINF_SUCCESS; + } + break; + + case TMTIMERSTATE_PENDING_RESCHEDULE: + if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP, enmState)) + { + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a); + return VINF_SUCCESS; + } + break; + + case TMTIMERSTATE_ACTIVE: + if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_STOP, enmState)) + { + tmSchedule(pVM, pQueueCC, pQueue, pTimer); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a); + return VINF_SUCCESS; + } + break; + + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: +#ifdef IN_RING3 + if (!RTThreadYield()) + RTThreadSleep(1); +#else +/** @todo call host and yield cpu after a while. */ +#endif + break; + + /* + * Invalid states. + */ + case TMTIMERSTATE_DESTROY: + case TMTIMERSTATE_FREE: + AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, pTimer->szName)); + return VERR_TM_INVALID_STATE; + default: + AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName)); + return VERR_TM_UNKNOWN_STATE; + } + } while (cRetries-- > 0); + + AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName)); + STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a); + return VERR_TM_TIMER_UNSTABLE_STATE; +} + + +/** + * Get the current clock time. + * Handy for calculating the new expire time. + * + * @returns Current clock time. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(uint64_t) TMTimerGet(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + STAM_COUNTER_INC(&pTimer->StatGet); + + uint64_t u64; + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + u64 = TMVirtualGet(pVM); + break; + case TMCLOCK_VIRTUAL_SYNC: + u64 = TMVirtualSyncGet(pVM); + break; + case TMCLOCK_REAL: + u64 = TMRealGet(pVM); + break; + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return UINT64_MAX; + } + //Log2(("TMTimerGet: returns %'RU64 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n", + // u64, pTimer, tmTimerState(pTimer->enmState), pTimer->szName)); + return u64; +} + + +/** + * Get the frequency of the timer clock. + * + * @returns Clock frequency (as Hz of course). + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(uint64_t) TMTimerGetFreq(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + case TMCLOCK_VIRTUAL_SYNC: + return TMCLOCK_FREQ_VIRTUAL; + + case TMCLOCK_REAL: + return TMCLOCK_FREQ_REAL; + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return 0; + } +} + + +/** + * Get the expire time of the timer. + * Only valid for active timers. + * + * @returns Expire time of the timer. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(uint64_t) TMTimerGetExpire(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, UINT64_MAX); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + TMTIMER_ASSERT_CRITSECT(pVM, pTimer); + int cRetries = 1000; + do + { + TMTIMERSTATE enmState = pTimer->enmState; + switch (enmState) + { + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_EXPIRED_DELIVER: + case TMTIMERSTATE_STOPPED: + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + Log2(("TMTimerGetExpire: returns ~0 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n", + pTimer, tmTimerState(pTimer->enmState), pTimer->szName)); + return UINT64_MAX; + + case TMTIMERSTATE_ACTIVE: + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_SCHEDULE: + Log2(("TMTimerGetExpire: returns %'RU64 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n", + pTimer->u64Expire, pTimer, tmTimerState(pTimer->enmState), pTimer->szName)); + return pTimer->u64Expire; + + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: +#ifdef IN_RING3 + if (!RTThreadYield()) + RTThreadSleep(1); +#endif + break; + + /* + * Invalid states. + */ + case TMTIMERSTATE_DESTROY: + case TMTIMERSTATE_FREE: + AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, pTimer->szName)); + Log2(("TMTimerGetExpire: returns ~0 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n", + pTimer, tmTimerState(pTimer->enmState), pTimer->szName)); + return UINT64_MAX; + default: + AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName)); + return UINT64_MAX; + } + } while (cRetries-- > 0); + + AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName)); + Log2(("TMTimerGetExpire: returns ~0 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n", + pTimer, tmTimerState(pTimer->enmState), pTimer->szName)); + return UINT64_MAX; +} + + +/** + * Checks if a timer is active or not. + * + * @returns True if active. + * @returns False if not active. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(bool) TMTimerIsActive(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, false); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + TMTIMERSTATE enmState = pTimer->enmState; + switch (enmState) + { + case TMTIMERSTATE_STOPPED: + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_EXPIRED_DELIVER: + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + Log2(("TMTimerIsActive: returns false (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n", + pTimer, tmTimerState(pTimer->enmState), pTimer->szName)); + return false; + + case TMTIMERSTATE_ACTIVE: + case TMTIMERSTATE_PENDING_RESCHEDULE: + case TMTIMERSTATE_PENDING_SCHEDULE: + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + Log2(("TMTimerIsActive: returns true (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n", + pTimer, tmTimerState(pTimer->enmState), pTimer->szName)); + return true; + + /* + * Invalid states. + */ + case TMTIMERSTATE_DESTROY: + case TMTIMERSTATE_FREE: + AssertMsgFailed(("Invalid timer state %s (%s)\n", tmTimerState(enmState), pTimer->szName)); + Log2(("TMTimerIsActive: returns false (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n", + pTimer, tmTimerState(pTimer->enmState), pTimer->szName)); + return false; + default: + AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName)); + return false; + } +} + + +/* -=-=-=-=-=-=- Convenience APIs -=-=-=-=-=-=- */ + + +/** + * Arm a timer with a (new) expire time relative to current time. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cMilliesToNext Number of milliseconds to the next tick. + */ +VMMDECL(int) TMTimerSetMillies(PVMCC pVM, TMTIMERHANDLE hTimer, uint32_t cMilliesToNext) +{ + TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return tmTimerSetRelative(pVM, pTimer, cMilliesToNext * UINT64_C(1000000), NULL, pQueueCC, pQueue); + + case TMCLOCK_VIRTUAL_SYNC: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return tmTimerSetRelative(pVM, pTimer, cMilliesToNext * UINT64_C(1000000), NULL, pQueueCC, pQueue); + + case TMCLOCK_REAL: + AssertCompile(TMCLOCK_FREQ_REAL == 1000); + return tmTimerSetRelative(pVM, pTimer, cMilliesToNext, NULL, pQueueCC, pQueue); + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return VERR_TM_TIMER_BAD_CLOCK; + } +} + + +/** + * Arm a timer with a (new) expire time relative to current time. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cMicrosToNext Number of microseconds to the next tick. + */ +VMMDECL(int) TMTimerSetMicro(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cMicrosToNext) +{ + TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return tmTimerSetRelative(pVM, pTimer, cMicrosToNext * 1000, NULL, pQueueCC, pQueue); + + case TMCLOCK_VIRTUAL_SYNC: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return tmTimerSetRelative(pVM, pTimer, cMicrosToNext * 1000, NULL, pQueueCC, pQueue); + + case TMCLOCK_REAL: + AssertCompile(TMCLOCK_FREQ_REAL == 1000); + return tmTimerSetRelative(pVM, pTimer, cMicrosToNext / 1000, NULL, pQueueCC, pQueue); + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return VERR_TM_TIMER_BAD_CLOCK; + } +} + + +/** + * Arm a timer with a (new) expire time relative to current time. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cNanosToNext Number of nanoseconds to the next tick. + */ +VMMDECL(int) TMTimerSetNano(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cNanosToNext) +{ + TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return tmTimerSetRelative(pVM, pTimer, cNanosToNext, NULL, pQueueCC, pQueue); + + case TMCLOCK_VIRTUAL_SYNC: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return tmTimerSetRelative(pVM, pTimer, cNanosToNext, NULL, pQueueCC, pQueue); + + case TMCLOCK_REAL: + AssertCompile(TMCLOCK_FREQ_REAL == 1000); + return tmTimerSetRelative(pVM, pTimer, cNanosToNext / 1000000, NULL, pQueueCC, pQueue); + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return VERR_TM_TIMER_BAD_CLOCK; + } +} + + +/** + * Get the current clock time as nanoseconds. + * + * @returns The timer clock as nanoseconds. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(uint64_t) TMTimerGetNano(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + return TMTimerToNano(pVM, hTimer, TMTimerGet(pVM, hTimer)); +} + + +/** + * Get the current clock time as microseconds. + * + * @returns The timer clock as microseconds. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(uint64_t) TMTimerGetMicro(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + return TMTimerToMicro(pVM, hTimer, TMTimerGet(pVM, hTimer)); +} + + +/** + * Get the current clock time as milliseconds. + * + * @returns The timer clock as milliseconds. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + */ +VMMDECL(uint64_t) TMTimerGetMilli(PVMCC pVM, TMTIMERHANDLE hTimer) +{ + return TMTimerToMilli(pVM, hTimer, TMTimerGet(pVM, hTimer)); +} + + +/** + * Converts the specified timer clock time to nanoseconds. + * + * @returns nanoseconds. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cTicks The clock ticks. + * @remark There could be rounding errors here. We just do a simple integer divide + * without any adjustments. + */ +VMMDECL(uint64_t) TMTimerToNano(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cTicks) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + case TMCLOCK_VIRTUAL_SYNC: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return cTicks; + + case TMCLOCK_REAL: + AssertCompile(TMCLOCK_FREQ_REAL == 1000); + return cTicks * 1000000; + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return 0; + } +} + + +/** + * Converts the specified timer clock time to microseconds. + * + * @returns microseconds. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cTicks The clock ticks. + * @remark There could be rounding errors here. We just do a simple integer divide + * without any adjustments. + */ +VMMDECL(uint64_t) TMTimerToMicro(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cTicks) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + case TMCLOCK_VIRTUAL_SYNC: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return cTicks / 1000; + + case TMCLOCK_REAL: + AssertCompile(TMCLOCK_FREQ_REAL == 1000); + return cTicks * 1000; + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return 0; + } +} + + +/** + * Converts the specified timer clock time to milliseconds. + * + * @returns milliseconds. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cTicks The clock ticks. + * @remark There could be rounding errors here. We just do a simple integer divide + * without any adjustments. + */ +VMMDECL(uint64_t) TMTimerToMilli(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cTicks) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + case TMCLOCK_VIRTUAL_SYNC: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return cTicks / 1000000; + + case TMCLOCK_REAL: + AssertCompile(TMCLOCK_FREQ_REAL == 1000); + return cTicks; + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return 0; + } +} + + +/** + * Converts the specified nanosecond timestamp to timer clock ticks. + * + * @returns timer clock ticks. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cNanoSecs The nanosecond value ticks to convert. + * @remark There could be rounding and overflow errors here. + */ +VMMDECL(uint64_t) TMTimerFromNano(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cNanoSecs) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + case TMCLOCK_VIRTUAL_SYNC: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return cNanoSecs; + + case TMCLOCK_REAL: + AssertCompile(TMCLOCK_FREQ_REAL == 1000); + return cNanoSecs / 1000000; + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return 0; + } +} + + +/** + * Converts the specified microsecond timestamp to timer clock ticks. + * + * @returns timer clock ticks. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cMicroSecs The microsecond value ticks to convert. + * @remark There could be rounding and overflow errors here. + */ +VMMDECL(uint64_t) TMTimerFromMicro(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cMicroSecs) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + case TMCLOCK_VIRTUAL_SYNC: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return cMicroSecs * 1000; + + case TMCLOCK_REAL: + AssertCompile(TMCLOCK_FREQ_REAL == 1000); + return cMicroSecs / 1000; + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return 0; + } +} + + +/** + * Converts the specified millisecond timestamp to timer clock ticks. + * + * @returns timer clock ticks. + * @param pVM The cross context VM structure. + * @param hTimer Timer handle as returned by one of the create functions. + * @param cMilliSecs The millisecond value ticks to convert. + * @remark There could be rounding and overflow errors here. + */ +VMMDECL(uint64_t) TMTimerFromMilli(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cMilliSecs) +{ + TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */ + switch (pQueue->enmClock) + { + case TMCLOCK_VIRTUAL: + case TMCLOCK_VIRTUAL_SYNC: + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return cMilliSecs * 1000000; + + case TMCLOCK_REAL: + AssertCompile(TMCLOCK_FREQ_REAL == 1000); + return cMilliSecs; + + default: + AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock)); + return 0; + } +} + + +/** + * Convert state to string. + * + * @returns Readonly status name. + * @param enmState State. + */ +const char *tmTimerState(TMTIMERSTATE enmState) +{ + switch (enmState) + { +#define CASE(num, state) \ + case TMTIMERSTATE_##state: \ + AssertCompile(TMTIMERSTATE_##state == (num)); \ + return #num "-" #state + CASE( 0,INVALID); + CASE( 1,STOPPED); + CASE( 2,ACTIVE); + CASE( 3,EXPIRED_GET_UNLINK); + CASE( 4,EXPIRED_DELIVER); + CASE( 5,PENDING_STOP); + CASE( 6,PENDING_STOP_SCHEDULE); + CASE( 7,PENDING_SCHEDULE_SET_EXPIRE); + CASE( 8,PENDING_SCHEDULE); + CASE( 9,PENDING_RESCHEDULE_SET_EXPIRE); + CASE(10,PENDING_RESCHEDULE); + CASE(11,DESTROY); + CASE(12,FREE); + default: + AssertMsgFailed(("Invalid state enmState=%d\n", enmState)); + return "Invalid state!"; +#undef CASE + } +} + + +#if defined(IN_RING0) || defined(IN_RING3) +/** + * Copies over old timers and initialized newly allocted ones. + * + * Helper for TMR0TimerQueueGrow an tmR3TimerQueueGrow. + * + * @param paTimers The new timer allocation. + * @param paOldTimers The old timers. + * @param cNewTimers Number of new timers. + * @param cOldTimers Number of old timers. + */ +void tmHCTimerQueueGrowInit(PTMTIMER paTimers, TMTIMER const *paOldTimers, uint32_t cNewTimers, uint32_t cOldTimers) +{ + Assert(cOldTimers < cNewTimers); + + /* + * Copy over the old info and initialize the new handles. + */ + if (cOldTimers > 0) + memcpy(paTimers, paOldTimers, sizeof(TMTIMER) * cOldTimers); + + size_t i = cNewTimers; + while (i-- > cOldTimers) + { + paTimers[i].u64Expire = UINT64_MAX; + paTimers[i].enmType = TMTIMERTYPE_INVALID; + paTimers[i].enmState = TMTIMERSTATE_FREE; + paTimers[i].idxScheduleNext = UINT32_MAX; + paTimers[i].idxNext = UINT32_MAX; + paTimers[i].idxPrev = UINT32_MAX; + paTimers[i].hSelf = NIL_TMTIMERHANDLE; + } + + /* + * Mark the zero'th entry as allocated but invalid if we just allocated it. + */ + if (cOldTimers == 0) + { + paTimers[0].enmState = TMTIMERSTATE_INVALID; + paTimers[0].szName[0] = 'n'; + paTimers[0].szName[1] = 'i'; + paTimers[0].szName[2] = 'l'; + paTimers[0].szName[3] = '\0'; + } +} +#endif /* IN_RING0 || IN_RING3 */ + + +/** + * The slow path of tmGetFrequencyHint() where we try to recalculate the value. + * + * @returns The highest frequency. 0 if no timers care. + * @param pVM The cross context VM structure. + * @param uOldMaxHzHint The old global hint. + */ +DECL_NO_INLINE(static, uint32_t) tmGetFrequencyHintSlow(PVMCC pVM, uint32_t uOldMaxHzHint) +{ + /* Set two bits, though not entirely sure it's needed (too exhaused to think clearly) + but it should force other callers thru the slow path while we're recalculating and + help us detect changes while we're recalculating. */ + AssertCompile(RT_ELEMENTS(pVM->tm.s.aTimerQueues) <= 16); + + /* + * The "right" highest frequency value isn't so important that we'll block + * waiting on the timer semaphores. + */ + uint32_t uMaxHzHint = 0; + for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++) + { + PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[idxQueue]; + + /* Get the max Hz hint for the queue. */ + uint32_t uMaxHzHintQueue; + if ( !(ASMAtomicUoReadU64(&pVM->tm.s.HzHint.u64Combined) & (RT_BIT_32(idxQueue) | RT_BIT_32(idxQueue + 16))) + || RT_FAILURE_NP(PDMCritSectTryEnter(pVM, &pQueue->TimerLock))) + uMaxHzHintQueue = ASMAtomicReadU32(&pQueue->uMaxHzHint); + else + { + /* Is it still necessary to do updating? */ + if (ASMAtomicUoReadU64(&pVM->tm.s.HzHint.u64Combined) & (RT_BIT_32(idxQueue) | RT_BIT_32(idxQueue + 16))) + { + ASMAtomicAndU64(&pVM->tm.s.HzHint.u64Combined, ~RT_BIT_64(idxQueue + 16)); /* clear one flag up front */ + + PTMTIMERQUEUECC pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, idxQueue, pQueue); + uMaxHzHintQueue = 0; + for (PTMTIMER pCur = tmTimerQueueGetHead(pQueueCC, pQueue); + pCur; + pCur = tmTimerGetNext(pQueueCC, pCur)) + { + uint32_t uHzHint = ASMAtomicUoReadU32(&pCur->uHzHint); + if (uHzHint > uMaxHzHintQueue) + { + TMTIMERSTATE enmState = pCur->enmState; + switch (enmState) + { + case TMTIMERSTATE_ACTIVE: + case TMTIMERSTATE_EXPIRED_GET_UNLINK: + case TMTIMERSTATE_EXPIRED_DELIVER: + case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_SCHEDULE: + case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE: + case TMTIMERSTATE_PENDING_RESCHEDULE: + uMaxHzHintQueue = uHzHint; + break; + + case TMTIMERSTATE_STOPPED: + case TMTIMERSTATE_PENDING_STOP: + case TMTIMERSTATE_PENDING_STOP_SCHEDULE: + case TMTIMERSTATE_DESTROY: + case TMTIMERSTATE_FREE: + case TMTIMERSTATE_INVALID: + break; + /* no default, want gcc warnings when adding more states. */ + } + } + } + + /* Write the new Hz hint for the quest and clear the other update flag. */ + ASMAtomicUoWriteU32(&pQueue->uMaxHzHint, uMaxHzHintQueue); + ASMAtomicAndU64(&pVM->tm.s.HzHint.u64Combined, ~RT_BIT_64(idxQueue)); + } + else + uMaxHzHintQueue = ASMAtomicUoReadU32(&pQueue->uMaxHzHint); + + PDMCritSectLeave(pVM, &pQueue->TimerLock); + } + + /* Update the global max Hz hint. */ + if (uMaxHzHint < uMaxHzHintQueue) + uMaxHzHint = uMaxHzHintQueue; + } + + /* + * Update the frequency hint if no pending frequency changes and we didn't race anyone thru here. + */ + uint64_t u64Actual = RT_MAKE_U64(0 /*no pending updates*/, uOldMaxHzHint); + if (ASMAtomicCmpXchgExU64(&pVM->tm.s.HzHint.u64Combined, RT_MAKE_U64(0, uMaxHzHint), u64Actual, &u64Actual)) + Log(("tmGetFrequencyHintSlow: New value %u Hz\n", uMaxHzHint)); + else + for (uint32_t iTry = 1;; iTry++) + { + if (RT_LO_U32(u64Actual) != 0) + Log(("tmGetFrequencyHintSlow: Outdated value %u Hz (%#x, try %u)\n", uMaxHzHint, RT_LO_U32(u64Actual), iTry)); + else if (iTry >= 4) + Log(("tmGetFrequencyHintSlow: Unable to set %u Hz (try %u)\n", uMaxHzHint, iTry)); + else if (ASMAtomicCmpXchgExU64(&pVM->tm.s.HzHint.u64Combined, RT_MAKE_U64(0, uMaxHzHint), u64Actual, &u64Actual)) + Log(("tmGetFrequencyHintSlow: New value %u Hz (try %u)\n", uMaxHzHint, iTry)); + else + continue; + break; + } + return uMaxHzHint; +} + + +/** + * Gets the highest frequency hint for all the important timers. + * + * @returns The highest frequency. 0 if no timers care. + * @param pVM The cross context VM structure. + */ +DECLINLINE(uint32_t) tmGetFrequencyHint(PVMCC pVM) +{ + /* + * Query the value, recalculate it if necessary. + */ + uint64_t u64Combined = ASMAtomicReadU64(&pVM->tm.s.HzHint.u64Combined); + if (RT_HI_U32(u64Combined) == 0) + return RT_LO_U32(u64Combined); /* hopefully somewhat likely */ + return tmGetFrequencyHintSlow(pVM, RT_LO_U32(u64Combined)); +} + + +/** + * Calculates a host timer frequency that would be suitable for the current + * timer load. + * + * This will take the highest timer frequency, adjust for catch-up and warp + * driver, and finally add a little fudge factor. The caller (VMM) will use + * the result to adjust the per-cpu preemption timer. + * + * @returns The highest frequency. 0 if no important timers around. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +VMM_INT_DECL(uint32_t) TMCalcHostTimerFrequency(PVMCC pVM, PVMCPUCC pVCpu) +{ + uint32_t uHz = tmGetFrequencyHint(pVM); + + /* Catch up, we have to be more aggressive than the % indicates at the + beginning of the effort. */ + if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + { + uint32_t u32Pct = ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage); + if (ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + { + if (u32Pct <= 100) + u32Pct = u32Pct * pVM->tm.s.cPctHostHzFudgeFactorCatchUp100 / 100; + else if (u32Pct <= 200) + u32Pct = u32Pct * pVM->tm.s.cPctHostHzFudgeFactorCatchUp200 / 100; + else if (u32Pct <= 400) + u32Pct = u32Pct * pVM->tm.s.cPctHostHzFudgeFactorCatchUp400 / 100; + uHz *= u32Pct + 100; + uHz /= 100; + } + } + + /* Warp drive. */ + if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualWarpDrive)) + { + uint32_t u32Pct = ASMAtomicReadU32(&pVM->tm.s.u32VirtualWarpDrivePercentage); + if (ASMAtomicReadBool(&pVM->tm.s.fVirtualWarpDrive)) + { + uHz *= u32Pct; + uHz /= 100; + } + } + + /* Fudge factor. */ + if (pVCpu->idCpu == pVM->tm.s.idTimerCpu) + uHz *= pVM->tm.s.cPctHostHzFudgeFactorTimerCpu; + else + uHz *= pVM->tm.s.cPctHostHzFudgeFactorOtherCpu; + uHz /= 100; + + /* Make sure it isn't too high. */ + if (uHz > pVM->tm.s.cHostHzMax) + uHz = pVM->tm.s.cHostHzMax; + + return uHz; +} + + +/** + * Whether the guest virtual clock is ticking. + * + * @returns true if ticking, false otherwise. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(bool) TMVirtualIsTicking(PVM pVM) +{ + return RT_BOOL(pVM->tm.s.cVirtualTicking); +} + diff --git a/src/VBox/VMM/VMMAll/TMAllCpu.cpp b/src/VBox/VMM/VMMAll/TMAllCpu.cpp new file mode 100644 index 00000000..7b68afe1 --- /dev/null +++ b/src/VBox/VMM/VMMAll/TMAllCpu.cpp @@ -0,0 +1,661 @@ +/* $Id: TMAllCpu.cpp $ */ +/** @file + * TM - Timeout Manager, CPU Time, All Contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_TM +#include +#include +#include +#include +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) +# include /* for SUPGetCpuHzFromGIP; ASMReadTSC */ +#elif defined(RT_ARCH_ARM64) || defined(RT_ARCH_ARM32) +# include +#endif +#include "TMInternal.h" +#include +#include + +#include +#include +#include +#include +#include + + + +/** + * Converts from virtual time to raw CPU ticks. + * + * Mainly to have the ASMMultU64ByU32DivByU32 overflow trickery in one place. + * + * @returns raw CPU ticks. + * @param pVM The cross context VM structure. + * @param u64VirtualTime The virtual time to convert. + */ +DECLINLINE(uint64_t) tmCpuTickCalcFromVirtual(PVMCC pVM, uint64_t u64VirtualTime) +{ + if (pVM->tm.s.cTSCTicksPerSecond <= UINT32_MAX) + return ASMMultU64ByU32DivByU32(u64VirtualTime, (uint32_t)pVM->tm.s.cTSCTicksPerSecond, TMCLOCK_FREQ_VIRTUAL); + Assert(pVM->tm.s.cTSCTicksPerSecond <= ((uint64_t)UINT32_MAX << 2)); /* <= 15.99 GHz */ + return ASMMultU64ByU32DivByU32(u64VirtualTime, (uint32_t)(pVM->tm.s.cTSCTicksPerSecond >> 2), TMCLOCK_FREQ_VIRTUAL >> 2); +} + + +/** + * Gets the raw cpu tick from current virtual time. + * + * @param pVM The cross context VM structure. + * @param fCheckTimers Whether to check timers. + */ +DECLINLINE(uint64_t) tmCpuTickGetRawVirtual(PVMCC pVM, bool fCheckTimers) +{ + if (fCheckTimers) + return tmCpuTickCalcFromVirtual(pVM, TMVirtualSyncGet(pVM)); + return tmCpuTickCalcFromVirtual(pVM, TMVirtualSyncGetNoCheck(pVM)); +} + + +#ifdef IN_RING3 +/** + * Used by tmR3CpuTickParavirtEnable and tmR3CpuTickParavirtDisable. + * + * @param pVM The cross context VM structure. + */ +uint64_t tmR3CpuTickGetRawVirtualNoCheck(PVM pVM) +{ + return tmCpuTickGetRawVirtual(pVM, false /*fCheckTimers*/); +} +#endif + + +/** + * Resumes the CPU timestamp counter ticking. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @internal + */ +int tmCpuTickResume(PVMCC pVM, PVMCPUCC pVCpu) +{ + if (!pVCpu->tm.s.fTSCTicking) + { + pVCpu->tm.s.fTSCTicking = true; + + /** @todo Test that pausing and resuming doesn't cause lag! (I.e. that we're + * unpaused before the virtual time and stopped after it. */ + switch (pVM->tm.s.enmTSCMode) + { + case TMTSCMODE_REAL_TSC_OFFSET: + pVCpu->tm.s.offTSCRawSrc = SUPReadTsc() - pVCpu->tm.s.u64TSC; + break; + case TMTSCMODE_VIRT_TSC_EMULATED: + case TMTSCMODE_DYNAMIC: + pVCpu->tm.s.offTSCRawSrc = tmCpuTickGetRawVirtual(pVM, false /* don't check for pending timers */) + - pVCpu->tm.s.u64TSC; + break; + case TMTSCMODE_NATIVE_API: + pVCpu->tm.s.offTSCRawSrc = 0; /** @todo ?? */ + /* Looks like this is only used by weird modes and MSR TSC writes. We cannot support either on NEM/win. */ + break; + default: + AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE); + } + return VINF_SUCCESS; + } + AssertFailed(); + return VERR_TM_TSC_ALREADY_TICKING; +} + + +/** + * Resumes the CPU timestamp counter ticking. + * + * @returns VINF_SUCCESS or VERR_TM_VIRTUAL_TICKING_IPE (asserted). + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + */ +int tmCpuTickResumeLocked(PVMCC pVM, PVMCPUCC pVCpu) +{ + if (!pVCpu->tm.s.fTSCTicking) + { + /* TSC must be ticking before calling tmCpuTickGetRawVirtual()! */ + pVCpu->tm.s.fTSCTicking = true; + uint32_t c = ASMAtomicIncU32(&pVM->tm.s.cTSCsTicking); + AssertMsgReturn(c <= pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE); + if (c == 1) + { + /* The first VCPU to resume. */ + uint64_t offTSCRawSrcOld = pVCpu->tm.s.offTSCRawSrc; + + STAM_COUNTER_INC(&pVM->tm.s.StatTSCResume); + + /* When resuming, use the TSC value of the last stopped VCPU to avoid the TSC going back. */ + switch (pVM->tm.s.enmTSCMode) + { + case TMTSCMODE_REAL_TSC_OFFSET: + pVCpu->tm.s.offTSCRawSrc = SUPReadTsc() - pVM->tm.s.u64LastPausedTSC; + break; + case TMTSCMODE_VIRT_TSC_EMULATED: + case TMTSCMODE_DYNAMIC: + pVCpu->tm.s.offTSCRawSrc = tmCpuTickGetRawVirtual(pVM, false /* don't check for pending timers */) + - pVM->tm.s.u64LastPausedTSC; + break; + case TMTSCMODE_NATIVE_API: + { + int rc = NEMHCResumeCpuTickOnAll(pVM, pVCpu, pVM->tm.s.u64LastPausedTSC); + AssertRCReturn(rc, rc); + pVCpu->tm.s.offTSCRawSrc = offTSCRawSrcOld = 0; + break; + } + default: + AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE); + } + + /* Calculate the offset addendum for other VCPUs to use. */ + pVM->tm.s.offTSCPause = pVCpu->tm.s.offTSCRawSrc - offTSCRawSrcOld; + } + else + { + /* All other VCPUs (if any). */ + pVCpu->tm.s.offTSCRawSrc += pVM->tm.s.offTSCPause; + } + } + return VINF_SUCCESS; +} + + +/** + * Pauses the CPU timestamp counter ticking. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @internal + */ +int tmCpuTickPause(PVMCPUCC pVCpu) +{ + if (pVCpu->tm.s.fTSCTicking) + { + pVCpu->tm.s.u64TSC = TMCpuTickGetNoCheck(pVCpu); + pVCpu->tm.s.fTSCTicking = false; + return VINF_SUCCESS; + } + AssertFailed(); + return VERR_TM_TSC_ALREADY_PAUSED; +} + + +/** + * Pauses the CPU timestamp counter ticking. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @internal + */ +int tmCpuTickPauseLocked(PVMCC pVM, PVMCPUCC pVCpu) +{ + if (pVCpu->tm.s.fTSCTicking) + { + pVCpu->tm.s.u64TSC = TMCpuTickGetNoCheck(pVCpu); + pVCpu->tm.s.fTSCTicking = false; + + uint32_t c = ASMAtomicDecU32(&pVM->tm.s.cTSCsTicking); + AssertMsgReturn(c < pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE); + if (c == 0) + { + /* When the last TSC stops, remember the value. */ + STAM_COUNTER_INC(&pVM->tm.s.StatTSCPause); + pVM->tm.s.u64LastPausedTSC = pVCpu->tm.s.u64TSC; + } + return VINF_SUCCESS; + } + AssertFailed(); + return VERR_TM_TSC_ALREADY_PAUSED; +} + + +#ifdef IN_RING0 /* Only used in ring-0 at present (AMD-V and VT-x). */ + +# ifdef VBOX_WITH_STATISTICS +/** + * Record why we refused to use offsetted TSC. + * + * Used by TMCpuTickCanUseRealTSC() and TMCpuTickGetDeadlineAndTscOffset(). + * + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +DECLINLINE(void) tmCpuTickRecordOffsettedTscRefusal(PVM pVM, PVMCPU pVCpu) +{ + /* Sample the reason for refusing. */ + if (pVM->tm.s.enmTSCMode != TMTSCMODE_DYNAMIC) + STAM_COUNTER_INC(&pVM->tm.s.StatTSCNotFixed); + else if (!pVCpu->tm.s.fTSCTicking) + STAM_COUNTER_INC(&pVM->tm.s.StatTSCNotTicking); + else if (pVM->tm.s.enmTSCMode != TMTSCMODE_REAL_TSC_OFFSET) + { + if (pVM->tm.s.fVirtualSyncCatchUp) + { + if (pVM->tm.s.u32VirtualSyncCatchUpPercentage <= 10) + STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupLE010); + else if (pVM->tm.s.u32VirtualSyncCatchUpPercentage <= 25) + STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupLE025); + else if (pVM->tm.s.u32VirtualSyncCatchUpPercentage <= 100) + STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupLE100); + else + STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupOther); + } + else if (!pVM->tm.s.fVirtualSyncTicking) + STAM_COUNTER_INC(&pVM->tm.s.StatTSCSyncNotTicking); + else if (pVM->tm.s.fVirtualWarpDrive) + STAM_COUNTER_INC(&pVM->tm.s.StatTSCWarp); + } +} +# endif /* VBOX_WITH_STATISTICS */ + +/** + * Checks if AMD-V / VT-x can use an offsetted hardware TSC or not. + * + * @returns true/false accordingly. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param poffRealTsc The offset against the TSC of the current host CPU, + * if pfOffsettedTsc is set to true. + * @param pfParavirtTsc Where to return whether paravirt TSC is enabled. + * + * @thread EMT(pVCpu). + * @see TMCpuTickGetDeadlineAndTscOffset(). + */ +VMM_INT_DECL(bool) TMCpuTickCanUseRealTSC(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *poffRealTsc, bool *pfParavirtTsc) +{ + Assert(pVCpu->tm.s.fTSCTicking || DBGFIsStepping(pVCpu)); + + *pfParavirtTsc = pVM->tm.s.fParavirtTscEnabled; + + /* + * In real TSC mode it's easy, we just need the delta & offTscRawSrc and + * the CPU will add them to RDTSC and RDTSCP at runtime. + * + * In tmCpuTickGetInternal we do: + * SUPReadTsc() - pVCpu->tm.s.offTSCRawSrc; + * Where SUPReadTsc() does: + * ASMReadTSC() - pGipCpu->i64TscDelta; + * Which means tmCpuTickGetInternal actually does: + * ASMReadTSC() - pGipCpu->i64TscDelta - pVCpu->tm.s.offTSCRawSrc; + * So, the offset to be ADDED to RDTSC[P] is: + * offRealTsc = -(pGipCpu->i64TscDelta + pVCpu->tm.s.offTSCRawSrc) + */ + if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET) + { + /** @todo We should negate both deltas! It's soo weird that we do the + * exact opposite of what the hardware implements. */ +# ifdef IN_RING3 + *poffRealTsc = (uint64_t)0 - pVCpu->tm.s.offTSCRawSrc - (uint64_t)SUPGetTscDelta(g_pSUPGlobalInfoPage); +# else + *poffRealTsc = (uint64_t)0 - pVCpu->tm.s.offTSCRawSrc - (uint64_t)SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet); +# endif + return true; + } + + /* + * We require: + * 1. A fixed TSC, this is checked at init time. + * 2. That the TSC is ticking (we shouldn't be here if it isn't) + * 3. Either that we're using the real TSC as time source or + * a) we don't have any lag to catch up, and + * b) the virtual sync clock hasn't been halted by an expired timer, and + * c) we're not using warp drive (accelerated virtual guest time). + */ + if ( pVM->tm.s.enmTSCMode == TMTSCMODE_DYNAMIC + && !pVM->tm.s.fVirtualSyncCatchUp + && RT_LIKELY(pVM->tm.s.fVirtualSyncTicking) + && !pVM->tm.s.fVirtualWarpDrive) + { + /* The source is the timer synchronous virtual clock. */ + uint64_t uTscNow; + uint64_t u64Now = tmCpuTickCalcFromVirtual(pVM, TMVirtualSyncGetNoCheckWithTsc(pVM, &uTscNow)) + - pVCpu->tm.s.offTSCRawSrc; + /** @todo When we start collecting statistics on how much time we spend executing + * guest code before exiting, we should check this against the next virtual sync + * timer timeout. If it's lower than the avg. length, we should trap rdtsc to increase + * the chance that we'll get interrupted right after the timer expired. */ + if (u64Now >= pVCpu->tm.s.u64TSCLastSeen) + { +# ifdef IN_RING3 + *poffRealTsc = u64Now - (uTscNow + (uint64_t)SUPGetTscDelta(g_pSUPGlobalInfoPage); +# else + *poffRealTsc = u64Now - (uTscNow + (uint64_t)SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet)); +# endif + return true; /** @todo count this? */ + } + } + +# ifdef VBOX_WITH_STATISTICS + tmCpuTickRecordOffsettedTscRefusal(pVM, pVCpu); +# endif + return false; +} + + +/** + * Calculates the number of host CPU ticks till the next virtual sync deadline. + * + * @note To save work, this function will not bother calculating the accurate + * tick count for deadlines that are more than a second ahead. + * + * @returns The number of host cpu ticks to the next deadline. Max one second. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param cNsToDeadline The number of nano seconds to the next virtual + * sync deadline. + */ +DECLINLINE(uint64_t) tmCpuCalcTicksToDeadline(PVMCPUCC pVCpu, uint64_t cNsToDeadline) +{ + AssertCompile(TMCLOCK_FREQ_VIRTUAL <= _4G); +# ifdef IN_RING3 + RT_NOREF_PV(pVCpu); + PSUPGIP const pGip = g_pSUPGlobalInfoPage; + uint64_t uCpuHz = pGip ? SUPGetCpuHzFromGip(pGip) : pVCpu->pVMR3->tm.s.cTSCTicksPerSecondHost; +# else + uint64_t uCpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet); +# endif + if (RT_UNLIKELY(cNsToDeadline >= TMCLOCK_FREQ_VIRTUAL)) + return uCpuHz; + AssertCompile(TMCLOCK_FREQ_VIRTUAL <= UINT32_MAX); + uint64_t cTicks = ASMMultU64ByU32DivByU32(uCpuHz, (uint32_t)cNsToDeadline, TMCLOCK_FREQ_VIRTUAL); + if (cTicks > 4000) + cTicks -= 4000; /* fudge to account for overhead */ + else + cTicks >>= 1; + return cTicks; +} + + +/** + * Gets the next deadline in host CPU clock ticks and the TSC offset if we can + * use the raw TSC. + * + * @returns The number of host CPU clock ticks to the next timer deadline. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param poffRealTsc The offset against the TSC of the current host CPU, + * if pfOffsettedTsc is set to true. + * @param pfOffsettedTsc Where to return whether TSC offsetting can be used. + * @param pfParavirtTsc Where to return whether paravirt TSC is enabled. + * @param puTscNow Where to return the TSC value that the return + * value is relative to. This is delta adjusted. + * @param puDeadlineVersion Where to return the deadline "version" number. + * Use with TMVirtualSyncIsCurrentDeadlineVersion() + * to check if the absolute deadline is still up to + * date and the caller can skip calling this + * function. + * + * @thread EMT(pVCpu). + * @see TMCpuTickCanUseRealTSC(). + */ +VMM_INT_DECL(uint64_t) TMCpuTickGetDeadlineAndTscOffset(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *poffRealTsc, + bool *pfOffsettedTsc, bool *pfParavirtTsc, + uint64_t *puTscNow, uint64_t *puDeadlineVersion) +{ + Assert(pVCpu->tm.s.fTSCTicking || DBGFIsStepping(pVCpu)); + + *pfParavirtTsc = pVM->tm.s.fParavirtTscEnabled; + + /* + * Same logic as in TMCpuTickCanUseRealTSC. + */ + if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET) + { + /** @todo We should negate both deltas! It's soo weird that we do the + * exact opposite of what the hardware implements. */ +# ifdef IN_RING3 + *poffRealTsc = (uint64_t)0 - pVCpu->tm.s.offTSCRawSrc - (uint64_t)SUPGetTscDelta(g_pSUPGlobalInfoPage); +# else + *poffRealTsc = (uint64_t)0 - pVCpu->tm.s.offTSCRawSrc - (uint64_t)SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet); +# endif + *pfOffsettedTsc = true; + return tmCpuCalcTicksToDeadline(pVCpu, TMVirtualSyncGetNsToDeadline(pVM, puDeadlineVersion, puTscNow)); + } + + /* + * Same logic as in TMCpuTickCanUseRealTSC. + */ + if ( pVM->tm.s.enmTSCMode == TMTSCMODE_DYNAMIC + && !pVM->tm.s.fVirtualSyncCatchUp + && RT_LIKELY(pVM->tm.s.fVirtualSyncTicking) + && !pVM->tm.s.fVirtualWarpDrive) + { + /* The source is the timer synchronous virtual clock. */ + uint64_t cNsToDeadline; + uint64_t u64NowVirtSync = TMVirtualSyncGetWithDeadlineNoCheck(pVM, &cNsToDeadline, puDeadlineVersion, puTscNow); + uint64_t u64Now = tmCpuTickCalcFromVirtual(pVM, u64NowVirtSync); + u64Now -= pVCpu->tm.s.offTSCRawSrc; + +# ifdef IN_RING3 + *poffRealTsc = u64Now - (*puTscNow + (uint64_t)SUPGetTscDelta(g_pSUPGlobalInfoPage)); /* undoing delta */ +# else + *poffRealTsc = u64Now - (*puTscNow + (uint64_t)SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet)); /* undoing delta */ +# endif + *pfOffsettedTsc = u64Now >= pVCpu->tm.s.u64TSCLastSeen; + return tmCpuCalcTicksToDeadline(pVCpu, cNsToDeadline); + } + +# ifdef VBOX_WITH_STATISTICS + tmCpuTickRecordOffsettedTscRefusal(pVM, pVCpu); +# endif + *pfOffsettedTsc = false; + *poffRealTsc = 0; + return tmCpuCalcTicksToDeadline(pVCpu, TMVirtualSyncGetNsToDeadline(pVM, puDeadlineVersion, puTscNow)); +} + +#endif /* IN_RING0 - at the moment */ + +/** + * Read the current CPU timestamp counter. + * + * @returns Gets the CPU tsc. + * @param pVCpu The cross context virtual CPU structure. + * @param fCheckTimers Whether to check timers. + */ +DECLINLINE(uint64_t) tmCpuTickGetInternal(PVMCPUCC pVCpu, bool fCheckTimers) +{ + uint64_t u64; + + if (RT_LIKELY(pVCpu->tm.s.fTSCTicking)) + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + switch (pVM->tm.s.enmTSCMode) + { + case TMTSCMODE_REAL_TSC_OFFSET: + u64 = SUPReadTsc(); + break; + case TMTSCMODE_VIRT_TSC_EMULATED: + case TMTSCMODE_DYNAMIC: + u64 = tmCpuTickGetRawVirtual(pVM, fCheckTimers); + break; + case TMTSCMODE_NATIVE_API: + { + u64 = 0; + int rcNem = NEMHCQueryCpuTick(pVCpu, &u64, NULL); + AssertLogRelRCReturn(rcNem, SUPReadTsc()); + break; + } + default: + AssertFailedBreakStmt(u64 = SUPReadTsc()); + } + u64 -= pVCpu->tm.s.offTSCRawSrc; + + /* Always return a value higher than what the guest has already seen. */ + if (RT_LIKELY(u64 > pVCpu->tm.s.u64TSCLastSeen)) + pVCpu->tm.s.u64TSCLastSeen = u64; + else + { + STAM_COUNTER_INC(&pVM->tm.s.StatTSCUnderflow); + pVCpu->tm.s.u64TSCLastSeen += 64; /** @todo choose a good increment here */ + u64 = pVCpu->tm.s.u64TSCLastSeen; + } + } + else + u64 = pVCpu->tm.s.u64TSC; + return u64; +} + + +/** + * Read the current CPU timestamp counter. + * + * @returns Gets the CPU tsc. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(uint64_t) TMCpuTickGet(PVMCPUCC pVCpu) +{ + return tmCpuTickGetInternal(pVCpu, true /* fCheckTimers */); +} + + +/** + * Read the current CPU timestamp counter, don't check for expired timers. + * + * @returns Gets the CPU tsc. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(uint64_t) TMCpuTickGetNoCheck(PVMCPUCC pVCpu) +{ + return tmCpuTickGetInternal(pVCpu, false /* fCheckTimers */); +} + + +/** + * Sets the current CPU timestamp counter. + * + * @returns VBox status code. + * @param pVM The cross context VM structure. + * @param pVCpu The cross context virtual CPU structure. + * @param u64Tick The new timestamp value. + * + * @thread EMT which TSC is to be set. + */ +VMM_INT_DECL(int) TMCpuTickSet(PVMCC pVM, PVMCPUCC pVCpu, uint64_t u64Tick) +{ + VMCPU_ASSERT_EMT(pVCpu); + STAM_COUNTER_INC(&pVM->tm.s.StatTSCSet); + + /* + * This is easier to do when the TSC is paused since resume will + * do all the calculations for us. Actually, we don't need to + * call tmCpuTickPause here since we overwrite u64TSC anyway. + */ + bool fTSCTicking = pVCpu->tm.s.fTSCTicking; + pVCpu->tm.s.fTSCTicking = false; + pVCpu->tm.s.u64TSC = u64Tick; + pVCpu->tm.s.u64TSCLastSeen = u64Tick; + if (fTSCTicking) + tmCpuTickResume(pVM, pVCpu); + /** @todo Try help synchronizing it better among the virtual CPUs? */ + + return VINF_SUCCESS; +} + +/** + * Sets the last seen CPU timestamp counter. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param u64LastSeenTick The last seen timestamp value. + * + * @thread EMT which TSC is to be set. + */ +VMM_INT_DECL(int) TMCpuTickSetLastSeen(PVMCPUCC pVCpu, uint64_t u64LastSeenTick) +{ + VMCPU_ASSERT_EMT(pVCpu); + + LogFlow(("TMCpuTickSetLastSeen %RX64\n", u64LastSeenTick)); + /** @todo deal with wraparound! */ + if (pVCpu->tm.s.u64TSCLastSeen < u64LastSeenTick) + pVCpu->tm.s.u64TSCLastSeen = u64LastSeenTick; + return VINF_SUCCESS; +} + +/** + * Gets the last seen CPU timestamp counter of the guest. + * + * @returns the last seen TSC. + * @param pVCpu The cross context virtual CPU structure. + * + * @thread EMT(pVCpu). + */ +VMM_INT_DECL(uint64_t) TMCpuTickGetLastSeen(PVMCPUCC pVCpu) +{ + VMCPU_ASSERT_EMT(pVCpu); + + return pVCpu->tm.s.u64TSCLastSeen; +} + + +/** + * Get the timestamp frequency. + * + * @returns Number of ticks per second. + * @param pVM The cross context VM structure. + */ +VMMDECL(uint64_t) TMCpuTicksPerSecond(PVMCC pVM) +{ + if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET) + { + PSUPGLOBALINFOPAGE const pGip = g_pSUPGlobalInfoPage; + if (pGip && pGip->u32Mode != SUPGIPMODE_INVARIANT_TSC) + { +#ifdef IN_RING3 + uint64_t cTSCTicksPerSecond = SUPGetCpuHzFromGip(pGip); +#elif defined(IN_RING0) + uint64_t cTSCTicksPerSecond = SUPGetCpuHzFromGipBySetIndex(pGip, (uint32_t)RTMpCpuIdToSetIndex(RTMpCpuId())); +#else + uint64_t cTSCTicksPerSecond = SUPGetCpuHzFromGipBySetIndex(pGip, VMMGetCpu(pVM)->iHostCpuSet); +#endif + if (RT_LIKELY(cTSCTicksPerSecond != ~(uint64_t)0)) + return cTSCTicksPerSecond; + } + } + return pVM->tm.s.cTSCTicksPerSecond; +} + + +/** + * Whether the TSC is ticking for the VCPU. + * + * @returns true if ticking, false otherwise. + * @param pVCpu The cross context virtual CPU structure. + */ +VMM_INT_DECL(bool) TMCpuTickIsTicking(PVMCPUCC pVCpu) +{ + return pVCpu->tm.s.fTSCTicking; +} + diff --git a/src/VBox/VMM/VMMAll/TMAllReal.cpp b/src/VBox/VMM/VMMAll/TMAllReal.cpp new file mode 100644 index 00000000..7540c87c --- /dev/null +++ b/src/VBox/VMM/VMMAll/TMAllReal.cpp @@ -0,0 +1,63 @@ +/* $Id: TMAllReal.cpp $ */ +/** @file + * TM - Timeout Manager, Real Time, All Contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_TM +#include +#include "TMInternal.h" +#include +#include + + +/** + * Gets the current TMCLOCK_REAL time. + * + * @returns Real time. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(uint64_t) TMRealGet(PVM pVM) +{ + NOREF(pVM); + return RTTimeMilliTS(); +} + + +/** + * Gets the frequency of the TMCLOCK_REAL clock. + * + * @returns frequency. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(uint64_t) TMRealGetFreq(PVM pVM) +{ + NOREF(pVM); + return TMCLOCK_FREQ_REAL; +} + diff --git a/src/VBox/VMM/VMMAll/TMAllVirtual.cpp b/src/VBox/VMM/VMMAll/TMAllVirtual.cpp new file mode 100644 index 00000000..9244bd85 --- /dev/null +++ b/src/VBox/VMM/VMMAll/TMAllVirtual.cpp @@ -0,0 +1,1154 @@ +/* $Id: TMAllVirtual.cpp $ */ +/** @file + * TM - Timeout Manager, Virtual Time, All Contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_TM +#include +#include +#ifdef IN_RING3 +# include +#endif +#include "TMInternal.h" +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + + +/** + * @interface_method_impl{RTTIMENANOTSDATA,pfnBad} + */ +DECLCALLBACK(DECLEXPORT(void)) tmVirtualNanoTSBad(PRTTIMENANOTSDATA pData, uint64_t u64NanoTS, uint64_t u64DeltaPrev, + uint64_t u64PrevNanoTS) +{ + PVMCC pVM = RT_FROM_CPP_MEMBER(pData, VMCC, VMCC_CTX(tm).s.VirtualGetRawData); + pData->cBadPrev++; + if ((int64_t)u64DeltaPrev < 0) + LogRel(("TM: u64DeltaPrev=%RI64 u64PrevNanoTS=0x%016RX64 u64NanoTS=0x%016RX64 pVM=%p\n", + u64DeltaPrev, u64PrevNanoTS, u64NanoTS, pVM)); + else + Log(("TM: u64DeltaPrev=%RI64 u64PrevNanoTS=0x%016RX64 u64NanoTS=0x%016RX64 pVM=%p (debugging?)\n", + u64DeltaPrev, u64PrevNanoTS, u64NanoTS, pVM)); +} + + +#ifdef IN_RING3 +/** + * @callback_method_impl{FNTIMENANOTSINTERNAL, For driverless mode.} + */ +static DECLCALLBACK(uint64_t) tmR3VirtualNanoTSDriverless(PRTTIMENANOTSDATA pData, PRTITMENANOTSEXTRA pExtra) +{ + RT_NOREF(pData); + if (pExtra) + pExtra->uTSCValue = ASMReadTSC(); + return RTTimeSystemNanoTS(); +} +#endif + + +/** + * @interface_method_impl{RTTIMENANOTSDATA,pfnRediscover} + * + * This is the initial worker, so the first call in each context ends up here. + * It is also used should the delta rating of the host CPUs change or if the + * fGetGipCpu feature the current worker relies upon becomes unavailable. The + * last two events may occur as CPUs are taken online. + */ +DECLCALLBACK(DECLEXPORT(uint64_t)) tmVirtualNanoTSRediscover(PRTTIMENANOTSDATA pData, PRTITMENANOTSEXTRA pExtra) +{ + PVMCC pVM = RT_FROM_CPP_MEMBER(pData, VMCC, VMCC_CTX(tm).s.VirtualGetRawData); + PFNTIMENANOTSINTERNAL pfnWorker; + + /* + * We require a valid GIP for the selection below. + * Invalid GIP is fatal, though we have to allow no GIP in driverless mode (ring-3 only). + */ + PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage; +#ifdef IN_RING3 + if (pGip) +#endif + { + AssertFatalMsg(RT_VALID_PTR(pGip), ("pVM=%p pGip=%p\n", pVM, pGip)); + AssertFatalMsg(pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC, ("pVM=%p pGip=%p u32Magic=%#x\n", pVM, pGip, pGip->u32Magic)); + AssertFatalMsg(pGip->u32Mode > SUPGIPMODE_INVALID && pGip->u32Mode < SUPGIPMODE_END, + ("pVM=%p pGip=%p u32Mode=%#x\n", pVM, pGip, pGip->u32Mode)); + + /* + * Determine the new worker. + */ +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) + bool const fLFence = RT_BOOL(ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_SSE2); +#endif + switch (pGip->u32Mode) + { +#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) + case SUPGIPMODE_SYNC_TSC: + case SUPGIPMODE_INVARIANT_TSC: +# ifdef IN_RING0 + if (pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO) + pfnWorker = fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta : RTTimeNanoTSLegacySyncInvarNoDelta; + else + pfnWorker = fLFence ? RTTimeNanoTSLFenceSyncInvarWithDelta : RTTimeNanoTSLegacySyncInvarWithDelta; +# else + if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS) + pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_PRACTICALLY_ZERO + ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta : RTTimeNanoTSLegacySyncInvarNoDelta + : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseIdtrLim : RTTimeNanoTSLegacySyncInvarWithDeltaUseIdtrLim; + else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS) + pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_PRACTICALLY_ZERO + ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta : RTTimeNanoTSLegacySyncInvarNoDelta + : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseRdtscp : RTTimeNanoTSLegacySyncInvarWithDeltaUseRdtscp; + else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_0B) + pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO + ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta : RTTimeNanoTSLegacySyncInvarNoDelta + : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseApicIdExt0B : RTTimeNanoTSLegacySyncInvarWithDeltaUseApicIdExt0B; + else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_8000001E) + pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO + ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta : RTTimeNanoTSLegacySyncInvarNoDelta + : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseApicIdExt8000001E : RTTimeNanoTSLegacySyncInvarWithDeltaUseApicIdExt8000001E; + else + pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO + ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta : RTTimeNanoTSLegacySyncInvarNoDelta + : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseApicId : RTTimeNanoTSLegacySyncInvarWithDeltaUseApicId; +# endif + break; + + case SUPGIPMODE_ASYNC_TSC: +# ifdef IN_RING0 + pfnWorker = fLFence ? RTTimeNanoTSLFenceAsync : RTTimeNanoTSLegacyAsync; +# else + if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS) + pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseIdtrLim : RTTimeNanoTSLegacyAsyncUseIdtrLim; + else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS) + pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseRdtscp : RTTimeNanoTSLegacyAsyncUseRdtscp; + else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL) + pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseRdtscpGroupChNumCl : RTTimeNanoTSLegacyAsyncUseRdtscpGroupChNumCl; + else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_0B) + pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseApicIdExt0B : RTTimeNanoTSLegacyAsyncUseApicIdExt0B; + else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_8000001E) + pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseApicIdExt8000001E : RTTimeNanoTSLegacyAsyncUseApicIdExt8000001E; + else + pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseApicId : RTTimeNanoTSLegacyAsyncUseApicId; +# endif + break; +#endif + default: + AssertFatalMsgFailed(("pVM=%p pGip=%p u32Mode=%#x\n", pVM, pGip, pGip->u32Mode)); + } + } +#ifdef IN_RING3 + else + pfnWorker = tmR3VirtualNanoTSDriverless; +#endif + + /* + * Update the pfnVirtualGetRaw pointer and call the worker we selected. + */ + ASMAtomicWritePtr((void * volatile *)&pVM->VMCC_CTX(tm).s.pfnVirtualGetRaw, (void *)(uintptr_t)pfnWorker); + return pfnWorker(pData, pExtra); +} + + +/** + * @interface_method_impl{RTTIMENANOTSDATA,pfnBadCpuIndex} + */ +DECLCALLBACK(DECLEXPORT(uint64_t)) tmVirtualNanoTSBadCpuIndex(PRTTIMENANOTSDATA pData, PRTITMENANOTSEXTRA pExtra, + uint16_t idApic, uint16_t iCpuSet, uint16_t iGipCpu) +{ + PVMCC pVM = RT_FROM_CPP_MEMBER(pData, VMCC, VMCC_CTX(tm).s.VirtualGetRawData); + AssertFatalMsgFailed(("pVM=%p idApic=%#x iCpuSet=%#x iGipCpu=%#x pExtra=%p\n", pVM, idApic, iCpuSet, iGipCpu, pExtra)); +#ifndef _MSC_VER + return UINT64_MAX; +#endif +} + + +/** + * Wrapper around the IPRT GIP time methods. + */ +DECLINLINE(uint64_t) tmVirtualGetRawNanoTS(PVMCC pVM) +{ +#ifdef IN_RING3 + uint64_t u64 = pVM->tm.s.pfnVirtualGetRaw(&pVM->tm.s.VirtualGetRawData, NULL /*pExtra*/); +#elif defined(IN_RING0) + uint32_t cPrevSteps = pVM->tmr0.s.VirtualGetRawData.c1nsSteps; + uint64_t u64 = pVM->tmr0.s.pfnVirtualGetRaw(&pVM->tmr0.s.VirtualGetRawData, NULL /*pExtra*/); + if (cPrevSteps != pVM->tmr0.s.VirtualGetRawData.c1nsSteps) + VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); +#else +# error "unsupported context" +#endif + /*DBGFTRACE_POS_U64(pVM, u64);*/ + return u64; +} + + +/** + * Wrapper around the IPRT GIP time methods, extended version. + */ +DECLINLINE(uint64_t) tmVirtualGetRawNanoTSEx(PVMCC pVM, uint64_t *puTscNow) +{ + RTITMENANOTSEXTRA Extra; +#ifdef IN_RING3 + uint64_t u64 = pVM->tm.s.pfnVirtualGetRaw(&pVM->tm.s.VirtualGetRawData, &Extra); +#elif defined(IN_RING0) + uint32_t cPrevSteps = pVM->tmr0.s.VirtualGetRawData.c1nsSteps; + uint64_t u64 = pVM->tmr0.s.pfnVirtualGetRaw(&pVM->tmr0.s.VirtualGetRawData, &Extra); + if (cPrevSteps != pVM->tmr0.s.VirtualGetRawData.c1nsSteps) + VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); +#else +# error "unsupported context" +#endif + if (puTscNow) + *puTscNow = Extra.uTSCValue; + /*DBGFTRACE_POS_U64(pVM, u64);*/ + return u64; +} + + +/** + * Get the time when we're not running at 100% + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * @param puTscNow Where to return the TSC corresponding to the returned + * timestamp (delta adjusted). Optional. + */ +static uint64_t tmVirtualGetRawNonNormal(PVMCC pVM, uint64_t *puTscNow) +{ + /* + * Recalculate the RTTimeNanoTS() value for the period where + * warp drive has been enabled. + */ + uint64_t u64 = tmVirtualGetRawNanoTSEx(pVM, puTscNow); + u64 -= pVM->tm.s.u64VirtualWarpDriveStart; + u64 *= pVM->tm.s.u32VirtualWarpDrivePercentage; + u64 /= 100; + u64 += pVM->tm.s.u64VirtualWarpDriveStart; + + /* + * Now we apply the virtual time offset. + * (Which is the negated tmVirtualGetRawNanoTS() value for when the virtual + * machine started if it had been running continuously without any suspends.) + */ + u64 -= pVM->tm.s.u64VirtualOffset; + return u64; +} + + +/** + * Get the raw virtual time. + * + * @returns The current time stamp. + * @param pVM The cross context VM structure. + */ +DECLINLINE(uint64_t) tmVirtualGetRaw(PVMCC pVM) +{ + if (RT_LIKELY(!pVM->tm.s.fVirtualWarpDrive)) + return tmVirtualGetRawNanoTS(pVM) - pVM->tm.s.u64VirtualOffset; + return tmVirtualGetRawNonNormal(pVM, NULL /*puTscNow*/); +} + + +/** + * Get the raw virtual time, extended version. + * + * @returns The current time stamp. + * @param pVM The cross context VM structure. + * @param puTscNow Where to return the TSC corresponding to the returned + * timestamp (delta adjusted). Optional. + */ +DECLINLINE(uint64_t) tmVirtualGetRawEx(PVMCC pVM, uint64_t *puTscNow) +{ + if (RT_LIKELY(!pVM->tm.s.fVirtualWarpDrive)) + return tmVirtualGetRawNanoTSEx(pVM, puTscNow) - pVM->tm.s.u64VirtualOffset; + return tmVirtualGetRawNonNormal(pVM, puTscNow); +} + + +/** + * Inlined version of tmVirtualGetEx. + */ +DECLINLINE(uint64_t) tmVirtualGet(PVMCC pVM, bool fCheckTimers) +{ + uint64_t u64; + if (RT_LIKELY(pVM->tm.s.cVirtualTicking)) + { + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualGet); + u64 = tmVirtualGetRaw(pVM); + + /* + * Use the chance to check for expired timers. + */ + if (fCheckTimers) + { + PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu); + if ( !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER) + && !pVM->tm.s.fRunningQueues + && ( pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire <= u64 + || ( pVM->tm.s.fVirtualSyncTicking + && pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64 - pVM->tm.s.offVirtualSync + ) + ) + && !pVM->tm.s.fRunningQueues + ) + { + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualGetSetFF); + Log5(("TMAllVirtual(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))); + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER); +#ifdef IN_RING3 + VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM); +#endif + } + } + } + else + u64 = pVM->tm.s.u64Virtual; + return u64; +} + + +/** + * Gets the current TMCLOCK_VIRTUAL time + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * + * @remark While the flow of time will never go backwards, the speed of the + * progress varies due to inaccurate RTTimeNanoTS and TSC. The latter can be + * influenced by power saving (SpeedStep, PowerNow!), while the former + * makes use of TSC and kernel timers. + */ +VMM_INT_DECL(uint64_t) TMVirtualGet(PVMCC pVM) +{ + return tmVirtualGet(pVM, true /*fCheckTimers*/); +} + + +/** + * Gets the current TMCLOCK_VIRTUAL time without checking + * timers or anything. + * + * Meaning, this has no side effect on FFs like TMVirtualGet may have. + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * + * @remarks See TMVirtualGet. + */ +VMM_INT_DECL(uint64_t) TMVirtualGetNoCheck(PVMCC pVM) +{ + return tmVirtualGet(pVM, false /*fCheckTimers*/); +} + + +/** + * Converts the dead line interval from TMCLOCK_VIRTUAL to host nano seconds. + * + * @returns Host nano second count. + * @param pVM The cross context VM structure. + * @param cVirtTicksToDeadline The TMCLOCK_VIRTUAL interval. + */ +DECLINLINE(uint64_t) tmVirtualVirtToNsDeadline(PVM pVM, uint64_t cVirtTicksToDeadline) +{ + if (RT_UNLIKELY(pVM->tm.s.fVirtualWarpDrive)) + return ASMMultU64ByU32DivByU32(cVirtTicksToDeadline, 100, pVM->tm.s.u32VirtualWarpDrivePercentage); + return cVirtTicksToDeadline; +} + + +/** + * tmVirtualSyncGetLocked worker for handling catch-up when owning the lock. + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * @param u64 raw virtual time. + * @param off offVirtualSync. + * @param pcNsToDeadline Where to return the number of nano seconds to + * the next virtual sync timer deadline. Can be + * NULL. + * @param pnsAbsDeadline Where to return the absolute deadline. + * Optional. + */ +DECLINLINE(uint64_t) tmVirtualSyncGetHandleCatchUpLocked(PVMCC pVM, uint64_t u64, uint64_t off, + uint64_t *pcNsToDeadline, uint64_t *pnsAbsDeadline) +{ + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked); + + /* + * Don't make updates until we've check the timer queue. + */ + bool fUpdatePrev = true; + bool fUpdateOff = true; + bool fStop = false; + const uint64_t u64Prev = pVM->tm.s.u64VirtualSyncCatchUpPrev; + uint64_t u64Delta = u64 - u64Prev; + if (RT_LIKELY(!(u64Delta >> 32))) + { + uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, pVM->tm.s.u32VirtualSyncCatchUpPercentage, 100); + if (off > u64Sub + pVM->tm.s.offVirtualSyncGivenUp) + { + off -= u64Sub; + Log4(("TM: %'RU64/-%'8RU64: sub %RU32 [vsghcul]\n", u64 - off, off - pVM->tm.s.offVirtualSyncGivenUp, u64Sub)); + } + else + { + /* we've completely caught up. */ + STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c); + off = pVM->tm.s.offVirtualSyncGivenUp; + fStop = true; + Log4(("TM: %'RU64/0: caught up [vsghcul]\n", u64)); + } + } + else + { + /* More than 4 seconds since last time (or negative), ignore it. */ + fUpdateOff = false; + fUpdatePrev = !(u64Delta & RT_BIT_64(63)); + Log(("TMVirtualGetSync: u64Delta=%RX64\n", u64Delta)); + } + + /* + * Complete the calculation of the current TMCLOCK_VIRTUAL_SYNC time. The current + * approach is to never pass the head timer. So, when we do stop the clock and + * set the timer pending flag. + */ + u64 -= off; + + uint64_t u64Last = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync); + if (u64Last > u64) + { + u64 = u64Last + 1; + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetAdjLast); + } + + uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); + if (pnsAbsDeadline) + *pnsAbsDeadline = u64Expire; /* Always return the unadjusted absolute deadline, or HM will waste time going + thru this code over an over again even if there aren't any timer changes. */ + if (u64 < u64Expire) + { + ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64); + if (fUpdateOff) + ASMAtomicWriteU64(&pVM->tm.s.offVirtualSync, off); + if (fStop) + ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false); + if (fUpdatePrev) + ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev, u64); + if (pcNsToDeadline) + { + uint64_t cNsToDeadline = u64Expire - u64; + if (pVM->tm.s.fVirtualSyncCatchUp) + cNsToDeadline = ASMMultU64ByU32DivByU32(cNsToDeadline, 100, + pVM->tm.s.u32VirtualSyncCatchUpPercentage + 100); + *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, cNsToDeadline); + } + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); + } + else + { + u64 = u64Expire; + ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64); + ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false); + + VM_FF_SET(pVM, VM_FF_TM_VIRTUAL_SYNC); + PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu); + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER); + Log5(("TMAllVirtual(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))); + Log4(("TM: %'RU64/-%'8RU64: exp tmr=>ff [vsghcul]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp)); + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); + + if (pcNsToDeadline) + *pcNsToDeadline = 0; +#ifdef IN_RING3 + VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM); +#endif + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF); + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetExpired); + } + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked); + + Log6(("tmVirtualSyncGetHandleCatchUpLocked -> %'RU64\n", u64)); + DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetHandleCatchUpLocked"); + return u64; +} + + +/** + * tmVirtualSyncGetEx worker for when we get the lock. + * + * @returns timesamp. + * @param pVM The cross context VM structure. + * @param u64 The virtual clock timestamp. + * @param pcNsToDeadline Where to return the number of nano seconds to + * the next virtual sync timer deadline. Can be + * NULL. + * @param pnsAbsDeadline Where to return the absolute deadline. + * Optional. + */ +DECLINLINE(uint64_t) tmVirtualSyncGetLocked(PVMCC pVM, uint64_t u64, uint64_t *pcNsToDeadline, uint64_t *pnsAbsDeadline) +{ + /* + * Not ticking? + */ + if (!pVM->tm.s.fVirtualSyncTicking) + { + u64 = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync); + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); + if (pcNsToDeadline) + *pcNsToDeadline = 0; + if (pnsAbsDeadline) + *pnsAbsDeadline = u64; + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked); + Log6(("tmVirtualSyncGetLocked -> %'RU64 [stopped]\n", u64)); + DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetLocked-stopped"); + return u64; + } + + /* + * Handle catch up in a separate function. + */ + uint64_t off = ASMAtomicUoReadU64(&pVM->tm.s.offVirtualSync); + if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + return tmVirtualSyncGetHandleCatchUpLocked(pVM, u64, off, pcNsToDeadline, pnsAbsDeadline); + + /* + * Complete the calculation of the current TMCLOCK_VIRTUAL_SYNC time. The current + * approach is to never pass the head timer. So, when we do stop the clock and + * set the timer pending flag. + */ + u64 -= off; + + uint64_t u64Last = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync); + if (u64Last > u64) + { + u64 = u64Last + 1; + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetAdjLast); + } + + uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); + if (pnsAbsDeadline) + *pnsAbsDeadline = u64Expire; + if (u64 < u64Expire) + { + ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64); + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); + if (pcNsToDeadline) + *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, u64Expire - u64); + } + else + { + u64 = u64Expire; + ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64); + ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false); + + VM_FF_SET(pVM, VM_FF_TM_VIRTUAL_SYNC); + PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu); + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER); + Log5(("TMAllVirtual(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))); + Log4(("TM: %'RU64/-%'8RU64: exp tmr=>ff [vsgl]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp)); + PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock); + +#ifdef IN_RING3 + VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM); +#endif + if (pcNsToDeadline) + *pcNsToDeadline = 0; + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF); + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetExpired); + } + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked); + Log6(("tmVirtualSyncGetLocked -> %'RU64\n", u64)); + DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetLocked"); + return u64; +} + + +/** + * Gets the current TMCLOCK_VIRTUAL_SYNC time. + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * @param fCheckTimers Check timers or not + * @param pcNsToDeadline Where to return the number of nano seconds to + * the next virtual sync timer deadline. Can be + * NULL. + * @param pnsAbsDeadline Where to return the absolute deadline. + * Optional. + * @param puTscNow Where to return the TSC corresponding to the + * returned timestamp (delta adjusted). Optional. + * @thread EMT. + */ +DECLINLINE(uint64_t) tmVirtualSyncGetEx(PVMCC pVM, bool fCheckTimers, uint64_t *pcNsToDeadline, + uint64_t *pnsAbsDeadline, uint64_t *puTscNow) +{ + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGet); + + uint64_t u64; + if (!pVM->tm.s.fVirtualSyncTicking) + { + if (pcNsToDeadline) + *pcNsToDeadline = 0; + u64 = pVM->tm.s.u64VirtualSync; + DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetEx-stopped1"); + return u64; + } + + /* + * Query the virtual clock and do the usual expired timer check. + */ + Assert(pVM->tm.s.cVirtualTicking); + u64 = tmVirtualGetRawEx(pVM, puTscNow); + if (fCheckTimers) + { + PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu); + if ( !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER) + && pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire <= u64) + { + Log5(("TMAllVirtual(%u): FF: 0 -> 1\n", __LINE__)); + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER); +#ifdef IN_RING3 + VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM /** @todo |VMNOTIFYFF_FLAGS_POKE*/); +#endif + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF); + } + } + + /* + * If we can get the lock, get it. The result is much more reliable. + * + * Note! This is where all clock source devices branch off because they + * will be owning the lock already. The 'else' is taken by code + * which is less picky or hasn't been adjusted yet + */ + /** @todo switch this around, have the tmVirtualSyncGetLocked code inlined + * here and the remainder of this function in a static worker. */ + if (PDMCritSectTryEnter(pVM, &pVM->tm.s.VirtualSyncLock) == VINF_SUCCESS) + return tmVirtualSyncGetLocked(pVM, u64, pcNsToDeadline, pnsAbsDeadline); + + /* + * When the clock is ticking, not doing catch ups and not running into an + * expired time, we can get away without locking. Try this first. + */ + uint64_t off; + if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)) + { + if (!ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + { + off = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync); + if (RT_LIKELY( ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking) + && !ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp) + && off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync))) + { + off = u64 - off; + uint64_t const u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); + if (off < u64Expire) + { + if (pnsAbsDeadline) + *pnsAbsDeadline = u64Expire; + if (pcNsToDeadline) + *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, u64Expire - off); + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLockless); + Log6(("tmVirtualSyncGetEx -> %'RU64 [lockless]\n", off)); + DBGFTRACE_U64_TAG(pVM, off, "tmVirtualSyncGetEx-lockless"); + return off; + } + } + } + } + else + { + off = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSync); + if (RT_LIKELY(!ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking))) + { + if (pcNsToDeadline) + *pcNsToDeadline = 0; + if (pnsAbsDeadline) + *pnsAbsDeadline = off; + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLockless); + Log6(("tmVirtualSyncGetEx -> %'RU64 [lockless/stopped]\n", off)); + DBGFTRACE_U64_TAG(pVM, off, "tmVirtualSyncGetEx-stopped2"); + return off; + } + } + + /* + * Read the offset and adjust if we're playing catch-up. + * + * The catch-up adjusting work by us decrementing the offset by a percentage of + * the time elapsed since the previous TMVirtualGetSync call. + * + * It's possible to get a very long or even negative interval between two read + * for the following reasons: + * - Someone might have suspended the process execution, frequently the case when + * debugging the process. + * - We might be on a different CPU which TSC isn't quite in sync with the + * other CPUs in the system. + * - Another thread is racing us and we might have been preempted while inside + * this function. + * + * Assuming nano second virtual time, we can simply ignore any intervals which has + * any of the upper 32 bits set. + */ + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + int cOuterTries = 42; + for (;; cOuterTries--) + { + /* Try grab the lock, things get simpler when owning the lock. */ + int rcLock = PDMCritSectTryEnter(pVM, &pVM->tm.s.VirtualSyncLock); + if (RT_SUCCESS_NP(rcLock)) + return tmVirtualSyncGetLocked(pVM, u64, pcNsToDeadline, pnsAbsDeadline); + + /* Re-check the ticking flag. */ + if (!ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking)) + { + off = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSync); + if ( ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking) + && cOuterTries > 0) + continue; + if (pcNsToDeadline) + *pcNsToDeadline = 0; + if (pnsAbsDeadline) + *pnsAbsDeadline = off; + Log6(("tmVirtualSyncGetEx -> %'RU64 [stopped]\n", off)); + DBGFTRACE_U64_TAG(pVM, off, "tmVirtualSyncGetEx-stopped3"); + return off; + } + + off = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync); + if (ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + { + /* No changes allowed, try get a consistent set of parameters. */ + uint64_t const u64Prev = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev); + uint64_t const offGivenUp = ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp); + uint32_t const u32Pct = ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage); + if ( ( u64Prev == ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev) + && offGivenUp == ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp) + && u32Pct == ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage) + && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + || cOuterTries <= 0) + { + uint64_t u64Delta = u64 - u64Prev; + if (RT_LIKELY(!(u64Delta >> 32))) + { + uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, u32Pct, 100); + if (off > u64Sub + offGivenUp) + { + off -= u64Sub; + Log4(("TM: %'RU64/-%'8RU64: sub %RU32 [NoLock]\n", u64 - off, pVM->tm.s.offVirtualSync - offGivenUp, u64Sub)); + } + else + { + /* we've completely caught up. */ + STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c); + off = offGivenUp; + Log4(("TM: %'RU64/0: caught up [NoLock]\n", u64)); + } + } + else + /* More than 4 seconds since last time (or negative), ignore it. */ + Log(("TMVirtualGetSync: u64Delta=%RX64 (NoLock)\n", u64Delta)); + + /* Check that we're still running and in catch up. */ + if ( ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking) + && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + break; + if (cOuterTries <= 0) + break; /* enough */ + } + } + else if ( off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) + && !ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + break; /* Got an consistent offset */ + else if (cOuterTries <= 0) + break; /* enough */ + } + if (cOuterTries <= 0) + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetELoop); + + /* + * Complete the calculation of the current TMCLOCK_VIRTUAL_SYNC time. The current + * approach is to never pass the head timer. So, when we do stop the clock and + * set the timer pending flag. + */ + u64 -= off; +/** @todo u64VirtualSyncLast */ + uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); + if (pnsAbsDeadline) + *pnsAbsDeadline = u64Expire; + if (u64 >= u64Expire) + { + PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu); + if (!VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)) + { + Log5(("TMAllVirtual(%u): FF: %d -> 1 (NoLock)\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))); + VM_FF_SET(pVM, VM_FF_TM_VIRTUAL_SYNC); /* Hmm? */ + VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER); +#ifdef IN_RING3 + VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM); +#endif + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF); + Log4(("TM: %'RU64/-%'8RU64: exp tmr=>ff [NoLock]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp)); + } + else + Log4(("TM: %'RU64/-%'8RU64: exp tmr [NoLock]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp)); + if (pcNsToDeadline) + *pcNsToDeadline = 0; + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetExpired); + } + else if (pcNsToDeadline) + { + uint64_t cNsToDeadline = u64Expire - u64; + if (ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)) + cNsToDeadline = ASMMultU64ByU32DivByU32(cNsToDeadline, 100, + ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage) + 100); + *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, cNsToDeadline); + } + + Log6(("tmVirtualSyncGetEx -> %'RU64\n", u64)); + DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetEx-nolock"); + return u64; +} + + +/** + * Gets the current TMCLOCK_VIRTUAL_SYNC time. + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * @thread EMT. + * @remarks May set the timer and virtual sync FFs. + */ +VMM_INT_DECL(uint64_t) TMVirtualSyncGet(PVMCC pVM) +{ + return tmVirtualSyncGetEx(pVM, true /*fCheckTimers*/, NULL /*pcNsToDeadline*/, NULL /*pnsAbsDeadline*/, NULL /*puTscNow*/); +} + + +/** + * Gets the current TMCLOCK_VIRTUAL_SYNC time without checking timers running on + * TMCLOCK_VIRTUAL. + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * @thread EMT. + * @remarks May set the timer and virtual sync FFs. + */ +VMM_INT_DECL(uint64_t) TMVirtualSyncGetNoCheck(PVMCC pVM) +{ + return tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, NULL /*pcNsToDeadline*/, NULL /*pnsAbsDeadline*/, NULL /*puTscNow*/); +} + + +/** + * Gets the current TMCLOCK_VIRTUAL_SYNC time without checking timers running on + * TMCLOCK_VIRTUAL, also returning corresponding TSC value. + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * @param puTscNow Where to return the TSC value that the return + * value is relative to. This is delta adjusted. + * @thread EMT. + * @remarks May set the timer and virtual sync FFs. + */ +VMM_INT_DECL(uint64_t) TMVirtualSyncGetNoCheckWithTsc(PVMCC pVM, uint64_t *puTscNow) +{ + return tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, NULL /*pcNsToDeadline*/, NULL /*pnsAbsDeadline*/, puTscNow); +} + + +/** + * Gets the current TMCLOCK_VIRTUAL_SYNC time. + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * @param fCheckTimers Check timers on the virtual clock or not. + * @thread EMT. + * @remarks May set the timer and virtual sync FFs. + */ +VMM_INT_DECL(uint64_t) TMVirtualSyncGetEx(PVMCC pVM, bool fCheckTimers) +{ + return tmVirtualSyncGetEx(pVM, fCheckTimers, NULL /*pcNsToDeadline*/, NULL /*pnsAbsDeadline*/, NULL /*puTscNow*/); +} + + +/** + * Gets the current TMCLOCK_VIRTUAL_SYNC time and ticks to the next deadline + * without checking timers running on TMCLOCK_VIRTUAL. + * + * @returns The timestamp. + * @param pVM The cross context VM structure. + * @param pcNsToDeadline Where to return the number of nano seconds to + * the next virtual sync timer deadline. + * @param puTscNow Where to return the TSC value that the return + * value is relative to. This is delta adjusted. + * @param puDeadlineVersion Where to return the deadline "version" number. + * Use with TMVirtualSyncIsCurrentDeadlineVersion() + * to check if the absolute deadline is still up to + * date and the caller can skip calling this + * function. + * @thread EMT. + * @remarks May set the timer and virtual sync FFs. + */ +VMM_INT_DECL(uint64_t) TMVirtualSyncGetWithDeadlineNoCheck(PVMCC pVM, uint64_t *pcNsToDeadline, + uint64_t *puDeadlineVersion, uint64_t *puTscNow) +{ + uint64_t cNsToDeadlineTmp; /* try convince the compiler to skip the if tests. */ + uint64_t u64Now = tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, &cNsToDeadlineTmp, puDeadlineVersion, puTscNow); + *pcNsToDeadline = cNsToDeadlineTmp; + return u64Now; +} + + +/** + * Gets the number of nano seconds to the next virtual sync deadline. + * + * @returns The number of TMCLOCK_VIRTUAL ticks. + * @param pVM The cross context VM structure. + * @param puTscNow Where to return the TSC value that the return + * value is relative to. This is delta adjusted. + * @param puDeadlineVersion Where to return the deadline "version" number. + * Use with TMVirtualSyncIsCurrentDeadlineVersion() + * to check if the absolute deadline is still up to + * date and the caller can skip calling this + * function. + * @thread EMT. + * @remarks May set the timer and virtual sync FFs. + */ +VMMDECL(uint64_t) TMVirtualSyncGetNsToDeadline(PVMCC pVM, uint64_t *puDeadlineVersion, uint64_t *puTscNow) +{ + uint64_t cNsToDeadline; + tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, &cNsToDeadline, puDeadlineVersion, puTscNow); + return cNsToDeadline; +} + + +/** + * Checks if the given deadline is still current. + * + * @retval true if the deadline is still current. + * @retval false if the deadline is outdated. + * @param pVM The cross context VM structure. + * @param uDeadlineVersion The deadline version to check. + */ +VMM_INT_DECL(bool) TMVirtualSyncIsCurrentDeadlineVersion(PVMCC pVM, uint64_t uDeadlineVersion) +{ + /** @todo Try use ASMAtomicUoReadU64 instead. */ + uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire); + return u64Expire == uDeadlineVersion; +} + + +/** + * Gets the current lag of the synchronous virtual clock (relative to the virtual clock). + * + * @return The current lag. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(uint64_t) TMVirtualSyncGetLag(PVMCC pVM) +{ + return pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp; +} + + +/** + * Get the current catch-up percent. + * + * @return The current catch0up percent. 0 means running at the same speed as the virtual clock. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(uint32_t) TMVirtualSyncGetCatchUpPct(PVMCC pVM) +{ + if (pVM->tm.s.fVirtualSyncCatchUp) + return pVM->tm.s.u32VirtualSyncCatchUpPercentage; + return 0; +} + + +/** + * Gets the current TMCLOCK_VIRTUAL frequency. + * + * @returns The frequency. + * @param pVM The cross context VM structure. + */ +VMM_INT_DECL(uint64_t) TMVirtualGetFreq(PVM pVM) +{ + NOREF(pVM); + return TMCLOCK_FREQ_VIRTUAL; +} + + +/** + * Worker for TMR3PauseClocks. + * + * @returns VINF_SUCCESS or VERR_TM_VIRTUAL_TICKING_IPE (asserted). + * @param pVM The cross context VM structure. + */ +int tmVirtualPauseLocked(PVMCC pVM) +{ + uint32_t c = ASMAtomicDecU32(&pVM->tm.s.cVirtualTicking); + AssertMsgReturn(c < pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE); + if (c == 0) + { + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualPause); + pVM->tm.s.u64Virtual = tmVirtualGetRaw(pVM); + ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false); + } + return VINF_SUCCESS; +} + + +/** + * Worker for TMR3ResumeClocks. + * + * @returns VINF_SUCCESS or VERR_TM_VIRTUAL_TICKING_IPE (asserted). + * @param pVM The cross context VM structure. + */ +int tmVirtualResumeLocked(PVMCC pVM) +{ + uint32_t c = ASMAtomicIncU32(&pVM->tm.s.cVirtualTicking); + AssertMsgReturn(c <= pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE); + if (c == 1) + { + STAM_COUNTER_INC(&pVM->tm.s.StatVirtualResume); + pVM->tm.s.u64VirtualRawPrev = 0; + pVM->tm.s.u64VirtualWarpDriveStart = tmVirtualGetRawNanoTS(pVM); + pVM->tm.s.u64VirtualOffset = pVM->tm.s.u64VirtualWarpDriveStart - pVM->tm.s.u64Virtual; + ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, true); + } + return VINF_SUCCESS; +} + + +/** + * Converts from virtual ticks to nanoseconds. + * + * @returns nanoseconds. + * @param pVM The cross context VM structure. + * @param u64VirtualTicks The virtual ticks to convert. + * @remark There could be rounding errors here. We just do a simple integer divide + * without any adjustments. + */ +VMM_INT_DECL(uint64_t) TMVirtualToNano(PVM pVM, uint64_t u64VirtualTicks) +{ + NOREF(pVM); + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return u64VirtualTicks; +} + + +/** + * Converts from virtual ticks to microseconds. + * + * @returns microseconds. + * @param pVM The cross context VM structure. + * @param u64VirtualTicks The virtual ticks to convert. + * @remark There could be rounding errors here. We just do a simple integer divide + * without any adjustments. + */ +VMM_INT_DECL(uint64_t) TMVirtualToMicro(PVM pVM, uint64_t u64VirtualTicks) +{ + NOREF(pVM); + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return u64VirtualTicks / 1000; +} + + +/** + * Converts from virtual ticks to milliseconds. + * + * @returns milliseconds. + * @param pVM The cross context VM structure. + * @param u64VirtualTicks The virtual ticks to convert. + * @remark There could be rounding errors here. We just do a simple integer divide + * without any adjustments. + */ +VMM_INT_DECL(uint64_t) TMVirtualToMilli(PVM pVM, uint64_t u64VirtualTicks) +{ + NOREF(pVM); + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return u64VirtualTicks / 1000000; +} + + +/** + * Converts from nanoseconds to virtual ticks. + * + * @returns virtual ticks. + * @param pVM The cross context VM structure. + * @param u64NanoTS The nanosecond value ticks to convert. + * @remark There could be rounding and overflow errors here. + */ +VMM_INT_DECL(uint64_t) TMVirtualFromNano(PVM pVM, uint64_t u64NanoTS) +{ + NOREF(pVM); + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return u64NanoTS; +} + + +/** + * Converts from microseconds to virtual ticks. + * + * @returns virtual ticks. + * @param pVM The cross context VM structure. + * @param u64MicroTS The microsecond value ticks to convert. + * @remark There could be rounding and overflow errors here. + */ +VMM_INT_DECL(uint64_t) TMVirtualFromMicro(PVM pVM, uint64_t u64MicroTS) +{ + NOREF(pVM); + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return u64MicroTS * 1000; +} + + +/** + * Converts from milliseconds to virtual ticks. + * + * @returns virtual ticks. + * @param pVM The cross context VM structure. + * @param u64MilliTS The millisecond value ticks to convert. + * @remark There could be rounding and overflow errors here. + */ +VMM_INT_DECL(uint64_t) TMVirtualFromMilli(PVM pVM, uint64_t u64MilliTS) +{ + NOREF(pVM); + AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000); + return u64MilliTS * 1000000; +} + diff --git a/src/VBox/VMM/VMMAll/TRPMAll.cpp b/src/VBox/VMM/VMMAll/TRPMAll.cpp new file mode 100644 index 00000000..a7901d52 --- /dev/null +++ b/src/VBox/VMM/VMMAll/TRPMAll.cpp @@ -0,0 +1,438 @@ +/* $Id: TRPMAll.cpp $ */ +/** @file + * TRPM - Trap Monitor - Any Context. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_TRPM +#include +#include +#include +#include +#include +#include +#include +#include "TRPMInternal.h" +#include +#include +#include +#include +#include +#include +#include +#include + + + +/** + * Query info about the current active trap/interrupt. + * If no trap is active active an error code is returned. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pu8TrapNo Where to store the trap number. + * @param penmType Where to store the trap type + */ +VMMDECL(int) TRPMQueryTrap(PVMCPU pVCpu, uint8_t *pu8TrapNo, TRPMEVENT *penmType) +{ + /* + * Check if we have a trap at present. + */ + if (pVCpu->trpm.s.uActiveVector != ~0U) + { + if (pu8TrapNo) + *pu8TrapNo = (uint8_t)pVCpu->trpm.s.uActiveVector; + if (penmType) + *penmType = pVCpu->trpm.s.enmActiveType; + return VINF_SUCCESS; + } + + return VERR_TRPM_NO_ACTIVE_TRAP; +} + + +/** + * Gets the trap number for the current trap. + * + * The caller is responsible for making sure there is an active trap which + * takes an error code when making this request. + * + * @returns The current trap number. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(uint8_t) TRPMGetTrapNo(PVMCPU pVCpu) +{ + AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n")); + return (uint8_t)pVCpu->trpm.s.uActiveVector; +} + + +/** + * Gets the error code for the current trap. + * + * The caller is responsible for making sure there is an active trap which + * takes an error code when making this request. + * + * @returns Error code. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(uint32_t) TRPMGetErrorCode(PVMCPU pVCpu) +{ + AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n")); +#ifdef VBOX_STRICT + switch (pVCpu->trpm.s.uActiveVector) + { + case X86_XCPT_TS: + case X86_XCPT_NP: + case X86_XCPT_SS: + case X86_XCPT_GP: + case X86_XCPT_PF: + case X86_XCPT_AC: + case X86_XCPT_DF: + break; + default: + AssertMsgFailed(("This trap (%#x) doesn't have any error code\n", pVCpu->trpm.s.uActiveVector)); + break; + } +#endif + return pVCpu->trpm.s.uActiveErrorCode; +} + + +/** + * Gets the fault address for the current trap. + * + * The caller is responsible for making sure there is an active trap 0x0e when + * making this request. + * + * @returns Fault address associated with the trap. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(RTGCUINTPTR) TRPMGetFaultAddress(PVMCPU pVCpu) +{ + AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n")); + AssertMsg(pVCpu->trpm.s.uActiveVector == X86_XCPT_PF, ("Not page-fault trap!\n")); + return pVCpu->trpm.s.uActiveCR2; +} + + +/** + * Gets the instruction-length for the current trap (only relevant for software + * interrupts and software exceptions \#BP and \#OF). + * + * The caller is responsible for making sure there is an active trap 0x0e when + * making this request. + * + * @returns Fault address associated with the trap. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(uint8_t) TRPMGetInstrLength(PVMCPU pVCpu) +{ + AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n")); + return pVCpu->trpm.s.cbInstr; +} + + +/** + * Checks if the current \#DB exception is due to an INT1/ICEBP instruction. + * + * The caller is responsible for making sure there is an active trap. + * + * @returns @c true if it's due to INT1/ICEBP, @c false if not. + * + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) TRPMIsTrapDueToIcebp(PVMCPU pVCpu) +{ + AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n")); + return pVCpu->trpm.s.fIcebp; +} + + +/** + * Clears the current active trap/exception/interrupt. + * + * The caller is responsible for making sure there is an active trap + * when making this request. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(int) TRPMResetTrap(PVMCPU pVCpu) +{ + /* + * Cannot reset non-existing trap! + */ + if (pVCpu->trpm.s.uActiveVector == ~0U) + { + AssertMsgFailed(("No active trap!\n")); + return VERR_TRPM_NO_ACTIVE_TRAP; + } + + /* + * Reset it. + */ + pVCpu->trpm.s.uActiveVector = ~0U; + return VINF_SUCCESS; +} + + +/** + * Assert trap/exception/interrupt. + * + * The caller is responsible for making sure there is no active trap + * when making this request. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param u8TrapNo The trap vector to assert. + * @param enmType Trap type. + */ +VMMDECL(int) TRPMAssertTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType) +{ + Log2(("TRPMAssertTrap: u8TrapNo=%02x type=%d\n", u8TrapNo, enmType)); + + /* + * Cannot assert a trap when one is already active. + */ + if (pVCpu->trpm.s.uActiveVector != ~0U) + { + AssertMsgFailed(("CPU%d: Active trap %#x\n", pVCpu->idCpu, pVCpu->trpm.s.uActiveVector)); + return VERR_TRPM_ACTIVE_TRAP; + } + + pVCpu->trpm.s.uActiveVector = u8TrapNo; + pVCpu->trpm.s.enmActiveType = enmType; + pVCpu->trpm.s.uActiveErrorCode = ~0U; + pVCpu->trpm.s.uActiveCR2 = 0xdeadface; + pVCpu->trpm.s.cbInstr = UINT8_MAX; + pVCpu->trpm.s.fIcebp = false; + return VINF_SUCCESS; +} + + +/** + * Assert a page-fault exception. + * + * The caller is responsible for making sure there is no active trap + * when making this request. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param uCR2 The new fault address. + * @param uErrorCode The error code for the page-fault. + */ +VMMDECL(int) TRPMAssertXcptPF(PVMCPUCC pVCpu, RTGCUINTPTR uCR2, uint32_t uErrorCode) +{ + Log2(("TRPMAssertXcptPF: uCR2=%RGv uErrorCode=%#RX32\n", uCR2, uErrorCode)); + + /* + * Cannot assert a trap when one is already active. + */ + if (pVCpu->trpm.s.uActiveVector != ~0U) + { + AssertMsgFailed(("CPU%d: Active trap %#x\n", pVCpu->idCpu, pVCpu->trpm.s.uActiveVector)); + return VERR_TRPM_ACTIVE_TRAP; + } + + pVCpu->trpm.s.uActiveVector = X86_XCPT_PF; + pVCpu->trpm.s.enmActiveType = TRPM_TRAP; + pVCpu->trpm.s.uActiveErrorCode = uErrorCode; + pVCpu->trpm.s.uActiveCR2 = uCR2; + pVCpu->trpm.s.cbInstr = UINT8_MAX; + return VINF_SUCCESS; +} + + +/** + * Sets the error code of the current trap. + * (This function is for use in trap handlers and such.) + * + * The caller is responsible for making sure there is an active trap + * which takes an errorcode when making this request. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uErrorCode The new error code. + */ +VMMDECL(void) TRPMSetErrorCode(PVMCPU pVCpu, uint32_t uErrorCode) +{ + Log2(("TRPMSetErrorCode: uErrorCode=%#RX32\n", uErrorCode)); + AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n")); + AssertMsg( pVCpu->trpm.s.enmActiveType == TRPM_TRAP + || ( pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT && pVCpu->trpm.s.uActiveVector == X86_XCPT_DB), + ("Not hardware exception or privileged software exception (INT1/ICEBP)!\n")); + pVCpu->trpm.s.uActiveErrorCode = uErrorCode; +#ifdef VBOX_STRICT + if (pVCpu->trpm.s.enmActiveType == TRPM_TRAP) + { + switch (pVCpu->trpm.s.uActiveVector) + { + case X86_XCPT_TS: case X86_XCPT_NP: case X86_XCPT_SS: case X86_XCPT_GP: case X86_XCPT_PF: + AssertMsg(uErrorCode != ~0U, ("Invalid uErrorCode=%#x u8TrapNo=%u\n", uErrorCode, pVCpu->trpm.s.uActiveVector)); + break; + case X86_XCPT_AC: case X86_XCPT_DF: + AssertMsg(uErrorCode == 0, ("Invalid uErrorCode=%#x u8TrapNo=%u\n", uErrorCode, pVCpu->trpm.s.uActiveVector)); + break; + default: + AssertMsg(uErrorCode == ~0U, ("Invalid uErrorCode=%#x u8TrapNo=%u\n", uErrorCode, pVCpu->trpm.s.uActiveVector)); + break; + } + } +#endif +} + + +/** + * Sets the fault address of the current \#PF trap. (This function is for use in + * trap handlers and such.) + * + * The caller is responsible for making sure there is an active trap 0e + * when making this request. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uCR2 The new fault address (cr2 register). + */ +VMMDECL(void) TRPMSetFaultAddress(PVMCPU pVCpu, RTGCUINTPTR uCR2) +{ + Log2(("TRPMSetFaultAddress: uCR2=%RGv\n", uCR2)); + AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n")); + AssertMsg(pVCpu->trpm.s.enmActiveType == TRPM_TRAP, ("Not hardware exception!\n")); + AssertMsg(pVCpu->trpm.s.uActiveVector == X86_XCPT_PF, ("Not trap 0e!\n")); + pVCpu->trpm.s.uActiveCR2 = uCR2; +} + + +/** + * Sets the instruction-length of the current trap (relevant for software + * interrupts and software exceptions like \#BP, \#OF). + * + * The caller is responsible for making sure there is an active trap when making + * this request. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr The instruction length. + */ +VMMDECL(void) TRPMSetInstrLength(PVMCPU pVCpu, uint8_t cbInstr) +{ + Log2(("TRPMSetInstrLength: cbInstr=%u\n", cbInstr)); + AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n")); + AssertMsg( pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT + || ( pVCpu->trpm.s.enmActiveType == TRPM_TRAP + && ( pVCpu->trpm.s.uActiveVector == X86_XCPT_BP + || pVCpu->trpm.s.uActiveVector == X86_XCPT_OF)), + ("Invalid trap type %#x\n", pVCpu->trpm.s.enmActiveType)); + pVCpu->trpm.s.cbInstr = cbInstr; +} + + +/** + * Sets if the current \#DB exception is due to an INT1/ICEBP instruction. + * + * The caller is responsible for making sure there is an active trap and it's a + * \#DB. + * + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(void) TRPMSetTrapDueToIcebp(PVMCPU pVCpu) +{ + AssertMsg(pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT, ("Trap type for INT1/ICEBP invalid!")); + AssertMsg(pVCpu->trpm.s.uActiveVector == X86_XCPT_DB, ("INT1/ICEBP must be indicated by a #DB!\n")); + pVCpu->trpm.s.fIcebp = true; +} + + +/** + * Checks if the current active trap/interrupt/exception/fault/whatever is a software + * interrupt or not. + * + * The caller is responsible for making sure there is an active trap + * when making this request. + * + * @returns true if software interrupt, false if not. + * + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) TRPMIsSoftwareInterrupt(PVMCPU pVCpu) +{ + AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n")); + return (pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT); +} + + +/** + * Check if there is an active trap. + * + * @returns true if trap active, false if not. + * @param pVCpu The cross context virtual CPU structure. + */ +VMMDECL(bool) TRPMHasTrap(PVMCPU pVCpu) +{ + return pVCpu->trpm.s.uActiveVector != ~0U; +} + + +/** + * Query all info about the current active trap/interrupt. + * If no trap is active active an error code is returned. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pu8TrapNo Where to store the trap number. + * @param pEnmType Where to store the trap type. + * @param puErrorCode Where to store the error code associated with some + * traps. ~0U is stored if the trap has no error code. + * @param puCR2 Where to store the CR2 associated with a trap 0E. + * @param pcbInstr Where to store the instruction-length associated with + * some traps. + * @param pfIcebp Where to store whether the trap is a \#DB caused by an + * INT1/ICEBP instruction. + */ +VMMDECL(int) TRPMQueryTrapAll(PVMCPU pVCpu, uint8_t *pu8TrapNo, TRPMEVENT *pEnmType, uint32_t *puErrorCode, PRTGCUINTPTR puCR2, + uint8_t *pcbInstr, bool *pfIcebp) +{ + /* + * Check if we have an active trap. + */ + if (pVCpu->trpm.s.uActiveVector == ~0U) + return VERR_TRPM_NO_ACTIVE_TRAP; + + if (pu8TrapNo) + *pu8TrapNo = (uint8_t)pVCpu->trpm.s.uActiveVector; + if (pEnmType) + *pEnmType = pVCpu->trpm.s.enmActiveType; + if (puErrorCode) + *puErrorCode = pVCpu->trpm.s.uActiveErrorCode; + if (puCR2) + *puCR2 = pVCpu->trpm.s.uActiveCR2; + if (pcbInstr) + *pcbInstr = pVCpu->trpm.s.cbInstr; + if (pfIcebp) + *pfIcebp = pVCpu->trpm.s.fIcebp; + return VINF_SUCCESS; +} + diff --git a/src/VBox/VMM/VMMAll/VMAll.cpp b/src/VBox/VMM/VMMAll/VMAll.cpp new file mode 100644 index 00000000..67909b83 --- /dev/null +++ b/src/VBox/VMM/VMMAll/VMAll.cpp @@ -0,0 +1,444 @@ +/* $Id: VMAll.cpp $ */ +/** @file + * VM - Virtual Machine All Contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_VM +#include "VMInternal.h" +#include +#include +#include +#include +#include + +#include +#include +#include + + +#ifdef IN_RING3 + +/** + * Sets the error message. + * + * @returns rc. Meaning you can do: + * @code + * return VM_SET_ERROR(pVM, VERR_OF_YOUR_CHOICE, "descriptive message"); + * @endcode + * @param pVM The cross context VM structure. + * @param rc VBox status code. + * @param SRC_POS Use RT_SRC_POS. + * @param pszFormat Error message format string. + * @param ... Error message arguments. + * @thread Any + */ +VMMDECL(int) VMSetError(PVMCC pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...) +{ + va_list args; + va_start(args, pszFormat); + int rc2 = VMSetErrorV(pVM, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc == rc2); NOREF(rc2); + va_end(args); + return rc; +} + + +/** + * Sets the error message. + * + * @returns rc. Meaning you can do: + * @code + * return VM_SET_ERROR(pVM, VERR_OF_YOUR_CHOICE, "descriptive message"); + * @endcode + * @param pVM The cross context VM structure. + * @param rc VBox status code. + * @param SRC_POS Use RT_SRC_POS. + * @param pszFormat Error message format string. + * @param args Error message arguments. + * @thread Any + */ +VMMDECL(int) VMSetErrorV(PVMCC pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args) +{ +# ifdef IN_RING3 + /* + * Switch to EMT. + */ + va_list va2; + va_copy(va2, args); /* Have to make a copy here or GCC will break. */ + VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)vmR3SetErrorUV, 7, /* ASSUMES 3 source pos args! */ + pVM->pUVM, rc, RT_SRC_POS_ARGS, pszFormat, &va2); + va_end(va2); + +# else + /* + * We're already on the EMT thread and can safely create a VMERROR chunk. + */ + vmSetErrorCopy(pVM, rc, RT_SRC_POS_ARGS, pszFormat, args); + VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_VM_SET_ERROR, 0); +# endif + return rc; +} + + +/** + * Copies the error to a VMERROR structure. + * + * This is mainly intended for Ring-0 and GC where the error must be copied to + * memory accessible from ring-3. But it's just possible that we might add + * APIs for retrieving the VMERROR copy later. + * + * @param pVM The cross context VM structure. + * @param rc VBox status code. + * @param SRC_POS Use RT_SRC_POS. + * @param pszFormat Error message format string. + * @param args Error message arguments. + * @thread EMT + */ +void vmSetErrorCopy(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args) +{ + NOREF(pVM); NOREF(rc); RT_SRC_POS_NOREF(); NOREF(pszFormat); NOREF(args); +# if 0 /// @todo implement Ring-0 and GC VMSetError + /* + * Create the untranslated message copy. + */ + /* free any old message. */ + MMHyperFree(pVM, MMHyperR32Ctx(pVM, pVM->vm.s.pError)); + pVM->vm.s.pError = NULL; + + /* calc reasonable start size. */ + size_t cchFile = pszFile ? strlen(pszFile) : 0; + size_t cchFunction = pszFunction ? strlen(pszFunction) : 0; + size_t cchFormat = strlen(pszFormat); + size_t cb = sizeof(VMERROR) + + cchFile + 1 + + cchFunction + 1 + + cchFormat + 32; + + /* allocate it */ + void *pv; + int rc2 = MMHyperAlloc(pVM, cb, 0, MM_TAG_VM, &pv); + if (RT_SUCCESS(rc2)) + { + /* initialize it. */ + PVMERROR pErr = (PVMERROR)pv; + pErr->cbAllocated = cb; + pErr->iLine = iLine; + pErr->off = sizeof(VMERROR); + pErr->offFile = pErr->offFunction = 0; + + if (cchFile) + { + pErr->offFile = pErr->off; + memcpy((uint8_t *)pErr + pErr->off, pszFile, cchFile + 1); + pErr->off += cchFile + 1; + } + + if (cchFunction) + { + pErr->offFunction = pErr->off; + memcpy((uint8_t *)pErr + pErr->off, pszFunction, cchFunction + 1); + pErr->off += cchFunction + 1; + } + + pErr->offMessage = pErr->off; + + /* format the message (pErr might be reallocated) */ + VMSETERRORFMTARGS Args; + Args.pVM = pVM; + Args.pErr = pErr; + + va_list va2; + va_copy(va2, args); + RTStrFormatV(vmSetErrorFmtOut, &pErr, NULL, NULL, &pszFormatTmp, args); + va_end(va2); + + /* done. */ + pVM->vm.s.pErrorR3 = MMHyper2HC(pVM, (uintptr_t)pArgs.pErr); + } +# endif +} + +#endif /* IN_RING3 */ +#ifdef IN_RING3 + +/** + * Sets the runtime error message. + * + * As opposed VMSetError(), this method is intended to inform the VM user about + * errors and error-like conditions that happen at an arbitrary point during VM + * execution (like "host memory low" or "out of host disk space"). + * + * @returns VBox status code. For some flags the status code must be + * propagated up the stack. + * + * @param pVM The cross context VM structure. + * + * @param fFlags Flags indicating which actions to take. + * See VMSETRTERR_FLAGS_* for details on each flag. + * + * @param pszErrorId Unique error identification string. This is used by + * the frontends and maybe other devices or drivers, so + * once an ID has been selected it's essentially + * unchangable. Employ camelcase when constructing the + * string, leave out spaces. + * + * The registered runtime error callbacks should string + * switch on this and handle the ones it knows + * specifically and the unknown ones generically. + * + * @param pszFormat Error message format string. + * @param ... Error message arguments. + * + * @thread Any + */ +VMMDECL(int) VMSetRuntimeError(PVMCC pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...) +{ + va_list va; + va_start(va, pszFormat); + int rc = VMSetRuntimeErrorV(pVM, fFlags, pszErrorId, pszFormat, va); + va_end(va); + return rc; +} + + +/** + * va_list version of VMSetRuntimeError. + * + * @returns VBox status code. For some flags the status code must be + * propagated up the stack. + * + * @param pVM The cross context VM structure. + * @param fFlags Flags indicating which actions to take. See + * VMSETRTERR_FLAGS_*. + * @param pszErrorId Error ID string. + * @param pszFormat Error message format string. + * @param va Error message arguments. + * + * @thread Any + */ +VMMDECL(int) VMSetRuntimeErrorV(PVMCC pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va) +{ + Log(("VMSetRuntimeErrorV: fFlags=%#x pszErrorId=%s\n", fFlags, pszErrorId)); + + /* + * Relaxed parameter validation. + */ + AssertPtr(pVM); + AssertMsg(!(fFlags & ~(VMSETRTERR_FLAGS_NO_WAIT | VMSETRTERR_FLAGS_SUSPEND | VMSETRTERR_FLAGS_FATAL)), ("%#x\n", fFlags)); + Assert(!(fFlags & VMSETRTERR_FLAGS_NO_WAIT) || !VM_IS_EMT(pVM)); + Assert(!(fFlags & VMSETRTERR_FLAGS_SUSPEND) || !(fFlags & VMSETRTERR_FLAGS_FATAL)); + AssertPtr(pszErrorId); + Assert(*pszErrorId); + Assert(RTStrEnd(pszErrorId, 128) != NULL); + AssertPtr(pszFormat); + Assert(RTStrEnd(pszFormat, 512) != NULL); + +# ifdef IN_RING3 + /* + * Switch to EMT. + * + * If it's a no-wait request, we have to format the message into a buffer + * here since the variable arguments list will become invalid once we call + * va_end and return. + */ + int rc; + if ( !(fFlags & VMSETRTERR_FLAGS_NO_WAIT) + || VM_IS_EMT(pVM)) + { + fFlags &= ~VMSETRTERR_FLAGS_NO_WAIT; + + va_list va2; + va_copy(va2, va); /* Have to make a copy here or GCC will break. */ + rc = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, + (PFNRT)vmR3SetRuntimeErrorV, 5, pVM, fFlags, pszErrorId, pszFormat, &va2); + va_end(va2); + } + else + { + char *pszMessage = MMR3HeapAPrintfV(pVM, MM_TAG_VM, pszFormat, va); + rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY, + (PFNRT)vmR3SetRuntimeError, 4, pVM, fFlags, pszErrorId, pszMessage); + if (RT_FAILURE(rc)) + MMR3HeapFree(pszMessage); + } + +# else + /* + * We're already on the EMT and can safely create a VMRUNTIMEERROR chunk. + */ + AssertReleaseMsgFailed(("Congratulations! You will have the pleasure of debugging the RC/R0 path.\n")); + vmSetRuntimeErrorCopy(pVM, fFlags, pszErrorId, pszFormat, va); + + int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_VM_SET_RUNTIME_ERROR, 0); +# endif + + Log(("VMSetRuntimeErrorV: returns %Rrc (pszErrorId=%s)\n", rc, pszErrorId)); + return rc; +} + + +/** + * Copies the error to a VMRUNTIMEERROR structure. + * + * This is mainly intended for Ring-0 and RC where the error must be copied to + * memory accessible from ring-3. But it's just possible that we might add + * APIs for retrieving the VMRUNTIMEERROR copy later. + * + * @param pVM The cross context VM structure. + * @param fFlags The error flags. + * @param pszErrorId Error ID string. + * @param pszFormat Error message format string. + * @param va Error message arguments. This is of course spoiled + * by this call. + * @thread EMT + */ +void vmSetRuntimeErrorCopy(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va) +{ + NOREF(pVM); NOREF(fFlags); NOREF(pszErrorId); NOREF(pszFormat); NOREF(va); +# if 0 /// @todo implement Ring-0 and GC VMSetError + /* + * Create the untranslated message copy. + */ + /* free any old message. */ + MMHyperFree(pVM, MMHyperR32Ctx(pVM, pVM->vm.s.pRuntimeErrorR3)); + pVM->vm.s.pRuntimeErrorR3 = NULL; + + /* calc reasonable start size. */ + size_t cchErrorID = pszErrorId ? strlen(pszErrorId) : 0; + size_t cchFormat = strlen(pszFormat); + size_t cb = sizeof(VMRUNTIMEERROR) + + cchErrorID + 1 + + cchFormat + 32; + + /* allocate it */ + void *pv; + int rc2 = MMHyperAlloc(pVM, cb, 0, MM_TAG_VM, &pv); + if (RT_SUCCESS(rc2)) + { + /* initialize it. */ + PVMRUNTIMEERROR pErr = (PVMRUNTIMEERROR)pv; + pErr->cbAllocated = cb; + pErr->fFlags = fFlags; + pErr->off = sizeof(PVMRUNTIMEERROR); + pErr->offErrorID = 0; + + if (cchErrorID) + { + pErr->offErrorID = pErr->off; + memcpy((uint8_t *)pErr + pErr->off, pszErrorId, cchErrorID + 1); + pErr->off += cchErrorID + 1; + } + + pErr->offMessage = pErr->off; + + /* format the message (pErr might be reallocated) */ + VMSETRUNTIMEERRORFMTARGS Args; + Args.pVM = pVM; + Args.pErr = pErr; + + va_list va2; + va_copy(va2, args); + RTStrFormatV(vmSetRuntimeErrorFmtOut, &pErr, NULL, NULL, &pszFormatTmp, args); + va_end(va2); + + /* done. */ + pVM->vm.s.pErrorRuntimeR3 = MMHyper2HC(pVM, (uintptr_t)pArgs.pErr); + } +# endif +} + +#endif /* IN_RING3 */ + +/** + * Gets the name of VM state. + * + * @returns Pointer to a read-only string with the state name. + * @param enmState The state. + */ +VMMDECL(const char *) VMGetStateName(VMSTATE enmState) +{ + switch (enmState) + { +#define MY_CASE(enm) case VMSTATE_##enm: return #enm; + MY_CASE(CREATING); + MY_CASE(CREATED); + MY_CASE(RUNNING); + MY_CASE(LOADING); + MY_CASE(LOAD_FAILURE); + MY_CASE(SAVING); + MY_CASE(SUSPENDED); + MY_CASE(RESETTING); + MY_CASE(GURU_MEDITATION); + MY_CASE(OFF); + MY_CASE(DESTROYING); + MY_CASE(TERMINATED); +#undef MY_CASE + default: + return "Unknown"; + } +} + + +/** + * Gets the total reset count. + * + * @returns Reset count. UINT32_MAX if @a pVM is invalid. + * @param pVM The VM handle. + */ +VMMDECL(uint32_t) VMGetResetCount(PVMCC pVM) +{ + VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX); + return pVM->vm.s.cResets; +} + + +/** + * Gets the soft reset count. + * + * @returns Soft reset count. UINT32_MAX if @a pVM is invalid. + * @param pVM The VM handle. + */ +VMMDECL(uint32_t) VMGetSoftResetCount(PVMCC pVM) +{ + VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX); + return pVM->vm.s.cSoftResets; +} + + +/** + * Gets the hard reset count. + * + * @returns Hard reset count. UINT32_MAX if @a pVM is invalid. + * @param pVM The VM handle. + */ +VMMDECL(uint32_t) VMGetHardResetCount(PVMCC pVM) +{ + VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX); + return pVM->vm.s.cHardResets; +} + diff --git a/src/VBox/VMM/VMMAll/VMMAll.cpp b/src/VBox/VMM/VMMAll/VMMAll.cpp new file mode 100644 index 00000000..b8ba8eb8 --- /dev/null +++ b/src/VBox/VMM/VMMAll/VMMAll.cpp @@ -0,0 +1,296 @@ +/* $Id: VMMAll.cpp $ */ +/** @file + * VMM All Contexts. + */ + +/* + * Copyright (C) 2006-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Header Files * +*********************************************************************************************************************************/ +#define LOG_GROUP LOG_GROUP_VMM +#include +#include "VMMInternal.h" +#include +#ifdef IN_RING0 +# include +#endif +#include +#include +#include +#include +#include + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +/** User counter for the vmmInitFormatTypes function (pro forma). */ +static volatile uint32_t g_cFormatTypeUsers = 0; + + +/** + * Helper that formats a decimal number in the range 0..9999. + * + * @returns The length of the formatted number. + * @param pszBuf Output buffer with sufficient space. + * @param uNumber The number to format. + */ +static unsigned vmmFormatTypeShortNumber(char *pszBuf, uint32_t uNumber) +{ + unsigned off = 0; + if (uNumber >= 10) + { + if (uNumber >= 100) + { + if (uNumber >= 1000) + pszBuf[off++] = ((uNumber / 1000) % 10) + '0'; + pszBuf[off++] = ((uNumber / 100) % 10) + '0'; + } + pszBuf[off++] = ((uNumber / 10) % 10) + '0'; + } + pszBuf[off++] = (uNumber % 10) + '0'; + pszBuf[off] = '\0'; + return off; +} + + +/** + * @callback_method_impl{FNRTSTRFORMATTYPE, vmsetcpu} + */ +static DECLCALLBACK(size_t) vmmFormatTypeVmCpuSet(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput, + const char *pszType, void const *pvValue, + int cchWidth, int cchPrecision, unsigned fFlags, + void *pvUser) +{ + NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(fFlags); + + PCVMCPUSET pSet = (PCVMCPUSET)pvValue; + uint32_t cCpus = 0; + uint32_t iCpu = RT_ELEMENTS(pSet->au32Bitmap) * 32; + while (iCpu--) + if (VMCPUSET_IS_PRESENT(pSet, iCpu)) + cCpus++; + + char szTmp[32]; + AssertCompile(RT_ELEMENTS(pSet->au32Bitmap) * 32 < 999); + if (cCpus == 1) + { + iCpu = RT_ELEMENTS(pSet->au32Bitmap) * 32; + while (iCpu--) + if (VMCPUSET_IS_PRESENT(pSet, iCpu)) + { + szTmp[0] = 'c'; + szTmp[1] = 'p'; + szTmp[2] = 'u'; + return pfnOutput(pvArgOutput, szTmp, 3 + vmmFormatTypeShortNumber(&szTmp[3], iCpu)); + } + cCpus = 0; + } + if (cCpus == 0) + return pfnOutput(pvArgOutput, RT_STR_TUPLE("")); + if (cCpus == RT_ELEMENTS(pSet->au32Bitmap) * 32) + return pfnOutput(pvArgOutput, RT_STR_TUPLE("")); + + /* + * Print cpus that are present: {1,2,7,9 ... } + */ + size_t cchRet = pfnOutput(pvArgOutput, "{", 1); + + cCpus = 0; + iCpu = 0; + while (iCpu < RT_ELEMENTS(pSet->au32Bitmap) * 32) + { + if (VMCPUSET_IS_PRESENT(pSet, iCpu)) + { + /* Output the first cpu number. */ + int off = 0; + if (cCpus != 0) + szTmp[off++] = ','; + cCpus++; + off += vmmFormatTypeShortNumber(&szTmp[off], iCpu); + + /* Check for sequence. */ + uint32_t const iStart = ++iCpu; + while ( iCpu < RT_ELEMENTS(pSet->au32Bitmap) * 32 + && VMCPUSET_IS_PRESENT(pSet, iCpu)) + { + iCpu++; + cCpus++; + } + if (iCpu != iStart) + { + szTmp[off++] = '-'; + off += vmmFormatTypeShortNumber(&szTmp[off], iCpu); + } + + /* Terminate and output. */ + szTmp[off] = '\0'; + cchRet += pfnOutput(pvArgOutput, szTmp, off); + } + iCpu++; + } + + cchRet += pfnOutput(pvArgOutput, "}", 1); + NOREF(pvUser); + return cchRet; +} + + +/** + * Registers the VMM wide format types. + * + * Called by VMMR3Init, VMMR0Init and VMMRCInit. + */ +int vmmInitFormatTypes(void) +{ + int rc = VINF_SUCCESS; + if (ASMAtomicIncU32(&g_cFormatTypeUsers) == 1) + rc = RTStrFormatTypeRegister("vmcpuset", vmmFormatTypeVmCpuSet, NULL); + return rc; +} + + +/** + * Counterpart to vmmInitFormatTypes, called by VMMR3Term and VMMR0Term. + */ +void vmmTermFormatTypes(void) +{ + if (ASMAtomicDecU32(&g_cFormatTypeUsers) == 0) + RTStrFormatTypeDeregister("vmcpuset"); +} + + +/** + * Gets the ID of the virtual CPU associated with the calling thread. + * + * @returns The CPU ID. NIL_VMCPUID if the thread isn't an EMT. + * + * @param pVM The cross context VM structure. + * @internal + */ +VMMDECL(VMCPUID) VMMGetCpuId(PVMCC pVM) +{ +#if defined(IN_RING3) + return VMR3GetVMCPUId(pVM); + +#elif defined(IN_RING0) + PVMCPUCC pVCpu = GVMMR0GetGVCpuByGVMandEMT(pVM, NIL_RTNATIVETHREAD); + return pVCpu ? pVCpu->idCpu : NIL_VMCPUID; + +#else /* RC: Always EMT(0) */ + NOREF(pVM); + return 0; +#endif +} + + +/** + * Returns the VMCPU of the calling EMT. + * + * @returns The VMCPU pointer. NULL if not an EMT. + * + * @param pVM The cross context VM structure. + * @internal + */ +VMMDECL(PVMCPUCC) VMMGetCpu(PVMCC pVM) +{ +#ifdef IN_RING3 + VMCPUID idCpu = VMR3GetVMCPUId(pVM); + if (idCpu == NIL_VMCPUID) + return NULL; + Assert(idCpu < pVM->cCpus); + return VMCC_GET_CPU(pVM, idCpu); + +#elif defined(IN_RING0) + return GVMMR0GetGVCpuByGVMandEMT(pVM, NIL_RTNATIVETHREAD); + +#else /* RC: Always EMT(0) */ + RT_NOREF(pVM); + return &g_VCpu0; +#endif /* IN_RING0 */ +} + + +/** + * Returns the VMCPU of the first EMT thread. + * + * @returns The VMCPU pointer. + * @param pVM The cross context VM structure. + * @internal + */ +VMMDECL(PVMCPUCC) VMMGetCpu0(PVMCC pVM) +{ + Assert(pVM->cCpus == 1); + return VMCC_GET_CPU_0(pVM); +} + + +/** + * Returns the VMCPU of the specified virtual CPU. + * + * @returns The VMCPU pointer. NULL if idCpu is invalid. + * + * @param pVM The cross context VM structure. + * @param idCpu The ID of the virtual CPU. + * @internal + */ +VMMDECL(PVMCPUCC) VMMGetCpuById(PVMCC pVM, RTCPUID idCpu) +{ + AssertReturn(idCpu < pVM->cCpus, NULL); + return VMCC_GET_CPU(pVM, idCpu); +} + + +/** + * Gets the VBOX_SVN_REV. + * + * This is just to avoid having to compile a bunch of big files + * and requires less Makefile mess. + * + * @returns VBOX_SVN_REV. + */ +VMM_INT_DECL(uint32_t) VMMGetSvnRev(void) +{ + return VBOX_SVN_REV; +} + + +/** + * Returns the build type for matching components. + * + * @returns Build type value. + */ +uint32_t vmmGetBuildType(void) +{ + uint32_t uRet = 0xbeef0000; +#ifdef DEBUG + uRet |= RT_BIT_32(0); +#endif +#ifdef VBOX_WITH_STATISTICS + uRet |= RT_BIT_32(1); +#endif + return uRet; +} + diff --git a/src/VBox/VMM/VMMAll/VMMAllA.asm b/src/VBox/VMM/VMMAll/VMMAllA.asm new file mode 100644 index 00000000..93a25e76 --- /dev/null +++ b/src/VBox/VMM/VMMAll/VMMAllA.asm @@ -0,0 +1,93 @@ +; $Id: VMMAllA.asm $ +;; @file +; VMM - All Contexts Assembly Routines. +; + +; +; Copyright (C) 2009-2023 Oracle and/or its affiliates. +; +; This file is part of VirtualBox base platform packages, as +; available from https://www.virtualbox.org. +; +; This program is free software; you can redistribute it and/or +; modify it under the terms of the GNU General Public License +; as published by the Free Software Foundation, in version 3 of the +; License. +; +; This program is distributed in the hope that it will be useful, but +; WITHOUT ANY WARRANTY; without even the implied warranty of +; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +; General Public License for more details. +; +; You should have received a copy of the GNU General Public License +; along with this program; if not, see . +; +; SPDX-License-Identifier: GPL-3.0-only +; + +;******************************************************************************* +;* Header Files * +;******************************************************************************* +%include "VBox/asmdefs.mac" + + +;******************************************************************************* +;* Defined Constants And Macros * +;******************************************************************************* +%ifdef IN_RING3 + %ifdef RT_ARCH_AMD64 + %define VMM_TRASH_XMM_REGS + %endif +%endif +%ifdef IN_RING0 + %ifdef RT_ARCH_AMD64 + %ifdef RT_OS_WINDOWS + %define VMM_TRASH_XMM_REGS + %endif + %endif +%endif +%ifndef VMM_TRASH_XMM_REGS + %ifdef VBOX_WITH_KERNEL_USING_XMM + %define VMM_TRASH_XMM_REGS + %endif +%endif + +BEGINCODE + + +;; +; Trashes the volatile XMM registers in the current ABI. +; +BEGINPROC VMMTrashVolatileXMMRegs +%ifdef VMM_TRASH_XMM_REGS + push xBP + mov xBP, xSP + + ; take whatever is on the stack. + and xSP, ~15 + sub xSP, 80h + + movdqa xmm0, [xSP + 0] + movdqa xmm1, [xSP + 010h] + movdqa xmm2, [xSP + 020h] + movdqa xmm3, [xSP + 030h] + movdqa xmm4, [xSP + 040h] + movdqa xmm5, [xSP + 050h] + %ifdef ASM_CALL64_GCC + movdqa xmm6, [xSP + 060h] + movdqa xmm7, [xSP + 070h] + movdqa xmm8, [xSP + 000h] + movdqa xmm9, [xSP + 010h] + movdqa xmm10,[xSP + 020h] + movdqa xmm11,[xSP + 030h] + movdqa xmm12,[xSP + 040h] + movdqa xmm13,[xSP + 050h] + movdqa xmm14,[xSP + 060h] + movdqa xmm15,[xSP + 070h] + %endif + leave +%endif ; VMM_TRASH_XMM_REGS + xor eax, eax ; for good measure. + ret +ENDPROC VMMTrashVolatileXMMRegs + diff --git a/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h b/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h new file mode 100644 index 00000000..85db8a47 --- /dev/null +++ b/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h @@ -0,0 +1,12003 @@ +/* $Id: VMXAllTemplate.cpp.h $ */ +/** @file + * HM VMX (Intel VT-x) - Code template for our own hypervisor and the NEM darwin backend using Apple's Hypervisor.framework. + */ + +/* + * Copyright (C) 2012-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * SPDX-License-Identifier: GPL-3.0-only + */ + + +/********************************************************************************************************************************* +* Defined Constants And Macros * +*********************************************************************************************************************************/ +#if !defined(VMX_VMCS_WRITE_16) || !defined(VMX_VMCS_WRITE_32) || !defined(VMX_VMCS_WRITE_64) || !defined(VMX_VMCS_WRITE_64) +# error "At least one of the VMX_VMCS_WRITE_16, VMX_VMCS_WRITE_32, VMX_VMCS_WRITE_64 or VMX_VMCS_WRITE_64 is missing" +#endif + + +#if !defined(VMX_VMCS_READ_16) || !defined(VMX_VMCS_READ_32) || !defined(VMX_VMCS_READ_64) || !defined(VMX_VMCS_READ_64) +# error "At least one of the VMX_VMCS_READ_16, VMX_VMCS_READ_32, VMX_VMCS_READ_64 or VMX_VMCS_READ_64 is missing" +#endif + +/** Enables condensing of VMREAD instructions, see vmxHCReadToTransient(). */ +#define HMVMX_WITH_CONDENSED_VMREADS + +/** Use the function table. */ +#define HMVMX_USE_FUNCTION_TABLE + +/** Determine which tagged-TLB flush handler to use. */ +#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0 +#define HMVMX_FLUSH_TAGGED_TLB_EPT 1 +#define HMVMX_FLUSH_TAGGED_TLB_VPID 2 +#define HMVMX_FLUSH_TAGGED_TLB_NONE 3 + +/** Assert that all the given fields have been read from the VMCS. */ +#ifdef VBOX_STRICT +# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \ + do { \ + uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \ + Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \ + } while (0) +#else +# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0) +#endif + +/** + * Subset of the guest-CPU state that is kept by VMX R0 code while executing the + * guest using hardware-assisted VMX. + * + * This excludes state like GPRs (other than RSP) which are always are + * swapped and restored across the world-switch and also registers like EFER, + * MSR which cannot be modified by the guest without causing a VM-exit. + */ +#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \ + | CPUMCTX_EXTRN_RFLAGS \ + | CPUMCTX_EXTRN_RSP \ + | CPUMCTX_EXTRN_SREG_MASK \ + | CPUMCTX_EXTRN_TABLE_MASK \ + | CPUMCTX_EXTRN_KERNEL_GS_BASE \ + | CPUMCTX_EXTRN_SYSCALL_MSRS \ + | CPUMCTX_EXTRN_SYSENTER_MSRS \ + | CPUMCTX_EXTRN_TSC_AUX \ + | CPUMCTX_EXTRN_OTHER_MSRS \ + | CPUMCTX_EXTRN_CR0 \ + | CPUMCTX_EXTRN_CR3 \ + | CPUMCTX_EXTRN_CR4 \ + | CPUMCTX_EXTRN_DR7 \ + | CPUMCTX_EXTRN_HWVIRT \ + | CPUMCTX_EXTRN_INHIBIT_INT \ + | CPUMCTX_EXTRN_INHIBIT_NMI) + +/** + * Exception bitmap mask for real-mode guests (real-on-v86). + * + * We need to intercept all exceptions manually except: + * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking + * due to bugs in Intel CPUs. + * - \#PF need not be intercepted even in real-mode if we have nested paging + * support. + */ +#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \ + | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \ + | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \ + | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \ + | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \ + | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \ + | RT_BIT(X86_XCPT_XF)) + +/** Maximum VM-instruction error number. */ +#define HMVMX_INSTR_ERROR_MAX 28 + +/** Profiling macro. */ +#ifdef HM_PROFILE_EXIT_DISPATCH +# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed) +# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed) +#else +# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0) +# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0) +#endif + +#ifndef IN_NEM_DARWIN +/** Assert that preemption is disabled or covered by thread-context hooks. */ +# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \ + || !RTThreadPreemptIsEnabled(NIL_RTTHREAD)) + +/** Assert that we haven't migrated CPUs when thread-context hooks are not + * used. */ +# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \ + || (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \ + ("Illegal migration! Entered on CPU %u Current %u\n", \ + (a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId())) +#else +# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) do { } while (0) +# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) do { } while (0) +#endif + +/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU + * context. */ +#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \ + ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \ + (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz))) + +/** Log the VM-exit reason with an easily visible marker to identify it in a + * potential sea of logging data. */ +#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \ + do { \ + Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \ + HMGetVmxExitName(a_uExitReason))); \ + } while (0) \ + + +/********************************************************************************************************************************* +* Structures and Typedefs * +*********************************************************************************************************************************/ +/** + * Memory operand read or write access. + */ +typedef enum VMXMEMACCESS +{ + VMXMEMACCESS_READ = 0, + VMXMEMACCESS_WRITE = 1 +} VMXMEMACCESS; + + +/** + * VMX VM-exit handler. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + */ +#ifndef HMVMX_USE_FUNCTION_TABLE +typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient); +#else +typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)); +/** Pointer to VM-exit handler. */ +typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER; +#endif + +/** + * VMX VM-exit handler, non-strict status code. + * + * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI. + * + * @returns VBox status code, no informational status code returned. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the + * use of that status code will be replaced with VINF_EM_SOMETHING + * later when switching over to IEM. + */ +#ifndef HMVMX_USE_FUNCTION_TABLE +typedef int FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient); +#else +typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC; +#endif + + +/********************************************************************************************************************************* +* Internal Functions * +*********************************************************************************************************************************/ +#ifndef HMVMX_USE_FUNCTION_TABLE +DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient); +# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC) +# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int) +#else +# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC) +# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL +#endif +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient); +#endif + +static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat); + +/** @name VM-exit handler prototypes. + * @{ + */ +static FNVMXEXITHANDLER vmxHCExitXcptOrNmi; +static FNVMXEXITHANDLER vmxHCExitExtInt; +static FNVMXEXITHANDLER vmxHCExitTripleFault; +static FNVMXEXITHANDLERNSRC vmxHCExitIntWindow; +static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindow; +static FNVMXEXITHANDLER vmxHCExitTaskSwitch; +static FNVMXEXITHANDLER vmxHCExitCpuid; +static FNVMXEXITHANDLER vmxHCExitGetsec; +static FNVMXEXITHANDLER vmxHCExitHlt; +static FNVMXEXITHANDLERNSRC vmxHCExitInvd; +static FNVMXEXITHANDLER vmxHCExitInvlpg; +static FNVMXEXITHANDLER vmxHCExitRdpmc; +static FNVMXEXITHANDLER vmxHCExitVmcall; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +static FNVMXEXITHANDLER vmxHCExitVmclear; +static FNVMXEXITHANDLER vmxHCExitVmlaunch; +static FNVMXEXITHANDLER vmxHCExitVmptrld; +static FNVMXEXITHANDLER vmxHCExitVmptrst; +static FNVMXEXITHANDLER vmxHCExitVmread; +static FNVMXEXITHANDLER vmxHCExitVmresume; +static FNVMXEXITHANDLER vmxHCExitVmwrite; +static FNVMXEXITHANDLER vmxHCExitVmxoff; +static FNVMXEXITHANDLER vmxHCExitVmxon; +static FNVMXEXITHANDLER vmxHCExitInvvpid; +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +static FNVMXEXITHANDLER vmxHCExitInvept; +# endif +#endif +static FNVMXEXITHANDLER vmxHCExitRdtsc; +static FNVMXEXITHANDLER vmxHCExitMovCRx; +static FNVMXEXITHANDLER vmxHCExitMovDRx; +static FNVMXEXITHANDLER vmxHCExitIoInstr; +static FNVMXEXITHANDLER vmxHCExitRdmsr; +static FNVMXEXITHANDLER vmxHCExitWrmsr; +static FNVMXEXITHANDLER vmxHCExitMwait; +static FNVMXEXITHANDLER vmxHCExitMtf; +static FNVMXEXITHANDLER vmxHCExitMonitor; +static FNVMXEXITHANDLER vmxHCExitPause; +static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThreshold; +static FNVMXEXITHANDLER vmxHCExitApicAccess; +static FNVMXEXITHANDLER vmxHCExitEptViolation; +static FNVMXEXITHANDLER vmxHCExitEptMisconfig; +static FNVMXEXITHANDLER vmxHCExitRdtscp; +static FNVMXEXITHANDLER vmxHCExitPreemptTimer; +static FNVMXEXITHANDLERNSRC vmxHCExitWbinvd; +static FNVMXEXITHANDLER vmxHCExitXsetbv; +static FNVMXEXITHANDLER vmxHCExitInvpcid; +#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +static FNVMXEXITHANDLERNSRC vmxHCExitSetPendingXcptUD; +#endif +static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestState; +static FNVMXEXITHANDLERNSRC vmxHCExitErrUnexpected; +/** @} */ + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** @name Nested-guest VM-exit handler prototypes. + * @{ + */ +static FNVMXEXITHANDLER vmxHCExitXcptOrNmiNested; +static FNVMXEXITHANDLER vmxHCExitTripleFaultNested; +static FNVMXEXITHANDLERNSRC vmxHCExitIntWindowNested; +static FNVMXEXITHANDLERNSRC vmxHCExitNmiWindowNested; +static FNVMXEXITHANDLER vmxHCExitTaskSwitchNested; +static FNVMXEXITHANDLER vmxHCExitHltNested; +static FNVMXEXITHANDLER vmxHCExitInvlpgNested; +static FNVMXEXITHANDLER vmxHCExitRdpmcNested; +static FNVMXEXITHANDLER vmxHCExitVmreadVmwriteNested; +static FNVMXEXITHANDLER vmxHCExitRdtscNested; +static FNVMXEXITHANDLER vmxHCExitMovCRxNested; +static FNVMXEXITHANDLER vmxHCExitMovDRxNested; +static FNVMXEXITHANDLER vmxHCExitIoInstrNested; +static FNVMXEXITHANDLER vmxHCExitRdmsrNested; +static FNVMXEXITHANDLER vmxHCExitWrmsrNested; +static FNVMXEXITHANDLER vmxHCExitMwaitNested; +static FNVMXEXITHANDLER vmxHCExitMtfNested; +static FNVMXEXITHANDLER vmxHCExitMonitorNested; +static FNVMXEXITHANDLER vmxHCExitPauseNested; +static FNVMXEXITHANDLERNSRC vmxHCExitTprBelowThresholdNested; +static FNVMXEXITHANDLER vmxHCExitApicAccessNested; +static FNVMXEXITHANDLER vmxHCExitApicWriteNested; +static FNVMXEXITHANDLER vmxHCExitVirtEoiNested; +static FNVMXEXITHANDLER vmxHCExitRdtscpNested; +static FNVMXEXITHANDLERNSRC vmxHCExitWbinvdNested; +static FNVMXEXITHANDLER vmxHCExitInvpcidNested; +static FNVMXEXITHANDLERNSRC vmxHCExitErrInvalidGuestStateNested; +static FNVMXEXITHANDLER vmxHCExitInstrNested; +static FNVMXEXITHANDLER vmxHCExitInstrWithInfoNested; +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +static FNVMXEXITHANDLER vmxHCExitEptViolationNested; +static FNVMXEXITHANDLER vmxHCExitEptMisconfigNested; +# endif +/** @} */ +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + + +/********************************************************************************************************************************* +* Global Variables * +*********************************************************************************************************************************/ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** + * Array of all VMCS fields. + * Any fields added to the VT-x spec. should be added here. + * + * Currently only used to derive shadow VMCS fields for hardware-assisted execution + * of nested-guests. + */ +static const uint32_t g_aVmcsFields[] = +{ + /* 16-bit control fields. */ + VMX_VMCS16_VPID, + VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR, + VMX_VMCS16_EPTP_INDEX, + VMX_VMCS16_HLAT_PREFIX_SIZE, + + /* 16-bit guest-state fields. */ + VMX_VMCS16_GUEST_ES_SEL, + VMX_VMCS16_GUEST_CS_SEL, + VMX_VMCS16_GUEST_SS_SEL, + VMX_VMCS16_GUEST_DS_SEL, + VMX_VMCS16_GUEST_FS_SEL, + VMX_VMCS16_GUEST_GS_SEL, + VMX_VMCS16_GUEST_LDTR_SEL, + VMX_VMCS16_GUEST_TR_SEL, + VMX_VMCS16_GUEST_INTR_STATUS, + VMX_VMCS16_GUEST_PML_INDEX, + + /* 16-bits host-state fields. */ + VMX_VMCS16_HOST_ES_SEL, + VMX_VMCS16_HOST_CS_SEL, + VMX_VMCS16_HOST_SS_SEL, + VMX_VMCS16_HOST_DS_SEL, + VMX_VMCS16_HOST_FS_SEL, + VMX_VMCS16_HOST_GS_SEL, + VMX_VMCS16_HOST_TR_SEL, + + /* 64-bit control fields. */ + VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, + VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH, + VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, + VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH, + VMX_VMCS64_CTRL_MSR_BITMAP_FULL, + VMX_VMCS64_CTRL_MSR_BITMAP_HIGH, + VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, + VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH, + VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, + VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH, + VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, + VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH, + VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL, + VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH, + VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL, + VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH, + VMX_VMCS64_CTRL_TSC_OFFSET_FULL, + VMX_VMCS64_CTRL_TSC_OFFSET_HIGH, + VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, + VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH, + VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, + VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH, + VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL, + VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH, + VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL, + VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH, + VMX_VMCS64_CTRL_EPTP_FULL, + VMX_VMCS64_CTRL_EPTP_HIGH, + VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL, + VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH, + VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL, + VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH, + VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL, + VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH, + VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL, + VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH, + VMX_VMCS64_CTRL_EPTP_LIST_FULL, + VMX_VMCS64_CTRL_EPTP_LIST_HIGH, + VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL, + VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH, + VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL, + VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH, + VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL, + VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH, + VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL, + VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH, + VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL, + VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH, + VMX_VMCS64_CTRL_SPPTP_FULL, + VMX_VMCS64_CTRL_SPPTP_HIGH, + VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL, + VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH, + VMX_VMCS64_CTRL_PROC_EXEC3_FULL, + VMX_VMCS64_CTRL_PROC_EXEC3_HIGH, + VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL, + VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH, + VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_FULL, + VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_HIGH, + VMX_VMCS64_CTRL_HLAT_PTR_FULL, + VMX_VMCS64_CTRL_HLAT_PTR_HIGH, + VMX_VMCS64_CTRL_EXIT2_FULL, + VMX_VMCS64_CTRL_EXIT2_HIGH, + + /* 64-bit read-only data fields. */ + VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, + VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH, + + /* 64-bit guest-state fields. */ + VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, + VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH, + VMX_VMCS64_GUEST_DEBUGCTL_FULL, + VMX_VMCS64_GUEST_DEBUGCTL_HIGH, + VMX_VMCS64_GUEST_PAT_FULL, + VMX_VMCS64_GUEST_PAT_HIGH, + VMX_VMCS64_GUEST_EFER_FULL, + VMX_VMCS64_GUEST_EFER_HIGH, + VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, + VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH, + VMX_VMCS64_GUEST_PDPTE0_FULL, + VMX_VMCS64_GUEST_PDPTE0_HIGH, + VMX_VMCS64_GUEST_PDPTE1_FULL, + VMX_VMCS64_GUEST_PDPTE1_HIGH, + VMX_VMCS64_GUEST_PDPTE2_FULL, + VMX_VMCS64_GUEST_PDPTE2_HIGH, + VMX_VMCS64_GUEST_PDPTE3_FULL, + VMX_VMCS64_GUEST_PDPTE3_HIGH, + VMX_VMCS64_GUEST_BNDCFGS_FULL, + VMX_VMCS64_GUEST_BNDCFGS_HIGH, + VMX_VMCS64_GUEST_RTIT_CTL_FULL, + VMX_VMCS64_GUEST_RTIT_CTL_HIGH, + VMX_VMCS64_GUEST_PKRS_FULL, + VMX_VMCS64_GUEST_PKRS_HIGH, + + /* 64-bit host-state fields. */ + VMX_VMCS64_HOST_PAT_FULL, + VMX_VMCS64_HOST_PAT_HIGH, + VMX_VMCS64_HOST_EFER_FULL, + VMX_VMCS64_HOST_EFER_HIGH, + VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL, + VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH, + VMX_VMCS64_HOST_PKRS_FULL, + VMX_VMCS64_HOST_PKRS_HIGH, + + /* 32-bit control fields. */ + VMX_VMCS32_CTRL_PIN_EXEC, + VMX_VMCS32_CTRL_PROC_EXEC, + VMX_VMCS32_CTRL_EXCEPTION_BITMAP, + VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, + VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, + VMX_VMCS32_CTRL_CR3_TARGET_COUNT, + VMX_VMCS32_CTRL_EXIT, + VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, + VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, + VMX_VMCS32_CTRL_ENTRY, + VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, + VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, + VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, + VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, + VMX_VMCS32_CTRL_TPR_THRESHOLD, + VMX_VMCS32_CTRL_PROC_EXEC2, + VMX_VMCS32_CTRL_PLE_GAP, + VMX_VMCS32_CTRL_PLE_WINDOW, + + /* 32-bits read-only fields. */ + VMX_VMCS32_RO_VM_INSTR_ERROR, + VMX_VMCS32_RO_EXIT_REASON, + VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, + VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, + VMX_VMCS32_RO_IDT_VECTORING_INFO, + VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, + VMX_VMCS32_RO_EXIT_INSTR_LENGTH, + VMX_VMCS32_RO_EXIT_INSTR_INFO, + + /* 32-bit guest-state fields. */ + VMX_VMCS32_GUEST_ES_LIMIT, + VMX_VMCS32_GUEST_CS_LIMIT, + VMX_VMCS32_GUEST_SS_LIMIT, + VMX_VMCS32_GUEST_DS_LIMIT, + VMX_VMCS32_GUEST_FS_LIMIT, + VMX_VMCS32_GUEST_GS_LIMIT, + VMX_VMCS32_GUEST_LDTR_LIMIT, + VMX_VMCS32_GUEST_TR_LIMIT, + VMX_VMCS32_GUEST_GDTR_LIMIT, + VMX_VMCS32_GUEST_IDTR_LIMIT, + VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, + VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, + VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, + VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, + VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, + VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, + VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, + VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, + VMX_VMCS32_GUEST_INT_STATE, + VMX_VMCS32_GUEST_ACTIVITY_STATE, + VMX_VMCS32_GUEST_SMBASE, + VMX_VMCS32_GUEST_SYSENTER_CS, + VMX_VMCS32_PREEMPT_TIMER_VALUE, + + /* 32-bit host-state fields. */ + VMX_VMCS32_HOST_SYSENTER_CS, + + /* Natural-width control fields. */ + VMX_VMCS_CTRL_CR0_MASK, + VMX_VMCS_CTRL_CR4_MASK, + VMX_VMCS_CTRL_CR0_READ_SHADOW, + VMX_VMCS_CTRL_CR4_READ_SHADOW, + VMX_VMCS_CTRL_CR3_TARGET_VAL0, + VMX_VMCS_CTRL_CR3_TARGET_VAL1, + VMX_VMCS_CTRL_CR3_TARGET_VAL2, + VMX_VMCS_CTRL_CR3_TARGET_VAL3, + + /* Natural-width read-only data fields. */ + VMX_VMCS_RO_EXIT_QUALIFICATION, + VMX_VMCS_RO_IO_RCX, + VMX_VMCS_RO_IO_RSI, + VMX_VMCS_RO_IO_RDI, + VMX_VMCS_RO_IO_RIP, + VMX_VMCS_RO_GUEST_LINEAR_ADDR, + + /* Natural-width guest-state field */ + VMX_VMCS_GUEST_CR0, + VMX_VMCS_GUEST_CR3, + VMX_VMCS_GUEST_CR4, + VMX_VMCS_GUEST_ES_BASE, + VMX_VMCS_GUEST_CS_BASE, + VMX_VMCS_GUEST_SS_BASE, + VMX_VMCS_GUEST_DS_BASE, + VMX_VMCS_GUEST_FS_BASE, + VMX_VMCS_GUEST_GS_BASE, + VMX_VMCS_GUEST_LDTR_BASE, + VMX_VMCS_GUEST_TR_BASE, + VMX_VMCS_GUEST_GDTR_BASE, + VMX_VMCS_GUEST_IDTR_BASE, + VMX_VMCS_GUEST_DR7, + VMX_VMCS_GUEST_RSP, + VMX_VMCS_GUEST_RIP, + VMX_VMCS_GUEST_RFLAGS, + VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, + VMX_VMCS_GUEST_SYSENTER_ESP, + VMX_VMCS_GUEST_SYSENTER_EIP, + VMX_VMCS_GUEST_S_CET, + VMX_VMCS_GUEST_SSP, + VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR, + + /* Natural-width host-state fields */ + VMX_VMCS_HOST_CR0, + VMX_VMCS_HOST_CR3, + VMX_VMCS_HOST_CR4, + VMX_VMCS_HOST_FS_BASE, + VMX_VMCS_HOST_GS_BASE, + VMX_VMCS_HOST_TR_BASE, + VMX_VMCS_HOST_GDTR_BASE, + VMX_VMCS_HOST_IDTR_BASE, + VMX_VMCS_HOST_SYSENTER_ESP, + VMX_VMCS_HOST_SYSENTER_EIP, + VMX_VMCS_HOST_RSP, + VMX_VMCS_HOST_RIP, + VMX_VMCS_HOST_S_CET, + VMX_VMCS_HOST_SSP, + VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR +}; +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + +#ifdef HMVMX_USE_FUNCTION_TABLE +/** + * VMX_EXIT dispatch table. + */ +static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] = +{ + /* 0 VMX_EXIT_XCPT_OR_NMI */ { vmxHCExitXcptOrNmi }, + /* 1 VMX_EXIT_EXT_INT */ { vmxHCExitExtInt }, + /* 2 VMX_EXIT_TRIPLE_FAULT */ { vmxHCExitTripleFault }, + /* 3 VMX_EXIT_INIT_SIGNAL */ { vmxHCExitErrUnexpected }, + /* 4 VMX_EXIT_SIPI */ { vmxHCExitErrUnexpected }, + /* 5 VMX_EXIT_IO_SMI */ { vmxHCExitErrUnexpected }, + /* 6 VMX_EXIT_SMI */ { vmxHCExitErrUnexpected }, + /* 7 VMX_EXIT_INT_WINDOW */ { vmxHCExitIntWindow }, + /* 8 VMX_EXIT_NMI_WINDOW */ { vmxHCExitNmiWindow }, + /* 9 VMX_EXIT_TASK_SWITCH */ { vmxHCExitTaskSwitch }, + /* 10 VMX_EXIT_CPUID */ { vmxHCExitCpuid }, + /* 11 VMX_EXIT_GETSEC */ { vmxHCExitGetsec }, + /* 12 VMX_EXIT_HLT */ { vmxHCExitHlt }, + /* 13 VMX_EXIT_INVD */ { vmxHCExitInvd }, + /* 14 VMX_EXIT_INVLPG */ { vmxHCExitInvlpg }, + /* 15 VMX_EXIT_RDPMC */ { vmxHCExitRdpmc }, + /* 16 VMX_EXIT_RDTSC */ { vmxHCExitRdtsc }, + /* 17 VMX_EXIT_RSM */ { vmxHCExitErrUnexpected }, + /* 18 VMX_EXIT_VMCALL */ { vmxHCExitVmcall }, +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitVmclear }, + /* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitVmlaunch }, + /* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitVmptrld }, + /* 22 VMX_EXIT_VMPTRST */ { vmxHCExitVmptrst }, + /* 23 VMX_EXIT_VMREAD */ { vmxHCExitVmread }, + /* 24 VMX_EXIT_VMRESUME */ { vmxHCExitVmresume }, + /* 25 VMX_EXIT_VMWRITE */ { vmxHCExitVmwrite }, + /* 26 VMX_EXIT_VMXOFF */ { vmxHCExitVmxoff }, + /* 27 VMX_EXIT_VMXON */ { vmxHCExitVmxon }, +#else + /* 19 VMX_EXIT_VMCLEAR */ { vmxHCExitSetPendingXcptUD }, + /* 20 VMX_EXIT_VMLAUNCH */ { vmxHCExitSetPendingXcptUD }, + /* 21 VMX_EXIT_VMPTRLD */ { vmxHCExitSetPendingXcptUD }, + /* 22 VMX_EXIT_VMPTRST */ { vmxHCExitSetPendingXcptUD }, + /* 23 VMX_EXIT_VMREAD */ { vmxHCExitSetPendingXcptUD }, + /* 24 VMX_EXIT_VMRESUME */ { vmxHCExitSetPendingXcptUD }, + /* 25 VMX_EXIT_VMWRITE */ { vmxHCExitSetPendingXcptUD }, + /* 26 VMX_EXIT_VMXOFF */ { vmxHCExitSetPendingXcptUD }, + /* 27 VMX_EXIT_VMXON */ { vmxHCExitSetPendingXcptUD }, +#endif + /* 28 VMX_EXIT_MOV_CRX */ { vmxHCExitMovCRx }, + /* 29 VMX_EXIT_MOV_DRX */ { vmxHCExitMovDRx }, + /* 30 VMX_EXIT_IO_INSTR */ { vmxHCExitIoInstr }, + /* 31 VMX_EXIT_RDMSR */ { vmxHCExitRdmsr }, + /* 32 VMX_EXIT_WRMSR */ { vmxHCExitWrmsr }, + /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ { vmxHCExitErrInvalidGuestState }, + /* 34 VMX_EXIT_ERR_MSR_LOAD */ { vmxHCExitErrUnexpected }, + /* 35 UNDEFINED */ { vmxHCExitErrUnexpected }, + /* 36 VMX_EXIT_MWAIT */ { vmxHCExitMwait }, + /* 37 VMX_EXIT_MTF */ { vmxHCExitMtf }, + /* 38 UNDEFINED */ { vmxHCExitErrUnexpected }, + /* 39 VMX_EXIT_MONITOR */ { vmxHCExitMonitor }, + /* 40 VMX_EXIT_PAUSE */ { vmxHCExitPause }, + /* 41 VMX_EXIT_ERR_MACHINE_CHECK */ { vmxHCExitErrUnexpected }, + /* 42 UNDEFINED */ { vmxHCExitErrUnexpected }, + /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ { vmxHCExitTprBelowThreshold }, + /* 44 VMX_EXIT_APIC_ACCESS */ { vmxHCExitApicAccess }, + /* 45 VMX_EXIT_VIRTUALIZED_EOI */ { vmxHCExitErrUnexpected }, + /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ { vmxHCExitErrUnexpected }, + /* 47 VMX_EXIT_LDTR_TR_ACCESS */ { vmxHCExitErrUnexpected }, + /* 48 VMX_EXIT_EPT_VIOLATION */ { vmxHCExitEptViolation }, + /* 49 VMX_EXIT_EPT_MISCONFIG */ { vmxHCExitEptMisconfig }, +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + /* 50 VMX_EXIT_INVEPT */ { vmxHCExitInvept }, +#else + /* 50 VMX_EXIT_INVEPT */ { vmxHCExitSetPendingXcptUD }, +#endif + /* 51 VMX_EXIT_RDTSCP */ { vmxHCExitRdtscp }, + /* 52 VMX_EXIT_PREEMPT_TIMER */ { vmxHCExitPreemptTimer }, +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* 53 VMX_EXIT_INVVPID */ { vmxHCExitInvvpid }, +#else + /* 53 VMX_EXIT_INVVPID */ { vmxHCExitSetPendingXcptUD }, +#endif + /* 54 VMX_EXIT_WBINVD */ { vmxHCExitWbinvd }, + /* 55 VMX_EXIT_XSETBV */ { vmxHCExitXsetbv }, + /* 56 VMX_EXIT_APIC_WRITE */ { vmxHCExitErrUnexpected }, + /* 57 VMX_EXIT_RDRAND */ { vmxHCExitErrUnexpected }, + /* 58 VMX_EXIT_INVPCID */ { vmxHCExitInvpcid }, + /* 59 VMX_EXIT_VMFUNC */ { vmxHCExitErrUnexpected }, + /* 60 VMX_EXIT_ENCLS */ { vmxHCExitErrUnexpected }, + /* 61 VMX_EXIT_RDSEED */ { vmxHCExitErrUnexpected }, + /* 62 VMX_EXIT_PML_FULL */ { vmxHCExitErrUnexpected }, + /* 63 VMX_EXIT_XSAVES */ { vmxHCExitErrUnexpected }, + /* 64 VMX_EXIT_XRSTORS */ { vmxHCExitErrUnexpected }, + /* 65 UNDEFINED */ { vmxHCExitErrUnexpected }, + /* 66 VMX_EXIT_SPP_EVENT */ { vmxHCExitErrUnexpected }, + /* 67 VMX_EXIT_UMWAIT */ { vmxHCExitErrUnexpected }, + /* 68 VMX_EXIT_TPAUSE */ { vmxHCExitErrUnexpected }, + /* 69 VMX_EXIT_LOADIWKEY */ { vmxHCExitErrUnexpected }, +}; +#endif /* HMVMX_USE_FUNCTION_TABLE */ + +#if defined(VBOX_STRICT) && defined(LOG_ENABLED) +static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] = +{ + /* 0 */ "(Not Used)", + /* 1 */ "VMCALL executed in VMX root operation.", + /* 2 */ "VMCLEAR with invalid physical address.", + /* 3 */ "VMCLEAR with VMXON pointer.", + /* 4 */ "VMLAUNCH with non-clear VMCS.", + /* 5 */ "VMRESUME with non-launched VMCS.", + /* 6 */ "VMRESUME after VMXOFF", + /* 7 */ "VM-entry with invalid control fields.", + /* 8 */ "VM-entry with invalid host state fields.", + /* 9 */ "VMPTRLD with invalid physical address.", + /* 10 */ "VMPTRLD with VMXON pointer.", + /* 11 */ "VMPTRLD with incorrect revision identifier.", + /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.", + /* 13 */ "VMWRITE to read-only VMCS component.", + /* 14 */ "(Not Used)", + /* 15 */ "VMXON executed in VMX root operation.", + /* 16 */ "VM-entry with invalid executive-VMCS pointer.", + /* 17 */ "VM-entry with non-launched executing VMCS.", + /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.", + /* 19 */ "VMCALL with non-clear VMCS.", + /* 20 */ "VMCALL with invalid VM-exit control fields.", + /* 21 */ "(Not Used)", + /* 22 */ "VMCALL with incorrect MSEG revision identifier.", + /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.", + /* 24 */ "VMCALL with invalid SMM-monitor features.", + /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.", + /* 26 */ "VM-entry with events blocked by MOV SS.", + /* 27 */ "(Not Used)", + /* 28 */ "Invalid operand to INVEPT/INVVPID." +}; +#endif /* VBOX_STRICT && LOG_ENABLED */ + + +/** + * Gets the CR0 guest/host mask. + * + * These bits typically does not change through the lifetime of a VM. Any bit set in + * this mask is owned by the host/hypervisor and would cause a VM-exit when modified + * by the guest. + * + * @returns The CR0 guest/host mask. + * @param pVCpu The cross context virtual CPU structure. + */ +static uint64_t vmxHCGetFixedCr0Mask(PCVMCPUCC pVCpu) +{ + /* + * Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and + * to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits. + * + * Furthermore, modifications to any bits that are reserved/unspecified currently + * by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior + * when future CPUs specify and use currently reserved/unspecified bits. + */ + /** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM + * enmGuestMode to be in-sync with the current mode. See @bugref{6398} + * and @bugref{6944}. */ + PCVMCC pVM = pVCpu->CTX_SUFF(pVM); + AssertCompile(RT_HI_U32(VMX_EXIT_HOST_CR0_IGNORE_MASK) == UINT32_C(0xffffffff)); /* Paranoia. */ + return ( X86_CR0_PE + | X86_CR0_NE + | (VM_IS_VMX_NESTED_PAGING(pVM) ? 0 : X86_CR0_WP) + | X86_CR0_PG + | VMX_EXIT_HOST_CR0_IGNORE_MASK); +} + + +/** + * Gets the CR4 guest/host mask. + * + * These bits typically does not change through the lifetime of a VM. Any bit set in + * this mask is owned by the host/hypervisor and would cause a VM-exit when modified + * by the guest. + * + * @returns The CR4 guest/host mask. + * @param pVCpu The cross context virtual CPU structure. + */ +static uint64_t vmxHCGetFixedCr4Mask(PCVMCPUCC pVCpu) +{ + /* + * We construct a mask of all CR4 bits that the guest can modify without causing + * a VM-exit. Then invert this mask to obtain all CR4 bits that should cause + * a VM-exit when the guest attempts to modify them when executing using + * hardware-assisted VMX. + * + * When a feature is not exposed to the guest (and may be present on the host), + * we want to intercept guest modifications to the bit so we can emulate proper + * behavior (e.g., #GP). + * + * Furthermore, only modifications to those bits that don't require immediate + * emulation is allowed. For e.g., PCIDE is excluded because the behavior + * depends on CR3 which might not always be the guest value while executing + * using hardware-assisted VMX. + */ + PCVMCC pVM = pVCpu->CTX_SUFF(pVM); + bool fFsGsBase = pVM->cpum.ro.GuestFeatures.fFsGsBase; +#ifdef IN_NEM_DARWIN + bool fXSaveRstor = pVM->cpum.ro.GuestFeatures.fXSaveRstor; +#endif + bool fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor; + + /* + * Paranoia. + * Ensure features exposed to the guest are present on the host. + */ + AssertStmt(!fFsGsBase || g_CpumHostFeatures.s.fFsGsBase, fFsGsBase = 0); +#ifdef IN_NEM_DARWIN + AssertStmt(!fXSaveRstor || g_CpumHostFeatures.s.fXSaveRstor, fXSaveRstor = 0); +#endif + AssertStmt(!fFxSaveRstor || g_CpumHostFeatures.s.fFxSaveRstor, fFxSaveRstor = 0); + + uint64_t const fGstMask = X86_CR4_PVI + | X86_CR4_TSD + | X86_CR4_DE + | X86_CR4_MCE + | X86_CR4_PCE + | X86_CR4_OSXMMEEXCPT + | (fFsGsBase ? X86_CR4_FSGSBASE : 0) +#ifdef IN_NEM_DARWIN /* On native VT-x setting OSXSAVE must exit as we need to load guest XCR0 (see + fLoadSaveGuestXcr0). These exits are not needed on Darwin as that's not our problem. */ + | (fXSaveRstor ? X86_CR4_OSXSAVE : 0) +#endif + | (fFxSaveRstor ? X86_CR4_OSFXSR : 0); + return ~fGstMask; +} + + +/** + * Adds one or more exceptions to the exception bitmap and commits it to the current + * VMCS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * @param uXcptMask The exception(s) to add. + */ +static void vmxHCAddXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask) +{ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap; + if ((uXcptBitmap & uXcptMask) != uXcptMask) + { + uXcptBitmap |= uXcptMask; + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap); + AssertRC(rc); + pVmcsInfo->u32XcptBitmap = uXcptBitmap; + } +} + + +/** + * Adds an exception to the exception bitmap and commits it to the current VMCS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * @param uXcpt The exception to add. + */ +static void vmxHCAddXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt) +{ + Assert(uXcpt <= X86_XCPT_LAST); + vmxHCAddXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT_32(uXcpt)); +} + + +/** + * Remove one or more exceptions from the exception bitmap and commits it to the + * current VMCS. + * + * This takes care of not removing the exception intercept if a nested-guest + * requires the exception to be intercepted. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * @param uXcptMask The exception(s) to remove. + */ +static int vmxHCRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask) +{ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap; + if (uXcptBitmap & uXcptMask) + { +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (!pVmxTransient->fIsNestedGuest) + { /* likely */ } + else + uXcptMask &= ~pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap; +#endif +#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS + uXcptMask &= ~( RT_BIT(X86_XCPT_BP) + | RT_BIT(X86_XCPT_DE) + | RT_BIT(X86_XCPT_NM) + | RT_BIT(X86_XCPT_TS) + | RT_BIT(X86_XCPT_UD) + | RT_BIT(X86_XCPT_NP) + | RT_BIT(X86_XCPT_SS) + | RT_BIT(X86_XCPT_GP) + | RT_BIT(X86_XCPT_PF) + | RT_BIT(X86_XCPT_MF)); +#elif defined(HMVMX_ALWAYS_TRAP_PF) + uXcptMask &= ~RT_BIT(X86_XCPT_PF); +#endif + if (uXcptMask) + { + /* Validate we are not removing any essential exception intercepts. */ +#ifndef IN_NEM_DARWIN + Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging || !(uXcptMask & RT_BIT(X86_XCPT_PF))); +#else + Assert(!(uXcptMask & RT_BIT(X86_XCPT_PF))); +#endif + NOREF(pVCpu); + Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB))); + Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC))); + + /* Remove it from the exception bitmap. */ + uXcptBitmap &= ~uXcptMask; + + /* Commit and update the cache if necessary. */ + if (pVmcsInfo->u32XcptBitmap != uXcptBitmap) + { + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap); + AssertRC(rc); + pVmcsInfo->u32XcptBitmap = uXcptBitmap; + } + } + } + return VINF_SUCCESS; +} + + +/** + * Remove an exceptions from the exception bitmap and commits it to the current + * VMCS. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * @param uXcpt The exception to remove. + */ +static int vmxHCRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt) +{ + return vmxHCRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt)); +} + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + +/** + * Loads the shadow VMCS specified by the VMCS info. object. + * + * @returns VBox status code. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks Can be called with interrupts disabled. + */ +static int vmxHCLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo) +{ + Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS); + + int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs); + if (RT_SUCCESS(rc)) + pVmcsInfo->fShadowVmcsState |= VMX_V_VMCS_LAUNCH_STATE_CURRENT; + return rc; +} + + +/** + * Clears the shadow VMCS specified by the VMCS info. object. + * + * @returns VBox status code. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks Can be called with interrupts disabled. + */ +static int vmxHCClearShadowVmcs(PVMXVMCSINFO pVmcsInfo) +{ + Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS); + + int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs); + if (RT_SUCCESS(rc)) + pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR; + return rc; +} + + +/** + * Switches from and to the specified VMCSes. + * + * @returns VBox status code. + * @param pVmcsInfoFrom The VMCS info. object we are switching from. + * @param pVmcsInfoTo The VMCS info. object we are switching to. + * + * @remarks Called with interrupts disabled. + */ +static int vmxHCSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo) +{ + /* + * Clear the VMCS we are switching out if it has not already been cleared. + * This will sync any CPU internal data back to the VMCS. + */ + if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR) + { + int rc = hmR0VmxClearVmcs(pVmcsInfoFrom); + if (RT_SUCCESS(rc)) + { + /* + * The shadow VMCS, if any, would not be active at this point since we + * would have cleared it while importing the virtual hardware-virtualization + * state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to + * clear the shadow VMCS here, just assert for safety. + */ + Assert(!pVmcsInfoFrom->pvShadowVmcs || pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR); + } + else + return rc; + } + + /* + * Clear the VMCS we are switching to if it has not already been cleared. + * This will initialize the VMCS launch state to "clear" required for loading it. + * + * See Intel spec. 31.6 "Preparation And Launching A Virtual Machine". + */ + if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR) + { + int rc = hmR0VmxClearVmcs(pVmcsInfoTo); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + return rc; + } + + /* + * Finally, load the VMCS we are switching to. + */ + return hmR0VmxLoadVmcs(pVmcsInfoTo); +} + + +/** + * Switches between the guest VMCS and the nested-guest VMCS as specified by the + * caller. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param fSwitchToNstGstVmcs Whether to switch to the nested-guest VMCS (pass + * true) or guest VMCS (pass false). + */ +static int vmxHCSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs) +{ + /* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */ + HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); + + PVMXVMCSINFO pVmcsInfoFrom; + PVMXVMCSINFO pVmcsInfoTo; + if (fSwitchToNstGstVmcs) + { + pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfo; + pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst; + } + else + { + pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst; + pVmcsInfoTo = &pVCpu->hmr0.s.vmx.VmcsInfo; + } + + /* + * Disable interrupts to prevent being preempted while we switch the current VMCS as the + * preemption hook code path acquires the current VMCS. + */ + RTCCUINTREG const fEFlags = ASMIntDisableFlags(); + + int rc = vmxHCSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo); + if (RT_SUCCESS(rc)) + { + pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs = fSwitchToNstGstVmcs; + pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3 = fSwitchToNstGstVmcs; + + /* + * If we are switching to a VMCS that was executed on a different host CPU or was + * never executed before, flag that we need to export the host state before executing + * guest/nested-guest code using hardware-assisted VMX. + * + * This could probably be done in a preemptible context since the preemption hook + * will flag the necessary change in host context. However, since preemption is + * already disabled and to avoid making assumptions about host specific code in + * RTMpCpuId when called with preemption enabled, we'll do this while preemption is + * disabled. + */ + if (pVmcsInfoTo->idHostCpuState == RTMpCpuId()) + { /* likely */ } + else + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE); + + ASMSetFlags(fEFlags); + + /* + * We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence, + * flag that we need to update the host MSR values there. Even if we decide in the + * future to share the VM-exit MSR-store area page between the guest and nested-guest, + * if its content differs, we would have to update the host MSRs anyway. + */ + pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false; + } + else + ASMSetFlags(fEFlags); + return rc; +} + +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ +#ifdef VBOX_STRICT + +/** + * Reads the VM-entry interruption-information field from the VMCS into the VMX + * transient structure. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + */ +DECLINLINE(void) vmxHCReadEntryIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo); + AssertRC(rc); +} + + +/** + * Reads the VM-entry exception error code field from the VMCS into + * the VMX transient structure. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + */ +DECLINLINE(void) vmxHCReadEntryXcptErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode); + AssertRC(rc); +} + + +/** + * Reads the VM-entry exception error code field from the VMCS into + * the VMX transient structure. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + */ +DECLINLINE(void) vmxHCReadEntryInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr); + AssertRC(rc); +} + +#endif /* VBOX_STRICT */ + + +/** + * Reads VMCS fields into the VMXTRANSIENT structure, slow path version. + * + * Don't call directly unless the it's likely that some or all of the fields + * given in @a a_fReadMask have already been read. + * + * @tparam a_fReadMask The fields to read. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + */ +template +static void vmxHCReadToTransientSlow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + AssertCompile((a_fReadMask & ~( HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE + | HMVMX_READ_EXIT_INTERRUPTION_INFO + | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE + | HMVMX_READ_GUEST_LINEAR_ADDR + | HMVMX_READ_GUEST_PHYSICAL_ADDR + | HMVMX_READ_GUEST_PENDING_DBG_XCPTS + )) == 0); + + if ((pVmxTransient->fVmcsFieldsRead & a_fReadMask) != a_fReadMask) + { + uint32_t const fVmcsFieldsRead = pVmxTransient->fVmcsFieldsRead; + + if ( (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION) + && !(fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION)) + { + int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); + AssertRC(rc); + } + if ( (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN) + && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN)) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr); + AssertRC(rc); + } + if ( (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO) + && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO)) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u); + AssertRC(rc); + } + if ( (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO) + && !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO)) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo); + AssertRC(rc); + } + if ( (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE) + && !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE)) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode); + AssertRC(rc); + } + if ( (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO) + && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO)) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo); + AssertRC(rc); + } + if ( (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE) + && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode); + AssertRC(rc); + } + if ( (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR) + && !(fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR)) + { + int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); + AssertRC(rc); + } + if ( (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR) + && !(fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR)) + { + int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr); + AssertRC(rc); + } + if ( (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS) + && !(fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)) + { + int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts); + AssertRC(rc); + } + + pVmxTransient->fVmcsFieldsRead |= a_fReadMask; + } +} + + +/** + * Reads VMCS fields into the VMXTRANSIENT structure. + * + * This optimizes for the case where none of @a a_fReadMask has been read yet, + * generating an optimized read sequences w/o any conditionals between in + * non-strict builds. + * + * @tparam a_fReadMask The fields to read. One or more of the + * HMVMX_READ_XXX fields ORed together. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + */ +template +DECLINLINE(void) vmxHCReadToTransient(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + AssertCompile((a_fReadMask & ~( HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE + | HMVMX_READ_EXIT_INTERRUPTION_INFO + | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE + | HMVMX_READ_GUEST_LINEAR_ADDR + | HMVMX_READ_GUEST_PHYSICAL_ADDR + | HMVMX_READ_GUEST_PENDING_DBG_XCPTS + )) == 0); + + if (RT_LIKELY(!(pVmxTransient->fVmcsFieldsRead & a_fReadMask))) + { + if (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION) + { + int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); + AssertRC(rc); + } + if (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr); + AssertRC(rc); + } + if (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u); + AssertRC(rc); + } + if (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo); + AssertRC(rc); + } + if (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode); + AssertRC(rc); + } + if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo); + AssertRC(rc); + } + if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE) + { + int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode); + AssertRC(rc); + } + if (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR) + { + int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); + AssertRC(rc); + } + if (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR) + { + int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr); + AssertRC(rc); + } + if (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS) + { + int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &pVmxTransient->uGuestPendingDbgXcpts); + AssertRC(rc); + } + + pVmxTransient->fVmcsFieldsRead |= a_fReadMask; + } + else + { + STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatReadToTransientFallback); + Log11Func(("a_fReadMask=%#x fVmcsFieldsRead=%#x => %#x - Taking inefficient code path!\n", + a_fReadMask, pVmxTransient->fVmcsFieldsRead, a_fReadMask & pVmxTransient->fVmcsFieldsRead)); + vmxHCReadToTransientSlow(pVCpu, pVmxTransient); + } +} + + +#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE +/** + * Reads all relevant read-only VMCS fields into the VMX transient structure. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + */ +static void vmxHCReadAllRoFieldsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); + rc |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbExitInstr); + rc |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u); + rc |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo); + rc |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode); + rc |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo); + rc |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode); + rc |= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); + rc |= VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &pVmxTransient->uGuestPhysicalAddr); + AssertRC(rc); + pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE + | HMVMX_READ_EXIT_INTERRUPTION_INFO + | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE + | HMVMX_READ_GUEST_LINEAR_ADDR + | HMVMX_READ_GUEST_PHYSICAL_ADDR; +} +#endif + +/** + * Verifies that our cached values of the VMCS fields are all consistent with + * what's actually present in the VMCS. + * + * @returns VBox status code. + * @retval VINF_SUCCESS if all our caches match their respective VMCS fields. + * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the + * VMCS content. HMCPU error-field is + * updated, see VMX_VCI_XXX. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * @param fIsNstGstVmcs Whether this is a nested-guest VMCS. + */ +static int vmxHCCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs) +{ + const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS"; + + uint32_t u32Val; + int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val); + AssertRC(rc); + AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val, + ("%s entry controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val), + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_ENTRY, + VERR_VMX_VMCS_FIELD_CACHE_INVALID); + + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXIT, &u32Val); + AssertRC(rc); + AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val, + ("%s exit controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val), + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_EXIT, + VERR_VMX_VMCS_FIELD_CACHE_INVALID); + + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); + AssertRC(rc); + AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val, + ("%s pin controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val), + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PIN_EXEC, + VERR_VMX_VMCS_FIELD_CACHE_INVALID); + + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); + AssertRC(rc); + AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val, + ("%s proc controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val), + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC, + VERR_VMX_VMCS_FIELD_CACHE_INVALID); + + if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS) + { + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); + AssertRC(rc); + AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val, + ("%s proc2 controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val), + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC2, + VERR_VMX_VMCS_FIELD_CACHE_INVALID); + } + + uint64_t u64Val; + if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS) + { + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_PROC_EXEC3_FULL, &u64Val); + AssertRC(rc); + AssertMsgReturnStmt(pVmcsInfo->u64ProcCtls3 == u64Val, + ("%s proc3 controls mismatch: Cache=%#RX32 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64ProcCtls3, u64Val), + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC3, + VERR_VMX_VMCS_FIELD_CACHE_INVALID); + } + + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); + AssertRC(rc); + AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val, + ("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val), + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_XCPT_BITMAP, + VERR_VMX_VMCS_FIELD_CACHE_INVALID); + + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val); + AssertRC(rc); + AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val, + ("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val), + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_TSC_OFFSET, + VERR_VMX_VMCS_FIELD_CACHE_INVALID); + + NOREF(pcszVmcs); + return VINF_SUCCESS; +} + + +/** + * Exports the guest state with appropriate VM-entry and VM-exit controls in the + * VMCS. + * + * This is typically required when the guest changes paging mode. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks Requires EFER. + * @remarks No-long-jump zone!!! + */ +static int vmxHCExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient) +{ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS) + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + + /* + * VM-entry controls. + */ + { + uint32_t fVal = g_HmMsrs.u.vmx.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */ + uint32_t const fZap = g_HmMsrs.u.vmx.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */ + + /* + * Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry. + * The first VT-x capable CPUs only supported the 1-setting of this bit. + * + * For nested-guests, this is a mandatory VM-entry control. It's also + * required because we do not want to leak host bits to the nested-guest. + */ + fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG; + + /* + * Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. + * + * For nested-guests, the "IA-32e mode guest" control we initialize with what is + * required to get the nested-guest working with hardware-assisted VMX execution. + * It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor + * can skip intercepting changes to the EFER MSR. This is why it needs to be done + * here rather than while merging the guest VMCS controls. + */ + if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)) + { + Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME); + fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST; + } + else + Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST)); + + /* + * If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. + * + * For nested-guests, we use the "load IA32_EFER" if the hardware supports it, + * regardless of whether the nested-guest VMCS specifies it because we are free to + * load whatever MSRs we require and we do not need to modify the guest visible copy + * of the VM-entry MSR load area. + */ + if ( g_fHmVmxSupportsVmcsEfer +#ifndef IN_NEM_DARWIN + && hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient) +#endif + ) + fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR; + else + Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR)); + + /* + * The following should -not- be set (since we're not in SMM mode): + * - VMX_ENTRY_CTLS_ENTRY_TO_SMM + * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON + */ + + /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR, + * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */ + + if ((fVal & fZap) == fVal) + { /* likely */ } + else + { + Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n", + g_HmMsrs.u.vmx.EntryCtls.n.allowed0, fVal, fZap)); + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_ENTRY; + return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO; + } + + /* Commit it to the VMCS. */ + if (pVmcsInfo->u32EntryCtls != fVal) + { + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, fVal); + AssertRC(rc); + pVmcsInfo->u32EntryCtls = fVal; + } + } + + /* + * VM-exit controls. + */ + { + uint32_t fVal = g_HmMsrs.u.vmx.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */ + uint32_t const fZap = g_HmMsrs.u.vmx.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */ + + /* + * Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only + * supported the 1-setting of this bit. + * + * For nested-guests, we set the "save debug controls" as the converse + * "load debug controls" is mandatory for nested-guests anyway. + */ + fVal |= VMX_EXIT_CTLS_SAVE_DEBUG; + + /* + * Set the host long mode active (EFER.LMA) bit (which Intel calls + * "Host address-space size") if necessary. On VM-exit, VT-x sets both the + * host EFER.LMA and EFER.LME bit to this value. See assertion in + * vmxHCExportHostMsrs(). + * + * For nested-guests, we always set this bit as we do not support 32-bit + * hosts. + */ + fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE; + +#ifndef IN_NEM_DARWIN + /* + * If the VMCS EFER MSR fields are supported by the hardware, we use it. + * + * For nested-guests, we should use the "save IA32_EFER" control if we also + * used the "load IA32_EFER" control while exporting VM-entry controls. + */ + if ( g_fHmVmxSupportsVmcsEfer + && hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient)) + { + fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR + | VMX_EXIT_CTLS_LOAD_EFER_MSR; + } +#endif + + /* + * Enable saving of the VMX-preemption timer value on VM-exit. + * For nested-guests, currently not exposed/used. + */ + /** @todo r=bird: Measure performance hit because of this vs. always rewriting + * the timer value. */ + if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM)) + { + Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER); + fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER; + } + + /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */ + Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT)); + + /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR, + * VMX_EXIT_CTLS_SAVE_PAT_MSR, + * VMX_EXIT_CTLS_LOAD_PAT_MSR. */ + + if ((fVal & fZap) == fVal) + { /* likely */ } + else + { + Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n", + g_HmMsrs.u.vmx.ExitCtls.n.allowed0, fVal, fZap)); + VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_EXIT; + return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO; + } + + /* Commit it to the VMCS. */ + if (pVmcsInfo->u32ExitCtls != fVal) + { + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT, fVal); + AssertRC(rc); + pVmcsInfo->u32ExitCtls = fVal; + } + } + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS); + } + return VINF_SUCCESS; +} + + +/** + * Sets the TPR threshold in the VMCS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * @param u32TprThreshold The TPR threshold (task-priority class only). + */ +DECLINLINE(void) vmxHCApicSetTprThreshold(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold) +{ + Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */ + Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); + RT_NOREF(pVmcsInfo); + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold); + AssertRC(rc); +} + + +/** + * Exports the guest APIC TPR state into the VMCS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks No-long-jump zone!!! + */ +static void vmxHCExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient) +{ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + if (!pVmxTransient->fIsNestedGuest) + { + if ( PDMHasApic(pVCpu->CTX_SUFF(pVM)) + && APICIsEnabled(pVCpu)) + { + /* + * Setup TPR shadowing. + */ + if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW) + { + bool fPendingIntr = false; + uint8_t u8Tpr = 0; + uint8_t u8PendingIntr = 0; + int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr); + AssertRC(rc); + + /* + * If there are interrupts pending but masked by the TPR, instruct VT-x to + * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the + * priority of the pending interrupt so we can deliver the interrupt. If there + * are no interrupts pending, set threshold to 0 to not cause any + * TPR-below-threshold VM-exits. + */ + uint32_t u32TprThreshold = 0; + if (fPendingIntr) + { + /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR + (which is the Task-Priority Class). */ + const uint8_t u8PendingPriority = u8PendingIntr >> 4; + const uint8_t u8TprPriority = u8Tpr >> 4; + if (u8PendingPriority <= u8TprPriority) + u32TprThreshold = u8PendingPriority; + } + + vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold); + } + } + } + /* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */ + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR); + } +} + + +/** + * Gets the guest interruptibility-state and updates related force-flags. + * + * @returns Guest's interruptibility-state. + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks No-long-jump zone!!! + */ +static uint32_t vmxHCGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu) +{ + uint32_t fIntrState; + + /* + * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS. + */ + if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx)) + fIntrState = 0; + else + { + /* If inhibition is active, RIP should've been imported from the VMCS already. */ + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP); + + if (CPUMIsInInterruptShadowAfterSs(&pVCpu->cpum.GstCtx)) + fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS; + else + { + fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI; + + /* Block-by-STI must not be set when interrupts are disabled. */ + AssertStmt(pVCpu->cpum.GstCtx.eflags.Bits.u1IF, fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS); + } + } + + /* + * Check if we should inhibit NMI delivery. + */ + if (!CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx)) + { /* likely */ } + else + fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI; + + /* + * Validate. + */ + /* We don't support block-by-SMI yet.*/ + Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); + + return fIntrState; +} + + +/** + * Exports the exception intercepts required for guest execution in the VMCS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks No-long-jump zone!!! + */ +static void vmxHCExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient) +{ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS) + { + /* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */ + if ( !pVmxTransient->fIsNestedGuest + && VCPU_2_VMXSTATE(pVCpu).fGIMTrapXcptUD) + vmxHCAddXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD); + else + vmxHCRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD); + + /* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */ + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS); + } +} + + +/** + * Exports the guest's RIP into the guest-state area in the VMCS. + * + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks No-long-jump zone!!! + */ +static void vmxHCExportGuestRip(PVMCPUCC pVCpu) +{ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RIP) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP); + + int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip); + AssertRC(rc); + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RIP); + Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip)); + } +} + + +/** + * Exports the guest's RFLAGS into the guest-state area in the VMCS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks No-long-jump zone!!! + */ +static void vmxHCExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient) +{ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RFLAGS) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); + + /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits + of RFLAGS are reserved (MBZ). We use bits 63:24 for internal purposes, so no need + to assert this, the CPUMX86EFLAGS/CPUMX86RFLAGS union masks these off for us. + Use 32-bit VMWRITE. */ + uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u; + Assert((fEFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK); + AssertMsg(!(fEFlags & ~(X86_EFL_LIVE_MASK | X86_EFL_RA1_MASK)), ("%#x\n", fEFlags)); + +#ifndef IN_NEM_DARWIN + /* + * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so + * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x + * can run the real-mode guest code under Virtual 8086 mode. + */ + PVMXVMCSINFOSHARED pVmcsInfo = pVmxTransient->pVmcsInfo->pShared; + if (pVmcsInfo->RealMode.fRealOnV86Active) + { + Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS); + Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM))); + Assert(!pVmxTransient->fIsNestedGuest); + pVmcsInfo->RealMode.Eflags.u32 = fEFlags; /* Save the original eflags of the real-mode guest. */ + fEFlags |= X86_EFL_VM; /* Set the Virtual 8086 mode bit. */ + fEFlags &= ~X86_EFL_IOPL; /* Change IOPL to 0, otherwise certain instructions won't fault. */ + } +#else + RT_NOREF(pVmxTransient); +#endif + + int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, fEFlags); + AssertRC(rc); + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS); + Log4Func(("eflags=%#RX32\n", fEFlags)); + } +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** + * Copies the nested-guest VMCS to the shadow VMCS. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks No-long-jump zone!!! + */ +static int vmxHCCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo) +{ + PVMCC const pVM = pVCpu->CTX_SUFF(pVM); + PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + /* + * Disable interrupts so we don't get preempted while the shadow VMCS is the + * current VMCS, as we may try saving guest lazy MSRs. + * + * Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk + * calling the import VMCS code which is currently performing the guest MSR reads + * (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts + * and the rest of the VMX leave session machinery. + */ + RTCCUINTREG const fEFlags = ASMIntDisableFlags(); + + int rc = vmxHCLoadShadowVmcs(pVmcsInfo); + if (RT_SUCCESS(rc)) + { + /* + * Copy all guest read/write VMCS fields. + * + * We don't check for VMWRITE failures here for performance reasons and + * because they are not expected to fail, barring irrecoverable conditions + * like hardware errors. + */ + uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields; + for (uint32_t i = 0; i < cShadowVmcsFields; i++) + { + uint64_t u64Val; + uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i]; + IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val); + VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val); + } + + /* + * If the host CPU supports writing all VMCS fields, copy the guest read-only + * VMCS fields, so the guest can VMREAD them without causing a VM-exit. + */ + if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL) + { + uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields; + for (uint32_t i = 0; i < cShadowVmcsRoFields; i++) + { + uint64_t u64Val; + uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsRoFields[i]; + IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val); + VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val); + } + } + + rc = vmxHCClearShadowVmcs(pVmcsInfo); + rc |= hmR0VmxLoadVmcs(pVmcsInfo); + } + + ASMSetFlags(fEFlags); + return rc; +} + + +/** + * Copies the shadow VMCS to the nested-guest VMCS. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks Called with interrupts disabled. + */ +static int vmxHCCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo) +{ + Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); + PVMCC const pVM = pVCpu->CTX_SUFF(pVM); + PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + + int rc = vmxHCLoadShadowVmcs(pVmcsInfo); + if (RT_SUCCESS(rc)) + { + /* + * Copy guest read/write fields from the shadow VMCS. + * Guest read-only fields cannot be modified, so no need to copy them. + * + * We don't check for VMREAD failures here for performance reasons and + * because they are not expected to fail, barring irrecoverable conditions + * like hardware errors. + */ + uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields; + for (uint32_t i = 0; i < cShadowVmcsFields; i++) + { + uint64_t u64Val; + uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i]; + VMX_VMCS_READ_64(pVCpu, uVmcsField, &u64Val); + IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val); + } + + rc = vmxHCClearShadowVmcs(pVmcsInfo); + rc |= hmR0VmxLoadVmcs(pVmcsInfo); + } + return rc; +} + + +/** + * Enables VMCS shadowing for the given VMCS info. object. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks No-long-jump zone!!! + */ +static void vmxHCEnableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo) +{ + uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2; + if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)) + { + Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS); + uProcCtls2 |= VMX_PROC_CTLS2_VMCS_SHADOWING; + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc); + rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs); AssertRC(rc); + pVmcsInfo->u32ProcCtls2 = uProcCtls2; + pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs; + Log4Func(("Enabled\n")); + } +} + + +/** + * Disables VMCS shadowing for the given VMCS info. object. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks No-long-jump zone!!! + */ +static void vmxHCDisableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo) +{ + /* + * We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the + * VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit + * to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS. + * Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail. + * + * See Intel spec. 26.2.1.1 "VM-Execution Control Fields". + * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State". + */ + uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2; + if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING) + { + uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING; + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2); AssertRC(rc); + rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS); AssertRC(rc); + pVmcsInfo->u32ProcCtls2 = uProcCtls2; + pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS; + Log4Func(("Disabled\n")); + } +} +#endif + + +/** + * Exports the guest CR0 control register into the guest-state area in the VMCS. + * + * The guest FPU state is always pre-loaded hence we don't need to bother about + * sharing FPU related CR0 bits between the guest and host. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks No-long-jump zone!!! + */ +static int vmxHCExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient) +{ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR0) + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + + uint64_t fSetCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed0; + uint64_t const fZapCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed1; + if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM)) + fSetCr0 &= ~(uint64_t)(X86_CR0_PE | X86_CR0_PG); + else + Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG)); + + if (!pVmxTransient->fIsNestedGuest) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); + uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0; + uint64_t const u64ShadowCr0 = u64GuestCr0; + Assert(!RT_HI_U32(u64GuestCr0)); + + /* + * Setup VT-x's view of the guest CR0. + */ + uint32_t uProcCtls = pVmcsInfo->u32ProcCtls; + if (VM_IS_VMX_NESTED_PAGING(pVM)) + { +#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS + if (CPUMIsGuestPagingEnabled(pVCpu)) + { + /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */ + uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT + | VMX_PROC_CTLS_CR3_STORE_EXIT); + } + else + { + /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */ + uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT + | VMX_PROC_CTLS_CR3_STORE_EXIT; + } + + /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */ + if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM)) + uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT; +#endif + } + else + { + /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */ + u64GuestCr0 |= X86_CR0_WP; + } + + /* + * Guest FPU bits. + * + * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state + * using CR0.TS. + * + * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be + * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks. + */ + u64GuestCr0 |= X86_CR0_NE; + + /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */ + bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE); + + /* + * Update exception intercepts. + */ + uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap; +#ifndef IN_NEM_DARWIN + if (pVmcsInfo->pShared->RealMode.fRealOnV86Active) + { + Assert(PDMVmmDevHeapIsEnabled(pVM)); + Assert(pVM->hm.s.vmx.pRealModeTSS); + uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK; + } + else +#endif + { + /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */ + uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK; + if (fInterceptMF) + uXcptBitmap |= RT_BIT(X86_XCPT_MF); + } + + /* Additional intercepts for debugging, define these yourself explicitly. */ +#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS + uXcptBitmap |= 0 + | RT_BIT(X86_XCPT_BP) + | RT_BIT(X86_XCPT_DE) + | RT_BIT(X86_XCPT_NM) + | RT_BIT(X86_XCPT_TS) + | RT_BIT(X86_XCPT_UD) + | RT_BIT(X86_XCPT_NP) + | RT_BIT(X86_XCPT_SS) + | RT_BIT(X86_XCPT_GP) + | RT_BIT(X86_XCPT_PF) + | RT_BIT(X86_XCPT_MF) + ; +#elif defined(HMVMX_ALWAYS_TRAP_PF) + uXcptBitmap |= RT_BIT(X86_XCPT_PF); +#endif + if (VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv) + uXcptBitmap |= RT_BIT(X86_XCPT_GP); + if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE) + uXcptBitmap |= RT_BIT(X86_XCPT_DE); + Assert(VM_IS_VMX_NESTED_PAGING(pVM) || (uXcptBitmap & RT_BIT(X86_XCPT_PF))); + + /* Apply the hardware specified CR0 fixed bits and enable caching. */ + u64GuestCr0 |= fSetCr0; + u64GuestCr0 &= fZapCr0; + u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW); + + Assert(!RT_HI_U32(u64GuestCr0)); + Assert(u64GuestCr0 & X86_CR0_NE); + + /* Commit the CR0 and related fields to the guest VMCS. */ + int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc); + if (uProcCtls != pVmcsInfo->u32ProcCtls) + { + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls); + AssertRC(rc); + } + if (uXcptBitmap != pVmcsInfo->u32XcptBitmap) + { + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap); + AssertRC(rc); + } + + /* Update our caches. */ + pVmcsInfo->u32ProcCtls = uProcCtls; + pVmcsInfo->u32XcptBitmap = uXcptBitmap; + + Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0)); + } + else + { + /* + * With nested-guests, we may have extended the guest/host mask here since we + * merged in the outer guest's mask. Thus, the merged mask can include more bits + * (to read from the nested-guest CR0 read-shadow) than the nested hypervisor + * originally supplied. We must copy those bits from the nested-guest CR0 into + * the nested-guest CR0 read-shadow. + */ + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); + uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0; + uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask); + + /* Apply the hardware specified CR0 fixed bits and enable caching. */ + u64GuestCr0 |= fSetCr0; + u64GuestCr0 &= fZapCr0; + u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW); + + Assert(!RT_HI_U32(u64GuestCr0)); + Assert(u64GuestCr0 & X86_CR0_NE); + + /* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */ + int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0); AssertRC(rc); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0); AssertRC(rc); + + Log4Func(("cr0=%#RX64 shadow=%#RX64 vmcs_read_shw=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0, + pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0ReadShadow.u, fSetCr0, fZapCr0)); + } + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR0); + } + + return VINF_SUCCESS; +} + + +/** + * Exports the guest control registers (CR3, CR4) into the guest-state area + * in the VMCS. + * + * @returns VBox strict status code. + * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code + * without unrestricted guest access and the VMMDev is not presently + * mapped (e.g. EFI32). + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks No-long-jump zone!!! + */ +static VBOXSTRICTRC vmxHCExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient) +{ + int rc = VINF_SUCCESS; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Guest CR2. + * It's always loaded in the assembler code. Nothing to do here. + */ + + /* + * Guest CR3. + */ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR3) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3); + + if (VM_IS_VMX_NESTED_PAGING(pVM)) + { +#ifndef IN_NEM_DARWIN + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu); + + /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */ + Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS); + Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000))); + Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff)); + + /* VMX_EPT_MEMTYPE_WB support is already checked in vmxHCSetupTaggedTlb(). */ + pVmcsInfo->HCPhysEPTP |= RT_BF_MAKE(VMX_BF_EPTP_MEMTYPE, VMX_EPTP_MEMTYPE_WB) + | RT_BF_MAKE(VMX_BF_EPTP_PAGE_WALK_LENGTH, VMX_EPTP_PAGE_WALK_LENGTH_4); + + /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */ + AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */ + && ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */ + ("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP)); + AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */ + || (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY), + ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP)); + + rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP); + AssertRC(rc); +#endif + + PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + uint64_t u64GuestCr3 = pCtx->cr3; + if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM) + || CPUMIsGuestPagingEnabledEx(pCtx)) + { + /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */ + if (CPUMIsGuestInPAEModeEx(pCtx)) + { + rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, pCtx->aPaePdpes[0].u); AssertRC(rc); + rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, pCtx->aPaePdpes[1].u); AssertRC(rc); + rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, pCtx->aPaePdpes[2].u); AssertRC(rc); + rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, pCtx->aPaePdpes[3].u); AssertRC(rc); + } + + /* + * The guest's view of its CR3 is unblemished with nested paging when the + * guest is using paging or we have unrestricted guest execution to handle + * the guest when it's not using paging. + */ + } +#ifndef IN_NEM_DARWIN + else + { + /* + * The guest is not using paging, but the CPU (VT-x) has to. While the guest + * thinks it accesses physical memory directly, we use our identity-mapped + * page table to map guest-linear to guest-physical addresses. EPT takes care + * of translating it to host-physical addresses. + */ + RTGCPHYS GCPhys; + Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable); + + /* We obtain it here every time as the guest could have relocated this PCI region. */ + rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys); + if (RT_SUCCESS(rc)) + { /* likely */ } + else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS) + { + Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n")); + return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */ + } + else + AssertMsgFailedReturn(("%Rrc\n", rc), rc); + + u64GuestCr3 = GCPhys; + } +#endif + + Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3)); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, u64GuestCr3); + AssertRC(rc); + } + else + { + Assert(!pVmxTransient->fIsNestedGuest); + /* Non-nested paging case, just use the hypervisor's CR3. */ + RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu); + + Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3)); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, HCPhysGuestCr3); + AssertRC(rc); + } + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR3); + } + + /* + * Guest CR4. + * ASSUMES this is done everytime we get in from ring-3! (XCR0) + */ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR4) + { + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + + uint64_t const fSetCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed0; + uint64_t const fZapCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed1; + + /* + * With nested-guests, we may have extended the guest/host mask here (since we + * merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the + * mask can include more bits (to read from the nested-guest CR4 read-shadow) than + * the nested hypervisor originally supplied. Thus, we should, in essence, copy + * those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow. + */ + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4); + uint64_t u64GuestCr4 = pCtx->cr4; + uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest + ? pCtx->cr4 + : CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask); + Assert(!RT_HI_U32(u64GuestCr4)); + +#ifndef IN_NEM_DARWIN + /* + * Setup VT-x's view of the guest CR4. + * + * If we're emulating real-mode using virtual-8086 mode, we want to redirect software + * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt + * redirection bitmap is already all 0, see hmR3InitFinalizeR0()) + * + * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode". + */ + if (pVmcsInfo->pShared->RealMode.fRealOnV86Active) + { + Assert(pVM->hm.s.vmx.pRealModeTSS); + Assert(PDMVmmDevHeapIsEnabled(pVM)); + u64GuestCr4 &= ~(uint64_t)X86_CR4_VME; + } +#endif + + if (VM_IS_VMX_NESTED_PAGING(pVM)) + { + if ( !CPUMIsGuestPagingEnabledEx(pCtx) + && !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)) + { + /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */ + u64GuestCr4 |= X86_CR4_PSE; + /* Our identity mapping is a 32-bit page directory. */ + u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE; + } + /* else use guest CR4.*/ + } + else + { + Assert(!pVmxTransient->fIsNestedGuest); + + /* + * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host + * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables. + */ + switch (VCPU_2_VMXSTATE(pVCpu).enmShadowMode) + { + case PGMMODE_REAL: /* Real-mode. */ + case PGMMODE_PROTECTED: /* Protected mode without paging. */ + case PGMMODE_32_BIT: /* 32-bit paging. */ + { + u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE; + break; + } + + case PGMMODE_PAE: /* PAE paging. */ + case PGMMODE_PAE_NX: /* PAE paging with NX. */ + { + u64GuestCr4 |= X86_CR4_PAE; + break; + } + + case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */ + case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */ + { +#ifdef VBOX_WITH_64_BITS_GUESTS + /* For our assumption in vmxHCShouldSwapEferMsr. */ + Assert(u64GuestCr4 & X86_CR4_PAE); + break; +#endif + } + default: + AssertFailed(); + return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE; + } + } + + /* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */ + u64GuestCr4 |= fSetCr4; + u64GuestCr4 &= fZapCr4; + + Assert(!RT_HI_U32(u64GuestCr4)); + Assert(u64GuestCr4 & X86_CR4_VMXE); + + /* Commit the CR4 and CR4 read-shadow to the guest VMCS. */ + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR4, u64GuestCr4); AssertRC(rc); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4); AssertRC(rc); + +#ifndef IN_NEM_DARWIN + /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */ + bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0(); + if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0) + { + pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0; + hmR0VmxUpdateStartVmFunction(pVCpu); + } +#endif + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR4); + + Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4)); + } + return rc; +} + + +#ifdef VBOX_STRICT +/** + * Strict function to validate segment registers. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks Will import guest CR0 on strict builds during validation of + * segments. + */ +static void vmxHCValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo) +{ + /* + * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". + * + * The reason we check for attribute value 0 in this function and not just the unusable bit is + * because vmxHCExportGuestSegReg() only updates the VMCS' copy of the value with the + * unusable bit and doesn't change the guest-context value. + */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0); + if ( !VM_IS_VMX_UNRESTRICTED_GUEST(pVM) + && ( !CPUMIsGuestInRealModeEx(pCtx) + && !CPUMIsGuestInV86ModeEx(pCtx))) + { + /* Protected mode checks */ + /* CS */ + Assert(pCtx->cs.Attr.n.u1Present); + Assert(!(pCtx->cs.Attr.u & 0xf00)); + Assert(!(pCtx->cs.Attr.u & 0xfffe0000)); + Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff + || !(pCtx->cs.Attr.n.u1Granularity)); + Assert( !(pCtx->cs.u32Limit & 0xfff00000) + || (pCtx->cs.Attr.n.u1Granularity)); + /* CS cannot be loaded with NULL in protected mode. */ + Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */ + if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11) + Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl); + else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15) + Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl); + else + AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl)); + /* SS */ + Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL)); + Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL)); + if ( !(pCtx->cr0 & X86_CR0_PE) + || pCtx->cs.Attr.n.u4Type == 3) + { + Assert(!pCtx->ss.Attr.n.u2Dpl); + } + if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE)) + { + Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL)); + Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7); + Assert(pCtx->ss.Attr.n.u1Present); + Assert(!(pCtx->ss.Attr.u & 0xf00)); + Assert(!(pCtx->ss.Attr.u & 0xfffe0000)); + Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff + || !(pCtx->ss.Attr.n.u1Granularity)); + Assert( !(pCtx->ss.u32Limit & 0xfff00000) + || (pCtx->ss.Attr.n.u1Granularity)); + } + /* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSegReg(). */ + if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE)) + { + Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED); + Assert(pCtx->ds.Attr.n.u1Present); + Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL)); + Assert(!(pCtx->ds.Attr.u & 0xf00)); + Assert(!(pCtx->ds.Attr.u & 0xfffe0000)); + Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff + || !(pCtx->ds.Attr.n.u1Granularity)); + Assert( !(pCtx->ds.u32Limit & 0xfff00000) + || (pCtx->ds.Attr.n.u1Granularity)); + Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE) + || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ)); + } + if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE)) + { + Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED); + Assert(pCtx->es.Attr.n.u1Present); + Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL)); + Assert(!(pCtx->es.Attr.u & 0xf00)); + Assert(!(pCtx->es.Attr.u & 0xfffe0000)); + Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff + || !(pCtx->es.Attr.n.u1Granularity)); + Assert( !(pCtx->es.u32Limit & 0xfff00000) + || (pCtx->es.Attr.n.u1Granularity)); + Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE) + || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ)); + } + if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE)) + { + Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED); + Assert(pCtx->fs.Attr.n.u1Present); + Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL)); + Assert(!(pCtx->fs.Attr.u & 0xf00)); + Assert(!(pCtx->fs.Attr.u & 0xfffe0000)); + Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff + || !(pCtx->fs.Attr.n.u1Granularity)); + Assert( !(pCtx->fs.u32Limit & 0xfff00000) + || (pCtx->fs.Attr.n.u1Granularity)); + Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE) + || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ)); + } + if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE)) + { + Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED); + Assert(pCtx->gs.Attr.n.u1Present); + Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL)); + Assert(!(pCtx->gs.Attr.u & 0xf00)); + Assert(!(pCtx->gs.Attr.u & 0xfffe0000)); + Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff + || !(pCtx->gs.Attr.n.u1Granularity)); + Assert( !(pCtx->gs.u32Limit & 0xfff00000) + || (pCtx->gs.Attr.n.u1Granularity)); + Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE) + || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ)); + } + /* 64-bit capable CPUs. */ + Assert(!RT_HI_U32(pCtx->cs.u64Base)); + Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base)); + Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base)); + Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base)); + } + else if ( CPUMIsGuestInV86ModeEx(pCtx) + || ( CPUMIsGuestInRealModeEx(pCtx) + && !VM_IS_VMX_UNRESTRICTED_GUEST(pVM))) + { + /* Real and v86 mode checks. */ + /* vmxHCExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */ + uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr; +#ifndef IN_NEM_DARWIN + if (pVmcsInfo->pShared->RealMode.fRealOnV86Active) + { + u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; + u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3; + } + else +#endif + { + u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u; + u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u; + } + + /* CS */ + AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel)); + Assert(pCtx->cs.u32Limit == 0xffff); + AssertMsg(u32CSAttr == 0xf3, ("cs=%#x %#x ", pCtx->cs.Sel, u32CSAttr)); + /* SS */ + Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4); + Assert(pCtx->ss.u32Limit == 0xffff); + Assert(u32SSAttr == 0xf3); + /* DS */ + Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4); + Assert(pCtx->ds.u32Limit == 0xffff); + Assert(u32DSAttr == 0xf3); + /* ES */ + Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4); + Assert(pCtx->es.u32Limit == 0xffff); + Assert(u32ESAttr == 0xf3); + /* FS */ + Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4); + Assert(pCtx->fs.u32Limit == 0xffff); + Assert(u32FSAttr == 0xf3); + /* GS */ + Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4); + Assert(pCtx->gs.u32Limit == 0xffff); + Assert(u32GSAttr == 0xf3); + /* 64-bit capable CPUs. */ + Assert(!RT_HI_U32(pCtx->cs.u64Base)); + Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base)); + Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base)); + Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base)); + } +} +#endif /* VBOX_STRICT */ + + +/** + * Exports a guest segment register into the guest-state area in the VMCS. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * @param iSegReg The segment register number (X86_SREG_XXX). + * @param pSelReg Pointer to the segment selector. + * + * @remarks No-long-jump zone!!! + */ +static int vmxHCExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg) +{ + Assert(iSegReg < X86_SREG_COUNT); + + uint32_t u32Access = pSelReg->Attr.u; +#ifndef IN_NEM_DARWIN + if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active) +#endif + { + /* + * The way to differentiate between whether this is really a null selector or was just + * a selector loaded with 0 in real-mode is using the segment attributes. A selector + * loaded in real-mode with the value 0 is valid and usable in protected-mode and we + * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures + * NULL selectors loaded in protected-mode have their attribute as 0. + */ + if (u32Access) + { } + else + u32Access = X86DESCATTR_UNUSABLE; + } +#ifndef IN_NEM_DARWIN + else + { + /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */ + u32Access = 0xf3; + Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS); + Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM))); + RT_NOREF_PV(pVCpu); + } +#else + RT_NOREF(pVmcsInfo); +#endif + + /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */ + AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED), + ("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u)); + + /* + * Commit it to the VMCS. + */ + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg), pSelReg->Sel); AssertRC(rc); + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg), pSelReg->u32Limit); AssertRC(rc); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg), pSelReg->u64Base); AssertRC(rc); + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access); AssertRC(rc); + return VINF_SUCCESS; +} + + +/** + * Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state + * area in the VMCS. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks Will import guest CR0 on strict builds during validation of + * segments. + * @remarks No-long-jump zone!!! + */ +static int vmxHCExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient) +{ + int rc = VERR_INTERNAL_ERROR_5; +#ifndef IN_NEM_DARWIN + PVMCC pVM = pVCpu->CTX_SUFF(pVM); +#endif + PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; +#ifndef IN_NEM_DARWIN + PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared; +#endif + + /* + * Guest Segment registers: CS, SS, DS, ES, FS, GS. + */ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK) + { + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CS) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS); +#ifndef IN_NEM_DARWIN + if (pVmcsInfoShared->RealMode.fRealOnV86Active) + pVmcsInfoShared->RealMode.AttrCS.u = pCtx->cs.Attr.u; +#endif + rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs); + AssertRC(rc); + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CS); + } + + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SS) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS); +#ifndef IN_NEM_DARWIN + if (pVmcsInfoShared->RealMode.fRealOnV86Active) + pVmcsInfoShared->RealMode.AttrSS.u = pCtx->ss.Attr.u; +#endif + rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss); + AssertRC(rc); + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SS); + } + + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_DS) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS); +#ifndef IN_NEM_DARWIN + if (pVmcsInfoShared->RealMode.fRealOnV86Active) + pVmcsInfoShared->RealMode.AttrDS.u = pCtx->ds.Attr.u; +#endif + rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds); + AssertRC(rc); + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_DS); + } + + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_ES) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES); +#ifndef IN_NEM_DARWIN + if (pVmcsInfoShared->RealMode.fRealOnV86Active) + pVmcsInfoShared->RealMode.AttrES.u = pCtx->es.Attr.u; +#endif + rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es); + AssertRC(rc); + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_ES); + } + + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_FS) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS); +#ifndef IN_NEM_DARWIN + if (pVmcsInfoShared->RealMode.fRealOnV86Active) + pVmcsInfoShared->RealMode.AttrFS.u = pCtx->fs.Attr.u; +#endif + rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs); + AssertRC(rc); + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_FS); + } + + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GS) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS); +#ifndef IN_NEM_DARWIN + if (pVmcsInfoShared->RealMode.fRealOnV86Active) + pVmcsInfoShared->RealMode.AttrGS.u = pCtx->gs.Attr.u; +#endif + rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs); + AssertRC(rc); + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GS); + } + +#ifdef VBOX_STRICT + vmxHCValidateSegmentRegs(pVCpu, pVmcsInfo); +#endif + Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, + pCtx->cs.Attr.u)); + } + + /* + * Guest TR. + */ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_TR) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR); + + /* + * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is + * achieved using the interrupt redirection bitmap (all bits cleared to let the guest + * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup. + */ + uint16_t u16Sel; + uint32_t u32Limit; + uint64_t u64Base; + uint32_t u32AccessRights; +#ifndef IN_NEM_DARWIN + if (!pVmcsInfoShared->RealMode.fRealOnV86Active) +#endif + { + u16Sel = pCtx->tr.Sel; + u32Limit = pCtx->tr.u32Limit; + u64Base = pCtx->tr.u64Base; + u32AccessRights = pCtx->tr.Attr.u; + } +#ifndef IN_NEM_DARWIN + else + { + Assert(!pVmxTransient->fIsNestedGuest); + Assert(pVM->hm.s.vmx.pRealModeTSS); + Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */ + + /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */ + RTGCPHYS GCPhys; + rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys); + AssertRCReturn(rc, rc); + + X86DESCATTR DescAttr; + DescAttr.u = 0; + DescAttr.n.u1Present = 1; + DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY; + + u16Sel = 0; + u32Limit = HM_VTX_TSS_SIZE; + u64Base = GCPhys; + u32AccessRights = DescAttr.u; + } +#endif + + /* Validate. */ + Assert(!(u16Sel & RT_BIT(2))); + AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY + || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights)); + AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights)); + Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/ + Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/ + Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */ + Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */ + Assert( (u32Limit & 0xfff) == 0xfff + || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */ + Assert( !(pCtx->tr.u32Limit & 0xfff00000) + || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */ + + rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, u16Sel); AssertRC(rc); + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRC(rc); + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRC(rc); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRC(rc); + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_TR); + Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit)); + } + + /* + * Guest GDTR. + */ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GDTR) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR); + + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt); AssertRC(rc); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt); AssertRC(rc); + + /* Validate. */ + Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */ + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GDTR); + Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt)); + } + + /* + * Guest LDTR. + */ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_LDTR) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR); + + /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */ + uint32_t u32Access; + if ( !pVmxTransient->fIsNestedGuest + && !pCtx->ldtr.Attr.u) + u32Access = X86DESCATTR_UNUSABLE; + else + u32Access = pCtx->ldtr.Attr.u; + + rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel); AssertRC(rc); + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit); AssertRC(rc); + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRC(rc); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); AssertRC(rc); + + /* Validate. */ + if (!(u32Access & X86DESCATTR_UNUSABLE)) + { + Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */ + Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */ + Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */ + Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */ + Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */ + Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */ + Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff + || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */ + Assert( !(pCtx->ldtr.u32Limit & 0xfff00000) + || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */ + } + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_LDTR); + Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit)); + } + + /* + * Guest IDTR. + */ + if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_IDTR) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR); + + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt); AssertRC(rc); + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt); AssertRC(rc); + + /* Validate. */ + Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */ + + ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_IDTR); + Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt)); + } + + return VINF_SUCCESS; +} + + +/** + * Gets the IEM exception flags for the specified vector and IDT vectoring / + * VM-exit interruption info type. + * + * @returns The IEM exception flags. + * @param uVector The event vector. + * @param uVmxEventType The VMX event type. + * + * @remarks This function currently only constructs flags required for + * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code + * and CR2 aspects of an exception are not included). + */ +static uint32_t vmxHCGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType) +{ + uint32_t fIemXcptFlags; + switch (uVmxEventType) + { + case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT: + case VMX_IDT_VECTORING_INFO_TYPE_NMI: + fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT; + break; + + case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT: + fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT; + break; + + case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: + fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR; + break; + + case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: + { + fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT; + if (uVector == X86_XCPT_BP) + fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR; + else if (uVector == X86_XCPT_OF) + fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR; + else + { + fIemXcptFlags = 0; + AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector)); + } + break; + } + + case VMX_IDT_VECTORING_INFO_TYPE_SW_INT: + fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT; + break; + + default: + fIemXcptFlags = 0; + AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector)); + break; + } + return fIemXcptFlags; +} + + +/** + * Sets an event as a pending event to be injected into the guest. + * + * @param pVCpu The cross context virtual CPU structure. + * @param u32IntInfo The VM-entry interruption-information field. + * @param cbInstr The VM-entry instruction length in bytes (for + * software interrupts, exceptions and privileged + * software exceptions). + * @param u32ErrCode The VM-entry exception error code. + * @param GCPtrFaultAddress The fault-address (CR2) in case it's a + * page-fault. + */ +DECLINLINE(void) vmxHCSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode, + RTGCUINTPTR GCPtrFaultAddress) +{ + Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending); + VCPU_2_VMXSTATE(pVCpu).Event.fPending = true; + VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo = u32IntInfo; + VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode = u32ErrCode; + VCPU_2_VMXSTATE(pVCpu).Event.cbInstr = cbInstr; + VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress = GCPtrFaultAddress; +} + + +/** + * Sets an external interrupt as pending-for-injection into the VM. + * + * @param pVCpu The cross context virtual CPU structure. + * @param u8Interrupt The external interrupt vector. + */ +DECLINLINE(void) vmxHCSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt) +{ + uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1); + vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */); +} + + +/** + * Sets an NMI (\#NMI) exception as pending-for-injection into the VM. + * + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(void) vmxHCSetPendingXcptNmi(PVMCPUCC pVCpu) +{ + uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1); + vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */); +} + + +/** + * Sets a double-fault (\#DF) exception as pending-for-injection into the VM. + * + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(void) vmxHCSetPendingXcptDF(PVMCPUCC pVCpu) +{ + uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1); + vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */); +} + + +/** + * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM. + * + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(void) vmxHCSetPendingXcptUD(PVMCPUCC pVCpu) +{ + uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1); + vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */); +} + + +/** + * Sets a debug (\#DB) exception as pending-for-injection into the VM. + * + * @param pVCpu The cross context virtual CPU structure. + */ +DECLINLINE(void) vmxHCSetPendingXcptDB(PVMCPUCC pVCpu) +{ + uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1); + vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */); +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** + * Sets a general-protection (\#GP) exception as pending-for-injection into the VM. + * + * @param pVCpu The cross context virtual CPU structure. + * @param u32ErrCode The error code for the general-protection exception. + */ +DECLINLINE(void) vmxHCSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode) +{ + uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1); + vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */); +} + + +/** + * Sets a stack (\#SS) exception as pending-for-injection into the VM. + * + * @param pVCpu The cross context virtual CPU structure. + * @param u32ErrCode The error code for the stack exception. + */ +DECLINLINE(void) vmxHCSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode) +{ + uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1); + vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */); +} +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + + +/** + * Fixes up attributes for the specified segment register. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pSelReg The segment register that needs fixing. + * @param pszRegName The register name (for logging and assertions). + */ +static void vmxHCFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName) +{ + Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE); + + /* + * If VT-x marks the segment as unusable, most other bits remain undefined: + * - For CS the L, D and G bits have meaning. + * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox). + * - For the remaining data segments no bits are defined. + * + * The present bit and the unusable bit has been observed to be set at the + * same time (the selector was supposed to be invalid as we started executing + * a V8086 interrupt in ring-0). + * + * What should be important for the rest of the VBox code, is that the P bit is + * cleared. Some of the other VBox code recognizes the unusable bit, but + * AMD-V certainly don't, and REM doesn't really either. So, to be on the + * safe side here, we'll strip off P and other bits we don't care about. If + * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed. + * + * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers". + */ +#ifdef VBOX_STRICT + uint32_t const uAttr = pSelReg->Attr.u; +#endif + + /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */ + pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G + | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT; + +#ifdef VBOX_STRICT +# ifndef IN_NEM_DARWIN + VMMRZCallRing3Disable(pVCpu); +# endif + Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u)); +# ifdef DEBUG_bird + AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u, + ("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n", + pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit)); +# endif +# ifndef IN_NEM_DARWIN + VMMRZCallRing3Enable(pVCpu); +# endif + NOREF(uAttr); +#endif + RT_NOREF2(pVCpu, pszRegName); +} + + +/** + * Imports a guest segment register from the current VMCS into the guest-CPU + * context. + * + * @param pVCpu The cross context virtual CPU structure. + * @tparam a_iSegReg The segment register number (X86_SREG_XXX). + * + * @remarks Called with interrupts and/or preemption disabled. + */ +template +DECLINLINE(void) vmxHCImportGuestSegReg(PVMCPUCC pVCpu) +{ + AssertCompile(a_iSegReg < X86_SREG_COUNT); + /* Check that the macros we depend upon here and in the export parenter function works: */ +#define MY_SEG_VMCS_FIELD(a_FieldPrefix, a_FieldSuff) \ + ( a_iSegReg == X86_SREG_ES ? a_FieldPrefix ## ES ## a_FieldSuff \ + : a_iSegReg == X86_SREG_CS ? a_FieldPrefix ## CS ## a_FieldSuff \ + : a_iSegReg == X86_SREG_SS ? a_FieldPrefix ## SS ## a_FieldSuff \ + : a_iSegReg == X86_SREG_DS ? a_FieldPrefix ## DS ## a_FieldSuff \ + : a_iSegReg == X86_SREG_FS ? a_FieldPrefix ## FS ## a_FieldSuff \ + : a_iSegReg == X86_SREG_GS ? a_FieldPrefix ## GS ## a_FieldSuff : 0) + AssertCompile(VMX_VMCS_GUEST_SEG_BASE(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS_GUEST_,_BASE)); + AssertCompile(VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS16_GUEST_,_SEL)); + AssertCompile(VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_LIMIT)); + AssertCompile(VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg) == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_ACCESS_RIGHTS)); + + PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[a_iSegReg]; + + uint16_t u16Sel; + int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg), &u16Sel); AssertRC(rc); + pSelReg->Sel = u16Sel; + pSelReg->ValidSel = u16Sel; + + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg), &pSelReg->u32Limit); AssertRC(rc); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(a_iSegReg), &pSelReg->u64Base); AssertRC(rc); + + uint32_t u32Attr; + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg), &u32Attr); AssertRC(rc); + pSelReg->Attr.u = u32Attr; + if (u32Attr & X86DESCATTR_UNUSABLE) + vmxHCFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + a_iSegReg * 3); + + pSelReg->fFlags = CPUMSELREG_FLAGS_VALID; +} + + +/** + * Imports the guest LDTR from the current VMCS into the guest-CPU context. + * + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks Called with interrupts and/or preemption disabled. + */ +DECLINLINE(void) vmxHCImportGuestLdtr(PVMCPUCC pVCpu) +{ + uint16_t u16Sel; + uint64_t u64Base; + uint32_t u32Limit, u32Attr; + int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL, &u16Sel); AssertRC(rc); + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit); AssertRC(rc); + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE, &u64Base); AssertRC(rc); + + pVCpu->cpum.GstCtx.ldtr.Sel = u16Sel; + pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel; + pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit; + pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base; + pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr; + if (u32Attr & X86DESCATTR_UNUSABLE) + vmxHCFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR"); +} + + +/** + * Imports the guest TR from the current VMCS into the guest-CPU context. + * + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks Called with interrupts and/or preemption disabled. + */ +DECLINLINE(void) vmxHCImportGuestTr(PVMCPUCC pVCpu) +{ + uint16_t u16Sel; + uint64_t u64Base; + uint32_t u32Limit, u32Attr; + int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL, &u16Sel); AssertRC(rc); + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit); AssertRC(rc); + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr); AssertRC(rc); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE, &u64Base); AssertRC(rc); + + pVCpu->cpum.GstCtx.tr.Sel = u16Sel; + pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel; + pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID; + pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit; + pVCpu->cpum.GstCtx.tr.u64Base = u64Base; + pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr; + /* TR is the only selector that can never be unusable. */ + Assert(!(u32Attr & X86DESCATTR_UNUSABLE)); +} + + +/** + * Core: Imports the guest RIP from the VMCS back into the guest-CPU context. + * + * @returns The RIP value. + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks Called with interrupts and/or preemption disabled, should not assert! + * @remarks Do -not- call this function directly! + */ +DECL_FORCE_INLINE(uint64_t) vmxHCImportGuestCoreRip(PVMCPUCC pVCpu) +{ + uint64_t u64Val; + int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val); + AssertRC(rc); + + pVCpu->cpum.GstCtx.rip = u64Val; + + return u64Val; +} + + +/** + * Imports the guest RIP from the VMCS back into the guest-CPU context. + * + * @param pVCpu The cross context virtual CPU structure. + * + * @remarks Called with interrupts and/or preemption disabled, should not assert! + * @remarks Do -not- call this function directly, use vmxHCImportGuestState() + * instead!!! + */ +DECLINLINE(void) vmxHCImportGuestRip(PVMCPUCC pVCpu) +{ + if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP) + { + EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false); + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP; + } +} + + +/** + * Core: Imports the guest RFLAGS from the VMCS back into the guest-CPU context. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks Called with interrupts and/or preemption disabled, should not assert! + * @remarks Do -not- call this function directly! + */ +DECL_FORCE_INLINE(void) vmxHCImportGuestCoreRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo) +{ + uint64_t fRFlags; + int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &fRFlags); + AssertRC(rc); + + Assert((fRFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK); + Assert((fRFlags & ~(uint64_t)(X86_EFL_1 | X86_EFL_LIVE_MASK)) == 0); + + pVCpu->cpum.GstCtx.rflags.u = fRFlags; +#ifndef IN_NEM_DARWIN + PCVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared; + if (!pVmcsInfoShared->RealMode.fRealOnV86Active) + { /* mostly likely */ } + else + { + pVCpu->cpum.GstCtx.eflags.Bits.u1VM = 0; + pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL = pVmcsInfoShared->RealMode.Eflags.Bits.u2IOPL; + } +#else + RT_NOREF(pVmcsInfo); +#endif +} + + +/** + * Imports the guest RFLAGS from the VMCS back into the guest-CPU context. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks Called with interrupts and/or preemption disabled, should not assert! + * @remarks Do -not- call this function directly, use vmxHCImportGuestState() + * instead!!! + */ +DECLINLINE(void) vmxHCImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo) +{ + if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RFLAGS) + { + vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo); + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RFLAGS; + } +} + + +/** + * Worker for vmxHCImportGuestIntrState that handles the case where any of the + * relevant VMX_VMCS32_GUEST_INT_STATE bits are set. + */ +DECL_NO_INLINE(static,void) vmxHCImportGuestIntrStateSlow(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t fGstIntState) +{ + /* + * We must import RIP here to set our EM interrupt-inhibited state. + * We also import RFLAGS as our code that evaluates pending interrupts + * before VM-entry requires it. + */ + vmxHCImportGuestRip(pVCpu); + vmxHCImportGuestRFlags(pVCpu, pVmcsInfo); + + CPUMUpdateInterruptShadowSsStiEx(&pVCpu->cpum.GstCtx, + RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS), + RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI), + pVCpu->cpum.GstCtx.rip); + CPUMUpdateInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx, RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)); +} + + +/** + * Imports the guest interruptibility-state from the VMCS back into the guest-CPU + * context. + * + * @note May import RIP and RFLAGS if interrupt or NMI are blocked. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks Called with interrupts and/or preemption disabled, try not to assert and + * do not log! + * @remarks Do -not- call this function directly, use vmxHCImportGuestState() + * instead!!! + */ +DECLINLINE(void) vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo) +{ + uint32_t u32Val; + int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32Val); AssertRC(rc); + if (!u32Val) + { + CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); + CPUMClearInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx); + } + else + vmxHCImportGuestIntrStateSlow(pVCpu, pVmcsInfo, u32Val); +} + + +/** + * Worker for VMXR0ImportStateOnDemand. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * @param fWhat What to import, CPUMCTX_EXTRN_XXX. + */ +static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat) +{ + int rc = VINF_SUCCESS; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + uint32_t u32Val; + + /* + * Note! This is hack to workaround a mysterious BSOD observed with release builds + * on Windows 10 64-bit hosts. Profile and debug builds are not affected and + * neither are other host platforms. + * + * Committing this temporarily as it prevents BSOD. + * + * Update: This is very likely a compiler optimization bug, see @bugref{9180}. + */ +#ifdef RT_OS_WINDOWS + if (pVM == 0 || pVM == (void *)(uintptr_t)-1) + return VERR_HM_IPE_1; +#endif + + STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x); + +#ifndef IN_NEM_DARWIN + /* + * We disable interrupts to make the updating of the state and in particular + * the fExtrn modification atomic wrt to preemption hooks. + */ + RTCCUINTREG const fEFlags = ASMIntDisableFlags(); +#endif + + fWhat &= pCtx->fExtrn; + if (fWhat) + { + do + { + if (fWhat & CPUMCTX_EXTRN_RIP) + vmxHCImportGuestRip(pVCpu); + + if (fWhat & CPUMCTX_EXTRN_RFLAGS) + vmxHCImportGuestRFlags(pVCpu, pVmcsInfo); + + /* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */ + if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI)) + vmxHCImportGuestIntrState(pVCpu, pVmcsInfo); + + if (fWhat & CPUMCTX_EXTRN_RSP) + { + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pCtx->rsp); + AssertRC(rc); + } + + if (fWhat & CPUMCTX_EXTRN_SREG_MASK) + { + PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared; +#ifndef IN_NEM_DARWIN + bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active; +#else + bool const fRealOnV86Active = false; /* HV supports only unrestricted guest execution. */ +#endif + if (fWhat & CPUMCTX_EXTRN_CS) + { + vmxHCImportGuestSegReg(pVCpu); + vmxHCImportGuestRip(pVCpu); /** @todo WTF? */ + if (fRealOnV86Active) + pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u; + EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */); + } + if (fWhat & CPUMCTX_EXTRN_SS) + { + vmxHCImportGuestSegReg(pVCpu); + if (fRealOnV86Active) + pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u; + } + if (fWhat & CPUMCTX_EXTRN_DS) + { + vmxHCImportGuestSegReg(pVCpu); + if (fRealOnV86Active) + pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u; + } + if (fWhat & CPUMCTX_EXTRN_ES) + { + vmxHCImportGuestSegReg(pVCpu); + if (fRealOnV86Active) + pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u; + } + if (fWhat & CPUMCTX_EXTRN_FS) + { + vmxHCImportGuestSegReg(pVCpu); + if (fRealOnV86Active) + pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u; + } + if (fWhat & CPUMCTX_EXTRN_GS) + { + vmxHCImportGuestSegReg(pVCpu); + if (fRealOnV86Active) + pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u; + } + } + + if (fWhat & CPUMCTX_EXTRN_TABLE_MASK) + { + if (fWhat & CPUMCTX_EXTRN_LDTR) + vmxHCImportGuestLdtr(pVCpu); + + if (fWhat & CPUMCTX_EXTRN_GDTR) + { + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pCtx->gdtr.pGdt); AssertRC(rc); + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc); + pCtx->gdtr.cbGdt = u32Val; + } + + /* Guest IDTR. */ + if (fWhat & CPUMCTX_EXTRN_IDTR) + { + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pCtx->idtr.pIdt); AssertRC(rc); + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc); + pCtx->idtr.cbIdt = u32Val; + } + + /* Guest TR. */ + if (fWhat & CPUMCTX_EXTRN_TR) + { +#ifndef IN_NEM_DARWIN + /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, + don't need to import that one. */ + if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active) +#endif + vmxHCImportGuestTr(pVCpu); + } + } + + if (fWhat & CPUMCTX_EXTRN_DR7) + { +#ifndef IN_NEM_DARWIN + if (!pVCpu->hmr0.s.fUsingHyperDR7) +#endif + { + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pCtx->dr[7]); + AssertRC(rc); + } + } + + if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS) + { + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip); AssertRC(rc); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp); AssertRC(rc); + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc); + pCtx->SysEnter.cs = u32Val; + } + +#ifndef IN_NEM_DARWIN + if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE) + { + if ( pVM->hmr0.s.fAllow64BitGuests + && (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)) + pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); + } + + if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS) + { + if ( pVM->hmr0.s.fAllow64BitGuests + && (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)) + { + pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); + pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR); + pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK); + } + } + + if (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS)) + { + PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared; + PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore; + uint32_t const cMsrs = pVmcsInfo->cExitMsrStore; + Assert(pMsrs); + Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc)); + Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE); + for (uint32_t i = 0; i < cMsrs; i++) + { + uint32_t const idMsr = pMsrs[i].u32Msr; + switch (idMsr) + { + case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break; + case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break; + case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break; + default: + { + uint32_t idxLbrMsr; + if (VM_IS_VMX_LBR(pVM)) + { + if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr)) + { + Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr)); + pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value; + break; + } + if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr)) + { + Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr)); + pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value; + break; + } + if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr) + { + pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value; + break; + } + /* Fallthru (no break) */ + } + pCtx->fExtrn = 0; + VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr; + ASMSetFlags(fEFlags); + AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs)); + return VERR_HM_UNEXPECTED_LD_ST_MSR; + } + } + } + } +#endif + + if (fWhat & CPUMCTX_EXTRN_CR_MASK) + { + if (fWhat & CPUMCTX_EXTRN_CR0) + { + uint64_t u64Cr0; + uint64_t u64Shadow; + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc); +#ifndef VBOX_WITH_NESTED_HWVIRT_VMX + u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask) + | (u64Shadow & pVmcsInfo->u64Cr0Mask); +#else + if (!CPUMIsGuestInVmxNonRootMode(pCtx)) + { + u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask) + | (u64Shadow & pVmcsInfo->u64Cr0Mask); + } + else + { + /* + * We've merged the guest and nested-guest's CR0 guest/host mask while executing + * the nested-guest using hardware-assisted VMX. Accordingly we need to + * re-construct CR0. See @bugref{9180#c95} for details. + */ + PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo; + PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + u64Cr0 = (u64Cr0 & ~(pVmcsInfoGst->u64Cr0Mask & pVmcsNstGst->u64Cr0Mask.u)) + | (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u) + | (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u)); + Assert(u64Cr0 & X86_CR0_NE); + } +#endif +#ifndef IN_NEM_DARWIN + VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */ +#endif + CPUMSetGuestCR0(pVCpu, u64Cr0); +#ifndef IN_NEM_DARWIN + VMMRZCallRing3Enable(pVCpu); +#endif + } + + if (fWhat & CPUMCTX_EXTRN_CR4) + { + uint64_t u64Cr4; + uint64_t u64Shadow; + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc); + rc |= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc); +#ifndef VBOX_WITH_NESTED_HWVIRT_VMX + u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask) + | (u64Shadow & pVmcsInfo->u64Cr4Mask); +#else + if (!CPUMIsGuestInVmxNonRootMode(pCtx)) + { + u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask) + | (u64Shadow & pVmcsInfo->u64Cr4Mask); + } + else + { + /* + * We've merged the guest and nested-guest's CR4 guest/host mask while executing + * the nested-guest using hardware-assisted VMX. Accordingly we need to + * re-construct CR4. See @bugref{9180#c95} for details. + */ + PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo; + PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + u64Cr4 = (u64Cr4 & ~(pVmcsInfo->u64Cr4Mask & pVmcsNstGst->u64Cr4Mask.u)) + | (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u) + | (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u)); + Assert(u64Cr4 & X86_CR4_VMXE); + } +#endif + pCtx->cr4 = u64Cr4; + } + + if (fWhat & CPUMCTX_EXTRN_CR3) + { + /* CR0.PG bit changes are always intercepted, so it's up to date. */ + if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM) + || ( VM_IS_VMX_NESTED_PAGING(pVM) + && CPUMIsGuestPagingEnabledEx(pCtx))) + { + uint64_t u64Cr3; + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc); + if (pCtx->cr3 != u64Cr3) + { + pCtx->cr3 = u64Cr3; + VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3); + } + + /* + * If the guest is in PAE mode, sync back the PDPE's into the guest state. + * CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. + */ + if (CPUMIsGuestInPAEModeEx(pCtx)) + { + X86PDPE aPaePdpes[4]; + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc); + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc); + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc); + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc); + if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes))) + { + memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes)); + /* PGM now updates PAE PDPTEs while updating CR3. */ + VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3); + } + } + } + } + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (fWhat & CPUMCTX_EXTRN_HWVIRT) + { + if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING) + && !CPUMIsGuestInVmxNonRootMode(pCtx)) + { + Assert(CPUMIsGuestInVmxRootMode(pCtx)); + rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo); + if (RT_SUCCESS(rc)) + { /* likely */ } + else + break; + } + } +#endif + } while (0); + + if (RT_SUCCESS(rc)) + { + /* Update fExtrn. */ + pCtx->fExtrn &= ~fWhat; + + /* If everything has been imported, clear the HM keeper bit. */ + if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL)) + { +#ifndef IN_NEM_DARWIN + pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM; +#else + pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM; +#endif + Assert(!pCtx->fExtrn); + } + } + } +#ifndef IN_NEM_DARWIN + else + AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn)); + + /* + * Restore interrupts. + */ + ASMSetFlags(fEFlags); +#endif + + STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x); + + if (RT_SUCCESS(rc)) + { /* likely */ } + else + return rc; + + /* + * Honor any pending CR3 updates. + * + * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback() + * -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp + * -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are. + * + * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus + * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that + * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should + * -NOT- check if CPUMCTX_EXTRN_CR3 is set! + * + * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here. + * + * The force-flag is checked first as it's cheaper for potential superfluous calls to this function. + */ + if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) +#ifndef IN_NEM_DARWIN + && VMMRZCallRing3IsEnabled(pVCpu) +#endif + ) + { + Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3)); + PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu)); + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)); + } + + return VINF_SUCCESS; +} + + +/** + * Internal state fetcher, inner version where we fetch all of a_fWhat. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * @param fEFlags Saved EFLAGS for restoring the interrupt flag. Ignored + * in NEM/darwin context. + * @tparam a_fWhat What to import, zero or more bits from + * HMVMX_CPUMCTX_EXTRN_ALL. + */ +template +static int vmxHCImportGuestStateInner(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t fEFlags) +{ + Assert(a_fWhat != 0); /* No AssertCompile as the assertion probably kicks in before the compiler (clang) discards it. */ + AssertCompile(!(a_fWhat & ~HMVMX_CPUMCTX_EXTRN_ALL)); + Assert( (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == a_fWhat + || (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == (a_fWhat & ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS))); + + STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x); + + PVMCC const pVM = pVCpu->CTX_SUFF(pVM); + + /* RIP and RFLAGS may have been imported already by the post exit code + together with the CPUMCTX_EXTRN_INHIBIT_INT/NMI state, so this part + of the code is skipping this part of the code. */ + if ( (a_fWhat & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)) + && pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)) + { + if (a_fWhat & CPUMCTX_EXTRN_RFLAGS) + vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo); + + if (a_fWhat & CPUMCTX_EXTRN_RIP) + { + if (!(a_fWhat & CPUMCTX_EXTRN_CS)) + EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false); + else + vmxHCImportGuestCoreRip(pVCpu); + } + } + + /* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */ + if (a_fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI)) + vmxHCImportGuestIntrState(pVCpu, pVmcsInfo); + + if (a_fWhat & (CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TR)) + { + if (a_fWhat & CPUMCTX_EXTRN_CS) + { + vmxHCImportGuestSegReg(pVCpu); + /** @todo try get rid of this carp, it smells and is probably never ever + * used: */ + if ( !(a_fWhat & CPUMCTX_EXTRN_RIP) + && (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP)) + { + vmxHCImportGuestCoreRip(pVCpu); + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP; + } + EMHistoryUpdatePC(pVCpu, pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, true /* fFlattened */); + } + if (a_fWhat & CPUMCTX_EXTRN_SS) + vmxHCImportGuestSegReg(pVCpu); + if (a_fWhat & CPUMCTX_EXTRN_DS) + vmxHCImportGuestSegReg(pVCpu); + if (a_fWhat & CPUMCTX_EXTRN_ES) + vmxHCImportGuestSegReg(pVCpu); + if (a_fWhat & CPUMCTX_EXTRN_FS) + vmxHCImportGuestSegReg(pVCpu); + if (a_fWhat & CPUMCTX_EXTRN_GS) + vmxHCImportGuestSegReg(pVCpu); + + /* Guest TR. + Real-mode emulation using virtual-8086 mode has the fake TSS + (pRealModeTSS) in TR, don't need to import that one. */ +#ifndef IN_NEM_DARWIN + PVMXVMCSINFOSHARED const pVmcsInfoShared = pVmcsInfo->pShared; + bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active; + if ((a_fWhat & CPUMCTX_EXTRN_TR) && !fRealOnV86Active) +#else + if (a_fWhat & CPUMCTX_EXTRN_TR) +#endif + vmxHCImportGuestTr(pVCpu); + +#ifndef IN_NEM_DARWIN /* NEM/Darwin: HV supports only unrestricted guest execution. */ + if (fRealOnV86Active) + { + if (a_fWhat & CPUMCTX_EXTRN_CS) + pVCpu->cpum.GstCtx.cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u; + if (a_fWhat & CPUMCTX_EXTRN_SS) + pVCpu->cpum.GstCtx.ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u; + if (a_fWhat & CPUMCTX_EXTRN_DS) + pVCpu->cpum.GstCtx.ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u; + if (a_fWhat & CPUMCTX_EXTRN_ES) + pVCpu->cpum.GstCtx.es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u; + if (a_fWhat & CPUMCTX_EXTRN_FS) + pVCpu->cpum.GstCtx.fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u; + if (a_fWhat & CPUMCTX_EXTRN_GS) + pVCpu->cpum.GstCtx.gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u; + } +#endif + } + + if (a_fWhat & CPUMCTX_EXTRN_RSP) + { + int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pVCpu->cpum.GstCtx.rsp); + AssertRC(rc); + } + + if (a_fWhat & CPUMCTX_EXTRN_LDTR) + vmxHCImportGuestLdtr(pVCpu); + + if (a_fWhat & CPUMCTX_EXTRN_GDTR) + { + int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &pVCpu->cpum.GstCtx.gdtr.pGdt); AssertRC(rc1); + uint32_t u32Val; + int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc2); + pVCpu->cpum.GstCtx.gdtr.cbGdt = (uint16_t)u32Val; + } + + /* Guest IDTR. */ + if (a_fWhat & CPUMCTX_EXTRN_IDTR) + { + int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &pVCpu->cpum.GstCtx.idtr.pIdt); AssertRC(rc1); + uint32_t u32Val; + int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc2); + pVCpu->cpum.GstCtx.idtr.cbIdt = (uint64_t)u32Val; + } + + if (a_fWhat & CPUMCTX_EXTRN_DR7) + { +#ifndef IN_NEM_DARWIN + if (!pVCpu->hmr0.s.fUsingHyperDR7) +#endif + { + int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pVCpu->cpum.GstCtx.dr[7]); + AssertRC(rc); + } + } + + if (a_fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS) + { + int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP, &pVCpu->cpum.GstCtx.SysEnter.eip); AssertRC(rc1); + int const rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP, &pVCpu->cpum.GstCtx.SysEnter.esp); AssertRC(rc2); + uint32_t u32Val; + int const rc3 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc3); + pVCpu->cpum.GstCtx.SysEnter.cs = u32Val; + } + +#ifndef IN_NEM_DARWIN + if (a_fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE) + { + if ( (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST) + && pVM->hmr0.s.fAllow64BitGuests) + pVCpu->cpum.GstCtx.msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); + } + + if (a_fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS) + { + if ( (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST) + && pVM->hmr0.s.fAllow64BitGuests) + { + pVCpu->cpum.GstCtx.msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); + pVCpu->cpum.GstCtx.msrSTAR = ASMRdMsr(MSR_K6_STAR); + pVCpu->cpum.GstCtx.msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK); + } + } + + if (a_fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS)) + { + PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared; + PCVMXAUTOMSR pMsrs = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore; + uint32_t const cMsrs = pVmcsInfo->cExitMsrStore; + Assert(pMsrs); + Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc)); + Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE); + for (uint32_t i = 0; i < cMsrs; i++) + { + uint32_t const idMsr = pMsrs[i].u32Msr; + switch (idMsr) + { + case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value); break; + case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value); break; + case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break; + default: + { + uint32_t idxLbrMsr; + if (VM_IS_VMX_LBR(pVM)) + { + if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr)) + { + Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr)); + pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value; + break; + } + if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr)) + { + Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr)); + pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value; + break; + } + if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr) + { + pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value; + break; + } + } + pVCpu->cpum.GstCtx.fExtrn = 0; + VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr; + ASMSetFlags(fEFlags); + AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs)); + return VERR_HM_UNEXPECTED_LD_ST_MSR; + } + } + } + } +#endif + + if (a_fWhat & CPUMCTX_EXTRN_CR0) + { + uint64_t u64Cr0; + uint64_t u64Shadow; + int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Cr0); AssertRC(rc1); + int const rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc2); +#ifndef VBOX_WITH_NESTED_HWVIRT_VMX + u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask) + | (u64Shadow & pVmcsInfo->u64Cr0Mask); +#else + if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)) + u64Cr0 = (u64Cr0 & ~pVmcsInfo->u64Cr0Mask) + | (u64Shadow & pVmcsInfo->u64Cr0Mask); + else + { + /* + * We've merged the guest and nested-guest's CR0 guest/host mask while executing + * the nested-guest using hardware-assisted VMX. Accordingly we need to + * re-construct CR0. See @bugref{9180#c95} for details. + */ + PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo; + PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + u64Cr0 = (u64Cr0 & ~(pVmcsInfoGst->u64Cr0Mask & pVmcsNstGst->u64Cr0Mask.u)) + | (pVmcsNstGst->u64GuestCr0.u & pVmcsNstGst->u64Cr0Mask.u) + | (u64Shadow & (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u)); + Assert(u64Cr0 & X86_CR0_NE); + } +#endif +#ifndef IN_NEM_DARWIN + VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */ +#endif + CPUMSetGuestCR0(pVCpu, u64Cr0); +#ifndef IN_NEM_DARWIN + VMMRZCallRing3Enable(pVCpu); +#endif + } + + if (a_fWhat & CPUMCTX_EXTRN_CR4) + { + uint64_t u64Cr4; + uint64_t u64Shadow; + int rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64Cr4); AssertRC(rc1); + int rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc2); +#ifndef VBOX_WITH_NESTED_HWVIRT_VMX + u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask) + | (u64Shadow & pVmcsInfo->u64Cr4Mask); +#else + if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)) + u64Cr4 = (u64Cr4 & ~pVmcsInfo->u64Cr4Mask) + | (u64Shadow & pVmcsInfo->u64Cr4Mask); + else + { + /* + * We've merged the guest and nested-guest's CR4 guest/host mask while executing + * the nested-guest using hardware-assisted VMX. Accordingly we need to + * re-construct CR4. See @bugref{9180#c95} for details. + */ + PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo; + PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + u64Cr4 = (u64Cr4 & ~(pVmcsInfo->u64Cr4Mask & pVmcsNstGst->u64Cr4Mask.u)) + | (pVmcsNstGst->u64GuestCr4.u & pVmcsNstGst->u64Cr4Mask.u) + | (u64Shadow & (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u)); + Assert(u64Cr4 & X86_CR4_VMXE); + } +#endif + pVCpu->cpum.GstCtx.cr4 = u64Cr4; + } + + if (a_fWhat & CPUMCTX_EXTRN_CR3) + { + /* CR0.PG bit changes are always intercepted, so it's up to date. */ + if ( VM_IS_VMX_UNRESTRICTED_GUEST(pVM) + || ( VM_IS_VMX_NESTED_PAGING(pVM) + && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))) + { + uint64_t u64Cr3; + int const rc0 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3); AssertRC(rc0); + if (pVCpu->cpum.GstCtx.cr3 != u64Cr3) + { + pVCpu->cpum.GstCtx.cr3 = u64Cr3; + VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3); + } + + /* + * If the guest is in PAE mode, sync back the PDPE's into the guest state. + * CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. + */ + if (CPUMIsGuestInPAEModeEx(&pVCpu->cpum.GstCtx)) + { + X86PDPE aPaePdpes[4]; + int const rc1 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc1); + int const rc2 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc2); + int const rc3 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc3); + int const rc4 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc4); + if (memcmp(&aPaePdpes[0], &pVCpu->cpum.GstCtx.aPaePdpes[0], sizeof(aPaePdpes))) + { + memcpy(&pVCpu->cpum.GstCtx.aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes)); + /* PGM now updates PAE PDPTEs while updating CR3. */ + VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3); + } + } + } + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if (a_fWhat & CPUMCTX_EXTRN_HWVIRT) + { + if ( (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING) + && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)) + { + Assert(CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)); + int const rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo); + AssertRCReturn(rc, rc); + } + } +#endif + + /* Update fExtrn. */ + pVCpu->cpum.GstCtx.fExtrn &= ~a_fWhat; + + /* If everything has been imported, clear the HM keeper bit. */ + if (!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL)) + { +#ifndef IN_NEM_DARWIN + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM; +#else + pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM; +#endif + Assert(!pVCpu->cpum.GstCtx.fExtrn); + } + + STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x); + + /* + * Honor any pending CR3 updates. + * + * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback() + * -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp + * -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are. + * + * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus + * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that + * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should + * -NOT- check if CPUMCTX_EXTRN_CR3 is set! + * + * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here. + * + * The force-flag is checked first as it's cheaper for potential superfluous calls to this function. + */ +#ifndef IN_NEM_DARWIN + if (!(a_fWhat & CPUMCTX_EXTRN_CR3) + ? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) || !VMMRZCallRing3IsEnabled(pVCpu)) + : !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) || !VMMRZCallRing3IsEnabled(pVCpu) ) + return VINF_SUCCESS; + ASMSetFlags(fEFlags); +#else + if (!(a_fWhat & CPUMCTX_EXTRN_CR3) + ? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)) + : !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) ) + return VINF_SUCCESS; + RT_NOREF_PV(fEFlags); +#endif + + Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3)); + PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu)); + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)); + return VINF_SUCCESS; +} + + +/** + * Internal state fetcher. + * + * @returns VBox status code. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * @param pszCaller For logging. + * @tparam a_fWhat What needs to be imported, CPUMCTX_EXTRN_XXX. + * @tparam a_fDoneLocal What's ASSUMED to have been retrieved locally + * already. This is ORed together with @a a_fWhat when + * calculating what needs fetching (just for safety). + * @tparam a_fDonePostExit What's ASSUMED to been been retrieved by + * hmR0VmxPostRunGuest()/nemR3DarwinHandleExitCommon() + * already. This is ORed together with @a a_fWhat when + * calculating what needs fetching (just for safety). + */ +template +DECLINLINE(int) vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, const char *pszCaller) +{ + RT_NOREF_PV(pszCaller); + if ((a_fWhat | a_fDoneLocal | a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL) + { +#ifndef IN_NEM_DARWIN + /* + * We disable interrupts to make the updating of the state and in particular + * the fExtrn modification atomic wrt to preemption hooks. + */ + RTCCUINTREG const fEFlags = ASMIntDisableFlags(); +#else + RTCCUINTREG const fEFlags = 0; +#endif + + /* + * We combine all three parameters and take the (probably) inlined optimized + * code path for the new things specified in a_fWhat. + * + * As a tweak to deal with exits that have INHIBIT_INT/NMI active, causing + * vmxHCImportGuestIntrState to automatically fetch both RIP & RFLAGS, we + * also take the streamlined path when both of these are cleared in fExtrn + * already. vmxHCImportGuestStateInner checks fExtrn before fetching. This + * helps with MWAIT and HLT exits that always inhibit IRQs on many platforms. + */ + uint64_t const fWhatToDo = pVCpu->cpum.GstCtx.fExtrn + & ((a_fWhat | a_fDoneLocal | a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL); + if (RT_LIKELY( ( fWhatToDo == (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal | a_fDonePostExit)) + || fWhatToDo == ( a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal | a_fDonePostExit) + & ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)) /* fetch with INHIBIT_INT/NMI */)) + && (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal | a_fDonePostExit)) != 0 /* just in case */) + { + int const rc = vmxHCImportGuestStateInner< a_fWhat + & HMVMX_CPUMCTX_EXTRN_ALL + & ~(a_fDoneLocal | a_fDonePostExit)>(pVCpu, pVmcsInfo, fEFlags); +#ifndef IN_NEM_DARWIN + ASMSetFlags(fEFlags); +#endif + return rc; + } + +#ifndef IN_NEM_DARWIN + ASMSetFlags(fEFlags); +#endif + + /* + * We shouldn't normally get here, but it may happen when executing + * in the debug run-loops. Typically, everything should already have + * been fetched then. Otherwise call the fallback state import function. + */ + if (fWhatToDo == 0) + { /* hope the cause was the debug loop or something similar */ } + else + { + STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestStateFallback); + Log11Func(("a_fWhat=%#RX64/%#RX64/%#RX64 fExtrn=%#RX64 => %#RX64 - Taking inefficient code path from %s!\n", + a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL, a_fDoneLocal & HMVMX_CPUMCTX_EXTRN_ALL, + a_fDonePostExit & HMVMX_CPUMCTX_EXTRN_ALL, pVCpu->cpum.GstCtx.fExtrn, fWhatToDo, pszCaller)); + return vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, a_fWhat | a_fDoneLocal | a_fDonePostExit); + } + } + return VINF_SUCCESS; +} + + +/** + * Check per-VM and per-VCPU force flag actions that require us to go back to + * ring-3 for one reason or another. + * + * @returns Strict VBox status code (i.e. informational status codes too) + * @retval VINF_SUCCESS if we don't have any actions that require going back to + * ring-3. + * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync. + * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware + * interrupts) + * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires + * all EMTs to be in ring-3. + * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests. + * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return + * to the EM loop. + * + * @param pVCpu The cross context virtual CPU structure. + * @param fIsNestedGuest Flag whether this is for a for a pending nested guest event. + * @param fStepping Whether we are single-stepping the guest using the + * hypervisor debugger. + * + * @remarks This might cause nested-guest VM-exits, caller must check if the guest + * is no longer in VMX non-root mode. + */ +static VBOXSTRICTRC vmxHCCheckForceFlags(PVMCPUCC pVCpu, bool fIsNestedGuest, bool fStepping) +{ +#ifndef IN_NEM_DARWIN + Assert(VMMRZCallRing3IsEnabled(pVCpu)); +#endif + + /* + * Update pending interrupts into the APIC's IRR. + */ + if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC)) + APICUpdatePendingInterrupts(pVCpu); + + /* + * Anything pending? Should be more likely than not if we're doing a good job. + */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if ( !fStepping + ? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK) + && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK) + : !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK) + && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) ) + return VINF_SUCCESS; + + /* Pending PGM C3 sync. */ + if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)) + { + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4))); + VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, + VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)); + if (rcStrict != VINF_SUCCESS) + { + AssertRC(VBOXSTRICTRC_VAL(rcStrict)); + Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } + } + + /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */ + if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK) + || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK)) + { + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHmToR3FF); + int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY; + Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d (fVM=%#RX64 fCpu=%#RX64)\n", + rc, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions)); + return rc; + } + + /* Pending VM request packets, such as hardware interrupts. */ + if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST) + || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST)) + { + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchVmReq); + Log4Func(("Pending VM request forcing us back to ring-3\n")); + return VINF_EM_PENDING_REQUEST; + } + + /* Pending PGM pool flushes. */ + if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING)) + { + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchPgmPoolFlush); + Log4Func(("PGM pool flush pending forcing us back to ring-3\n")); + return VINF_PGM_POOL_FLUSH_PENDING; + } + + /* Pending DMA requests. */ + if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA)) + { + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchDma); + Log4Func(("Pending DMA request forcing us back to ring-3\n")); + return VINF_EM_RAW_TO_R3; + } + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /* + * Pending nested-guest events. + * + * Please note the priority of these events are specified and important. + * See Intel spec. 29.4.3.2 "APIC-Write Emulation". + * See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts". + */ + if (fIsNestedGuest) + { + /* Pending nested-guest APIC-write. */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE)) + { + Log4Func(("Pending nested-guest APIC-write\n")); + VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu); + Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE); + return rcStrict; + } + + /* Pending nested-guest monitor-trap flag (MTF). */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF)) + { + Log4Func(("Pending nested-guest MTF\n")); + VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */); + Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE); + return rcStrict; + } + + /* Pending nested-guest VMX-preemption timer expired. */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER)) + { + Log4Func(("Pending nested-guest preempt timer\n")); + VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu); + Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE); + return rcStrict; + } + } +#else + NOREF(fIsNestedGuest); +#endif + + return VINF_SUCCESS; +} + + +/** + * Converts any TRPM trap into a pending HM event. This is typically used when + * entering from ring-3 (not longjmp returns). + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void vmxHCTrpmTrapToPendingEvent(PVMCPUCC pVCpu) +{ + Assert(TRPMHasTrap(pVCpu)); + Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending); + + uint8_t uVector; + TRPMEVENT enmTrpmEvent; + uint32_t uErrCode; + RTGCUINTPTR GCPtrFaultAddress; + uint8_t cbInstr; + bool fIcebp; + + int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp); + AssertRC(rc); + + uint32_t u32IntInfo; + u32IntInfo = uVector | VMX_IDT_VECTORING_INFO_VALID; + u32IntInfo |= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp); + + rc = TRPMResetTrap(pVCpu); + AssertRC(rc); + Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n", + u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress)); + + vmxHCSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress); +} + + +/** + * Converts the pending HM event into a TRPM trap. + * + * @param pVCpu The cross context virtual CPU structure. + */ +static void vmxHCPendingEventToTrpmTrap(PVMCPUCC pVCpu) +{ + Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending); + + /* If a trap was already pending, we did something wrong! */ + Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP); + + uint32_t const u32IntInfo = VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo; + uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo); + TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo); + + Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType)); + + int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType); + AssertRC(rc); + + if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo)) + TRPMSetErrorCode(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode); + + if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo)) + TRPMSetFaultAddress(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress); + else + { + uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo); + switch (uVectorType) + { + case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: + TRPMSetTrapDueToIcebp(pVCpu); + RT_FALL_THRU(); + case VMX_IDT_VECTORING_INFO_TYPE_SW_INT: + case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: + { + AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT + || ( uVector == X86_XCPT_BP /* INT3 */ + || uVector == X86_XCPT_OF /* INTO */ + || uVector == X86_XCPT_DB /* INT1 (ICEBP) */), + ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType)); + TRPMSetInstrLength(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.cbInstr); + break; + } + } + } + + /* We're now done converting the pending event. */ + VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; +} + + +/** + * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to + * cause a VM-exit as soon as the guest is in a state to receive interrupts. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + */ +static void vmxHCSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo) +{ + if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT) + { + if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)) + { + pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT; + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls); + AssertRC(rc); + } + } /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */ +} + + +/** + * Clears the interrupt-window exiting control in the VMCS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + */ +DECLINLINE(void) vmxHCClearIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo) +{ + if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT) + { + pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT; + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls); + AssertRC(rc); + } +} + + +/** + * Sets the NMI-window exiting control in the VMCS which instructs VT-x to + * cause a VM-exit as soon as the guest is in a state to receive NMIs. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + */ +static void vmxHCSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo) +{ + if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT) + { + if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)) + { + pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT; + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls); + AssertRC(rc); + Log4Func(("Setup NMI-window exiting\n")); + } + } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */ +} + + +/** + * Clears the NMI-window exiting control in the VMCS. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + */ +DECLINLINE(void) vmxHCClearNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo) +{ + if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT) + { + pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT; + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls); + AssertRC(rc); + } +} + + +/** + * Injects an event into the guest upon VM-entry by updating the relevant fields + * in the VM-entry area in the VMCS. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @retval VINF_SUCCESS if the event is successfully injected into the VMCS. + * @retval VINF_EM_RESET if event injection resulted in a triple-fault. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info object. + * @param fIsNestedGuest Flag whether this is for a for a pending nested guest event. + * @param pEvent The event being injected. + * @param pfIntrState Pointer to the VT-x guest-interruptibility-state. This + * will be updated if necessary. This cannot not be NULL. + * @param fStepping Whether we're single-stepping guest execution and should + * return VINF_EM_DBG_STEPPED if the event is injected + * directly (registers modified by us, not by hardware on + * VM-entry). + */ +static VBOXSTRICTRC vmxHCInjectEventVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, PCHMEVENT pEvent, + bool fStepping, uint32_t *pfIntrState) +{ + /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */ + AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo)); + Assert(pfIntrState); + +#ifdef IN_NEM_DARWIN + RT_NOREF(fIsNestedGuest, fStepping, pfIntrState); +#endif + + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + uint32_t u32IntInfo = pEvent->u64IntInfo; + uint32_t const u32ErrCode = pEvent->u32ErrCode; + uint32_t const cbInstr = pEvent->cbInstr; + RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress; + uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo); + uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo); + +#ifdef VBOX_STRICT + /* + * Validate the error-code-valid bit for hardware exceptions. + * No error codes for exceptions in real-mode. + * + * See Intel spec. 20.1.4 "Interrupt and Exception Handling" + */ + if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT + && !CPUMIsGuestInRealModeEx(pCtx)) + { + switch (uVector) + { + case X86_XCPT_PF: + case X86_XCPT_DF: + case X86_XCPT_TS: + case X86_XCPT_NP: + case X86_XCPT_SS: + case X86_XCPT_GP: + case X86_XCPT_AC: + AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo), + ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector)); + RT_FALL_THRU(); + default: + break; + } + } + + /* Cannot inject an NMI when block-by-MOV SS is in effect. */ + Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI + || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)); +#endif + + RT_NOREF(uVector); + if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT + || uIntType == VMX_EXIT_INT_INFO_TYPE_NMI + || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT + || uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT) + { + Assert(uVector <= X86_XCPT_LAST); + Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI || uVector == X86_XCPT_NMI); + Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT || uVector == X86_XCPT_DB); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedXcpts[uVector]); + } + else + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedIrqs[uVector & MASK_INJECT_IRQ_STAT]); + + /* + * Hardware interrupts & exceptions cannot be delivered through the software interrupt + * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the + * interrupt handler in the (real-mode) guest. + * + * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode". + * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling. + */ + if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */ + { +#ifndef IN_NEM_DARWIN + if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest) +#endif + { + /* + * For CPUs with unrestricted guest execution enabled and with the guest + * in real-mode, we must not set the deliver-error-code bit. + * + * See Intel spec. 26.2.1.3 "VM-Entry Control Fields". + */ + u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID; + } +#ifndef IN_NEM_DARWIN + else + { + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + Assert(PDMVmmDevHeapIsEnabled(pVM)); + Assert(pVM->hm.s.vmx.pRealModeTSS); + Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)); + + /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */ + int rc2 = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK + | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS); + AssertRCReturn(rc2, rc2); + + /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */ + size_t const cbIdtEntry = sizeof(X86IDTR16); + if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt) + { + /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */ + if (uVector == X86_XCPT_DF) + return VINF_EM_RESET; + + /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. + No error codes for exceptions in real-mode. */ + if (uVector == X86_XCPT_GP) + { + static HMEVENT const s_EventXcptDf + = HMEVENT_INIT_ONLY_INT_INFO( RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1)); + return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptDf, fStepping, pfIntrState); + } + + /* + * If we're injecting an event with no valid IDT entry, inject a #GP. + * No error codes for exceptions in real-mode. + * + * See Intel spec. 20.1.4 "Interrupt and Exception Handling" + */ + static HMEVENT const s_EventXcptGp + = HMEVENT_INIT_ONLY_INT_INFO( RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0) + | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1)); + return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptGp, fStepping, pfIntrState); + } + + /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */ + uint16_t uGuestIp = pCtx->ip; + if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT) + { + Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF); + /* #BP and #OF are both benign traps, we need to resume the next instruction. */ + uGuestIp = pCtx->ip + (uint16_t)cbInstr; + } + else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT) + uGuestIp = pCtx->ip + (uint16_t)cbInstr; + + /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */ + X86IDTR16 IdtEntry; + RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry; + rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry); + AssertRCReturn(rc2, rc2); + + /* Construct the stack frame for the interrupt/exception handler. */ + VBOXSTRICTRC rcStrict; + rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, (uint16_t)pCtx->eflags.u); + if (rcStrict == VINF_SUCCESS) + { + rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel); + if (rcStrict == VINF_SUCCESS) + rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp); + } + + /* Clear the required eflag bits and jump to the interrupt/exception handler. */ + if (rcStrict == VINF_SUCCESS) + { + pCtx->eflags.u &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC); + pCtx->rip = IdtEntry.offSel; + pCtx->cs.Sel = IdtEntry.uSel; + pCtx->cs.ValidSel = IdtEntry.uSel; + pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry; + if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT + && uVector == X86_XCPT_PF) + pCtx->cr2 = GCPtrFault; + + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2 + | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS + | HM_CHANGED_GUEST_RSP); + + /* + * If we delivered a hardware exception (other than an NMI) and if there was + * block-by-STI in effect, we should clear it. + */ + if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI) + { + Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI + && uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT); + Log4Func(("Clearing inhibition due to STI\n")); + *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI; + } + + Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n", + u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip)); + + /* + * The event has been truly dispatched to the guest. Mark it as no longer pending so + * we don't attempt to undo it if we are returning to ring-3 before executing guest code. + */ + VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; + + /* + * If we eventually support nested-guest execution without unrestricted guest execution, + * we should set fInterceptEvents here. + */ + Assert(!fIsNestedGuest); + + /* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */ + if (fStepping) + rcStrict = VINF_EM_DBG_STEPPED; + } + AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping), + ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + } +#else + RT_NOREF(pVmcsInfo); +#endif + } + + /* + * Validate. + */ + Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */ + Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */ + + /* + * Inject the event into the VMCS. + */ + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo); + if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo)) + rc |= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode); + rc |= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr); + AssertRC(rc); + + /* + * Update guest CR2 if this is a page-fault. + */ + if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo)) + pCtx->cr2 = GCPtrFault; + + Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2)); + return VINF_SUCCESS; +} + + +/** + * Evaluates the event to be delivered to the guest and sets it as the pending + * event. + * + * Toggling of interrupt force-flags here is safe since we update TRPM on premature + * exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must + * NOT restore these force-flags. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS information structure. + * @param fIsNestedGuest Flag whether the evaluation happens for a nested guest. + * @param pfIntrState Where to store the VT-x guest-interruptibility state. + */ +static VBOXSTRICTRC vmxHCEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t *pfIntrState) +{ + Assert(pfIntrState); + Assert(!TRPMHasTrap(pVCpu)); + + /* + * Compute/update guest-interruptibility state related FFs. + * The FFs will be used below while evaluating events to be injected. + */ + *pfIntrState = vmxHCGetGuestIntrStateAndUpdateFFs(pVCpu); + + /* + * Evaluate if a new event needs to be injected. + * An event that's already pending has already performed all necessary checks. + */ + if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending + && !CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx)) + { + /** @todo SMI. SMIs take priority over NMIs. */ + + /* + * NMIs. + * NMIs take priority over external interrupts. + */ +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx; +#endif + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) + { + /* + * For a guest, the FF always indicates the guest's ability to receive an NMI. + * + * For a nested-guest, the FF always indicates the outer guest's ability to + * receive an NMI while the guest-interruptibility state bit depends on whether + * the nested-hypervisor is using virtual-NMIs. + */ + if (!CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx)) + { +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( fIsNestedGuest + && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT)) + return IEMExecVmxVmexitXcptNmi(pVCpu); +#endif + vmxHCSetPendingXcptNmi(pVCpu); + VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI); + Log4Func(("NMI pending injection\n")); + + /* We've injected the NMI, bail. */ + return VINF_SUCCESS; + } + if (!fIsNestedGuest) + vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo); + } + + /* + * External interrupts (PIC/APIC). + * Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it. + * We cannot re-request the interrupt from the controller again. + */ + if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC) + && !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction) + { + Assert(!DBGFIsStepping(pVCpu)); + int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS); + AssertRC(rc); + + /* + * We must not check EFLAGS directly when executing a nested-guest, use + * CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of + * external interrupts when "External interrupt exiting" is set. This fixes a nasty + * SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by + * other VM-exits (like a preemption timer), see @bugref{9562#c18}. + * + * See Intel spec. 25.4.1 "Event Blocking". + */ + if (CPUMIsGuestPhysIntrEnabled(pVCpu)) + { +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( fIsNestedGuest + && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)) + { + VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector */, true /* fIntPending */); + if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE) + return rcStrict; + } +#endif + uint8_t u8Interrupt; + rc = PDMGetInterrupt(pVCpu, &u8Interrupt); + if (RT_SUCCESS(rc)) + { +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( fIsNestedGuest + && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)) + { + VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */); + Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE); + return rcStrict; + } +#endif + vmxHCSetPendingExtInt(pVCpu, u8Interrupt); + Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt)); + } + else if (rc == VERR_APIC_INTR_MASKED_BY_TPR) + { + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchTprMaskedIrq); + + if ( !fIsNestedGuest + && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)) + vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4); + /* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */ + + /* + * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and + * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no + * need to re-set this force-flag here. + */ + } + else + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchGuestIrq); + + /* We've injected the interrupt or taken necessary action, bail. */ + return VINF_SUCCESS; + } + if (!fIsNestedGuest) + vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo); + } + } + else if (!fIsNestedGuest) + { + /* + * An event is being injected or we are in an interrupt shadow. Check if another event is + * pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept + * the pending event. + */ + if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) + vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo); + else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC) + && !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction) + vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo); + } + /* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */ + + return VINF_SUCCESS; +} + + +/** + * Injects any pending events into the guest if the guest is in a state to + * receive them. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS information structure. + * @param fIsNestedGuest Flag whether the event injection happens for a nested guest. + * @param fIntrState The VT-x guest-interruptibility state. + * @param fStepping Whether we are single-stepping the guest using the + * hypervisor debugger and should return + * VINF_EM_DBG_STEPPED if the event was dispatched + * directly. + */ +static VBOXSTRICTRC vmxHCInjectPendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, + uint32_t fIntrState, bool fStepping) +{ + HMVMX_ASSERT_PREEMPT_SAFE(pVCpu); +#ifndef IN_NEM_DARWIN + Assert(VMMRZCallRing3IsEnabled(pVCpu)); +#endif + +#ifdef VBOX_STRICT + /* + * Verify guest-interruptibility state. + * + * We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS, + * since injecting an event may modify the interruptibility state and we must thus always + * use fIntrState. + */ + { + bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS); + bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI); + Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS)); + Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */ + Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/ + Assert(!TRPMHasTrap(pVCpu)); + NOREF(fBlockMovSS); NOREF(fBlockSti); + } +#endif + + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + if (VCPU_2_VMXSTATE(pVCpu).Event.fPending) + { + /* + * Do -not- clear any interrupt-window exiting control here. We might have an interrupt + * pending even while injecting an event and in this case, we want a VM-exit as soon as + * the guest is ready for the next interrupt, see @bugref{6208#c45}. + * + * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery". + */ + uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo); +#ifdef VBOX_STRICT + if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT) + { + Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF); + Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)); + Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)); + } + else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI) + { + Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)); + Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)); + Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)); + } +#endif + Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo, + uIntType)); + + /* + * Inject the event and get any changes to the guest-interruptibility state. + * + * The guest-interruptibility state may need to be updated if we inject the event + * into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts). + */ + rcStrict = vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &VCPU_2_VMXSTATE(pVCpu).Event, fStepping, &fIntrState); + AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict); + + if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT) + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterrupt); + else + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectXcpt); + } + + /* + * Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and + * is an interrupt shadow (block-by-STI or block-by-MOV SS). + */ + if ( (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)) + && !fIsNestedGuest) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); + + if (!VCPU_2_VMXSTATE(pVCpu).fSingleInstruction) + { + /* + * Set or clear the BS bit depending on whether the trap flag is active or not. We need + * to do both since we clear the BS bit from the VMCS while exiting to ring-3. + */ + Assert(!DBGFIsStepping(pVCpu)); + uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_TF); + int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, + fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT); + AssertRC(rc); + } + else + { + /* + * We must not deliver a debug exception when single-stepping over STI/Mov-SS in the + * hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However, + * we take care of this case in vmxHCExportSharedDebugState and also the case if + * we use MTF, so just make sure it's called before executing guest-code. + */ + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR_MASK); + } + } + /* else: for nested-guest currently handling while merging controls. */ + + /* + * Finally, update the guest-interruptibility state. + * + * This is required for the real-on-v86 software interrupt injection, for + * pending debug exceptions as well as updates to the guest state from ring-3 (IEM). + */ + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState); + AssertRC(rc); + + /* + * There's no need to clear the VM-entry interruption-information field here if we're not + * injecting anything. VT-x clears the valid bit on every VM-exit. + * + * See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection". + */ + + Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping)); + return rcStrict; +} + + +/** + * Tries to determine what part of the guest-state VT-x has deemed as invalid + * and update error record fields accordingly. + * + * @returns VMX_IGS_* error codes. + * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything + * wrong with the guest state. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmcsInfo The VMCS info. object. + * + * @remarks This function assumes our cache of the VMCS controls + * are valid, i.e. vmxHCCheckCachedVmcsCtls() succeeded. + */ +static uint32_t vmxHCCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo) +{ +#define HMVMX_ERROR_BREAK(err) { uError = (err); break; } +#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { uError = (err); break; } else do { } while (0) + + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + uint32_t uError = VMX_IGS_ERROR; + uint32_t u32IntrState = 0; +#ifndef IN_NEM_DARWIN + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + bool const fUnrestrictedGuest = VM_IS_VMX_UNRESTRICTED_GUEST(pVM); +#else + bool const fUnrestrictedGuest = true; +#endif + do + { + int rc; + + /* + * Guest-interruptibility state. + * + * Read this first so that any check that fails prior to those that actually + * require the guest-interruptibility state would still reflect the correct + * VMCS value and avoids causing further confusion. + */ + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32IntrState); + AssertRC(rc); + + uint32_t u32Val; + uint64_t u64Val; + + /* + * CR0. + */ + /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */ + uint64_t fSetCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 & g_HmMsrs.u.vmx.u64Cr0Fixed1); + uint64_t const fZapCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 | g_HmMsrs.u.vmx.u64Cr0Fixed1); + /* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG). + See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */ + if (fUnrestrictedGuest) + fSetCr0 &= ~(uint64_t)(X86_CR0_PE | X86_CR0_PG); + + uint64_t u64GuestCr0; + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64GuestCr0); + AssertRC(rc); + HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1); + HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0); + if ( !fUnrestrictedGuest + && (u64GuestCr0 & X86_CR0_PG) + && !(u64GuestCr0 & X86_CR0_PE)) + HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO); + + /* + * CR4. + */ + /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */ + uint64_t const fSetCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 & g_HmMsrs.u.vmx.u64Cr4Fixed1); + uint64_t const fZapCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 | g_HmMsrs.u.vmx.u64Cr4Fixed1); + + uint64_t u64GuestCr4; + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64GuestCr4); + AssertRC(rc); + HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1); + HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0); + + /* + * IA32_DEBUGCTL MSR. + */ + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val); + AssertRC(rc); + if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG) + && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */ + { + HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED); + } + uint64_t u64DebugCtlMsr = u64Val; + +#ifdef VBOX_STRICT + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val); + AssertRC(rc); + Assert(u32Val == pVmcsInfo->u32EntryCtls); +#endif + bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST); + + /* + * RIP and RFLAGS. + */ + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val); + AssertRC(rc); + /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */ + if ( !fLongModeGuest + || !pCtx->cs.Attr.n.u1Long) + { + HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID); + } + /** @todo If the processor supports N < 64 linear-address bits, bits 63:N + * must be identical if the "IA-32e mode guest" VM-entry + * control is 1 and CS.L is 1. No check applies if the + * CPU supports 64 linear-address bits. */ + + /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */ + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */ + VMX_IGS_RFLAGS_RESERVED); + HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */ + uint32_t const u32Eflags = u64Val; + + if ( fLongModeGuest + || ( fUnrestrictedGuest + && !(u64GuestCr0 & X86_CR0_PE))) + { + HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID); + } + + uint32_t u32EntryInfo; + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo); + AssertRC(rc); + if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo)) + { + HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID); + } + + /* + * 64-bit checks. + */ + if (fLongModeGuest) + { + HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE); + HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE); + } + + if ( !fLongModeGuest + && (u64GuestCr4 & X86_CR4_PCIDE)) + HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE); + + /** @todo CR3 field must be such that bits 63:52 and bits in the range + * 51:32 beyond the processor's physical-address width are 0. */ + + if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG) + && (pCtx->dr[7] & X86_DR7_MBZ_MASK)) + HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED); + +#ifndef IN_NEM_DARWIN + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_ESP, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL); + + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_EIP, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL); +#endif + + /* + * PERF_GLOBAL MSR. + */ + if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR) + { + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)), + VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */ + } + + /* + * PAT MSR. + */ + if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR) + { + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PAT_FULL, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED); + for (unsigned i = 0; i < 8; i++) + { + uint8_t u8Val = (u64Val & 0xff); + if ( u8Val != 0 /* UC */ + && u8Val != 1 /* WC */ + && u8Val != 4 /* WT */ + && u8Val != 5 /* WP */ + && u8Val != 6 /* WB */ + && u8Val != 7 /* UC- */) + HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID); + u64Val >>= 8; + } + } + + /* + * EFER MSR. + */ + if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR) + { + Assert(g_fHmVmxSupportsVmcsEfer); + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)), + VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */ + HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls + & VMX_ENTRY_CTLS_IA32E_MODE_GUEST), + VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH); + /** @todo r=ramshankar: Unrestricted check here is probably wrong, see + * iemVmxVmentryCheckGuestState(). */ + HMVMX_CHECK_BREAK( fUnrestrictedGuest + || !(u64GuestCr0 & X86_CR0_PG) + || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME), + VMX_IGS_EFER_LMA_LME_MISMATCH); + } + + /* + * Segment registers. + */ + HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE) + || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID); + if (!(u32Eflags & X86_EFL_VM)) + { + /* CS */ + HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID); + HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED); + HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED); + HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff + || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000) + || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID); + /* CS cannot be loaded with NULL in protected mode. */ + HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE); + HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID); + if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11) + HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL); + else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15) + HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH); + else if (fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3) + HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID); + else + HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID); + + /* SS */ + HMVMX_CHECK_BREAK( fUnrestrictedGuest + || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL); + HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL); + if ( !(pCtx->cr0 & X86_CR0_PE) + || pCtx->cs.Attr.n.u4Type == 3) + { + HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID); + } + + if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE)) + { + HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID); + HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID); + HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED); + HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED); + HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff + || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000) + || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID); + } + + /* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSReg(). */ + if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE)) + { + HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID); + HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID); + HMVMX_CHECK_BREAK( fUnrestrictedGuest + || pCtx->ds.Attr.n.u4Type > 11 + || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL); + HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED); + HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED); + HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff + || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000) + || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE) + || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID); + } + if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE)) + { + HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID); + HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID); + HMVMX_CHECK_BREAK( fUnrestrictedGuest + || pCtx->es.Attr.n.u4Type > 11 + || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL); + HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED); + HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED); + HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff + || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000) + || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE) + || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID); + } + if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE)) + { + HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID); + HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID); + HMVMX_CHECK_BREAK( fUnrestrictedGuest + || pCtx->fs.Attr.n.u4Type > 11 + || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL); + HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED); + HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED); + HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff + || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000) + || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE) + || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID); + } + if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE)) + { + HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID); + HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID); + HMVMX_CHECK_BREAK( fUnrestrictedGuest + || pCtx->gs.Attr.n.u4Type > 11 + || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL); + HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED); + HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED); + HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff + || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000) + || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE) + || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID); + } + /* 64-bit capable CPUs. */ + HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL); + HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL); + HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE) + || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL); + HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID); + HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base), + VMX_IGS_LONGMODE_SS_BASE_INVALID); + HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base), + VMX_IGS_LONGMODE_DS_BASE_INVALID); + HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base), + VMX_IGS_LONGMODE_ES_BASE_INVALID); + } + else + { + /* V86 mode checks. */ + uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr; + if (pVmcsInfo->pShared->RealMode.fRealOnV86Active) + { + u32CSAttr = 0xf3; u32SSAttr = 0xf3; + u32DSAttr = 0xf3; u32ESAttr = 0xf3; + u32FSAttr = 0xf3; u32GSAttr = 0xf3; + } + else + { + u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; + u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u; + u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u; + } + + /* CS */ + HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID); + HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID); + HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID); + /* SS */ + HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID); + HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID); + HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID); + /* DS */ + HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID); + HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID); + HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID); + /* ES */ + HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID); + HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID); + HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID); + /* FS */ + HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID); + HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID); + HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID); + /* GS */ + HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID); + HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID); + HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID); + /* 64-bit capable CPUs. */ + HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL); + HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL); + HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE) + || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL); + HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID); + HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base), + VMX_IGS_LONGMODE_SS_BASE_INVALID); + HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base), + VMX_IGS_LONGMODE_DS_BASE_INVALID); + HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base), + VMX_IGS_LONGMODE_ES_BASE_INVALID); + } + + /* + * TR. + */ + HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID); + /* 64-bit capable CPUs. */ + HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL); + if (fLongModeGuest) + HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */ + VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID); + else + HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */ + || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/ + VMX_IGS_TR_ATTR_TYPE_INVALID); + HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID); + HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID); + HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */ + HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff + || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000) + || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID); + HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE); + + /* + * GDTR and IDTR (64-bit capable checks). + */ + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL); + + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL); + + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */ + + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */ + + /* + * Guest Non-Register State. + */ + /* Activity State. */ + uint32_t u32ActivityState; + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState); + AssertRC(rc); + HMVMX_CHECK_BREAK( !u32ActivityState + || (u32ActivityState & RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)), + VMX_IGS_ACTIVITY_STATE_INVALID); + HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl) + || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID); + + if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS + || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI) + { + HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID); + } + + /** @todo Activity state and injecting interrupts. Left as a todo since we + * currently don't use activity states but ACTIVE. */ + + HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM) + || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID); + + /* Guest interruptibility-state. */ + HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED); + HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)) + != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS), + VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID); + HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF) + || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI), + VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID); + if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo)) + { + HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI) + && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS), + VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID); + } + else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo)) + { + HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS), + VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID); + HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI), + VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID); + } + /** @todo Assumes the processor is not in SMM. */ + HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI), + VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID); + HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM) + || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI), + VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID); + if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI) + && VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo)) + { + HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID); + } + + /* Pending debug exceptions. */ + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val); + AssertRC(rc); + /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */ + HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED); + u32Val = u64Val; /* For pending debug exceptions checks below. */ + + if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI) + || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS) + || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT) + { + if ( (u32Eflags & X86_EFL_TF) + && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */ + { + /* Bit 14 is PendingDebug.BS. */ + HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET); + } + if ( !(u32Eflags & X86_EFL_TF) + || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */ + { + /* Bit 14 is PendingDebug.BS. */ + HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR); + } + } + +#ifndef IN_NEM_DARWIN + /* VMCS link pointer. */ + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val); + AssertRC(rc); + if (u64Val != UINT64_C(0xffffffffffffffff)) + { + HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED); + /** @todo Bits beyond the processor's physical-address width MBZ. */ + /** @todo SMM checks. */ + Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val); + Assert(pVmcsInfo->pvShadowVmcs); + VMXVMCSREVID VmcsRevId; + VmcsRevId.u = *(uint32_t *)pVmcsInfo->pvShadowVmcs; + HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID), + VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID); + HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING), + VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW); + } + + /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is + * not using nested paging? */ + if ( VM_IS_VMX_NESTED_PAGING(pVM) + && !fLongModeGuest + && CPUMIsGuestInPAEModeEx(pCtx)) + { + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED); + + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED); + + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED); + + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val); + AssertRC(rc); + HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED); + } +#endif + + /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */ + if (uError == VMX_IGS_ERROR) + uError = VMX_IGS_REASON_NOT_FOUND; + } while (0); + + VCPU_2_VMXSTATE(pVCpu).u32HMError = uError; + VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32GuestIntrState = u32IntrState; + return uError; + +#undef HMVMX_ERROR_BREAK +#undef HMVMX_CHECK_BREAK +} + + +#ifndef HMVMX_USE_FUNCTION_TABLE +/** + * Handles a guest VM-exit from hardware-assisted VMX execution. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + */ +DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ +#ifdef DEBUG_ramshankar +# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \ + do { \ + if (a_fSave != 0) \ + vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); \ + VBOXSTRICTRC rcStrict = a_CallExpr; \ + if (a_fSave != 0) \ + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); \ + return rcStrict; \ + } while (0) +#else +# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr +#endif + uint32_t const uExitReason = pVmxTransient->uExitReason; + switch (uExitReason) + { + case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, vmxHCExitEptMisconfig(pVCpu, pVmxTransient)); + case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, vmxHCExitEptViolation(pVCpu, pVmxTransient)); + case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, vmxHCExitIoInstr(pVCpu, pVmxTransient)); + case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, vmxHCExitCpuid(pVCpu, pVmxTransient)); + case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, vmxHCExitRdtsc(pVCpu, pVmxTransient)); + case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, vmxHCExitRdtscp(pVCpu, pVmxTransient)); + case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, vmxHCExitApicAccess(pVCpu, pVmxTransient)); + case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, vmxHCExitXcptOrNmi(pVCpu, pVmxTransient)); + case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, vmxHCExitMovCRx(pVCpu, pVmxTransient)); + case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, vmxHCExitExtInt(pVCpu, pVmxTransient)); + case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, vmxHCExitIntWindow(pVCpu, pVmxTransient)); + case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient)); + case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, vmxHCExitMwait(pVCpu, pVmxTransient)); + case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, vmxHCExitMonitor(pVCpu, pVmxTransient)); + case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, vmxHCExitTaskSwitch(pVCpu, pVmxTransient)); + case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, vmxHCExitPreemptTimer(pVCpu, pVmxTransient)); + case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, vmxHCExitRdmsr(pVCpu, pVmxTransient)); + case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, vmxHCExitWrmsr(pVCpu, pVmxTransient)); + case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, vmxHCExitVmcall(pVCpu, pVmxTransient)); + case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, vmxHCExitMovDRx(pVCpu, pVmxTransient)); + case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, vmxHCExitHlt(pVCpu, pVmxTransient)); + case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, vmxHCExitInvd(pVCpu, pVmxTransient)); + case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, vmxHCExitInvlpg(pVCpu, pVmxTransient)); + case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, vmxHCExitMtf(pVCpu, pVmxTransient)); + case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, vmxHCExitPause(pVCpu, pVmxTransient)); + case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, vmxHCExitWbinvd(pVCpu, pVmxTransient)); + case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, vmxHCExitXsetbv(pVCpu, pVmxTransient)); + case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, vmxHCExitInvpcid(pVCpu, pVmxTransient)); + case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, vmxHCExitGetsec(pVCpu, pVmxTransient)); + case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, vmxHCExitRdpmc(pVCpu, pVmxTransient)); +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, vmxHCExitVmclear(pVCpu, pVmxTransient)); + case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, vmxHCExitVmlaunch(pVCpu, pVmxTransient)); + case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, vmxHCExitVmptrld(pVCpu, pVmxTransient)); + case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, vmxHCExitVmptrst(pVCpu, pVmxTransient)); + case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, vmxHCExitVmread(pVCpu, pVmxTransient)); + case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, vmxHCExitVmwrite(pVCpu, pVmxTransient)); + case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, vmxHCExitVmresume(pVCpu, pVmxTransient)); + case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, vmxHCExitVmxoff(pVCpu, pVmxTransient)); + case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, vmxHCExitVmxon(pVCpu, pVmxTransient)); + case VMX_EXIT_INVVPID: VMEXIT_CALL_RET(0, vmxHCExitInvvpid(pVCpu, pVmxTransient)); +#else + case VMX_EXIT_VMCLEAR: + case VMX_EXIT_VMLAUNCH: + case VMX_EXIT_VMPTRLD: + case VMX_EXIT_VMPTRST: + case VMX_EXIT_VMREAD: + case VMX_EXIT_VMRESUME: + case VMX_EXIT_VMWRITE: + case VMX_EXIT_VMXOFF: + case VMX_EXIT_VMXON: + case VMX_EXIT_INVVPID: + return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient); +#endif +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case VMX_EXIT_INVEPT: VMEXIT_CALL_RET(0, vmxHCExitInvept(pVCpu, pVmxTransient)); +#else + case VMX_EXIT_INVEPT: return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient); +#endif + + case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFault(pVCpu, pVmxTransient); + case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindow(pVCpu, pVmxTransient); + case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient); + + case VMX_EXIT_INIT_SIGNAL: + case VMX_EXIT_SIPI: + case VMX_EXIT_IO_SMI: + case VMX_EXIT_SMI: + case VMX_EXIT_ERR_MSR_LOAD: + case VMX_EXIT_ERR_MACHINE_CHECK: + case VMX_EXIT_PML_FULL: + case VMX_EXIT_VIRTUALIZED_EOI: + case VMX_EXIT_GDTR_IDTR_ACCESS: + case VMX_EXIT_LDTR_TR_ACCESS: + case VMX_EXIT_APIC_WRITE: + case VMX_EXIT_RDRAND: + case VMX_EXIT_RSM: + case VMX_EXIT_VMFUNC: + case VMX_EXIT_ENCLS: + case VMX_EXIT_RDSEED: + case VMX_EXIT_XSAVES: + case VMX_EXIT_XRSTORS: + case VMX_EXIT_UMWAIT: + case VMX_EXIT_TPAUSE: + case VMX_EXIT_LOADIWKEY: + default: + return vmxHCExitErrUnexpected(pVCpu, pVmxTransient); + } +#undef VMEXIT_CALL_RET +} +#endif /* !HMVMX_USE_FUNCTION_TABLE */ + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** + * Handles a nested-guest VM-exit from hardware-assisted VMX execution. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + */ +DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + uint32_t const uExitReason = pVmxTransient->uExitReason; + switch (uExitReason) + { +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + case VMX_EXIT_EPT_MISCONFIG: return vmxHCExitEptMisconfigNested(pVCpu, pVmxTransient); + case VMX_EXIT_EPT_VIOLATION: return vmxHCExitEptViolationNested(pVCpu, pVmxTransient); +# else + case VMX_EXIT_EPT_MISCONFIG: return vmxHCExitEptMisconfig(pVCpu, pVmxTransient); + case VMX_EXIT_EPT_VIOLATION: return vmxHCExitEptViolation(pVCpu, pVmxTransient); +# endif + case VMX_EXIT_XCPT_OR_NMI: return vmxHCExitXcptOrNmiNested(pVCpu, pVmxTransient); + case VMX_EXIT_IO_INSTR: return vmxHCExitIoInstrNested(pVCpu, pVmxTransient); + case VMX_EXIT_HLT: return vmxHCExitHltNested(pVCpu, pVmxTransient); + + /* + * We shouldn't direct host physical interrupts to the nested-guest. + */ + case VMX_EXIT_EXT_INT: + return vmxHCExitExtInt(pVCpu, pVmxTransient); + + /* + * Instructions that cause VM-exits unconditionally or the condition is + * always taken solely from the nested hypervisor (meaning if the VM-exit + * happens, it's guaranteed to be a nested-guest VM-exit). + * + * - Provides VM-exit instruction length ONLY. + */ + case VMX_EXIT_CPUID: /* Unconditional. */ + case VMX_EXIT_VMCALL: + case VMX_EXIT_GETSEC: + case VMX_EXIT_INVD: + case VMX_EXIT_XSETBV: + case VMX_EXIT_VMLAUNCH: + case VMX_EXIT_VMRESUME: + case VMX_EXIT_VMXOFF: + case VMX_EXIT_ENCLS: /* Condition specified solely by nested hypervisor. */ + case VMX_EXIT_VMFUNC: + return vmxHCExitInstrNested(pVCpu, pVmxTransient); + + /* + * Instructions that cause VM-exits unconditionally or the condition is + * always taken solely from the nested hypervisor (meaning if the VM-exit + * happens, it's guaranteed to be a nested-guest VM-exit). + * + * - Provides VM-exit instruction length. + * - Provides VM-exit information. + * - Optionally provides Exit qualification. + * + * Since Exit qualification is 0 for all VM-exits where it is not + * applicable, reading and passing it to the guest should produce + * defined behavior. + * + * See Intel spec. 27.2.1 "Basic VM-Exit Information". + */ + case VMX_EXIT_INVEPT: /* Unconditional. */ + case VMX_EXIT_INVVPID: + case VMX_EXIT_VMCLEAR: + case VMX_EXIT_VMPTRLD: + case VMX_EXIT_VMPTRST: + case VMX_EXIT_VMXON: + case VMX_EXIT_GDTR_IDTR_ACCESS: /* Condition specified solely by nested hypervisor. */ + case VMX_EXIT_LDTR_TR_ACCESS: + case VMX_EXIT_RDRAND: + case VMX_EXIT_RDSEED: + case VMX_EXIT_XSAVES: + case VMX_EXIT_XRSTORS: + case VMX_EXIT_UMWAIT: + case VMX_EXIT_TPAUSE: + return vmxHCExitInstrWithInfoNested(pVCpu, pVmxTransient); + + case VMX_EXIT_RDTSC: return vmxHCExitRdtscNested(pVCpu, pVmxTransient); + case VMX_EXIT_RDTSCP: return vmxHCExitRdtscpNested(pVCpu, pVmxTransient); + case VMX_EXIT_RDMSR: return vmxHCExitRdmsrNested(pVCpu, pVmxTransient); + case VMX_EXIT_WRMSR: return vmxHCExitWrmsrNested(pVCpu, pVmxTransient); + case VMX_EXIT_INVLPG: return vmxHCExitInvlpgNested(pVCpu, pVmxTransient); + case VMX_EXIT_INVPCID: return vmxHCExitInvpcidNested(pVCpu, pVmxTransient); + case VMX_EXIT_TASK_SWITCH: return vmxHCExitTaskSwitchNested(pVCpu, pVmxTransient); + case VMX_EXIT_WBINVD: return vmxHCExitWbinvdNested(pVCpu, pVmxTransient); + case VMX_EXIT_MTF: return vmxHCExitMtfNested(pVCpu, pVmxTransient); + case VMX_EXIT_APIC_ACCESS: return vmxHCExitApicAccessNested(pVCpu, pVmxTransient); + case VMX_EXIT_APIC_WRITE: return vmxHCExitApicWriteNested(pVCpu, pVmxTransient); + case VMX_EXIT_VIRTUALIZED_EOI: return vmxHCExitVirtEoiNested(pVCpu, pVmxTransient); + case VMX_EXIT_MOV_CRX: return vmxHCExitMovCRxNested(pVCpu, pVmxTransient); + case VMX_EXIT_INT_WINDOW: return vmxHCExitIntWindowNested(pVCpu, pVmxTransient); + case VMX_EXIT_NMI_WINDOW: return vmxHCExitNmiWindowNested(pVCpu, pVmxTransient); + case VMX_EXIT_TPR_BELOW_THRESHOLD: return vmxHCExitTprBelowThresholdNested(pVCpu, pVmxTransient); + case VMX_EXIT_MWAIT: return vmxHCExitMwaitNested(pVCpu, pVmxTransient); + case VMX_EXIT_MONITOR: return vmxHCExitMonitorNested(pVCpu, pVmxTransient); + case VMX_EXIT_PAUSE: return vmxHCExitPauseNested(pVCpu, pVmxTransient); + + case VMX_EXIT_PREEMPT_TIMER: + { + /** @todo NSTVMX: Preempt timer. */ + return vmxHCExitPreemptTimer(pVCpu, pVmxTransient); + } + + case VMX_EXIT_MOV_DRX: return vmxHCExitMovDRxNested(pVCpu, pVmxTransient); + case VMX_EXIT_RDPMC: return vmxHCExitRdpmcNested(pVCpu, pVmxTransient); + + case VMX_EXIT_VMREAD: + case VMX_EXIT_VMWRITE: return vmxHCExitVmreadVmwriteNested(pVCpu, pVmxTransient); + + case VMX_EXIT_TRIPLE_FAULT: return vmxHCExitTripleFaultNested(pVCpu, pVmxTransient); + case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestStateNested(pVCpu, pVmxTransient); + + case VMX_EXIT_INIT_SIGNAL: + case VMX_EXIT_SIPI: + case VMX_EXIT_IO_SMI: + case VMX_EXIT_SMI: + case VMX_EXIT_ERR_MSR_LOAD: + case VMX_EXIT_ERR_MACHINE_CHECK: + case VMX_EXIT_PML_FULL: + case VMX_EXIT_RSM: + default: + return vmxHCExitErrUnexpected(pVCpu, pVmxTransient); + } +} +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + + +/** @name VM-exit helpers. + * @{ + */ +/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */ +/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */ +/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */ + +/** Macro for VM-exits called unexpectedly. */ +#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \ + do { \ + VCPU_2_VMXSTATE((a_pVCpu)).u32HMError = (a_HmError); \ + return VERR_VMX_UNEXPECTED_EXIT; \ + } while (0) + +#ifdef VBOX_STRICT +# ifndef IN_NEM_DARWIN +/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */ +# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \ + RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId() + +# define HMVMX_ASSERT_PREEMPT_CPUID() \ + do { \ + RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \ + AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \ + } while (0) + +# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \ + do { \ + AssertPtr((a_pVCpu)); \ + AssertPtr((a_pVmxTransient)); \ + Assert( (a_pVmxTransient)->fVMEntryFailed == false \ + || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \ + || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \ + || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \ + Assert((a_pVmxTransient)->pVmcsInfo); \ + Assert(ASMIntAreEnabled()); \ + HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \ + HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \ + Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \ + HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \ + if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \ + HMVMX_ASSERT_PREEMPT_CPUID(); \ + HMVMX_STOP_EXIT_DISPATCH_PROF(); \ + } while (0) +# else +# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() do { } while(0) +# define HMVMX_ASSERT_PREEMPT_CPUID() do { } while(0) +# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \ + do { \ + AssertPtr((a_pVCpu)); \ + AssertPtr((a_pVmxTransient)); \ + Assert( (a_pVmxTransient)->fVMEntryFailed == false \ + || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \ + || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \ + || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \ + Assert((a_pVmxTransient)->pVmcsInfo); \ + Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \ + HMVMX_STOP_EXIT_DISPATCH_PROF(); \ + } while (0) +# endif + +# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \ + do { \ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \ + Assert((a_pVmxTransient)->fIsNestedGuest); \ + } while (0) + +# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \ + do { \ + Log4Func(("\n")); \ + } while (0) +#else +# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \ + do { \ + HMVMX_STOP_EXIT_DISPATCH_PROF(); \ + NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \ + } while (0) + +# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \ + do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0) + +# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0) +#endif + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** Macro that does the necessary privilege checks and intercepted VM-exits for + * guests that attempted to execute a VMX instruction. */ +# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \ + do \ + { \ + VBOXSTRICTRC rcStrictTmp = vmxHCCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \ + if (rcStrictTmp == VINF_SUCCESS) \ + { /* likely */ } \ + else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \ + { \ + Assert((a_pVCpu)->hm.s.Event.fPending); \ + Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \ + return VINF_SUCCESS; \ + } \ + else \ + { \ + int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \ + AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \ + } \ + } while (0) + +/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */ +# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \ + do \ + { \ + VBOXSTRICTRC rcStrictTmp = vmxHCDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \ + (a_pGCPtrEffAddr)); \ + if (rcStrictTmp == VINF_SUCCESS) \ + { /* likely */ } \ + else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \ + { \ + uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \ + Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \ + NOREF(uXcptTmp); \ + return VINF_SUCCESS; \ + } \ + else \ + { \ + Log4Func(("vmxHCDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \ + return rcStrictTmp; \ + } \ + } while (0) +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + + +/** + * Advances the guest RIP by the specified number of bytes. + * + * @param pVCpu The cross context virtual CPU structure. + * @param cbInstr Number of bytes to advance the RIP by. + * + * @remarks No-long-jump zone!!! + */ +DECLINLINE(void) vmxHCAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr) +{ + CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI); + + /* + * Advance RIP. + * + * The upper 32 bits are only set when in 64-bit mode, so we have to detect + * when the addition causes a "carry" into the upper half and check whether + * we're in 64-bit and can go on with it or wether we should zap the top + * half. (Note! The 8086, 80186 and 80286 emulation is done exclusively in + * IEM, so we don't need to bother with pre-386 16-bit wraparound.) + * + * See PC wrap around tests in bs3-cpu-weird-1. + */ + uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip; + uint64_t const uRipNext = uRipPrev + cbInstr; + if (RT_LIKELY( !((uRipNext ^ uRipPrev) & RT_BIT_64(32)) + || CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))) + pVCpu->cpum.GstCtx.rip = uRipNext; + else + pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext; + + /* + * Clear RF and interrupt shadowing. + */ + if (RT_LIKELY(!(pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_RF | X86_EFL_TF)))) + pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_INHIBIT_SHADOW; + else + { + if ((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_RF | X86_EFL_TF)) == X86_EFL_TF) + { + /** @todo \#DB - single step. */ + } + pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW); + } + AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX); + + /* Mark both RIP and RFLAGS as updated. */ + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); +} + + +/** + * Advances the guest RIP after reading it from the VMCS. + * + * @returns VBox status code, no informational status codes. + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks No-long-jump zone!!! + */ +static int vmxHCAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + vmxHCReadToTransientSlow(pVCpu, pVmxTransient); + /** @todo consider template here after checking callers. */ + int rc = vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS); + AssertRCReturn(rc, rc); + + vmxHCAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr); + return VINF_SUCCESS; +} + + +/** + * Handle a condition that occurred while delivering an event through the guest or + * nested-guest IDT. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @retval VINF_SUCCESS if we should continue handling the VM-exit. + * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought + * to continue execution of the guest which will delivery the \#DF. + * @retval VINF_EM_RESET if we detected a triple-fault condition. + * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * + * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS. + * Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit + * is due to an EPT violation, PML full or SPP-related event. + * + * @remarks No-long-jump zone!!! + */ +static VBOXSTRICTRC vmxHCCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending); + HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO); + if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION + || pVmxTransient->uExitReason == VMX_EXIT_PML_FULL + || pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT) + HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION); + + VBOXSTRICTRC rcStrict = VINF_SUCCESS; + PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo; + uint32_t const uExitIntInfo = pVmxTransient->uExitIntInfo; + if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo)) + { + uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo); + uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo); + + /* + * If the event was a software interrupt (generated with INT n) or a software exception + * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we + * can handle the VM-exit and continue guest execution which will re-execute the + * instruction rather than re-injecting the exception, as that can cause premature + * trips to ring-3 before injection and involve TRPM which currently has no way of + * storing that these exceptions were caused by these instructions (ICEBP's #DB poses + * the problem). + */ + IEMXCPTRAISE enmRaise; + IEMXCPTRAISEINFO fRaiseInfo; + if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT + || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT + || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT) + { + enmRaise = IEMXCPTRAISE_REEXEC_INSTR; + fRaiseInfo = IEMXCPTRAISEINFO_NONE; + } + else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)) + { + uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo); + uint8_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo); + Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT); + + uint32_t const fIdtVectorFlags = vmxHCGetIemXcptFlags(uIdtVector, uIdtVectorType); + uint32_t const fExitVectorFlags = vmxHCGetIemXcptFlags(uExitVector, uExitVectorType); + + enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo); + + /* Determine a vectoring #PF condition, see comment in vmxHCExitXcptPF(). */ + if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF)) + { + pVmxTransient->fVectoringPF = true; + enmRaise = IEMXCPTRAISE_PREV_EVENT; + } + } + else + { + /* + * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access + * VM-exit, then the VM-exit interruption-information will not be valid and we end up here. + * It is sufficient to reflect the original event to the guest after handling the VM-exit. + */ + Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT + || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI + || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT); + enmRaise = IEMXCPTRAISE_PREV_EVENT; + fRaiseInfo = IEMXCPTRAISEINFO_NONE; + } + + /* + * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig + * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest + * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the + * subsequent VM-entry would fail, see @bugref{7445}. + * + * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". + */ + if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI + && enmRaise == IEMXCPTRAISE_PREV_EVENT + && (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI) + && CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx)) + CPUMClearInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx); + + switch (enmRaise) + { + case IEMXCPTRAISE_CURRENT_XCPT: + { + Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo)); + Assert(rcStrict == VINF_SUCCESS); + break; + } + + case IEMXCPTRAISE_PREV_EVENT: + { + uint32_t u32ErrCode; + if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo)) + u32ErrCode = pVmxTransient->uIdtVectoringErrorCode; + else + u32ErrCode = 0; + + /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see vmxHCExitXcptPF(). */ + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflect); + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */, u32ErrCode, + pVCpu->cpum.GstCtx.cr2); + + Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo, + VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode)); + Assert(rcStrict == VINF_SUCCESS); + break; + } + + case IEMXCPTRAISE_REEXEC_INSTR: + Assert(rcStrict == VINF_SUCCESS); + break; + + case IEMXCPTRAISE_DOUBLE_FAULT: + { + /* + * Determine a vectoring double #PF condition. Used later, when PGM evaluates the + * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF. + */ + if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF) + { + pVmxTransient->fVectoringDoublePF = true; + Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo, + pVCpu->cpum.GstCtx.cr2)); + rcStrict = VINF_SUCCESS; + } + else + { + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectConvertDF); + vmxHCSetPendingXcptDF(pVCpu); + Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo, + uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo))); + rcStrict = VINF_HM_DOUBLE_FAULT; + } + break; + } + + case IEMXCPTRAISE_TRIPLE_FAULT: + { + Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, + VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo))); + rcStrict = VINF_EM_RESET; + break; + } + + case IEMXCPTRAISE_CPU_HANG: + { + Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo)); + rcStrict = VERR_EM_GUEST_CPU_HANG; + break; + } + + default: + { + AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise)); + rcStrict = VERR_VMX_IPE_2; + break; + } + } + } + else if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI) + && !CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx)) + { + if ( VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo) + && VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF + && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo)) + { + /* + * Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in + * the guest or nested-guest NMI handler). We need to set the block-by-NMI field so + * that virtual NMIs remain blocked until the IRET execution is completed. + * + * See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception". + */ + CPUMSetInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx); + Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason)); + } + else if ( pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION + || pVmxTransient->uExitReason == VMX_EXIT_PML_FULL + || pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT) + { + /* + * Execution of IRET caused an EPT violation, page-modification log-full event or + * SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the + * guest or nested-guest NMI handler). We need to set the block-by-NMI field so + * that virtual NMIs remain blocked until the IRET execution is completed. + * + * See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET" + */ + if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual)) + { + CPUMSetInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx); + Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason)); + } + } + } + + Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT + || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG); + return rcStrict; +} + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** + * Perform the relevant VMX instruction checks for VM-exits that occurred due to the + * guest attempting to execute a VMX instruction. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @retval VINF_SUCCESS if we should continue handling the VM-exit. + * @retval VINF_HM_PENDING_XCPT if an exception was raised. + * + * @param pVCpu The cross context virtual CPU structure. + * @param uExitReason The VM-exit reason. + * + * @todo NSTVMX: Document other error codes when VM-exit is implemented. + * @remarks No-long-jump zone!!! + */ +static VBOXSTRICTRC vmxHCCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason) +{ + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS + | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER); + + /* + * The physical CPU would have already checked the CPU mode/code segment. + * We shall just assert here for paranoia. + * See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits". + */ + Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)); + Assert( !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx) + || CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)); + + if (uExitReason == VMX_EXIT_VMXON) + { + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4); + + /* + * We check CR4.VMXE because it is required to be always set while in VMX operation + * by physical CPUs and our CR4 read-shadow is only consulted when executing specific + * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation + * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0). + */ + if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx)) + { + Log4Func(("CR4.VMXE is not set -> #UD\n")); + vmxHCSetPendingXcptUD(pVCpu); + return VINF_HM_PENDING_XCPT; + } + } + else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx)) + { + /* + * The guest has not entered VMX operation but attempted to execute a VMX instruction + * (other than VMXON), we need to raise a #UD. + */ + Log4Func(("Not in VMX root mode -> #UD\n")); + vmxHCSetPendingXcptUD(pVCpu); + return VINF_HM_PENDING_XCPT; + } + + /* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */ + return VINF_SUCCESS; +} + + +/** + * Decodes the memory operand of an instruction that caused a VM-exit. + * + * The Exit qualification field provides the displacement field for memory + * operand instructions, if any. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @retval VINF_SUCCESS if the operand was successfully decoded. + * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the + * operand. + * @param pVCpu The cross context virtual CPU structure. + * @param uExitInstrInfo The VM-exit instruction information field. + * @param enmMemAccess The memory operand's access type (read or write). + * @param GCPtrDisp The instruction displacement field, if any. For + * RIP-relative addressing pass RIP + displacement here. + * @param pGCPtrMem Where to store the effective destination memory address. + * + * @remarks Warning! This function ASSUMES the instruction cannot be used in real or + * virtual-8086 mode hence skips those checks while verifying if the + * segment is valid. + */ +static VBOXSTRICTRC vmxHCDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess, + PRTGCPTR pGCPtrMem) +{ + Assert(pGCPtrMem); + Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu)); + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER + | CPUMCTX_EXTRN_CR0); + + static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) }; + static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) }; + AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks)); + + VMXEXITINSTRINFO ExitInstrInfo; + ExitInstrInfo.u = uExitInstrInfo; + uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize; + uint8_t const iSegReg = ExitInstrInfo.All.iSegReg; + bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid; + uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg; + uint8_t const uScale = ExitInstrInfo.All.u2Scaling; + bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid; + uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg; + bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand; + bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx); + + /* + * Validate instruction information. + * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code. + */ + AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks), + ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1); + AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT, + ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2); + AssertLogRelMsgReturn(fIsMemOperand, + ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3); + + /* + * Compute the complete effective address. + * + * See AMD instruction spec. 1.4.2 "SIB Byte Format" + * See AMD spec. 4.5.2 "Segment Registers". + */ + RTGCPTR GCPtrMem = GCPtrDisp; + if (fBaseRegValid) + GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64; + if (fIdxRegValid) + GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale; + + RTGCPTR const GCPtrOff = GCPtrMem; + if ( !fIsLongMode + || iSegReg >= X86_SREG_FS) + GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base; + GCPtrMem &= s_auAddrSizeMasks[uAddrSize]; + + /* + * Validate effective address. + * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode". + */ + uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize]; + Assert(cbAccess > 0); + if (fIsLongMode) + { + if (X86_IS_CANONICAL(GCPtrMem)) + { + *pGCPtrMem = GCPtrMem; + return VINF_SUCCESS; + } + + /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2 + * "Data Limit Checks in 64-bit Mode". */ + Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem)); + vmxHCSetPendingXcptGP(pVCpu, 0); + return VINF_HM_PENDING_XCPT; + } + + /* + * This is a watered down version of iemMemApplySegment(). + * Parts that are not applicable for VMX instructions like real-or-v8086 mode + * and segment CPL/DPL checks are skipped. + */ + RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff; + RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1; + PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg]; + + /* Check if the segment is present and usable. */ + if ( pSel->Attr.n.u1Present + && !pSel->Attr.n.u1Unusable) + { + Assert(pSel->Attr.n.u1DescType); + if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE)) + { + /* Check permissions for the data segment. */ + if ( enmMemAccess == VMXMEMACCESS_WRITE + && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE)) + { + Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u)); + vmxHCSetPendingXcptGP(pVCpu, iSegReg); + return VINF_HM_PENDING_XCPT; + } + + /* Check limits if it's a normal data segment. */ + if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN)) + { + if ( GCPtrFirst32 > pSel->u32Limit + || GCPtrLast32 > pSel->u32Limit) + { + Log4Func(("Data segment limit exceeded. " + "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32, + GCPtrLast32, pSel->u32Limit)); + if (iSegReg == X86_SREG_SS) + vmxHCSetPendingXcptSS(pVCpu, 0); + else + vmxHCSetPendingXcptGP(pVCpu, 0); + return VINF_HM_PENDING_XCPT; + } + } + else + { + /* Check limits if it's an expand-down data segment. + Note! The upper boundary is defined by the B bit, not the G bit! */ + if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1) + || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))) + { + Log4Func(("Expand-down data segment limit exceeded. " + "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32, + GCPtrLast32, pSel->u32Limit)); + if (iSegReg == X86_SREG_SS) + vmxHCSetPendingXcptSS(pVCpu, 0); + else + vmxHCSetPendingXcptGP(pVCpu, 0); + return VINF_HM_PENDING_XCPT; + } + } + } + else + { + /* Check permissions for the code segment. */ + if ( enmMemAccess == VMXMEMACCESS_WRITE + || ( enmMemAccess == VMXMEMACCESS_READ + && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ))) + { + Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u)); + Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)); + vmxHCSetPendingXcptGP(pVCpu, 0); + return VINF_HM_PENDING_XCPT; + } + + /* Check limits for the code segment (normal/expand-down not applicable for code segments). */ + if ( GCPtrFirst32 > pSel->u32Limit + || GCPtrLast32 > pSel->u32Limit) + { + Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", + GCPtrFirst32, GCPtrLast32, pSel->u32Limit)); + if (iSegReg == X86_SREG_SS) + vmxHCSetPendingXcptSS(pVCpu, 0); + else + vmxHCSetPendingXcptGP(pVCpu, 0); + return VINF_HM_PENDING_XCPT; + } + } + } + else + { + Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u)); + vmxHCSetPendingXcptGP(pVCpu, 0); + return VINF_HM_PENDING_XCPT; + } + + *pGCPtrMem = GCPtrMem; + return VINF_SUCCESS; +} +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + + +/** + * VM-exit helper for LMSW. + */ +static VBOXSTRICTRC vmxHCExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst) +{ + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst); + AssertMsg( rcStrict == VINF_SUCCESS + || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0); + if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitLmsw); + Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +} + + +/** + * VM-exit helper for CLTS. + */ +static VBOXSTRICTRC vmxHCExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr) +{ + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr); + AssertMsg( rcStrict == VINF_SUCCESS + || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0); + if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitClts); + Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +} + + +/** + * VM-exit helper for MOV from CRx (CRx read). + */ +static VBOXSTRICTRC vmxHCExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg) +{ + Assert(iCrReg < 16); + Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs)); + + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg); + AssertMsg( rcStrict == VINF_SUCCESS + || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + + if (iGReg == X86_GREG_xSP) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP); + else + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); +#ifdef VBOX_WITH_STATISTICS + switch (iCrReg) + { + case 0: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Read); break; + case 2: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Read); break; + case 3: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Read); break; + case 4: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Read); break; + case 8: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Read); break; + } +#endif + Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +} + + +/** + * VM-exit helper for MOV to CRx (CRx write). + */ +static VBOXSTRICTRC vmxHCExitMovToCrX(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg) +{ + HMVMX_CPUMCTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK); + + VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg); + AssertMsg( rcStrict == VINF_SUCCESS + || rcStrict == VINF_IEM_RAISED_XCPT + || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + + switch (iCrReg) + { + case 0: + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0 + | HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_EXIT_CTLS); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Write); + Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0)); + break; + + case 2: + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Write); + /* Nothing to do here, CR2 it's not part of the VMCS. */ + break; + + case 3: + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Write); + Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3)); + break; + + case 4: + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Write); +#ifndef IN_NEM_DARWIN + Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict), + pVCpu->cpum.GstCtx.cr4, pVCpu->hmr0.s.fLoadSaveGuestXcr0)); +#else + Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr4)); +#endif + break; + + case 8: + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, + HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Write); + break; + + default: + AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg)); + break; + } + + if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit exception handler for \#PF (Page-fault exception). + * + * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS. + */ +static VBOXSTRICTRC vmxHCExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient); + vmxHCReadToTransient(pVCpu, pVmxTransient); + +#ifndef IN_NEM_DARWIN + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if (!VM_IS_VMX_NESTED_PAGING(pVM)) + { /* likely */ } + else +#endif + { +#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF) && !defined(IN_NEM_DARWIN) + Assert(pVmxTransient->fIsNestedGuest || pVCpu->hmr0.s.fUsingDebugLoop); +#endif + VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory or vectoring #PF. */ + if (!pVmxTransient->fVectoringDoublePF) + { + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */, + pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual); + } + else + { + /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */ + Assert(!pVmxTransient->fIsNestedGuest); + vmxHCSetPendingXcptDF(pVCpu); + Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n")); + } + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF); + return VINF_SUCCESS; + } + + Assert(!pVmxTransient->fIsNestedGuest); + + /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable + of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */ + if (pVmxTransient->fVectoringPF) + { + Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending); + return VINF_EM_RAW_INJECT_TRPM_EVENT; + } + + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + Log4Func(("#PF: cs:rip=%#04x:%08RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pVCpu->cpum.GstCtx.cs.Sel, + pVCpu->cpum.GstCtx.rip, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pVCpu->cpum.GstCtx.cr3)); + + TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode); + rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, &pVCpu->cpum.GstCtx, (RTGCPTR)pVmxTransient->uExitQual); + + Log4Func(("#PF: rc=%Rrc\n", rc)); + if (rc == VINF_SUCCESS) + { + /* + * This is typically a shadow page table sync or a MMIO instruction. But we may have + * emulated something like LTR or a far jump. Any part of the CPU context may have changed. + */ + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + TRPMResetTrap(pVCpu); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPF); + return rc; + } + + if (rc == VINF_EM_RAW_GUEST_TRAP) + { + if (!pVmxTransient->fVectoringDoublePF) + { + /* It's a guest page fault and needs to be reflected to the guest. */ + uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu); + TRPMResetTrap(pVCpu); + VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* In case it's a contributory #PF. */ + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */, + uGstErrorCode, pVmxTransient->uExitQual); + } + else + { + /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */ + TRPMResetTrap(pVCpu); + VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; /* Clear pending #PF to replace it with #DF. */ + vmxHCSetPendingXcptDF(pVCpu); + Log4Func(("#PF: Pending #DF due to vectoring #PF\n")); + } + + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF); + return VINF_SUCCESS; + } + + TRPMResetTrap(pVCpu); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPFEM); + return rc; +} + + +/** + * VM-exit exception handler for \#MF (Math Fault: floating point exception). + * + * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS. + */ +static VBOXSTRICTRC vmxHCExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF); + + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE)) + { + /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */ + rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */); + + /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit + * provides VM-exit instruction length. If this causes problem later, + * disassemble the instruction like it's done on AMD-V. */ + int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient); + AssertRCReturn(rc2, rc2); + return rc; + } + + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr, + pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */); + return VINF_SUCCESS; +} + + +/** + * VM-exit exception handler for \#BP (Breakpoint exception). + * + * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS. + */ +static VBOXSTRICTRC vmxHCExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP); + + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict; + if (!pVmxTransient->fIsNestedGuest) + rcStrict = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx); + else + rcStrict = VINF_EM_RAW_GUEST_TRAP; + + if (rcStrict == VINF_EM_RAW_GUEST_TRAP) + { + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), + pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */); + rcStrict = VINF_SUCCESS; + } + + Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_DBG_BREAKPOINT); + return rcStrict; +} + + +/** + * VM-exit exception handler for \#AC (Alignment-check exception). + * + * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS. + */ +static VBOXSTRICTRC vmxHCExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* + * Detect #ACs caused by host having enabled split-lock detection. + * Emulate such instructions. + */ +#define VMX_HC_EXIT_XCPT_AC_INITIAL_REGS (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS) + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + /** @todo detect split lock in cpu feature? */ + if ( /* 1. If 486-style alignment checks aren't enabled, then this must be a split-lock exception */ + !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM) + /* 2. #AC cannot happen in rings 0-2 except for split-lock detection. */ + || CPUMGetGuestCPL(pVCpu) != 3 + /* 3. When the EFLAGS.AC != 0 this can only be a split-lock case. */ + || !(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_AC) ) + { + /* + * Check for debug/trace events and import state accordingly. + */ + STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestACSplitLock); + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if ( !DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK) +#ifndef IN_NEM_DARWIN + && !VBOXVMM_VMX_SPLIT_LOCK_ENABLED() +#endif + ) + { + if (pVM->cCpus == 1) + { +#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */ + rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); +#else + rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); +#endif + AssertRCReturn(rc, rc); + } + } + else + { + rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXVMM_XCPT_DF(pVCpu, &pVCpu->cpum.GstCtx); + + if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK)) + { + VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, DBGFEVENT_VMX_SPLIT_LOCK, DBGFEVENTCTX_HM, 0); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + } + + /* + * Emulate the instruction. + * + * We have to ignore the LOCK prefix here as we must not retrigger the + * detection on the host. This isn't all that satisfactory, though... + */ + if (pVM->cCpus == 1) + { + Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC\n", pVCpu->cpum.GstCtx.cs.Sel, + pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0)); + + /** @todo For SMP configs we should do a rendezvous here. */ + VBOXSTRICTRC rcStrict = IEMExecOneIgnoreLock(pVCpu); + if (rcStrict == VINF_SUCCESS) +#if 0 /** @todo r=bird: This is potentially wrong. Might have to just do a whole state sync above and mark everything changed to be safe... */ + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, + HM_CHANGED_GUEST_RIP + | HM_CHANGED_GUEST_RFLAGS + | HM_CHANGED_GUEST_GPRS_MASK + | HM_CHANGED_GUEST_CS + | HM_CHANGED_GUEST_SS); +#else + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); +#endif + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; + } + Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC -> VINF_EM_EMULATE_SPLIT_LOCK\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0)); + return VINF_EM_EMULATE_SPLIT_LOCK; + } + + STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC); + Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 cpl=%d -> #AC\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0, CPUMGetGuestCPL(pVCpu) )); + + /* Re-inject it. We'll detect any nesting before getting here. */ + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), + pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */); + return VINF_SUCCESS; +} + + +/** + * VM-exit exception handler for \#DB (Debug exception). + * + * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS. + */ +static VBOXSTRICTRC vmxHCExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB); + + /* + * Get the DR6-like values from the Exit qualification and pass it to DBGF for processing. + */ + vmxHCReadToTransient(pVCpu, pVmxTransient); + + /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */ + uint64_t const uDR6 = X86_DR6_INIT_VAL + | (pVmxTransient->uExitQual & ( X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 + | X86_DR6_BD | X86_DR6_BS)); + Log6Func(("uDR6=%#RX64 uExitQual=%#RX64\n", uDR6, pVmxTransient->uExitQual)); + + int rc; + if (!pVmxTransient->fIsNestedGuest) + { + rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx, uDR6, VCPU_2_VMXSTATE(pVCpu).fSingleInstruction); + + /* + * Prevents stepping twice over the same instruction when the guest is stepping using + * EFLAGS.TF and the hypervisor debugger is stepping using MTF. + * Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t"). + */ + if ( rc == VINF_EM_DBG_STEPPED + && (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)) + { + Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction); + rc = VINF_EM_RAW_GUEST_TRAP; + } + } + else + rc = VINF_EM_RAW_GUEST_TRAP; + Log6Func(("rc=%Rrc\n", rc)); + if (rc == VINF_EM_RAW_GUEST_TRAP) + { + /* + * The exception was for the guest. Update DR6, DR7.GD and + * IA32_DEBUGCTL.LBR before forwarding it. + * See Intel spec. 27.1 "Architectural State before a VM-Exit" + * and @sdmv3{077,622,17.2.3,Debug Status Register (DR6)}. + */ +#ifndef IN_NEM_DARWIN + VMMRZCallRing3Disable(pVCpu); + HM_DISABLE_PREEMPT(pVCpu); + + pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK; + pVCpu->cpum.GstCtx.dr[6] |= uDR6; + if (CPUMIsGuestDebugStateActive(pVCpu)) + ASMSetDR6(pVCpu->cpum.GstCtx.dr[6]); + + HM_RESTORE_PREEMPT(); + VMMRZCallRing3Enable(pVCpu); +#else + /** @todo */ +#endif + + rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */ + pVCpu->cpum.GstCtx.dr[7] &= ~(uint64_t)X86_DR7_GD; + + /* Paranoia. */ + pVCpu->cpum.GstCtx.dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK; + pVCpu->cpum.GstCtx.dr[7] |= X86_DR7_RA1_MASK; + + rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, pVCpu->cpum.GstCtx.dr[7]); + AssertRC(rc); + + /* + * Raise #DB in the guest. + * + * It is important to reflect exactly what the VM-exit gave us (preserving the + * interruption-type) rather than use vmxHCSetPendingXcptDB() as the #DB could've + * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may + * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}. + * + * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of + * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection". + */ + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), + pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */); + return VINF_SUCCESS; + } + + /* + * Not a guest trap, must be a hypervisor related debug event then. + * Update DR6 in case someone is interested in it. + */ + AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc)); + AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5); + CPUMSetHyperDR6(pVCpu, uDR6); + + return rc; +} + + +/** + * Hacks its way around the lovely mesa driver's backdoor accesses. + * + * @sa hmR0SvmHandleMesaDrvGp. + */ +static int vmxHCHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx) +{ + LogFunc(("cs:rip=%#04x:%08RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx)); + RT_NOREF(pCtx); + + /* For now we'll just skip the instruction. */ + return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient); +} + + +/** + * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely + * backdoor logging w/o checking what it is running inside. + * + * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the + * backdoor port and magic numbers loaded in registers. + * + * @returns true if it is, false if it isn't. + * @sa hmR0SvmIsMesaDrvGp. + */ +DECLINLINE(bool) vmxHCIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx) +{ + /* 0xed: IN eAX,dx */ + uint8_t abInstr[1]; + if (pVmxTransient->cbExitInstr != sizeof(abInstr)) + return false; + + /* Check that it is #GP(0). */ + if (pVmxTransient->uExitIntErrorCode != 0) + return false; + + /* Check magic and port. */ + Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX))); + /*Log(("vmxHCIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));*/ + if (pCtx->rax != UINT32_C(0x564d5868)) + return false; + if (pCtx->dx != UINT32_C(0x5658)) + return false; + + /* Flat ring-3 CS. */ + AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS); + Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS)); + /*Log(("vmxHCIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));*/ + if (pCtx->cs.Attr.n.u2Dpl != 3) + return false; + if (pCtx->cs.u64Base != 0) + return false; + + /* Check opcode. */ + AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP); + Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP)); + int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr)); + /*Log(("vmxHCIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));*/ + if (RT_FAILURE(rc)) + return false; + if (abInstr[0] != 0xed) + return false; + + return true; +} + + +/** + * VM-exit exception handler for \#GP (General-protection exception). + * + * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS. + */ +static VBOXSTRICTRC vmxHCExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP); + + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; +#ifndef IN_NEM_DARWIN + PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared; + if (pVmcsInfoShared->RealMode.fRealOnV86Active) + { /* likely */ } + else +#endif + { +#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS +# ifndef IN_NEM_DARWIN + Assert(pVCpu->hmr0.s.fUsingDebugLoop || VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv || pVmxTransient->fIsNestedGuest); +# else + Assert(/*pVCpu->hmr0.s.fUsingDebugLoop ||*/ VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv || pVmxTransient->fIsNestedGuest); +# endif +#endif + /* + * If the guest is not in real-mode or we have unrestricted guest execution support, or if we are + * executing a nested-guest, reflect #GP to the guest or nested-guest. + */ + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + Log4Func(("Gst: cs:rip=%#04x:%08RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip, + pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel)); + + if ( pVmxTransient->fIsNestedGuest + || !VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv + || !vmxHCIsMesaDrvGp(pVCpu, pVmxTransient, pCtx)) + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), + pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */); + else + rc = vmxHCHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx); + return rc; + } + +#ifndef IN_NEM_DARWIN + Assert(CPUMIsGuestInRealModeEx(pCtx)); + Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest); + Assert(!pVmxTransient->fIsNestedGuest); + + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu); + if (rcStrict == VINF_SUCCESS) + { + if (!CPUMIsGuestInRealModeEx(pCtx)) + { + /* + * The guest is no longer in real-mode, check if we can continue executing the + * guest using hardware-assisted VMX. Otherwise, fall back to emulation. + */ + pVmcsInfoShared->RealMode.fRealOnV86Active = false; + if (HMCanExecuteVmxGuest(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx)) + { + Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n")); + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + } + else + { + Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n")); + rcStrict = VINF_EM_RESCHEDULE; + } + } + else + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + } + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + rcStrict = VINF_SUCCESS; + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + } + return VBOXSTRICTRC_VAL(rcStrict); +#endif +} + + +/** + * VM-exit exception handler for \#DE (Divide Error). + * + * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS. + */ +static VBOXSTRICTRC vmxHCExitXcptDE(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE); + + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE; + if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE) + { + uint8_t cbInstr = 0; + VBOXSTRICTRC rc2 = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr); + if (rc2 == VINF_SUCCESS) + rcStrict = VINF_SUCCESS; /* Restart instruction with modified guest register context. */ + else if (rc2 == VERR_NOT_FOUND) + rcStrict = VERR_NOT_FOUND; /* Deliver the exception. */ + else + Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict))); + } + else + rcStrict = VINF_SUCCESS; /* Do nothing. */ + + /* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */ + if (RT_FAILURE(rcStrict)) + { + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), + pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */); + rcStrict = VINF_SUCCESS; + } + + Assert(rcStrict == VINF_SUCCESS || rcStrict == VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE); + return VBOXSTRICTRC_VAL(rcStrict); +} + + +/** + * VM-exit exception handler wrapper for all other exceptions that are not handled + * by a specific handler. + * + * This simply re-injects the exception back into the VM without any special + * processing. + * + * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS. + */ +static VBOXSTRICTRC vmxHCExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient); + +#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS +# ifndef IN_NEM_DARWIN + PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + AssertMsg(pVCpu->hmr0.s.fUsingDebugLoop || pVmcsInfo->pShared->RealMode.fRealOnV86Active || pVmxTransient->fIsNestedGuest, + ("uVector=%#x u32XcptBitmap=%#X32\n", + VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap)); + NOREF(pVmcsInfo); +# endif +#endif + + /* + * Re-inject the exception into the guest. This cannot be a double-fault condition which + * would have been handled while checking exits due to event delivery. + */ + uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo); + +#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%08RX64\n", uVector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); +#endif + +#ifdef VBOX_WITH_STATISTICS + switch (uVector) + { + case X86_XCPT_DE: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE); break; + case X86_XCPT_DB: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB); break; + case X86_XCPT_BP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP); break; + case X86_XCPT_OF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break; + case X86_XCPT_BR: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBR); break; + case X86_XCPT_UD: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestUD); break; + case X86_XCPT_NM: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF); break; + case X86_XCPT_DF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDF); break; + case X86_XCPT_TS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestTS); break; + case X86_XCPT_NP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestNP); break; + case X86_XCPT_SS: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestSS); break; + case X86_XCPT_GP: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP); break; + case X86_XCPT_PF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF); break; + case X86_XCPT_MF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF); break; + case X86_XCPT_AC: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC); break; + case X86_XCPT_XF: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXF); break; + default: + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXcpUnk); + break; + } +#endif + + /* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */ + Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo)); + NOREF(uVector); + + /* Re-inject the original exception into the guest. */ + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), + pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */); + return VINF_SUCCESS; +} + + +/** + * VM-exit exception handler for all exceptions (except NMIs!). + * + * @remarks This may be called for both guests and nested-guests. Take care to not + * make assumptions and avoid doing anything that is not relevant when + * executing a nested-guest (e.g., Mesa driver hacks). + */ +static VBOXSTRICTRC vmxHCExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO); + + /* + * If this VM-exit occurred while delivering an event through the guest IDT, take + * action based on the return code and additional hints (e.g. for page-faults) + * that will be updated in the VMX transient structure. + */ + VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient); + if (rcStrict == VINF_SUCCESS) + { + /* + * If an exception caused a VM-exit due to delivery of an event, the original + * event may have to be re-injected into the guest. We shall reinject it and + * continue guest execution. However, page-fault is a complicated case and + * needs additional processing done in vmxHCExitXcptPF(). + */ + Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)); + uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo); + if ( !VCPU_2_VMXSTATE(pVCpu).Event.fPending + || uVector == X86_XCPT_PF) + { + switch (uVector) + { + case X86_XCPT_PF: return vmxHCExitXcptPF(pVCpu, pVmxTransient); + case X86_XCPT_GP: return vmxHCExitXcptGP(pVCpu, pVmxTransient); + case X86_XCPT_MF: return vmxHCExitXcptMF(pVCpu, pVmxTransient); + case X86_XCPT_DB: return vmxHCExitXcptDB(pVCpu, pVmxTransient); + case X86_XCPT_BP: return vmxHCExitXcptBP(pVCpu, pVmxTransient); + case X86_XCPT_AC: return vmxHCExitXcptAC(pVCpu, pVmxTransient); + case X86_XCPT_DE: return vmxHCExitXcptDE(pVCpu, pVmxTransient); + default: + return vmxHCExitXcptOthers(pVCpu, pVmxTransient); + } + } + /* else: inject pending event before resuming guest execution. */ + } + else if (rcStrict == VINF_HM_DOUBLE_FAULT) + { + Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending); + rcStrict = VINF_SUCCESS; + } + + return rcStrict; +} +/** @} */ + + +/** @name VM-exit handlers. + * @{ + */ +/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */ +/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */ +/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */ + +/** + * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT). + */ +HMVMX_EXIT_DECL vmxHCExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitExtInt); + +#ifndef IN_NEM_DARWIN + /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */ + if (VMMR0ThreadCtxHookIsEnabled(pVCpu)) + return VINF_SUCCESS; + return VINF_EM_RAW_INTERRUPT; +#else + return VINF_SUCCESS; +#endif +} + + +/** + * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional + * VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3); + + vmxHCReadToTransient(pVCpu, pVmxTransient); + + uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo); + uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo); + Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)); + + PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT) + && uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT); + NOREF(pVmcsInfo); + + VBOXSTRICTRC rcStrict; + switch (uExitIntType) + { +#ifndef IN_NEM_DARWIN /* NMIs should never reach R3. */ + /* + * Host physical NMIs: + * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we + * injected it ourselves and anything we inject is not going to cause a VM-exit directly + * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2]. + * + * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery". + * See Intel spec. 27.5.5 "Updating Non-Register State". + */ + case VMX_EXIT_INT_INFO_TYPE_NMI: + { + rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo); + break; + } +#endif + + /* + * Privileged software exceptions (#DB from ICEBP), + * Software exceptions (#BP and #OF), + * Hardware exceptions: + * Process the required exceptions and resume guest execution if possible. + */ + case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: + Assert(uVector == X86_XCPT_DB); + RT_FALL_THRU(); + case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: + Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT); + RT_FALL_THRU(); + case VMX_EXIT_INT_INFO_TYPE_HW_XCPT: + { + NOREF(uVector); + vmxHCReadToTransient< HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient); + rcStrict = vmxHCExitXcpt(pVCpu, pVmxTransient); + break; + } + + default: + { + VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo; + rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE; + AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo)); + break; + } + } + + STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3); + return rcStrict; +} + + +/** + * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW). + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo); + + /* Evaluate and deliver pending events and resume guest execution. */ + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIntWindow); + return VINF_SUCCESS; +} + + +/** + * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW). + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */ + { + AssertMsgFailed(("Unexpected NMI-window exit.\n")); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason); + } + + Assert(!CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx)); + + /* + * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI. + * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}. + */ + uint32_t fIntrState; + int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState); + AssertRC(rc); + Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)); + if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI) + { + CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); + + fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI; + rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState); + AssertRC(rc); + } + + /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */ + vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo); + + /* Evaluate and deliver pending events and resume guest execution. */ + return VINF_SUCCESS; +} + + +/** + * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit. + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient); +} + + +/** + * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit. + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient); +} + + +/** + * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* + * Get the state we need and update the exit history entry. + */ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict; + PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu, + EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID), + pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base); + if (!pExitRec) + { + /* + * Regular CPUID instruction execution. + */ + rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr); + if (rcStrict == VINF_SUCCESS) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + } + else + { + /* + * Frequent exit or something needing probing. Get state and call EMHistoryExec. + */ + int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc2, rc2); + + Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx)); + + rcStrict = EMHistoryExec(pVCpu, pExitRec, 0); + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + + Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + } + return rcStrict; +} + + +/** + * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE) + return VINF_EM_RAW_EMULATE_INSTR; + + AssertMsgFailed(("vmxHCExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n")); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason); +} + + +/** + * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + /* If we get a spurious VM-exit when TSC offsetting is enabled, + we must reset offsetting on VM-entry. See @bugref{6634}. */ + if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING) + pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false; + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + } + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + /* If we get a spurious VM-exit when TSC offsetting is enabled, + we must reset offsetting on VM-reentry. See @bugref{6634}. */ + if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING) + pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false; + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + } + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedRdpmc(pVCpu, pVmxTransient->cbExitInstr); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3; + if (EMAreHypercallInstructionsEnabled(pVCpu)) + { + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RIP + | CPUMCTX_EXTRN_RFLAGS + | CPUMCTX_EXTRN_CR0 + | CPUMCTX_EXTRN_SS + | CPUMCTX_EXTRN_CS + | CPUMCTX_EXTRN_EFER>(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + /* Perform the hypercall. */ + rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx); + if (rcStrict == VINF_SUCCESS) + { + rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient); + AssertRCReturn(rc, rc); + } + else + Assert( rcStrict == VINF_GIM_R3_HYPERCALL + || rcStrict == VINF_GIM_HYPERCALL_CONTINUING + || RT_FAILURE(rcStrict)); + + /* If the hypercall changes anything other than guest's general-purpose registers, + we would need to reload the guest changed bits here before VM-entry. */ + } + else + Log4Func(("Hypercalls not enabled\n")); + + /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */ + if (RT_FAILURE(rcStrict)) + { + vmxHCSetPendingXcptUD(pVCpu); + rcStrict = VINF_SUCCESS; + } + + return rcStrict; +} + + +/** + * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); +#ifndef IN_NEM_DARWIN + Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging || pVCpu->hmr0.s.fUsingDebugLoop); +#endif + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual); + + if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + else + AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual, + VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +} + + +/** + * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr); + if (rcStrict == VINF_SUCCESS) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + + return rcStrict; +} + + +/** + * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr); + if (RT_SUCCESS(rcStrict)) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx)) + rcStrict = VINF_SUCCESS; + } + + return rcStrict; +} + + +/** + * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional + * VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + return VINF_EM_RESET; +} + + +/** + * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient); + AssertRCReturn(rc, rc); + + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */ + if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */ + rc = VINF_SUCCESS; + else + rc = VINF_EM_HALT; + + if (rc != VINF_SUCCESS) + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHltToR3); + return rc; +} + + +#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** + * VM-exit handler for instructions that result in a \#UD exception delivered to + * the guest. + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + vmxHCSetPendingXcptUD(pVCpu); + return VINF_SUCCESS; +} +#endif + + +/** + * VM-exit handler for expiry of the VMX-preemption timer. + */ +HMVMX_EXIT_DECL vmxHCExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */ + pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false; +Log12(("vmxHCExitPreemptTimer:\n")); + + /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + bool fTimersPending = TMTimerPollBool(pVM, pVCpu); + STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitPreemptTimer); + return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS; +} + + +/** + * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr); + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS + : HM_CHANGED_RAISED_XCPT_MASK); + +#ifndef IN_NEM_DARWIN + PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0(); + if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0) + { + pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0; + hmR0VmxUpdateStartVmFunction(pVCpu); + } +#endif + + return rcStrict; +} + + +/** + * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /** @todo Enable the new code after finding a reliably guest test-case. */ +#if 1 + return VERR_EM_INTERPRETER; +#else + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK); + AssertRCReturn(rc, rc); + + /* Paranoia. Ensure this has a memory operand. */ + Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0); + + uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2; + Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs)); + uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64 + : pVCpu->cpum.GstCtx.aGRegs[iGReg].u32; + + RTGCPTR GCPtrDesc; + HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc); + + VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg, + GCPtrDesc, uType); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +#endif +} + + +/** + * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error + * VM-exit. + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); + AssertRCReturn(rc, rc); + + rc = vmxHCCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest); + if (RT_FAILURE(rc)) + return rc; + + uint32_t const uInvalidReason = vmxHCCheckGuestState(pVCpu, pVmcsInfo); + NOREF(uInvalidReason); + +#ifdef VBOX_STRICT + uint32_t fIntrState; + uint64_t u64Val; + vmxHCReadToTransient< HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + vmxHCReadEntryXcptErrorCodeVmcs(pVCpu, pVmxTransient); + + Log4(("uInvalidReason %u\n", uInvalidReason)); + Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo)); + Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode)); + Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr)); + + rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState); AssertRC(rc); + Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState)); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc); + Log4(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val)); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, &u64Val); AssertRC(rc); + Log4(("VMX_VMCS_CTRL_CR0_MASK %#RX64\n", u64Val)); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val); AssertRC(rc); + Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val)); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, &u64Val); AssertRC(rc); + Log4(("VMX_VMCS_CTRL_CR4_MASK %#RX64\n", u64Val)); + rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val); AssertRC(rc); + Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RX64\n", u64Val)); +# ifndef IN_NEM_DARWIN + if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging) + { + rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc); + Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val)); + } + + hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL); +# endif +#endif + + return VERR_VMX_INVALID_GUEST_STATE; +} + +/** + * VM-exit handler for all undefined/unexpected reasons. Should never happen. + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + /* + * Cumulative notes of all recognized but unexpected VM-exits. + * + * 1. This does -not- cover scenarios like a page-fault VM-exit occurring when + * nested-paging is used. + * + * 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be + * emulated or a #UD must be raised in the guest. Therefore, we should -not- be using + * this function (and thereby stop VM execution) for handling such instructions. + * + * + * VMX_EXIT_INIT_SIGNAL: + * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. + * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these + * VM-exits. However, we should not receive INIT signals VM-exit while executing a VM. + * + * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" + * See Intel spec. 29.3 "VMX Instructions" for "VMXON". + * See Intel spec. "23.8 Restrictions on VMX operation". + * + * VMX_EXIT_SIPI: + * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest + * activity state is used. We don't make use of it as our guests don't have direct + * access to the host local APIC. + * + * See Intel spec. 25.3 "Other Causes of VM-exits". + * + * VMX_EXIT_IO_SMI: + * VMX_EXIT_SMI: + * This can only happen if we support dual-monitor treatment of SMI, which can be + * activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer + * monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in + * VMX root mode or receive an SMI. If we get here, something funny is going on. + * + * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment" + * See Intel spec. 25.3 "Other Causes of VM-Exits" + * + * VMX_EXIT_ERR_MSR_LOAD: + * Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected + * and typically indicates a bug in the hypervisor code. We thus cannot not resume + * execution. + * + * See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State". + * + * VMX_EXIT_ERR_MACHINE_CHECK: + * Machine check exceptions indicates a fatal/unrecoverable hardware condition + * including but not limited to system bus, ECC, parity, cache and TLB errors. A + * #MC exception abort class exception is raised. We thus cannot assume a + * reasonable chance of continuing any sort of execution and we bail. + * + * See Intel spec. 15.1 "Machine-check Architecture". + * See Intel spec. 27.1 "Architectural State Before A VM Exit". + * + * VMX_EXIT_PML_FULL: + * VMX_EXIT_VIRTUALIZED_EOI: + * VMX_EXIT_APIC_WRITE: + * We do not currently support any of these features and thus they are all unexpected + * VM-exits. + * + * VMX_EXIT_GDTR_IDTR_ACCESS: + * VMX_EXIT_LDTR_TR_ACCESS: + * VMX_EXIT_RDRAND: + * VMX_EXIT_RSM: + * VMX_EXIT_VMFUNC: + * VMX_EXIT_ENCLS: + * VMX_EXIT_RDSEED: + * VMX_EXIT_XSAVES: + * VMX_EXIT_XRSTORS: + * VMX_EXIT_UMWAIT: + * VMX_EXIT_TPAUSE: + * VMX_EXIT_LOADIWKEY: + * These VM-exits are -not- caused unconditionally by execution of the corresponding + * instruction. Any VM-exit for these instructions indicate a hardware problem, + * unsupported CPU modes (like SMM) or potentially corrupt VMCS controls. + * + * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally". + */ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason)); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason); +} + + +/** + * VM-exit handler for RDMSR (VMX_EXIT_RDMSR). + */ +HMVMX_EXIT_DECL vmxHCExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient(pVCpu, pVmxTransient); + + /** @todo Optimize this: We currently drag in the whole MSR state + * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get + * MSRs required. That would require changes to IEM and possibly CPUM too. + * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; + int rc; + switch (idMsr) + { + default: + rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, + __FUNCTION__); + AssertRCReturn(rc, rc); + break; + case MSR_K8_FS_BASE: + rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS + | CPUMCTX_EXTRN_FS>(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + break; + case MSR_K8_GS_BASE: + rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS + | CPUMCTX_EXTRN_GS>(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + break; + } + + Log4Func(("ecx=%#RX32\n", idMsr)); + +#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN) + Assert(!pVmxTransient->fIsNestedGuest); + if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS) + { + if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr) + && idMsr != MSR_K6_EFER) + { + AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr)); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr); + } + if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr)) + { + Assert(pVmcsInfo->pvMsrBitmap); + uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr); + if (fMsrpm & VMXMSRPM_ALLOW_RD) + { + AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr)); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr); + } + } + } +#endif + + VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitRdmsr); + if (rcStrict == VINF_SUCCESS) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + else + AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ || rcStrict == VINF_EM_TRIPLE_FAULT, + ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + + return rcStrict; +} + + +/** + * VM-exit handler for WRMSR (VMX_EXIT_WRMSR). + */ +HMVMX_EXIT_DECL vmxHCExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient(pVCpu, pVmxTransient); + + /* + * The FS and GS base MSRs are not part of the above all-MSRs mask. + * Although we don't need to fetch the base as it will be overwritten shortly, while + * loading guest-state we would also load the entire segment register including limit + * and attributes and thus we need to load them here. + */ + /** @todo Optimize this: We currently drag in the whole MSR state + * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get + * MSRs required. That would require changes to IEM and possibly CPUM too. + * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; + int rc; + switch (idMsr) + { + default: + rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, + __FUNCTION__); + AssertRCReturn(rc, rc); + break; + + case MSR_K8_FS_BASE: + rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS + | CPUMCTX_EXTRN_FS>(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + break; + case MSR_K8_GS_BASE: + rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS + | CPUMCTX_EXTRN_GS>(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + break; + } + Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax)); + + VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitWrmsr); + + if (rcStrict == VINF_SUCCESS) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + + /* If this is an X2APIC WRMSR access, update the APIC state as well. */ + if ( idMsr == MSR_IA32_APICBASE + || ( idMsr >= MSR_IA32_X2APIC_START + && idMsr <= MSR_IA32_X2APIC_END)) + { + /* + * We've already saved the APIC related guest-state (TPR) in post-run phase. + * When full APIC register virtualization is implemented we'll have to make + * sure APIC state is saved from the VMCS before IEM changes it. + */ + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_APIC_TPR); + } + else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */ + pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false; + else if (idMsr == MSR_K6_EFER) + { + /* + * If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls + * as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME). + * We care about the other bits as well, SCE and NXE. See @bugref{7368}. + */ + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_EXIT_CTLS); + } + + /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */ + if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)) + { + switch (idMsr) + { + case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break; + case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break; + case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break; + case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_FS); break; + case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_GS); break; + case MSR_K6_EFER: /* Nothing to do, already handled above. */ break; + default: + { +#ifndef IN_NEM_DARWIN + if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS); + else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS); +#else + AssertMsgFailed(("TODO\n")); +#endif + break; + } + } + } +#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN) + else + { + /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */ + switch (idMsr) + { + case MSR_IA32_SYSENTER_CS: + case MSR_IA32_SYSENTER_EIP: + case MSR_IA32_SYSENTER_ESP: + case MSR_K8_FS_BASE: + case MSR_K8_GS_BASE: + { + AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr)); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr); + } + + /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */ + default: + { + if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)) + { + /* EFER MSR writes are always intercepted. */ + if (idMsr != MSR_K6_EFER) + { + AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", + idMsr)); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr); + } + } + + if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr)) + { + Assert(pVmcsInfo->pvMsrBitmap); + uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr); + if (fMsrpm & VMXMSRPM_ALLOW_WR) + { + AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr)); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr); + } + } + break; + } + } + } +#endif /* VBOX_STRICT */ + } + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + else + AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE || rcStrict == VINF_EM_TRIPLE_FAULT, + ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict))); + + return rcStrict; +} + + +/** + * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /** @todo The guest has likely hit a contended spinlock. We might want to + * poke a schedule different guest VCPU. */ + int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient); + if (RT_SUCCESS(rc)) + return VINF_EM_RAW_INTERRUPT; + + AssertMsgFailed(("vmxHCExitPause: Failed to increment RIP. rc=%Rrc\n", rc)); + return rc; +} + + +/** + * VM-exit handler for when the TPR value is lowered below the specified + * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit. + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); + + /* + * The TPR shadow would've been synced with the APIC TPR in the post-run phase. + * We'll re-evaluate pending interrupts and inject them before the next VM + * entry so we can just continue execution here. + */ + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTprBelowThreshold); + return VINF_SUCCESS; +} + + +/** + * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional + * VM-exit. + * + * @retval VINF_SUCCESS when guest execution can continue. + * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3. + * @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to + * incompatible guest state for VMX execution (real-on-v86 case). + */ +HMVMX_EXIT_DECL vmxHCExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + + VBOXSTRICTRC rcStrict; + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + uint64_t const uExitQual = pVmxTransient->uExitQual; + uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual); + switch (uAccessType) + { + /* + * MOV to CRx. + */ + case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: + { + /* + * When PAE paging is used, the CPU will reload PAE PDPTEs from CR3 when the guest + * changes certain bits even in CR0, CR4 (and not just CR3). We are currently fine + * since IEM_CPUMCTX_EXTRN_MUST_MASK (used below) includes CR3 which will import + * PAE PDPTEs as well. + */ + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0); +#ifndef IN_NEM_DARWIN + uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0; +#endif + uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual); + uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual); + + /* + * MOV to CR3 only cause a VM-exit when one or more of the following are true: + * - When nested paging isn't used. + * - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables). + * - We are executing in the VM debug loop. + */ +#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS +# ifndef IN_NEM_DARWIN + Assert( iCrReg != 3 + || !VM_IS_VMX_NESTED_PAGING(pVM) + || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx) + || pVCpu->hmr0.s.fUsingDebugLoop); +# else + Assert( iCrReg != 3 + || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)); +# endif +#endif + + /* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */ + Assert( iCrReg != 8 + || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)); + + rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg); + AssertMsg( rcStrict == VINF_SUCCESS + || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + +#ifndef IN_NEM_DARWIN + /* + * This is a kludge for handling switches back to real mode when we try to use + * V86 mode to run real mode code directly. Problem is that V86 mode cannot + * deal with special selector values, so we have to return to ring-3 and run + * there till the selector values are V86 mode compatible. + * + * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the + * latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of + * this function. + */ + if ( iCrReg == 0 + && rcStrict == VINF_SUCCESS + && !VM_IS_VMX_UNRESTRICTED_GUEST(pVM) + && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx) + && (uOldCr0 & X86_CR0_PE) + && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)) + { + /** @todo Check selectors rather than returning all the time. */ + Assert(!pVmxTransient->fIsNestedGuest); + Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n")); + rcStrict = VINF_EM_RESCHEDULE_REM; + } +#endif + + break; + } + + /* + * MOV from CRx. + */ + case VMX_EXIT_QUAL_CRX_ACCESS_READ: + { + uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(uExitQual); + uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual); + + /* + * MOV from CR3 only cause a VM-exit when one or more of the following are true: + * - When nested paging isn't used. + * - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3). + * - We are executing in the VM debug loop. + */ +#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS +# ifndef IN_NEM_DARWIN + Assert( iCrReg != 3 + || !VM_IS_VMX_NESTED_PAGING(pVM) + || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx) + || pVCpu->hmr0.s.fLeaveDone); +# else + Assert( iCrReg != 3 + || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)); +# endif +#endif + + /* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */ + Assert( iCrReg != 8 + || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)); + + rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg); + break; + } + + /* + * CLTS (Clear Task-Switch Flag in CR0). + */ + case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: + { + rcStrict = vmxHCExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr); + break; + } + + /* + * LMSW (Load Machine-Status Word into CR0). + * LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. + */ + case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: + { + RTGCPTR GCPtrEffDst; + uint8_t const cbInstr = pVmxTransient->cbExitInstr; + uint16_t const uMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual); + bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual); + if (fMemOperand) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + GCPtrEffDst = pVmxTransient->uGuestLinearAddr; + } + else + GCPtrEffDst = NIL_RTGCPTR; + rcStrict = vmxHCExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst); + break; + } + + default: + { + AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType)); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType); + } + } + + Assert((VCPU_2_VMXSTATE(pVCpu).fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS)) + == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS)); + Assert(rcStrict != VINF_IEM_RAISED_XCPT); + + STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2); + NOREF(pVM); + return rcStrict; +} + + +/** + * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional + * VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1); + + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); +#define VMX_HC_EXIT_IO_INSTR_INITIAL_REGS (IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER) + /* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */ + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */ + uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual); + uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual); + bool const fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT); + bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual); + bool const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF); + bool const fDbgStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction; + AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1); + + /* + * Update exit history to see if this exit can be optimized. + */ + VBOXSTRICTRC rcStrict; + PCEMEXITREC pExitRec = NULL; + if ( !fGstStepping + && !fDbgStepping) + pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu, + !fIOString + ? !fIOWrite + ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ) + : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE) + : !fIOWrite + ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ) + : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE), + pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base); + if (!pExitRec) + { + static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */ + static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */ + + uint32_t const cbValue = s_aIOSizes[uIOSize]; + uint32_t const cbInstr = pVmxTransient->cbExitInstr; + bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if (fIOString) + { + /* + * INS/OUTS - I/O String instruction. + * + * Use instruction-information if available, otherwise fall back on + * interpreting the instruction. + */ + Log4Func(("cs:rip=%#04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r')); + AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2); + bool const fInsOutsInfo = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS); + if (fInsOutsInfo) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3); + AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2); + IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize; + bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual); + if (fIOWrite) + rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, + pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/); + else + { + /* + * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES. + * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS. + * See Intel Instruction spec. for "INS". + * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS". + */ + rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/); + } + } + else + rcStrict = IEMExecOne(pVCpu); + + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP); + fUpdateRipAlready = true; + } + else + { + /* + * IN/OUT - I/O instruction. + */ + Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r')); + uint32_t const uAndVal = s_aIOOpAnd[uIOSize]; + Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual)); + if (fIOWrite) + { + rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite); +#ifndef IN_NEM_DARWIN + if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE + && !pCtx->eflags.Bits.u1TF) + rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal); +#endif + } + else + { + uint32_t u32Result = 0; + rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue); + if (IOM_SUCCESS(rcStrict)) + { + /* Save result of I/O IN instr. in AL/AX/EAX. */ + pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal); + } +#ifndef IN_NEM_DARWIN + if ( rcStrict == VINF_IOM_R3_IOPORT_READ + && !pCtx->eflags.Bits.u1TF) + rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue); +#endif + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIORead); + } + } + + if (IOM_SUCCESS(rcStrict)) + { + if (!fUpdateRipAlready) + { + vmxHCAdvanceGuestRipBy(pVCpu, cbInstr); + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP); + } + + /* + * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru + * while booting Fedora 17 64-bit guest. + * + * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ. + */ + if (fIOString) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RFLAGS); + + /* + * If any I/O breakpoints are armed, we need to check if one triggered + * and take appropriate action. + * Note that the I/O breakpoint type is undefined if CR4.DE is 0. + */ +#if 1 + AssertCompile(VMX_HC_EXIT_IO_INSTR_INITIAL_REGS & CPUMCTX_EXTRN_DR7); +#else + AssertCompile(!(VMX_HC_EXIT_IO_INSTR_INITIAL_REGS & CPUMCTX_EXTRN_DR7)); + rc = vmxHCImportGuestState(pVCpu, pVmcsInfo); + AssertRCReturn(rc, rc); +#endif + + /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the + * execution engines about whether hyper BPs and such are pending. */ + uint32_t const uDr7 = pCtx->dr[7]; + if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK) + && X86_DR7_ANY_RW_IO(uDr7) + && (pCtx->cr4 & X86_CR4_DE)) + || DBGFBpIsHwIoArmed(pVM))) + { + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxIoCheck); + +#ifndef IN_NEM_DARWIN + /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */ + VMMRZCallRing3Disable(pVCpu); + HM_DISABLE_PREEMPT(pVCpu); + + bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */); + + VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue); + if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP) + { + /* Raise #DB. */ + if (fIsGuestDbgActive) + ASMSetDR6(pCtx->dr[6]); + if (pCtx->dr[7] != uDr7) + VCPU_2_VMXSTATE(pVCpu).fCtxChanged |= HM_CHANGED_GUEST_DR7; + + vmxHCSetPendingXcptDB(pVCpu); + } + /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST], + however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */ + else if ( rcStrict2 != VINF_SUCCESS + && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict)) + rcStrict = rcStrict2; + AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE); + + HM_RESTORE_PREEMPT(); + VMMRZCallRing3Enable(pVCpu); +#else + /** @todo */ +#endif + } + } + +#ifdef VBOX_STRICT + if ( rcStrict == VINF_IOM_R3_IOPORT_READ + || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ) + Assert(!fIOWrite); + else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE + || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE + || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE) + Assert(fIOWrite); + else + { +# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST + * statuses, that the VMM device and some others may return. See + * IOM_SUCCESS() for guidance. */ + AssertMsg( RT_FAILURE(rcStrict) + || rcStrict == VINF_SUCCESS + || rcStrict == VINF_EM_RAW_EMULATE_INSTR + || rcStrict == VINF_EM_DBG_BREAKPOINT + || rcStrict == VINF_EM_RAW_GUEST_TRAP + || rcStrict == VINF_EM_RAW_TO_R3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); +# endif + } +#endif + STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1); + } + else + { + /* + * Frequent exit or something needing probing. Get state and call EMHistoryExec. + */ + int rc2 = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc2, rc2); + STAM_COUNTER_INC(!fIOString ? fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIORead + : fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringRead); + Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "", + fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize)); + + rcStrict = EMHistoryExec(pVCpu, pExitRec, 0); + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + + Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + } + return rcStrict; +} + + +/** + * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional + * VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* Check if this task-switch occurred while delivery an event through the guest IDT. */ + vmxHCReadToTransient(pVCpu, pVmxTransient); + if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo)) + { + uint32_t uErrCode; + if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + uErrCode = pVmxTransient->uIdtVectoringErrorCode; + } + else + uErrCode = 0; + + RTGCUINTPTR GCPtrFaultAddress; + if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo)) + GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2; + else + GCPtrFaultAddress = 0; + + vmxHCReadToTransient(pVCpu, pVmxTransient); + + vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo), + pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress); + + Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo), + VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo))); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch); + return VINF_EM_RAW_INJECT_TRPM_EVENT; + } + } + + /* Fall back to the interpreter to emulate the task-switch. */ + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch); + return VERR_EM_INTERPRETER; +} + + +/** + * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG; + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls); + AssertRC(rc); + return VINF_EM_DBG_STEPPED; +} + + +/** + * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitApicAccess); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_EXIT_INTERRUPTION_INFO + | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient); + + /* + * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. + */ + VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */ + if (RT_UNLIKELY(VCPU_2_VMXSTATE(pVCpu).Event.fPending)) + { + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret); + return VINF_EM_RAW_INJECT_TRPM_EVENT; + } + } + else + { + Assert(rcStrict != VINF_HM_DOUBLE_FAULT); + return rcStrict; + } + + /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */ + uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual); + switch (uAccessType) + { +#ifndef IN_NEM_DARWIN + case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE: + case VMX_APIC_ACCESS_TYPE_LINEAR_READ: + { + AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW) + || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR, + ("vmxHCExitApicAccess: can't access TPR offset while using TPR shadowing.\n")); + + RTGCPHYS GCPhys = VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */ + GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; + GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual); + Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys, + VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual))); + + rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu, + uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys); + Log4Func(("IOMR0MmioPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + if ( rcStrict == VINF_SUCCESS + || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT + || rcStrict == VERR_PAGE_NOT_PRESENT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS + | HM_CHANGED_GUEST_APIC_TPR); + rcStrict = VINF_SUCCESS; + } + break; + } +#else + /** @todo */ +#endif + + default: + { + Log4Func(("uAccessType=%#x\n", uAccessType)); + rcStrict = VINF_EM_RAW_EMULATE_INSTR; + break; + } + } + + if (rcStrict != VINF_SUCCESS) + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchApicAccessToR3); + return rcStrict; +} + + +/** + * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional + * VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + + /* + * We might also get this VM-exit if the nested-guest isn't intercepting MOV DRx accesses. + * In such a case, rather than disabling MOV DRx intercepts and resuming execution, we + * must emulate the MOV DRx access. + */ + if (!pVmxTransient->fIsNestedGuest) + { + /* We should -not- get this VM-exit if the guest's debug registers were active. */ + if ( pVmxTransient->fWasGuestDebugStateActive +#ifdef VMX_WITH_MAYBE_ALWAYS_INTERCEPT_MOV_DRX + && !pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fAlwaysInterceptMovDRx +#endif + ) + { + AssertMsgFailed(("Unexpected MOV DRx exit\n")); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason); + } + + if ( !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction + && !pVmxTransient->fWasHyperDebugStateActive) + { + Assert(!DBGFIsStepping(pVCpu)); + Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB)); + + /* Whether we disable intercepting MOV DRx instructions and resume + the current one, or emulate it and keep intercepting them is + configurable. Though it usually comes down to whether there are + any new DR6 & DR7 bits (RTM) we want to hide from the guest. */ +#ifdef VMX_WITH_MAYBE_ALWAYS_INTERCEPT_MOV_DRX + bool const fResumeInstruction = !pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fAlwaysInterceptMovDRx; +#else + bool const fResumeInstruction = true; +#endif + if (fResumeInstruction) + { + pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT; + int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls); + AssertRC(rc); + } + +#ifndef IN_NEM_DARWIN + /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */ + VMMRZCallRing3Disable(pVCpu); + HM_DISABLE_PREEMPT(pVCpu); + + /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */ + CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */); + Assert(CPUMIsGuestDebugStateActive(pVCpu)); + + HM_RESTORE_PREEMPT(); + VMMRZCallRing3Enable(pVCpu); +#else + CPUMR3NemActivateGuestDebugState(pVCpu); + Assert(CPUMIsGuestDebugStateActive(pVCpu)); + Assert(!CPUMIsHyperDebugStateActive(pVCpu)); +#endif + + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxContextSwitch); + if (fResumeInstruction) + { +#ifdef VBOX_WITH_STATISTICS + vmxHCReadToTransient(pVCpu, pVmxTransient); + if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE) + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite); + else + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead); +#endif + return VINF_SUCCESS; + } + } + } + + /* + * Import state. We must have DR7 loaded here as it's always consulted, + * both for reading and writing. The other debug registers are never + * exported as such. + */ + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK + | CPUMCTX_EXTRN_GPRS_MASK + | CPUMCTX_EXTRN_DR7>(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + uint8_t const iGReg = VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual); + uint8_t const iDrReg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual); + Log4Func(("cs:rip=%#04x:%08RX64 r%d %s dr%d\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, iGReg, + VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE ? "->" : "<-", iDrReg)); + + VBOXSTRICTRC rcStrict; + if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE) + { + /* + * Write DRx register. + */ + rcStrict = IEMExecDecodedMovDRxWrite(pVCpu, pVmxTransient->cbExitInstr, iDrReg, iGReg); + AssertMsg( rcStrict == VINF_SUCCESS + || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + + if (rcStrict == VINF_SUCCESS) + { + /** @todo r=bird: Not sure why we always flag DR7 as modified here, but I've + * kept it for now to avoid breaking something non-obvious. */ + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS + | HM_CHANGED_GUEST_DR7); + /* Update the DR6 register if guest debug state is active, otherwise we'll + trash it when calling CPUMR0DebugStateMaybeSaveGuestAndRestoreHost. */ + if (iDrReg == 6 && CPUMIsGuestDebugStateActive(pVCpu)) + ASMSetDR6(pVCpu->cpum.GstCtx.dr[6]); + Log4Func(("r%d=%#RX64 => dr%d=%#RX64\n", iGReg, pVCpu->cpum.GstCtx.aGRegs[iGReg].u, + iDrReg, pVCpu->cpum.GstCtx.dr[iDrReg])); + } + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite); + } + else + { + /* + * Read DRx register into a general purpose register. + */ + rcStrict = IEMExecDecodedMovDRxRead(pVCpu, pVmxTransient->cbExitInstr, iGReg, iDrReg); + AssertMsg( rcStrict == VINF_SUCCESS + || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + + if (rcStrict == VINF_SUCCESS) + { + if (iGReg == X86_GREG_xSP) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS + | HM_CHANGED_GUEST_RSP); + else + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + } + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead); + } + + return rcStrict; +} + + +/** + * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG). + * Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + +#ifndef IN_NEM_DARWIN + Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging); + + vmxHCReadToTransient< HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_EXIT_INTERRUPTION_INFO + | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE + | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient); + + /* + * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. + */ + VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + /* + * In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to + * instruction emulation to inject the original event. Otherwise, injecting the original event + * using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again. + */ + if (!VCPU_2_VMXSTATE(pVCpu).Event.fPending) + { /* likely */ } + else + { + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret); +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX + /** @todo NSTVMX: Think about how this should be handled. */ + if (pVmxTransient->fIsNestedGuest) + return VERR_VMX_IPE_3; +# endif + return VINF_EM_RAW_INJECT_TRPM_EVENT; + } + } + else + { + Assert(rcStrict != VINF_HM_DOUBLE_FAULT); + return rcStrict; + } + + /* + * Get sufficient state and update the exit history entry. + */ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr; + PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu, + EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO), + pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base); + if (!pExitRec) + { + /* + * If we succeed, resume guest execution. + * If we fail in interpreting the instruction because we couldn't get the guest physical address + * of the page containing the instruction via the guest's page tables (we would invalidate the guest page + * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this + * weird case. See @bugref{6043}. + */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); +/** @todo bird: We can probably just go straight to IOM here and assume that + * it's MMIO, then fall back on PGM if that hunch didn't work out so + * well. However, we need to address that aliasing workarounds that + * PGMR0Trap0eHandlerNPMisconfig implements. So, some care is needed. + * + * Might also be interesting to see if we can get this done more or + * less locklessly inside IOM. Need to consider the lookup table + * updating and use a bit more carefully first (or do all updates via + * rendezvous) */ + rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, &pVCpu->cpum.GstCtx, GCPhys, UINT32_MAX); + Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict))); + if ( rcStrict == VINF_SUCCESS + || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT + || rcStrict == VERR_PAGE_NOT_PRESENT) + { + /* Successfully handled MMIO operation. */ + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS + | HM_CHANGED_GUEST_APIC_TPR); + rcStrict = VINF_SUCCESS; + } + } + else + { + /* + * Frequent exit or something needing probing. Call EMHistoryExec. + */ + Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys)); + + rcStrict = EMHistoryExec(pVCpu, pExitRec, 0); + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + + Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + } + return rcStrict; +#else + AssertFailed(); + return VERR_VMX_IPE_3; /* Should never happen with Apple HV in R3. */ +#endif +} + + +/** + * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional + * VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); +#ifndef IN_NEM_DARWIN + Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_EXIT_INTERRUPTION_INFO + | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE + | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient); + + /* + * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. + */ + VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + /* + * If delivery of an event causes an EPT violation (true nested #PF and not MMIO), + * we shall resolve the nested #PF and re-inject the original event. + */ + if (VCPU_2_VMXSTATE(pVCpu).Event.fPending) + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflectNPF); + } + else + { + Assert(rcStrict != VINF_HM_DOUBLE_FAULT); + return rcStrict; + } + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr; + uint64_t const uExitQual = pVmxTransient->uExitQual; + AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual)); + + RTGCUINT uErrorCode = 0; + if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH) + uErrorCode |= X86_TRAP_PF_ID; + if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE) + uErrorCode |= X86_TRAP_PF_RW; + if (uExitQual & (VMX_EXIT_QUAL_EPT_ENTRY_READ | VMX_EXIT_QUAL_EPT_ENTRY_WRITE | VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE)) + uErrorCode |= X86_TRAP_PF_P; + + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%08RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip)); + + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + + /* + * Handle the pagefault trap for the nested shadow table. + */ + TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode); + rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, pCtx, GCPhys); + TRPMResetTrap(pVCpu); + + /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */ + if ( rcStrict == VINF_SUCCESS + || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT + || rcStrict == VERR_PAGE_NOT_PRESENT) + { + /* Successfully synced our nested page tables. */ + STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitReasonNpf); + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS); + return VINF_SUCCESS; + } + Log4Func(("EPT return to ring-3 rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; + +#else /* IN_NEM_DARWIN */ + PVM pVM = pVCpu->CTX_SUFF(pVM); + uint64_t const uHostTsc = ASMReadTSC(); RT_NOREF(uHostTsc); + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient); + vmxHCImportGuestRip(pVCpu); + vmxHCImportGuestSegReg(pVCpu); + + /* + * Ask PGM for information about the given GCPhys. We need to check if we're + * out of sync first. + */ + NEMHCDARWINHMACPCCSTATE State = { RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE), + false, + false }; + PGMPHYSNEMPAGEINFO Info; + int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pVmxTransient->uGuestPhysicalAddr, State.fWriteAccess, &Info, + nemR3DarwinHandleMemoryAccessPageCheckerCallback, &State); + if (RT_SUCCESS(rc)) + { + if (Info.fNemProt & ( RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE) + ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ)) + { + if (State.fCanResume) + { + Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt, + Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "", + State.fDidSomething ? "" : " no-change")); + EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS), + pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc); + return VINF_SUCCESS; + } + } + + Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt, + Info.fHasHandlers ? " handlers" : "", Info.fZeroPage ? " zero-pg" : "", + State.fDidSomething ? "" : " no-change")); + } + else + Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + pVmxTransient->uGuestPhysicalAddr, rc, State.fDidSomething ? " modified-backing" : "")); + + /* + * Emulate the memory access, either access handler or special memory. + */ + PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, + RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE) + ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE) + : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ), + pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc); + + rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + VBOXSTRICTRC rcStrict; + if (!pExitRec) + rcStrict = IEMExecOne(pVCpu); + else + { + /* Frequent access or probing. */ + rcStrict = EMHistoryExec(pVCpu, pExitRec, 0); + Log4(("MemExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + } + + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + + Log4Func(("EPT return rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); + return rcStrict; +#endif /* IN_NEM_DARWIN */ +} + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + +/** + * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP + | CPUMCTX_EXTRN_SREG_MASK + | CPUMCTX_EXTRN_HWVIRT + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr); + + VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH, + otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */ + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z); + VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH); + STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)) + rcStrict = VINF_VMX_VMLAUNCH_VMRESUME; + } + Assert(rcStrict != VINF_IEM_RAISED_XCPT); + return rcStrict; +} + + +/** + * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP + | CPUMCTX_EXTRN_SREG_MASK + | CPUMCTX_EXTRN_HWVIRT + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr); + + VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP + | CPUMCTX_EXTRN_SREG_MASK + | CPUMCTX_EXTRN_HWVIRT + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr); + + VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* + * Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and + * thus might not need to import the shadow VMCS state, it's safer just in case + * code elsewhere dares look at unsynced VMCS fields. + */ + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP + | CPUMCTX_EXTRN_SREG_MASK + | CPUMCTX_EXTRN_HWVIRT + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand) + HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr); + + VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + +# if 0 //ndef IN_NEM_DARWIN /** @todo this needs serious tuning still, slows down things enormously. */ + /* Try for exit optimization. This is on the following instruction + because it would be a waste of time to have to reinterpret the + already decoded vmwrite instruction. */ + PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndType(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_VMREAD)); + if (pExitRec) + { + /* Frequent access or probing. */ + rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); + AssertRCReturn(rc, rc); + + rcStrict = EMHistoryExec(pVCpu, pExitRec, 0); + Log4(("vmread/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n", + pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, + VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + } +# endif + } + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME, + otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */ + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z); + VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME); + STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); + if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)) + rcStrict = VINF_VMX_VMLAUNCH_VMRESUME; + } + Assert(rcStrict != VINF_IEM_RAISED_XCPT); + return rcStrict; +} + + +/** + * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* + * Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook + * gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it + * flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here. + */ + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP + | CPUMCTX_EXTRN_SREG_MASK + | CPUMCTX_EXTRN_HWVIRT + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand) + HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr); + + VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_CR4 + | CPUMCTX_EXTRN_HWVIRT + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_HWVIRT); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP + | CPUMCTX_EXTRN_SREG_MASK + | CPUMCTX_EXTRN_HWVIRT + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr); + + VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP + | CPUMCTX_EXTRN_SREG_MASK + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr); + + VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT +/** + * VM-exit handler for INVEPT (VMX_EXIT_INVEPT). Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitInvept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState< CPUMCTX_EXTRN_RSP + | CPUMCTX_EXTRN_SREG_MASK + | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason); + + VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr); + + VBOXSTRICTRC rcStrict = IEMExecDecodedInvept(pVCpu, &ExitInfo); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS); + else if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} +# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ +/** @} */ + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX +/** @name Nested-guest VM-exit handlers. + * @{ + */ +/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */ +/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */ +/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */ + +/** + * Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). + * Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient(pVCpu, pVmxTransient); + + uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo; + uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo); + Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)); + + switch (uExitIntType) + { +# ifndef IN_NEM_DARWIN + /* + * Physical NMIs: + * We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host. + */ + case VMX_EXIT_INT_INFO_TYPE_NMI: + return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo); +# endif + + /* + * Hardware exceptions, + * Software exceptions, + * Privileged software exceptions: + * Figure out if the exception must be delivered to the guest or the nested-guest. + */ + case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: + case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: + case VMX_EXIT_INT_INFO_TYPE_HW_XCPT: + { + vmxHCReadToTransient< HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient); + + PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + if (CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo), pVmxTransient->uExitIntErrorCode)) + { + /* Exit qualification is required for debug and page-fault exceptions. */ + vmxHCReadToTransient(pVCpu, pVmxTransient); + + /* + * For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged + * software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction + * length. However, if delivery of a software interrupt, software exception or privileged + * software exception causes a VM-exit, that too provides the VM-exit instruction length. + */ + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient); + VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT(pVmxTransient->uExitIntInfo, + pVmxTransient->uExitIntErrorCode, + pVmxTransient->uIdtVectoringInfo, + pVmxTransient->uIdtVectoringErrorCode); +#ifdef DEBUG_ramshankar + vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL); + Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n", + pVmxTransient->uExitIntInfo, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual)); + if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo)) + Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n", + pVmxTransient->uIdtVectoringInfo, pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2)); +#endif + return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo); + } + + /* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in vmxHCExitXcptPF. */ + Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging); + return vmxHCExitXcpt(pVCpu, pVmxTransient); + } + + /* + * Software interrupts: + * VM-exits cannot be caused by software interrupts. + * + * External interrupts: + * This should only happen when "acknowledge external interrupts on VM-exit" + * control is set. However, we never set this when executing a guest or + * nested-guest. For nested-guests it is emulated while injecting interrupts into + * the guest. + */ + case VMX_EXIT_INT_INFO_TYPE_SW_INT: + case VMX_EXIT_INT_INFO_TYPE_EXT_INT: + default: + { + VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo; + return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE; + } + } +} + + +/** + * Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). + * Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + return IEMExecVmxVmexitTripleFault(pVCpu); +} + + +/** + * Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW). + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT)) + return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */); + return vmxHCExitIntWindow(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW). + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT)) + return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */); + return vmxHCExitIntWindow(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). + * Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient); + + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient); + VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo, + pVmxTransient->uIdtVectoringErrorCode); + return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo); +} + + +/** + * Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + return vmxHCExitHlt(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT)) + { + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient); + return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return vmxHCExitInvlpg(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + return vmxHCExitRdpmc(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE + * (VMX_EXIT_VMWRITE). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + Assert( pVmxTransient->uExitReason == VMX_EXIT_VMREAD + || pVmxTransient->uExitReason == VMX_EXIT_VMWRITE); + + vmxHCReadToTransient(pVCpu, pVmxTransient); + + uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2; + Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs)); + uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64; + + HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER); + if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)) + u64VmcsField &= UINT64_C(0xffffffff); + + if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField)) + { + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + + if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD) + return vmxHCExitVmread(pVCpu, pVmxTransient); + return vmxHCExitVmwrite(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + + return vmxHCExitRdtsc(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). + * Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + + VBOXSTRICTRC rcStrict; + uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual); + switch (uAccessType) + { + case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: + { + uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual); + uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual); + Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs)); + uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64; + + bool fIntercept; + switch (iCrReg) + { + case 0: + case 4: + fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX); + break; + + case 3: + fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX); + break; + + case 8: + fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT); + break; + + default: + fIntercept = false; + break; + } + if (fIntercept) + { + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient); + rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + else + { + int const rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg); + } + break; + } + + case VMX_EXIT_QUAL_CRX_ACCESS_READ: + { + /* + * CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking). + * CR2 reads do not cause a VM-exit. + * CR3 reads cause a VM-exit depending on the "CR3 store exiting" control. + * CR8 reads cause a VM-exit depending on the "CR8 store exiting" control. + */ + uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual); + if ( iCrReg == 3 + || iCrReg == 8) + { + static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0, + 0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT }; + uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg]; + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept)) + { + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient); + rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + else + { + uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual); + rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg); + } + } + else + { + AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg)); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg); + } + break; + } + + case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: + { + PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs; + uint64_t const uGstHostMask = pVmcsNstGst->u64Cr0Mask.u; + uint64_t const uReadShadow = pVmcsNstGst->u64Cr0ReadShadow.u; + if ( (uGstHostMask & X86_CR0_TS) + && (uReadShadow & X86_CR0_TS)) + { + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient); + rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + else + rcStrict = vmxHCExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr); + break; + } + + case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */ + { + RTGCPTR GCPtrEffDst; + uint16_t const uNewMsw = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual); + bool const fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual); + if (fMemOperand) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + GCPtrEffDst = pVmxTransient->uGuestLinearAddr; + } + else + GCPtrEffDst = NIL_RTGCPTR; + + if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw)) + { + VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient); + ExitInfo.u64GuestLinearAddr = GCPtrEffDst; + rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + else + rcStrict = vmxHCExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst); + break; + } + + default: + { + AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType)); + HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType); + } + } + + if (rcStrict == VINF_IEM_RAISED_XCPT) + { + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); + rcStrict = VINF_SUCCESS; + } + return rcStrict; +} + + +/** + * Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). + * Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT)) + { + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient); + return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return vmxHCExitMovDRx(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). + * Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient(pVCpu, pVmxTransient); + + uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual); + uint8_t const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual); + AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1); + + static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses in bytes. */ + uint8_t const cbAccess = s_aIOSizes[uIOSize]; + if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess)) + { + /* + * IN/OUT instruction: + * - Provides VM-exit instruction length. + * + * INS/OUTS instruction: + * - Provides VM-exit instruction length. + * - Provides Guest-linear address. + * - Optionally provides VM-exit instruction info (depends on CPU feature). + */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + vmxHCReadToTransient(pVCpu, pVmxTransient); + + /* Make sure we don't use stale/uninitialized VMX-transient info. below. */ + pVmxTransient->ExitInstrInfo.u = 0; + pVmxTransient->uGuestLinearAddr = 0; + + bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo; + bool const fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual); + if (fIOString) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + if (fVmxInsOutsInfo) + { + Assert(RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */ + vmxHCReadToTransient(pVCpu, pVmxTransient); + } + } + + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_AND_LIN_ADDR_FROM_TRANSIENT(pVmxTransient); + return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return vmxHCExitIoInstr(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR). + */ +HMVMX_EXIT_DECL vmxHCExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + uint32_t fMsrpm; + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS)) + fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx); + else + fMsrpm = VMXMSRPM_EXIT_RD; + + if (fMsrpm & VMXMSRPM_EXIT_RD) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + return vmxHCExitRdmsr(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR). + */ +HMVMX_EXIT_DECL vmxHCExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + uint32_t fMsrpm; + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS)) + fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx); + else + fMsrpm = VMXMSRPM_EXIT_WR; + + if (fMsrpm & VMXMSRPM_EXIT_WR) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + return vmxHCExitWrmsr(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + return vmxHCExitMwait(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional + * VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */ + vmxHCReadToTransient(pVCpu, pVmxTransient); + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient); + return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo); +} + + +/** + * Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + return vmxHCExitMonitor(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /** @todo NSTVMX: Think about this more. Does the outer guest need to intercept + * PAUSE when executing a nested-guest? If it does not, we would not need + * to check for the intercepts here. Just call VM-exit... */ + + /* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */ + if ( CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT) + || CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + return vmxHCExitPause(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for when the TPR value is lowered below the + * specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit. + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient); + return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo); + } + return vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional + * VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient); + + Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS)); + + Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual), + VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual))); + + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient); + VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo, + pVmxTransient->uIdtVectoringErrorCode); + return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo); +} + + +/** + * Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE). + * Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT)); + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual); +} + + +/** + * Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI). + * Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY)); + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual); +} + + +/** + * Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT)) + { + Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP)); + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + return vmxHCExitRdtscp(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit. + */ +HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); + } + return vmxHCExitWbinvd(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT)) + { + Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID)); + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_INFO + | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient); + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); + } + return vmxHCExitInvpcid(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for invalid-guest state + * (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + + /* + * Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM. + * So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code. + * Handle it like it's in an invalid guest state of the outer guest. + * + * When the fast path is implemented, this should be changed to cause the corresponding + * nested-guest VM-exit. + */ + return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for instructions that cause VM-exits unconditionally + * and only provide the instruction length. + * + * Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + +#ifdef VBOX_STRICT + PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + switch (pVmxTransient->uExitReason) + { + case VMX_EXIT_ENCLS: + Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT)); + break; + + case VMX_EXIT_VMFUNC: + Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC)); + break; + } +#endif + + vmxHCReadToTransient(pVCpu, pVmxTransient); + return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr); +} + + +/** + * Nested-guest VM-exit handler for instructions that provide instruction length as + * well as more information. + * + * Unconditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + +# ifdef VBOX_STRICT + PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + switch (pVmxTransient->uExitReason) + { + case VMX_EXIT_GDTR_IDTR_ACCESS: + case VMX_EXIT_LDTR_TR_ACCESS: + Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT)); + break; + + case VMX_EXIT_RDRAND: + Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT)); + break; + + case VMX_EXIT_RDSEED: + Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT)); + break; + + case VMX_EXIT_XSAVES: + case VMX_EXIT_XRSTORS: + /** @todo NSTVMX: Verify XSS-bitmap. */ + Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS)); + break; + + case VMX_EXIT_UMWAIT: + case VMX_EXIT_TPAUSE: + Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT)); + Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE)); + break; + + case VMX_EXIT_LOADIWKEY: + Assert(CPUMIsGuestVmxProcCtls3Set(pCtx, VMX_PROC_CTLS3_LOADIWKEY_EXIT)); + break; + } +# endif + + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient); + VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient); + return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo); +} + +# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT + +/** + * Nested-guest VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). + * Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitEptViolationNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT)) + { + vmxHCReadToTransient< HMVMX_READ_EXIT_QUALIFICATION + | HMVMX_READ_EXIT_INSTR_LEN + | HMVMX_READ_EXIT_INTERRUPTION_INFO + | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE + | HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE + | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + /* + * If it's our VMEXIT, we're responsible for re-injecting any event which delivery + * might have triggered this VMEXIT. If we forward the problem to the inner VMM, + * it's its problem to deal with that issue and we'll clear the recovered event. + */ + VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /*likely*/ } + else + { + Assert(rcStrict != VINF_HM_DOUBLE_FAULT); + return rcStrict; + } + uint32_t const fClearEventOnForward = VCPU_2_VMXSTATE(pVCpu).Event.fPending; /* paranoia. should not inject events below. */ + + RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr; + uint64_t const uExitQual = pVmxTransient->uExitQual; + + RTGCPTR GCPtrNestedFault; + bool const fIsLinearAddrValid = RT_BOOL(uExitQual & VMX_EXIT_QUAL_EPT_LINEAR_ADDR_VALID); + if (fIsLinearAddrValid) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + GCPtrNestedFault = pVmxTransient->uGuestLinearAddr; + } + else + GCPtrNestedFault = 0; + + RTGCUINT const uErr = ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH) ? X86_TRAP_PF_ID : 0) + | ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE) ? X86_TRAP_PF_RW : 0) + | ((uExitQual & ( VMX_EXIT_QUAL_EPT_ENTRY_READ + | VMX_EXIT_QUAL_EPT_ENTRY_WRITE + | VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE)) ? X86_TRAP_PF_P : 0); + + PGMPTWALK Walk; + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, uErr, pCtx, GCPhysNestedFault, + fIsLinearAddrValid, GCPtrNestedFault, &Walk); + Log7Func(("PGM (uExitQual=%#RX64, %RGp, %RGv) -> %Rrc (fFailed=%d)\n", + uExitQual, GCPhysNestedFault, GCPtrNestedFault, VBOXSTRICTRC_VAL(rcStrict), Walk.fFailed)); + if (RT_SUCCESS(rcStrict)) + return rcStrict; + + if (fClearEventOnForward) + VCPU_2_VMXSTATE(pVCpu).Event.fPending = false; + + VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo, + pVmxTransient->uIdtVectoringErrorCode); + if (Walk.fFailed & PGM_WALKFAIL_EPT_VIOLATION) + { + VMXVEXITINFO const ExitInfo + = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_AND_GST_ADDRESSES(VMX_EXIT_EPT_VIOLATION, + pVmxTransient->uExitQual, + pVmxTransient->cbExitInstr, + pVmxTransient->uGuestLinearAddr, + pVmxTransient->uGuestPhysicalAddr); + return IEMExecVmxVmexitEptViolation(pVCpu, &ExitInfo, &ExitEventInfo); + } + + AssertMsgReturn(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG, + ("uErr=%#RX32 uExitQual=%#RX64 GCPhysNestedFault=%#RGp GCPtrNestedFault=%#RGv\n", + (uint32_t)uErr, uExitQual, GCPhysNestedFault, GCPtrNestedFault), + rcStrict); + return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo); + } + + return vmxHCExitEptViolation(pVCpu, pVmxTransient); +} + + +/** + * Nested-guest VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG). + * Conditional VM-exit. + */ +HMVMX_EXIT_DECL vmxHCExitEptMisconfigNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient) +{ + HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient); + Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging); + + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + + PGMPTWALK Walk; + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr; + VBOXSTRICTRC rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, X86_TRAP_PF_RSVD, pCtx, + GCPhysNestedFault, false /* fIsLinearAddrValid */, + 0 /* GCPtrNestedFault */, &Walk); + if (RT_SUCCESS(rcStrict)) + { + AssertMsgFailed(("Shouldn't happen with the way we have programmed the EPT shadow tables\n")); + return rcStrict; + } + + AssertMsg(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG, ("GCPhysNestedFault=%#RGp\n", GCPhysNestedFault)); + vmxHCReadToTransient< HMVMX_READ_IDT_VECTORING_INFO + | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient); + + VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo, + pVmxTransient->uIdtVectoringErrorCode); + return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo); + } + + return vmxHCExitEptMisconfig(pVCpu, pVmxTransient); +} + +# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */ + +/** @} */ +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + + +/** @name Execution loop for single stepping, DBGF events and expensive Dtrace + * probes. + * + * The following few functions and associated structure contains the bloat + * necessary for providing detailed debug events and dtrace probes as well as + * reliable host side single stepping. This works on the principle of + * "subclassing" the normal execution loop and workers. We replace the loop + * method completely and override selected helpers to add necessary adjustments + * to their core operation. + * + * The goal is to keep the "parent" code lean and mean, so as not to sacrifice + * any performance for debug and analysis features. + * + * @{ + */ + +/** + * Transient per-VCPU debug state of VMCS and related info. we save/restore in + * the debug run loop. + */ +typedef struct VMXRUNDBGSTATE +{ + /** The RIP we started executing at. This is for detecting that we stepped. */ + uint64_t uRipStart; + /** The CS we started executing with. */ + uint16_t uCsStart; + + /** Whether we've actually modified the 1st execution control field. */ + bool fModifiedProcCtls : 1; + /** Whether we've actually modified the 2nd execution control field. */ + bool fModifiedProcCtls2 : 1; + /** Whether we've actually modified the exception bitmap. */ + bool fModifiedXcptBitmap : 1; + + /** We desire the modified the CR0 mask to be cleared. */ + bool fClearCr0Mask : 1; + /** We desire the modified the CR4 mask to be cleared. */ + bool fClearCr4Mask : 1; + /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */ + uint32_t fCpe1Extra; + /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */ + uint32_t fCpe1Unwanted; + /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */ + uint32_t fCpe2Extra; + /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */ + uint32_t bmXcptExtra; + /** The sequence number of the Dtrace provider settings the state was + * configured against. */ + uint32_t uDtraceSettingsSeqNo; + /** VM-exits to check (one bit per VM-exit). */ + uint32_t bmExitsToCheck[3]; + + /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */ + uint32_t fProcCtlsInitial; + /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */ + uint32_t fProcCtls2Initial; + /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */ + uint32_t bmXcptInitial; +} VMXRUNDBGSTATE; +AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4); +typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE; + + +/** + * Initializes the VMXRUNDBGSTATE structure. + * + * @param pVCpu The cross context virtual CPU structure of the + * calling EMT. + * @param pVmxTransient The VMX-transient structure. + * @param pDbgState The debug state to initialize. + */ +static void vmxHCRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState) +{ + pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip; + pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel; + + pDbgState->fModifiedProcCtls = false; + pDbgState->fModifiedProcCtls2 = false; + pDbgState->fModifiedXcptBitmap = false; + pDbgState->fClearCr0Mask = false; + pDbgState->fClearCr4Mask = false; + pDbgState->fCpe1Extra = 0; + pDbgState->fCpe1Unwanted = 0; + pDbgState->fCpe2Extra = 0; + pDbgState->bmXcptExtra = 0; + pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls; + pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2; + pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap; +} + + +/** + * Updates the VMSC fields with changes requested by @a pDbgState. + * + * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well + * immediately before executing guest code, i.e. when interrupts are disabled. + * We don't check status codes here as we cannot easily assert or return in the + * latter case. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * @param pDbgState The debug state. + */ +static void vmxHCPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState) +{ + /* + * Ensure desired flags in VMCS control fields are set. + * (Ignoring write failure here, as we're committed and it's just debug extras.) + * + * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so + * there should be no stale data in pCtx at this point. + */ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra + || (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted)) + { + pVmcsInfo->u32ProcCtls |= pDbgState->fCpe1Extra; + pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted; + VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls); + Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls)); + pDbgState->fModifiedProcCtls = true; + } + + if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra) + { + pVmcsInfo->u32ProcCtls2 |= pDbgState->fCpe2Extra; + VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2); + Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2)); + pDbgState->fModifiedProcCtls2 = true; + } + + if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra) + { + pVmcsInfo->u32XcptBitmap |= pDbgState->bmXcptExtra; + VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap); + Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap)); + pDbgState->fModifiedXcptBitmap = true; + } + + if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0) + { + pVmcsInfo->u64Cr0Mask = 0; + VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, 0); + Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n")); + } + + if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0) + { + pVmcsInfo->u64Cr4Mask = 0; + VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, 0); + Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n")); + } + + NOREF(pVCpu); +} + + +/** + * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for + * re-entry next time around. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * @param pDbgState The debug state. + * @param rcStrict The return code from executing the guest using single + * stepping. + */ +static VBOXSTRICTRC vmxHCRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState, + VBOXSTRICTRC rcStrict) +{ + /* + * Restore VM-exit control settings as we may not reenter this function the + * next time around. + */ + PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo; + + /* We reload the initial value, trigger what we can of recalculations the + next time around. From the looks of things, that's all that's required atm. */ + if (pDbgState->fModifiedProcCtls) + { + if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu)) + pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */ + int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial); + AssertRC(rc2); + pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial; + } + + /* We're currently the only ones messing with this one, so just restore the + cached value and reload the field. */ + if ( pDbgState->fModifiedProcCtls2 + && pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial) + { + int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial); + AssertRC(rc2); + pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial; + } + + /* If we've modified the exception bitmap, we restore it and trigger + reloading and partial recalculation the next time around. */ + if (pDbgState->fModifiedXcptBitmap) + { + int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pDbgState->bmXcptInitial); + AssertRC(rc2); + pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial; + } + + return rcStrict; +} + + +/** + * Configures VM-exit controls for current DBGF and DTrace settings. + * + * This updates @a pDbgState and the VMCS execution control fields to reflect + * the necessary VM-exits demanded by DBGF and DTrace. + * + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. May update + * fUpdatedTscOffsettingAndPreemptTimer. + * @param pDbgState The debug state. + */ +static void vmxHCPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState) +{ +#ifndef IN_NEM_DARWIN + /* + * Take down the dtrace serial number so we can spot changes. + */ + pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO(); + ASMCompilerBarrier(); +#endif + + /* + * We'll rebuild most of the middle block of data members (holding the + * current settings) as we go along here, so start by clearing it all. + */ + pDbgState->bmXcptExtra = 0; + pDbgState->fCpe1Extra = 0; + pDbgState->fCpe1Unwanted = 0; + pDbgState->fCpe2Extra = 0; + for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++) + pDbgState->bmExitsToCheck[i] = 0; + + /* + * Software interrupts (INT XXh) - no idea how to trigger these... + */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE) + || VBOXVMM_INT_SOFTWARE_ENABLED()) + { + ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI); + } + + /* + * INT3 breakpoints - triggered by #BP exceptions. + */ + if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0) + pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP); + + /* + * Exception bitmap and XCPT events+probes. + */ + for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++) + if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt))) + pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt); + + if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE); + if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB); + if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP); + if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF); + if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR); + if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD); + if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM); + if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF); + if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS); + if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP); + if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS); + if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP); + if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF); + if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF); + if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC); + if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF); + if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE); + if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX); + + if (pDbgState->bmXcptExtra) + ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI); + + /* + * Process events and probes for VM-exits, making sure we get the wanted VM-exits. + * + * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does. + * So, when adding/changing/removing please don't forget to update it. + * + * Some of the macros are picking up local variables to save horizontal space, + * (being able to see it in a table is the lesser evil here). + */ +#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \ + ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \ + || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() ) +#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \ + if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \ + { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \ + ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \ + } else do { } while (0) +#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \ + if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \ + { \ + (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \ + AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \ + ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \ + } else do { } while (0) +#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \ + if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \ + { \ + (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \ + AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \ + ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \ + } else do { } while (0) +#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \ + if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \ + { \ + (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \ + AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \ + ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \ + } else do { } while (0) + + SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */ + SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */ + SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */ + + SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC); + SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD); + SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG); + SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC); + SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON); + + if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) + || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE)) + { + int rc = vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo, + CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR); + AssertRC(rc); + +#if 0 /** @todo fix me */ + pDbgState->fClearCr0Mask = true; + pDbgState->fClearCr4Mask = true; +#endif + if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)) + pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT; + if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE)) + pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT; + pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */ + /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would + require clearing here and in the loop if we start using it. */ + ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX); + } + else + { + if (pDbgState->fClearCr0Mask) + { + pDbgState->fClearCr0Mask = false; + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR0); + } + if (pDbgState->fClearCr4Mask) + { + pDbgState->fClearCr4Mask = false; + ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR4); + } + } + SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX); + + if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) + || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE)) + { + /** @todo later, need to fix handler as it assumes this won't usually happen. */ + ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX); + } + SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX); + + SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR); + SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR); + SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT); + SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR); +#if 0 /** @todo too slow, fix handler. */ + SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT); +#endif + SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE); + + if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT) + || IS_EITHER_ENABLED(pVM, INSTR_SIDT) + || IS_EITHER_ENABLED(pVM, INSTR_LGDT) + || IS_EITHER_ENABLED(pVM, INSTR_LIDT)) + { + pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT; + ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS); + } + SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS); + + if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT) + || IS_EITHER_ENABLED(pVM, INSTR_STR) + || IS_EITHER_ENABLED(pVM, INSTR_LLDT) + || IS_EITHER_ENABLED(pVM, INSTR_LTR)) + { + pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT; + ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS); + } + SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS); + + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT); + SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID); + SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV); + SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND); + SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC); + SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT); + SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */ + SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES); + SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */ + SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS); + +#undef IS_EITHER_ENABLED +#undef SET_ONLY_XBM_IF_EITHER_EN +#undef SET_CPE1_XBM_IF_EITHER_EN +#undef SET_CPEU_XBM_IF_EITHER_EN +#undef SET_CPE2_XBM_IF_EITHER_EN + + /* + * Sanitize the control stuff. + */ + pDbgState->fCpe2Extra &= g_HmMsrs.u.vmx.ProcCtls2.n.allowed1; + if (pDbgState->fCpe2Extra) + pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS; + pDbgState->fCpe1Extra &= g_HmMsrs.u.vmx.ProcCtls.n.allowed1; + pDbgState->fCpe1Unwanted &= ~g_HmMsrs.u.vmx.ProcCtls.n.allowed0; +#ifndef IN_NEM_DARWIN + if (pVCpu->hmr0.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT)) + { + pVCpu->hmr0.s.fDebugWantRdTscExit ^= true; + pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false; + } +#else + if (pVCpu->nem.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT)) + { + pVCpu->nem.s.fDebugWantRdTscExit ^= true; + pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false; + } +#endif + + Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n", + pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra, + pDbgState->fClearCr0Mask ? " clr-cr0" : "", + pDbgState->fClearCr4Mask ? " clr-cr4" : "")); +} + + +/** + * Fires off DBGF events and dtrace probes for a VM-exit, when it's + * appropriate. + * + * The caller has checked the VM-exit against the + * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs + * already, so we don't have to do that either. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @param pVCpu The cross context virtual CPU structure. + * @param pVmxTransient The VMX-transient structure. + * @param uExitReason The VM-exit reason. + * + * @remarks The name of this function is displayed by dtrace, so keep it short + * and to the point. No longer than 33 chars long, please. + */ +static VBOXSTRICTRC vmxHCHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason) +{ + /* + * Translate the event into a DBGF event (enmEvent + uEventArg) and at the + * same time check whether any corresponding Dtrace event is enabled (fDtrace). + * + * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate + * does. Must add/change/remove both places. Same ordering, please. + * + * Added/removed events must also be reflected in the next section + * where we dispatch dtrace events. + */ + bool fDtrace1 = false; + bool fDtrace2 = false; + DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END; + DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END; + uint32_t uEventArg = 0; +#define SET_EXIT(a_EventSubName) \ + do { \ + enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \ + fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \ + } while (0) +#define SET_BOTH(a_EventSubName) \ + do { \ + enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \ + enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \ + fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \ + fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \ + } while (0) + switch (uExitReason) + { + case VMX_EXIT_MTF: + return vmxHCExitMtf(pVCpu, pVmxTransient); + + case VMX_EXIT_XCPT_OR_NMI: + { + uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo); + switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo)) + { + case VMX_EXIT_INT_INFO_TYPE_HW_XCPT: + case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: + case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: + if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST)) + { + if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo)) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + uEventArg = pVmxTransient->uExitIntErrorCode; + } + enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector); + switch (enmEvent1) + { + case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break; + case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break; + case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break; + case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break; + case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break; + case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break; + case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break; + case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break; + case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break; + case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break; + case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break; + case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break; + case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break; + case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break; + case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break; + case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break; + case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break; + case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break; + default: break; + } + } + else + AssertFailed(); + break; + + case VMX_EXIT_INT_INFO_TYPE_SW_INT: + uEventArg = idxVector; + enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE; + fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED(); + break; + } + break; + } + + case VMX_EXIT_TRIPLE_FAULT: + enmEvent1 = DBGFEVENT_TRIPLE_FAULT; + //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED(); + break; + case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break; + case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break; + case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break; + case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break; + case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break; + + /* Instruction specific VM-exits: */ + case VMX_EXIT_CPUID: SET_BOTH(CPUID); break; + case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break; + case VMX_EXIT_HLT: SET_BOTH(HALT); break; + case VMX_EXIT_INVD: SET_BOTH(INVD); break; + case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break; + case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break; + case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break; + case VMX_EXIT_RSM: SET_BOTH(RSM); break; + case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break; + case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break; + case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break; + case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break; + case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break; + case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break; + case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break; + case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break; + case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break; + case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break; + case VMX_EXIT_MOV_CRX: + vmxHCReadToTransient(pVCpu, pVmxTransient); + if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ) + SET_BOTH(CRX_READ); + else + SET_BOTH(CRX_WRITE); + uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual); + break; + case VMX_EXIT_MOV_DRX: + vmxHCReadToTransient(pVCpu, pVmxTransient); + if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) + == VMX_EXIT_QUAL_DRX_DIRECTION_READ) + SET_BOTH(DRX_READ); + else + SET_BOTH(DRX_WRITE); + uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual); + break; + case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break; + case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break; + case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break; + case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break; + case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break; + case VMX_EXIT_GDTR_IDTR_ACCESS: + vmxHCReadToTransient(pVCpu, pVmxTransient); + switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID)) + { + case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break; + case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break; + case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break; + case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break; + } + break; + + case VMX_EXIT_LDTR_TR_ACCESS: + vmxHCReadToTransient(pVCpu, pVmxTransient); + switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID)) + { + case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break; + case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break; + case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break; + case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break; + } + break; + + case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break; + case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break; + case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break; + case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break; + case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break; + case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break; + case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break; + case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break; + case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break; + case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break; + case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break; + + /* Events that aren't relevant at this point. */ + case VMX_EXIT_EXT_INT: + case VMX_EXIT_INT_WINDOW: + case VMX_EXIT_NMI_WINDOW: + case VMX_EXIT_TPR_BELOW_THRESHOLD: + case VMX_EXIT_PREEMPT_TIMER: + case VMX_EXIT_IO_INSTR: + break; + + /* Errors and unexpected events. */ + case VMX_EXIT_INIT_SIGNAL: + case VMX_EXIT_SIPI: + case VMX_EXIT_IO_SMI: + case VMX_EXIT_SMI: + case VMX_EXIT_ERR_INVALID_GUEST_STATE: + case VMX_EXIT_ERR_MSR_LOAD: + case VMX_EXIT_ERR_MACHINE_CHECK: + case VMX_EXIT_PML_FULL: + case VMX_EXIT_VIRTUALIZED_EOI: + break; + + default: + AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason)); + break; + } +#undef SET_BOTH +#undef SET_EXIT + + /* + * Dtrace tracepoints go first. We do them here at once so we don't + * have to copy the guest state saving and stuff a few dozen times. + * Down side is that we've got to repeat the switch, though this time + * we use enmEvent since the probes are a subset of what DBGF does. + */ + if (fDtrace1 || fDtrace2) + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; + switch (enmEvent1) + { + /** @todo consider which extra parameters would be helpful for each probe. */ + case DBGFEVENT_END: break; + case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break; + case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break; + case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break; + case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break; + case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break; + case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break; + case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break; + case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break; + case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break; + case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break; + case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break; + case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break; + case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break; + case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break; + case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break; + case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx, + RT_MAKE_U64(pCtx->eax, pCtx->edx)); break; + case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break; + case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break; + default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break; + } + switch (enmEvent2) + { + /** @todo consider which extra parameters would be helpful for each probe. */ + case DBGFEVENT_END: break; + case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break; + case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break; + case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break; + case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break; + case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break; + case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break; + case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx, + RT_MAKE_U64(pCtx->eax, pCtx->edx)); break; + case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break; + case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break; + default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break; + } + } + + /* + * Fire of the DBGF event, if enabled (our check here is just a quick one, + * the DBGF call will do a full check). + * + * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap. + * Note! If we have to events, we prioritize the first, i.e. the instruction + * one, in order to avoid event nesting. + */ + PVMCC pVM = pVCpu->CTX_SUFF(pVM); + if ( enmEvent1 != DBGFEVENT_END + && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1)) + { + vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + else if ( enmEvent2 != DBGFEVENT_END + && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2)) + { + vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + return VINF_SUCCESS; +} + + +/** + * Single-stepping VM-exit filtering. + * + * This is preprocessing the VM-exits and deciding whether we've gotten far + * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit + * handling is performed. + * + * @returns Strict VBox status code (i.e. informational status codes too). + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + * @param pVmxTransient The VMX-transient structure. + * @param pDbgState The debug state. + */ +DECLINLINE(VBOXSTRICTRC) vmxHCRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState) +{ + /* + * Expensive (saves context) generic dtrace VM-exit probe. + */ + uint32_t const uExitReason = pVmxTransient->uExitReason; + if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()) + { /* more likely */ } + else + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRC(rc); + VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual); + } + +#ifndef IN_NEM_DARWIN + /* + * Check for host NMI, just to get that out of the way. + */ + if (uExitReason != VMX_EXIT_XCPT_OR_NMI) + { /* normally likely */ } + else + { + vmxHCReadToTransient(pVCpu, pVmxTransient); + uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo); + if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI) + return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo); + } +#endif + + /* + * Check for single stepping event if we're stepping. + */ + if (VCPU_2_VMXSTATE(pVCpu).fSingleInstruction) + { + switch (uExitReason) + { + case VMX_EXIT_MTF: + return vmxHCExitMtf(pVCpu, pVmxTransient); + + /* Various events: */ + case VMX_EXIT_XCPT_OR_NMI: + case VMX_EXIT_EXT_INT: + case VMX_EXIT_TRIPLE_FAULT: + case VMX_EXIT_INT_WINDOW: + case VMX_EXIT_NMI_WINDOW: + case VMX_EXIT_TASK_SWITCH: + case VMX_EXIT_TPR_BELOW_THRESHOLD: + case VMX_EXIT_APIC_ACCESS: + case VMX_EXIT_EPT_VIOLATION: + case VMX_EXIT_EPT_MISCONFIG: + case VMX_EXIT_PREEMPT_TIMER: + + /* Instruction specific VM-exits: */ + case VMX_EXIT_CPUID: + case VMX_EXIT_GETSEC: + case VMX_EXIT_HLT: + case VMX_EXIT_INVD: + case VMX_EXIT_INVLPG: + case VMX_EXIT_RDPMC: + case VMX_EXIT_RDTSC: + case VMX_EXIT_RSM: + case VMX_EXIT_VMCALL: + case VMX_EXIT_VMCLEAR: + case VMX_EXIT_VMLAUNCH: + case VMX_EXIT_VMPTRLD: + case VMX_EXIT_VMPTRST: + case VMX_EXIT_VMREAD: + case VMX_EXIT_VMRESUME: + case VMX_EXIT_VMWRITE: + case VMX_EXIT_VMXOFF: + case VMX_EXIT_VMXON: + case VMX_EXIT_MOV_CRX: + case VMX_EXIT_MOV_DRX: + case VMX_EXIT_IO_INSTR: + case VMX_EXIT_RDMSR: + case VMX_EXIT_WRMSR: + case VMX_EXIT_MWAIT: + case VMX_EXIT_MONITOR: + case VMX_EXIT_PAUSE: + case VMX_EXIT_GDTR_IDTR_ACCESS: + case VMX_EXIT_LDTR_TR_ACCESS: + case VMX_EXIT_INVEPT: + case VMX_EXIT_RDTSCP: + case VMX_EXIT_INVVPID: + case VMX_EXIT_WBINVD: + case VMX_EXIT_XSETBV: + case VMX_EXIT_RDRAND: + case VMX_EXIT_INVPCID: + case VMX_EXIT_VMFUNC: + case VMX_EXIT_RDSEED: + case VMX_EXIT_XSAVES: + case VMX_EXIT_XRSTORS: + { + int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); + AssertRCReturn(rc, rc); + if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart + || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart) + return VINF_EM_DBG_STEPPED; + break; + } + + /* Errors and unexpected events: */ + case VMX_EXIT_INIT_SIGNAL: + case VMX_EXIT_SIPI: + case VMX_EXIT_IO_SMI: + case VMX_EXIT_SMI: + case VMX_EXIT_ERR_INVALID_GUEST_STATE: + case VMX_EXIT_ERR_MSR_LOAD: + case VMX_EXIT_ERR_MACHINE_CHECK: + case VMX_EXIT_PML_FULL: + case VMX_EXIT_VIRTUALIZED_EOI: + case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */ + break; + + default: + AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason)); + break; + } + } + + /* + * Check for debugger event breakpoints and dtrace probes. + */ + if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U + && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) ) + { + VBOXSTRICTRC rcStrict = vmxHCHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason); + if (rcStrict != VINF_SUCCESS) + return rcStrict; + } + + /* + * Normal processing. + */ +#ifdef HMVMX_USE_FUNCTION_TABLE + return g_aVMExitHandlers[uExitReason].pfn(pVCpu, pVmxTransient); +#else + return vmxHCHandleExit(pVCpu, pVmxTransient, uExitReason); +#endif +} + +/** @} */ -- cgit v1.2.3