diff options
Diffstat (limited to 'src/VBox/VMM/include/IEMInline.h')
| -rw-r--r-- | src/VBox/VMM/include/IEMInline.h | 2880 |
1 files changed, 2880 insertions, 0 deletions
diff --git a/src/VBox/VMM/include/IEMInline.h b/src/VBox/VMM/include/IEMInline.h new file mode 100644 index 00000000..0d6e3975 --- /dev/null +++ b/src/VBox/VMM/include/IEMInline.h @@ -0,0 +1,2880 @@ +/* $Id: IEMInline.h $ */ +/** @file + * IEM - Interpreted Execution Manager - Inlined Functions. + */ + +/* + * Copyright (C) 2011-2023 Oracle and/or its affiliates. + * + * This file is part of VirtualBox base platform packages, as + * available from https://www.virtualbox.org. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, in version 3 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see <https://www.gnu.org/licenses>. + * + * SPDX-License-Identifier: GPL-3.0-only + */ + +#ifndef VMM_INCLUDED_SRC_include_IEMInline_h +#define VMM_INCLUDED_SRC_include_IEMInline_h +#ifndef RT_WITHOUT_PRAGMA_ONCE +# pragma once +#endif + + + +/** + * Makes status code addjustments (pass up from I/O and access handler) + * as well as maintaining statistics. + * + * @returns Strict VBox status code to pass up. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param rcStrict The status from executing an instruction. + */ +DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT +{ + if (rcStrict != VINF_SUCCESS) + { + if (RT_SUCCESS(rcStrict)) + { + AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST) + || rcStrict == VINF_IOM_R3_IOPORT_READ + || rcStrict == VINF_IOM_R3_IOPORT_WRITE + || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE + || rcStrict == VINF_IOM_R3_MMIO_READ + || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE + || rcStrict == VINF_IOM_R3_MMIO_WRITE + || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE + || rcStrict == VINF_CPUM_R3_MSR_READ + || rcStrict == VINF_CPUM_R3_MSR_WRITE + || rcStrict == VINF_EM_RAW_EMULATE_INSTR + || rcStrict == VINF_EM_RAW_TO_R3 + || rcStrict == VINF_EM_TRIPLE_FAULT + || rcStrict == VINF_GIM_R3_HYPERCALL + /* raw-mode / virt handlers only: */ + || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT + || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT + || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT + || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT + || rcStrict == VINF_SELM_SYNC_GDT + || rcStrict == VINF_CSAM_PENDING_ACTION + || rcStrict == VINF_PATM_CHECK_PATCH_PAGE + /* nested hw.virt codes: */ + || rcStrict == VINF_VMX_VMEXIT + || rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE + || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR + || rcStrict == VINF_SVM_VMEXIT + , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); +/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */ + int32_t const rcPassUp = pVCpu->iem.s.rcPassUp; +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + if ( rcStrict == VINF_VMX_VMEXIT + && rcPassUp == VINF_SUCCESS) + rcStrict = VINF_SUCCESS; + else +#endif +#ifdef VBOX_WITH_NESTED_HWVIRT_SVM + if ( rcStrict == VINF_SVM_VMEXIT + && rcPassUp == VINF_SUCCESS) + rcStrict = VINF_SUCCESS; + else +#endif + if (rcPassUp == VINF_SUCCESS) + pVCpu->iem.s.cRetInfStatuses++; + else if ( rcPassUp < VINF_EM_FIRST + || rcPassUp > VINF_EM_LAST + || rcPassUp < VBOXSTRICTRC_VAL(rcStrict)) + { + Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict))); + pVCpu->iem.s.cRetPassUpStatus++; + rcStrict = rcPassUp; + } + else + { + Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict))); + pVCpu->iem.s.cRetInfStatuses++; + } + } + else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED) + pVCpu->iem.s.cRetAspectNotImplemented++; + else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED) + pVCpu->iem.s.cRetInstrNotImplemented++; + else + pVCpu->iem.s.cRetErrStatuses++; + } + else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS) + { + pVCpu->iem.s.cRetPassUpStatus++; + rcStrict = pVCpu->iem.s.rcPassUp; + } + + return rcStrict; +} + + +/** + * Sets the pass up status. + * + * @returns VINF_SUCCESS. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param rcPassUp The pass up status. Must be informational. + * VINF_SUCCESS is not allowed. + */ +DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT +{ + AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS); + + int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp; + if (rcOldPassUp == VINF_SUCCESS) + pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp); + /* If both are EM scheduling codes, use EM priority rules. */ + else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST + && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST) + { + if (rcPassUp < rcOldPassUp) + { + Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp)); + pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp); + } + else + Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp)); + } + /* Override EM scheduling with specific status code. */ + else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST) + { + Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp)); + pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp); + } + /* Don't override specific status code, first come first served. */ + else + Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp)); + return VINF_SUCCESS; +} + + +/** + * Calculates the CPU mode. + * + * This is mainly for updating IEMCPU::enmCpuMode. + * + * @returns CPU mode. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + */ +DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)) + return IEMMODE_64BIT; + if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */ + return IEMMODE_32BIT; + return IEMMODE_16BIT; +} + + +#if defined(VBOX_INCLUDED_vmm_dbgf_h) || defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */ +/** + * Initializes the execution state. + * + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param fBypassHandlers Whether to bypass access handlers. + * + * @remarks Callers of this must call iemUninitExec() to undo potentially fatal + * side-effects in strict builds. + */ +DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers) RT_NOEXCEPT +{ + IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK); + Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr)); + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr)); + + pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); + pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu); +# ifdef VBOX_STRICT + pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe; + pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe; + pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe; + pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe; + pVCpu->iem.s.fPrefixes = 0xfeedbeef; + pVCpu->iem.s.uRexReg = 127; + pVCpu->iem.s.uRexB = 127; + pVCpu->iem.s.offModRm = 127; + pVCpu->iem.s.uRexIndex = 127; + pVCpu->iem.s.iEffSeg = 127; + pVCpu->iem.s.idxPrefix = 127; + pVCpu->iem.s.uVex3rdReg = 127; + pVCpu->iem.s.uVexLength = 127; + pVCpu->iem.s.fEvexStuff = 127; + pVCpu->iem.s.uFpuOpcode = UINT16_MAX; +# ifdef IEM_WITH_CODE_TLB + pVCpu->iem.s.offInstrNextByte = UINT16_MAX; + pVCpu->iem.s.pbInstrBuf = NULL; + pVCpu->iem.s.cbInstrBuf = UINT16_MAX; + pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX; + pVCpu->iem.s.offCurInstrStart = INT16_MAX; + pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff); +# else + pVCpu->iem.s.offOpcode = 127; + pVCpu->iem.s.cbOpcode = 127; +# endif +# endif /* VBOX_STRICT */ + + pVCpu->iem.s.cActiveMappings = 0; + pVCpu->iem.s.iNextMapping = 0; + pVCpu->iem.s.rcPassUp = VINF_SUCCESS; + pVCpu->iem.s.fBypassHandlers = fBypassHandlers; + pVCpu->iem.s.fDisregardLock = false; + pVCpu->iem.s.fPendingInstructionBreakpoints = false; + pVCpu->iem.s.fPendingDataBreakpoints = false; + pVCpu->iem.s.fPendingIoBreakpoints = false; + if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK) + && pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0)) + { /* likely */ } + else + iemInitPendingBreakpointsSlow(pVCpu); +} +#endif /* VBOX_INCLUDED_vmm_dbgf_h */ + + +#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX) +/** + * Performs a minimal reinitialization of the execution state. + * + * This is intended to be used by VM-exits, SMM, LOADALL and other similar + * 'world-switch' types operations on the CPU. Currently only nested + * hardware-virtualization uses it. + * + * @param pVCpu The cross context virtual CPU structure of the calling EMT. + */ +DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + IEMMODE const enmMode = iemCalcCpuMode(pVCpu); + uint8_t const uCpl = CPUMGetGuestCPL(pVCpu); + + pVCpu->iem.s.uCpl = uCpl; + pVCpu->iem.s.enmCpuMode = enmMode; + pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */ + pVCpu->iem.s.enmEffAddrMode = enmMode; + if (enmMode != IEMMODE_64BIT) + { + pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */ + pVCpu->iem.s.enmEffOpSize = enmMode; + } + else + { + pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT; + pVCpu->iem.s.enmEffOpSize = enmMode; + } + pVCpu->iem.s.iEffSeg = X86_SREG_DS; +# ifndef IEM_WITH_CODE_TLB + /** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */ + pVCpu->iem.s.offOpcode = 0; + pVCpu->iem.s.cbOpcode = 0; +# endif + pVCpu->iem.s.rcPassUp = VINF_SUCCESS; +} +#endif + + +/** + * Counterpart to #iemInitExec that undoes evil strict-build stuff. + * + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + */ +DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */ +#ifdef VBOX_STRICT +# ifdef IEM_WITH_CODE_TLB + NOREF(pVCpu); +# else + pVCpu->iem.s.cbOpcode = 0; +# endif +#else + NOREF(pVCpu); +#endif +} + + +/** + * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter. + * + * Only calling iemRCRawMaybeReenter in raw-mode, obviously. + * + * @returns Fiddled strict vbox status code, ready to return to non-IEM caller. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param rcStrict The status code to fiddle. + */ +DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT +{ + iemUninitExec(pVCpu); + return iemExecStatusCodeFiddling(pVCpu, rcStrict); +} + + +/** + * Macro used by the IEMExec* method to check the given instruction length. + * + * Will return on failure! + * + * @param a_cbInstr The given instruction length. + * @param a_cbMin The minimum length. + */ +#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \ + AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \ + ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH) + + +#ifndef IEM_WITH_SETJMP + +/** + * Fetches the first opcode byte. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param pu8 Where to return the opcode byte. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT +{ + /* + * Check for hardware instruction breakpoints. + */ + if (RT_LIKELY(!pVCpu->iem.s.fPendingInstructionBreakpoints)) + { /* likely */ } + else + { + VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu, + pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else if (rcStrict == VINF_EM_RAW_GUEST_TRAP) + return iemRaiseDebugException(pVCpu); + else + return rcStrict; + } + + /* + * Fetch the first opcode byte. + */ + uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; + *pu8 = pVCpu->iem.s.abOpcode[offOpcode]; + return VINF_SUCCESS; + } + return iemOpcodeGetNextU8Slow(pVCpu, pu8); +} + +#else /* IEM_WITH_SETJMP */ + +/** + * Fetches the first opcode byte, longjmp on error. + * + * @returns The opcode byte. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ + /* + * Check for hardware instruction breakpoints. + */ + if (RT_LIKELY(!pVCpu->iem.s.fPendingInstructionBreakpoints)) + { /* likely */ } + else + { + VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu, + pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base); + if (RT_LIKELY(rcStrict == VINF_SUCCESS)) + { /* likely */ } + else + { + if (rcStrict == VINF_EM_RAW_GUEST_TRAP) + rcStrict = iemRaiseDebugException(pVCpu); + IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)); + } + } + + /* + * Fetch the first opcode byte. + */ +# ifdef IEM_WITH_CODE_TLB + uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; + uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; + if (RT_LIKELY( pbBuf != NULL + && offBuf < pVCpu->iem.s.cbInstrBuf)) + { + pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1; + return pbBuf[offBuf]; + } +# else + uintptr_t offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; + return pVCpu->iem.s.abOpcode[offOpcode]; + } +# endif + return iemOpcodeGetNextU8SlowJmp(pVCpu); +} + +#endif /* IEM_WITH_SETJMP */ + +/** + * Fetches the first opcode byte, returns/throws automatically on failure. + * + * @param a_pu8 Where to return the opcode byte. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \ + if (rcStrict2 == VINF_SUCCESS) \ + { /* likely */ } \ + else \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu)) +#endif /* IEM_WITH_SETJMP */ + + +#ifndef IEM_WITH_SETJMP + +/** + * Fetches the next opcode byte. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param pu8 Where to return the opcode byte. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT +{ + uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; + *pu8 = pVCpu->iem.s.abOpcode[offOpcode]; + return VINF_SUCCESS; + } + return iemOpcodeGetNextU8Slow(pVCpu, pu8); +} + +#else /* IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode byte, longjmp on error. + * + * @returns The opcode byte. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IEM_WITH_CODE_TLB + uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; + uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; + if (RT_LIKELY( pbBuf != NULL + && offBuf < pVCpu->iem.s.cbInstrBuf)) + { + pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1; + return pbBuf[offBuf]; + } +# else + uintptr_t offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; + return pVCpu->iem.s.abOpcode[offOpcode]; + } +# endif + return iemOpcodeGetNextU8SlowJmp(pVCpu); +} + +#endif /* IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode byte, returns automatically on failure. + * + * @param a_pu8 Where to return the opcode byte. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \ + if (rcStrict2 == VINF_SUCCESS) \ + { /* likely */ } \ + else \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu)) +#endif /* IEM_WITH_SETJMP */ + + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next signed byte from the opcode stream. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pi8 Where to return the signed byte. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT +{ + return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8); +} +#endif /* !IEM_WITH_SETJMP */ + + +/** + * Fetches the next signed byte from the opcode stream, returning automatically + * on failure. + * + * @param a_pi8 Where to return the signed byte. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else /* IEM_WITH_SETJMP */ +# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu)) + +#endif /* IEM_WITH_SETJMP */ + + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next signed byte from the opcode stream, extending it to + * unsigned 16-bit. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu16 Where to return the unsigned word. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT +{ + uint8_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode)) + return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16); + + *pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode]; + pVCpu->iem.s.offOpcode = offOpcode + 1; + return VINF_SUCCESS; +} +#endif /* !IEM_WITH_SETJMP */ + +/** + * Fetches the next signed byte from the opcode stream and sign-extending it to + * a word, returning automatically on failure. + * + * @param a_pu16 Where to return the word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu)) +#endif + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next signed byte from the opcode stream, extending it to + * unsigned 32-bit. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32 Where to return the unsigned dword. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT +{ + uint8_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode)) + return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32); + + *pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode]; + pVCpu->iem.s.offOpcode = offOpcode + 1; + return VINF_SUCCESS; +} +#endif /* !IEM_WITH_SETJMP */ + +/** + * Fetches the next signed byte from the opcode stream and sign-extending it to + * a word, returning automatically on failure. + * + * @param a_pu32 Where to return the word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu)) +#endif + + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next signed byte from the opcode stream, extending it to + * unsigned 64-bit. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the unsigned qword. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + uint8_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode)) + return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64); + + *pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode]; + pVCpu->iem.s.offOpcode = offOpcode + 1; + return VINF_SUCCESS; +} +#endif /* !IEM_WITH_SETJMP */ + +/** + * Fetches the next signed byte from the opcode stream and sign-extending it to + * a word, returning automatically on failure. + * + * @param a_pu64 Where to return the word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu)) +#endif + + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next opcode byte. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the + * calling thread. + * @param pu8 Where to return the opcode byte. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT +{ + uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; + pVCpu->iem.s.offModRm = offOpcode; + if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; + *pu8 = pVCpu->iem.s.abOpcode[offOpcode]; + return VINF_SUCCESS; + } + return iemOpcodeGetNextU8Slow(pVCpu, pu8); +} +#else /* IEM_WITH_SETJMP */ +/** + * Fetches the next opcode byte, longjmp on error. + * + * @returns The opcode byte. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IEM_WITH_CODE_TLB + uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; + pVCpu->iem.s.offModRm = offBuf; + uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; + if (RT_LIKELY( pbBuf != NULL + && offBuf < pVCpu->iem.s.cbInstrBuf)) + { + pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1; + return pbBuf[offBuf]; + } +# else + uintptr_t offOpcode = pVCpu->iem.s.offOpcode; + pVCpu->iem.s.offModRm = offOpcode; + if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1; + return pVCpu->iem.s.abOpcode[offOpcode]; + } +# endif + return iemOpcodeGetNextU8SlowJmp(pVCpu); +} +#endif /* IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode byte, which is a ModR/M byte, returns automatically + * on failure. + * + * Will note down the position of the ModR/M byte for VT-x exits. + * + * @param a_pbRm Where to return the RM opcode byte. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \ + if (rcStrict2 == VINF_SUCCESS) \ + { /* likely */ } \ + else \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu)) +#endif /* IEM_WITH_SETJMP */ + + +#ifndef IEM_WITH_SETJMP + +/** + * Fetches the next opcode word. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu16 Where to return the opcode word. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT +{ + uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); +# endif + return VINF_SUCCESS; + } + return iemOpcodeGetNextU16Slow(pVCpu, pu16); +} + +#else /* IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode word, longjmp on error. + * + * @returns The opcode word. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IEM_WITH_CODE_TLB + uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; + uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; + if (RT_LIKELY( pbBuf != NULL + && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf)) + { + pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + return *(uint16_t const *)&pbBuf[offBuf]; +# else + return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]); +# endif + } +# else /* !IEM_WITH_CODE_TLB */ + uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); +# endif + } +# endif /* !IEM_WITH_CODE_TLB */ + return iemOpcodeGetNextU16SlowJmp(pVCpu); +} + +#endif /* IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode word, returns automatically on failure. + * + * @param a_pu16 Where to return the opcode word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu)) +#endif + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next opcode word, zero extending it to a double word. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32 Where to return the opcode double word. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT +{ + uint8_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode)) + return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32); + + *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); + pVCpu->iem.s.offOpcode = offOpcode + 2; + return VINF_SUCCESS; +} +#endif /* !IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode word and zero extends it to a double word, returns + * automatically on failure. + * + * @param a_pu32 Where to return the opcode double word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu)) +#endif + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next opcode word, zero extending it to a quad word. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the opcode quad word. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + uint8_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode)) + return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64); + + *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]); + pVCpu->iem.s.offOpcode = offOpcode + 2; + return VINF_SUCCESS; +} +#endif /* !IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode word and zero extends it to a quad word, returns + * automatically on failure. + * + * @param a_pu64 Where to return the opcode quad word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu)) +#endif + + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next signed word from the opcode stream. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pi16 Where to return the signed word. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT +{ + return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16); +} +#endif /* !IEM_WITH_SETJMP */ + + +/** + * Fetches the next signed word from the opcode stream, returning automatically + * on failure. + * + * @param a_pi16 Where to return the signed word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu)) +#endif + +#ifndef IEM_WITH_SETJMP + +/** + * Fetches the next opcode dword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu32 Where to return the opcode double word. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT +{ + uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3]); +# endif + return VINF_SUCCESS; + } + return iemOpcodeGetNextU32Slow(pVCpu, pu32); +} + +#else /* IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode dword, longjmp on error. + * + * @returns The opcode dword. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IEM_WITH_CODE_TLB + uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; + uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; + if (RT_LIKELY( pbBuf != NULL + && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf)) + { + pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + return *(uint32_t const *)&pbBuf[offBuf]; +# else + return RT_MAKE_U32_FROM_U8(pbBuf[offBuf], + pbBuf[offBuf + 1], + pbBuf[offBuf + 2], + pbBuf[offBuf + 3]); +# endif + } +# else + uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3]); +# endif + } +# endif + return iemOpcodeGetNextU32SlowJmp(pVCpu); +} + +#endif /* IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode dword, returns automatically on failure. + * + * @param a_pu32 Where to return the opcode dword. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu)) +#endif + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next opcode dword, zero extending it to a quad word. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the opcode quad word. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + uint8_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode)) + return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64); + + *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3]); + pVCpu->iem.s.offOpcode = offOpcode + 4; + return VINF_SUCCESS; +} +#endif /* !IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode dword and zero extends it to a quad word, returns + * automatically on failure. + * + * @param a_pu64 Where to return the opcode quad word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu)) +#endif + + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next signed double word from the opcode stream. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pi32 Where to return the signed double word. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT +{ + return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32); +} +#endif + +/** + * Fetches the next signed double word from the opcode stream, returning + * automatically on failure. + * + * @param a_pi32 Where to return the signed double word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu)) +#endif + +#ifndef IEM_WITH_SETJMP +/** + * Fetches the next opcode dword, sign extending it into a quad word. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the opcode quad word. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + uint8_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode)) + return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64); + + int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3]); + *pu64 = i32; + pVCpu->iem.s.offOpcode = offOpcode + 4; + return VINF_SUCCESS; +} +#endif /* !IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode double word and sign extends it to a quad word, + * returns automatically on failure. + * + * @param a_pu64 Where to return the opcode quad word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu)) +#endif + +#ifndef IEM_WITH_SETJMP + +/** + * Fetches the next opcode qword. + * + * @returns Strict VBox status code. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pu64 Where to return the opcode qword. + */ +DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT +{ + uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode)) + { +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3], + pVCpu->iem.s.abOpcode[offOpcode + 4], + pVCpu->iem.s.abOpcode[offOpcode + 5], + pVCpu->iem.s.abOpcode[offOpcode + 6], + pVCpu->iem.s.abOpcode[offOpcode + 7]); +# endif + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8; + return VINF_SUCCESS; + } + return iemOpcodeGetNextU64Slow(pVCpu, pu64); +} + +#else /* IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode qword, longjmp on error. + * + * @returns The opcode qword. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP +{ +# ifdef IEM_WITH_CODE_TLB + uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte; + uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf; + if (RT_LIKELY( pbBuf != NULL + && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf)) + { + pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + return *(uint64_t const *)&pbBuf[offBuf]; +# else + return RT_MAKE_U64_FROM_U8(pbBuf[offBuf], + pbBuf[offBuf + 1], + pbBuf[offBuf + 2], + pbBuf[offBuf + 3], + pbBuf[offBuf + 4], + pbBuf[offBuf + 5], + pbBuf[offBuf + 6], + pbBuf[offBuf + 7]); +# endif + } +# else + uintptr_t const offOpcode = pVCpu->iem.s.offOpcode; + if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode)) + { + pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8; +# ifdef IEM_USE_UNALIGNED_DATA_ACCESS + return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode]; +# else + return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode], + pVCpu->iem.s.abOpcode[offOpcode + 1], + pVCpu->iem.s.abOpcode[offOpcode + 2], + pVCpu->iem.s.abOpcode[offOpcode + 3], + pVCpu->iem.s.abOpcode[offOpcode + 4], + pVCpu->iem.s.abOpcode[offOpcode + 5], + pVCpu->iem.s.abOpcode[offOpcode + 6], + pVCpu->iem.s.abOpcode[offOpcode + 7]); +# endif + } +# endif + return iemOpcodeGetNextU64SlowJmp(pVCpu); +} + +#endif /* IEM_WITH_SETJMP */ + +/** + * Fetches the next opcode quad word, returns automatically on failure. + * + * @param a_pu64 Where to return the opcode quad word. + * @remark Implicitly references pVCpu. + */ +#ifndef IEM_WITH_SETJMP +# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \ + do \ + { \ + VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \ + if (rcStrict2 != VINF_SUCCESS) \ + return rcStrict2; \ + } while (0) +#else +# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) ) +#endif + + +/** @name Misc Worker Functions. + * @{ + */ + +/** + * Gets the correct EFLAGS regardless of whether PATM stores parts of them or + * not (kind of obsolete now). + * + * @param a_pVCpu The cross context virtual CPU structure of the calling thread. + */ +#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u ) + +/** + * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete). + * + * @param a_pVCpu The cross context virtual CPU structure of the calling thread. + * @param a_fEfl The new EFLAGS. + */ +#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0) + + +/** + * Loads a NULL data selector into a selector register, both the hidden and + * visible parts, in protected mode. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pSReg Pointer to the segment register. + * @param uRpl The RPL. + */ +DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT +{ + /** @todo Testcase: write a testcase checking what happends when loading a NULL + * data selector in protected mode. */ + pSReg->Sel = uRpl; + pSReg->ValidSel = uRpl; + pSReg->fFlags = CPUMSELREG_FLAGS_VALID; + if (IEM_IS_GUEST_CPU_INTEL(pVCpu)) + { + /* VT-x (Intel 3960x) observed doing something like this. */ + pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT); + pSReg->u32Limit = UINT32_MAX; + pSReg->u64Base = 0; + } + else + { + pSReg->Attr.u = X86DESCATTR_UNUSABLE; + pSReg->u32Limit = 0; + pSReg->u64Base = 0; + } +} + +/** @} */ + + +/* + * + * Helpers routines. + * Helpers routines. + * Helpers routines. + * + */ + +/** + * Recalculates the effective operand size. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + switch (pVCpu->iem.s.enmCpuMode) + { + case IEMMODE_16BIT: + pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT; + break; + case IEMMODE_32BIT: + pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT; + break; + case IEMMODE_64BIT: + switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) + { + case 0: + pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize; + break; + case IEM_OP_PRF_SIZE_OP: + pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT; + break; + case IEM_OP_PRF_SIZE_REX_W: + case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP: + pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT; + break; + } + break; + default: + AssertFailed(); + } +} + + +/** + * Sets the default operand size to 64-bit and recalculates the effective + * operand size. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT); + pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; + if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP) + pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT; + else + pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT; +} + + +/** + * Sets the default operand size to 64-bit and recalculates the effective + * operand size, with intel ignoring any operand size prefix (AMD respects it). + * + * This is for the relative jumps. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT); + pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; + if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP + || pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL) + pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT; + else + pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT; +} + + + + +/** @name Register Access. + * @{ + */ + +/** + * Gets a reference (pointer) to the specified hidden segment register. + * + * @returns Hidden register reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The segment register. + */ +DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT +{ + Assert(iSegReg < X86_SREG_COUNT); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg]; + + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)); + return pSReg; +} + + +/** + * Ensures that the given hidden segment register is up to date. + * + * @returns Hidden register reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pSReg The segment register. + */ +DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT +{ + Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)); + NOREF(pVCpu); + return pSReg; +} + + +/** + * Gets a reference (pointer) to the specified segment register (the selector + * value). + * + * @returns Pointer to the selector variable. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The segment register. + */ +DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT +{ + Assert(iSegReg < X86_SREG_COUNT); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel; +} + + +/** + * Fetches the selector value of a segment register. + * + * @returns The selector value. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The segment register. + */ +DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT +{ + Assert(iSegReg < X86_SREG_COUNT); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel; +} + + +/** + * Fetches the base address value of a segment register. + * + * @returns The selector value. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The segment register. + */ +DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT +{ + Assert(iSegReg < X86_SREG_COUNT); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base; +} + + +/** + * Gets a reference (pointer) to the specified general purpose register. + * + * @returns Register reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The general purpose register. + */ +DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + Assert(iReg < 16); + return &pVCpu->cpum.GstCtx.aGRegs[iReg]; +} + + +/** + * Gets a reference (pointer) to the specified 8-bit general purpose register. + * + * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here. + * + * @returns Register reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + if (iReg < 4 || (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX)) + { + Assert(iReg < 16); + return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8; + } + /* high 8-bit register. */ + Assert(iReg < 8); + return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi; +} + + +/** + * Gets a reference (pointer) to the specified 16-bit general purpose register. + * + * @returns Register reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + Assert(iReg < 16); + return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16; +} + + +/** + * Gets a reference (pointer) to the specified 32-bit general purpose register. + * + * @returns Register reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + Assert(iReg < 16); + return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32; +} + + +/** + * Gets a reference (pointer) to the specified signed 32-bit general purpose register. + * + * @returns Register reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + Assert(iReg < 16); + return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32; +} + + +/** + * Gets a reference (pointer) to the specified 64-bit general purpose register. + * + * @returns Register reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + Assert(iReg < 64); + return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64; +} + + +/** + * Gets a reference (pointer) to the specified signed 64-bit general purpose register. + * + * @returns Register reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + Assert(iReg < 16); + return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64; +} + + +/** + * Gets a reference (pointer) to the specified segment register's base address. + * + * @returns Segment register base address reference. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iSegReg The segment selector. + */ +DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT +{ + Assert(iSegReg < X86_SREG_COUNT); + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base; +} + + +/** + * Fetches the value of a 8-bit general purpose register. + * + * @returns The register value. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + return *iemGRegRefU8(pVCpu, iReg); +} + + +/** + * Fetches the value of a 16-bit general purpose register. + * + * @returns The register value. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + Assert(iReg < 16); + return pVCpu->cpum.GstCtx.aGRegs[iReg].u16; +} + + +/** + * Fetches the value of a 32-bit general purpose register. + * + * @returns The register value. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + Assert(iReg < 16); + return pVCpu->cpum.GstCtx.aGRegs[iReg].u32; +} + + +/** + * Fetches the value of a 64-bit general purpose register. + * + * @returns The register value. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iReg The register. + */ +DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT +{ + Assert(iReg < 16); + return pVCpu->cpum.GstCtx.aGRegs[iReg].u64; +} + + +/** + * Get the address of the top of the stack. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT +{ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + return pVCpu->cpum.GstCtx.rsp; + if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + return pVCpu->cpum.GstCtx.esp; + return pVCpu->cpum.GstCtx.sp; +} + + +/** + * Updates the RIP/EIP/IP to point to the next instruction. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr The number of bytes to add. + */ +DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT +{ + /* + * Advance RIP. + * + * When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit, + * while in all other modes except LM64 the updates are 32-bit. This means + * we need to watch for both 32-bit and 16-bit "carry" situations, i.e. + * 4GB and 64KB rollovers, and decide whether anything needs masking. + * + * See PC wrap around tests in bs3-cpu-weird-1. + */ + uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip; + uint64_t const uRipNext = uRipPrev + cbInstr; + if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) | RT_BIT_64(16))) + || pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)) + pVCpu->cpum.GstCtx.rip = uRipNext; + else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386) + pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext; + else + pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext; +} + + +/** + * Called by iemRegAddToRipAndFinishingClearingRF and others when any of the + * following EFLAGS bits are set: + * - X86_EFL_RF - clear it. + * - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them. + * - X86_EFL_TF - generate single step \#DB trap. + * - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not + * instruction). + * + * According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}, + * a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint + * takes priority over both NMIs and hardware interrupts. So, neither is + * considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are + * either unsupported will be triggered on-top of any \#DB raised here.) + * + * The RF flag only needs to be cleared here as it only suppresses instruction + * breakpoints which are not raised here (happens synchronously during + * instruction fetching). + * + * The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its + * status has no bearing on whether \#DB exceptions are raised. + * + * @note This must *NOT* be called by the two instructions setting the + * CPUMCTX_INHIBIT_SHADOW_SS flag. + * + * @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events} + * @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching + * Stacks} + */ +static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Normally we're just here to clear RF and/or interrupt shadow bits. + */ + if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) == 0)) + pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW); + else + { + /* + * Raise a #DB or/and DBGF event. + */ + VBOXSTRICTRC rcStrict; + if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK)) + { + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6); + pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK; + if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF) + pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS; + pVCpu->cpum.GstCtx.dr[6] |= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT; + LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6], + pVCpu->cpum.GstCtx.rflags.uBoth)); + + pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK); + rcStrict = iemRaiseDebugException(pVCpu); + + /* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */ + if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict)) + { + rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT; + LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict))); + } + } + else + { + Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK); + rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT; + LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict))); + } + pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK; + return rcStrict; + } + return VINF_SUCCESS; +} + + +/** + * Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * We assume that most of the time nothing actually needs doing here. + */ + AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX); + if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth + & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) )) + return VINF_SUCCESS; + return iemFinishInstructionWithFlagsSet(pVCpu); +} + + +/** + * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF + * and CPUMCTX_INHIBIT_SHADOW. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr The number of bytes to add. + */ +DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT +{ + iemRegAddToRip(pVCpu, cbInstr); + return iemRegFinishClearingRF(pVCpu); +} + + +/** + * Extended version of iemFinishInstructionWithFlagsSet that goes with + * iemRegAddToRipAndFinishingClearingRfEx. + * + * See iemFinishInstructionWithFlagsSet() for details. + */ +static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + /* + * Raise a #DB. + */ + IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6); + pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK; + pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS + | (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT; + /** @todo Do we set all pending \#DB events, or just one? */ + LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n", + pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6], + pVCpu->cpum.GstCtx.rflags.uBoth)); + pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK); + return iemRaiseDebugException(pVCpu); +} + + +/** + * Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and + * others potentially updating EFLAGS.TF. + * + * The single step event must be generated using the TF value at the start of + * the instruction, not the new value set by it. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbInstr The number of bytes to add. + * @param fEflOld The EFLAGS at the start of the instruction + * execution. + */ +DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT +{ + iemRegAddToRip(pVCpu, cbInstr); + if (!(fEflOld & X86_EFL_TF)) + return iemRegFinishClearingRF(pVCpu); + return iemFinishInstructionWithTfSet(pVCpu); +} + + +/** + * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu)); +} + + +/** + * Adds to the stack pointer. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbToAdd The number of bytes to add (8-bit!). + */ +DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT +{ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pVCpu->cpum.GstCtx.rsp += cbToAdd; + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + pVCpu->cpum.GstCtx.esp += cbToAdd; + else + pVCpu->cpum.GstCtx.sp += cbToAdd; +} + + +/** + * Subtracts from the stack pointer. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbToSub The number of bytes to subtract (8-bit!). + */ +DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT +{ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pVCpu->cpum.GstCtx.rsp -= cbToSub; + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + pVCpu->cpum.GstCtx.esp -= cbToSub; + else + pVCpu->cpum.GstCtx.sp -= cbToSub; +} + + +/** + * Adds to the temporary stack pointer. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pTmpRsp The temporary SP/ESP/RSP to update. + * @param cbToAdd The number of bytes to add (16-bit). + */ +DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT +{ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pTmpRsp->u += cbToAdd; + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + pTmpRsp->DWords.dw0 += cbToAdd; + else + pTmpRsp->Words.w0 += cbToAdd; +} + + +/** + * Subtracts from the temporary stack pointer. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pTmpRsp The temporary SP/ESP/RSP to update. + * @param cbToSub The number of bytes to subtract. + * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is + * expecting that. + */ +DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT +{ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + pTmpRsp->u -= cbToSub; + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + pTmpRsp->DWords.dw0 -= cbToSub; + else + pTmpRsp->Words.w0 -= cbToSub; +} + + +/** + * Calculates the effective stack address for a push of the specified size as + * well as the new RSP value (upper bits may be masked). + * + * @returns Effective stack addressf for the push. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbItem The size of the stack item to pop. + * @param puNewRsp Where to return the new RSP value. + */ +DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT +{ + RTUINT64U uTmpRsp; + RTGCPTR GCPtrTop; + uTmpRsp.u = pVCpu->cpum.GstCtx.rsp; + + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + GCPtrTop = uTmpRsp.u -= cbItem; + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem; + else + GCPtrTop = uTmpRsp.Words.w0 -= cbItem; + *puNewRsp = uTmpRsp.u; + return GCPtrTop; +} + + +/** + * Gets the current stack pointer and calculates the value after a pop of the + * specified size. + * + * @returns Current stack pointer. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param cbItem The size of the stack item to pop. + * @param puNewRsp Where to return the new RSP value. + */ +DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT +{ + RTUINT64U uTmpRsp; + RTGCPTR GCPtrTop; + uTmpRsp.u = pVCpu->cpum.GstCtx.rsp; + + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + GCPtrTop = uTmpRsp.u; + uTmpRsp.u += cbItem; + } + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + { + GCPtrTop = uTmpRsp.DWords.dw0; + uTmpRsp.DWords.dw0 += cbItem; + } + else + { + GCPtrTop = uTmpRsp.Words.w0; + uTmpRsp.Words.w0 += cbItem; + } + *puNewRsp = uTmpRsp.u; + return GCPtrTop; +} + + +/** + * Calculates the effective stack address for a push of the specified size as + * well as the new temporary RSP value (upper bits may be masked). + * + * @returns Effective stack addressf for the push. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pTmpRsp The temporary stack pointer. This is updated. + * @param cbItem The size of the stack item to pop. + */ +DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT +{ + RTGCPTR GCPtrTop; + + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + GCPtrTop = pTmpRsp->u -= cbItem; + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem; + else + GCPtrTop = pTmpRsp->Words.w0 -= cbItem; + return GCPtrTop; +} + + +/** + * Gets the effective stack address for a pop of the specified size and + * calculates and updates the temporary RSP. + * + * @returns Current stack pointer. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pTmpRsp The temporary stack pointer. This is updated. + * @param cbItem The size of the stack item to pop. + */ +DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT +{ + RTGCPTR GCPtrTop; + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + GCPtrTop = pTmpRsp->u; + pTmpRsp->u += cbItem; + } + else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig) + { + GCPtrTop = pTmpRsp->DWords.dw0; + pTmpRsp->DWords.dw0 += cbItem; + } + else + { + GCPtrTop = pTmpRsp->Words.w0; + pTmpRsp->Words.w0 += cbItem; + } + return GCPtrTop; +} + +/** @} */ + + +/** @name FPU access and helpers. + * + * @{ + */ + + +/** + * Hook for preparing to use the host FPU. + * + * This is necessary in ring-0 and raw-mode context (nop in ring-3). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT +{ +#ifdef IN_RING3 + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM); +#else + CPUMRZFpuStatePrepareHostCpuForUse(pVCpu); +#endif + IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); +} + + +/** + * Hook for preparing to use the host FPU for SSE. + * + * This is necessary in ring-0 and raw-mode context (nop in ring-3). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + iemFpuPrepareUsage(pVCpu); +} + + +/** + * Hook for preparing to use the host FPU for AVX. + * + * This is necessary in ring-0 and raw-mode context (nop in ring-3). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + iemFpuPrepareUsage(pVCpu); +} + + +/** + * Hook for actualizing the guest FPU state before the interpreter reads it. + * + * This is necessary in ring-0 and raw-mode context (nop in ring-3). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT +{ +#ifdef IN_RING3 + NOREF(pVCpu); +#else + CPUMRZFpuStateActualizeForRead(pVCpu); +#endif + IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); +} + + +/** + * Hook for actualizing the guest FPU state before the interpreter changes it. + * + * This is necessary in ring-0 and raw-mode context (nop in ring-3). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT +{ +#ifdef IN_RING3 + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM); +#else + CPUMRZFpuStateActualizeForChange(pVCpu); +#endif + IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); +} + + +/** + * Hook for actualizing the guest XMM0..15 and MXCSR register state for read + * only. + * + * This is necessary in ring-0 and raw-mode context (nop in ring-3). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT +{ +#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM) + NOREF(pVCpu); +#else + CPUMRZFpuStateActualizeSseForRead(pVCpu); +#endif + IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); +} + + +/** + * Hook for actualizing the guest XMM0..15 and MXCSR register state for + * read+write. + * + * This is necessary in ring-0 and raw-mode context (nop in ring-3). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT +{ +#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM) + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM); +#else + CPUMRZFpuStateActualizeForChange(pVCpu); +#endif + IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); + + /* Make sure any changes are loaded the next time around. */ + pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_SSE; +} + + +/** + * Hook for actualizing the guest YMM0..15 and MXCSR register state for read + * only. + * + * This is necessary in ring-0 and raw-mode context (nop in ring-3). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT +{ +#ifdef IN_RING3 + NOREF(pVCpu); +#else + CPUMRZFpuStateActualizeAvxForRead(pVCpu); +#endif + IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); +} + + +/** + * Hook for actualizing the guest YMM0..15 and MXCSR register state for + * read+write. + * + * This is necessary in ring-0 and raw-mode context (nop in ring-3). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT +{ +#ifdef IN_RING3 + CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM); +#else + CPUMRZFpuStateActualizeForChange(pVCpu); +#endif + IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx); + + /* Just assume we're going to make changes to the SSE and YMM_HI parts. */ + pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_YMM | XSAVE_C_SSE; +} + + +/** + * Stores a QNaN value into a FPU register. + * + * @param pReg Pointer to the register. + */ +DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT +{ + pReg->au32[0] = UINT32_C(0x00000000); + pReg->au32[1] = UINT32_C(0xc0000000); + pReg->au16[4] = UINT16_C(0xffff); +} + + +/** + * Updates the FOP, FPU.CS and FPUIP registers. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pFpuCtx The FPU context. + */ +DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT +{ + Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX); + pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode; + /** @todo x87.CS and FPUIP needs to be kept seperately. */ + if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) + { + /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled + * happens in real mode here based on the fnsave and fnstenv images. */ + pFpuCtx->CS = 0; + pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4); + } + else if (!IEM_IS_LONG_MODE(pVCpu)) + { + pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel; + pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip; + } + else + *(uint64_t *)&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip; +} + + + + + +/** + * Marks the specified stack register as free (for FFREE). + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param iStReg The register to free. + */ +DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT +{ + Assert(iStReg < 8); + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK; + pFpuCtx->FTW &= ~RT_BIT(iReg); +} + + +/** + * Increments FSW.TOP, i.e. pops an item off the stack without freeing it. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + uint16_t uFsw = pFpuCtx->FSW; + uint16_t uTop = uFsw & X86_FSW_TOP_MASK; + uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK; + uFsw &= ~X86_FSW_TOP_MASK; + uFsw |= uTop; + pFpuCtx->FSW = uFsw; +} + + +/** + * Decrements FSW.TOP, i.e. push an item off the stack without storing anything. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + uint16_t uFsw = pFpuCtx->FSW; + uint16_t uTop = uFsw & X86_FSW_TOP_MASK; + uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK; + uFsw &= ~X86_FSW_TOP_MASK; + uFsw |= uTop; + pFpuCtx->FSW = uFsw; +} + + + + +DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK; + if (pFpuCtx->FTW & RT_BIT(iReg)) + return VINF_SUCCESS; + return VERR_NOT_FOUND; +} + + +DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK; + if (pFpuCtx->FTW & RT_BIT(iReg)) + { + *ppRef = &pFpuCtx->aRegs[iStReg].r80; + return VINF_SUCCESS; + } + return VERR_NOT_FOUND; +} + + +DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, + uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW); + uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK; + uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK; + if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1))) + { + *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80; + *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80; + return VINF_SUCCESS; + } + return VERR_NOT_FOUND; +} + + +DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT +{ + PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87; + uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW); + uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK; + uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK; + if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1))) + { + *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80; + return VINF_SUCCESS; + } + return VERR_NOT_FOUND; +} + + +/** + * Rotates the stack registers when setting new TOS. + * + * @param pFpuCtx The FPU context. + * @param iNewTop New TOS value. + * @remarks We only do this to speed up fxsave/fxrstor which + * arrange the FP registers in stack order. + * MUST be done before writing the new TOS (FSW). + */ +DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT +{ + uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW); + RTFLOAT80U ar80Temp[8]; + + if (iOldTop == iNewTop) + return; + + /* Unscrew the stack and get it into 'native' order. */ + ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80; + ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80; + ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80; + ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80; + ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80; + ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80; + ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80; + ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80; + + /* Now rotate the stack to the new position. */ + pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK]; + pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK]; + pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK]; + pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK]; + pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK]; + pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK]; + pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK]; + pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK]; +} + + +/** + * Updates the FPU exception status after FCW is changed. + * + * @param pFpuCtx The FPU context. + */ +DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT +{ + uint16_t u16Fsw = pFpuCtx->FSW; + if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK)) + u16Fsw |= X86_FSW_ES | X86_FSW_B; + else + u16Fsw &= ~(X86_FSW_ES | X86_FSW_B); + pFpuCtx->FSW = u16Fsw; +} + + +/** + * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE. + * + * @returns The full FTW. + * @param pFpuCtx The FPU context. + */ +DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT +{ + uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW; + uint16_t u16Ftw = 0; + unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW); + for (unsigned iSt = 0; iSt < 8; iSt++) + { + unsigned const iReg = (iSt + iTop) & 7; + if (!(u8Ftw & RT_BIT(iReg))) + u16Ftw |= 3 << (iReg * 2); /* empty */ + else + { + uint16_t uTag; + PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80; + if (pr80Reg->s.uExponent == 0x7fff) + uTag = 2; /* Exponent is all 1's => Special. */ + else if (pr80Reg->s.uExponent == 0x0000) + { + if (pr80Reg->s.uMantissa == 0x0000) + uTag = 1; /* All bits are zero => Zero. */ + else + uTag = 2; /* Must be special. */ + } + else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */ + uTag = 0; /* Valid. */ + else + uTag = 2; /* Must be special. */ + + u16Ftw |= uTag << (iReg * 2); + } + } + + return u16Ftw; +} + + +/** + * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR). + * + * @returns The compressed FTW. + * @param u16FullFtw The full FTW to convert. + */ +DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT +{ + uint8_t u8Ftw = 0; + for (unsigned i = 0; i < 8; i++) + { + if ((u16FullFtw & 3) != 3 /*empty*/) + u8Ftw |= RT_BIT(i); + u16FullFtw >>= 2; + } + + return u8Ftw; +} + +/** @} */ + + +/** @name Memory access. + * + * @{ + */ + + +/** + * Checks whether alignment checks are enabled or not. + * + * @returns true if enabled, false if not. + * @param pVCpu The cross context virtual CPU structure of the calling thread. + */ +DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT +{ + AssertCompile(X86_CR0_AM == X86_EFL_AC); + return pVCpu->iem.s.uCpl == 3 + && (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM); +} + +/** + * Checks if the given segment can be written to, raise the appropriate + * exception if not. + * + * @returns VBox strict status code. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pHid Pointer to the hidden register. + * @param iSegReg The register number. + * @param pu64BaseAddr Where to return the base address to use for the + * segment. (In 64-bit code it may differ from the + * base in the hidden segment.) + */ +DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, + uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base; + else + { + if (!pHid->Attr.n.u1Present) + { + uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg); + AssertRelease(uSel == 0); + Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE) + || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) ) + && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT ) + return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W); + *pu64BaseAddr = pHid->u64Base; + } + return VINF_SUCCESS; +} + + +/** + * Checks if the given segment can be read from, raise the appropriate + * exception if not. + * + * @returns VBox strict status code. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param pHid Pointer to the hidden register. + * @param iSegReg The register number. + * @param pu64BaseAddr Where to return the base address to use for the + * segment. (In 64-bit code it may differ from the + * base in the hidden segment.) + */ +DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, + uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT +{ + IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base; + else + { + if (!pHid->Attr.n.u1Present) + { + uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg); + AssertRelease(uSel == 0); + Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg)); + return iemRaiseGeneralProtectionFault0(pVCpu); + } + + if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE) + return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R); + *pu64BaseAddr = pHid->u64Base; + } + return VINF_SUCCESS; +} + + +/** + * Maps a physical page. + * + * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr). + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param GCPhysMem The physical address. + * @param fAccess The intended access. + * @param ppvMem Where to return the mapping address. + * @param pLock The PGM lock. + */ +DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, + void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT +{ +#ifdef IEM_LOG_MEMORY_WRITES + if (fAccess & IEM_ACCESS_TYPE_WRITE) + return VERR_PGM_PHYS_TLB_CATCH_ALL; +#endif + + /** @todo This API may require some improving later. A private deal with PGM + * regarding locking and unlocking needs to be struct. A couple of TLBs + * living in PGM, but with publicly accessible inlined access methods + * could perhaps be an even better solution. */ + int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu, + GCPhysMem, + RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE), + pVCpu->iem.s.fBypassHandlers, + ppvMem, + pLock); + /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/ + AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc)); + + return rc; +} + + +/** + * Unmap a page previously mapped by iemMemPageMap. + * + * @param pVCpu The cross context virtual CPU structure of the calling thread. + * @param GCPhysMem The physical address. + * @param fAccess The intended access. + * @param pvMem What iemMemPageMap returned. + * @param pLock The PGM lock. + */ +DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, + const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT +{ + NOREF(pVCpu); + NOREF(GCPhysMem); + NOREF(fAccess); + NOREF(pvMem); + PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock); +} + +#ifdef IEM_WITH_SETJMP + +/** @todo slim this down */ +DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, + size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + Assert(cbMem >= 1); + Assert(iSegReg < X86_SREG_COUNT); + + /* + * 64-bit mode is simpler. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX) + { + IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg); + GCPtrMem += pSel->u64Base; + } + + if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1))) + return GCPtrMem; + iemRaiseGeneralProtectionFault0Jmp(pVCpu); + } + /* + * 16-bit and 32-bit segmentation. + */ + else if (iSegReg != UINT8_MAX) + { + /** @todo Does this apply to segments with 4G-1 limit? */ + uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1; + if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem)) + { + IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg); + switch (pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE + | X86_SEL_TYPE_READ | X86_SEL_TYPE_WRITE /* same as read */ + | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_CONF /* same as down */ + | X86_SEL_TYPE_CODE)) + { + case X86DESCATTR_P: /* readonly data, expand up */ + case X86DESCATTR_P | X86_SEL_TYPE_WRITE: /* writable data, expand up */ + case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ: /* code, read-only */ + case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_CONF: /* conforming code, read-only */ + /* expand up */ + if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit)) + return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base; + Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n", + (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit)); + break; + + case X86DESCATTR_P | X86_SEL_TYPE_DOWN: /* readonly data, expand down */ + case X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_WRITE: /* writable data, expand down */ + /* expand down */ + if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit + && ( pSel->Attr.n.u1DefBig + || GCPtrLast32 <= UINT32_C(0xffff)) )) + return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base; + Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n", + (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX)); + break; + + default: + Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u)); + iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R); + break; + } + } + Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32)); + iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R); + } + /* + * 32-bit flat address. + */ + else + return GCPtrMem; +} + + +/** @todo slim this down */ +DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, + RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP +{ + Assert(cbMem >= 1); + Assert(iSegReg < X86_SREG_COUNT); + + /* + * 64-bit mode is simpler. + */ + if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) + { + if (iSegReg >= X86_SREG_FS) + { + IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg); + GCPtrMem += pSel->u64Base; + } + + if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1))) + return GCPtrMem; + } + /* + * 16-bit and 32-bit segmentation. + */ + else + { + IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg)); + PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg); + uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE + | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN); + if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE)) /* data, expand up */ + { + /* expand up */ + uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem; + if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit + && GCPtrLast32 > (uint32_t)GCPtrMem)) + return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base; + } + else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */ + { + /* expand down */ + uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem; + if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit + && GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)) + && GCPtrLast32 > (uint32_t)GCPtrMem)) + return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base; + } + else + iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W); + iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W); + } + iemRaiseGeneralProtectionFault0Jmp(pVCpu); +} + +#endif /* IEM_WITH_SETJMP */ + +/** + * Fakes a long mode stack selector for SS = 0. + * + * @param pDescSs Where to return the fake stack descriptor. + * @param uDpl The DPL we want. + */ +DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT +{ + pDescSs->Long.au64[0] = 0; + pDescSs->Long.au64[1] = 0; + pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC; + pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */ + pDescSs->Long.Gen.u2Dpl = uDpl; + pDescSs->Long.Gen.u1Present = 1; + pDescSs->Long.Gen.u1Long = 1; +} + +/** @} */ + + +#ifdef VBOX_WITH_NESTED_HWVIRT_VMX + +/** + * Gets CR0 fixed-0 bits in VMX operation. + * + * We do this rather than fetching what we report to the guest (in + * IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same + * values regardless of whether unrestricted-guest feature is available on the CPU. + * + * @returns CR0 fixed-0 bits. + * @param pVCpu The cross context virtual CPU structure. + * @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode + * must be returned. When @c false, the CR0 fixed-0 + * bits for VMX root mode is returned. + * + */ +DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT +{ + Assert(IEM_VMX_IS_ROOT_MODE(pVCpu)); + + PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs; + if ( fVmxNonRootMode + && (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST)) + return VMX_V_CR0_FIXED0_UX; + return VMX_V_CR0_FIXED0; +} + + +/** + * Sets virtual-APIC write emulation as pending. + * + * @param pVCpu The cross context virtual CPU structure. + * @param offApic The offset in the virtual-APIC page that was written. + */ +DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT +{ + Assert(offApic < XAPIC_OFF_END + 4); + + /* + * Record the currently updated APIC offset, as we need this later for figuring + * out whether to perform TPR, EOI or self-IPI virtualization as well as well + * as for supplying the exit qualification when causing an APIC-write VM-exit. + */ + pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic; + + /* + * Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI + * virtualization or APIC-write emulation). + */ + if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE)) + VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE); +} + +#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */ + +#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */ |