diff options
Diffstat (limited to 'src/VBox/VMM/VMMR0/HMR0A.asm')
| -rw-r--r-- | src/VBox/VMM/VMMR0/HMR0A.asm | 1705 |
1 files changed, 1705 insertions, 0 deletions
diff --git a/src/VBox/VMM/VMMR0/HMR0A.asm b/src/VBox/VMM/VMMR0/HMR0A.asm new file mode 100644 index 00000000..fc1fc67f --- /dev/null +++ b/src/VBox/VMM/VMMR0/HMR0A.asm @@ -0,0 +1,1705 @@ +; $Id: HMR0A.asm $ +;; @file +; HM - Ring-0 VMX, SVM world-switch and helper routines. +; + +; +; Copyright (C) 2006-2020 Oracle Corporation +; +; This file is part of VirtualBox Open Source Edition (OSE), as +; available from http://www.virtualbox.org. This file is free software; +; you can redistribute it and/or modify it under the terms of the GNU +; General Public License (GPL) as published by the Free Software +; Foundation, in version 2 as it comes in the "COPYING" file of the +; VirtualBox OSE distribution. VirtualBox OSE is distributed in the +; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. +; + +;********************************************************************************************************************************* +;* Header Files * +;********************************************************************************************************************************* +%include "VBox/asmdefs.mac" +%include "VBox/err.mac" +%include "VBox/vmm/hm_vmx.mac" +%include "VBox/vmm/cpum.mac" +%include "VBox/vmm/vm.mac" +%include "iprt/x86.mac" +%include "HMInternal.mac" + +%ifdef RT_OS_OS2 ;; @todo fix OMF support in yasm and kick nasm out completely. + %macro vmwrite 2, + int3 + %endmacro + %define vmlaunch int3 + %define vmresume int3 + %define vmsave int3 + %define vmload int3 + %define vmrun int3 + %define clgi int3 + %define stgi int3 + %macro invlpga 2, + int3 + %endmacro +%endif + +;********************************************************************************************************************************* +;* Defined Constants And Macros * +;********************************************************************************************************************************* +;; The offset of the XMM registers in X86FXSTATE. +; Use define because I'm too lazy to convert the struct. +%define XMM_OFF_IN_X86FXSTATE 160 + +;; Spectre filler for 32-bit mode. +; Some user space address that points to a 4MB page boundrary in hope that it +; will somehow make it less useful. +%define SPECTRE_FILLER32 0x227fffff +;; Spectre filler for 64-bit mode. +; Choosen to be an invalid address (also with 5 level paging). +%define SPECTRE_FILLER64 0x02204204207fffff +;; Spectre filler for the current CPU mode. +%ifdef RT_ARCH_AMD64 + %define SPECTRE_FILLER SPECTRE_FILLER64 +%else + %define SPECTRE_FILLER SPECTRE_FILLER32 +%endif + +;; +; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation. +; +%ifdef RT_ARCH_AMD64 + %define VMX_SKIP_GDTR + %define VMX_SKIP_TR + %define VBOX_SKIP_RESTORE_SEG + %ifdef RT_OS_DARWIN + ; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't + ; risk loading a stale LDT value or something invalid. + %define HM_64_BIT_USE_NULL_SEL + ; Darwin (Mavericks) uses IDTR limit to store the CPU Id so we need to restore it always. + ; See @bugref{6875}. + %else + %define VMX_SKIP_IDTR + %endif +%endif + +;; @def MYPUSHAD +; Macro generating an equivalent to PUSHAD instruction. + +;; @def MYPOPAD +; Macro generating an equivalent to POPAD instruction. + +;; @def MYPUSHSEGS +; Macro saving all segment registers on the stack. +; @param 1 Full width register name. +; @param 2 16-bit register name for \a 1. + +;; @def MYPOPSEGS +; Macro restoring all segment registers on the stack. +; @param 1 Full width register name. +; @param 2 16-bit register name for \a 1. + +%ifdef ASM_CALL64_GCC + %macro MYPUSHAD64 0 + push r15 + push r14 + push r13 + push r12 + push rbx + %endmacro + %macro MYPOPAD64 0 + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + %endmacro + +%else ; ASM_CALL64_MSC + %macro MYPUSHAD64 0 + push r15 + push r14 + push r13 + push r12 + push rbx + push rsi + push rdi + %endmacro + %macro MYPOPAD64 0 + pop rdi + pop rsi + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + %endmacro +%endif + +%ifdef VBOX_SKIP_RESTORE_SEG + %macro MYPUSHSEGS64 2 + %endmacro + + %macro MYPOPSEGS64 2 + %endmacro +%else ; !VBOX_SKIP_RESTORE_SEG + ; Trashes, rax, rdx & rcx. + %macro MYPUSHSEGS64 2 + %ifndef HM_64_BIT_USE_NULL_SEL + mov %2, es + push %1 + mov %2, ds + push %1 + %endif + + ; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode, + ; Solaris OTOH doesn't and we must save it. + mov ecx, MSR_K8_FS_BASE + rdmsr + push rdx + push rax + %ifndef HM_64_BIT_USE_NULL_SEL + push fs + %endif + + ; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel. + ; The same happens on exit. + mov ecx, MSR_K8_GS_BASE + rdmsr + push rdx + push rax + %ifndef HM_64_BIT_USE_NULL_SEL + push gs + %endif + %endmacro + + ; trashes, rax, rdx & rcx + %macro MYPOPSEGS64 2 + ; Note: do not step through this code with a debugger! + %ifndef HM_64_BIT_USE_NULL_SEL + xor eax, eax + mov ds, ax + mov es, ax + mov fs, ax + mov gs, ax + %endif + + %ifndef HM_64_BIT_USE_NULL_SEL + pop gs + %endif + pop rax + pop rdx + mov ecx, MSR_K8_GS_BASE + wrmsr + + %ifndef HM_64_BIT_USE_NULL_SEL + pop fs + %endif + pop rax + pop rdx + mov ecx, MSR_K8_FS_BASE + wrmsr + ; Now it's safe to step again + + %ifndef HM_64_BIT_USE_NULL_SEL + pop %1 + mov ds, %2 + pop %1 + mov es, %2 + %endif + %endmacro +%endif ; VBOX_SKIP_RESTORE_SEG + +%macro MYPUSHAD32 0 + pushad +%endmacro +%macro MYPOPAD32 0 + popad +%endmacro + +%macro MYPUSHSEGS32 2 + push ds + push es + push fs + push gs +%endmacro +%macro MYPOPSEGS32 2 + pop gs + pop fs + pop es + pop ds +%endmacro + +%ifdef RT_ARCH_AMD64 + %define MYPUSHAD MYPUSHAD64 + %define MYPOPAD MYPOPAD64 + %define MYPUSHSEGS MYPUSHSEGS64 + %define MYPOPSEGS MYPOPSEGS64 +%else + %define MYPUSHAD MYPUSHAD32 + %define MYPOPAD MYPOPAD32 + %define MYPUSHSEGS MYPUSHSEGS32 + %define MYPOPSEGS MYPOPSEGS32 +%endif + +;; +; Creates an indirect branch prediction barrier on CPUs that need and supports that. +; @clobbers eax, edx, ecx +; @param 1 How to address CPUMCTX. +; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT) +%macro INDIRECT_BRANCH_PREDICTION_BARRIER 2 + test byte [%1 + CPUMCTX.fWorldSwitcher], %2 + jz %%no_indirect_branch_barrier + mov ecx, MSR_IA32_PRED_CMD + mov eax, MSR_IA32_PRED_CMD_F_IBPB + xor edx, edx + wrmsr +%%no_indirect_branch_barrier: +%endmacro + +;; +; Creates an indirect branch prediction and L1D barrier on CPUs that need and supports that. +; @clobbers eax, edx, ecx +; @param 1 How to address CPUMCTX. +; @param 2 Which IBPB flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT) +; @param 3 Which FLUSH flag to test for (CPUMCTX_WSF_L1D_ENTRY) +; @param 4 Which MDS flag to test for (CPUMCTX_WSF_MDS_ENTRY) +%macro INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER 4 + ; Only one test+jmp when disabled CPUs. + test byte [%1 + CPUMCTX.fWorldSwitcher], (%2 | %3 | %4) + jz %%no_barrier_needed + + ; The eax:edx value is the same for both. + AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D) + mov eax, MSR_IA32_PRED_CMD_F_IBPB + xor edx, edx + + ; Indirect branch barrier. + test byte [%1 + CPUMCTX.fWorldSwitcher], %2 + jz %%no_indirect_branch_barrier + mov ecx, MSR_IA32_PRED_CMD + wrmsr +%%no_indirect_branch_barrier: + + ; Level 1 data cache flush. + test byte [%1 + CPUMCTX.fWorldSwitcher], %3 + jz %%no_cache_flush_barrier + mov ecx, MSR_IA32_FLUSH_CMD + wrmsr + jmp %%no_mds_buffer_flushing ; MDS flushing is included in L1D_FLUSH +%%no_cache_flush_barrier: + + ; MDS buffer flushing. + test byte [%1 + CPUMCTX.fWorldSwitcher], %4 + jz %%no_mds_buffer_flushing + sub xSP, xSP + mov [xSP], ds + verw [xSP] + add xSP, xSP +%%no_mds_buffer_flushing: + +%%no_barrier_needed: +%endmacro + + +;********************************************************************************************************************************* +;* External Symbols * +;********************************************************************************************************************************* +%ifdef VBOX_WITH_KERNEL_USING_XMM +extern NAME(CPUMIsGuestFPUStateActive) +%endif + + +BEGINCODE + + +;; +; Restores host-state fields. +; +; @returns VBox status code +; @param f32RestoreHost x86: [ebp + 08h] msc: ecx gcc: edi RestoreHost flags. +; @param pRestoreHost x86: [ebp + 0ch] msc: rdx gcc: rsi Pointer to the RestoreHost struct. +; +ALIGNCODE(16) +BEGINPROC VMXRestoreHostState +%ifdef RT_ARCH_AMD64 + %ifndef ASM_CALL64_GCC + ; Use GCC's input registers since we'll be needing both rcx and rdx further + ; down with the wrmsr instruction. Use the R10 and R11 register for saving + ; RDI and RSI since MSC preserve the two latter registers. + mov r10, rdi + mov r11, rsi + mov rdi, rcx + mov rsi, rdx + %endif + + test edi, VMX_RESTORE_HOST_GDTR + jz .test_idtr + lgdt [rsi + VMXRESTOREHOST.HostGdtr] + +.test_idtr: + test edi, VMX_RESTORE_HOST_IDTR + jz .test_ds + lidt [rsi + VMXRESTOREHOST.HostIdtr] + +.test_ds: + test edi, VMX_RESTORE_HOST_SEL_DS + jz .test_es + mov ax, [rsi + VMXRESTOREHOST.uHostSelDS] + mov ds, eax + +.test_es: + test edi, VMX_RESTORE_HOST_SEL_ES + jz .test_tr + mov ax, [rsi + VMXRESTOREHOST.uHostSelES] + mov es, eax + +.test_tr: + test edi, VMX_RESTORE_HOST_SEL_TR + jz .test_fs + ; When restoring the TR, we must first clear the busy flag or we'll end up faulting. + mov dx, [rsi + VMXRESTOREHOST.uHostSelTR] + mov ax, dx + and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset + test edi, VMX_RESTORE_HOST_GDT_READ_ONLY | VMX_RESTORE_HOST_GDT_NEED_WRITABLE + jnz .gdt_readonly + add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt. + and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit) + ltr dx + jmp short .test_fs +.gdt_readonly: + test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE + jnz .gdt_readonly_need_writable + mov rcx, cr0 + mov r9, rcx + add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt. + and rcx, ~X86_CR0_WP + mov cr0, rcx + and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit) + ltr dx + mov cr0, r9 + jmp short .test_fs +.gdt_readonly_need_writable: + add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw + and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit) + lgdt [rsi + VMXRESTOREHOST.HostGdtrRw] + ltr dx + lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; load the original GDT + +.test_fs: + ; + ; When restoring the selector values for FS and GS, we'll temporarily trash + ; the base address (at least the high 32-bit bits, but quite possibly the + ; whole base address), the wrmsr will restore it correctly. (VT-x actually + ; restores the base correctly when leaving guest mode, but not the selector + ; value, so there is little problem with interrupts being enabled prior to + ; this restore job.) + ; We'll disable ints once for both FS and GS as that's probably faster. + ; + test edi, VMX_RESTORE_HOST_SEL_FS | VMX_RESTORE_HOST_SEL_GS + jz .restore_success + pushfq + cli ; (see above) + + test edi, VMX_RESTORE_HOST_SEL_FS + jz .test_gs + mov ax, word [rsi + VMXRESTOREHOST.uHostSelFS] + mov fs, eax + mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo + mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi + mov ecx, MSR_K8_FS_BASE + wrmsr + +.test_gs: + test edi, VMX_RESTORE_HOST_SEL_GS + jz .restore_flags + mov ax, word [rsi + VMXRESTOREHOST.uHostSelGS] + mov gs, eax + mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo + mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi + mov ecx, MSR_K8_GS_BASE + wrmsr + +.restore_flags: + popfq + +.restore_success: + mov eax, VINF_SUCCESS + %ifndef ASM_CALL64_GCC + ; Restore RDI and RSI on MSC. + mov rdi, r10 + mov rsi, r11 + %endif +%else ; RT_ARCH_X86 + mov eax, VERR_NOT_IMPLEMENTED +%endif + ret +ENDPROC VMXRestoreHostState + + +;; +; Dispatches an NMI to the host. +; +ALIGNCODE(16) +BEGINPROC VMXDispatchHostNmi + ; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts". + int 2 + ret +ENDPROC VMXDispatchHostNmi + + +;; +; Executes VMWRITE, 64-bit value. +; +; @returns VBox status code. +; @param idxField x86: [ebp + 08h] msc: rcx gcc: rdi VMCS index. +; @param u64Data x86: [ebp + 0ch] msc: rdx gcc: rsi VM field value. +; +ALIGNCODE(16) +BEGINPROC VMXWriteVmcs64 +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_GCC + and edi, 0ffffffffh + xor rax, rax + vmwrite rdi, rsi + %else + and ecx, 0ffffffffh + xor rax, rax + vmwrite rcx, rdx + %endif +%else ; RT_ARCH_X86 + mov ecx, [esp + 4] ; idxField + lea edx, [esp + 8] ; &u64Data + vmwrite ecx, [edx] ; low dword + jz .done + jc .done + inc ecx + xor eax, eax + vmwrite ecx, [edx + 4] ; high dword +.done: +%endif ; RT_ARCH_X86 + jnc .valid_vmcs + mov eax, VERR_VMX_INVALID_VMCS_PTR + ret +.valid_vmcs: + jnz .the_end + mov eax, VERR_VMX_INVALID_VMCS_FIELD +.the_end: + ret +ENDPROC VMXWriteVmcs64 + + +;; +; Executes VMREAD, 64-bit value. +; +; @returns VBox status code. +; @param idxField VMCS index. +; @param pData Where to store VM field value. +; +;DECLASM(int) VMXReadVmcs64(uint32_t idxField, uint64_t *pData); +ALIGNCODE(16) +BEGINPROC VMXReadVmcs64 +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_GCC + and edi, 0ffffffffh + xor rax, rax + vmread [rsi], rdi + %else + and ecx, 0ffffffffh + xor rax, rax + vmread [rdx], rcx + %endif +%else ; RT_ARCH_X86 + mov ecx, [esp + 4] ; idxField + mov edx, [esp + 8] ; pData + vmread [edx], ecx ; low dword + jz .done + jc .done + inc ecx + xor eax, eax + vmread [edx + 4], ecx ; high dword +.done: +%endif ; RT_ARCH_X86 + jnc .valid_vmcs + mov eax, VERR_VMX_INVALID_VMCS_PTR + ret +.valid_vmcs: + jnz .the_end + mov eax, VERR_VMX_INVALID_VMCS_FIELD +.the_end: + ret +ENDPROC VMXReadVmcs64 + + +;; +; Executes VMREAD, 32-bit value. +; +; @returns VBox status code. +; @param idxField VMCS index. +; @param pu32Data Where to store VM field value. +; +;DECLASM(int) VMXReadVmcs32(uint32_t idxField, uint32_t *pu32Data); +ALIGNCODE(16) +BEGINPROC VMXReadVmcs32 +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_GCC + and edi, 0ffffffffh + xor rax, rax + vmread r10, rdi + mov [rsi], r10d + %else + and ecx, 0ffffffffh + xor rax, rax + vmread r10, rcx + mov [rdx], r10d + %endif +%else ; RT_ARCH_X86 + mov ecx, [esp + 4] ; idxField + mov edx, [esp + 8] ; pu32Data + xor eax, eax + vmread [edx], ecx +%endif ; RT_ARCH_X86 + jnc .valid_vmcs + mov eax, VERR_VMX_INVALID_VMCS_PTR + ret +.valid_vmcs: + jnz .the_end + mov eax, VERR_VMX_INVALID_VMCS_FIELD +.the_end: + ret +ENDPROC VMXReadVmcs32 + + +;; +; Executes VMWRITE, 32-bit value. +; +; @returns VBox status code. +; @param idxField VMCS index. +; @param u32Data Where to store VM field value. +; +;DECLASM(int) VMXWriteVmcs32(uint32_t idxField, uint32_t u32Data); +ALIGNCODE(16) +BEGINPROC VMXWriteVmcs32 +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_GCC + and edi, 0ffffffffh + and esi, 0ffffffffh + xor rax, rax + vmwrite rdi, rsi + %else + and ecx, 0ffffffffh + and edx, 0ffffffffh + xor rax, rax + vmwrite rcx, rdx + %endif +%else ; RT_ARCH_X86 + mov ecx, [esp + 4] ; idxField + mov edx, [esp + 8] ; u32Data + xor eax, eax + vmwrite ecx, edx +%endif ; RT_ARCH_X86 + jnc .valid_vmcs + mov eax, VERR_VMX_INVALID_VMCS_PTR + ret +.valid_vmcs: + jnz .the_end + mov eax, VERR_VMX_INVALID_VMCS_FIELD +.the_end: + ret +ENDPROC VMXWriteVmcs32 + + +;; +; Executes VMXON. +; +; @returns VBox status code. +; @param HCPhysVMXOn Physical address of VMXON structure. +; +;DECLASM(int) VMXEnable(RTHCPHYS HCPhysVMXOn); +BEGINPROC VMXEnable +%ifdef RT_ARCH_AMD64 + xor rax, rax + %ifdef ASM_CALL64_GCC + push rdi + %else + push rcx + %endif + vmxon [rsp] +%else ; RT_ARCH_X86 + xor eax, eax + vmxon [esp + 4] +%endif ; RT_ARCH_X86 + jnc .good + mov eax, VERR_VMX_INVALID_VMXON_PTR + jmp .the_end + +.good: + jnz .the_end + mov eax, VERR_VMX_VMXON_FAILED + +.the_end: +%ifdef RT_ARCH_AMD64 + add rsp, 8 +%endif + ret +ENDPROC VMXEnable + + +;; +; Executes VMXOFF. +; +;DECLASM(void) VMXDisable(void); +BEGINPROC VMXDisable + vmxoff +.the_end: + ret +ENDPROC VMXDisable + + +;; +; Executes VMCLEAR. +; +; @returns VBox status code. +; @param HCPhysVmcs Physical address of VM control structure. +; +;DECLASM(int) VMXClearVmcs(RTHCPHYS HCPhysVmcs); +ALIGNCODE(16) +BEGINPROC VMXClearVmcs +%ifdef RT_ARCH_AMD64 + xor rax, rax + %ifdef ASM_CALL64_GCC + push rdi + %else + push rcx + %endif + vmclear [rsp] +%else ; RT_ARCH_X86 + xor eax, eax + vmclear [esp + 4] +%endif ; RT_ARCH_X86 + jnc .the_end + mov eax, VERR_VMX_INVALID_VMCS_PTR +.the_end: +%ifdef RT_ARCH_AMD64 + add rsp, 8 +%endif + ret +ENDPROC VMXClearVmcs + + +;; +; Executes VMPTRLD. +; +; @returns VBox status code. +; @param HCPhysVmcs Physical address of VMCS structure. +; +;DECLASM(int) VMXLoadVmcs(RTHCPHYS HCPhysVmcs); +ALIGNCODE(16) +BEGINPROC VMXLoadVmcs +%ifdef RT_ARCH_AMD64 + xor rax, rax + %ifdef ASM_CALL64_GCC + push rdi + %else + push rcx + %endif + vmptrld [rsp] +%else + xor eax, eax + vmptrld [esp + 4] +%endif + jnc .the_end + mov eax, VERR_VMX_INVALID_VMCS_PTR +.the_end: +%ifdef RT_ARCH_AMD64 + add rsp, 8 +%endif + ret +ENDPROC VMXLoadVmcs + + +;; +; Executes VMPTRST. +; +; @returns VBox status code. +; @param [esp + 04h] gcc:rdi msc:rcx Param 1 - First parameter - Address that will receive the current pointer. +; +;DECLASM(int) VMXGetCurrentVmcs(RTHCPHYS *pVMCS); +BEGINPROC VMXGetCurrentVmcs +%ifdef RT_OS_OS2 + mov eax, VERR_NOT_SUPPORTED + ret +%else + %ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_GCC + vmptrst qword [rdi] + %else + vmptrst qword [rcx] + %endif + %else + vmptrst qword [esp+04h] + %endif + xor eax, eax +.the_end: + ret +%endif +ENDPROC VMXGetCurrentVmcs + +;; +; Invalidate a page using INVEPT. +; +; @param enmTlbFlush msc:ecx gcc:edi x86:[esp+04] Type of flush. +; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer. +; +;DECLASM(int) VMXR0InvEPT(VMXTLBFLUSHEPT enmTlbFlush, uint64_t *pDescriptor); +BEGINPROC VMXR0InvEPT +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_GCC + and edi, 0ffffffffh + xor rax, rax +; invept rdi, qword [rsi] + DB 0x66, 0x0F, 0x38, 0x80, 0x3E + %else + and ecx, 0ffffffffh + xor rax, rax +; invept rcx, qword [rdx] + DB 0x66, 0x0F, 0x38, 0x80, 0xA + %endif +%else + mov ecx, [esp + 4] + mov edx, [esp + 8] + xor eax, eax +; invept ecx, qword [edx] + DB 0x66, 0x0F, 0x38, 0x80, 0xA +%endif + jnc .valid_vmcs + mov eax, VERR_VMX_INVALID_VMCS_PTR + ret +.valid_vmcs: + jnz .the_end + mov eax, VERR_INVALID_PARAMETER +.the_end: + ret +ENDPROC VMXR0InvEPT + + +;; +; Invalidate a page using INVVPID. +; +; @param enmTlbFlush msc:ecx gcc:edi x86:[esp+04] Type of flush +; @param pDescriptor msc:edx gcc:esi x86:[esp+08] Descriptor pointer +; +;DECLASM(int) VMXR0InvVPID(VMXTLBFLUSHVPID enmTlbFlush, uint64_t *pDescriptor); +BEGINPROC VMXR0InvVPID +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_GCC + and edi, 0ffffffffh + xor rax, rax +; invvpid rdi, qword [rsi] + DB 0x66, 0x0F, 0x38, 0x81, 0x3E + %else + and ecx, 0ffffffffh + xor rax, rax +; invvpid rcx, qword [rdx] + DB 0x66, 0x0F, 0x38, 0x81, 0xA + %endif +%else + mov ecx, [esp + 4] + mov edx, [esp + 8] + xor eax, eax +; invvpid ecx, qword [edx] + DB 0x66, 0x0F, 0x38, 0x81, 0xA +%endif + jnc .valid_vmcs + mov eax, VERR_VMX_INVALID_VMCS_PTR + ret +.valid_vmcs: + jnz .the_end + mov eax, VERR_INVALID_PARAMETER +.the_end: + ret +ENDPROC VMXR0InvVPID + + +%if GC_ARCH_BITS == 64 +;; +; Executes INVLPGA. +; +; @param pPageGC msc:rcx gcc:rdi x86:[esp+04] Virtual page to invalidate +; @param uASID msc:rdx gcc:rsi x86:[esp+0C] Tagged TLB id +; +;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID); +BEGINPROC SVMR0InvlpgA +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_GCC + mov rax, rdi + mov rcx, rsi + %else + mov rax, rcx + mov rcx, rdx + %endif +%else + mov eax, [esp + 4] + mov ecx, [esp + 0Ch] +%endif + invlpga [xAX], ecx + ret +ENDPROC SVMR0InvlpgA + +%else ; GC_ARCH_BITS != 64 +;; +; Executes INVLPGA +; +; @param pPageGC msc:ecx gcc:edi x86:[esp+04] Virtual page to invalidate +; @param uASID msc:edx gcc:esi x86:[esp+08] Tagged TLB id +; +;DECLASM(void) SVMR0InvlpgA(RTGCPTR pPageGC, uint32_t uASID); +BEGINPROC SVMR0InvlpgA +%ifdef RT_ARCH_AMD64 + %ifdef ASM_CALL64_GCC + movzx rax, edi + mov ecx, esi + %else + ; from http://www.cs.cmu.edu/~fp/courses/15213-s06/misc/asm64-handout.pdf: + ; "Perhaps unexpectedly, instructions that move or generate 32-bit register + ; values also set the upper 32 bits of the register to zero. Consequently + ; there is no need for an instruction movzlq." + mov eax, ecx + mov ecx, edx + %endif +%else + mov eax, [esp + 4] + mov ecx, [esp + 8] +%endif + invlpga [xAX], ecx + ret +ENDPROC SVMR0InvlpgA + +%endif ; GC_ARCH_BITS != 64 + + +%ifdef VBOX_WITH_KERNEL_USING_XMM + +;; +; Wrapper around vmx.pfnStartVM that preserves host XMM registers and +; load the guest ones when necessary. +; +; @cproto DECLASM(int) hmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, void *pvUnused, PVM pVM, +; PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM); +; +; @returns eax +; +; @param fResumeVM msc:rcx +; @param pCtx msc:rdx +; @param pvUnused msc:r8 +; @param pVM msc:r9 +; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT. +; @param pfnStartVM msc:[rbp+38h] +; +; @remarks This is essentially the same code as hmR0SVMRunWrapXMM, only the parameters differ a little bit. +; +; @remarks Drivers shouldn't use AVX registers without saving+loading: +; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396 +; However the compiler docs have different idea: +; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx +; We'll go with the former for now. +; +; ASSUMING 64-bit and windows for now. +; +ALIGNCODE(16) +BEGINPROC hmR0VMXStartVMWrapXMM + push xBP + mov xBP, xSP + sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size. + + ; Spill input parameters. + mov [xBP + 010h], rcx ; fResumeVM + mov [xBP + 018h], rdx ; pCtx + mov [xBP + 020h], r8 ; pvUnused + mov [xBP + 028h], r9 ; pVM + + ; Ask CPUM whether we've started using the FPU yet. + mov rcx, [xBP + 30h] ; pVCpu + call NAME(CPUMIsGuestFPUStateActive) + test al, al + jnz .guest_fpu_state_active + + ; No need to mess with XMM registers just call the start routine and return. + mov r11, [xBP + 38h] ; pfnStartVM + mov r10, [xBP + 30h] ; pVCpu + mov [xSP + 020h], r10 + mov rcx, [xBP + 010h] ; fResumeVM + mov rdx, [xBP + 018h] ; pCtx + mov r8, [xBP + 020h] ; pvUnused + mov r9, [xBP + 028h] ; pVM + call r11 + + leave + ret + +ALIGNCODE(8) +.guest_fpu_state_active: + ; Save the non-volatile host XMM registers. + movdqa [rsp + 040h + 000h], xmm6 + movdqa [rsp + 040h + 010h], xmm7 + movdqa [rsp + 040h + 020h], xmm8 + movdqa [rsp + 040h + 030h], xmm9 + movdqa [rsp + 040h + 040h], xmm10 + movdqa [rsp + 040h + 050h], xmm11 + movdqa [rsp + 040h + 060h], xmm12 + movdqa [rsp + 040h + 070h], xmm13 + movdqa [rsp + 040h + 080h], xmm14 + movdqa [rsp + 040h + 090h], xmm15 + stmxcsr [rsp + 040h + 0a0h] + + mov r10, [xBP + 018h] ; pCtx + mov eax, [r10 + CPUMCTX.fXStateMask] + test eax, eax + jz .guest_fpu_state_manually + + ; + ; Using XSAVE to load the guest XMM, YMM and ZMM registers. + ; + and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS + xor edx, edx + mov r10, [r10 + CPUMCTX.pXStateR0] + xrstor [r10] + + ; Make the call (same as in the other case). + mov r11, [xBP + 38h] ; pfnStartVM + mov r10, [xBP + 30h] ; pVCpu + mov [xSP + 020h], r10 + mov rcx, [xBP + 010h] ; fResumeVM + mov rdx, [xBP + 018h] ; pCtx + mov r8, [xBP + 020h] ; pvUnused + mov r9, [xBP + 028h] ; pVM + call r11 + + mov r11d, eax ; save return value (xsave below uses eax) + + ; Save the guest XMM registers. + mov r10, [xBP + 018h] ; pCtx + mov eax, [r10 + CPUMCTX.fXStateMask] + and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS + xor edx, edx + mov r10, [r10 + CPUMCTX.pXStateR0] + xsave [r10] + + mov eax, r11d ; restore return value + +.restore_non_volatile_host_xmm_regs: + ; Load the non-volatile host XMM registers. + movdqa xmm6, [rsp + 040h + 000h] + movdqa xmm7, [rsp + 040h + 010h] + movdqa xmm8, [rsp + 040h + 020h] + movdqa xmm9, [rsp + 040h + 030h] + movdqa xmm10, [rsp + 040h + 040h] + movdqa xmm11, [rsp + 040h + 050h] + movdqa xmm12, [rsp + 040h + 060h] + movdqa xmm13, [rsp + 040h + 070h] + movdqa xmm14, [rsp + 040h + 080h] + movdqa xmm15, [rsp + 040h + 090h] + ldmxcsr [rsp + 040h + 0a0h] + leave + ret + + ; + ; No XSAVE, load and save the guest XMM registers manually. + ; +.guest_fpu_state_manually: + ; Load the full guest XMM register state. + mov r10, [r10 + CPUMCTX.pXStateR0] + movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h] + movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h] + movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h] + movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h] + movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h] + movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h] + movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h] + movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h] + movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h] + movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h] + movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h] + movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h] + movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h] + movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h] + movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h] + movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h] + ldmxcsr [r10 + X86FXSTATE.MXCSR] + + ; Make the call (same as in the other case). + mov r11, [xBP + 38h] ; pfnStartVM + mov r10, [xBP + 30h] ; pVCpu + mov [xSP + 020h], r10 + mov rcx, [xBP + 010h] ; fResumeVM + mov rdx, [xBP + 018h] ; pCtx + mov r8, [xBP + 020h] ; pvUnused + mov r9, [xBP + 028h] ; pVM + call r11 + + ; Save the guest XMM registers. + mov r10, [xBP + 018h] ; pCtx + mov r10, [r10 + CPUMCTX.pXStateR0] + stmxcsr [r10 + X86FXSTATE.MXCSR] + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15 + jmp .restore_non_volatile_host_xmm_regs +ENDPROC hmR0VMXStartVMWrapXMM + +;; +; Wrapper around svm.pfnVMRun that preserves host XMM registers and +; load the guest ones when necessary. +; +; @cproto DECLASM(int) hmR0SVMRunWrapXMM(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu, +; PFNHMSVMVMRUN pfnVMRun); +; +; @returns eax +; +; @param HCPhysVmcbHost msc:rcx +; @param HCPhysVmcb msc:rdx +; @param pCtx msc:r8 +; @param pVM msc:r9 +; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT. +; @param pfnVMRun msc:[rbp+38h] +; +; @remarks This is essentially the same code as hmR0VMXStartVMWrapXMM, only the parameters differ a little bit. +; +; @remarks Drivers shouldn't use AVX registers without saving+loading: +; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396 +; However the compiler docs have different idea: +; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx +; We'll go with the former for now. +; +; ASSUMING 64-bit and windows for now. +ALIGNCODE(16) +BEGINPROC hmR0SVMRunWrapXMM + push xBP + mov xBP, xSP + sub xSP, 0b0h + 040h ; don't bother optimizing the frame size + + ; Spill input parameters. + mov [xBP + 010h], rcx ; HCPhysVmcbHost + mov [xBP + 018h], rdx ; HCPhysVmcb + mov [xBP + 020h], r8 ; pCtx + mov [xBP + 028h], r9 ; pVM + + ; Ask CPUM whether we've started using the FPU yet. + mov rcx, [xBP + 30h] ; pVCpu + call NAME(CPUMIsGuestFPUStateActive) + test al, al + jnz .guest_fpu_state_active + + ; No need to mess with XMM registers just call the start routine and return. + mov r11, [xBP + 38h] ; pfnVMRun + mov r10, [xBP + 30h] ; pVCpu + mov [xSP + 020h], r10 + mov rcx, [xBP + 010h] ; HCPhysVmcbHost + mov rdx, [xBP + 018h] ; HCPhysVmcb + mov r8, [xBP + 020h] ; pCtx + mov r9, [xBP + 028h] ; pVM + call r11 + + leave + ret + +ALIGNCODE(8) +.guest_fpu_state_active: + ; Save the non-volatile host XMM registers. + movdqa [rsp + 040h + 000h], xmm6 + movdqa [rsp + 040h + 010h], xmm7 + movdqa [rsp + 040h + 020h], xmm8 + movdqa [rsp + 040h + 030h], xmm9 + movdqa [rsp + 040h + 040h], xmm10 + movdqa [rsp + 040h + 050h], xmm11 + movdqa [rsp + 040h + 060h], xmm12 + movdqa [rsp + 040h + 070h], xmm13 + movdqa [rsp + 040h + 080h], xmm14 + movdqa [rsp + 040h + 090h], xmm15 + stmxcsr [rsp + 040h + 0a0h] + + mov r10, [xBP + 020h] ; pCtx + mov eax, [r10 + CPUMCTX.fXStateMask] + test eax, eax + jz .guest_fpu_state_manually + + ; + ; Using XSAVE. + ; + and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS + xor edx, edx + mov r10, [r10 + CPUMCTX.pXStateR0] + xrstor [r10] + + ; Make the call (same as in the other case). + mov r11, [xBP + 38h] ; pfnVMRun + mov r10, [xBP + 30h] ; pVCpu + mov [xSP + 020h], r10 + mov rcx, [xBP + 010h] ; HCPhysVmcbHost + mov rdx, [xBP + 018h] ; HCPhysVmcb + mov r8, [xBP + 020h] ; pCtx + mov r9, [xBP + 028h] ; pVM + call r11 + + mov r11d, eax ; save return value (xsave below uses eax) + + ; Save the guest XMM registers. + mov r10, [xBP + 020h] ; pCtx + mov eax, [r10 + CPUMCTX.fXStateMask] + and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS + xor edx, edx + mov r10, [r10 + CPUMCTX.pXStateR0] + xsave [r10] + + mov eax, r11d ; restore return value + +.restore_non_volatile_host_xmm_regs: + ; Load the non-volatile host XMM registers. + movdqa xmm6, [rsp + 040h + 000h] + movdqa xmm7, [rsp + 040h + 010h] + movdqa xmm8, [rsp + 040h + 020h] + movdqa xmm9, [rsp + 040h + 030h] + movdqa xmm10, [rsp + 040h + 040h] + movdqa xmm11, [rsp + 040h + 050h] + movdqa xmm12, [rsp + 040h + 060h] + movdqa xmm13, [rsp + 040h + 070h] + movdqa xmm14, [rsp + 040h + 080h] + movdqa xmm15, [rsp + 040h + 090h] + ldmxcsr [rsp + 040h + 0a0h] + leave + ret + + ; + ; No XSAVE, load and save the guest XMM registers manually. + ; +.guest_fpu_state_manually: + ; Load the full guest XMM register state. + mov r10, [r10 + CPUMCTX.pXStateR0] + movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h] + movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h] + movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h] + movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h] + movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h] + movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h] + movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h] + movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h] + movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h] + movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h] + movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h] + movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h] + movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h] + movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h] + movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h] + movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h] + ldmxcsr [r10 + X86FXSTATE.MXCSR] + + ; Make the call (same as in the other case). + mov r11, [xBP + 38h] ; pfnVMRun + mov r10, [xBP + 30h] ; pVCpu + mov [xSP + 020h], r10 + mov rcx, [xBP + 010h] ; HCPhysVmcbHost + mov rdx, [xBP + 018h] ; HCPhysVmcb + mov r8, [xBP + 020h] ; pCtx + mov r9, [xBP + 028h] ; pVM + call r11 + + ; Save the guest XMM registers. + mov r10, [xBP + 020h] ; pCtx + mov r10, [r10 + CPUMCTX.pXStateR0] + stmxcsr [r10 + X86FXSTATE.MXCSR] + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14 + movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15 + jmp .restore_non_volatile_host_xmm_regs +ENDPROC hmR0SVMRunWrapXMM + +%endif ; VBOX_WITH_KERNEL_USING_XMM + + +%ifdef RT_ARCH_AMD64 +;; @def RESTORE_STATE_VM64 +; Macro restoring essential host state and updating guest state +; for 64-bit host, 64-bit guest for VT-x. +; +%macro RESTORE_STATE_VM64 0 + ; Restore base and limit of the IDTR & GDTR. + %ifndef VMX_SKIP_IDTR + lidt [xSP] + add xSP, xCB * 2 + %endif + %ifndef VMX_SKIP_GDTR + lgdt [xSP] + add xSP, xCB * 2 + %endif + + push xDI + %ifndef VMX_SKIP_TR + mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR) + %else + mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR) + %endif + + mov qword [xDI + CPUMCTX.eax], rax + mov rax, SPECTRE_FILLER64 + mov qword [xDI + CPUMCTX.ebx], rbx + mov rbx, rax + mov qword [xDI + CPUMCTX.ecx], rcx + mov rcx, rax + mov qword [xDI + CPUMCTX.edx], rdx + mov rdx, rax + mov qword [xDI + CPUMCTX.esi], rsi + mov rsi, rax + mov qword [xDI + CPUMCTX.ebp], rbp + mov rbp, rax + mov qword [xDI + CPUMCTX.r8], r8 + mov r8, rax + mov qword [xDI + CPUMCTX.r9], r9 + mov r9, rax + mov qword [xDI + CPUMCTX.r10], r10 + mov r10, rax + mov qword [xDI + CPUMCTX.r11], r11 + mov r11, rax + mov qword [xDI + CPUMCTX.r12], r12 + mov r12, rax + mov qword [xDI + CPUMCTX.r13], r13 + mov r13, rax + mov qword [xDI + CPUMCTX.r14], r14 + mov r14, rax + mov qword [xDI + CPUMCTX.r15], r15 + mov r15, rax + mov rax, cr2 + mov qword [xDI + CPUMCTX.cr2], rax + + pop xAX ; The guest rdi we pushed above + mov qword [xDI + CPUMCTX.edi], rax + + ; Fight spectre. + INDIRECT_BRANCH_PREDICTION_BARRIER xDI, CPUMCTX_WSF_IBPB_EXIT + + %ifndef VMX_SKIP_TR + ; Restore TSS selector; must mark it as not busy before using ltr! + ; ASSUME that this is supposed to be 'BUSY' (saves 20-30 ticks on the T42p). + ; @todo get rid of sgdt + pop xBX ; Saved TR + sub xSP, xCB * 2 + sgdt [xSP] + mov xAX, xBX + and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset + add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset + and dword [xAX + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit) + ltr bx + add xSP, xCB * 2 + %endif + + pop xAX ; Saved LDTR + cmp eax, 0 + je %%skip_ldt_write64 + lldt ax + +%%skip_ldt_write64: + pop xSI ; pCtx (needed in rsi by the macros below) + + ; Restore segment registers. + MYPOPSEGS xAX, ax + + ; Restore the host XCR0 if necessary. + pop xCX + test ecx, ecx + jnz %%xcr0_after_skip + pop xAX + pop xDX + xsetbv ; ecx is already zero. +%%xcr0_after_skip: + + ; Restore general purpose registers. + MYPOPAD +%endmacro + + +;; +; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode) +; +; @returns VBox status code +; @param fResume msc:rcx, gcc:rdi Whether to use vmlauch/vmresume. +; @param pCtx msc:rdx, gcc:rsi Pointer to the guest-CPU context. +; @param pvUnused msc:r8, gcc:rdx Unused argument. +; @param pVM msc:r9, gcc:rcx The cross context VM structure. +; @param pVCpu msc:[ebp+30], gcc:r8 The cross context virtual CPU structure of the calling EMT. +; +ALIGNCODE(16) +BEGINPROC VMXR0StartVM64 + push xBP + mov xBP, xSP + + pushf + cli + + ; Save all general purpose host registers. + MYPUSHAD + + ; First we have to save some final CPU context registers. + lea r10, [.vmlaunch64_done wrt rip] + mov rax, VMX_VMCS_HOST_RIP ; return address (too difficult to continue after VMLAUNCH?) + vmwrite rax, r10 + ; Note: ASSUMES success! + + ; + ; Unify the input parameter registers. + ; +%ifdef ASM_CALL64_GCC + ; fResume already in rdi + ; pCtx already in rsi + mov rbx, rdx ; pvUnused +%else + mov rdi, rcx ; fResume + mov rsi, rdx ; pCtx + mov rbx, r8 ; pvUnused +%endif + + ; + ; Save the host XCR0 and load the guest one if necessary. + ; Note! Trashes rdx and rcx. + ; +%ifdef ASM_CALL64_MSC + mov rax, [xBP + 30h] ; pVCpu +%else + mov rax, r8 ; pVCpu +%endif + test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1 + jz .xcr0_before_skip + + xor ecx, ecx + xgetbv ; save the host one on the stack + push xDX + push xAX + + mov eax, [xSI + CPUMCTX.aXcr] ; load the guest one + mov edx, [xSI + CPUMCTX.aXcr + 4] + xor ecx, ecx ; paranoia + xsetbv + + push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0) + jmp .xcr0_before_done + +.xcr0_before_skip: + push 3fh ; indicate that we need not +.xcr0_before_done: + + ; + ; Save segment registers. + ; Note! Trashes rdx & rcx, so we moved it here (amd64 case). + ; + MYPUSHSEGS xAX, ax + + ; Save the pCtx pointer. + push xSI + + ; Save host LDTR. + xor eax, eax + sldt ax + push xAX + +%ifndef VMX_SKIP_TR + ; The host TR limit is reset to 0x67; save & restore it manually. + str eax + push xAX +%endif + +%ifndef VMX_SKIP_GDTR + ; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly! + sub xSP, xCB * 2 + sgdt [xSP] +%endif +%ifndef VMX_SKIP_IDTR + sub xSP, xCB * 2 + sidt [xSP] +%endif + + ; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction). + mov rbx, qword [xSI + CPUMCTX.cr2] + mov rdx, cr2 + cmp rbx, rdx + je .skip_cr2_write + mov cr2, rbx + +.skip_cr2_write: + mov eax, VMX_VMCS_HOST_RSP + vmwrite xAX, xSP + ; Note: ASSUMES success! + ; Don't mess with ESP anymore!!! + + ; Fight spectre and similar. + INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY, CPUMCTX_WSF_L1D_ENTRY, CPUMCTX_WSF_MDS_ENTRY + + ; Load guest general purpose registers. + mov rax, qword [xSI + CPUMCTX.eax] + mov rbx, qword [xSI + CPUMCTX.ebx] + mov rcx, qword [xSI + CPUMCTX.ecx] + mov rdx, qword [xSI + CPUMCTX.edx] + mov rbp, qword [xSI + CPUMCTX.ebp] + mov r8, qword [xSI + CPUMCTX.r8] + mov r9, qword [xSI + CPUMCTX.r9] + mov r10, qword [xSI + CPUMCTX.r10] + mov r11, qword [xSI + CPUMCTX.r11] + mov r12, qword [xSI + CPUMCTX.r12] + mov r13, qword [xSI + CPUMCTX.r13] + mov r14, qword [xSI + CPUMCTX.r14] + mov r15, qword [xSI + CPUMCTX.r15] + + ; Resume or start VM? + cmp xDI, 0 ; fResume + + ; Load guest rdi & rsi. + mov rdi, qword [xSI + CPUMCTX.edi] + mov rsi, qword [xSI + CPUMCTX.esi] + + je .vmlaunch64_launch + + vmresume + jc near .vmxstart64_invalid_vmcs_ptr + jz near .vmxstart64_start_failed + jmp .vmlaunch64_done; ; here if vmresume detected a failure + +.vmlaunch64_launch: + vmlaunch + jc near .vmxstart64_invalid_vmcs_ptr + jz near .vmxstart64_start_failed + jmp .vmlaunch64_done; ; here if vmlaunch detected a failure + +ALIGNCODE(16) +.vmlaunch64_done: + RESTORE_STATE_VM64 + mov eax, VINF_SUCCESS + +.vmstart64_end: + popf + pop xBP + ret + +.vmxstart64_invalid_vmcs_ptr: + RESTORE_STATE_VM64 + mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM + jmp .vmstart64_end + +.vmxstart64_start_failed: + RESTORE_STATE_VM64 + mov eax, VERR_VMX_UNABLE_TO_START_VM + jmp .vmstart64_end +ENDPROC VMXR0StartVM64 +%endif ; RT_ARCH_AMD64 + + +;; +; Clears the MDS buffers using VERW. +ALIGNCODE(16) +BEGINPROC hmR0MdsClear + sub xSP, xCB + mov [xSP], ds + verw [xSP] + add xSP, xCB + ret +ENDPROC hmR0MdsClear + + +%ifdef RT_ARCH_AMD64 +;; +; Prepares for and executes VMRUN (32-bit and 64-bit guests). +; +; @returns VBox status code +; @param HCPhysVmcbHost msc:rcx,gcc:rdi Physical address of host VMCB. +; @param HCPhysVmcb msc:rdx,gcc:rsi Physical address of guest VMCB. +; @param pCtx msc:r8,gcc:rdx Pointer to the guest-CPU context. +; @param pVM msc:r9,gcc:rcx The cross context VM structure. +; @param pVCpu msc:[rsp+28],gcc:r8 The cross context virtual CPU structure of the calling EMT. +; +ALIGNCODE(16) +BEGINPROC SVMR0VMRun + ; Fake a cdecl stack frame + %ifdef ASM_CALL64_GCC + push r8 ; pVCpu + push rcx ; pVM + push rdx ; pCtx + push rsi ; HCPhysVmcb + push rdi ; HCPhysVmcbHost + %else + mov rax, [rsp + 28h] + push rax ; rbp + 30h pVCpu + push r9 ; rbp + 28h pVM + push r8 ; rbp + 20h pCtx + push rdx ; rbp + 18h HCPhysVmcb + push rcx ; rbp + 10h HCPhysVmcbHost + %endif + push 0 ; rbp + 08h "fake ret addr" + push rbp ; rbp + 00h + mov rbp, rsp + pushf + + ; Manual save and restore: + ; - General purpose registers except RIP, RSP, RAX + ; + ; Trashed: + ; - CR2 (we don't care) + ; - LDTR (reset to 0) + ; - DRx (presumably not changed at all) + ; - DR7 (reset to 0x400) + + ; Save all general purpose host registers. + MYPUSHAD + + ; Load pCtx into xSI. + mov xSI, [rbp + xCB * 2 + RTHCPHYS_CB * 2] + + ; Save the host XCR0 and load the guest one if necessary. + mov rax, [xBP + 30h] ; pVCpu + test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1 + jz .xcr0_before_skip + + xor ecx, ecx + xgetbv ; save the host XCR0 on the stack + push xDX + push xAX + + mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx + mov eax, [xSI + CPUMCTX.aXcr] ; load the guest XCR0 + mov edx, [xSI + CPUMCTX.aXcr + 4] + xor ecx, ecx ; paranoia + xsetbv + + push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0) + jmp .xcr0_before_done + +.xcr0_before_skip: + push 3fh ; indicate that we need not restore XCR0 +.xcr0_before_done: + + ; Save guest CPU-context pointer for simplifying saving of the GPRs afterwards. + push rsi + + ; Save host fs, gs, sysenter msr etc. + mov rax, [rbp + xCB * 2] ; HCPhysVmcbHost (64 bits physical address; x86: take low dword only) + push rax ; save for the vmload after vmrun + vmsave + + ; Fight spectre. + INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY + + ; Setup rax for VMLOAD. + mov rax, [rbp + xCB * 2 + RTHCPHYS_CB] ; HCPhysVmcb (64 bits physical address; take low dword only) + + ; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN). + mov rbx, qword [xSI + CPUMCTX.ebx] + mov rcx, qword [xSI + CPUMCTX.ecx] + mov rdx, qword [xSI + CPUMCTX.edx] + mov rdi, qword [xSI + CPUMCTX.edi] + mov rbp, qword [xSI + CPUMCTX.ebp] + mov r8, qword [xSI + CPUMCTX.r8] + mov r9, qword [xSI + CPUMCTX.r9] + mov r10, qword [xSI + CPUMCTX.r10] + mov r11, qword [xSI + CPUMCTX.r11] + mov r12, qword [xSI + CPUMCTX.r12] + mov r13, qword [xSI + CPUMCTX.r13] + mov r14, qword [xSI + CPUMCTX.r14] + mov r15, qword [xSI + CPUMCTX.r15] + mov rsi, qword [xSI + CPUMCTX.esi] + + ; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch. + clgi + sti + + ; Load guest FS, GS, Sysenter MSRs etc. + vmload + + ; Run the VM. + vmrun + + ; Save guest fs, gs, sysenter msr etc. + vmsave + + ; Load host fs, gs, sysenter msr etc. + pop rax ; load HCPhysVmcbHost (pushed above) + vmload + + ; Set the global interrupt flag again, but execute cli to make sure IF=0. + cli + stgi + + ; Pop the context pointer (pushed above) and save the guest GPRs (sans RSP and RAX). + pop rax + + mov qword [rax + CPUMCTX.ebx], rbx + mov rbx, SPECTRE_FILLER64 + mov qword [rax + CPUMCTX.ecx], rcx + mov rcx, rbx + mov qword [rax + CPUMCTX.edx], rdx + mov rdx, rbx + mov qword [rax + CPUMCTX.esi], rsi + mov rsi, rbx + mov qword [rax + CPUMCTX.edi], rdi + mov rdi, rbx + mov qword [rax + CPUMCTX.ebp], rbp + mov rbp, rbx + mov qword [rax + CPUMCTX.r8], r8 + mov r8, rbx + mov qword [rax + CPUMCTX.r9], r9 + mov r9, rbx + mov qword [rax + CPUMCTX.r10], r10 + mov r10, rbx + mov qword [rax + CPUMCTX.r11], r11 + mov r11, rbx + mov qword [rax + CPUMCTX.r12], r12 + mov r12, rbx + mov qword [rax + CPUMCTX.r13], r13 + mov r13, rbx + mov qword [rax + CPUMCTX.r14], r14 + mov r14, rbx + mov qword [rax + CPUMCTX.r15], r15 + mov r15, rbx + + ; Fight spectre. Note! Trashes rax! + INDIRECT_BRANCH_PREDICTION_BARRIER rax, CPUMCTX_WSF_IBPB_EXIT + + ; Restore the host xcr0 if necessary. + pop xCX + test ecx, ecx + jnz .xcr0_after_skip + pop xAX + pop xDX + xsetbv ; ecx is already zero +.xcr0_after_skip: + + ; Restore host general purpose registers. + MYPOPAD + + mov eax, VINF_SUCCESS + + popf + pop rbp + add rsp, 6 * xCB + ret +ENDPROC SVMR0VMRun +%endif ; RT_ARCH_AMD64 + |