path: root/src/VBox/VMM/VMMAll/IEMAllAImpl.asm

; $Id: IEMAllAImpl.asm $
;; @file
; IEM - Instruction Implementation in Assembly.
;

;
; Copyright (C) 2011-2023 Oracle and/or its affiliates.
;
; This file is part of VirtualBox base platform packages, as
; available from https://www.virtualbox.org.
;
; This program is free software; you can redistribute it and/or
; modify it under the terms of the GNU General Public License
; as published by the Free Software Foundation, in version 3 of the
; License.
;
; This program is distributed in the hope that it will be useful, but
; WITHOUT ANY WARRANTY; without even the implied warranty of
; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
; General Public License for more details.
;
; You should have received a copy of the GNU General Public License
; along with this program; if not, see <https://www.gnu.org/licenses>.
;
; SPDX-License-Identifier: GPL-3.0-only
;


;*********************************************************************************************************************************
;*  Header Files                                                                                                                 *
;*********************************************************************************************************************************
%include "VBox/asmdefs.mac"
%include "VBox/err.mac"
%include "iprt/x86.mac"


;*********************************************************************************************************************************
;*  Defined Constants And Macros                                                                                                 *
;*********************************************************************************************************************************

;;
; RET XX / RET wrapper for fastcall.
;
%macro RET_FASTCALL 1
%ifdef RT_ARCH_X86
 %ifdef RT_OS_WINDOWS
    ret %1
 %else
    ret
 %endif
%else
    ret
%endif
%endmacro

;;
; NAME for fastcall functions.
;
;; @todo 'global @fastcall@12' is still broken in yasm and requires dollar
;         escaping (or whatever the dollar is good for here).  Thus the ugly
;         prefix argument.
;
%define NAME_FASTCALL(a_Name, a_cbArgs, a_Prefix)   NAME(a_Name)
%ifdef RT_ARCH_X86
 %ifdef RT_OS_WINDOWS
  %undef NAME_FASTCALL
  %define NAME_FASTCALL(a_Name, a_cbArgs, a_Prefix) a_Prefix %+ a_Name %+ @ %+ a_cbArgs
 %endif
%endif

;;
; BEGINPROC for fastcall functions.
;
; @param        1       The function name (C).
; @param        2       The argument size on x86.
;
%macro BEGINPROC_FASTCALL 2
 %ifdef ASM_FORMAT_PE
  export %1=NAME_FASTCALL(%1,%2,$@)
 %endif
 %ifdef __NASM__
  %ifdef ASM_FORMAT_OMF
   export NAME(%1) NAME_FASTCALL(%1,%2,$@)
  %endif
 %endif
 %ifndef ASM_FORMAT_BIN
  global NAME_FASTCALL(%1,%2,$@)
 %endif
NAME_FASTCALL(%1,%2,@):
        IBT_ENDBRxx
%endmacro


;
; We employ some macro assembly here to hid the calling convention differences.
;
%ifdef RT_ARCH_AMD64
 %macro PROLOGUE_1_ARGS 0
 %endmacro
 %macro EPILOGUE_1_ARGS 0
        ret
 %endmacro
 %macro EPILOGUE_1_ARGS_EX 0
        ret
 %endmacro

 %macro PROLOGUE_2_ARGS 0
 %endmacro
 %macro EPILOGUE_2_ARGS 0
        ret
 %endmacro
 %macro EPILOGUE_2_ARGS_EX 1
        ret
 %endmacro

 %macro PROLOGUE_3_ARGS 0
 %endmacro
 %macro EPILOGUE_3_ARGS 0
        ret
 %endmacro
 %macro EPILOGUE_3_ARGS_EX 1
        ret
 %endmacro

 %macro PROLOGUE_4_ARGS 0
 %endmacro
 %macro EPILOGUE_4_ARGS 0
        ret
 %endmacro
 %macro EPILOGUE_4_ARGS_EX 1
        ret
 %endmacro

 %ifdef ASM_CALL64_GCC
  %define A0        rdi
  %define A0_32     edi
  %define A0_16     di
  %define A0_8      dil

  %define A1        rsi
  %define A1_32     esi
  %define A1_16     si
  %define A1_8      sil

  %define A2        rdx
  %define A2_32     edx
  %define A2_16     dx
  %define A2_8      dl

  %define A3        rcx
  %define A3_32     ecx
  %define A3_16     cx
 %endif

 %ifdef ASM_CALL64_MSC
  %define A0        rcx
  %define A0_32     ecx
  %define A0_16     cx
  %define A0_8      cl

  %define A1        rdx
  %define A1_32     edx
  %define A1_16     dx
  %define A1_8      dl

  %define A2        r8
  %define A2_32     r8d
  %define A2_16     r8w
  %define A2_8      r8b

  %define A3        r9
  %define A3_32     r9d
  %define A3_16     r9w
 %endif

 %define T0         rax
 %define T0_32      eax
 %define T0_16      ax
 %define T0_8       al

 %define T1         r11
 %define T1_32      r11d
 %define T1_16      r11w
 %define T1_8       r11b

 %define T2         r10                 ; only AMD64
 %define T2_32      r10d
 %define T2_16      r10w
 %define T2_8       r10b

%else
 ; x86
 %macro PROLOGUE_1_ARGS 0
        push    edi
 %endmacro
 %macro EPILOGUE_1_ARGS 0
        pop     edi
        ret     0
 %endmacro
 %macro EPILOGUE_1_ARGS_EX 1
        pop     edi
        ret     %1
 %endmacro

 %macro PROLOGUE_2_ARGS 0
        push    edi
 %endmacro
 %macro EPILOGUE_2_ARGS 0
        pop     edi
        ret     0
 %endmacro
 %macro EPILOGUE_2_ARGS_EX 1
        pop     edi
        ret     %1
 %endmacro

 %macro PROLOGUE_3_ARGS 0
        push    ebx
        mov     ebx, [esp + 4 + 4]
        push    edi
 %endmacro
 %macro EPILOGUE_3_ARGS_EX 1
  %if (%1) < 4
   %error "With three args, at least 4 bytes must be remove from the stack upon return (32-bit)."
  %endif
        pop     edi
        pop     ebx
        ret     %1
 %endmacro
 %macro EPILOGUE_3_ARGS 0
        EPILOGUE_3_ARGS_EX 4
 %endmacro

 %macro PROLOGUE_4_ARGS 0
        push    ebx
        push    edi
        push    esi
        mov     ebx, [esp + 12 + 4 + 0]
        mov     esi, [esp + 12 + 4 + 4]
 %endmacro
 %macro EPILOGUE_4_ARGS_EX 1
  %if (%1) < 8
   %error "With four args, at least 8 bytes must be remove from the stack upon return (32-bit)."
  %endif
        pop     esi
        pop     edi
        pop     ebx
        ret     %1
 %endmacro
 %macro EPILOGUE_4_ARGS 0
        EPILOGUE_4_ARGS_EX 8
 %endmacro

 %define A0         ecx
 %define A0_32      ecx
 %define A0_16       cx
 %define A0_8        cl

 %define A1         edx
 %define A1_32      edx
 %define A1_16      dx
 %define A1_8       dl

 %define A2         ebx
 %define A2_32      ebx
 %define A2_16      bx
 %define A2_8       bl

 %define A3         esi
 %define A3_32      esi
 %define A3_16      si

 %define T0         eax
 %define T0_32      eax
 %define T0_16      ax
 %define T0_8       al

 %define T1         edi
 %define T1_32      edi
 %define T1_16      di
%endif


;;
; Load the relevant flags from [%1] if there are undefined flags (%3).
;
; @remarks      Clobbers T0, stack. Changes EFLAGS.
; @param        A2      The register pointing to the flags.
; @param        1       The parameter (A0..A3) pointing to the eflags.
; @param        2       The set of modified flags.
; @param        3       The set of undefined flags.
;
%macro IEM_MAYBE_LOAD_FLAGS 3
 ;%if (%3) != 0
        pushf                           ; store current flags
        mov     T0_32, [%1]             ; load the guest flags
        and     dword [xSP], ~(%2 | %3) ; mask out the modified and undefined flags
        and     T0_32, (%2 | %3)        ; select the modified and undefined flags.
        or      [xSP], T0               ; merge guest flags with host flags.
        popf                            ; load the mixed flags.
 ;%endif
%endmacro

;;
; Update the flag.
;
; @remarks  Clobbers T0, T1, stack.
; @param        1       The register pointing to the EFLAGS.
; @param        2       The mask of modified flags to save.
; @param        3       The mask of undefined flags to (maybe) save.
;
%macro IEM_SAVE_FLAGS 3
 %if (%2 | %3) != 0
        pushf
        pop     T1
        mov     T0_32, [%1]             ; flags
        and     T0_32, ~(%2 | %3)       ; clear the modified & undefined flags.
        and     T1_32, (%2 | %3)        ; select the modified and undefined flags.
        or      T0_32, T1_32            ; combine the flags.
        mov     [%1], T0_32             ; save the flags.
 %endif
%endmacro

;;
; Calculates the new EFLAGS based on the CPU EFLAGS and fixed clear and set bit masks.
;
; @remarks  Clobbers T0, T1, stack.
; @param        1       The register pointing to the EFLAGS.
; @param        2       The mask of modified flags to save.
; @param        3       Mask of additional flags to always clear
; @param        4       Mask of additional flags to always set.
;
%macro IEM_SAVE_AND_ADJUST_FLAGS 4
 %if (%2 | %3 | %4) != 0
        pushf
        pop     T1
        mov     T0_32, [%1]             ; load flags.
        and     T0_32, ~(%2 | %3)       ; clear the modified and always cleared flags.
        and     T1_32, (%2)             ; select the modified flags.
        or      T0_32, T1_32            ; combine the flags.
  %if (%4) != 0
        or      T0_32, %4               ; add the always set flags.
  %endif
        mov     [%1], T0_32             ; save the result.
 %endif
%endmacro

;;
; Calculates the new EFLAGS based on the CPU EFLAGS (%2), a clear mask (%3),
; signed input (%4[%5]) and parity index (%6).
;
; This is used by MUL and IMUL, where we got result (%4 & %6) in xAX which is
; also T0. So, we have to use T1 for the EFLAGS calculation and save T0/xAX
; while we extract the %2 flags from the CPU EFLAGS or use T2 (only AMD64).
;
; @remarks  Clobbers T0, T1, stack, %6, EFLAGS.
; @param        1       The register pointing to the EFLAGS.
; @param        2       The mask of modified flags to save.
; @param        3       Mask of additional flags to always clear
; @param        4       The result register to set SF by.
; @param        5       The width of the %4 register in bits (8, 16, 32, or 64).
; @param        6       The (full) register containing the parity table index. Will be modified!

%macro IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF 6
 %ifdef RT_ARCH_AMD64
        pushf
        pop     T2
 %else
        push    T0
        pushf
        pop     T0
 %endif
        mov     T1_32, [%1]             ; load flags.
        and     T1_32, ~(%2 | %3 | X86_EFL_PF | X86_EFL_SF)  ; clear the modified, always cleared flags and the two flags we calc.
 %ifdef RT_ARCH_AMD64
        and     T2_32, (%2)             ; select the modified flags.
        or      T1_32, T2_32            ; combine the flags.
 %else
        and     T0_32, (%2)             ; select the modified flags.
        or      T1_32, T0_32            ; combine the flags.
        pop     T0
 %endif

        ; First calculate SF as it's likely to be refereing to the same register as %6 does.
        bt      %4, %5 - 1
        jnc     %%sf_clear
        or      T1_32, X86_EFL_SF
 %%sf_clear:

        ; Parity last.
        and     %6, 0xff
 %ifdef RT_ARCH_AMD64
        lea     T2, [NAME(g_afParity) xWrtRIP]
        or      T1_8, [T2 + %6]
 %else
        or      T1_8, [NAME(g_afParity) + %6]
 %endif

        mov     [%1], T1_32             ; save the result.
%endmacro

;;
; Calculates the new EFLAGS using fixed clear and set bit masks.
;
; @remarks  Clobbers T0.
; @param        1       The register pointing to the EFLAGS.
; @param        2       Mask of additional flags to always clear
; @param        3       Mask of additional flags to always set.
;
%macro IEM_ADJUST_FLAGS 3
 %if (%2 | %3) != 0
        mov     T0_32, [%1]             ; Load flags.
  %if (%2) != 0
        and     T0_32, ~(%2)            ; Remove the always cleared flags.
  %endif
  %if (%3) != 0
        or      T0_32, %3               ; Add the always set flags.
  %endif
        mov     [%1], T0_32             ; Save the result.
 %endif
%endmacro

;;
; Calculates the new EFLAGS using fixed clear and set bit masks.
;
; @remarks  Clobbers T0, %4, EFLAGS.
; @param        1       The register pointing to the EFLAGS.
; @param        2       Mask of additional flags to always clear
; @param        3       Mask of additional flags to always set.
; @param        4       The (full) register containing the parity table index. Will be modified!
;
%macro IEM_ADJUST_FLAGS_WITH_PARITY 4
        mov     T0_32, [%1]                 ; Load flags.
        and     T0_32, ~(%2 | X86_EFL_PF)   ; Remove PF and the always cleared flags.
 %if (%3) != 0
        or      T0_32, %3                   ; Add the always set flags.
 %endif
        and     %4, 0xff
 %ifdef RT_ARCH_AMD64
        lea     T2, [NAME(g_afParity) xWrtRIP]
        or      T0_8, [T2 + %4]
 %else
        or      T0_8, [NAME(g_afParity) + %4]
 %endif
        mov     [%1], T0_32             ; Save the result.
%endmacro


;;
; Checks that the size expression %1 matches %2 adjusted according to
; RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK and for 256 entries.
; @param 1  The jump array size assembly expression.
; @param 2  The size without accounting for the IBT_ENDBRxx_WITHOUT_NOTRACK instruction.
;
%macro IEMCHECK_256_JUMP_ARRAY_SIZE 2
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        dw      (0xffff - %2 - 256*4) + %1  ; will cause warning if entries are too big.
        dw      (0xffff + %2 + 256*4) - %1  ; will cause warning if entries are too small.
 %else
        dw      (0xffff - %2)         + %1  ; will cause warning if entries are too big.
        dw      (0xffff + %2)         - %1  ; will cause warning if entries are too small.
 %endif
%endmacro


;*********************************************************************************************************************************
;*  External Symbols                                                                                                             *
;*********************************************************************************************************************************
extern NAME(g_afParity)


;;
; Macro for implementing a binary operator.
;
; This will generate code for the 8, 16, 32 and 64 bit accesses with locked
; variants, except on 32-bit system where the 64-bit accesses requires hand
; coding.
;
; All the functions takes a pointer to the destination memory operand in A0,
; the source register operand in A1 and a pointer to eflags in A2.
;
; @param        1       The instruction mnemonic.
; @param        2       Non-zero if there should be a locked version.
; @param        3       The modified flags.
; @param        4       The undefined flags.
;
%macro IEMIMPL_BIN_OP 4
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        %1      byte [A0], A1_8
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u8

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        %1      word [A0], A1_16
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        %1      dword [A0], A1_32
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        %1      qword [A0], A1
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS_EX 8
ENDPROC iemAImpl_ %+ %1 %+ _u64
  %endif ; RT_ARCH_AMD64

 %if %2 != 0 ; locked versions requested?

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8_locked, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        lock %1 byte [A0], A1_8
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u8_locked

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        lock %1 word [A0], A1_16
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16_locked

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        lock %1 dword [A0], A1_32
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32_locked

  %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 16
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        lock %1 qword [A0], A1
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS_EX 8
ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
  %endif ; RT_ARCH_AMD64
 %endif ; locked
%endmacro

;            instr,lock, modified-flags,                                                               undefined flags
IEMIMPL_BIN_OP add,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
IEMIMPL_BIN_OP adc,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
IEMIMPL_BIN_OP sub,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
IEMIMPL_BIN_OP sbb,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
IEMIMPL_BIN_OP or,   1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF
IEMIMPL_BIN_OP xor,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF
IEMIMPL_BIN_OP and,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF
IEMIMPL_BIN_OP cmp,  0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
IEMIMPL_BIN_OP test, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF


;;
; Macro for implementing a binary operator, VEX variant with separate input/output.
;
; This will generate code for the 32 and 64 bit accesses, except on 32-bit system
; where the 64-bit accesses requires hand coding.
;
; All the functions takes a pointer to the destination memory operand in A0,
; the first source register operand in A1, the second source register operand
; in A2 and a pointer to eflags in A3.
;
; @param        1       The instruction mnemonic.
; @param        2       The modified flags.
; @param        3       The undefined flags.
;
%macro IEMIMPL_VEX_BIN_OP 3
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS           A3, %2, %3
        %1      T0_32, A1_32, A2_32
        mov     [A0], T0_32
        IEM_SAVE_FLAGS                 A3, %2, %3
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS           A3, %2, %3
        %1      T0, A1, A2
        mov     [A0], T0
        IEM_SAVE_FLAGS                 A3, %2, %3
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64
 %endif ; RT_ARCH_AMD64
%endmacro

;                 instr,  modified-flags,                                                                undefined-flags
IEMIMPL_VEX_BIN_OP andn,  (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF),                           (X86_EFL_AF | X86_EFL_PF)
IEMIMPL_VEX_BIN_OP bextr, (X86_EFL_OF | X86_EFL_ZF | X86_EFL_CF),                                        (X86_EFL_SF | X86_EFL_AF | X86_EFL_PF)
IEMIMPL_VEX_BIN_OP bzhi,  (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF),                           (X86_EFL_AF | X86_EFL_PF)

;;
; Macro for implementing BLSR, BLCMSK and BLSI (fallbacks implemented in C).
;
; This will generate code for the 32 and 64 bit accesses, except on 32-bit system
; where the 64-bit accesses requires hand coding.
;
; All the functions takes a pointer to the destination memory operand in A0,
; the source register operand in A1 and a pointer to eflags in A2.
;
; @param        1       The instruction mnemonic.
; @param        2       The modified flags.
; @param        3       The undefined flags.
;
%macro IEMIMPL_VEX_BIN_OP_2 3
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
        mov     T0_32, [A0]
        %1      T0_32, A1_32
        mov     [A0], T0_32
        IEM_SAVE_FLAGS                 A2, %2, %3
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
        mov     T0, [A0]
        %1      T0, A1
        mov     [A0], T0
        IEM_SAVE_FLAGS                 A2, %2, %3
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64
 %endif ; RT_ARCH_AMD64
%endmacro

;                  instr,  modified-flags,                                      undefined-flags
IEMIMPL_VEX_BIN_OP_2 blsr,   (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF)
IEMIMPL_VEX_BIN_OP_2 blsmsk, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF)
IEMIMPL_VEX_BIN_OP_2 blsi,   (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF)


;;
; Macro for implementing a binary operator w/o flags, VEX variant with separate input/output.
;
; This will generate code for the 32 and 64 bit accesses, except on 32-bit system
; where the 64-bit accesses requires hand coding.
;
; All the functions takes a pointer to the destination memory operand in A0,
; the first source register operand in A1, the second source register operand
; in A2 and a pointer to eflags in A3.
;
; @param        1       The instruction mnemonic.
; @param        2       Fallback instruction if applicable.
; @param        3       Whether to emit fallback or not.
;
%macro IEMIMPL_VEX_BIN_OP_NOEFL 3
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
        PROLOGUE_3_ARGS
        %1      T0_32, A1_32, A2_32
        mov     [A0], T0_32
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

 %if %3
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_fallback, 12
        PROLOGUE_3_ARGS
  %ifdef ASM_CALL64_GCC
        mov     cl, A2_8
        %2      A1_32, cl
        mov     [A0], A1_32
  %else
        xchg    A2, A0
        %2      A1_32, cl
        mov     [A2], A1_32
  %endif
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32_fallback
 %endif

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
        PROLOGUE_3_ARGS
        %1      T0, A1, A2
        mov     [A0], T0
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64

 %if %3
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_fallback, 12
        PROLOGUE_3_ARGS
  %ifdef ASM_CALL64_GCC
        mov     cl, A2_8
        %2      A1, cl
        mov     [A0], A1_32
  %else
        xchg    A2, A0
        %2      A1, cl
        mov     [A2], A1_32
  %endif
        mov     [A0], A1
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64_fallback
  %endif
 %endif ; RT_ARCH_AMD64
%endmacro

;                           instr, fallback instr, emit fallback
IEMIMPL_VEX_BIN_OP_NOEFL    sarx,  sar,            1
IEMIMPL_VEX_BIN_OP_NOEFL    shlx,  shl,            1
IEMIMPL_VEX_BIN_OP_NOEFL    shrx,  shr,            1
IEMIMPL_VEX_BIN_OP_NOEFL    pdep,  nop,            0
IEMIMPL_VEX_BIN_OP_NOEFL    pext,  nop,            0


;
; RORX uses a immediate byte for the shift count, so we only do
; fallback implementation of that one.
;
BEGINPROC_FASTCALL iemAImpl_rorx_u32, 12
        PROLOGUE_3_ARGS
 %ifdef ASM_CALL64_GCC
        mov     cl, A2_8
        ror     A1_32, cl
        mov     [A0], A1_32
 %else
        xchg    A2, A0
        ror     A1_32, cl
        mov     [A2], A1_32
 %endif
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_rorx_u32

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_rorx_u64, 12
        PROLOGUE_3_ARGS
 %ifdef ASM_CALL64_GCC
        mov     cl, A2_8
        ror     A1, cl
        mov     [A0], A1_32
 %else
        xchg    A2, A0
        ror     A1, cl
        mov     [A2], A1_32
 %endif
        mov     [A0], A1
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_rorx_u64
 %endif ; RT_ARCH_AMD64


;
; MULX
;
BEGINPROC_FASTCALL iemAImpl_mulx_u32, 16
        PROLOGUE_4_ARGS
%ifdef ASM_CALL64_GCC
        ; A2_32 is EDX - prefect
        mulx    T0_32, T1_32, A3_32
        mov     [A1], T1_32 ; Low value first, as we should return the high part if same destination registers.
        mov     [A0], T0_32
%else
        ; A1 is xDX - must switch A1 and A2, so EDX=uSrc1
        xchg    A1, A2
        mulx    T0_32, T1_32, A3_32
        mov     [A2], T1_32 ; Low value first, as we should return the high part if same destination registers.
        mov     [A0], T0_32
%endif
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_mulx_u32


BEGINPROC_FASTCALL iemAImpl_mulx_u32_fallback, 16
        PROLOGUE_4_ARGS
%ifdef ASM_CALL64_GCC
        ; A2_32 is EDX, T0_32 is EAX
        mov     eax, A3_32
        mul     A2_32
        mov     [A1], eax ; Low value first, as we should return the high part if same destination registers.
        mov     [A0], edx
%else
        ; A1 is xDX, T0_32 is EAX - must switch A1 and A2, so EDX=uSrc1
        xchg    A1, A2
        mov     eax, A3_32
        mul     A2_32
        mov     [A2], eax ; Low value first, as we should return the high part if same destination registers.
        mov     [A0], edx
%endif
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_mulx_u32_fallback

%ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_mulx_u64, 16
        PROLOGUE_4_ARGS
%ifdef ASM_CALL64_GCC
        ; A2 is RDX - prefect
        mulx    T0, T1, A3
        mov     [A1], T1 ; Low value first, as we should return the high part if same destination registers.
        mov     [A0], T0
%else
        ; A1 is xDX - must switch A1 and A2, so RDX=uSrc1
        xchg    A1, A2
        mulx    T0, T1, A3
        mov     [A2], T1 ; Low value first, as we should return the high part if same destination registers.
        mov     [A0], T0
%endif
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_mulx_u64


BEGINPROC_FASTCALL iemAImpl_mulx_u64_fallback, 16
        PROLOGUE_4_ARGS
%ifdef ASM_CALL64_GCC
        ; A2 is RDX, T0 is RAX
        mov     rax, A3
        mul     A2
        mov     [A1], rax ; Low value first, as we should return the high part if same destination registers.
        mov     [A0], rdx
%else
        ; A1 is xDX, T0 is RAX - must switch A1 and A2, so RDX=uSrc1
        xchg    A1, A2
        mov     rax, A3
        mul     A2
        mov     [A2], rax ; Low value first, as we should return the high part if same destination registers.
        mov     [A0], rdx
%endif
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_mulx_u64_fallback

%endif


;;
; Macro for implementing a bit operator.
;
; This will generate code for the 16, 32 and 64 bit accesses with locked
; variants, except on 32-bit system where the 64-bit accesses requires hand
; coding.
;
; All the functions takes a pointer to the destination memory operand in A0,
; the source register operand in A1 and a pointer to eflags in A2.
;
; @param        1       The instruction mnemonic.
; @param        2       Non-zero if there should be a locked version.
; @param        3       The modified flags.
; @param        4       The undefined flags.
;
%macro IEMIMPL_BIT_OP 4
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        %1      word [A0], A1_16
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        %1      dword [A0], A1_32
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        %1      qword [A0], A1
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS_EX 8
ENDPROC iemAImpl_ %+ %1 %+ _u64
  %endif ; RT_ARCH_AMD64

 %if %2 != 0 ; locked versions requested?

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        lock %1 word [A0], A1_16
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16_locked

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        lock %1 dword [A0], A1_32
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32_locked

  %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 16
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
        lock %1 qword [A0], A1
        IEM_SAVE_FLAGS                 A2, %3, %4
        EPILOGUE_3_ARGS_EX 8
ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
  %endif ; RT_ARCH_AMD64
 %endif ; locked
%endmacro
IEMIMPL_BIT_OP bt,  0, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
IEMIMPL_BIT_OP btc, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
IEMIMPL_BIT_OP bts, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
IEMIMPL_BIT_OP btr, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)

;;
; Macro for implementing a bit search operator.
;
; This will generate code for the 16, 32 and 64 bit accesses, except on 32-bit
; system where the 64-bit accesses requires hand coding.
;
; All the functions takes a pointer to the destination memory operand in A0,
; the source register operand in A1 and a pointer to eflags in A2.
;
; In the ZF case the destination register is 'undefined', however it seems that
; both AMD and Intel just leaves it as is.  The undefined EFLAGS differs between
; AMD and Intel and accoridng to https://www.sandpile.org/x86/flags.htm between
; Intel microarchitectures.  We only implement 'intel' and 'amd' variation with
; the behaviour of more recent CPUs (Intel 10980X and AMD 3990X).
;
; @param        1       The instruction mnemonic.
; @param        2       The modified flags.
; @param        3       The undefined flags.
; @param        4       Non-zero if destination isn't written when ZF=1.  Zero if always written.
;
%macro IEMIMPL_BIT_OP2 4
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
        %1      T0_16, A1_16
%if %4 != 0
        jz      .unchanged_dst
%endif
        mov     [A0], T0_16
.unchanged_dst:
        IEM_SAVE_FLAGS                 A2, %2, %3
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _intel, 12
        PROLOGUE_3_ARGS
        %1      T1_16, A1_16
%if %4 != 0
        jz      .unchanged_dst
%endif
        mov     [A0], T1_16
        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
        EPILOGUE_3_ARGS
.unchanged_dst:
        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16_intel

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _amd, 12
        PROLOGUE_3_ARGS
        %1      T0_16, A1_16
%if %4 != 0
        jz      .unchanged_dst
%endif
        mov     [A0], T0_16
.unchanged_dst:
        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16_amd


BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
        %1      T0_32, A1_32
%if %4 != 0
        jz      .unchanged_dst
%endif
        mov     [A0], T0_32
.unchanged_dst:
        IEM_SAVE_FLAGS                 A2, %2, %3
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _intel, 12
        PROLOGUE_3_ARGS
        %1      T1_32, A1_32
%if %4 != 0
        jz      .unchanged_dst
%endif
        mov     [A0], T1_32
        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
        EPILOGUE_3_ARGS
.unchanged_dst:
        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32_intel

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _amd, 12
        PROLOGUE_3_ARGS
        %1      T0_32, A1_32
%if %4 != 0
        jz      .unchanged_dst
%endif
        mov     [A0], T0_32
.unchanged_dst:
        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32_amd


 %ifdef RT_ARCH_AMD64

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
        %1      T0, A1
%if %4 != 0
        jz      .unchanged_dst
%endif
        mov     [A0], T0
.unchanged_dst:
        IEM_SAVE_FLAGS                 A2, %2, %3
        EPILOGUE_3_ARGS_EX 8
ENDPROC iemAImpl_ %+ %1 %+ _u64

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _intel, 16
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
        %1      T1, A1
%if %4 != 0
        jz      .unchanged_dst
%endif
        mov     [A0], T1
        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
        EPILOGUE_3_ARGS
.unchanged_dst:
        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64_intel

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _amd, 16
        PROLOGUE_3_ARGS
        %1      T0, A1
%if %4 != 0
        jz      .unchanged_dst
%endif
        mov     [A0], T0
.unchanged_dst:
        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
        EPILOGUE_3_ARGS_EX 8
ENDPROC iemAImpl_ %+ %1 %+ _u64_amd

 %endif ; RT_ARCH_AMD64
%endmacro

IEMIMPL_BIT_OP2 bsf,   (X86_EFL_ZF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1
IEMIMPL_BIT_OP2 bsr,   (X86_EFL_ZF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1
IEMIMPL_BIT_OP2 tzcnt, (X86_EFL_ZF | X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0
IEMIMPL_BIT_OP2 lzcnt, (X86_EFL_ZF | X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0


;;
; Macro for implementing POPCNT.
;
; This will generate code for the 16, 32 and 64 bit accesses, except on 32-bit
; system where the 64-bit accesses requires hand coding.
;
; All the functions takes a pointer to the destination memory operand in A0,
; the source register operand in A1 and a pointer to eflags in A2.
;
; ASSUMES Intel and AMD set EFLAGS the same way.
;
; ASSUMES the instruction does not support memory destination.
;
; @param        1       The instruction mnemonic.
; @param        2       The modified flags.
; @param        3       The undefined flags.
;
%macro IEMIMPL_BIT_OP3 3
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
        %1      T0_16, A1_16
        mov     [A0], T0_16
        IEM_SAVE_FLAGS                 A2, %2, %3
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
        %1      T0_32, A1_32
        mov     [A0], T0_32
        IEM_SAVE_FLAGS                 A2, %2, %3
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
        %1      T0, A1
        mov     [A0], T0
        IEM_SAVE_FLAGS                 A2, %2, %3
        EPILOGUE_3_ARGS_EX 8
ENDPROC iemAImpl_ %+ %1 %+ _u64
 %endif ; RT_ARCH_AMD64
%endmacro
IEMIMPL_BIT_OP3 popcnt, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0


;
; IMUL is also a similar but yet different case (no lock, no mem dst).
; The rDX:rAX variant of imul is handled together with mul further down.
;
BEGINCODE
; @param        1       EFLAGS that are modified.
; @param        2       Undefined EFLAGS.
; @param        3       Function suffix.
; @param        4       EFLAGS variation: 0 for native, 1 for intel (ignored),
;                       2 for AMD (set AF, clear PF, ZF and SF).
%macro IEMIMPL_IMUL_TWO 4
BEGINPROC_FASTCALL iemAImpl_imul_two_u16 %+ %3, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2
        imul    A1_16, word [A0]
        mov     [A0], A1_16
 %if %4 != 1
        IEM_SAVE_FLAGS                          A2, %1, %2
 %else
        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_16, 16, A1
 %endif
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_imul_two_u16 %+ %3

BEGINPROC_FASTCALL iemAImpl_imul_two_u32 %+ %3, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2
        imul    A1_32, dword [A0]
        mov     [A0], A1_32
 %if %4 != 1
        IEM_SAVE_FLAGS                          A2, %1, %2
 %else
        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_32, 32, A1
 %endif
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_imul_two_u32 %+ %3

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_imul_two_u64 %+ %3, 16
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2
        imul    A1, qword [A0]
        mov     [A0], A1
 %if %4 != 1
        IEM_SAVE_FLAGS                          A2, %1, %2
 %else
        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1, 64, A1
 %endif
        EPILOGUE_3_ARGS_EX 8
ENDPROC iemAImpl_imul_two_u64 %+ %3
 %endif ; RT_ARCH_AMD64
%endmacro
IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF,       , 0
IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, 0,                                                 _intel, 1
IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, 0,                                                 _amd,   2


;
; XCHG for memory operands.  This implies locking.  No flag changes.
;
; Each function takes two arguments, first the pointer to the memory,
; then the pointer to the register.  They all return void.
;
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_xchg_u8_locked, 8
        PROLOGUE_2_ARGS
        mov     T0_8, [A1]
        xchg    [A0], T0_8
        mov     [A1], T0_8
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_xchg_u8_locked

BEGINPROC_FASTCALL iemAImpl_xchg_u16_locked, 8
        PROLOGUE_2_ARGS
        mov     T0_16, [A1]
        xchg    [A0], T0_16
        mov     [A1], T0_16
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_xchg_u16_locked

BEGINPROC_FASTCALL iemAImpl_xchg_u32_locked, 8
        PROLOGUE_2_ARGS
        mov     T0_32, [A1]
        xchg    [A0], T0_32
        mov     [A1], T0_32
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_xchg_u32_locked

%ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_xchg_u64_locked, 8
        PROLOGUE_2_ARGS
        mov     T0, [A1]
        xchg    [A0], T0
        mov     [A1], T0
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_xchg_u64_locked
%endif

; Unlocked variants for fDisregardLock mode.

BEGINPROC_FASTCALL iemAImpl_xchg_u8_unlocked, 8
        PROLOGUE_2_ARGS
        mov     T0_8, [A1]
        mov     T1_8, [A0]
        mov     [A0], T0_8
        mov     [A1], T1_8
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_xchg_u8_unlocked

BEGINPROC_FASTCALL iemAImpl_xchg_u16_unlocked, 8
        PROLOGUE_2_ARGS
        mov     T0_16, [A1]
        mov     T1_16, [A0]
        mov     [A0], T0_16
        mov     [A1], T1_16
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_xchg_u16_unlocked

BEGINPROC_FASTCALL iemAImpl_xchg_u32_unlocked, 8
        PROLOGUE_2_ARGS
        mov     T0_32, [A1]
        mov     T1_32, [A0]
        mov     [A0], T0_32
        mov     [A1], T1_32
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_xchg_u32_unlocked

%ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_xchg_u64_unlocked, 8
        PROLOGUE_2_ARGS
        mov     T0, [A1]
        mov     T1, [A0]
        mov     [A0], T0
        mov     [A1], T1
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_xchg_u64_unlocked
%endif


;
; XADD for memory operands.
;
; Each function takes three arguments, first the pointer to the
; memory/register, then the pointer to the register, and finally a pointer to
; eflags.  They all return void.
;
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_xadd_u8, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        mov     T0_8, [A1]
        xadd    [A0], T0_8
        mov     [A1], T0_8
        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_xadd_u8

BEGINPROC_FASTCALL iemAImpl_xadd_u16, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        mov     T0_16, [A1]
        xadd    [A0], T0_16
        mov     [A1], T0_16
        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_xadd_u16

BEGINPROC_FASTCALL iemAImpl_xadd_u32, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        mov     T0_32, [A1]
        xadd    [A0], T0_32
        mov     [A1], T0_32
        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_xadd_u32

%ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_xadd_u64, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        mov     T0, [A1]
        xadd    [A0], T0
        mov     [A1], T0
        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_xadd_u64
%endif ; RT_ARCH_AMD64

BEGINPROC_FASTCALL iemAImpl_xadd_u8_locked, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        mov     T0_8, [A1]
        lock xadd [A0], T0_8
        mov     [A1], T0_8
        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_xadd_u8_locked

BEGINPROC_FASTCALL iemAImpl_xadd_u16_locked, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        mov     T0_16, [A1]
        lock xadd [A0], T0_16
        mov     [A1], T0_16
        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_xadd_u16_locked

BEGINPROC_FASTCALL iemAImpl_xadd_u32_locked, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        mov     T0_32, [A1]
        lock xadd [A0], T0_32
        mov     [A1], T0_32
        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_xadd_u32_locked

%ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_xadd_u64_locked, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        mov     T0, [A1]
        lock xadd [A0], T0
        mov     [A1], T0
        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_xadd_u64_locked
%endif ; RT_ARCH_AMD64


;
; CMPXCHG8B.
;
; These are tricky register wise, so the code is duplicated for each calling
; convention.
;
; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to!
;
; C-proto:
; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
;                                             uint32_t *pEFlags));
;
; Note! Identical to iemAImpl_cmpxchg16b.
;
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_cmpxchg8b, 16
%ifdef RT_ARCH_AMD64
 %ifdef ASM_CALL64_MSC
        push    rbx

        mov     r11, rdx                ; pu64EaxEdx (is also T1)
        mov     r10, rcx                ; pu64Dst

        mov     ebx, [r8]
        mov     ecx, [r8 + 4]
        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
        mov     eax, [r11]
        mov     edx, [r11 + 4]

        cmpxchg8b [r10]

        mov     [r11], eax
        mov     [r11 + 4], edx
        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)

        pop     rbx
        ret
 %else
        push    rbx

        mov     r10, rcx                ; pEFlags
        mov     r11, rdx                ; pu64EbxEcx (is also T1)

        mov     ebx, [r11]
        mov     ecx, [r11 + 4]
        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
        mov     eax, [rsi]
        mov     edx, [rsi + 4]

        cmpxchg8b [rdi]

        mov     [rsi], eax
        mov     [rsi + 4], edx
        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)

        pop     rbx
        ret

 %endif
%else
        push    esi
        push    edi
        push    ebx
        push    ebp

        mov     edi, ecx                ; pu64Dst
        mov     esi, edx                ; pu64EaxEdx
        mov     ecx, [esp + 16 + 4 + 0] ; pu64EbxEcx
        mov     ebp, [esp + 16 + 4 + 4] ; pEFlags

        mov     ebx, [ecx]
        mov     ecx, [ecx + 4]
        IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF), 0  ; clobbers T0 (eax)
        mov     eax, [esi]
        mov     edx, [esi + 4]

        cmpxchg8b [edi]

        mov     [esi], eax
        mov     [esi + 4], edx
        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, edi)

        pop     ebp
        pop     ebx
        pop     edi
        pop     esi
        ret     8
%endif
ENDPROC iemAImpl_cmpxchg8b

BEGINPROC_FASTCALL iemAImpl_cmpxchg8b_locked, 16
%ifdef RT_ARCH_AMD64
 %ifdef ASM_CALL64_MSC
        push    rbx

        mov     r11, rdx                ; pu64EaxEdx (is also T1)
        mov     r10, rcx                ; pu64Dst

        mov     ebx, [r8]
        mov     ecx, [r8 + 4]
        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
        mov     eax, [r11]
        mov     edx, [r11 + 4]

        lock cmpxchg8b [r10]

        mov     [r11], eax
        mov     [r11 + 4], edx
        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)

        pop     rbx
        ret
 %else
        push    rbx

        mov     r10, rcx                ; pEFlags
        mov     r11, rdx                ; pu64EbxEcx (is also T1)

        mov     ebx, [r11]
        mov     ecx, [r11 + 4]
        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
        mov     eax, [rsi]
        mov     edx, [rsi + 4]

        lock cmpxchg8b [rdi]

        mov     [rsi], eax
        mov     [rsi + 4], edx
        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)

        pop     rbx
        ret

 %endif
%else
        push    esi
        push    edi
        push    ebx
        push    ebp

        mov     edi, ecx                ; pu64Dst
        mov     esi, edx                ; pu64EaxEdx
        mov     ecx, [esp + 16 + 4 + 0] ; pu64EbxEcx
        mov     ebp, [esp + 16 + 4 + 4] ; pEFlags

        mov     ebx, [ecx]
        mov     ecx, [ecx + 4]
        IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF), 0  ; clobbers T0 (eax)
        mov     eax, [esi]
        mov     edx, [esi + 4]

        lock cmpxchg8b [edi]

        mov     [esi], eax
        mov     [esi + 4], edx
        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, edi)

        pop     ebp
        pop     ebx
        pop     edi
        pop     esi
        ret     8
%endif
ENDPROC iemAImpl_cmpxchg8b_locked

%ifdef RT_ARCH_AMD64

;
; CMPXCHG16B.
;
; These are tricky register wise, so the code is duplicated for each calling
; convention.
;
; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to!
;
; C-proto:
; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu1284RaxRdx, PRTUINT128U pu128RbxRcx,
;                                              uint32_t *pEFlags));
;
; Note! Identical to iemAImpl_cmpxchg8b.
;
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_cmpxchg16b, 16
 %ifdef ASM_CALL64_MSC
        push    rbx

        mov     r11, rdx                ; pu64RaxRdx (is also T1)
        mov     r10, rcx                ; pu64Dst

        mov     rbx, [r8]
        mov     rcx, [r8 + 8]
        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
        mov     rax, [r11]
        mov     rdx, [r11 + 8]

        cmpxchg16b [r10]

        mov     [r11], rax
        mov     [r11 + 8], rdx
        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)

        pop     rbx
        ret
 %else
        push    rbx

        mov     r10, rcx                ; pEFlags
        mov     r11, rdx                ; pu64RbxRcx (is also T1)

        mov     rbx, [r11]
        mov     rcx, [r11 + 8]
        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
        mov     rax, [rsi]
        mov     rdx, [rsi + 8]

        cmpxchg16b [rdi]

        mov     [rsi], rax
        mov     [rsi + 8], rdx
        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)

        pop     rbx
        ret

 %endif
ENDPROC iemAImpl_cmpxchg16b

BEGINPROC_FASTCALL iemAImpl_cmpxchg16b_locked, 16
 %ifdef ASM_CALL64_MSC
        push    rbx

        mov     r11, rdx                ; pu64RaxRdx (is also T1)
        mov     r10, rcx                ; pu64Dst

        mov     rbx, [r8]
        mov     rcx, [r8 + 8]
        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
        mov     rax, [r11]
        mov     rdx, [r11 + 8]

        lock cmpxchg16b [r10]

        mov     [r11], rax
        mov     [r11 + 8], rdx
        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)

        pop     rbx
        ret
 %else
        push    rbx

        mov     r10, rcx                ; pEFlags
        mov     r11, rdx                ; pu64RbxRcx (is also T1)

        mov     rbx, [r11]
        mov     rcx, [r11 + 8]
        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
        mov     rax, [rsi]
        mov     rdx, [rsi + 8]

        lock cmpxchg16b [rdi]

        mov     [rsi], rax
        mov     [rsi + 8], rdx
        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)

        pop     rbx
        ret

 %endif
ENDPROC iemAImpl_cmpxchg16b_locked

%endif ; RT_ARCH_AMD64


;
; CMPXCHG.
;
; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to!
;
; C-proto:
; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg,(uintX_t *puXDst, uintX_t puEax, uintX_t uReg, uint32_t *pEFlags));
;
BEGINCODE
%macro IEMIMPL_CMPXCHG 2
BEGINPROC_FASTCALL iemAImpl_cmpxchg_u8 %+ %2, 16
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
        mov     al, [A1]
        %1 cmpxchg [A0], A2_8
        mov     [A1], al
        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cmpxchg_u8 %+ %2

BEGINPROC_FASTCALL iemAImpl_cmpxchg_u16 %+ %2, 16
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
        mov     ax, [A1]
        %1 cmpxchg [A0], A2_16
        mov     [A1], ax
        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cmpxchg_u16 %+ %2

BEGINPROC_FASTCALL iemAImpl_cmpxchg_u32 %+ %2, 16
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
        mov     eax, [A1]
        %1 cmpxchg [A0], A2_32
        mov     [A1], eax
        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cmpxchg_u32 %+ %2

BEGINPROC_FASTCALL iemAImpl_cmpxchg_u64 %+ %2, 16
%ifdef RT_ARCH_AMD64
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
        mov     rax, [A1]
        %1 cmpxchg [A0], A2
        mov     [A1], rax
        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
        EPILOGUE_4_ARGS
%else
        ;
        ; Must use cmpxchg8b here. See also iemAImpl_cmpxchg8b.
        ;
        push    esi
        push    edi
        push    ebx
        push    ebp

        mov     edi, ecx                ; pu64Dst
        mov     esi, edx                ; pu64Rax
        mov     ecx, [esp + 16 + 4 + 0] ; pu64Reg - Note! Pointer on 32-bit hosts!
        mov     ebp, [esp + 16 + 4 + 4] ; pEFlags

        mov     ebx, [ecx]
        mov     ecx, [ecx + 4]
        IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0  ; clobbers T0 (eax)
        mov     eax, [esi]
        mov     edx, [esi + 4]

        lock cmpxchg8b [edi]

        ; cmpxchg8b doesn't set CF, PF, AF, SF and OF, so we have to do that.
        jz      .cmpxchg8b_not_equal
        cmp     eax, eax                ; just set the other flags.
.store:
        mov     [esi], eax
        mov     [esi + 4], edx
        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, edi)

        pop     ebp
        pop     ebx
        pop     edi
        pop     esi
        ret     8

.cmpxchg8b_not_equal:
        cmp     [esi + 4], edx          ;; @todo FIXME - verify 64-bit compare implementation
        jne     .store
        cmp     [esi], eax
        jmp     .store

%endif
ENDPROC iemAImpl_cmpxchg_u64 %+ %2
%endmacro ; IEMIMPL_CMPXCHG

IEMIMPL_CMPXCHG , ,
IEMIMPL_CMPXCHG lock, _locked

;;
; Macro for implementing a unary operator.
;
; This will generate code for the 8, 16, 32 and 64 bit accesses with locked
; variants, except on 32-bit system where the 64-bit accesses requires hand
; coding.
;
; All the functions takes a pointer to the destination memory operand in A0,
; the source register operand in A1 and a pointer to eflags in A2.
;
; @param        1       The instruction mnemonic.
; @param        2       The modified flags.
; @param        3       The undefined flags.
;
%macro IEMIMPL_UNARY_OP 3
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 8
        PROLOGUE_2_ARGS
        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
        %1      byte [A0]
        IEM_SAVE_FLAGS       A1, %2, %3
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u8

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8_locked, 8
        PROLOGUE_2_ARGS
        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
        lock %1 byte [A0]
        IEM_SAVE_FLAGS       A1, %2, %3
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u8_locked

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 8
        PROLOGUE_2_ARGS
        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
        %1      word [A0]
        IEM_SAVE_FLAGS       A1, %2, %3
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 8
        PROLOGUE_2_ARGS
        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
        lock %1 word [A0]
        IEM_SAVE_FLAGS       A1, %2, %3
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16_locked

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 8
        PROLOGUE_2_ARGS
        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
        %1      dword [A0]
        IEM_SAVE_FLAGS       A1, %2, %3
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 8
        PROLOGUE_2_ARGS
        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
        lock %1 dword [A0]
        IEM_SAVE_FLAGS       A1, %2, %3
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32_locked

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8
        PROLOGUE_2_ARGS
        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
        %1      qword [A0]
        IEM_SAVE_FLAGS       A1, %2, %3
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 8
        PROLOGUE_2_ARGS
        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
        lock %1 qword [A0]
        IEM_SAVE_FLAGS       A1, %2, %3
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
 %endif ; RT_ARCH_AMD64

%endmacro

IEMIMPL_UNARY_OP inc, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), 0
IEMIMPL_UNARY_OP dec, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), 0
IEMIMPL_UNARY_OP neg, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
IEMIMPL_UNARY_OP not, 0, 0


;
; BSWAP. No flag changes.
;
; Each function takes one argument, pointer to the value to bswap
; (input/output). They all return void.
;
BEGINPROC_FASTCALL iemAImpl_bswap_u16, 4
        PROLOGUE_1_ARGS
        mov     T0_32, [A0]             ; just in case any of the upper bits are used.
        db 66h
        bswap   T0_32
        mov     [A0], T0_32
        EPILOGUE_1_ARGS
ENDPROC iemAImpl_bswap_u16

BEGINPROC_FASTCALL iemAImpl_bswap_u32, 4
        PROLOGUE_1_ARGS
        mov     T0_32, [A0]
        bswap   T0_32
        mov     [A0], T0_32
        EPILOGUE_1_ARGS
ENDPROC iemAImpl_bswap_u32

BEGINPROC_FASTCALL iemAImpl_bswap_u64, 4
%ifdef RT_ARCH_AMD64
        PROLOGUE_1_ARGS
        mov     T0, [A0]
        bswap   T0
        mov     [A0], T0
        EPILOGUE_1_ARGS
%else
        PROLOGUE_1_ARGS
        mov     T0, [A0]
        mov     T1, [A0 + 4]
        bswap   T0
        bswap   T1
        mov     [A0 + 4], T0
        mov     [A0], T1
        EPILOGUE_1_ARGS
%endif
ENDPROC iemAImpl_bswap_u64


;;
; Macro for implementing a shift operation.
;
; This will generate code for the 8, 16, 32 and 64 bit accesses, except on
; 32-bit system where the 64-bit accesses requires hand coding.
;
; All the functions takes a pointer to the destination memory operand in A0,
; the shift count in A1 and a pointer to eflags in A2.
;
; @param        1       The instruction mnemonic.
; @param        2       The modified flags.
; @param        3       The undefined flags.
;
; Makes ASSUMPTIONS about A0, A1 and A2 assignments.
;
; @note the _intel and _amd variants are implemented in C.
;
%macro IEMIMPL_SHIFT_OP 3
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
 %ifdef ASM_CALL64_GCC
        mov     cl, A1_8
        %1      byte [A0], cl
 %else
        xchg    A1, A0
        %1      byte [A1], cl
 %endif
        IEM_SAVE_FLAGS       A2, %2, %3
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u8

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
 %ifdef ASM_CALL64_GCC
        mov     cl, A1_8
        %1      word [A0], cl
 %else
        xchg    A1, A0
        %1      word [A1], cl
 %endif
        IEM_SAVE_FLAGS       A2, %2, %3
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
 %ifdef ASM_CALL64_GCC
        mov     cl, A1_8
        %1      dword [A0], cl
 %else
        xchg    A1, A0
        %1      dword [A1], cl
 %endif
        IEM_SAVE_FLAGS       A2, %2, %3
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
  %ifdef ASM_CALL64_GCC
        mov     cl, A1_8
        %1      qword [A0], cl
  %else
        xchg    A1, A0
        %1      qword [A1], cl
  %endif
        IEM_SAVE_FLAGS       A2, %2, %3
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64
 %endif ; RT_ARCH_AMD64

%endmacro

IEMIMPL_SHIFT_OP rol, (X86_EFL_OF | X86_EFL_CF), 0
IEMIMPL_SHIFT_OP ror, (X86_EFL_OF | X86_EFL_CF), 0
IEMIMPL_SHIFT_OP rcl, (X86_EFL_OF | X86_EFL_CF), 0
IEMIMPL_SHIFT_OP rcr, (X86_EFL_OF | X86_EFL_CF), 0
IEMIMPL_SHIFT_OP shl, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
IEMIMPL_SHIFT_OP shr, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
IEMIMPL_SHIFT_OP sar, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)


;;
; Macro for implementing a double precision shift operation.
;
; This will generate code for the 16, 32 and 64 bit accesses, except on
; 32-bit system where the 64-bit accesses requires hand coding.
;
; The functions takes the destination operand (r/m) in A0, the source (reg) in
; A1, the shift count in A2 and a pointer to the eflags variable/register in A3.
;
; @param        1       The instruction mnemonic.
; @param        2       The modified flags.
; @param        3       The undefined flags.
;
; Makes ASSUMPTIONS about A0, A1, A2 and A3 assignments.
;
; @note the _intel and _amd variants are implemented in C.
;
%macro IEMIMPL_SHIFT_DBL_OP 3
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 16
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
 %ifdef ASM_CALL64_GCC
        xchg    A3, A2
        %1      [A0], A1_16, cl
        xchg    A3, A2
 %else
        xchg    A0, A2
        %1      [A2], A1_16, cl
 %endif
        IEM_SAVE_FLAGS       A3, %2, %3
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
 %ifdef ASM_CALL64_GCC
        xchg    A3, A2
        %1      [A0], A1_32, cl
        xchg    A3, A2
 %else
        xchg    A0, A2
        %1      [A2], A1_32, cl
 %endif
        IEM_SAVE_FLAGS       A3, %2, %3
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32

 %ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 20
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
 %ifdef ASM_CALL64_GCC
        xchg    A3, A2
        %1      [A0], A1, cl
        xchg    A3, A2
 %else
        xchg    A0, A2
        %1      [A2], A1, cl
 %endif
        IEM_SAVE_FLAGS       A3, %2, %3
        EPILOGUE_4_ARGS_EX 12
ENDPROC iemAImpl_ %+ %1 %+ _u64
 %endif ; RT_ARCH_AMD64

%endmacro

IEMIMPL_SHIFT_DBL_OP shld, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
IEMIMPL_SHIFT_DBL_OP shrd, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)


;;
; Macro for implementing a multiplication operations.
;
; This will generate code for the 8, 16, 32 and 64 bit accesses, except on
; 32-bit system where the 64-bit accesses requires hand coding.
;
; The 8-bit function only operates on AX, so it takes no DX pointer.  The other
; functions takes a pointer to rAX in A0, rDX in A1, the operand in A2 and a
; pointer to eflags in A3.
;
; The functions all return 0 so the caller can be used for div/idiv as well as
; for the mul/imul implementation.
;
; @param        1       The instruction mnemonic.
; @param        2       The modified flags.
; @param        3       The undefined flags.
; @param        4       Name suffix.
; @param        5       EFLAGS behaviour: 0 for native, 1 for intel and 2 for AMD.
;
; Makes ASSUMPTIONS about A0, A1, A2, A3, T0 and T1 assignments.
;
%macro IEMIMPL_MUL_OP 5
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8 %+ %4, 12
        PROLOGUE_3_ARGS
        IEM_MAYBE_LOAD_FLAGS                     A2, %2, %3
        mov     al, [A0]
        %1      A1_8
        mov     [A0], ax
 %if %5 != 1
        IEM_SAVE_FLAGS                           A2, %2, %3
 %else
        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A2, %2, X86_EFL_AF | X86_EFL_ZF, ax, 8, xAX
 %endif
        xor     eax, eax
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u8 %+ %4

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ %4, 16
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3
        mov     ax, [A0]
 %ifdef ASM_CALL64_GCC
        %1      A2_16
        mov     [A0], ax
        mov     [A1], dx
 %else
        mov     T1, A1
        %1      A2_16
        mov     [A0], ax
        mov     [T1], dx
 %endif
 %if %5 != 1
        IEM_SAVE_FLAGS                           A3, %2, %3
 %else
        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, ax, 16, xAX
 %endif
        xor     eax, eax
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u16 %+ %4

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ %4, 16
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3
        mov     eax, [A0]
 %ifdef ASM_CALL64_GCC
        %1      A2_32
        mov     [A0], eax
        mov     [A1], edx
 %else
        mov     T1, A1
        %1      A2_32
        mov     [A0], eax
        mov     [T1], edx
 %endif
 %if %5 != 1
        IEM_SAVE_FLAGS                           A3, %2, %3
 %else
        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, eax, 32, xAX
 %endif
        xor     eax, eax
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u32 %+ %4

 %ifdef RT_ARCH_AMD64 ; The 32-bit host version lives in IEMAllAImplC.cpp.
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ %4, 20
        PROLOGUE_4_ARGS
        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3
        mov     rax, [A0]
  %ifdef ASM_CALL64_GCC
        %1      A2
        mov     [A0], rax
        mov     [A1], rdx
  %else
        mov     T1, A1
        %1      A2
        mov     [A0], rax
        mov     [T1], rdx
  %endif
  %if %5 != 1
        IEM_SAVE_FLAGS                           A3, %2, %3
  %else
        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, rax, 64, xAX
  %endif
        xor     eax, eax
        EPILOGUE_4_ARGS_EX 12
ENDPROC iemAImpl_ %+ %1 %+ _u64 %+ %4
 %endif ; !RT_ARCH_AMD64

%endmacro

IEMIMPL_MUL_OP mul,  (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF),       , 0
IEMIMPL_MUL_OP mul,  (X86_EFL_OF | X86_EFL_CF), 0,                                                   _intel, 1
IEMIMPL_MUL_OP mul,  (X86_EFL_OF | X86_EFL_CF), 0,                                                   _amd,   2
IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF),       , 0
IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), 0,                                                   _intel, 1
IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), 0,                                                   _amd,   2


BEGINCODE
;;
; Worker function for negating a 32-bit number in T1:T0
; @uses None (T0,T1)
BEGINPROC   iemAImpl_negate_T0_T1_u32
        push    0
        push    0
        xchg    T0_32, [xSP]
        xchg    T1_32, [xSP + xCB]
        sub     T0_32, [xSP]
        sbb     T1_32, [xSP + xCB]
        add     xSP, xCB*2
        ret
ENDPROC     iemAImpl_negate_T0_T1_u32

%ifdef RT_ARCH_AMD64
;;
; Worker function for negating a 64-bit number in T1:T0
; @uses None (T0,T1)
BEGINPROC   iemAImpl_negate_T0_T1_u64
        push    0
        push    0
        xchg    T0, [xSP]
        xchg    T1, [xSP + xCB]
        sub     T0, [xSP]
        sbb     T1, [xSP + xCB]
        add     xSP, xCB*2
        ret
ENDPROC     iemAImpl_negate_T0_T1_u64
%endif


;;
; Macro for implementing a division operations.
;
; This will generate code for the 8, 16, 32 and 64 bit accesses, except on
; 32-bit system where the 64-bit accesses requires hand coding.
;
; The 8-bit function only operates on AX, so it takes no DX pointer.  The other
; functions takes a pointer to rAX in A0, rDX in A1, the operand in A2 and a
; pointer to eflags in A3.
;
; The functions all return 0 on success and -1 if a divide error should be
; raised by the caller.
;
; @param        1       The instruction mnemonic.
; @param        2       The modified flags.
; @param        3       The undefined flags.
; @param        4       1 if signed, 0 if unsigned.
; @param        5       Function suffix.
; @param        6       EFLAGS variation: 0 for native, 1 for intel (ignored),
;                       2 for AMD (set AF, clear PF, ZF and SF).
;
; Makes ASSUMPTIONS about A0, A1, A2, A3, T0 and T1 assignments.
;
%macro IEMIMPL_DIV_OP 6
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8 %+ %5, 12
        PROLOGUE_3_ARGS

        ; div by chainsaw check.
        test    A1_8, A1_8
        jz      .div_zero

        ; Overflow check - unsigned division is simple to verify, haven't
        ; found a simple way to check signed division yet unfortunately.
 %if %4 == 0
        cmp     [A0 + 1], A1_8
        jae     .div_overflow
 %else
        mov     T0_16, [A0]             ; T0 = dividend
        mov     T1, A1                  ; T1 = saved divisor (because of missing T1_8 in 32-bit)
        test    A1_8, A1_8
        js      .divisor_negative
        test    T0_16, T0_16
        jns     .both_positive
        neg     T0_16
.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
        push    T0                      ; Start off like unsigned below.
        shr     T0_16, 7
        cmp     T0_8, A1_8
        pop     T0
        jb      .div_no_overflow
        ja      .div_overflow
        and     T0_8, 0x7f              ; Special case for covering (divisor - 1).
        cmp     T0_8, A1_8
        jae     .div_overflow
        jmp     .div_no_overflow

.divisor_negative:
        neg     A1_8
        test    T0_16, T0_16
        jns     .one_of_each
        neg     T0_16
.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
        shr     T0_16, 7
        cmp     T0_8, A1_8
        jae     .div_overflow
.div_no_overflow:
        mov     A1, T1                  ; restore divisor
 %endif

        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
        mov     ax, [A0]
        %1      A1_8
        mov     [A0], ax
 %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF.
        IEM_ADJUST_FLAGS    A2, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
 %else
        IEM_SAVE_FLAGS      A2, %2, %3
 %endif
        xor     eax, eax

.return:
        EPILOGUE_3_ARGS

.div_zero:
.div_overflow:
        mov     eax, -1
        jmp     .return
ENDPROC iemAImpl_ %+ %1 %+ _u8 %+ %5

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ %5, 16
        PROLOGUE_4_ARGS

        ; div by chainsaw check.
        test    A2_16, A2_16
        jz      .div_zero

        ; Overflow check - unsigned division is simple to verify, haven't
        ; found a simple way to check signed division yet unfortunately.
 %if %4 == 0
        cmp     [A1], A2_16
        jae     .div_overflow
 %else
        mov     T0_16, [A1]
        shl     T0_32, 16
        mov     T0_16, [A0]              ; T0 = dividend
        mov     T1, A2                   ; T1 = divisor
        test    T1_16, T1_16
        js      .divisor_negative
        test    T0_32, T0_32
        jns     .both_positive
        neg     T0_32
.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
        push    T0                      ; Start off like unsigned below.
        shr     T0_32, 15
        cmp     T0_16, T1_16
        pop     T0
        jb      .div_no_overflow
        ja      .div_overflow
        and     T0_16, 0x7fff           ; Special case for covering (divisor - 1).
        cmp     T0_16, T1_16
        jae     .div_overflow
        jmp     .div_no_overflow

.divisor_negative:
        neg     T1_16
        test    T0_32, T0_32
        jns     .one_of_each
        neg     T0_32
.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
        shr     T0_32, 15
        cmp     T0_16, T1_16
        jae     .div_overflow
.div_no_overflow:
 %endif

        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
 %ifdef ASM_CALL64_GCC
        mov     T1, A2
        mov     ax, [A0]
        mov     dx, [A1]
        %1      T1_16
        mov     [A0], ax
        mov     [A1], dx
 %else
        mov     T1, A1
        mov     ax, [A0]
        mov     dx, [T1]
        %1      A2_16
        mov     [A0], ax
        mov     [T1], dx
 %endif
 %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF.
        IEM_ADJUST_FLAGS    A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
 %else
        IEM_SAVE_FLAGS      A3, %2, %3
 %endif
        xor     eax, eax

.return:
        EPILOGUE_4_ARGS

.div_zero:
.div_overflow:
        mov     eax, -1
        jmp     .return
ENDPROC iemAImpl_ %+ %1 %+ _u16 %+ %5

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ %5, 16
        PROLOGUE_4_ARGS

        ; div by chainsaw check.
        test    A2_32, A2_32
        jz      .div_zero

        ; Overflow check - unsigned division is simple to verify, haven't
        ; found a simple way to check signed division yet unfortunately.
 %if %4 == 0
        cmp     [A1], A2_32
        jae     .div_overflow
 %else
        push    A2                      ; save A2 so we modify it (we out of regs on x86).
        mov     T0_32, [A0]             ; T0 = dividend low
        mov     T1_32, [A1]             ; T1 = dividend high
        test    A2_32, A2_32
        js      .divisor_negative
        test    T1_32, T1_32
        jns     .both_positive
        call    NAME(iemAImpl_negate_T0_T1_u32)
.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
        push    T0                      ; Start off like unsigned below.
        shl     T1_32, 1
        shr     T0_32, 31
        or      T1_32, T0_32
        cmp     T1_32, A2_32
        pop     T0
        jb      .div_no_overflow
        ja      .div_overflow
        and     T0_32, 0x7fffffff       ; Special case for covering (divisor - 1).
        cmp     T0_32, A2_32
        jae     .div_overflow
        jmp     .div_no_overflow

.divisor_negative:
        neg     A2_32
        test    T1_32, T1_32
        jns     .one_of_each
        call    NAME(iemAImpl_negate_T0_T1_u32)
.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
        shl     T1_32, 1
        shr     T0_32, 31
        or      T1_32, T0_32
        cmp     T1_32, A2_32
        jae     .div_overflow
.div_no_overflow:
        pop     A2
 %endif

        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
        mov     eax, [A0]
 %ifdef ASM_CALL64_GCC
        mov     T1, A2
        mov     eax, [A0]
        mov     edx, [A1]
        %1      T1_32
        mov     [A0], eax
        mov     [A1], edx
 %else
        mov     T1, A1
        mov     eax, [A0]
        mov     edx, [T1]
        %1      A2_32
        mov     [A0], eax
        mov     [T1], edx
 %endif
 %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF.
        IEM_ADJUST_FLAGS    A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
 %else
        IEM_SAVE_FLAGS      A3, %2, %3
 %endif
        xor     eax, eax

.return:
        EPILOGUE_4_ARGS

.div_overflow:
 %if %4 != 0
        pop     A2
 %endif
.div_zero:
        mov     eax, -1
        jmp     .return
ENDPROC iemAImpl_ %+ %1 %+ _u32 %+ %5

 %ifdef RT_ARCH_AMD64 ; The 32-bit host version lives in IEMAllAImplC.cpp.
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ %5, 20
        PROLOGUE_4_ARGS

        test    A2, A2
        jz      .div_zero
  %if %4 == 0
        cmp     [A1], A2
        jae     .div_overflow
  %else
        push    A2                      ; save A2 so we modify it (we out of regs on x86).
        mov     T0, [A0]                ; T0 = dividend low
        mov     T1, [A1]                ; T1 = dividend high
        test    A2, A2
        js      .divisor_negative
        test    T1, T1
        jns     .both_positive
        call    NAME(iemAImpl_negate_T0_T1_u64)
.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
        push    T0                      ; Start off like unsigned below.
        shl     T1, 1
        shr     T0, 63
        or      T1, T0
        cmp     T1, A2
        pop     T0
        jb      .div_no_overflow
        ja      .div_overflow
        mov     T1, 0x7fffffffffffffff
        and     T0, T1                  ; Special case for covering (divisor - 1).
        cmp     T0, A2
        jae     .div_overflow
        jmp     .div_no_overflow

.divisor_negative:
        neg     A2
        test    T1, T1
        jns     .one_of_each
        call    NAME(iemAImpl_negate_T0_T1_u64)
.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
        shl     T1, 1
        shr     T0, 63
        or      T1, T0
        cmp     T1, A2
        jae     .div_overflow
.div_no_overflow:
        pop     A2
  %endif

        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
        mov     rax, [A0]
  %ifdef ASM_CALL64_GCC
        mov     T1, A2
        mov     rax, [A0]
        mov     rdx, [A1]
        %1      T1
        mov     [A0], rax
        mov     [A1], rdx
  %else
        mov     T1, A1
        mov     rax, [A0]
        mov     rdx, [T1]
        %1      A2
        mov     [A0], rax
        mov     [T1], rdx
  %endif
  %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF.
        IEM_ADJUST_FLAGS    A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
  %else
        IEM_SAVE_FLAGS      A3, %2, %3
  %endif
        xor     eax, eax

.return:
        EPILOGUE_4_ARGS_EX 12

.div_overflow:
  %if %4 != 0
        pop     A2
  %endif
.div_zero:
        mov     eax, -1
        jmp     .return
ENDPROC iemAImpl_ %+ %1 %+ _u64 %+ %5
 %endif ; !RT_ARCH_AMD64

%endmacro

IEMIMPL_DIV_OP div,  0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,       , 0
IEMIMPL_DIV_OP div,  0, 0,                                                                             0, _intel, 1
IEMIMPL_DIV_OP div,  0, 0,                                                                             0, _amd,   2
IEMIMPL_DIV_OP idiv, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1,       , 0
IEMIMPL_DIV_OP idiv, 0, 0,                                                                             1, _intel, 1
IEMIMPL_DIV_OP idiv, 0, 0,                                                                             1, _amd,   2


;;
; Macro for implementing memory fence operation.
;
; No return value, no operands or anything.
;
; @param        1      The instruction.
;
%macro IEMIMPL_MEM_FENCE 1
BEGINCODE
BEGINPROC_FASTCALL iemAImpl_ %+ %1, 0
        %1
        ret
ENDPROC iemAImpl_ %+ %1
%endmacro

IEMIMPL_MEM_FENCE lfence
IEMIMPL_MEM_FENCE sfence
IEMIMPL_MEM_FENCE mfence

;;
; Alternative for non-SSE2 host.
;
BEGINPROC_FASTCALL iemAImpl_alt_mem_fence, 0
        push    xAX
        xchg    xAX, [xSP]
        add     xSP, xCB
        ret
ENDPROC iemAImpl_alt_mem_fence


;;
; Initialize the FPU for the actual instruction being emulated, this means
; loading parts of the guest's control word and status word.
;
; @uses     24 bytes of stack. T0, T1
; @param    1       Expression giving the address of the FXSTATE of the guest.
;
%macro FPU_LD_FXSTATE_FCW_AND_SAFE_FSW 1
        fnstenv [xSP]

        ; FCW - for exception, precision and rounding control.
        movzx   T0, word [%1 + X86FXSTATE.FCW]
        and     T0, X86_FCW_MASK_ALL | X86_FCW_PC_MASK | X86_FCW_RC_MASK
        mov     [xSP + X86FSTENV32P.FCW], T0_16

        ; FSW - for undefined C0, C1, C2, and C3.
        movzx   T1, word [%1 + X86FXSTATE.FSW]
        and     T1, X86_FSW_C_MASK
        movzx   T0, word [xSP + X86FSTENV32P.FSW]
        and     T0, X86_FSW_TOP_MASK
        or      T0, T1
        mov     [xSP + X86FSTENV32P.FSW], T0_16

        fldenv  [xSP]
%endmacro


;;
; Initialize the FPU for the actual instruction being emulated, this means
; loading parts of the guest's control word, status word, and update the
; tag word for the top register if it's empty.
;
; ASSUMES actual TOP=7
;
; @uses     24 bytes of stack.  T0, T1
; @param    1       Expression giving the address of the FXSTATE of the guest.
;
%macro FPU_LD_FXSTATE_FCW_AND_SAFE_FSW_AND_FTW_0 1
        fnstenv [xSP]

        ; FCW - for exception, precision and rounding control.
        movzx   T0_32, word [%1 + X86FXSTATE.FCW]
        and     T0_32, X86_FCW_MASK_ALL | X86_FCW_PC_MASK | X86_FCW_RC_MASK
        mov     [xSP + X86FSTENV32P.FCW], T0_16

        ; FSW - for undefined C0, C1, C2, and C3.
        movzx   T1_32, word [%1 + X86FXSTATE.FSW]
        and     T1_32, X86_FSW_C_MASK
        movzx   T0_32, word [xSP + X86FSTENV32P.FSW]
        and     T0_32, X86_FSW_TOP_MASK
        or      T0_32, T1_32
        mov     [xSP + X86FSTENV32P.FSW], T0_16

        ; FTW - Only for ST0 (in/out).
        movzx   T1_32, word [%1 + X86FXSTATE.FSW]
        shr     T1_32, X86_FSW_TOP_SHIFT
        and     T1_32, X86_FSW_TOP_SMASK
        bt      [%1 + X86FXSTATE.FTW], T1_16     ; Empty if FTW bit is clear.  Fixed register order.
        jc      %%st0_not_empty
        or      word [xSP + X86FSTENV32P.FTW], 0c000h ; TOP=7, so set TAG(7)=3
%%st0_not_empty:

        fldenv  [xSP]
%endmacro


;;
; Need to move this as well somewhere better?
;
struc IEMFPURESULT
    .r80Result  resw 5
    .FSW        resw 1
endstruc


;;
; Need to move this as well somewhere better?
;
struc IEMFPURESULTTWO
    .r80Result1 resw 5
    .FSW        resw 1
    .r80Result2 resw 5
endstruc


;
;---------------------- 16-bit signed integer operations ----------------------
;


;;
; Converts a 16-bit floating point value to a 80-bit one (fpu register).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 16-bit floating point value to convert.
;
BEGINPROC_FASTCALL iemAImpl_fild_r80_from_i16, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fild    word [A2]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_fild_r80_from_i16


;;
; Store a 80-bit floating point value (register) as a 16-bit signed integer (memory).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 16-bit signed integer value.
; @param    A3      Pointer to the 80-bit value.
;
BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i16, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fistp   word [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fist_r80_to_i16


;;
; Store a 80-bit floating point value (register) as a 16-bit signed integer
; (memory) with truncation.
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 16-bit signed integer value.
; @param    A3      Pointer to the 80-bit value.
;
BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i16, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fisttp  word [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fistt_r80_to_i16


;;
; FPU instruction working on one 80-bit and one 16-bit signed integer value.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 80-bit value.
; @param    A3      Pointer to the 16-bit value.
;
%macro IEMIMPL_FPU_R80_BY_I16 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i16, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      word [A3]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i16
%endmacro

IEMIMPL_FPU_R80_BY_I16 fiadd
IEMIMPL_FPU_R80_BY_I16 fimul
IEMIMPL_FPU_R80_BY_I16 fisub
IEMIMPL_FPU_R80_BY_I16 fisubr
IEMIMPL_FPU_R80_BY_I16 fidiv
IEMIMPL_FPU_R80_BY_I16 fidivr


;;
; FPU instruction working on one 80-bit and one 16-bit signed integer value,
; only returning FSW.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to store the output FSW.
; @param    A2      Pointer to the 80-bit value.
; @param    A3      Pointer to the 64-bit value.
;
%macro IEMIMPL_FPU_R80_BY_I16_FSW 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i16, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      word [A3]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i16
%endmacro

IEMIMPL_FPU_R80_BY_I16_FSW ficom



;
;---------------------- 32-bit signed integer operations ----------------------
;


;;
; Converts a 32-bit floating point value to a 80-bit one (fpu register).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 32-bit floating point value to convert.
;
BEGINPROC_FASTCALL iemAImpl_fild_r80_from_i32, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fild    dword [A2]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_fild_r80_from_i32


;;
; Store a 80-bit floating point value (register) as a 32-bit signed integer (memory).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 32-bit signed integer value.
; @param    A3      Pointer to the 80-bit value.
;
BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i32, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fistp   dword [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fist_r80_to_i32


;;
; Store a 80-bit floating point value (register) as a 32-bit signed integer
; (memory) with truncation.
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 32-bit signed integer value.
; @param    A3      Pointer to the 80-bit value.
;
BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i32, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fisttp  dword [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fistt_r80_to_i32


;;
; FPU instruction working on one 80-bit and one 32-bit signed integer value.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 80-bit value.
; @param    A3      Pointer to the 32-bit value.
;
%macro IEMIMPL_FPU_R80_BY_I32 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i32, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      dword [A3]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i32
%endmacro

IEMIMPL_FPU_R80_BY_I32 fiadd
IEMIMPL_FPU_R80_BY_I32 fimul
IEMIMPL_FPU_R80_BY_I32 fisub
IEMIMPL_FPU_R80_BY_I32 fisubr
IEMIMPL_FPU_R80_BY_I32 fidiv
IEMIMPL_FPU_R80_BY_I32 fidivr


;;
; FPU instruction working on one 80-bit and one 32-bit signed integer value,
; only returning FSW.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to store the output FSW.
; @param    A2      Pointer to the 80-bit value.
; @param    A3      Pointer to the 64-bit value.
;
%macro IEMIMPL_FPU_R80_BY_I32_FSW 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i32, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      dword [A3]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i32
%endmacro

IEMIMPL_FPU_R80_BY_I32_FSW ficom



;
;---------------------- 64-bit signed integer operations ----------------------
;


;;
; Converts a 64-bit floating point value to a 80-bit one (fpu register).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 64-bit floating point value to convert.
;
BEGINPROC_FASTCALL iemAImpl_fild_r80_from_i64, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fild    qword [A2]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_fild_r80_from_i64


;;
; Store a 80-bit floating point value (register) as a 64-bit signed integer (memory).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 64-bit signed integer value.
; @param    A3      Pointer to the 80-bit value.
;
BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i64, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fistp   qword [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fist_r80_to_i64


;;
; Store a 80-bit floating point value (register) as a 64-bit signed integer
; (memory) with truncation.
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 64-bit signed integer value.
; @param    A3      Pointer to the 80-bit value.
;
BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i64, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fisttp  qword [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fistt_r80_to_i64



;
;---------------------- 32-bit floating point operations ----------------------
;

;;
; Converts a 32-bit floating point value to a 80-bit one (fpu register).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 32-bit floating point value to convert.
;
BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r32, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fld     dword [A2]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_fld_r80_from_r32


;;
; Store a 80-bit floating point value (register) as a 32-bit one (memory).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 32-bit value.
; @param    A3      Pointer to the 80-bit value.
;
BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r32, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fst     dword [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fst_r80_to_r32


;;
; FPU instruction working on one 80-bit and one 32-bit floating point value.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 80-bit value.
; @param    A3      Pointer to the 32-bit value.
;
%macro IEMIMPL_FPU_R80_BY_R32 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r32, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      dword [A3]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r32
%endmacro

IEMIMPL_FPU_R80_BY_R32 fadd
IEMIMPL_FPU_R80_BY_R32 fmul
IEMIMPL_FPU_R80_BY_R32 fsub
IEMIMPL_FPU_R80_BY_R32 fsubr
IEMIMPL_FPU_R80_BY_R32 fdiv
IEMIMPL_FPU_R80_BY_R32 fdivr


;;
; FPU instruction working on one 80-bit and one 32-bit floating point value,
; only returning FSW.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to store the output FSW.
; @param    A2      Pointer to the 80-bit value.
; @param    A3      Pointer to the 64-bit value.
;
%macro IEMIMPL_FPU_R80_BY_R32_FSW 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r32, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      dword [A3]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r32
%endmacro

IEMIMPL_FPU_R80_BY_R32_FSW fcom



;
;---------------------- 64-bit floating point operations ----------------------
;

;;
; Converts a 64-bit floating point value to a 80-bit one (fpu register).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 64-bit floating point value to convert.
;
BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r64, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fld     qword [A2]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_fld_r80_from_r64


;;
; Store a 80-bit floating point value (register) as a 64-bit one (memory).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 64-bit value.
; @param    A3      Pointer to the 80-bit value.
;
BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r64, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fst     qword [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fst_r80_to_r64


;;
; FPU instruction working on one 80-bit and one 64-bit floating point value.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 80-bit value.
; @param    A3      Pointer to the 64-bit value.
;
%macro IEMIMPL_FPU_R80_BY_R64 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r64, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      qword [A3]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r64
%endmacro

IEMIMPL_FPU_R80_BY_R64 fadd
IEMIMPL_FPU_R80_BY_R64 fmul
IEMIMPL_FPU_R80_BY_R64 fsub
IEMIMPL_FPU_R80_BY_R64 fsubr
IEMIMPL_FPU_R80_BY_R64 fdiv
IEMIMPL_FPU_R80_BY_R64 fdivr

;;
; FPU instruction working on one 80-bit and one 64-bit floating point value,
; only returning FSW.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to store the output FSW.
; @param    A2      Pointer to the 80-bit value.
; @param    A3      Pointer to the 64-bit value.
;
%macro IEMIMPL_FPU_R80_BY_R64_FSW 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r64, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      qword [A3]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r64
%endmacro

IEMIMPL_FPU_R80_BY_R64_FSW fcom



;
;---------------------- 80-bit floating point operations ----------------------
;

;;
; Loads a 80-bit floating point register value from memory.
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 80-bit floating point value to load.
;
BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r80, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fld     tword [A2]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_fld_r80_from_r80


;;
; Store a 80-bit floating point register to memory
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 80-bit value.
; @param    A3      Pointer to the 80-bit register value.
;
BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r80, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fstp    tword [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fst_r80_to_r80


;;
; Loads an 80-bit floating point register value in BCD format from memory.
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 80-bit BCD value to load.
;
BEGINPROC_FASTCALL iemAImpl_fld_r80_from_d80, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fbld    tword [A2]

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_fld_r80_from_d80


;;
; Store a 80-bit floating point register to memory as BCD
;
; @param    A0      FPU context (fxsave).
; @param    A1      Where to return the output FSW.
; @param    A2      Where to store the 80-bit BCD value.
; @param    A3      Pointer to the 80-bit register value.
;
BEGINPROC_FASTCALL iemAImpl_fst_r80_to_d80, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        fbstp   tword [A2]

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_fst_r80_to_d80


;;
; FPU instruction working on two 80-bit floating point values.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the first 80-bit value (ST0)
; @param    A3      Pointer to the second 80-bit value (STn).
;
%macro IEMIMPL_FPU_R80_BY_R80 2
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      %2

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
%endmacro

IEMIMPL_FPU_R80_BY_R80 fadd,   {st0, st1}
IEMIMPL_FPU_R80_BY_R80 fmul,   {st0, st1}
IEMIMPL_FPU_R80_BY_R80 fsub,   {st0, st1}
IEMIMPL_FPU_R80_BY_R80 fsubr,  {st0, st1}
IEMIMPL_FPU_R80_BY_R80 fdiv,   {st0, st1}
IEMIMPL_FPU_R80_BY_R80 fdivr,  {st0, st1}
IEMIMPL_FPU_R80_BY_R80 fprem,  {}
IEMIMPL_FPU_R80_BY_R80 fprem1, {}
IEMIMPL_FPU_R80_BY_R80 fscale, {}


;;
; FPU instruction working on two 80-bit floating point values, ST1 and ST0,
; storing the result in ST1 and popping the stack.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the first 80-bit value (ST1).
; @param    A3      Pointer to the second 80-bit value (ST0).
;
%macro IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        fld     tword [A3]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
%endmacro

IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fpatan
IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fyl2x
IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fyl2xp1


;;
; FPU instruction working on two 80-bit floating point values, only
; returning FSW.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a uint16_t for the resulting FSW.
; @param    A2      Pointer to the first 80-bit value.
; @param    A3      Pointer to the second 80-bit value.
;
%macro IEMIMPL_FPU_R80_BY_R80_FSW 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      st0, st1

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
%endmacro

IEMIMPL_FPU_R80_BY_R80_FSW fcom
IEMIMPL_FPU_R80_BY_R80_FSW fucom


;;
; FPU instruction working on two 80-bit floating point values,
; returning FSW and EFLAGS (eax).
;
; @param    1       The instruction
;
; @returns  EFLAGS in EAX.
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a uint16_t for the resulting FSW.
; @param    A2      Pointer to the first 80-bit value.
; @param    A3      Pointer to the second 80-bit value.
;
%macro IEMIMPL_FPU_R80_BY_R80_EFL 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
        PROLOGUE_4_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A3]
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1      st1

        fnstsw  word  [A1]
        pushf
        pop     xAX

        fninit
        add     xSP, 20h
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
%endmacro

IEMIMPL_FPU_R80_BY_R80_EFL fcomi
IEMIMPL_FPU_R80_BY_R80_EFL fucomi


;;
; FPU instruction working on one 80-bit floating point value.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
; @param    A2      Pointer to the 80-bit value.
;
%macro IEMIMPL_FPU_R80 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80
%endmacro

IEMIMPL_FPU_R80 fchs
IEMIMPL_FPU_R80 fabs
IEMIMPL_FPU_R80 f2xm1
IEMIMPL_FPU_R80 fsqrt
IEMIMPL_FPU_R80 frndint
IEMIMPL_FPU_R80 fsin
IEMIMPL_FPU_R80 fcos


;;
; FPU instruction working on one 80-bit floating point value, only
; returning FSW.
;
; @param    1       The instruction
; @param    2       Non-zero to also restore FTW.
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a uint16_t for the resulting FSW.
; @param    A2      Pointer to the 80-bit value.
;
%macro IEMIMPL_FPU_R80_FSW 2
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
%if %2 != 0
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW_AND_FTW_0 A0
%else
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
%endif
        %1

        fnstsw  word  [A1]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80
%endmacro

IEMIMPL_FPU_R80_FSW ftst, 0
IEMIMPL_FPU_R80_FSW fxam, 1 ; No #IS or any other FP exceptions.



;;
; FPU instruction loading a 80-bit floating point constant.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULT for the output.
;
%macro IEMIMPL_FPU_R80_CONST 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1, 8
        PROLOGUE_2_ARGS
        sub     xSP, 20h

        fninit
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1

        fnstsw  word  [A1 + IEMFPURESULT.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULT.r80Result]

        fninit
        add     xSP, 20h
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+
%endmacro

IEMIMPL_FPU_R80_CONST fld1
IEMIMPL_FPU_R80_CONST fldl2t
IEMIMPL_FPU_R80_CONST fldl2e
IEMIMPL_FPU_R80_CONST fldpi
IEMIMPL_FPU_R80_CONST fldlg2
IEMIMPL_FPU_R80_CONST fldln2
IEMIMPL_FPU_R80_CONST fldz


;;
; FPU instruction working on one 80-bit floating point value, outputing two.
;
; @param    1       The instruction
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to a IEMFPURESULTTWO for the output.
; @param    A2      Pointer to the 80-bit value.
;
%macro IEMIMPL_FPU_R80_R80 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_r80, 12
        PROLOGUE_3_ARGS
        sub     xSP, 20h

        fninit
        fld     tword [A2]
        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
        %1

        fnstsw  word  [A1 + IEMFPURESULTTWO.FSW]
        fnclex
        fstp    tword [A1 + IEMFPURESULTTWO.r80Result2]
        fnclex
        fstp    tword [A1 + IEMFPURESULTTWO.r80Result1]

        fninit
        add     xSP, 20h
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _r80_r80
%endmacro

IEMIMPL_FPU_R80_R80 fptan
IEMIMPL_FPU_R80_R80 fxtract
IEMIMPL_FPU_R80_R80 fsincos




;---------------------- SSE and MMX Operations ----------------------

;; @todo what do we need to do for MMX?
%macro IEMIMPL_MMX_PROLOGUE 0
%endmacro
%macro IEMIMPL_MMX_EPILOGUE 0
%endmacro

;; @todo what do we need to do for SSE?
%macro IEMIMPL_SSE_PROLOGUE 0
%endmacro
%macro IEMIMPL_SSE_EPILOGUE 0
%endmacro

;; @todo what do we need to do for AVX?
%macro IEMIMPL_AVX_PROLOGUE 0
%endmacro
%macro IEMIMPL_AVX_EPILOGUE 0
%endmacro


;;
; Media instruction working on two full sized registers.
;
; @param    1       The instruction
; @param    2       Whether there is an MMX variant (1) or not (0).
;
; @param    A0      FPU context (fxsave).
; @param    A1      Pointer to the first media register size operand (input/output).
; @param    A2      Pointer to the second media register size operand (input).
;
%macro IEMIMPL_MEDIA_F2 2
%if %2 != 0
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
        PROLOGUE_3_ARGS
        IEMIMPL_MMX_PROLOGUE

        movq    mm0, [A1]
        movq    mm1, [A2]
        %1      mm0, mm1
        movq    [A1], mm0

        IEMIMPL_MMX_EPILOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64
%endif

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu   xmm0, [A1]
        movdqu   xmm1, [A2]
        %1       xmm0, xmm1
        movdqu   [A1], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_F2 pshufb,  1
IEMIMPL_MEDIA_F2 pand,    1
IEMIMPL_MEDIA_F2 pandn,   1
IEMIMPL_MEDIA_F2 por,     1
IEMIMPL_MEDIA_F2 pxor,    1
IEMIMPL_MEDIA_F2 pcmpeqb, 1
IEMIMPL_MEDIA_F2 pcmpeqw, 1
IEMIMPL_MEDIA_F2 pcmpeqd, 1
IEMIMPL_MEDIA_F2 pcmpeqq, 0
IEMIMPL_MEDIA_F2 pcmpgtb, 1
IEMIMPL_MEDIA_F2 pcmpgtw, 1
IEMIMPL_MEDIA_F2 pcmpgtd, 1
IEMIMPL_MEDIA_F2 pcmpgtq, 0
IEMIMPL_MEDIA_F2 paddb,   1
IEMIMPL_MEDIA_F2 paddw,   1
IEMIMPL_MEDIA_F2 paddd,   1
IEMIMPL_MEDIA_F2 paddq,   1
IEMIMPL_MEDIA_F2 paddsb,  1
IEMIMPL_MEDIA_F2 paddsw,  1
IEMIMPL_MEDIA_F2 paddusb, 1
IEMIMPL_MEDIA_F2 paddusw, 1
IEMIMPL_MEDIA_F2 psubb,   1
IEMIMPL_MEDIA_F2 psubw,   1
IEMIMPL_MEDIA_F2 psubd,   1
IEMIMPL_MEDIA_F2 psubq,   1
IEMIMPL_MEDIA_F2 psubsb,  1
IEMIMPL_MEDIA_F2 psubsw,  1
IEMIMPL_MEDIA_F2 psubusb, 1
IEMIMPL_MEDIA_F2 psubusw, 1
IEMIMPL_MEDIA_F2 pmullw,  1
IEMIMPL_MEDIA_F2 pmulld,  0
IEMIMPL_MEDIA_F2 pmulhw,  1
IEMIMPL_MEDIA_F2 pmaddwd, 1
IEMIMPL_MEDIA_F2 pminub,  1
IEMIMPL_MEDIA_F2 pminuw,  0
IEMIMPL_MEDIA_F2 pminud,  0
IEMIMPL_MEDIA_F2 pminsb,  0
IEMIMPL_MEDIA_F2 pminsw,  1
IEMIMPL_MEDIA_F2 pminsd,  0
IEMIMPL_MEDIA_F2 pmaxub,  1
IEMIMPL_MEDIA_F2 pmaxuw,  0
IEMIMPL_MEDIA_F2 pmaxud,  0
IEMIMPL_MEDIA_F2 pmaxsb,  0
IEMIMPL_MEDIA_F2 pmaxsw,  1
IEMIMPL_MEDIA_F2 pmaxsd,  0
IEMIMPL_MEDIA_F2 pabsb,   1
IEMIMPL_MEDIA_F2 pabsw,   1
IEMIMPL_MEDIA_F2 pabsd,   1
IEMIMPL_MEDIA_F2 psignb,  1
IEMIMPL_MEDIA_F2 psignw,  1
IEMIMPL_MEDIA_F2 psignd,  1
IEMIMPL_MEDIA_F2 phaddw,  1
IEMIMPL_MEDIA_F2 phaddd,  1
IEMIMPL_MEDIA_F2 phsubw,  1
IEMIMPL_MEDIA_F2 phsubd,  1
IEMIMPL_MEDIA_F2 phaddsw, 1
IEMIMPL_MEDIA_F2 phsubsw, 1
IEMIMPL_MEDIA_F2 pmaddubsw, 1
IEMIMPL_MEDIA_F2 pmulhrsw,  1
IEMIMPL_MEDIA_F2 pmuludq,   1


;;
; Media instruction working on two full sized registers, but no FXSAVE state argument.
;
; @param    1       The instruction
; @param    2       Whether there is an MMX variant (1) or not (0).
;
; @param    A0      Pointer to the first media register size operand (input/output).
; @param    A1      Pointer to the second media register size operand (input).
;
%macro IEMIMPL_MEDIA_OPT_F2 2
%if %2 != 0
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8
        PROLOGUE_2_ARGS
        IEMIMPL_MMX_PROLOGUE

        movq    mm0, [A0]
        movq    mm1, [A1]
        %1      mm0, mm1
        movq    [A0], mm0

        IEMIMPL_MMX_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64
%endif

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 8
        PROLOGUE_2_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu   xmm0, [A0]
        movdqu   xmm1, [A1]
        %1       xmm0, xmm1
        movdqu   [A0], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_OPT_F2 packsswb, 1
IEMIMPL_MEDIA_OPT_F2 packssdw, 1
IEMIMPL_MEDIA_OPT_F2 packuswb, 1
IEMIMPL_MEDIA_OPT_F2 packusdw, 0
IEMIMPL_MEDIA_OPT_F2 psllw,    1
IEMIMPL_MEDIA_OPT_F2 pslld,    1
IEMIMPL_MEDIA_OPT_F2 psllq,    1
IEMIMPL_MEDIA_OPT_F2 psrlw,    1
IEMIMPL_MEDIA_OPT_F2 psrld,    1
IEMIMPL_MEDIA_OPT_F2 psrlq,    1
IEMIMPL_MEDIA_OPT_F2 psraw,    1
IEMIMPL_MEDIA_OPT_F2 psrad,    1
IEMIMPL_MEDIA_OPT_F2 pmulhuw,  1
IEMIMPL_MEDIA_OPT_F2 pavgb,    1
IEMIMPL_MEDIA_OPT_F2 pavgw,    1
IEMIMPL_MEDIA_OPT_F2 psadbw,   1
IEMIMPL_MEDIA_OPT_F2 pmuldq,   0
IEMIMPL_MEDIA_OPT_F2 unpcklps, 0
IEMIMPL_MEDIA_OPT_F2 unpcklpd, 0
IEMIMPL_MEDIA_OPT_F2 unpckhps, 0
IEMIMPL_MEDIA_OPT_F2 unpckhpd, 0
IEMIMPL_MEDIA_OPT_F2 phminposuw, 0
IEMIMPL_MEDIA_OPT_F2 aesimc,   0
IEMIMPL_MEDIA_OPT_F2 aesenc,   0
IEMIMPL_MEDIA_OPT_F2 aesdec,   0
IEMIMPL_MEDIA_OPT_F2 aesenclast, 0
IEMIMPL_MEDIA_OPT_F2 aesdeclast, 0

;;
; Media instruction working on one full sized and one half sized register (lower half).
;
; @param    1       The instruction
; @param    2       1 if MMX is included, 0 if not.
;
; @param    A0      Pointer to the first full sized media register operand (input/output).
; @param    A1      Pointer to the second half sized media register operand (input).
;
%macro IEMIMPL_MEDIA_F1L1 2
 %if %2 != 0
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8
        PROLOGUE_2_ARGS
        IEMIMPL_MMX_PROLOGUE

        movq    mm0, [A0]
        movq    mm1, [A1]
        %1      mm0, mm1
        movq    [A0], mm0

        IEMIMPL_MMX_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u64
 %endif

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 8
        PROLOGUE_2_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu   xmm0, [A0]
        movdqu   xmm1, [A1]
        %1       xmm0, xmm1
        movdqu   [A0], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_F1L1 punpcklbw,  1
IEMIMPL_MEDIA_F1L1 punpcklwd,  1
IEMIMPL_MEDIA_F1L1 punpckldq,  1
IEMIMPL_MEDIA_F1L1 punpcklqdq, 0


;;
; Media instruction working two half sized input registers (lower half) and a full sized
; destination register (vpunpckh*).
;
; @param    1       The instruction
;
; @param    A0      Pointer to the destination register (full sized, output only).
; @param    A1      Pointer to the first full sized media source register operand, where we
;                   will only use the lower half as input - but we'll be loading it in full.
; @param    A2      Pointer to the second full sized media source register operand, where we
;                   will only use the lower half as input - but we'll be loading it in full.
;
%macro IEMIMPL_MEDIA_F1L1L1 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
        PROLOGUE_3_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu xmm0, [A1]
        vmovdqu xmm1, [A2]
        %1      xmm0, xmm0, xmm1
        vmovdqu [A0], xmm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 12
        PROLOGUE_3_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu  ymm0, [A1]
        vmovdqu  ymm1, [A2]
        %1       ymm0, ymm0, ymm1
        vmovdqu  [A0], ymm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u256
%endmacro

IEMIMPL_MEDIA_F1L1L1 vpunpcklbw
IEMIMPL_MEDIA_F1L1L1 vpunpcklwd
IEMIMPL_MEDIA_F1L1L1 vpunpckldq
IEMIMPL_MEDIA_F1L1L1 vpunpcklqdq


;;
; Media instruction working on one full sized and one half sized register (high half).
;
; @param    1       The instruction
; @param    2       1 if MMX is included, 0 if not.
;
; @param    A0      Pointer to the first full sized media register operand (input/output).
; @param    A1      Pointer to the second full sized media register operand, where we
;                   will only use the upper half as input - but we'll load it in full.
;
%macro IEMIMPL_MEDIA_F1H1 2
IEMIMPL_MEDIA_F1L1 %1, %2
%endmacro

IEMIMPL_MEDIA_F1L1 punpckhbw,  1
IEMIMPL_MEDIA_F1L1 punpckhwd,  1
IEMIMPL_MEDIA_F1L1 punpckhdq,  1
IEMIMPL_MEDIA_F1L1 punpckhqdq, 0


;;
; Media instruction working two half sized input registers (high half) and a full sized
; destination register (vpunpckh*).
;
; @param    1       The instruction
;
; @param    A0      Pointer to the destination register (full sized, output only).
; @param    A1      Pointer to the first full sized media source register operand, where we
;                   will only use the upper half as input - but we'll be loading it in full.
; @param    A2      Pointer to the second full sized media source register operand, where we
;                   will only use the upper half as input - but we'll be loading it in full.
;
%macro IEMIMPL_MEDIA_F1H1H1 1
IEMIMPL_MEDIA_F1L1L1 %1
%endmacro

IEMIMPL_MEDIA_F1H1H1 vpunpckhbw
IEMIMPL_MEDIA_F1H1H1 vpunpckhwd
IEMIMPL_MEDIA_F1H1H1 vpunpckhdq
IEMIMPL_MEDIA_F1H1H1 vpunpckhqdq


;
; Shufflers with evil 8-bit immediates.
;

BEGINPROC_FASTCALL iemAImpl_pshufw_u64, 16
        PROLOGUE_3_ARGS
        IEMIMPL_MMX_PROLOGUE

        movq    mm1, [A1]
        movq    mm0, mm0                ; paranoia!
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*8]         ; sizeof(pshufw+ret) == 9
 %else
        lea     T0, [A2 + A2*4]         ; sizeof(pshufw+ret) == 5
 %endif
        lea     T1, [T1 + T0]
        IBT_NOTRACK
        call    T1
        movq    [A0], mm0

        IEMIMPL_MMX_EPILOGUE
        EPILOGUE_3_ARGS
%assign bImm 0
%rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        pshufw  mm0, mm1, bImm
        ret
 %assign bImm bImm + 1
%endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
ENDPROC iemAImpl_pshufw_u64


%macro IEMIMPL_MEDIA_SSE_PSHUFXX 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm1, [A1]
        movdqu  xmm0, xmm1              ; paranoia!
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*4]         ; sizeof(pshufXX+ret) == 10: A2 * 10 = (A2 * 5) * 2
 %else
        lea     T0, [A2 + A2*2]         ; sizeof(pshufXX+ret) ==  6: A2 *  6 = (A2 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        movdqu  [A0], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS

 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      xmm0, xmm1, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_SSE_PSHUFXX pshufhw
IEMIMPL_MEDIA_SSE_PSHUFXX pshuflw
IEMIMPL_MEDIA_SSE_PSHUFXX pshufd


%macro IEMIMPL_MEDIA_AVX_VPSHUFXX 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        vmovdqu  ymm1, [A1]
        vmovdqu  ymm0, ymm1             ; paranoia!
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*4]         ; sizeof(pshufXX+ret) == 10: A2 * 10 = (A2 * 5) * 2
 %else
        lea     T0, [A2 + A2*2]         ; sizeof(pshufXX+ret) ==  6: A2 *  6 = (A2 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        vmovdqu  [A0], ymm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      ymm0, ymm1, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_ %+ %1 %+ _u256
%endmacro

IEMIMPL_MEDIA_AVX_VPSHUFXX vpshufhw
IEMIMPL_MEDIA_AVX_VPSHUFXX vpshuflw
IEMIMPL_MEDIA_AVX_VPSHUFXX vpshufd


;
; Shifts with evil 8-bit immediates.
;

%macro IEMIMPL_MEDIA_MMX_PSHIFTXX 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _imm_u64, 16
        PROLOGUE_2_ARGS
        IEMIMPL_MMX_PROLOGUE

        movq    mm0, [A0]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A1 + A1*8]         ; sizeof(psXX+ret) == 9
 %else
        lea     T0, [A1 + A1*4]         ; sizeof(psXX+ret) == 5
 %endif
        lea     T1, [T1 + T0]
        IBT_NOTRACK
        call    T1
        movq    [A0], mm0

        IEMIMPL_MMX_EPILOGUE
        EPILOGUE_2_ARGS
%assign bImm 0
%rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      mm0, bImm
        ret
 %assign bImm bImm + 1
%endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
ENDPROC iemAImpl_ %+ %1 %+ _imm_u64
%endmacro

IEMIMPL_MEDIA_MMX_PSHIFTXX psllw
IEMIMPL_MEDIA_MMX_PSHIFTXX pslld
IEMIMPL_MEDIA_MMX_PSHIFTXX psllq
IEMIMPL_MEDIA_MMX_PSHIFTXX psrlw
IEMIMPL_MEDIA_MMX_PSHIFTXX psrld
IEMIMPL_MEDIA_MMX_PSHIFTXX psrlq
IEMIMPL_MEDIA_MMX_PSHIFTXX psraw
IEMIMPL_MEDIA_MMX_PSHIFTXX psrad


%macro IEMIMPL_MEDIA_SSE_PSHIFTXX 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _imm_u128, 16
        PROLOGUE_2_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A0]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A1 + A1*4]         ; sizeof(psXX+ret) == 10: A1 * 10 = (A1 * 5) * 2
 %else
        lea     T0, [A1 + A1*2]         ; sizeof(psXX+ret) ==  6: A1 *  6 = (A1 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        movdqu  [A0], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_2_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      xmm0, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_ %+ %1 %+ _imm_u128
%endmacro

IEMIMPL_MEDIA_SSE_PSHIFTXX psllw
IEMIMPL_MEDIA_SSE_PSHIFTXX pslld
IEMIMPL_MEDIA_SSE_PSHIFTXX psllq
IEMIMPL_MEDIA_SSE_PSHIFTXX psrlw
IEMIMPL_MEDIA_SSE_PSHIFTXX psrld
IEMIMPL_MEDIA_SSE_PSHIFTXX psrlq
IEMIMPL_MEDIA_SSE_PSHIFTXX psraw
IEMIMPL_MEDIA_SSE_PSHIFTXX psrad
IEMIMPL_MEDIA_SSE_PSHIFTXX pslldq
IEMIMPL_MEDIA_SSE_PSHIFTXX psrldq


;
; Move byte mask.
;

BEGINPROC_FASTCALL iemAImpl_pmovmskb_u64, 8
        PROLOGUE_2_ARGS
        IEMIMPL_MMX_PROLOGUE

        movq    mm1, [A1]
        pmovmskb T0, mm1
        mov     [A0], T0
%ifdef RT_ARCH_X86
        mov     dword [A0 + 4], 0
%endif
        IEMIMPL_MMX_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_pmovmskb_u64

BEGINPROC_FASTCALL iemAImpl_pmovmskb_u128, 8
        PROLOGUE_2_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm1, [A1]
        pmovmskb T0, xmm1
        mov     [A0], T0
%ifdef RT_ARCH_X86
        mov     dword [A0 + 4], 0
%endif
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_pmovmskb_u128

BEGINPROC_FASTCALL iemAImpl_vpmovmskb_u256, 8
        PROLOGUE_2_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu  ymm1, [A1]
        vpmovmskb T0, ymm1
        mov     [A0], T0
%ifdef RT_ARCH_X86
        mov     dword [A0 + 4], 0
%endif
        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_vpmovmskb_u256


;;
; Media instruction working on two full sized source registers and one destination (AVX).
;
; @param    1       The instruction
;
; @param    A0      Pointer to the extended CPU/FPU state (X86XSAVEAREA).
; @param    A1      Pointer to the destination media register size operand (output).
; @param    A2      Pointer to the first source media register size operand (input).
; @param    A3      Pointer to the second source media register size operand (input).
;
%macro IEMIMPL_MEDIA_F3 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu  xmm0, [A2]
        vmovdqu  xmm1, [A3]
        %1       xmm0, xmm0, xmm1
        vmovdqu  [A1], xmm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu  ymm0, [A2]
        vmovdqu  ymm1, [A3]
        %1       ymm0, ymm0, ymm1
        vmovdqu  [A1], ymm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u256
%endmacro

IEMIMPL_MEDIA_F3 vpshufb
IEMIMPL_MEDIA_F3 vpand
IEMIMPL_MEDIA_F3 vpminub
IEMIMPL_MEDIA_F3 vpminuw
IEMIMPL_MEDIA_F3 vpminud
IEMIMPL_MEDIA_F3 vpminsb
IEMIMPL_MEDIA_F3 vpminsw
IEMIMPL_MEDIA_F3 vpminsd
IEMIMPL_MEDIA_F3 vpmaxub
IEMIMPL_MEDIA_F3 vpmaxuw
IEMIMPL_MEDIA_F3 vpmaxud
IEMIMPL_MEDIA_F3 vpmaxsb
IEMIMPL_MEDIA_F3 vpmaxsw
IEMIMPL_MEDIA_F3 vpmaxsd
IEMIMPL_MEDIA_F3 vpandn
IEMIMPL_MEDIA_F3 vpor
IEMIMPL_MEDIA_F3 vpxor
IEMIMPL_MEDIA_F3 vpcmpeqb
IEMIMPL_MEDIA_F3 vpcmpeqw
IEMIMPL_MEDIA_F3 vpcmpeqd
IEMIMPL_MEDIA_F3 vpcmpeqq
IEMIMPL_MEDIA_F3 vpcmpgtb
IEMIMPL_MEDIA_F3 vpcmpgtw
IEMIMPL_MEDIA_F3 vpcmpgtd
IEMIMPL_MEDIA_F3 vpcmpgtq
IEMIMPL_MEDIA_F3 vpaddb
IEMIMPL_MEDIA_F3 vpaddw
IEMIMPL_MEDIA_F3 vpaddd
IEMIMPL_MEDIA_F3 vpaddq
IEMIMPL_MEDIA_F3 vpsubb
IEMIMPL_MEDIA_F3 vpsubw
IEMIMPL_MEDIA_F3 vpsubd
IEMIMPL_MEDIA_F3 vpsubq


;;
; Media instruction working on two full sized source registers and one destination (AVX),
; but no XSAVE state pointer argument.
;
; @param    1       The instruction
;
; @param    A0      Pointer to the destination media register size operand (output).
; @param    A1      Pointer to the first source media register size operand (input).
; @param    A2      Pointer to the second source media register size operand (input).
;
%macro IEMIMPL_MEDIA_OPT_F3 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
        PROLOGUE_3_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu  xmm0, [A1]
        vmovdqu  xmm1, [A2]
        %1       xmm0, xmm0, xmm1
        vmovdqu  [A0], xmm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 12
        PROLOGUE_3_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu  ymm0, [A1]
        vmovdqu  ymm1, [A2]
        %1       ymm0, ymm0, ymm1
        vmovdqu  [A0], ymm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u256
%endmacro

IEMIMPL_MEDIA_OPT_F3 vpacksswb
IEMIMPL_MEDIA_OPT_F3 vpackssdw
IEMIMPL_MEDIA_OPT_F3 vpackuswb
IEMIMPL_MEDIA_OPT_F3 vpackusdw
IEMIMPL_MEDIA_OPT_F3 vpmullw
IEMIMPL_MEDIA_OPT_F3 vpmulld
IEMIMPL_MEDIA_OPT_F3 vpmulhw
IEMIMPL_MEDIA_OPT_F3 vpmulhuw
IEMIMPL_MEDIA_OPT_F3 vpavgb
IEMIMPL_MEDIA_OPT_F3 vpavgw
IEMIMPL_MEDIA_OPT_F3 vpsignb
IEMIMPL_MEDIA_OPT_F3 vpsignw
IEMIMPL_MEDIA_OPT_F3 vpsignd
IEMIMPL_MEDIA_OPT_F3 vphaddw
IEMIMPL_MEDIA_OPT_F3 vphaddd
IEMIMPL_MEDIA_OPT_F3 vphsubw
IEMIMPL_MEDIA_OPT_F3 vphsubd
IEMIMPL_MEDIA_OPT_F3 vphaddsw
IEMIMPL_MEDIA_OPT_F3 vphsubsw
IEMIMPL_MEDIA_OPT_F3 vpmaddubsw
IEMIMPL_MEDIA_OPT_F3 vpmulhrsw
IEMIMPL_MEDIA_OPT_F3 vpsadbw
IEMIMPL_MEDIA_OPT_F3 vpmuldq
IEMIMPL_MEDIA_OPT_F3 vpmuludq
IEMIMPL_MEDIA_OPT_F3 vunpcklps
IEMIMPL_MEDIA_OPT_F3 vunpcklpd
IEMIMPL_MEDIA_OPT_F3 vunpckhps
IEMIMPL_MEDIA_OPT_F3 vunpckhpd

;;
; Media instruction working on one full sized source registers and one destination (AVX),
; but no XSAVE state pointer argument.
;
; @param    1       The instruction
; @param    2       Flag whether the isntruction has a 256-bit (AVX2) variant (1) or not (0).
;
; @param    A0      Pointer to the destination media register size operand (output).
; @param    A1      Pointer to the source media register size operand (input).
;
%macro IEMIMPL_MEDIA_OPT_F2_AVX 2
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
        PROLOGUE_2_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu  xmm0, [A1]
        %1       xmm0, xmm0
        vmovdqu  [A0], xmm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128

 %if %2 == 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 12
        PROLOGUE_2_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu  ymm0, [A1]
        %1       ymm0, ymm0
        vmovdqu  [A0], ymm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u256
 %endif
%endmacro

IEMIMPL_MEDIA_OPT_F2_AVX vpabsb, 1
IEMIMPL_MEDIA_OPT_F2_AVX vpabsw, 1
IEMIMPL_MEDIA_OPT_F2_AVX vpabsd, 1
IEMIMPL_MEDIA_OPT_F2_AVX vphminposuw, 0


;
; The SSE 4.2 crc32
;
; @param    A1      Pointer to the 32-bit destination.
; @param    A2      The source operand, sized according to the suffix.
;
BEGINPROC_FASTCALL iemAImpl_crc32_u8, 8
        PROLOGUE_2_ARGS

        mov     T0_32, [A0]
        crc32   T0_32, A1_8
        mov     [A0], T0_32

        EPILOGUE_2_ARGS
ENDPROC iemAImpl_crc32_u8

BEGINPROC_FASTCALL iemAImpl_crc32_u16, 8
        PROLOGUE_2_ARGS

        mov     T0_32, [A0]
        crc32   T0_32, A1_16
        mov     [A0], T0_32

        EPILOGUE_2_ARGS
ENDPROC iemAImpl_crc32_u16

BEGINPROC_FASTCALL iemAImpl_crc32_u32, 8
        PROLOGUE_2_ARGS

        mov     T0_32, [A0]
        crc32   T0_32, A1_32
        mov     [A0], T0_32

        EPILOGUE_2_ARGS
ENDPROC iemAImpl_crc32_u32

%ifdef RT_ARCH_AMD64
BEGINPROC_FASTCALL iemAImpl_crc32_u64, 8
        PROLOGUE_2_ARGS

        mov     T0_32, [A0]
        crc32   T0, A1
        mov     [A0], T0_32

        EPILOGUE_2_ARGS
ENDPROC iemAImpl_crc32_u64
%endif


;
; PTEST (SSE 4.1)
;
; @param    A0      Pointer to the first source operand (aka readonly destination).
; @param    A1      Pointer to the second source operand.
; @param    A2      Pointer to the EFLAGS register.
;
BEGINPROC_FASTCALL  iemAImpl_ptest_u128, 12
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A0]
        movdqu  xmm1, [A1]
        ptest   xmm0, xmm1
        IEM_SAVE_FLAGS A2, X86_EFL_STATUS_BITS, 0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
ENDPROC             iemAImpl_ptest_u128

BEGINPROC_FASTCALL  iemAImpl_vptest_u256, 12
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        vmovdqu ymm0, [A0]
        vmovdqu ymm1, [A1]
        vptest  ymm0, ymm1
        IEM_SAVE_FLAGS A2, X86_EFL_STATUS_BITS, 0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
ENDPROC             iemAImpl_vptest_u256


;;
; Template for the [v]pmov{s,z}x* instructions
;
; @param    1       The instruction
;
; @param    A0      Pointer to the destination media register size operand (output).
; @param    A1      The source operand value (input).
;
%macro IEMIMPL_V_PMOV_SZ_X 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
        PROLOGUE_2_ARGS
        IEMIMPL_SSE_PROLOGUE

        movd     xmm0, A1
        %1       xmm0, xmm0
        vmovdqu  [A0], xmm0

        IEMIMPL_SSE_PROLOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 12
        PROLOGUE_2_ARGS
        IEMIMPL_AVX_PROLOGUE

        movd     xmm0, A1
        v %+ %1  xmm0, xmm0
        vmovdqu  [A0], xmm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 12
        PROLOGUE_2_ARGS
        IEMIMPL_AVX_PROLOGUE

        movdqu   xmm0, [A1]
        v %+ %1  ymm0, xmm0
        vmovdqu  [A0], ymm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u256
%endmacro

IEMIMPL_V_PMOV_SZ_X pmovsxbw
IEMIMPL_V_PMOV_SZ_X pmovsxbd
IEMIMPL_V_PMOV_SZ_X pmovsxbq
IEMIMPL_V_PMOV_SZ_X pmovsxwd
IEMIMPL_V_PMOV_SZ_X pmovsxwq
IEMIMPL_V_PMOV_SZ_X pmovsxdq

IEMIMPL_V_PMOV_SZ_X pmovzxbw
IEMIMPL_V_PMOV_SZ_X pmovzxbd
IEMIMPL_V_PMOV_SZ_X pmovzxbq
IEMIMPL_V_PMOV_SZ_X pmovzxwd
IEMIMPL_V_PMOV_SZ_X pmovzxwq
IEMIMPL_V_PMOV_SZ_X pmovzxdq


;;
; Need to move this as well somewhere better?
;
struc IEMSSERESULT
    .uResult      resd 4
    .MXCSR        resd 1
endstruc


;;
; Need to move this as well somewhere better?
;
struc IEMAVX128RESULT
    .uResult      resd 4
    .MXCSR        resd 1
endstruc


;;
; Need to move this as well somewhere better?
;
struc IEMAVX256RESULT
    .uResult      resd 8
    .MXCSR        resd 1
endstruc


;;
; Initialize the SSE MXCSR register using the guest value partially to
; account for rounding mode.
;
; @uses     4 bytes of stack to save the original value, T0.
; @param    1       Expression giving the address of the FXSTATE of the guest.
;
%macro SSE_LD_FXSTATE_MXCSR 1
        sub     xSP, 4

        stmxcsr [xSP]
        mov     T0_32, [%1 + X86FXSTATE.MXCSR]
        and     T0_32, X86_MXCSR_FZ | X86_MXCSR_RC_MASK | X86_MXCSR_DAZ
        or      T0_32, X86_MXCSR_XCPT_MASK
        sub     xSP, 4
        mov     [xSP], T0_32
        ldmxcsr [xSP]
        add     xSP, 4
%endmacro


;;
; Restores the SSE MXCSR register with the original value.
;
; @uses     4 bytes of stack to save the content of MXCSR value, T0, T1.
; @param    1       Expression giving the address where to return the MXCSR value.
; @param    2       Expression giving the address of the FXSTATE of the guest.
;
; @note Restores the stack pointer.
;
%macro SSE_ST_FXSTATE_MXCSR 2
        sub     xSP, 4
        stmxcsr [xSP]
        mov     T0_32, [xSP]
        add     xSP, 4
        ; Merge the status bits into the original MXCSR value.
        mov     T1_32, [%2 + X86FXSTATE.MXCSR]
        and     T0_32, X86_MXCSR_XCPT_FLAGS
        or      T0_32, T1_32
        mov     [%1 + IEMSSERESULT.MXCSR], T0_32

        ldmxcsr [xSP]
        add     xSP, 4
%endmacro


;;
; Initialize the SSE MXCSR register using the guest value partially to
; account for rounding mode.
;
; @uses     4 bytes of stack to save the original value.
; @param    1       Expression giving the address of the FXSTATE of the guest.
;
%macro AVX_LD_XSAVEAREA_MXCSR 1
        sub     xSP, 4

        stmxcsr [xSP]
        mov     T0_32, [%1 + X86FXSTATE.MXCSR]
        and     T0_32, X86_MXCSR_FZ | X86_MXCSR_RC_MASK | X86_MXCSR_DAZ
        sub     xSP, 4
        mov     [xSP], T0_32
        ldmxcsr [xSP]
        add     xSP, 4
%endmacro


;;
; Restores the AVX128 MXCSR register with the original value.
;
; @param    1       Expression giving the address where to return the MXCSR value.
;
; @note Restores the stack pointer.
;
%macro AVX128_ST_XSAVEAREA_MXCSR 1
        stmxcsr [%1 + IEMAVX128RESULT.MXCSR]

        ldmxcsr [xSP]
        add     xSP, 4
%endmacro


;;
; Restores the AVX256 MXCSR register with the original value.
;
; @param    1       Expression giving the address where to return the MXCSR value.
;
; @note Restores the stack pointer.
;
%macro AVX256_ST_XSAVEAREA_MXCSR 1
        stmxcsr [%1 + IEMAVX256RESULT.MXCSR]

        ldmxcsr [xSP]
        add     xSP, 4
%endmacro


;;
; Floating point instruction working on two full sized registers.
;
; @param    1       The instruction
; @param    2       Flag whether the AVX variant of the instruction takes two or three operands, 0 to disable AVX variants
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the result including the MXCSR value.
; @param    A2      Pointer to the first media register size operand (input/output).
; @param    A3      Pointer to the second media register size operand (input).
;
%macro IEMIMPL_FP_F2 2
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        movdqu   xmm0, [A2]
        movdqu   xmm1, [A3]
        %1       xmm0, xmm1
        movdqu   [A1 + IEMSSERESULT.uResult], xmm0

        SSE_ST_FXSTATE_MXCSR A1, A0
        IEMIMPL_SSE_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128

 %if %2 == 3
BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 12
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE
        AVX_LD_XSAVEAREA_MXCSR A0

        vmovdqu  xmm0, [A2]
        vmovdqu  xmm1, [A3]
        v %+ %1  xmm0, xmm0, xmm1
        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0

        AVX128_ST_XSAVEAREA_MXCSR A1
        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 12
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE
        AVX_LD_XSAVEAREA_MXCSR A0

        vmovdqu  ymm0, [A2]
        vmovdqu  ymm1, [A3]
        v %+ %1  ymm0, ymm0, ymm1
        vmovdqu  [A1 + IEMAVX256RESULT.uResult], ymm0

        AVX256_ST_XSAVEAREA_MXCSR A1
        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u256
 %elif %2 == 2
BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 12
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE
        AVX_LD_XSAVEAREA_MXCSR A0

        vmovdqu  xmm0, [A2]
        vmovdqu  xmm1, [A3]
        v %+ %1  xmm0, xmm1
        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0

        AVX128_ST_XSAVEAREA_MXCSR A1
        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 12
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE
        AVX_LD_XSAVEAREA_MXCSR A0

        vmovdqu  ymm0, [A2]
        vmovdqu  ymm1, [A3]
        v %+ %1  ymm0, ymm1
        vmovdqu  [A1 + IEMAVX256RESULT.uResult], ymm0

        AVX256_ST_XSAVEAREA_MXCSR A1
        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u256
 %endif
%endmacro

IEMIMPL_FP_F2 addps, 3
IEMIMPL_FP_F2 addpd, 3
IEMIMPL_FP_F2 mulps, 3
IEMIMPL_FP_F2 mulpd, 3
IEMIMPL_FP_F2 subps, 3
IEMIMPL_FP_F2 subpd, 3
IEMIMPL_FP_F2 minps, 3
IEMIMPL_FP_F2 minpd, 3
IEMIMPL_FP_F2 divps, 3
IEMIMPL_FP_F2 divpd, 3
IEMIMPL_FP_F2 maxps, 3
IEMIMPL_FP_F2 maxpd, 3
IEMIMPL_FP_F2 haddps, 3
IEMIMPL_FP_F2 haddpd, 3
IEMIMPL_FP_F2 hsubps, 3
IEMIMPL_FP_F2 hsubpd, 3
IEMIMPL_FP_F2 addsubps, 3
IEMIMPL_FP_F2 addsubpd, 3


;;
; These are actually unary operations but to keep it simple
; we treat them as binary for now, so the output result is
; always in sync with the register where the result might get written
; to.
IEMIMPL_FP_F2 sqrtps,    2
IEMIMPL_FP_F2 rsqrtps,   2
IEMIMPL_FP_F2 sqrtpd,    2
IEMIMPL_FP_F2 cvtdq2ps,  2
IEMIMPL_FP_F2 cvtps2dq,  2
IEMIMPL_FP_F2 cvttps2dq, 2
IEMIMPL_FP_F2 cvttpd2dq, 0 ; @todo AVX variants due to register size differences missing right now
IEMIMPL_FP_F2 cvtdq2pd,  0 ; @todo AVX variants due to register size differences missing right now
IEMIMPL_FP_F2 cvtpd2dq,  0 ; @todo AVX variants due to register size differences missing right now


;;
; Floating point instruction working on a full sized register and a single precision operand.
;
; @param    1       The instruction
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the result including the MXCSR value.
; @param    A2      Pointer to the first media register size operand (input/output).
; @param    A3      Pointer to the second single precision floating point value (input).
;
%macro IEMIMPL_FP_F2_R32 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128_r32, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        movdqu   xmm0, [A2]
        movd     xmm1, [A3]
        %1       xmm0, xmm1
        movdqu   [A1 + IEMSSERESULT.uResult], xmm0

        SSE_ST_FXSTATE_MXCSR A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128_r32

BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128_r32, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE
        AVX_LD_XSAVEAREA_MXCSR A0

        vmovdqu  xmm0, [A2]
        vmovd    xmm1, [A3]
        v %+ %1  xmm0, xmm0, xmm1
        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0

        AVX128_ST_XSAVEAREA_MXCSR A1
        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u128_r32
%endmacro

IEMIMPL_FP_F2_R32 addss
IEMIMPL_FP_F2_R32 mulss
IEMIMPL_FP_F2_R32 subss
IEMIMPL_FP_F2_R32 minss
IEMIMPL_FP_F2_R32 divss
IEMIMPL_FP_F2_R32 maxss
IEMIMPL_FP_F2_R32 cvtss2sd
IEMIMPL_FP_F2_R32 sqrtss
IEMIMPL_FP_F2_R32 rsqrtss


;;
; Floating point instruction working on a full sized register and a double precision operand.
;
; @param    1       The instruction
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the result including the MXCSR value.
; @param    A2      Pointer to the first media register size operand (input/output).
; @param    A3      Pointer to the second double precision floating point value (input).
;
%macro IEMIMPL_FP_F2_R64 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128_r64, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        movdqu   xmm0, [A2]
        movq     xmm1, [A3]
        %1       xmm0, xmm1
        movdqu   [A1 + IEMSSERESULT.uResult], xmm0

        SSE_ST_FXSTATE_MXCSR A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128_r64

BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128_r64, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE
        AVX_LD_XSAVEAREA_MXCSR A0

        vmovdqu  xmm0, [A2]
        vmovq    xmm1, [A3]
        v %+ %1  xmm0, xmm0, xmm1
        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0

        AVX128_ST_XSAVEAREA_MXCSR A1
        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u128_r64
%endmacro

IEMIMPL_FP_F2_R64 addsd
IEMIMPL_FP_F2_R64 mulsd
IEMIMPL_FP_F2_R64 subsd
IEMIMPL_FP_F2_R64 minsd
IEMIMPL_FP_F2_R64 divsd
IEMIMPL_FP_F2_R64 maxsd
IEMIMPL_FP_F2_R64 cvtsd2ss
IEMIMPL_FP_F2_R64 sqrtsd


;;
; Macro for the cvtpd2ps/cvtps2pd instructions.
;
;           1       The instruction name.
;           2       Whether the AVX256 result is 128-bit (0) or 256-bit (1).
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the result including the MXCSR value.
; @param    A2      Pointer to the first media register size operand (input/output).
; @param    A3      Pointer to the second media register size operand (input).
;
%macro IEMIMPL_CVT_F2 2
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        movdqu   xmm0, [A2]
        movdqu   xmm1, [A3]
        %1       xmm0, xmm1
        movdqu   [A1 + IEMSSERESULT.uResult], xmm0

        SSE_ST_FXSTATE_MXCSR A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE
        AVX_LD_XSAVEAREA_MXCSR A0

        vmovdqu   xmm0, [A2]
        vmovdqu   xmm1, [A3]
        v %+ %1   xmm0, xmm1
        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0

        AVX128_ST_XSAVEAREA_MXCSR A1
        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE
        AVX_LD_XSAVEAREA_MXCSR A0

        vmovdqu    ymm0, [A2]
        vmovdqu    ymm1, [A3]
 %if %2 == 0
        v %+ %1    xmm0, ymm1
 %else
        v %+ %1    ymm0, xmm1
 %endif
        vmovdqu    [A1 + IEMAVX256RESULT.uResult], ymm0

        AVX256_ST_XSAVEAREA_MXCSR A1
        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_v %+ %1 %+ _u256
%endmacro

IEMIMPL_CVT_F2 cvtpd2ps, 0
IEMIMPL_CVT_F2 cvtps2pd, 1


;;
; shufps instructions with 8-bit immediates.
;
; @param    A0      Pointer to the destination media register size operand (input/output).
; @param    A1      Pointer to the first source media register size operand (input).
; @param    A2      The 8-bit immediate
;
BEGINPROC_FASTCALL iemAImpl_shufps_u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A0]
        movdqu  xmm1, [A1]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*4]         ; sizeof(shufpX+ret+int3) == 10: A2 * 10 = (A2 * 5) * 2
 %else
        lea     T0, [A2 + A2*2]         ; sizeof(shufpX+ret+int3) ==  6: A2 *  6 = (A2 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        movdqu  [A0], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        shufps  xmm0, xmm1, bImm
        ret
        int3
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_shufps_u128


;;
; shufpd instruction with 8-bit immediates.
;
; @param    A0      Pointer to the destination media register size operand (input/output).
; @param    A1      Pointer to the first source media register size operand (input).
; @param    A2      The 8-bit immediate
;
BEGINPROC_FASTCALL iemAImpl_shufpd_u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A0]
        movdqu  xmm1, [A1]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*4]         ; sizeof(shufpX+ret) == 10: A2 * 10 = (A2 * 5) * 2
 %else
        lea     T0, [A2 + A2*2]         ; sizeof(shufpX+ret) ==  6: A2 *  6 = (A2 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        movdqu  [A0], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        shufpd  xmm0, xmm1, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_shufpd_u128


;;
; vshufp{s,d} instructions with 8-bit immediates.
;
; @param    1       The instruction name.
;
; @param    A0      Pointer to the destination media register size operand (output).
; @param    A1      Pointer to the first source media register size operand (input).
; @param    A2      Pointer to the second source media register size operand (input).
; @param    A3      The 8-bit immediate
;
%macro IEMIMPL_MEDIA_AVX_VSHUFPX 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE

        movdqu  xmm0, [A1]
        movdqu  xmm1, [A2]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A3 + A3*4]         ; sizeof(vshufpX+ret) == 10: A3 * 10 = (A3 * 5) * 2
 %else
        lea     T0, [A3 + A3*2]         ; sizeof(vshufpX+ret) ==  6: A3 *  6 = (A3 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        movdqu  [A0], xmm0

        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_4_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      xmm0, xmm0, xmm1, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_ %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu ymm0, [A1]
        vmovdqu ymm1, [A2]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A3 + A3*4]         ; sizeof(vshufpX+ret) == 10: A3 * 10 = (A3 * 5) * 2
 %else
        lea     T0, [A3 + A3*2]         ; sizeof(vshufpX+ret) ==  6: A3 *  6 = (A3 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        vmovdqu [A0], ymm0

        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_4_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      ymm0, ymm0, ymm1, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_ %+ %1 %+ _u256
%endmacro

IEMIMPL_MEDIA_AVX_VSHUFPX vshufps
IEMIMPL_MEDIA_AVX_VSHUFPX vshufpd


;;
; One of the [p]blendv{b,ps,pd} variants
;
; @param    1       The instruction
;
; @param    A0      Pointer to the first media register sized operand (input/output).
; @param    A1      Pointer to the second media sized value (input).
; @param    A2      Pointer to the media register sized mask value (input).
;
%macro IEMIMPL_P_BLEND 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu   xmm0, [A2] ; This is implicit
        movdqu   xmm1, [A0]
        movdqu   xmm2, [A1] ; @todo Do I need to save the original value here first?
        %1       xmm1, xmm2
        movdqu   [A0], xmm1

        IEMIMPL_SSE_PROLOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_P_BLEND pblendvb
IEMIMPL_P_BLEND blendvps
IEMIMPL_P_BLEND blendvpd


;;
; One of the v[p]blendv{b,ps,pd} variants
;
; @param    1       The instruction
;
; @param    A0      Pointer to the first media register sized operand (output).
; @param    A1      Pointer to the first media register sized operand (input).
; @param    A2      Pointer to the second media register sized operand (input).
; @param    A3      Pointer to the media register sized mask value (input).
%macro IEMIMPL_AVX_P_BLEND 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu   xmm0, [A1]
        vmovdqu   xmm1, [A2]
        vmovdqu   xmm2, [A3]
        %1        xmm0, xmm0, xmm1, xmm2
        vmovdqu   [A0], xmm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu   ymm0, [A1]
        vmovdqu   ymm1, [A2]
        vmovdqu   ymm2, [A3]
        %1        ymm0, ymm0, ymm1, ymm2
        vmovdqu   [A0], ymm0

        IEMIMPL_AVX_PROLOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u256
%endmacro

IEMIMPL_AVX_P_BLEND vpblendvb
IEMIMPL_AVX_P_BLEND vblendvps
IEMIMPL_AVX_P_BLEND vblendvpd


;;
; palignr mm1, mm2/m64 instruction.
;
; @param    A0      Pointer to the first media register sized operand (output).
; @param    A1      The second register sized operand (input).
; @param    A2      The 8-bit immediate.
BEGINPROC_FASTCALL iemAImpl_palignr_u64, 16
        PROLOGUE_3_ARGS
        IEMIMPL_MMX_PROLOGUE

        movq    mm0, [A0]
        movq    mm1, A1
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*4]         ; sizeof(endbrxx+palignr+ret) == 10: A2 * 10 = (A2 * 5) * 2
 %else
        lea     T0, [A2 + A2*2]         ; sizeof(palignr+ret)         ==  6: A2 *  6 = (A2 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        movq    [A0], mm0

        IEMIMPL_MMX_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        palignr mm0, mm1, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_palignr_u64


;;
; SSE instructions with 8-bit immediates of the form
;    xxx xmm1, xmm2, imm8.
; where the instruction encoding takes up 6 bytes.
;
; @param    1       The instruction name.
;
; @param    A0      Pointer to the first media register size operand (input/output).
; @param    A1      Pointer to the second source media register size operand (input).
; @param    A2      The 8-bit immediate
;
%macro IEMIMPL_MEDIA_SSE_INSN_IMM8_6 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A0]
        movdqu  xmm1, [A1]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*2]         ; sizeof(endbrxx+insnX+ret+int3) == 12: A2 * 12 = (A2 * 3) * 4
        lea     T1, [T1 + T0*4]
 %else
        lea     T1, [T1 + A2*8]         ; sizeof(insnX+ret+int3)         ==  8: A2 * 8
 %endif
        IBT_NOTRACK
        call    T1
        movdqu  [A0], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      xmm0, xmm1, bImm
        ret
        int3
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_SSE_INSN_IMM8_6 blendps
IEMIMPL_MEDIA_SSE_INSN_IMM8_6 blendpd
IEMIMPL_MEDIA_SSE_INSN_IMM8_6 pblendw
IEMIMPL_MEDIA_SSE_INSN_IMM8_6 palignr
IEMIMPL_MEDIA_SSE_INSN_IMM8_6 pclmulqdq
IEMIMPL_MEDIA_SSE_INSN_IMM8_6 aeskeygenassist


;;
; AVX instructions with 8-bit immediates of the form
;    xxx {x,y}mm1, {x,y}mm2, {x,y}mm3, imm8.
; where the instruction encoding takes up 6 bytes.
;
; @param    1       The instruction name.
; @param    2       Whether the instruction has a 256-bit variant (1) or not (0).
;
; @param    A0      Pointer to the destination media register size operand (output).
; @param    A1      Pointer to the first source media register size operand (input).
; @param    A2      Pointer to the second source media register size operand (input).
; @param    A3      The 8-bit immediate
;
%macro IEMIMPL_MEDIA_AVX_INSN_IMM8_6 2
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE

        movdqu  xmm0, [A1]
        movdqu  xmm1, [A2]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A3 + A3*2]         ; sizeof(endbrxx+insnX+ret+int3) == 12: A3 * 12 = (A3 * 3) * 4
        lea     T1, [T1 + T0*4]
 %else
        lea     T1, [T1 + A3*8]         ; sizeof(insnX+ret+int3)         ==  8: A3 * 8
 %endif
        IBT_NOTRACK
        call    T1
        movdqu  [A0], xmm0

        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_4_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      xmm0, xmm0, xmm1, bImm
        ret
        int3
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
ENDPROC iemAImpl_ %+ %1 %+ _u128

 %if %2 == 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
        PROLOGUE_4_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu ymm0, [A1]
        vmovdqu ymm1, [A2]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A3 + A3*2]         ; sizeof(endbrxx+insnX+ret+int3) == 12: A3 * 12 = (A3 * 3) * 4
        lea     T1, [T1 + T0*4]
 %else
        lea     T1, [T1 + A3*8]         ; sizeof(insnX+ret+int3)         ==  8: A3 *  8
 %endif
        IBT_NOTRACK
        call    T1
        vmovdqu [A0], ymm0

        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_4_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      ymm0, ymm0, ymm1, bImm
        ret
        int3
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
ENDPROC iemAImpl_ %+ %1 %+ _u256
 %endif
%endmacro

IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vblendps,   1
IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vblendpd,   1
IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vpblendw,   1
IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vpalignr,   1
IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vpclmulqdq, 0


;;
; Need to move this as well somewhere better?
;
struc IEMPCMPISTRISRC
    .uSrc1        resd 4
    .uSrc2        resd 4
endstruc

;;
; The pcmpistri instruction.
;
; @param    A0      Pointer to the ECX register to store the result to (output).
; @param    A1      Pointer to the EFLAGS register.
; @param    A2      Pointer to the structure containing the source operands (input).
; @param    A3      The 8-bit immediate
;
BEGINPROC_FASTCALL iemAImpl_pcmpistri_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A2 + IEMPCMPISTRISRC.uSrc1]
        movdqu  xmm1, [A2 + IEMPCMPISTRISRC.uSrc2]
        mov     T2, A0                  ; A0 can be ecx/rcx in some calling conventions which gets overwritten later (T2 only available on AMD64)
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A3 + A3*2]         ; sizeof(endbrxx+insnX+ret) == 12: A3 * 12 = (A3 * 3) * 4
        lea     T1, [T1 + T0*4]
 %else
        lea     T1, [T1 + A3*8]         ; sizeof(insnX+ret)         ==  8: A3 * 8
 %endif
        IBT_NOTRACK
        call    T1

        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
        mov    [T2], ecx

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        pcmpistri xmm0, xmm1, bImm
        ret
        int3
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
ENDPROC iemAImpl_pcmpistri_u128


;;
; pinsrw instruction.
;
; @param    A0      Pointer to the first media register size operand (input/output).
; @param    A1      The 16 bit input operand (input).
; @param    A2      The 8-bit immediate
;
BEGINPROC_FASTCALL iemAImpl_pinsrw_u64, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movq    mm0,  [A0]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*8]         ; sizeof(endbrxx+pinsrw+ret) == 9: A2 * 9
 %else
        lea     T0, [A2 + A2*4]         ; sizeof(pinsrw+ret)         == 5: A2 * 5
 %endif
        lea     T1, [T1 + T0]
        IBT_NOTRACK
        call    T1
        movq    [A0], mm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        pinsrw  mm0, A1_32, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
ENDPROC iemAImpl_pinsrw_u64

BEGINPROC_FASTCALL iemAImpl_pinsrw_u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A0]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*4]         ; sizeof(endbrxx+pinsrw+ret) == 10: A2 * 10 = (A2 * 5) * 2
 %else
        lea     T0, [A2 + A2*2]         ; sizeof(pinsrw+ret)         ==  6: A2 *  6 = (A2 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        movdqu  [A0], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        pinsrw  xmm0, A1_32, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_pinsrw_u128

;;
; vpinsrw instruction.
;
; @param    A0      Pointer to the first media register size operand (output).
; @param    A1      Pointer to the source media register size operand (input).
; @param    A2      The 16 bit input operand (input).
; @param    A3      The 8-bit immediate
;
BEGINPROC_FASTCALL iemAImpl_vpinsrw_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A1]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A3 + A3*4]         ; sizeof(endbrxx+vpinsrw+ret) == 10: A3 * 10 = (A3 * 5) * 2
 %else
        lea     T0, [A3 + A3*2]         ; sizeof(vpinsrw+ret)         ==  6: A3 *  6 = (A3 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        mov     A1, A2                  ; A2 requires longer encoding on Windows
        IBT_NOTRACK
        call    T1
        movdqu  [A0], xmm0

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        vpinsrw xmm0, xmm0, A1_32, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_vpinsrw_u128


;;
; pextrw instruction.
;
; @param    A0      Pointer to the 16bit output operand (output).
; @param    A1      Pointer to the media register size operand (input).
; @param    A2      The 8-bit immediate
;
BEGINPROC_FASTCALL iemAImpl_pextrw_u64, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movq    mm0,  A1
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*8]         ; sizeof(endbrxx+pextrw+ret) == 9: A2 * 9
 %else
        lea     T0, [A2 + A2*4]         ; sizeof(pextrw+ret)         == 5: A2 * 5
 %endif
        lea     T1, [T1 + T0]
        IBT_NOTRACK
        call    T1
        mov     word [A0], T0_16

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        pextrw  T0_32, mm0, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
ENDPROC iemAImpl_pextrw_u64

BEGINPROC_FASTCALL iemAImpl_pextrw_u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A1]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*4]         ; sizeof(endbrxx+pextrw+ret) == 10: A2 * 10 = (A2 * 5) * 2
 %else
        lea     T0, [A2 + A2*2]         ; sizeof(pextrw+ret)         ==  6: A2 *  6 = (A2 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        mov     word [A0], T0_16

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        pextrw  T0_32, xmm0, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_pextrw_u128

;;
; vpextrw instruction.
;
; @param    A0      Pointer to the 16bit output operand (output).
; @param    A1      Pointer to the source media register size operand (input).
; @param    A2      The 8-bit immediate
;
BEGINPROC_FASTCALL iemAImpl_vpextrw_u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A1]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A2 + A2*4]         ; sizeof(endbrxx+vpextrw+ret) == 10: A2 * 10 = (A2 * 5) * 2
 %else
        lea     T0, [A2 + A2*2]         ; sizeof(vpextrw+ret)         ==  6: A2 *  6 = (A2 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        mov     word [A0], T0_16

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        vpextrw T0_32, xmm0, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_vpextrw_u128


;;
; movmskp{s,d} SSE instruction template
;
; @param    1       The SSE instruction name.
; @param    2       The AVX instruction name.
;
; @param    A0      Pointer to the output register (output/byte sized).
; @param    A1      Pointer to the source media register size operand (input).
;
%macro IEMIMPL_MEDIA_MOVMSK_P 2
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_2_ARGS
        IEMIMPL_SSE_PROLOGUE

        movdqu  xmm0, [A1]
        %1      T0, xmm0
        mov     byte [A0], T0_8

        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128

BEGINPROC_FASTCALL iemAImpl_ %+ %2 %+ _u128, 16
        PROLOGUE_2_ARGS
        IEMIMPL_AVX_PROLOGUE

        movdqu  xmm0, [A1]
        %2      T0, xmm0
        mov     byte [A0], T0_8

        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %2 %+ _u128

BEGINPROC_FASTCALL iemAImpl_ %+ %2 %+ _u256, 16
        PROLOGUE_2_ARGS
        IEMIMPL_AVX_PROLOGUE

        vmovdqu ymm0, [A1]
        %2      T0, ymm0
        mov     byte [A0], T0_8

        IEMIMPL_AVX_EPILOGUE
        EPILOGUE_2_ARGS
ENDPROC iemAImpl_ %+ %2 %+ _u256
%endmacro

IEMIMPL_MEDIA_MOVMSK_P movmskps, vmovmskps
IEMIMPL_MEDIA_MOVMSK_P movmskpd, vmovmskpd


;;
; Restores the SSE MXCSR register with the original value.
;
; @uses     4 bytes of stack to save the content of MXCSR value, T0, T1.
; @param    1       Expression giving the address where to return the MXCSR value - only the MXCSR is stored, no IEMSSERESULT is used.
; @param    2       Expression giving the address of the FXSTATE of the guest.
;
; @note Restores the stack pointer.
;
%macro SSE_ST_FXSTATE_MXCSR_ONLY 2
        sub     xSP, 4
        stmxcsr [xSP]
        mov     T0_32, [xSP]
        add     xSP, 4
        ; Merge the status bits into the original MXCSR value.
        mov     T1_32, [%2 + X86FXSTATE.MXCSR]
        and     T0_32, X86_MXCSR_XCPT_FLAGS
        or      T0_32, T1_32
        mov     [%1], T0_32

        ldmxcsr [xSP]
        add     xSP, 4
%endmacro


;;
; cvttsd2si instruction - 32-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvttsd2si_i32_r64, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvttsd2si T0_32, [A3]
        mov       dword [A2], T0_32

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvttsd2si_i32_r64

;;
; cvttsd2si instruction - 64-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvttsd2si_i64_r64, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvttsd2si T0, [A3]
        mov       qword [A2], T0

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvttsd2si_i64_r64


;;
; cvtsd2si instruction - 32-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvtsd2si_i32_r64, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvtsd2si  T0_32, [A3]
        mov       dword [A2], T0_32

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvtsd2si_i32_r64

;;
; cvtsd2si instruction - 64-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvtsd2si_i64_r64, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvtsd2si  T0, [A3]
        mov       qword [A2], T0

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvtsd2si_i64_r64


;;
; cvttss2si instruction - 32-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvttss2si_i32_r32, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvttss2si T0_32, [A3]
        mov       dword [A2], T0_32

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvttss2si_i32_r32

;;
; cvttss2si instruction - 64-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvttss2si_i64_r32, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvttss2si T0, [A3]
        mov       qword [A2], T0

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvttss2si_i64_r32


;;
; cvtss2si instruction - 32-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvtss2si_i32_r32, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvtss2si  T0_32, [A3]
        mov       dword [A2], T0_32

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvtss2si_i32_r32

;;
; cvtss2si instruction - 64-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvtss2si_i64_r32, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvtss2si  T0, [A3]
        mov       qword [A2], T0

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvtss2si_i64_r32


;;
; cvtsi2ss instruction - 32-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvtsi2ss_r32_i32, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvtsi2ss  xmm0, dword [A3]
        movd      dword [A2], xmm0

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvtsi2ss_r32_i32

;;
; cvtsi2ss instruction - 64-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvtsi2ss_r32_i64, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvtsi2ss  xmm0, qword [A3]
        movd      dword [A2], xmm0

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvtsi2ss_r32_i64


;;
; cvtsi2sd instruction - 32-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvtsi2sd_r64_i32, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvtsi2sd  xmm0, dword [A3]
        movq      [A2], xmm0

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvtsi2sd_r64_i32

;;
; cvtsi2sd instruction - 64-bit variant.
;
; @param    A0      FPU context (FXSTATE or XSAVEAREA).
; @param    A1      Where to return the MXCSR value.
; @param    A2      Pointer to the result operand (output).
; @param    A3      Pointer to the second operand (input).
;
BEGINPROC_FASTCALL iemAImpl_cvtsi2sd_r64_i64, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR A0

        cvtsi2sd  xmm0, qword [A3]
        movq      [A2], xmm0

        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC iemAImpl_cvtsi2sd_r64_i64


;;
; Initialize the SSE MXCSR register using the guest value partially to
; account for rounding mode.
;
; @uses     4 bytes of stack to save the original value, T0.
; @param    1       Expression giving the address of the MXCSR register of the guest.
;
%macro SSE_LD_FXSTATE_MXCSR_ONLY 1
        sub     xSP, 4

        stmxcsr [xSP]
        mov     T0_32, [%1]
        and     T0_32, X86_MXCSR_FZ | X86_MXCSR_RC_MASK | X86_MXCSR_DAZ
        or      T0_32, X86_MXCSR_XCPT_MASK
        sub     xSP, 4
        mov     [xSP], T0_32
        ldmxcsr [xSP]
        add     xSP, 4
%endmacro


;;
; Restores the SSE MXCSR register with the original value.
;
; @uses     4 bytes of stack to save the content of MXCSR value, T0, T1.
; @param    1       Expression giving the address where to return the MXCSR value - only the MXCSR is stored, no IEMSSERESULT is used.
;
; @note Restores the stack pointer.
;
%macro SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE 1
        sub     xSP, 4
        stmxcsr [xSP]
        mov     T0_32, [xSP]
        add     xSP, 4
        ; Merge the status bits into the original MXCSR value.
        mov     T1_32, [%1]
        and     T0_32, X86_MXCSR_XCPT_FLAGS
        or      T0_32, T1_32
        mov     [%1], T0_32

        ldmxcsr [xSP]
        add     xSP, 4
%endmacro


;
; UCOMISS (SSE)
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the EFLAGS value (input/output).
; @param    A2      Pointer to the first source operand (aka readonly destination).
; @param    A3      Pointer to the second source operand.
;
BEGINPROC_FASTCALL  iemAImpl_ucomiss_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2]
        movdqu  xmm1, [A3]
        ucomiss xmm0, xmm1
        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC             iemAImpl_ucomiss_u128

BEGINPROC_FASTCALL  iemAImpl_vucomiss_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2]
        movdqu  xmm1, [A3]
        vucomiss xmm0, xmm1
        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC             iemAImpl_vucomiss_u128


;
; UCOMISD (SSE)
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the EFLAGS value (input/output).
; @param    A2      Pointer to the first source operand (aka readonly destination).
; @param    A3      Pointer to the second source operand.
;
BEGINPROC_FASTCALL  iemAImpl_ucomisd_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2]
        movdqu  xmm1, [A3]
        ucomisd xmm0, xmm1
        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC             iemAImpl_ucomisd_u128

BEGINPROC_FASTCALL  iemAImpl_vucomisd_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2]
        movdqu  xmm1, [A3]
        vucomisd xmm0, xmm1
        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC             iemAImpl_vucomisd_u128

;
; COMISS (SSE)
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the EFLAGS value (input/output).
; @param    A2      Pointer to the first source operand (aka readonly destination).
; @param    A3      Pointer to the second source operand.
;
BEGINPROC_FASTCALL  iemAImpl_comiss_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2]
        movdqu  xmm1, [A3]
        comiss xmm0, xmm1
        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC             iemAImpl_comiss_u128

BEGINPROC_FASTCALL  iemAImpl_vcomiss_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2]
        movdqu  xmm1, [A3]
        vcomiss xmm0, xmm1
        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC             iemAImpl_vcomiss_u128


;
; COMISD (SSE)
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the EFLAGS value (input/output).
; @param    A2      Pointer to the first source operand (aka readonly destination).
; @param    A3      Pointer to the second source operand.
;
BEGINPROC_FASTCALL  iemAImpl_comisd_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2]
        movdqu  xmm1, [A3]
        comisd xmm0, xmm1
        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC             iemAImpl_comisd_u128

BEGINPROC_FASTCALL  iemAImpl_vcomisd_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2]
        movdqu  xmm1, [A3]
        vcomisd xmm0, xmm1
        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
ENDPROC             iemAImpl_vcomisd_u128


;;
; Need to move this as well somewhere better?
;
struc IEMMEDIAF2XMMSRC
    .uSrc1        resd 4
    .uSrc2        resd 4
endstruc


;
; CMPPS (SSE)
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the first media register size operand (output).
; @param    A2      Pointer to the two media register sized inputs - IEMMEDIAF2XMMSRC (input).
; @param    A3      The 8-bit immediate (input).
;
BEGINPROC_FASTCALL iemAImpl_cmpps_u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2 + IEMMEDIAF2XMMSRC.uSrc1]
        movdqu  xmm1, [A2 + IEMMEDIAF2XMMSRC.uSrc2]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A3 + A3*8]         ; sizeof(endbrxx+cmpps+ret) == 9: A3 * 9
 %else
        lea     T0, [A3 + A3*4]         ; sizeof(cmpps+ret)         == 5: A3 * 5
 %endif
        lea     T1, [T1 + T0]
        IBT_NOTRACK
        call    T1
        movdqu  [A1], xmm0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        cmpps   xmm0, xmm1, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
ENDPROC iemAImpl_cmpps_u128

;;
; SSE instructions with 8-bit immediates of the form
;    xxx xmm1, xmm2, imm8.
; where the instruction encoding takes up 5 bytes and we need to load and save the MXCSR
; register.
;
; @param    1       The instruction name.
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the first media register size operand (output).
; @param    A2      Pointer to the two media register sized inputs - IEMMEDIAF2XMMSRC (input).
; @param    A3      The 8-bit immediate (input).
;
%macro IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2 + IEMMEDIAF2XMMSRC.uSrc1]
        movdqu  xmm1, [A2 + IEMMEDIAF2XMMSRC.uSrc2]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A3 + A3*4]         ; sizeof(endbrxx+cmpXX+ret) == 10: A3 * 10 = (A3 * 5) * 2
 %else
        lea     T0, [A3 + A3*2]         ; sizeof(cmpXX+ret)         ==  6: A3 *  6 = (A3 * 3) * 2
 %endif
        lea     T1, [T1 + T0*2]
        IBT_NOTRACK
        call    T1
        movdqu  [A1], xmm0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      xmm0, xmm1, bImm
        ret
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 cmppd
IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 cmpss
IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 cmpsd

;;
; SSE instructions with 8-bit immediates of the form
;    xxx xmm1, xmm2, imm8.
; where the instruction encoding takes up 6 bytes and we need to load and save the MXCSR
; register.
;
; @param    1       The instruction name.
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the first media register size operand (output).
; @param    A2      Pointer to the two media register sized inputs - IEMMEDIAF2XMMSRC (input).
; @param    A3      The 8-bit immediate (input).
;
%macro IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_4_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2 + IEMMEDIAF2XMMSRC.uSrc1]
        movdqu  xmm1, [A2 + IEMMEDIAF2XMMSRC.uSrc2]
        lea     T1, [.imm0 xWrtRIP]
 %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
        lea     T0, [A3 + A3*2]         ; sizeof(endbrxx+insn+ret+int3) == 12: A3 * 12 = (A3 * 3) * 4
        lea     T1, [T1 + T0*4]
 %else
        lea     T1, [T1 + A3*8]         ; sizeof(insn+ret+int3)         ==  8: A3 * 8
 %endif
        IBT_NOTRACK
        call    T1
        movdqu  [A1], xmm0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_4_ARGS
 %assign bImm 0
 %rep 256
.imm %+ bImm:
        IBT_ENDBRxx_WITHOUT_NOTRACK
        %1      xmm0, xmm1, bImm
        ret
        int3
  %assign bImm bImm + 1
 %endrep
.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundps
IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundpd
IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundss
IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundsd


;;
; SSE instructions of the form
;    xxx mm, xmm.
; and we need to load and save the MXCSR register.
;
; @param    1       The instruction name.
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the first MMX register sized operand (output).
; @param    A2      Pointer to the media register sized operand (input).
;
%macro IEMIMPL_MEDIA_SSE_MXCSR_I64_U128 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A2]
        %1      mm0, xmm0
        movq    [A1], mm0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_SSE_MXCSR_I64_U128 cvtpd2pi
IEMIMPL_MEDIA_SSE_MXCSR_I64_U128 cvttpd2pi

;;
; SSE instructions of the form
;    xxx xmm, xmm/m64.
; and we need to load and save the MXCSR register.
;
; @param    1       The instruction name.
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the first media register sized operand (input/output).
; @param    A2      The 64bit source value from a MMX media register (input)
;
%macro IEMIMPL_MEDIA_SSE_MXCSR_U128_U64 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movdqu  xmm0, [A1]
        movq    mm0, A2
        %1      xmm0, mm0
        movdqu  [A1], xmm0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_SSE_MXCSR_U128_U64 cvtpi2ps
IEMIMPL_MEDIA_SSE_MXCSR_U128_U64 cvtpi2pd

;;
; SSE instructions of the form
;    xxx mm, xmm/m64.
; and we need to load and save the MXCSR register.
;
; @param    1       The instruction name.
;
; @param    A0      Pointer to the MXCSR value (input/output).
; @param    A1      Pointer to the first MMX media register sized operand (output).
; @param    A2      The 64bit source value (input).
;
%macro IEMIMPL_MEDIA_SSE_MXCSR_U64_U64 1
BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
        PROLOGUE_3_ARGS
        IEMIMPL_SSE_PROLOGUE
        SSE_LD_FXSTATE_MXCSR_ONLY A0

        movq    xmm0, A2
        %1      mm0, xmm0
        movq    [A1], mm0

        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
        IEMIMPL_SSE_EPILOGUE
        EPILOGUE_3_ARGS
ENDPROC iemAImpl_ %+ %1 %+ _u128
%endmacro

IEMIMPL_MEDIA_SSE_MXCSR_U64_U64 cvtps2pi
IEMIMPL_MEDIA_SSE_MXCSR_U64_U64 cvttps2pi

;
; All forms of RDRAND and RDSEED
;
; @param    A0      Pointer to the destination operand.
; @param    A1      Pointer to the EFLAGS value (input/output).
;
%macro IEMIMPL_RDRAND_RDSEED 3
BEGINPROC_FASTCALL  iemAImpl_ %+ %1 %+ _u %+ %3, 8
        PROLOGUE_2_ARGS

        %1      %2
        mov     [A0], %2
        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0

        EPILOGUE_2_ARGS
ENDPROC             iemAImpl_ %+ %1 %+ _u %+ %3
%endmacro

IEMIMPL_RDRAND_RDSEED rdrand, ax,  16
IEMIMPL_RDRAND_RDSEED rdrand, eax, 32
IEMIMPL_RDRAND_RDSEED rdrand, rax, 64
IEMIMPL_RDRAND_RDSEED rdseed, ax,  16
IEMIMPL_RDRAND_RDSEED rdseed, eax, 32
IEMIMPL_RDRAND_RDSEED rdseed, rax, 64