summaryrefslogtreecommitdiffstats
path: root/third_party/aom/aom_dsp/x86/sad4d_sse2.asm
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--third_party/aom/aom_dsp/x86/sad4d_sse2.asm437
1 files changed, 437 insertions, 0 deletions
diff --git a/third_party/aom/aom_dsp/x86/sad4d_sse2.asm b/third_party/aom/aom_dsp/x86/sad4d_sse2.asm
new file mode 100644
index 0000000000..6edad99516
--- /dev/null
+++ b/third_party/aom/aom_dsp/x86/sad4d_sse2.asm
@@ -0,0 +1,437 @@
+;
+; Copyright (c) 2016, Alliance for Open Media. All rights reserved
+;
+; This source code is subject to the terms of the BSD 2 Clause License and
+; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+; was not distributed with this source code in the LICENSE file, you can
+; obtain it at www.aomedia.org/license/software. If the Alliance for Open
+; Media Patent License 1.0 was not distributed with this source code in the
+; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+;
+
+;
+
+%include "third_party/x86inc/x86inc.asm"
+
+SECTION .text
+
+; 'spill_src_stride' affect a lot how the code works.
+;
+; When 'spill_src_stride' is false, the 'src_strideq' resides in
+; register, [srcq + src_strideq + offset] is allowed, so we can simply
+; use such form to access src memory and don't bother to update 'srcq'
+; at each line. We only update 'srcq' each two-lines using a compact
+; LEA instruction like [srcq+src_strideq*2].
+;
+; When 'spill_src_stride' is true, the 'src_strideq' resides in memory.
+; we cannot use above form to access memory, we have to update
+; 'srcq' at each line break. As we process two parts (first,second)
+; together in each macro function, the second part may also sit
+; in the next line, which means we also need to possibly add
+; one 'src_strideq' to 'srcq' before processing second part.
+
+%macro HANDLE_SECOND_OFFSET 0
+ %if spill_src_stride
+ %define second_offset 0
+ add srcq, src_strideq
+ %else
+ %define second_offset (src_strideq)
+ %endif
+%endmacro
+
+; This is specically designed to handle when src_strideq is a
+; memory position, under such case, we can not accomplish
+; complex address calculation using LEA, and fall back to
+; using simple ADD instruction at each line ending.
+%macro ADVANCE_END_OF_TWO_LINES 0
+ %if spill_src_stride
+ add srcq, src_strideq
+ %else
+ lea srcq, [srcq+src_strideq*2]
+ %endif
+
+; note: ref_stride is never spilled when processing two lines
+ lea ref1q, [ref1q+ref_strideq*2]
+ lea ref2q, [ref2q+ref_strideq*2]
+ lea ref3q, [ref3q+ref_strideq*2]
+ lea ref4q, [ref4q+ref_strideq*2]
+%endmacro
+
+; PROCESS_4x2x4 first
+%macro PROCESS_4x2x4 1
+ movd m0, [srcq]
+ HANDLE_SECOND_OFFSET
+%if %1 == 1
+ movd m6, [ref1q]
+ movd m4, [ref2q]
+ movd m7, [ref3q]
+ movd m5, [ref4q]
+
+ movd m1, [srcq + second_offset]
+ movd m2, [ref1q+ref_strideq]
+ punpckldq m0, m1
+ punpckldq m6, m2
+ movd m1, [ref2q+ref_strideq]
+ movd m2, [ref3q+ref_strideq]
+ movd m3, [ref4q+ref_strideq]
+ punpckldq m4, m1
+ punpckldq m7, m2
+ punpckldq m5, m3
+ movlhps m0, m0
+ movlhps m6, m4
+ movlhps m7, m5
+ psadbw m6, m0
+ psadbw m7, m0
+%else
+ movd m1, [ref1q]
+ movd m5, [ref1q+ref_strideq]
+ movd m2, [ref2q]
+ movd m4, [ref2q+ref_strideq]
+ punpckldq m1, m5
+ punpckldq m2, m4
+ movd m3, [ref3q]
+ movd m5, [ref3q+ref_strideq]
+ punpckldq m3, m5
+ movd m4, [ref4q]
+ movd m5, [ref4q+ref_strideq]
+ punpckldq m4, m5
+ movd m5, [srcq + second_offset]
+ punpckldq m0, m5
+ movlhps m0, m0
+ movlhps m1, m2
+ movlhps m3, m4
+ psadbw m1, m0
+ psadbw m3, m0
+ paddd m6, m1
+ paddd m7, m3
+%endif
+%endmacro
+
+; PROCESS_8x2x4 first
+%macro PROCESS_8x2x4 1
+ movh m0, [srcq]
+ HANDLE_SECOND_OFFSET
+%if %1 == 1
+ movh m4, [ref1q]
+ movh m5, [ref2q]
+ movh m6, [ref3q]
+ movh m7, [ref4q]
+ movhps m0, [srcq + second_offset]
+ movhps m4, [ref1q+ref_strideq]
+ movhps m5, [ref2q+ref_strideq]
+ movhps m6, [ref3q+ref_strideq]
+ movhps m7, [ref4q+ref_strideq]
+ psadbw m4, m0
+ psadbw m5, m0
+ psadbw m6, m0
+ psadbw m7, m0
+%else
+ movh m1, [ref1q]
+ movh m2, [ref2q]
+ movhps m0, [srcq + second_offset]
+ movhps m1, [ref1q+ref_strideq]
+ movhps m2, [ref2q+ref_strideq]
+ psadbw m1, m0
+ psadbw m2, m0
+ paddd m4, m1
+ paddd m5, m2
+
+ movh m1, [ref3q]
+ movhps m1, [ref3q+ref_strideq]
+ movh m2, [ref4q]
+ movhps m2, [ref4q+ref_strideq]
+ psadbw m1, m0
+ psadbw m2, m0
+ paddd m6, m1
+ paddd m7, m2
+%endif
+%endmacro
+
+; PROCESS_FIRST_MMSIZE
+%macro PROCESS_FIRST_MMSIZE 0
+ mova m0, [srcq]
+ movu m4, [ref1q]
+ movu m5, [ref2q]
+ movu m6, [ref3q]
+ movu m7, [ref4q]
+ psadbw m4, m0
+ psadbw m5, m0
+ psadbw m6, m0
+ psadbw m7, m0
+%endmacro
+
+; PROCESS_16x1x4 offset
+%macro PROCESS_16x1x4 1
+ mova m0, [srcq + %1]
+ movu m1, [ref1q + ref_offsetq + %1]
+ movu m2, [ref2q + ref_offsetq + %1]
+ psadbw m1, m0
+ psadbw m2, m0
+ paddd m4, m1
+ paddd m5, m2
+
+ movu m1, [ref3q + ref_offsetq + %1]
+ movu m2, [ref4q + ref_offsetq + %1]
+ psadbw m1, m0
+ psadbw m2, m0
+ paddd m6, m1
+ paddd m7, m2
+%endmacro
+
+; void aom_sadNxNx4d_sse2(uint8_t *src, int src_stride,
+; uint8_t *ref[4], int ref_stride,
+; uint32_t res[4]);
+; Macro Arguments:
+; 1: Width
+; 2: Height
+; 3: If 0, then normal sad, else skip rows
+%macro SADNXN4D 2-3 0
+
+%define spill_src_stride 0
+%define spill_ref_stride 0
+%define spill_cnt 0
+
+; Whether a shared offset should be used instead of adding strides to
+; each reference array. With this option, only one line will be processed
+; per loop iteration.
+%define use_ref_offset (%1 >= mmsize)
+
+; Remove loops in the 4x4 and 8x4 case
+%define use_loop (use_ref_offset || %2 > 4)
+
+%if %3 == 1 ; skip rows
+%if AOM_ARCH_X86_64
+%if use_ref_offset
+cglobal sad_skip_%1x%2x4d, 5, 10, 8, src, src_stride, ref1, ref_stride, res, \
+ ref2, ref3, ref4, cnt, ref_offset
+%elif use_loop
+cglobal sad_skip_%1x%2x4d, 5, 9, 8, src, src_stride, ref1, ref_stride, res, \
+ ref2, ref3, ref4, cnt
+%else
+cglobal sad_skip_%1x%2x4d, 5, 8, 8, src, src_stride, ref1, ref_stride, res, \
+ ref2, ref3, ref4
+%endif
+%else
+%if use_ref_offset
+cglobal sad_skip_%1x%2x4d, 4, 7, 8, src, ref_offset, ref1, cnt, ref2, ref3, \
+ ref4
+%define spill_src_stride 1
+%define spill_ref_stride 1
+%elif use_loop
+cglobal sad_skip_%1x%2x4d, 4, 7, 8, src, cnt, ref1, ref_stride, ref2, \
+ ref3, ref4
+%define spill_src_stride 1
+%else
+cglobal sad_skip_%1x%2x4d, 4, 7, 8, src, src_stride, ref1, ref_stride, ref2, \
+ ref3, ref4
+%endif
+%endif
+%else ; normal sad
+%if AOM_ARCH_X86_64
+%if use_ref_offset
+cglobal sad%1x%2x4d, 5, 10, 8, src, src_stride, ref1, ref_stride, res, ref2, \
+ ref3, ref4, cnt, ref_offset
+%elif use_loop
+cglobal sad%1x%2x4d, 5, 9, 8, src, src_stride, ref1, ref_stride, res, ref2, \
+ ref3, ref4, cnt
+%else
+cglobal sad%1x%2x4d, 5, 8, 8, src, src_stride, ref1, ref_stride, res, ref2, \
+ ref3, ref4
+%endif
+%else
+%if use_ref_offset
+cglobal sad%1x%2x4d, 4, 7, 8, src, ref_offset, ref1, cnt, ref2, ref3, ref4
+ %define spill_src_stride 1
+ %define spill_ref_stride 1
+%elif use_loop
+cglobal sad%1x%2x4d, 4, 7, 8, src, cnt, ref1, ref_stride, ref2, ref3, ref4
+ %define spill_src_stride 1
+%else
+cglobal sad%1x%2x4d, 4, 7, 8, src, src_stride, ref1, ref_stride, ref2, ref3, \
+ ref4
+%endif
+%endif
+%endif
+
+%if spill_src_stride
+ %define src_strideq r1mp
+ %define src_strided r1mp
+%endif
+%if spill_ref_stride
+ %define ref_strideq r3mp
+ %define ref_strided r3mp
+%endif
+
+%if spill_cnt
+ SUB rsp, 4
+ %define cntd word [rsp]
+%endif
+
+%if %3 == 1
+ sal src_strided, 1
+ sal ref_strided, 1
+%endif
+ movsxdifnidn src_strideq, src_strided
+ movsxdifnidn ref_strideq, ref_strided
+
+ mov ref2q, [ref1q+gprsize*1]
+ mov ref3q, [ref1q+gprsize*2]
+ mov ref4q, [ref1q+gprsize*3]
+ mov ref1q, [ref1q+gprsize*0]
+
+; Is the loop for this wxh in another function?
+; If so, we jump into that function for the loop and returning
+%define external_loop (use_ref_offset && %1 > mmsize && %1 != %2)
+
+%if use_ref_offset
+ PROCESS_FIRST_MMSIZE
+%if %1 > mmsize
+ mov ref_offsetq, 0
+ mov cntd, %2 >> %3
+; Jump part way into the loop for the square version of this width
+%if %3 == 1
+ jmp mangle(private_prefix %+ _sad_skip_%1x%1x4d %+ SUFFIX).midloop
+%else
+ jmp mangle(private_prefix %+ _sad%1x%1x4d %+ SUFFIX).midloop
+%endif
+%else
+ mov ref_offsetq, ref_strideq
+ add srcq, src_strideq
+ mov cntd, (%2 >> %3) - 1
+%endif
+%if external_loop == 0
+.loop:
+; Unrolled horizontal loop
+%assign h_offset 0
+%rep %1/mmsize
+ PROCESS_16x1x4 h_offset
+%if h_offset == 0
+; The first row of the first column is done outside the loop and jumps here
+.midloop:
+%endif
+%assign h_offset h_offset+mmsize
+%endrep
+
+ add srcq, src_strideq
+ add ref_offsetq, ref_strideq
+ sub cntd, 1
+ jnz .loop
+%endif
+%else
+ PROCESS_%1x2x4 1
+ ADVANCE_END_OF_TWO_LINES
+%if use_loop
+ mov cntd, (%2/2 >> %3) - 1
+.loop:
+%endif
+ PROCESS_%1x2x4 0
+%if use_loop
+ ADVANCE_END_OF_TWO_LINES
+ sub cntd, 1
+ jnz .loop
+%endif
+%endif
+
+%if spill_cnt
+; Undo stack allocation for cnt
+ ADD rsp, 4
+%endif
+
+%if external_loop == 0
+%if %3 == 0
+ %define resultq r4
+ %define resultmp r4mp
+%endif
+
+; Undo modifications on parameters on the stack
+%if %3 == 1
+%if spill_src_stride
+ shr src_strided, 1
+%endif
+%if spill_ref_stride
+ shr ref_strided, 1
+%endif
+%endif
+
+%if %1 > 4
+ pslldq m5, 4
+ pslldq m7, 4
+ por m4, m5
+ por m6, m7
+ mova m5, m4
+ mova m7, m6
+ punpcklqdq m4, m6
+ punpckhqdq m5, m7
+ paddd m4, m5
+%if %3 == 1
+ pslld m4, 1
+%endif
+ movifnidn resultq, resultmp
+ movu [resultq], m4
+ RET
+%else
+ pshufd m6, m6, 0x08
+ pshufd m7, m7, 0x08
+%if %3 == 1
+ pslld m6, 1
+ pslld m7, 1
+%endif
+ movifnidn resultq, resultmp
+ movq [resultq+0], m6
+ movq [resultq+8], m7
+ RET
+%endif
+%endif ; external_loop == 0
+%endmacro
+
+INIT_XMM sse2
+SADNXN4D 128, 128
+SADNXN4D 128, 64
+SADNXN4D 64, 128
+SADNXN4D 64, 64
+SADNXN4D 64, 32
+SADNXN4D 32, 64
+SADNXN4D 32, 32
+SADNXN4D 32, 16
+SADNXN4D 16, 32
+SADNXN4D 16, 16
+SADNXN4D 16, 8
+SADNXN4D 8, 16
+SADNXN4D 8, 8
+SADNXN4D 8, 4
+SADNXN4D 4, 8
+SADNXN4D 4, 4
+%if CONFIG_REALTIME_ONLY==0
+SADNXN4D 4, 16
+SADNXN4D 16, 4
+SADNXN4D 8, 32
+SADNXN4D 32, 8
+SADNXN4D 16, 64
+SADNXN4D 64, 16
+%endif
+SADNXN4D 128, 128, 1
+SADNXN4D 128, 64, 1
+SADNXN4D 64, 128, 1
+SADNXN4D 64, 64, 1
+SADNXN4D 64, 32, 1
+SADNXN4D 32, 64, 1
+SADNXN4D 32, 32, 1
+SADNXN4D 32, 16, 1
+SADNXN4D 16, 32, 1
+SADNXN4D 16, 16, 1
+SADNXN4D 16, 8, 1
+SADNXN4D 8, 16, 1
+SADNXN4D 8, 8, 1
+SADNXN4D 4, 8, 1
+%if CONFIG_REALTIME_ONLY==0
+SADNXN4D 4, 16, 1
+SADNXN4D 8, 32, 1
+SADNXN4D 32, 8, 1
+SADNXN4D 16, 64, 1
+SADNXN4D 64, 16, 1
+%endif
+
+; Different assembly is needed when the height gets subsampled to 2
+; SADNXN4D 16, 4, 1
+; SADNXN4D 8, 4, 1
+; SADNXN4D 4, 4, 1
generated by cgit v1.2.3 (git 2.39.1) at 2024-08-27 22:03:25 +0000