summaryrefslogtreecommitdiffstats
path: root/third_party/dav1d/src/x86/mc_avx512.asm
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--third_party/dav1d/src/x86/mc_avx512.asm2953
1 files changed, 1971 insertions, 982 deletions
diff --git a/third_party/dav1d/src/x86/mc_avx512.asm b/third_party/dav1d/src/x86/mc_avx512.asm
index f9043f1ad3..50e670ec25 100644
--- a/third_party/dav1d/src/x86/mc_avx512.asm
+++ b/third_party/dav1d/src/x86/mc_avx512.asm
@@ -89,55 +89,47 @@ wm_444_mask: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 3
db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63
db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62
-bilin_h_perm16: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7
- db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15
- db 33, 32, 34, 33, 35, 34, 36, 35, 37, 36, 38, 37, 39, 38, 40, 39
- db 41, 40, 42, 41, 43, 42, 44, 43, 45, 44, 46, 45, 47, 46, 48, 47
-bilin_h_perm32: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7
- db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15
- db 17, 16, 18, 17, 19, 18, 20, 19, 21, 20, 22, 21, 23, 22, 24, 23
- db 25, 24, 26, 25, 27, 26, 28, 27, 29, 28, 30, 29, 31, 30, 32, 31
-bilin_v_perm8: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7
- db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23
- db 32, 80, 33, 81, 34, 82, 35, 83, 36, 84, 37, 85, 38, 86, 39, 87
- db 64, 32, 65, 33, 66, 34, 67, 35, 68, 36, 69, 37, 70, 38, 71, 39
-bilin_v_perm16: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7
- db 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15
- db 64, 16, 65, 17, 66, 18, 67, 19, 68, 20, 69, 21, 70, 22, 71, 23
- db 72, 24, 73, 25, 74, 26, 75, 27, 76, 28, 77, 29, 78, 30, 79, 31
-bilin_v_perm32: db 64, 0, 65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7
- db 72, 8, 73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15
- db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23
- db 88, 24, 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31
-bilin_v_perm64: dq 0, 4, 1, 5, 2, 6, 3, 7
-spel_h_perm16a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
+bilin_h_perm16: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
+ db 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16
+ db 32, 33, 33, 34, 34, 35, 35, 36, 36, 37, 37, 38, 38, 39, 39, 40
+ db 40, 41, 41, 42, 42, 43, 43, 44, 44, 45, 45, 46, 46, 47, 47, 48
+bilin_h_perm32: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
+ db 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16
+ db 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24
+ db 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32
+bilin_v_perm8: db 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23
+ db 16, 80, 17, 81, 18, 82, 19, 83, 20, 84, 21, 85, 22, 86, 23, 87
+ db 80, 32, 81, 33, 82, 34, 83, 35, 84, 36, 85, 37, 86, 38, 87, 39
+ db 32, 64, 33, 65, 34, 66, 35, 67, 36, 68, 37, 69, 38, 70, 39, 71
+bilin_v_perm16: db 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23
+ db 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31
+ db 16, 64, 17, 65, 18, 66, 19, 67, 20, 68, 21, 69, 22, 70, 23, 71
+ db 24, 72, 25, 73, 26, 74, 27, 75, 28, 76, 29, 77, 30, 78, 31, 79
+bilin_v_perm32: db 0, 64, 1, 65, 2, 66, 3, 67, 4, 68, 5, 69, 6, 70, 7, 71
+ db 8, 72, 9, 73, 10, 74, 11, 75, 12, 76, 13, 77, 14, 78, 15, 79
+ db 16, 80, 17, 81, 18, 82, 19, 83, 20, 84, 21, 85, 22, 86, 23, 87
+ db 24, 88, 25, 89, 26, 90, 27, 91, 28, 92, 29, 93, 30, 94, 31, 95
+bilin_v_perm64: dd 0, 0, 4, 8, 1, 1, 5, 9, 2, 2, 6, 10, 3, 3, 7, 11
+spel_h_perm16: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38
db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46
-spel_h_perm16b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10
- db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18
- db 36, 37, 38, 39, 37, 38, 39, 40, 38, 39, 40, 41, 39, 40, 41, 42
- db 44, 45, 46, 47, 45, 46, 47, 48, 46, 47, 48, 49, 47, 48, 49, 50
-spel_h_perm16c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
- db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
- db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46
- db 48, 49, 50, 51, 49, 50, 51, 52, 50, 51, 52, 53, 51, 52, 53, 54
-spel_h_perm32a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
+spel_h_perm32: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30
-spel_h_perm32b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10
- db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18
- db 20, 21, 22, 23, 21, 22, 23, 24, 22, 23, 24, 25, 23, 24, 25, 26
- db 28, 29, 30, 31, 29, 30, 31, 32, 30, 31, 32, 33, 31, 32, 33, 34
-spel_h_perm32c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
- db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
- db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30
- db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38
-spel_v_perm16: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7
+spel_v_perm8: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
+ db 8, 16, 9, 17, 10, 18, 11, 19, 12, 20, 13, 21, 14, 22, 15, 23
+ db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31
+ db 24, 32, 25, 33, 26, 34, 27, 35, 28, 36, 29, 37, 30, 38, 31, 39
+spel_v_perm16a: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7
db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
db 40, 16, 41, 17, 42, 18, 43, 19, 44, 20, 45, 21, 46, 22, 47, 23
db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31
+spel_v_perm16b: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7
+ db 40, 16, 41, 17, 42, 18, 43, 19, 44, 20, 45, 21, 46, 22, 47, 23
+ db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
+ db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31
spel_v_perm32: db 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39
db 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47
db 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55
@@ -154,34 +146,20 @@ spel_hv_perm8a: db 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 2
db 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31
db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39
db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47
-spel_hv_perm8b: db 32, 33, 48, 49, 34, 35, 50, 51, 36, 37, 52, 53, 38, 39, 54, 55
- db 40, 41, 56, 57, 42, 43, 58, 59, 44, 45, 60, 61, 46, 47, 62, 63
- db 48, 49, 64, 65, 50, 51, 66, 67, 52, 53, 68, 69, 54, 55, 70, 71
- db 56, 57, 72, 73, 58, 59, 74, 75, 60, 61, 76, 77, 62, 63, 78, 79
-spel_hv_perm8c: db 34, 35, 0, 1, 38, 39, 4, 5, 42, 43, 8, 9, 46, 47, 12, 13
+spel_hv_perm8b: db 34, 35, 0, 1, 38, 39, 4, 5, 42, 43, 8, 9, 46, 47, 12, 13
db 50, 51, 16, 17, 54, 55, 20, 21, 58, 59, 24, 25, 62, 63, 28, 29
db 0, 1, 32, 33, 4, 5, 36, 37, 8, 9, 40, 41, 12, 13, 44, 45
db 16, 17, 48, 49, 20, 21, 52, 53, 24, 25, 56, 57, 28, 29, 60, 61
-spel_hv_end16: db 1, 3, 17, 19, 5, 7, 21, 23, 33, 35, 49, 51, 37, 39, 53, 55
- db 9, 11, 25, 27, 13, 15, 29, 31, 41, 43, 57, 59, 45, 47, 61, 63
spel_hv_perm16a:db 0, 1, 2, 3, 32, 33, 34, 35, 1, 2, 3, 4, 33, 34, 35, 36
db 2, 3, 4, 5, 34, 35, 36, 37, 3, 4, 5, 6, 35, 36, 37, 38
-spel_hv_perm16c:db 8, 9, 10, 11, 40, 41, 42, 43, 9, 10, 11, 12, 41, 42, 43, 44
+ db 8, 9, 10, 11, 40, 41, 42, 43, 9, 10, 11, 12, 41, 42, 43, 44
db 10, 11, 12, 13, 42, 43, 44, 45, 11, 12, 13, 14, 43, 44, 45, 46
- db 16, 17, 18, 19, 48, 49, 50, 51, 17, 18, 19, 20, 49, 50, 51, 52
- db 18, 19, 20, 21, 50, 51, 52, 53, 19, 20, 21, 22, 51, 52, 53, 54
-spel_hv_perm16b:db 4, 5, 6, 7, 36, 37, 38, 39, 5, 6, 7, 8, 37, 38, 39, 40
- db 6, 7, 8, 9, 38, 39, 40, 41, 7, 8, 9, 10, 39, 40, 41, 42
- db 12, 13, 14, 15, 44, 45, 46, 47, 13, 14, 15, 16, 45, 46, 47, 48
- db 14, 15, 16, 17, 46, 47, 48, 49, 15, 16, 17, 18, 47, 48, 49, 50
-spel_hv_perm16d:db 0, 1, 2, 3, 1, 2, 3, 4, 4, 5, 6, 7, 5, 6, 7, 8
+spel_hv_perm16b:db 0, 1, 2, 3, 1, 2, 3, 4, 4, 5, 6, 7, 5, 6, 7, 8
db 2, 3, 4, 5, 3, 4, 5, 6, 6, 7, 8, 9, 7, 8, 9, 10
db 8, 9, 10, 11, 9, 10, 11, 12, 12, 13, 14, 15, 13, 14, 15, 16
db 10, 11, 12, 13, 11, 12, 13, 14, 14, 15, 16, 17, 15, 16, 17, 18
-spel_hv_perm16e:db 4, 5, 6, 7, 5, 6, 7, 8, 8, 9, 10, 11, 9, 10, 11, 12
- db 6, 7, 8, 9, 7, 8, 9, 10, 10, 11, 12, 13, 11, 12, 13, 14
- db 12, 13, 14, 15, 13, 14, 15, 16, 16, 17, 18, 19, 17, 18, 19, 20
- db 14, 15, 16, 17, 15, 16, 17, 18, 18, 19, 20, 21, 19, 20, 21, 22
+spel_hv_end16: db 1, 3, 17, 19, 5, 7, 21, 23, 33, 35, 49, 51, 37, 39, 53, 55
+ db 9, 11, 25, 27, 13, 15, 29, 31, 41, 43, 57, 59, 45, 47, 61, 63
spel_hv_end: db 1, 3, 5, 7, 17, 19, 21, 23, 33, 35, 37, 39, 49, 51, 53, 55
deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11
subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12
@@ -189,15 +167,14 @@ subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 1
subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10
subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
-bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11
-bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7
-bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7
+bilin_h_shuf4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12
+bilin_v_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11
blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3
rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
-resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7
resize_permA: dd 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
resize_permB: dd 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
resize_permC: dd 0, 4, 8, 12
+resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7
pb_02461357: db 0, 2, 4, 6, 1, 3, 5, 7
wm_420_perm64: dq 0xfedcba9876543210
@@ -205,6 +182,8 @@ wm_sign: dd 0x40804080, 0xc0c0c0c0, 0x40404040
pb_8x0_8x8: times 8 db 0
times 8 db 8
+pb_4: times 4 db 4
+pb_32: times 4 db 32
pb_127: times 4 db 127
pw_m128 times 2 dw -128
pw_m256: times 2 dw -256
@@ -216,7 +195,6 @@ pd_32: dd 32
pd_34: dd 34
pd_63: dd 63
pd_512: dd 512
-pd_32768: dd 32768
%define pb_m64 (wm_sign+4)
%define pb_64 (wm_sign+8)
@@ -289,8 +267,10 @@ BASE_JMP_TABLE put, avx512icl, 2, 4, 8, 16, 32, 64, 128
BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE put, bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128
+HV_JMP_TABLE put, 6tap, avx512icl, 2, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE put, 8tap, avx512icl, 3, 2, 4, 8, 16, 32, 64, 128
-HV_JMP_TABLE prep, 8tap, avx512icl, 7, 4, 8, 16, 32, 64, 128
+HV_JMP_TABLE prep, 6tap, avx512icl, 2, 4, 8, 16, 32, 64, 128
+HV_JMP_TABLE prep, 8tap, avx512icl, 3, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128
@@ -401,9 +381,9 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
.h:
; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4
; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4
- imul mxyd, 0xff01
- vbroadcasti128 m4, [bilin_h_shuf8]
- add mxyd, 16 << 8
+ imul mxyd, 255
+ vbroadcasti128 m4, [bilin_h_perm16]
+ add mxyd, 16
vpbroadcastw m5, mxyd
mov mxyd, r7m ; my
test mxyd, mxyd
@@ -526,9 +506,9 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
RET
.v:
movzx wd, word [r7+wq*2+table_offset(put, _bilin_v)]
- imul mxyd, 0xff01
+ imul mxyd, 255
vpbroadcastd m5, [pw_2048]
- add mxyd, 16 << 8
+ add mxyd, 16
add wq, r7
vpbroadcastw m4, mxyd
jmp wq
@@ -539,7 +519,7 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
lea srcq, [srcq+ssq*2]
pinsrw xmm0, xmm1, [srcq+ssq*0], 0 ; 2 1
pshuflw xmm1, xmm1, q2301 ; 1 0
- punpcklbw xmm1, xmm0, xmm1
+ punpcklbw xmm1, xmm0
pmaddubsw xmm1, xm4
pmulhrsw xmm1, xm5
packuswb xmm1, xmm1
@@ -552,11 +532,11 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
.v_w4:
movd xmm0, [srcq+ssq*0]
.v_w4_loop:
- vpbroadcastd xmm1, [srcq+ssq*1]
+ vpbroadcastd xmm2, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
- vpblendd xmm2, xmm1, xmm0, 0x01 ; 0 1
+ vpblendd xmm1, xmm2, xmm0, 0x01 ; 0 1
vpbroadcastd xmm0, [srcq+ssq*0]
- vpblendd xmm1, xmm0, 0x02 ; 1 2
+ vpblendd xmm2, xmm0, 0x02 ; 1 2
punpcklbw xmm1, xmm2
pmaddubsw xmm1, xm4
pmulhrsw xmm1, xm5
@@ -570,11 +550,11 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
.v_w8:
movq xmm0, [srcq+ssq*0]
.v_w8_loop:
- movq xmm3, [srcq+ssq*1]
+ movq xmm2, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
- punpcklbw xmm1, xmm3, xmm0
+ punpcklbw xmm1, xmm0, xmm2
movq xmm0, [srcq+ssq*0]
- punpcklbw xmm2, xmm0, xmm3
+ punpcklbw xmm2, xmm0
pmaddubsw xmm1, xm4
pmaddubsw xmm2, xm4
pmulhrsw xmm1, xm5
@@ -589,11 +569,11 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
.v_w16:
movu xmm0, [srcq+ssq*0]
.v_w16_loop:
- vbroadcasti128 ymm2, [srcq+ssq*1]
+ vbroadcasti128 ymm3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
- vpblendd ymm3, ymm2, ymm0, 0x0f ; 0 1
+ vpblendd ymm2, ymm3, ymm0, 0x0f ; 0 1
vbroadcasti128 ymm0, [srcq+ssq*0]
- vpblendd ymm2, ymm2, ymm0, 0xf0 ; 1 2
+ vpblendd ymm3, ymm0, 0xf0 ; 1 2
punpcklbw ymm1, ymm2, ymm3
punpckhbw ymm2, ymm3
pmaddubsw ymm1, ym4
@@ -612,11 +592,11 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
movu ym0, [srcq+ssq*0]
kxnorb k1, k1, k1
.v_w32_loop:
- vbroadcasti32x8 m2, [srcq+ssq*1]
+ vbroadcasti32x8 m3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
- vpblendmd m3{k1}, m2, m0 ; 0 1
+ vpblendmd m2{k1}, m3, m0 ; 0 1
vbroadcasti32x8 m0, [srcq+ssq*0]
- vpblendmd m2{k1}, m0, m2 ; 1 2
+ vpblendmd m3{k1}, m0, m3 ; 1 2
punpcklbw m1, m2, m3
punpckhbw m2, m3
pmaddubsw m1, m4
@@ -635,18 +615,18 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
.v_w64_loop:
movu m3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
- punpcklbw m1, m3, m0
- punpckhbw m6, m3, m0
+ punpcklbw m1, m0, m3
+ punpckhbw m6, m0, m3
movu m0, [srcq+ssq*0]
pmaddubsw m1, m4
pmaddubsw m6, m4
- punpcklbw m2, m0, m3
- punpckhbw m7, m0, m3
+ punpcklbw m2, m3, m0
+ punpckhbw m3, m0
pmaddubsw m2, m4
- pmaddubsw m7, m4
- REPX {pmulhrsw x, m5}, m1, m6, m2, m7
+ pmaddubsw m3, m4
+ REPX {pmulhrsw x, m5}, m1, m6, m2, m3
packuswb m1, m6
- packuswb m2, m7
+ packuswb m2, m3
mova [dstq+dsq*0], m1
mova [dstq+dsq*1], m2
lea dstq, [dstq+dsq*2]
@@ -660,13 +640,13 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
add srcq, ssq
movu m2, [srcq+64*0]
movu m3, [srcq+64*1]
- punpcklbw m6, m2, m0
+ punpcklbw m6, m0, m2
pmaddubsw m6, m4
- punpckhbw m0, m2, m0
+ punpckhbw m0, m2
pmaddubsw m0, m4
- punpcklbw m7, m3, m1
+ punpcklbw m7, m1, m3
pmaddubsw m7, m4
- punpckhbw m1, m3, m1
+ punpckhbw m1, m3
pmaddubsw m1, m4
REPX {pmulhrsw x, m5}, m6, m0, m7, m1
packuswb m6, m0
@@ -1005,8 +985,8 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
.h:
; 16 * src[x] + (mx * (src[x + 1] - src[x]))
; = (16 - mx) * src[x] + mx * src[x + 1]
- imul mxyd, 0xff01
- add mxyd, 16 << 8
+ imul mxyd, 255
+ add mxyd, 16
vpbroadcastw m5, mxyd
mov mxyd, r6m ; my
test mxyd, mxyd
@@ -1032,7 +1012,7 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
jg .h_w4_loop
RET
.h_w8:
- vbroadcasti32x4 m4, [bilin_h_shuf8]
+ vbroadcasti32x4 m4, [bilin_h_perm16]
.h_w8_loop:
movu xmm0, [srcq+strideq*0]
vinserti32x4 ym0, ymm0, [srcq+strideq*1], 1
@@ -1127,8 +1107,8 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
.v:
WIN64_SPILL_XMM 7
movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)]
- imul mxyd, 0xff01
- add mxyd, 16 << 8
+ imul mxyd, 255
+ add mxyd, 16
add wq, t2
lea stride3q, [strideq*3]
vpbroadcastw m6, mxyd
@@ -1218,11 +1198,11 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
.v_w64_loop:
vpermq m1, m5, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
- punpcklbw m4, m1, m0
- punpckhbw m2, m1, m0
+ punpcklbw m4, m0, m1
+ punpckhbw m2, m0, m1
vpermq m0, m5, [srcq+strideq*0]
- punpcklbw m3, m0, m1
- punpckhbw m1, m0, m1
+ punpcklbw m3, m1, m0
+ punpckhbw m1, m0
pmaddubsw m4, m6
pmaddubsw m2, m6
pmaddubsw m3, m6
@@ -1243,28 +1223,28 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
vpermq m2, m5, [srcq+strideq*1+ 0]
vpermq m3, m5, [srcq+strideq*1+64]
lea srcq, [srcq+strideq*2]
- punpcklbw m4, m2, m0
- punpckhbw m0, m2, m0
+ punpcklbw m4, m0, m2
+ punpckhbw m0, m2
pmaddubsw m4, m6
pmaddubsw m0, m6
mova [tmpq+64*0], m4
mova [tmpq+64*1], m0
- punpcklbw m4, m3, m1
- punpckhbw m1, m3, m1
+ punpcklbw m4, m1, m3
+ punpckhbw m1, m3
pmaddubsw m4, m6
pmaddubsw m1, m6
mova [tmpq+64*2], m4
mova [tmpq+64*3], m1
vpermq m0, m5, [srcq+strideq*0+ 0]
vpermq m1, m5, [srcq+strideq*0+64]
- punpcklbw m4, m0, m2
- punpckhbw m2, m0, m2
+ punpcklbw m4, m2, m0
+ punpckhbw m2, m0
pmaddubsw m4, m6
pmaddubsw m2, m6
mova [tmpq+64*4], m4
mova [tmpq+64*5], m2
- punpcklbw m4, m1, m3
- punpckhbw m3, m1, m3
+ punpcklbw m4, m3, m1
+ punpckhbw m3, m1
pmaddubsw m4, m6
pmaddubsw m3, m6
mova [tmpq+64*6], m4
@@ -1308,7 +1288,7 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
jg .hv_w4_loop
RET
.hv_w8:
- vbroadcasti32x4 m4, [bilin_h_shuf8]
+ vbroadcasti32x4 m4, [bilin_h_perm16]
vbroadcasti32x4 m0, [srcq+strideq*0]
pshufb m0, m4
pmaddubsw m0, m5
@@ -1448,7 +1428,7 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
%assign FILTER_SMOOTH (1*15 << 16) | 4*15
%assign FILTER_SHARP (2*15 << 16) | 3*15
-%macro FN 4 ; fn, type, type_h, type_v
+%macro FN 4-5 ; fn, type, type_h, type_v, jmp_to
cglobal %1_%2_8bpc
mov t0d, FILTER_%3
%ifidn %3, %4
@@ -1456,8 +1436,8 @@ cglobal %1_%2_8bpc
%else
mov t1d, FILTER_%4
%endif
-%ifnidn %2, regular ; skip the jump in the last filter
- jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX)
+%if %0 == 5 ; skip the jump in the last filter
+ jmp mangle(private_prefix %+ _%5 %+ SUFFIX)
%endif
%endmacro
@@ -1489,24 +1469,22 @@ DECLARE_REG_TMP 4, 5
DECLARE_REG_TMP 7, 8
%endif
+; Due to the use of vpdpbusd (which does 4 pixels per instruction) in
+; the horizontal filter, 6-tap is only used for the vertical filter.
%define PUT_8TAP_FN FN put_8tap,
-
-PUT_8TAP_FN sharp, SHARP, SHARP
-PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH
-PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP
-PUT_8TAP_FN smooth, SMOOTH, SMOOTH
-PUT_8TAP_FN sharp_regular, SHARP, REGULAR
-PUT_8TAP_FN regular_sharp, REGULAR, SHARP
-PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR
-PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH
+PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH, put_6tap_8bpc
+PUT_8TAP_FN sharp_regular, SHARP, REGULAR, put_6tap_8bpc
+PUT_8TAP_FN smooth, SMOOTH, SMOOTH, put_6tap_8bpc
+PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR, put_6tap_8bpc
+PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH, put_6tap_8bpc
PUT_8TAP_FN regular, REGULAR, REGULAR
-cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
+cglobal put_6tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ns
%define base r8-put_avx512icl
imul mxd, mxm, 0x010101
- add mxd, t0d ; 8tap_h, mx, 4tap_h
+ add mxd, t0d ; 6tap_h, mx, 4tap_h
imul myd, mym, 0x010101
- add myd, t1d ; 8tap_v, my, 4tap_v
+ add myd, t1d ; 6tap_v, my, 4tap_v
lea r8, [put_avx512icl]
movsxd wq, wm
movifnidn hd, hm
@@ -1514,6 +1492,7 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
jnz .h
test myd, 0xf00
jnz .v
+.put:
tzcnt wd, wd
movzx wd, word [r8+wq*2+table_offset(put,)]
add wq, r8
@@ -1523,158 +1502,577 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
pop r8
%endif
jmp wq
-.h:
- test myd, 0xf00
- jnz .hv
- vpbroadcastd m5, [pd_34] ; 2 + (8 << 2)
- WIN64_SPILL_XMM 11
- cmp wd, 4
- jl .h_w2
- vbroadcasti128 m6, [subpel_h_shufA]
- je .h_w4
- tzcnt wd, wd
- vbroadcasti128 m7, [subpel_h_shufB]
- vbroadcasti128 m8, [subpel_h_shufC]
- shr mxd, 16
- sub srcq, 3
- movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)]
- vpbroadcastd m9, [base+mxq*8+subpel_filters+0]
- vpbroadcastd m10, [base+mxq*8+subpel_filters+4]
- add wq, r8
- jmp wq
-.h_w2:
- movzx mxd, mxb
- dec srcq
- mova xmm4, [subpel_h_shuf4]
- vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2]
-.h_w2_loop:
- movq xmm0, [srcq+ssq*0]
- movhps xmm0, [srcq+ssq*1]
+.v:
+ movzx mxd, myb
+ shr myd, 16
+ cmp hd, 6
+ cmovs myd, mxd
+ tzcnt r6d, wd
+ movzx r6d, word [r8+r6*2+table_offset(put, _6tap_v)]
+ vpbroadcastd m6, [pw_512]
+ lea myq, [base+subpel_filters+1+myq*8]
+ vpbroadcastw m7, [myq+0]
+ add r6, r8
+ vpbroadcastw m8, [myq+2]
+ mov nsq, ssq
+ vpbroadcastw m9, [myq+4]
+ neg nsq
+ jmp r6
+.v_w2:
+ movd xmm2, [srcq+nsq*2]
+ pinsrw xmm2, [srcq+nsq*1], 2
+ pinsrw xmm2, [srcq+ssq*0], 4
+ pinsrw xmm2, [srcq+ssq*1], 6 ; 0 1 2 3
lea srcq, [srcq+ssq*2]
- pshufb xmm0, xmm4
- mova xmm1, xm5
- vpdpbusd xmm1, xmm0, xmm3
- packssdw xmm0, xmm1, xmm1
- psraw xmm0, 6
- packuswb xmm0, xm0
- pextrw [dstq+dsq*0], xmm0, 0
- pextrw [dstq+dsq*1], xmm0, 1
+ vpbroadcastd xmm0, [srcq+ssq*0]
+ palignr xmm3, xmm0, xmm2, 4 ; 1 2 3 4
+ punpcklbw xmm1, xmm2, xmm3 ; 01 12
+ punpckhbw xmm2, xmm3 ; 23 34
+.v_w2_loop:
+ vpbroadcastd xmm4, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ pmaddubsw xmm3, xmm1, xm7 ; a0 b0
+ mova xmm1, xmm2
+ pmaddubsw xmm2, xm8 ; a1 b1
+ paddw xmm3, xmm2
+ vpblendd xmm2, xmm0, xmm4, 0x02 ; 4 5
+ vpbroadcastd xmm0, [srcq+ssq*0]
+ vpblendd xmm4, xmm0, 0x02 ; 5 6
+ punpcklbw xmm2, xmm4 ; 67 78
+ pmaddubsw xmm4, xmm2, xm9 ; a3 b3
+ paddw xmm3, xmm4
+ pmulhrsw xmm3, xm6
+ packuswb xmm3, xmm3
+ pextrw [dstq+dsq*0], xmm3, 0
+ pextrw [dstq+dsq*1], xmm3, 2
lea dstq, [dstq+dsq*2]
sub hd, 2
- jg .h_w2_loop
+ jg .v_w2_loop
RET
-.h_w4:
- movzx mxd, mxb
- dec srcq
- vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2]
-.h_w4_loop:
- movq xmm0, [srcq+ssq*0]
- movq xmm1, [srcq+ssq*1]
+.v_w4:
+ movd xmm2, [srcq+nsq*2]
+ pinsrd xmm2, [srcq+nsq*1], 1
+ pinsrd xmm2, [srcq+ssq*0], 2
+ pinsrd xmm2, [srcq+ssq*1], 3 ; 0 1 2 3
lea srcq, [srcq+ssq*2]
- pshufb xmm0, xm6
- pshufb xmm1, xm6
- mova xmm2, xm5
- vpdpbusd xmm2, xmm0, xmm3
- mova xmm0, xm5
- vpdpbusd xmm0, xmm1, xmm3
- packssdw xmm0, xmm2, xmm0
- psraw xmm0, 6
- packuswb xmm0, xmm0
- movd [dstq+dsq*0], xmm0
- pextrd [dstq+dsq*1], xmm0, 1
+ vpbroadcastd xmm0, [srcq+ssq*0]
+ palignr xmm3, xmm0, xmm2, 4 ; 1 2 3 4
+ punpcklbw xmm1, xmm2, xmm3 ; 01 12
+ punpckhbw xmm2, xmm3 ; 23 34
+.v_w4_loop:
+ vpbroadcastd xmm4, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ pmaddubsw xmm3, xmm1, xm7 ; a0 b0
+ mova xmm1, xmm2
+ pmaddubsw xmm2, xm8 ; a1 b1
+ paddw xmm3, xmm2
+ vpblendd xmm2, xmm0, xmm4, 0x02 ; 4 5
+ vpbroadcastd xmm0, [srcq+ssq*0]
+ vpblendd xmm4, xmm0, 0x02 ; 5 6
+ punpcklbw xmm2, xmm4 ; 45 56
+ pmaddubsw xmm4, xmm2, xm9 ; a2 b2
+ paddw xmm3, xmm4
+ pmulhrsw xmm3, xm6
+ packuswb xmm3, xmm3
+ movd [dstq+dsq*0], xmm3
+ pextrd [dstq+dsq*1], xmm3, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
- jg .h_w4_loop
+ jg .v_w4_loop
RET
-.h_w8:
- movu xm0, [srcq+ssq*0]
- vinserti32x4 ym0, [srcq+ssq*1], 1
+.v_w8:
+ movq xmm1, [srcq+nsq*2]
+ vpbroadcastq ymm3, [srcq+nsq*1]
+ vpbroadcastq ymm2, [srcq+ssq*0]
+ vpbroadcastq ymm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
- WRAP_YMM PUT_8TAP_H 0, 1, 2, 3
- vpmovuswb xm0, ym0
- movq [dstq+dsq*0], xm0
- movhps [dstq+dsq*1], xm0
+ vpbroadcastq ymm0, [srcq+ssq*0]
+ vpblendd ymm1, ymm3, 0x30
+ vpblendd ymm3, ymm2, 0x30
+ punpcklbw ymm1, ymm3 ; 01 12
+ vpblendd ymm2, ymm4, 0x30
+ vpblendd ymm4, ymm0, 0x30
+ punpcklbw ymm2, ymm4 ; 23 34
+.v_w8_loop:
+ vpbroadcastq ymm4, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ pmaddubsw ymm3, ymm1, ym7 ; a0 b0
+ mova ymm1, ymm2
+ pmaddubsw ymm2, ym8 ; a1 b1
+ paddw ymm3, ymm2
+ vpblendd ymm2, ymm0, ymm4, 0x30
+ vpbroadcastq ymm0, [srcq+ssq*0]
+ vpblendd ymm4, ymm0, 0x30
+ punpcklbw ymm2, ymm4 ; 45 56
+ pmaddubsw ymm4, ymm2, ym9 ; a2 b2
+ paddw ymm3, ymm4
+ pmulhrsw ymm3, ym6
+ vextracti128 xmm4, ymm3, 1
+ packuswb xmm3, xmm4
+ movq [dstq+dsq*0], xmm3
+ movhps [dstq+dsq*1], xmm3
lea dstq, [dstq+dsq*2]
sub hd, 2
- jg .h_w8
+ jg .v_w8_loop
+ vzeroupper
RET
-.h_w16:
- mova m6, [spel_h_perm16a]
- mova m7, [spel_h_perm16b]
- mova m8, [spel_h_perm16c]
-.h_w16_loop:
- movu ym0, [srcq+ssq*0]
+.v_w16:
+ mova m5, [spel_v_perm16a]
+ vbroadcasti32x4 m1, [srcq+nsq*2]
+ vbroadcasti32x4 ym3, [srcq+nsq*1]
+ mov r6d, 0x0f
+ vbroadcasti32x4 m2, [srcq+ssq*0]
+ kmovb k1, r6d
+ vbroadcasti32x4 ym4, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ vbroadcasti32x4 m0, [srcq+ssq*0]
+ vshufpd m1{k1}, m3, m2, 0xcc
+ vshufpd m2{k1}, m4, m0, 0xcc
+ vpermb m1, m5, m1 ; 01 12
+ vpermb m2, m5, m2 ; 23 34
+.v_w16_loop:
+ vbroadcasti32x4 ym4, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ pmaddubsw m3, m1, m7 ; a0 b0
+ mova m1, m2
+ pmaddubsw m2, m8 ; a1 b1
+ paddw m3, m2
+ mova m2, m0
+ vbroadcasti32x4 m0, [srcq+ssq*0]
+ vshufpd m2{k1}, m4, m0, 0xcc
+ vpermb m2, m5, m2 ; 45 56
+ pmaddubsw m4, m2, m9 ; a2 b2
+ paddw m3, m4
+ pmulhrsw m3, m6
+ vextracti32x8 ym4, m3, 1
+ packuswb ym3, ym4
+ mova [dstq+dsq*0], xm3
+ vextracti32x4 [dstq+dsq*1], ym3, 1
+ lea dstq, [dstq+dsq*2]
+ sub hd, 2
+ jg .v_w16_loop
+ RET
+.v_w32:
+ mova m10, [spel_v_perm32]
+ pmovzxbq m5, [pb_02461357]
+ vpshrdw m11, m10, m10, 8
+ movu ym0, [srcq+nsq*2]
+ vinserti32x8 m0, [srcq+nsq*1], 1
+ vpermb m1, m10, m0 ; 01
+ vinserti32x8 m0, [srcq+ssq*0], 0
+ vpermb m2, m11, m0 ; 12
vinserti32x8 m0, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
- PUT_8TAP_H 0, 1, 2, 3, 1
- vpmovuswb ym0, m0
- mova [dstq+dsq*0], xm0
- vextracti128 [dstq+dsq*1], ym0, 1
+ vpermb m3, m10, m0 ; 23
+ vinserti32x8 m0, [srcq+ssq*0], 0
+ vpermb m4, m11, m0 ; 34
+.v_w32_loop:
+ vinserti32x8 m0, [srcq+ssq*1], 1
+ lea srcq, [srcq+ssq*2]
+ pmaddubsw m12, m1, m7
+ mova m1, m3
+ pmaddubsw m13, m2, m7
+ mova m2, m4
+ pmaddubsw m14, m3, m8
+ vpermb m3, m10, m0 ; 45
+ vinserti32x8 m0, [srcq+ssq*0], 0
+ pmaddubsw m15, m4, m8
+ vpermb m4, m11, m0 ; 56
+ paddw m12, m14
+ pmaddubsw m14, m3, m9
+ paddw m13, m15
+ pmaddubsw m15, m4, m9
+ paddw m12, m14
+ paddw m13, m15
+ pmulhrsw m12, m6
+ pmulhrsw m13, m6
+ packuswb m12, m13
+ vpermq m12, m5, m12
+ mova [dstq+dsq*0], ym12
+ vextracti32x8 [dstq+dsq*1], m12, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
- jg .h_w16_loop
+ jg .v_w32_loop
RET
-.h_w32:
- movu ym0, [srcq+ssq*0+8*0]
- vinserti32x8 m0, [srcq+ssq*1+8*0], 1
- movu ym1, [srcq+ssq*0+8*1]
- vinserti32x8 m1, [srcq+ssq*1+8*1], 1
+.v_w64:
+.v_w128:
+ lea r6d, [hq+wq*4-256]
+.v_loop0:
+ movu m2, [srcq+nsq*2]
+ movu m4, [srcq+nsq*1]
+ lea r4, [srcq+ssq*2]
+ movu m11, [srcq+ssq*0]
+ movu m13, [srcq+ssq*1]
+ mov r7, dstq
+ movu m0, [r4 +ssq*0]
+ punpcklbw m1, m2, m4 ; 01l
+ punpckhbw m2, m4 ; 01h
+ punpcklbw m3, m4, m11 ; 12l
+ punpckhbw m4, m11 ; 12h
+ punpcklbw m10, m11, m13 ; 23l
+ punpckhbw m11, m13 ; 23h
+ punpcklbw m12, m13, m0 ; 34l
+ punpckhbw m13, m0 ; 34h
+.v_loop:
+ movu m5, [r4+ssq*1]
+ pmaddubsw m14, m1, m7 ; a0l
+ mova m1, m10
+ pmaddubsw m10, m8 ; a1l
+ lea r4, [r4+ssq*2]
+ pmaddubsw m15, m2, m7 ; a0h
+ mova m2, m11
+ pmaddubsw m11, m8 ; a1h
+ paddw m14, m10
+ punpcklbw m10, m0, m5 ; 45l
+ paddw m15, m11
+ punpckhbw m11, m0, m5 ; 45h
+ pmaddubsw m0, m10, m9 ; a2l
+ paddw m14, m0
+ pmaddubsw m0, m11, m9 ; a2h
+ paddw m15, m0
+ movu m0, [r4+ssq*0]
+ pmulhrsw m14, m6
+ pmulhrsw m15, m6
+ packuswb m14, m15
+ pmaddubsw m15, m3, m7 ; b0l
+ mova m3, m12
+ pmaddubsw m12, m8 ; b1l
+ mova [r7+dsq*0], m14
+ pmaddubsw m14, m4, m7 ; b0h
+ mova m4, m13
+ pmaddubsw m13, m8 ; b1h
+ paddw m15, m12
+ punpcklbw m12, m5, m0 ; 56l
+ paddw m14, m13
+ punpckhbw m13, m5, m0 ; 56h
+ pmaddubsw m5, m12, m9 ; b2l
+ paddw m15, m5
+ pmaddubsw m5, m13, m9 ; b2h
+ paddw m14, m5
+ pmulhrsw m15, m6
+ pmulhrsw m14, m6
+ packuswb m15, m14
+ mova [r7+dsq*1], m15
+ lea r7, [r7+dsq*2]
+ sub hd, 2
+ jg .v_loop
+ add srcq, 64
+ add dstq, 64
+ movzx hd, r6b
+ sub r6d, 256
+ jg .v_loop0
+ RET
+.h:
+ test myd, 0xf00
+ jz mangle(private_prefix %+ _put_8tap_8bpc_avx512icl).h2
+.hv:
+ vpbroadcastd m9, [pd_34]
+ mova xm10, [spel_hv_end]
+ pxor xm0, xm0
+ cmp wd, 4
+ jg .hv_w8
+ movzx mxd, mxb
+ dec srcq
+ vpbroadcastd m7, [base+subpel_filters+mxq*8+2]
+ movzx mxd, myb
+ shr myd, 16
+ cmp hd, 6
+ cmovs myd, mxd
+ vpbroadcastq ym1, [base+subpel_filters+1+myq*8]
+ mov nsq, ssq
+ punpcklbw ym0, ym1
+ neg nsq
+ psraw ym0, 2 ; << 6
+ pshufd ym11, ym0, q0000
+ pshufd ym12, ym0, q1111
+ pshufd ym13, ym0, q2222
+ cmp wd, 4
+ je .hv_w4
+ vbroadcasti128 ym5, [subpel_h_shuf4]
+ movq xmm0, [srcq+nsq*2]
+ movhps xmm0, [srcq+nsq*1]
+ movq xmm2, [srcq+ssq*0]
+ movhps xmm2, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
- PUT_8TAP_H 0, 2, 3, 4
- PUT_8TAP_H 1, 4, 3, 2
- packuswb m0, m1
- mova [dstq+dsq*0], ym0
- vextracti32x8 [dstq+dsq*1], m0, 1
+ vpbroadcastq ymm1, [srcq+ssq*0]
+ vpblendd ymm0, ymm1, 0x30
+ pshufb xmm2, xm5 ; 2 3
+ pshufb ymm0, ym5 ; 0 1 4
+ mova xmm1, xm9
+ vpdpbusd xmm1, xmm2, xm7
+ mova ymm2, ym9
+ vpdpbusd ymm2, ymm0, ym7
+ packssdw ymm2, ymm1
+ psraw ymm2, 2
+ vextracti128 xmm0, ymm2, 1
+ vzeroupper
+ palignr xmm0, xmm2, 4
+ punpcklwd xmm1, xmm2, xmm0 ; 01 12
+ punpckhwd xmm2, xmm0 ; 23 34
+.hv_w2_loop:
+ movq xmm3, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ movhps xmm3, [srcq+ssq*0]
+ pmaddwd xmm4, xmm1, xm11 ; a0 b0
+ mova xmm1, xmm2
+ vpdpwssd xmm4, xmm2, xm12 ; a1 b1
+ pshufb xmm3, xm5
+ mova xmm2, xm9
+ vpdpbusd xmm2, xmm3, xm7
+ packssdw xmm3, xmm2, xmm2
+ psraw xmm3, 2
+ palignr xmm2, xmm3, xmm0, 12
+ mova xmm0, xmm3
+ punpcklwd xmm2, xmm3 ; 45 56
+ vpdpwssd xmm4, xmm2, xm13 ; a2 b2
+ packuswb xmm4, xmm4
+ pshufb xmm4, xm10
+ pextrw [dstq+dsq*0], xmm4, 0
+ pextrw [dstq+dsq*1], xmm4, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
- jg .h_w32
+ jg .hv_w2_loop
RET
-.h_w64:
- movu m0, [srcq+8*0]
- movu m1, [srcq+8*1]
- add srcq, ssq
- PUT_8TAP_H 0, 2, 3, 4
- PUT_8TAP_H 1, 4, 3, 2
- packuswb m0, m1
- mova [dstq], m0
- add dstq, dsq
- dec hd
- jg .h_w64
+.hv_w4:
+ movq xm2, [srcq+nsq*2]
+ vpbroadcastq ym1, [srcq+nsq*1]
+ vinserti32x4 ym2, [srcq+ssq*0], 1
+ vinserti32x4 m1, [srcq+ssq*1], 2 ; _ 1 3
+ lea srcq, [srcq+ssq*2]
+ vbroadcasti32x4 m5, [subpel_h_shufA]
+ vinserti32x4 m2, [srcq+ssq*0], 2 ; 0 2 4
+ pshufb m1, m5
+ mova m0, m9
+ pshufb m2, m5
+ mova m3, m9
+ vpdpbusd m0, m1, m7
+ mova ym1, [spel_hv_perm4a]
+ vpdpbusd m3, m2, m7
+ mova ym2, [spel_hv_perm4b]
+ mov r6d, 0x5555
+ mova ym6, [spel_hv_perm4d]
+ packssdw m0, m3
+ kmovw k1, r6d
+ psraw m0, 2 ; _ 0 1 2 3 4 5 6
+ vpermb ym1, ym1, ym0 ; 01 12
+ vpermb m2, m2, m0 ; 23 34
+.hv_w4_loop:
+ movq xm3, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ vinserti32x4 ym3, [srcq+ssq*0], 1
+ pmaddwd ym4, ym1, ym11 ; a0 b0
+ mova ym1, ym2
+ pshufb ym3, ym5
+ mova ym0, ym9
+ vpdpbusd ym0, ym3, ym7
+ vpdpwssd ym4, ym2, ym12 ; a1 b1
+ vpsraw ym2{k1}, ym0, 2 ; 5 6
+ vpermb ym2, ym6, ym2 ; 45 56
+ vpdpwssd ym4, ym2, ym13 ; a2 b2
+ packuswb ym4, ym4
+ vpermb ym4, ym10, ym4
+ movd [dstq+dsq*0], xm4
+ pextrd [dstq+dsq*1], xm4, 1
+ lea dstq, [dstq+dsq*2]
+ sub hd, 2
+ jg .hv_w4_loop
RET
-.h_w128:
- movu m0, [srcq+8*0]
- movu m2, [srcq+8*1]
- movu m1, [srcq+8*8]
- movu m3, [srcq+8*9]
- add srcq, ssq
- PUT_8TAP_H 0, 4, 11, 12
- PUT_8TAP_H 2, 12, 11, 4
- PUT_8TAP_H 1, 4, 11, 12
- PUT_8TAP_H 3, 12, 11, 4
- packuswb m0, m2
- packuswb m1, m3
- mova [dstq+64*0], m0
- mova [dstq+64*1], m1
- add dstq, dsq
- dec hd
- jg .h_w128
+.hv_w8:
+ shr mxd, 16
+ sub srcq, 3
+ vpbroadcastd m11, [base+subpel_filters+mxq*8+0]
+ vpbroadcastd m12, [base+subpel_filters+mxq*8+4]
+ movzx mxd, myb
+ shr myd, 16
+ cmp hd, 6
+ cmovs myd, mxd
+ vpbroadcastq m1, [base+subpel_filters+1+myq*8]
+ mov nsq, ssq
+ punpcklbw m0, m1
+ neg nsq
+ psraw m0, 2 ; << 6
+ pshufd m13, m0, q0000
+ pshufd m14, m0, q1111
+ pshufd m15, m0, q2222
+ cmp wd, 8
+ jne .hv_w16
+ movu xm0, [srcq+nsq*2]
+ vinserti32x4 ym0, [srcq+nsq*1], 1
+ vbroadcasti32x4 m1, [subpel_h_shufA]
+ vinserti32x4 m0, [srcq+ssq*0], 2
+ vbroadcasti32x4 m4, [subpel_h_shufB]
+ vinserti32x4 m0, [srcq+ssq*1], 3
+ lea srcq, [srcq+ssq*2]
+ vbroadcasti32x4 m7, [subpel_h_shufC]
+ vbroadcasti32x4 ym5, [srcq+ssq*0]
+ vbroadcasti32x8 m6, [subpel_h_shufA]
+ pshufb m1, m0, m1 ; 0 1 2 3 0123
+ mova m2, m9
+ vpdpbusd m2, m1, m11
+ pshufb m4, m0, m4 ; 0 1 2 3 4567
+ mova m1, m9
+ vpdpbusd m1, m4, m11
+ pshufb m0, m7 ; 0 1 2 3 89ab
+ pshufb ym7, ym5, ym6 ; 4 0123 4567
+ mova ym3, ym9
+ vpdpbusd ym3, ym7, ym11
+ vbroadcasti32x8 m7, [subpel_h_shufB]
+ vpdpbusd m2, m4, m12
+ mova m4, [spel_hv_perm8a]
+ pshufb ym5, ym7 ; 4 4567 89ab
+ vpdpbusd m1, m0, m12
+ vpaddd m0, m4, [pb_32] {1to16}
+ vpdpbusd ym3, ym5, ym12
+ mova m5, [spel_hv_perm8b]
+ mov r6, 0x55555555ff00
+ packssdw m2, m1
+ vpmovsdw xm3, ym3
+ kmovq k1, r6
+ psraw m2, 2 ; 0 1 2 3
+ psraw xm3, 2 ; 4
+ vpermb m1, m4, m2 ; 01 12
+ kshiftrq k2, k1, 16
+ vpermt2b m2, m0, m3 ; 23 34
+.hv_w8_loop:
+ vbroadcasti32x4 ym3, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ vbroadcasti32x4 m3{k1}, [srcq+ssq*0]
+ pmaddwd m0, m1, m13 ; a0 b0
+ pshufb m1, m3, m6 ; 5 6 0123 4567
+ mova m4, m9
+ vpdpbusd m4, m1, m11
+ pshufb m3, m7 ; 5 6 4567 89ab
+ vpdpwssd m0, m2, m14 ; a1 b1
+ mova m1, m2
+ vpdpbusd m4, m3, m12
+ psraw m2{k2}, m4, 2 ; 53 64
+ vpermb m2, m5, m2 ; 45 56
+ vpdpwssd m0, m2, m15 ; a2 b2
+ packuswb m0, m0
+ vpermb m0, m10, m0
+ movq [dstq+dsq*0], xm0
+ movhps [dstq+dsq*1], xm0
+ lea dstq, [dstq+dsq*2]
+ sub hd, 2
+ jg .hv_w8_loop
+ RET
+.hv_w16:
+ movu m19, [spel_hv_perm16a]
+ vpbroadcastd m7, [pb_4]
+ lea r6d, [wq*2-32]
+ mova m6, [spel_hv_perm16b]
+ paddb m20, m7, m19
+ lea r6d, [hq+r6*8]
+ paddb m21, m7, m20
+ mova ym10, [spel_hv_end16]
+ paddb m7, m6
+.hv_w16_loop0:
+ movu ym16, [srcq+nsq*2]
+ vinserti32x8 m16, [srcq+nsq*1], 1
+ lea r4, [srcq+ssq*2]
+ movu ym17, [srcq+ssq*0]
+ vinserti32x8 m17, [srcq+ssq*1], 1
+ mov r7, dstq
+ movu ym18, [r4 +ssq*0]
+ vpermb m2, m19, m16 ; 0 1 0123 89ab
+ mova m1, m9
+ vpermb m3, m21, m16 ; 0 1 89ab ghij
+ vpdpbusd m1, m2, m11
+ mova m2, m9
+ vpermb m4, m19, m17 ; 2 3 0123 89ab
+ vpdpbusd m2, m3, m12
+ mova m3, m9
+ vpermb m5, m21, m17 ; 2 3 89ab ghij
+ vpdpbusd m3, m4, m11
+ mova m4, m9
+ vpermb m0, m6, m18 ; 4 0145 2367 89cd abef
+ vpdpbusd m4, m5, m12
+ mova m5, m9
+ vpermb m16, m20, m16 ; 0 1 4567 cdef
+ vpdpbusd m5, m0, m11
+ vpermb m17, m20, m17 ; 2 3 4567 cdef
+ vpdpbusd m1, m16, m12
+ vpermb m18, m7, m18 ; 4 4589 67ab cdgh efij
+ vpdpbusd m2, m16, m11
+ vpdpbusd m3, m17, m12
+ vpdpbusd m4, m17, m11
+ vpdpbusd m5, m18, m12
+ packssdw m1, m2 ; 01
+ packssdw m3, m4 ; 23
+ REPX {psraw x, 2}, m1, m3, m5
+ vpshrdd m2, m1, m3, 16 ; 12
+ vpshrdd m4, m3, m5, 16 ; 34
+.hv_w16_loop:
+ movu ym18, [r4+ssq*1]
+ lea r4, [r4+ssq*2]
+ vinserti32x8 m18, [r4+ssq*0], 1
+ pmaddwd m16, m1, m13 ; a0
+ vpermb m1, m19, m18 ; 5 6 0123 89ab
+ pmaddwd m17, m2, m13 ; b0
+ vpermb m2, m20, m18 ; 5 6 4567 cdef
+ mova m0, m9
+ vpdpbusd m0, m1, m11
+ vpermb m18, m21, m18
+ mova m1, m9
+ vpdpbusd m1, m2, m11
+ vpdpwssd m16, m3, m14 ; a1
+ vpdpwssd m17, m4, m14 ; b1
+ vpdpbusd m0, m2, m12
+ mova m2, m4
+ vpdpbusd m1, m18, m12
+ packssdw m0, m1
+ mova m1, m3
+ psraw m4, m0, 2 ; 5 6
+ vpshrdd m3, m2, m4, 16 ; 4 5
+ vpdpwssd m17, m4, m15 ; b2
+ vpdpwssd m16, m3, m15 ; a2
+ packuswb m16, m17
+ vpermb m16, m10, m16
+ mova [r7+dsq*0], xm16
+ vextracti128 [r7+dsq*1], ym16, 1
+ lea r7, [r7+dsq*2]
+ sub hd, 2
+ jg .hv_w16_loop
+ add srcq, 16
+ add dstq, 16
+ movzx hd, r6b
+ sub r6d, 1<<8
+ jg .hv_w16_loop0
+ vzeroupper
RET
+
+PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP, put_8tap_8bpc
+PUT_8TAP_FN regular_sharp, REGULAR, SHARP, put_8tap_8bpc
+PUT_8TAP_FN sharp, SHARP, SHARP
+
+cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
+ imul mxd, mxm, 0x010101
+ add mxd, t0d ; 8tap_h, mx, 4tap_h
+ imul myd, mym, 0x010101
+ add myd, t1d ; 8tap_v, my, 4tap_v
+ lea r8, [put_avx512icl]
+ movsxd wq, wm
+ movifnidn hd, hm
+ test mxd, 0xf00
+ jnz .h
+ test myd, 0xf00
+ jz mangle(private_prefix %+ _put_6tap_8bpc_avx512icl).put
.v:
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
tzcnt r6d, wd
+ lea myq, [base+subpel_filters+myq*8]
movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)]
vpbroadcastd m7, [pw_512]
- lea myq, [base+subpel_filters+myq*8]
vpbroadcastw m8, [myq+0]
- vpbroadcastw m9, [myq+2]
- vpbroadcastw m10, [myq+4]
- vpbroadcastw m11, [myq+6]
add r6, r8
+ vpbroadcastw m9, [myq+2]
lea ss3q, [ssq*3]
+ vpbroadcastw m10, [myq+4]
sub srcq, ss3q
+ vpbroadcastw m11, [myq+6]
jmp r6
.v_w2:
movd xmm2, [srcq+ssq*0]
@@ -1802,7 +2200,7 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
vzeroupper
RET
.v_w16:
- mova m12, [spel_v_perm16]
+ mova m12, [spel_v_perm16a]
vbroadcasti32x4 m1, [srcq+ssq*0]
vbroadcasti32x4 ym4, [srcq+ssq*1]
mov r6d, 0x0f
@@ -1990,7 +2388,146 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
jg .v_loop0
vzeroupper
RET
+.h:
+ test myd, 0xf00
+ jnz .hv
+.h2:
+ vpbroadcastd m5, [pd_34] ; 2 + (8 << 2)
+ cmp wd, 4
+ jl .h_w2
+ vbroadcasti128 m6, [subpel_h_shufA]
+ je .h_w4
+ tzcnt wd, wd
+ vbroadcasti128 m7, [subpel_h_shufB]
+ vbroadcasti128 m8, [subpel_h_shufC]
+ shr mxd, 16
+ sub srcq, 3
+ movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)]
+ vpbroadcastd m9, [base+mxq*8+subpel_filters+0]
+ vpbroadcastd m10, [base+mxq*8+subpel_filters+4]
+ add wq, r8
+ jmp wq
+.h_w2:
+ movzx mxd, mxb
+ dec srcq
+ mova xmm4, [subpel_h_shuf4]
+ vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2]
+.h_w2_loop:
+ movq xmm0, [srcq+ssq*0]
+ movhps xmm0, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ pshufb xmm0, xmm4
+ mova xmm1, xm5
+ vpdpbusd xmm1, xmm0, xmm3
+ packssdw xmm0, xmm1, xmm1
+ psraw xmm0, 6
+ packuswb xmm0, xm0
+ pextrw [dstq+dsq*0], xmm0, 0
+ pextrw [dstq+dsq*1], xmm0, 1
+ lea dstq, [dstq+dsq*2]
+ sub hd, 2
+ jg .h_w2_loop
+ RET
+.h_w4:
+ movzx mxd, mxb
+ dec srcq
+ vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2]
+.h_w4_loop:
+ movq xmm0, [srcq+ssq*0]
+ movq xmm1, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ pshufb xmm0, xm6
+ pshufb xmm1, xm6
+ mova xmm2, xm5
+ vpdpbusd xmm2, xmm0, xmm3
+ mova xmm0, xm5
+ vpdpbusd xmm0, xmm1, xmm3
+ packssdw xmm0, xmm2, xmm0
+ psraw xmm0, 6
+ packuswb xmm0, xmm0
+ movd [dstq+dsq*0], xmm0
+ pextrd [dstq+dsq*1], xmm0, 1
+ lea dstq, [dstq+dsq*2]
+ sub hd, 2
+ jg .h_w4_loop
+ RET
+.h_w8:
+ movu xm0, [srcq+ssq*0]
+ vinserti32x4 ym0, [srcq+ssq*1], 1
+ lea srcq, [srcq+ssq*2]
+ WRAP_YMM PUT_8TAP_H 0, 1, 2, 3
+ vpmovuswb xm0, ym0
+ movq [dstq+dsq*0], xm0
+ movhps [dstq+dsq*1], xm0
+ lea dstq, [dstq+dsq*2]
+ sub hd, 2
+ jg .h_w8
+ RET
+.h_w16:
+ mova m6, [spel_h_perm16]
+ vpbroadcastd m8, [pb_4]
+ paddb m7, m8, m6
+ paddb m8, m7
+.h_w16_loop:
+ movu ym0, [srcq+ssq*0]
+ vinserti32x8 m0, [srcq+ssq*1], 1
+ lea srcq, [srcq+ssq*2]
+ PUT_8TAP_H 0, 1, 2, 3, 1
+ vpmovuswb ym0, m0
+ mova [dstq+dsq*0], xm0
+ vextracti128 [dstq+dsq*1], ym0, 1
+ lea dstq, [dstq+dsq*2]
+ sub hd, 2
+ jg .h_w16_loop
+ RET
+.h_w32:
+ movu ym0, [srcq+ssq*0+8*0]
+ vinserti32x8 m0, [srcq+ssq*1+8*0], 1
+ movu ym1, [srcq+ssq*0+8*1]
+ vinserti32x8 m1, [srcq+ssq*1+8*1], 1
+ lea srcq, [srcq+ssq*2]
+ PUT_8TAP_H 0, 2, 3, 4
+ PUT_8TAP_H 1, 4, 3, 2
+ packuswb m0, m1
+ mova [dstq+dsq*0], ym0
+ vextracti32x8 [dstq+dsq*1], m0, 1
+ lea dstq, [dstq+dsq*2]
+ sub hd, 2
+ jg .h_w32
+ RET
+.h_w64:
+ movu m0, [srcq+8*0]
+ movu m1, [srcq+8*1]
+ add srcq, ssq
+ PUT_8TAP_H 0, 2, 3, 4
+ PUT_8TAP_H 1, 4, 3, 2
+ packuswb m0, m1
+ mova [dstq], m0
+ add dstq, dsq
+ dec hd
+ jg .h_w64
+ RET
+.h_w128:
+ movu m0, [srcq+8*0]
+ movu m2, [srcq+8*1]
+ movu m1, [srcq+8*8]
+ movu m3, [srcq+8*9]
+ add srcq, ssq
+ PUT_8TAP_H 0, 4, 11, 12
+ PUT_8TAP_H 2, 12, 11, 4
+ PUT_8TAP_H 1, 4, 11, 12
+ PUT_8TAP_H 3, 12, 11, 4
+ packuswb m0, m2
+ packuswb m1, m3
+ mova [dstq+64*0], m0
+ mova [dstq+64*1], m1
+ add dstq, dsq
+ dec hd
+ jg .h_w128
+ RET
.hv:
+ vpbroadcastd m9, [pd_34]
+ pxor xm0, xm0
cmp wd, 4
jg .hv_w8
movzx mxd, mxb
@@ -2000,12 +2537,10 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
shr myd, 16
cmp hd, 6
cmovs myd, mxd
- vpbroadcastd m8, [pd_2]
- vpbroadcastq ym0, [base+subpel_filters+myq*8]
+ vpbroadcastq ym1, [base+subpel_filters+myq*8]
lea ss3q, [ssq*3]
- vpbroadcastd ym9, [pd_32768]
mov r6, srcq
- punpcklbw ym0, ym8, ym0
+ punpcklbw ym0, ym1
sub r6, ss3q
psraw ym0, 2 ; << 6
mova xm14, [spel_hv_end]
@@ -2029,9 +2564,9 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
vpblendd ymm2, ymm4, 0xc0 ; 0 1 4 5
pshufb ymm2, ym6
pshufb ymm0, ym6
- mova ymm1, ym8
+ mova ymm1, ym9
vpdpbusd ymm1, ymm2, ym7
- mova ymm2, ym8
+ mova ymm2, ym9
vpdpbusd ymm2, ymm0, ym7
packssdw ymm2, ymm1, ymm2
psraw ymm2, 2
@@ -2045,14 +2580,13 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
movq xmm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
movhps xmm4, [srcq+ssq*0]
- mova xmm5, xm9
- vpdpwssd xmm5, xmm1, xm10 ; a0 b0
+ pmaddwd xmm5, xmm1, xm10 ; a0 b0
mova xmm1, xmm2
vpdpwssd xmm5, xmm2, xm11 ; a1 b1
pshufb xmm4, xm6
mova xmm2, xmm3
vpdpwssd xmm5, xmm3, xm12 ; a2 b2
- mova xmm3, xm8
+ mova xmm3, xm9
vpdpbusd xmm3, xmm4, xm7
packssdw xmm4, xmm3, xmm3
psraw xmm4, 2
@@ -2081,9 +2615,9 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
vinserti32x4 m1, [srcq+ssq*0], 3 ; 0 2 4 6
pshufb m2, m6
pshufb m1, m6
- mova m0, m8
+ mova m0, m9
vpdpbusd m0, m2, m7
- mova m4, m8
+ mova m4, m9
vpdpbusd m4, m1, m7
mova ym1, [spel_hv_perm4a]
mova ym2, [spel_hv_perm4b]
@@ -2100,11 +2634,10 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
movq xmm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vinserti32x4 ym4, ymm4, [srcq+ssq*0], 1
- mova ym5, ym9
- vpdpwssd ym5, ym1, ym10 ; a0 b0
+ pmaddwd ym5, ym1, ym10 ; a0 b0
mova ym1, ym2
pshufb ym4, ym6
- mova ym0, ym8
+ mova ym0, ym9
vpdpbusd ym0, ym4, ym7
vpdpwssd ym5, ym2, ym11 ; a1 b1
mova ym2, ym3
@@ -2129,10 +2662,8 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
shr myd, 16
cmp hd, 6
cmovs myd, mxd
- vpbroadcastd m8, [pd_2]
- vpbroadcastq m0, [base+subpel_filters+myq*8]
- vpbroadcastd m9, [pd_32768]
- punpcklbw m0, m8, m0
+ vpbroadcastq m1, [base+subpel_filters+myq*8]
+ punpcklbw m0, m1
lea ss3q, [ssq*3]
psraw m0, 2 ; << 6
pshufd m12, m0, q0000
@@ -2153,31 +2684,31 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
vbroadcasti32x4 m4, [subpel_h_shufA]
vinserti32x4 m0, zmm2, [srcq+ssq*0], 2 ; 4 5 6 _
vbroadcasti32x4 m7, [subpel_h_shufB]
- vbroadcasti32x4 m17, [subpel_h_shufC]
+ vbroadcasti32x4 m8, [subpel_h_shufC]
pshufb m1, m6, m4 ; 0 1 2 3 0123
- mova m2, m8
+ mova m2, m9
vpdpbusd m2, m1, m10
pshufb m5, m6, m7 ; 0 1 2 3 4567
- mova m1, m8
+ mova m1, m9
vpdpbusd m1, m5, m10
pshufb m4, m0, m4 ; 4 5 6 _ 0123
- mova m3, m8
+ mova m3, m9
vpdpbusd m3, m4, m10
pshufb m7, m0, m7 ; 4 5 6 _ 4567
- mova m4, m8
+ mova m4, m9
vpdpbusd m4, m7, m10
- pshufb m6, m17
+ pshufb m6, m8
vpdpbusd m2, m5, m11
vpdpbusd m1, m6, m11
- pshufb m6, m0, m17
+ pshufb m6, m0, m8
vpdpbusd m3, m7, m11
vpdpbusd m4, m6, m11
mova m5, [spel_hv_perm8a]
- mova m0, [spel_hv_perm8b]
+ vpaddd m0, m5, [pb_32] {1to16}
mov r6, 0x55555555ff00
packssdw m2, m1
packssdw m3, m4
- mova m18, [spel_hv_perm8c]
+ mova m8, [spel_hv_perm8b]
psraw m2, 2 ; 0 1 2 3
psraw m3, 2 ; 4 5 6 _
vpermb m1, m5, m2 ; 01 12
@@ -2192,10 +2723,9 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
vbroadcasti32x4 ym4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vbroadcasti32x4 m4{k1}, [srcq+ssq*0]
- mova m0, m9
- vpdpwssd m0, m1, m12 ; a0 b0
+ pmaddwd m0, m1, m12 ; a0 b0
pshufb m1, m4, m6 ; 7 8 0123 4567
- mova m5, m8
+ mova m5, m9
vpdpbusd m5, m1, m10
pshufb m4, m7 ; 7 8 4567 89ab
vpdpwssd m0, m2, m13 ; a1 b1
@@ -2204,7 +2734,7 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
mova m2, m3
vpdpwssd m0, m3, m14 ; a2 b2
psraw m3{k2}, m5, 2 ; 75 86
- vpermb m3, m18, m3 ; 67 78
+ vpermb m3, m8, m3 ; 67 78
vpdpwssd m0, m3, m15 ; a3 b3
packuswb m0, m0
vpermb zmm1, m16, m0
@@ -2216,111 +2746,652 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
vzeroupper
RET
.hv_w16:
- movu m7, [spel_hv_perm16a]
+ WIN64_SPILL_XMM 23
+ movu m22, [spel_hv_perm16a]
sub srcq, ss3q
- mova m20, [spel_hv_perm16b]
+ vpbroadcastd m8, [pb_4]
lea r6d, [wq*2-32]
- mova m21, [spel_hv_perm16c]
- mov r4, srcq
- mov r7, dstq
+ mova m7, [spel_hv_perm16b]
+ paddb m20, m8, m22
mova ym16, [spel_hv_end16]
+ paddb m21, m8, m20
lea r6d, [hq+r6*8]
+ paddb m8, m7
.hv_w16_loop0:
movu ym17, [srcq+ssq*0]
vinserti32x8 m17, [srcq+ssq*1], 1 ; 0 1
+ lea r4, [srcq+ss3q]
movu ym18, [srcq+ssq*2]
- add srcq, ss3q
- vinserti32x8 m18, [srcq+ssq*0], 1 ; 2 3
- movu ym19, [srcq+ssq*1]
- vinserti32x8 m19, [srcq+ssq*2], 1 ; 4 5
- add srcq, ss3q
- vpermb m2, m7, m17 ; 0 1 0123 89ab
- vpermb m0, m20, m17 ; 0 1 4567 cdef
- vpermb m4, m7, m18 ; 2 3 0123 89ab
- mova m1, m8
+ vinserti32x8 m18, [r4 +ssq*0], 1 ; 2 3
+ mov r7, dstq
+ movu ym19, [r4 +ssq*1]
+ vinserti32x8 m19, [r4 +ssq*2], 1 ; 4 5
+ add r4, ss3q
+ vpermb m2, m22, m17 ; 0 1 0123 89ab
+ mova m1, m9
+ vpermb m3, m21, m17 ; 0 1 89ab ghij
vpdpbusd m1, m2, m10
- vpermb m5, m20, m18 ; 2 3 4567 cdef
- mova m2, m8
- vpdpbusd m2, m0, m10
- vpermb m17, m21, m17 ; 0 1 89ab ghij
- mova m3, m8
+ mova m2, m9
+ vpermb m4, m22, m18 ; 2 3 0123 89ab
+ vpdpbusd m2, m3, m11
+ mova m3, m9
+ vpermb m5, m21, m18 ; 2 3 89ab ghij
vpdpbusd m3, m4, m10
- vpermb m6, m7, m19 ; 4 5 0123 89ab
- mova m4, m8
- vpdpbusd m4, m5, m10
- vpermb m18, m21, m18 ; 2 3 89ab ghij
- vpdpbusd m1, m0, m11
- movu ym0, [srcq+ssq*0] ; 6
- vpdpbusd m2, m17, m11
- vpermb m17, m20, m19 ; 4 5 4567 cdef
- vpdpbusd m3, m5, m11
- mova m5, m8
+ mova m4, m9
+ vpermb m6, m22, m19 ; 4 5 0123 89ab
+ vpdpbusd m4, m5, m11
+ mova m5, m9
+ vpermb m17, m20, m17 ; 0 1 4567 cdef
vpdpbusd m5, m6, m10
- mova m6, m8
- vpdpbusd m6, m17, m10
- vpdpbusd m4, m18, m11
- mova m18, [spel_hv_perm16d]
- vpermb m18, m18, m0 ; 6 0145 2367 89cd abef
- vpdpbusd m5, m17, m11
- vpermb m19, m21, m19 ; 4 5 89ab ghij
- mova m17, m8
- vpdpbusd m17, m18, m10
- mova m18, [spel_hv_perm16e]
- vpermb m0, m18, m0 ; 6 4589 67ab cdgh efij
- packssdw m1, m2 ; 01
- vpdpbusd m6, m19, m11
- packssdw m3, m4 ; 23
- vpdpbusd m17, m0, m11
- psraw m1, 2
- packssdw m5, m6 ; 45
- psraw m3, 2
+ mova m6, m9
+ vpermb m0, m21, m19 ; 4 5 89ab ghij
+ vpdpbusd m1, m17, m11
+ vpdpbusd m2, m17, m10
+ movu ym17, [r4+ssq*0] ; 6
+ vpermb m18, m20, m18 ; 2 3 4567 cdef
+ vpdpbusd m6, m0, m11
+ vpermb m0, m7, m17 ; 6 0145 2367 89cd abef
+ vpdpbusd m3, m18, m11
+ vpermb m19, m20, m19 ; 4 5 4567 cdef
+ vpdpbusd m4, m18, m10
+ mova m18, m9
+ vpermb m17, m8, m17 ; 6 4589 67ab cdgh efij
+ vpdpbusd m18, m0, m10
+ packssdw m1, m2
+ vpdpbusd m5, m19, m11
+ vpdpbusd m6, m19, m10
+ packssdw m3, m4
+ vpdpbusd m18, m17, m11
+ psraw m1, 2 ; 01
+ psraw m3, 2 ; 23
+ packssdw m5, m6
vpshrdd m2, m1, m3, 16 ; 12
- psraw m5, 2
+ psraw m5, 2 ; 45
vpshrdd m4, m3, m5, 16 ; 34
- psraw m17, 2
- vpshrdd m6, m5, m17, 16 ; 56
+ psraw m18, 2
+ vpshrdd m6, m5, m18, 16 ; 56
.hv_w16_loop:
- movu ym18, [srcq+ssq*1]
- lea srcq, [srcq+ssq*2]
- vinserti32x8 m18, [srcq+ssq*0], 1
+ movu ym19, [r4+ssq*1]
+ lea r4, [r4+ssq*2]
+ vinserti32x8 m19, [r4+ssq*0], 1
+ pmaddwd m17, m1, m12 ; a0
+ vpermb m1, m22, m19 ; 7 8 0123 89ab
+ pmaddwd m18, m2, m12 ; b0
mova m0, m9
- vpdpwssd m0, m1, m12 ; a0
- vpermb m1, m7, m18 ; 7 8 0123 89ab
- mova m17, m9
- vpdpwssd m17, m2, m12 ; b0
- vpermb m2, m20, m18 ; 7 8 4567 cdef
- mova m19, m8
- vpdpbusd m19, m1, m10
- vpermb m18, m21, m18
- mova m1, m8
- vpdpbusd m1, m2, m10
- vpdpwssd m0, m3, m13 ; a1
- vpdpwssd m17, m4, m13 ; b1
- vpdpbusd m19, m2, m11
+ vpermb m2, m21, m19 ; 7 8 89ab ghij
+ vpdpbusd m0, m1, m10
+ mova m1, m9
+ vpermb m19, m20, m19 ; 7 8 4567 cdef
+ vpdpbusd m1, m2, m11
mova m2, m4
- vpdpbusd m1, m18, m11
+ vpdpwssd m17, m3, m13 ; a1
+ vpdpwssd m18, m4, m13 ; b1
mova m4, m6
- vpdpwssd m0, m5, m14 ; a2
- vpdpwssd m17, m6, m14 ; b2
- packssdw m19, m1
+ vpdpbusd m0, m19, m11
+ vpdpbusd m1, m19, m10
+ vpdpwssd m17, m5, m14 ; a2
+ vpdpwssd m18, m6, m14 ; b2
+ packssdw m0, m1
mova m1, m3
+ psraw m6, m0, 2 ; 78
mova m3, m5
- psraw m6, m19, 2 ; 7 8
- vpshrdd m5, m4, m6, 16 ; 6 7
- vpdpwssd m17, m6, m15 ; b3
- vpdpwssd m0, m5, m15 ; a3
- packuswb m0, m17
- vpermb zmm1, m16, m0
- mova [dstq+dsq*0], xmm1
- vextracti128 [dstq+dsq*1], ymm1, 1
- lea dstq, [dstq+dsq*2]
+ vpshrdd m5, m4, m6, 16 ; 67
+ vpdpwssd m18, m6, m15 ; b3
+ vpdpwssd m17, m5, m15 ; a3
+ packuswb m17, m18
+ vpermb m17, m16, m17
+ mova [r7+dsq*0], xm17
+ vextracti128 [r7+dsq*1], ym17, 1
+ lea r7, [r7+dsq*2]
sub hd, 2
jg .hv_w16_loop
- add r4, 16
- add r7, 16
+ add srcq, 16
+ add dstq, 16
+ movzx hd, r6b
+ sub r6d, 1<<8
+ jg .hv_w16_loop0
+ RET
+
+%if WIN64
+DECLARE_REG_TMP 6, 4
+%else
+DECLARE_REG_TMP 6, 7
+%endif
+
+%define PREP_8TAP_FN FN prep_8tap,
+PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH, prep_6tap_8bpc
+PREP_8TAP_FN sharp_regular, SHARP, REGULAR, prep_6tap_8bpc
+PREP_8TAP_FN smooth, SMOOTH, SMOOTH, prep_6tap_8bpc
+PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR, prep_6tap_8bpc
+PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH, prep_6tap_8bpc
+PREP_8TAP_FN regular, REGULAR, REGULAR
+
+cglobal prep_6tap_8bpc, 4, 8, 0, tmp, src, ss, w, h, mx, my, ss3
+%define base r7-prep_avx512icl
+ imul mxd, mxm, 0x010101
+ add mxd, t0d ; 6tap_h, mx, 4tap_h
+ imul myd, mym, 0x010101
+ add myd, t1d ; 6tap_v, my, 4tap_v
+ lea r7, [prep_avx512icl]
+ movifnidn hd, hm
+ test mxd, 0xf00
+ jnz .h
+ test myd, 0xf00
+ jnz .v
+.prep:
+ tzcnt wd, wd
+ movzx wd, word [r7+wq*2+table_offset(prep,)]
+ add wq, r7
+ lea r6, [ssq*3]
+%if WIN64
+ pop r7
+%endif
+ jmp wq
+.v:
+ movzx mxd, myb
+ shr myd, 16
+ cmp hd, 4
+ cmove myd, mxd
+ tzcnt r5d, wd
+ lea myq, [base+subpel_filters+1+myq*8]
+ movzx r5d, word [r7+r5*2+table_offset(prep, _6tap_v)]
+ vpbroadcastd m7, [pw_8192]
+ sub srcq, ssq
+ vpbroadcastw m8, [myq+0]
+ add r5, r7
+ vpbroadcastw m9, [myq+2]
+ lea ss3q, [ssq*3]
+ vpbroadcastw m10, [myq+4]
+ sub srcq, ssq
+ jmp r5
+.v_w4:
+ movd xmm2, [srcq+ssq*0]
+ pinsrd xmm2, [srcq+ssq*1], 1
+ vpbroadcastd ymm1, [srcq+ssq*2]
+ add srcq, ss3q
+ vpbroadcastd ymm3, [srcq+ssq*0]
+ vpbroadcastd ymm0, [srcq+ssq*1]
+ vbroadcasti128 ymm5, [deint_shuf4]
+ vpblendd ymm1, ymm2, 0xeb
+ punpcklqdq ymm3, ymm0
+ vpblendd ymm1, ymm3, 0x60 ; 0 1 2 _ 2 3 4 _
+ pshufb ymm1, ymm5 ; 01 12 23 34
+.v_w4_loop:
+ pinsrd xmm0, [srcq+ssq*2], 1
+ vpbroadcastd ymm2, [srcq+ss3q ]
+ lea srcq, [srcq+ssq*4]
+ vpbroadcastd ymm3, [srcq+ssq*0]
+ vpblendd ymm2, ymm0, 0xeb
+ vpbroadcastd ymm0, [srcq+ssq*1]
+ punpcklqdq ymm3, ymm0
+ vpblendd ymm2, ymm3, 0x60 ; 4 5 6 _ 6 7 8 _
+ pshufb ymm2, ymm5 ; 45 56 67 78
+ pmaddubsw ymm3, ymm1, ym8 ; a0 b0 c0 d0
+ vperm2i128 ymm1, ymm2, 0x21 ; 23 34 45 56
+ pmaddubsw ymm4, ymm2, ym10 ; a2 b2 c2 d2
+ pmaddubsw ymm1, ym9 ; a1 b1 c1 d1
+ paddw ymm3, ymm4
+ paddw ymm3, ymm1
+ pmulhrsw ymm3, ym7
+ mova ymm1, ymm2
+ mova [tmpq], ymm3
+ add tmpq, 32
+ sub hd, 4
+ jg .v_w4_loop
+ vzeroupper
+ RET
+.v_w8:
+ mova m6, [spel_v_perm8]
+ movq xm1, [srcq+ssq*0]
+ mov r6d, 0x3e
+ movq xm2, [srcq+ssq*1]
+ kmovb k1, r6d
+ vpbroadcastq ym3, [srcq+ssq*2]
+ add srcq, ss3q
+ vpunpcklqdq ym2, [srcq+ssq*0] {1to4}
+ vpunpcklqdq m1{k1}, m3, [srcq+ssq*1] {1to8}
+ movq xm0, [srcq+ssq*1]
+ kshiftlb k2, k1, 2
+ shufpd m1, m2, 0x18 ; 0 1 2 3 4
+ vpermb m1, m6, m1 ; 01 12 23 34
+.v_w8_loop:
+ vpbroadcastq ym3, [srcq+ss3q ]
+ vpunpcklqdq ym0{k1}, ym3, [srcq+ssq*2] {1to4}
+ lea srcq, [srcq+ssq*4]
+ vpbroadcastq m3, [srcq+ssq*1]
+ vpunpcklqdq m0{k2}, m3, [srcq+ssq*0] {1to8}
+ pmaddubsw m4, m1, m8 ; a0 b0 c0 d0
+ vpermb m2, m6, m0 ; 45 56 67 78
+ mova xm0, xm3
+ vshufi32x4 m1, m2, q1032 ; 23 34 45 56
+ pmaddubsw m3, m2, m10 ; a3 b3 c3 d3
+ pmaddubsw m5, m1, m9 ; a2 b2 c2 d2
+ mova m1, m2
+ paddw m4, m3
+ paddw m4, m5
+ pmulhrsw m4, m7
+ mova [tmpq], m4
+ add tmpq, 64
+ sub hd, 4
+ jg .v_w8_loop
+ RET
+.v_w16:
+ mova m11, [spel_v_perm16b]
+ vbroadcasti32x4 m1, [srcq+ssq*0]
+ mov r6d, 0x0f
+ vbroadcasti32x4 ym3, [srcq+ssq*1]
+ vbroadcasti32x4 m2, [srcq+ssq*2]
+ kmovb k1, r6d
+ add srcq, ss3q
+ vbroadcasti32x4 ym4, [srcq+ssq*0]
+ vbroadcasti32x4 m0, [srcq+ssq*1]
+ vshufpd m1{k1}, m3, m2, 0xcc
+ vshufpd m2{k1}, m4, m0, 0xcc
+ vpermb m1, m11, m1 ; 01 12
+ vpermb m2, m11, m2 ; 23 34
+.v_w16_loop:
+ pmaddubsw m3, m1, m8 ; a0 b0
+ pmaddubsw m5, m2, m9 ; a1 b1
+ vbroadcasti32x4 ym6, [srcq+ssq*2]
+ pmaddubsw m4, m2, m8 ; c0 d0
+ vbroadcasti32x4 m2, [srcq+ss3q ]
+ lea srcq, [srcq+ssq*4]
+ vshufpd m0{k1}, m6, m2, 0xcc
+ vbroadcasti32x4 ym6, [srcq+ssq*0]
+ vpermb m1, m11, m0 ; 45 56
+ vbroadcasti32x4 m0, [srcq+ssq*1]
+ vshufpd m2{k1}, m6, m0, 0xcc
+ pmaddubsw m6, m1, m9 ; c1 d1
+ vpermb m2, m11, m2 ; 67 78
+ paddw m3, m5
+ pmaddubsw m5, m1, m10 ; a2 b2
+ paddw m4, m6
+ pmaddubsw m6, m2, m10 ; c2 d2
+ paddw m3, m5
+ paddw m4, m6
+ pmulhrsw m3, m7
+ pmulhrsw m4, m7
+ mova [tmpq+ 0], m3
+ mova [tmpq+64], m4
+ add tmpq, 64*2
+ sub hd, 4
+ jg .v_w16_loop
+ RET
+.v_w32:
+ movshdup m6, [bilin_v_perm64]
+ movu ym16, [srcq+ssq*0]
+ movu ym17, [srcq+ssq*1]
+ movu ym18, [srcq+ssq*2]
+ add srcq, ss3q
+ movu ym19, [srcq+ssq*0]
+ add srcq, ssq
+ movu ym20, [srcq+ssq*0]
+ vpermt2q m16, m6, m18 ; 0 2
+ vpermt2q m17, m6, m19 ; 1 3
+ vpermt2q m18, m6, m20 ; 2 4
+ punpcklbw m0, m16, m17 ; 01
+ punpcklbw m1, m17, m18 ; 12
+ punpckhbw m2, m16, m17 ; 23
+ punpckhbw m3, m17, m18 ; 34
+.v_w32_loop:
+ movu ym16, [srcq+ssq*1]
+ lea srcq, [srcq+ssq*2]
+ movu ym17, [srcq+ssq*0]
+ pmaddubsw m4, m0, m8 ; a0
+ mova m0, m2
+ pmaddubsw m2, m9 ; a1
+ vpermt2q m16, m6, m17 ; 5 6
+ pmaddubsw m5, m1, m8 ; b0
+ mova m1, m3
+ pmaddubsw m3, m9 ; b1
+ shufpd m18, m16, 0x55 ; 4 5
+ paddw m4, m2
+ punpcklbw m2, m18, m16 ; 45
+ paddw m5, m3
+ punpckhbw m3, m18, m16 ; 56
+ mova m18, m16
+ pmaddubsw m16, m2, m10 ; a2
+ pmaddubsw m17, m3, m10 ; b2
+ paddw m4, m16
+ paddw m5, m17
+ pmulhrsw m4, m7
+ pmulhrsw m5, m7
+ mova [tmpq+ 0], m4
+ mova [tmpq+64], m5
+ add tmpq, 64*2
+ sub hd, 2
+ jg .v_w32_loop
+ vzeroupper
+ RET
+.v_w64:
+.v_w128:
+ mova m6, [bilin_v_perm64]
+ add wd, wd
+ lea r6d, [hq+wq]
+.v_loop0:
+ vpermq m12, m6, [srcq+ssq*0]
+ vpermq m13, m6, [srcq+ssq*1]
+ lea r5, [srcq+ssq*2]
+ vpermq m14, m6, [r5 +ssq*0]
+ vpermq m15, m6, [r5 +ssq*1]
+ lea r5, [r5+ssq*2]
+ vpermq m16, m6, [r5 +ssq*0]
+ mov r7, tmpq
+ punpcklbw m0, m12, m13 ; 01
+ punpckhbw m12, m13
+ punpcklbw m1, m13, m14 ; 12
+ punpckhbw m13, m14
+ punpcklbw m2, m14, m15 ; 23
+ punpckhbw m14, m15
+ punpcklbw m3, m15, m16 ; 34
+ punpckhbw m15, m16
+.v_loop:
+ pmaddubsw m17, m0, m8 ; a0
+ vpermq m5, m6, [r5+ssq*1]
+ pmaddubsw m18, m12, m8
+ mova m0, m2
+ pmaddubsw m2, m9 ; a1
+ mova m12, m14
+ pmaddubsw m14, m9
+ lea r5, [r5+ssq*2]
+ pmaddubsw m19, m1, m8 ; b0
+ pmaddubsw m20, m13, m8
+ mova m1, m3
+ pmaddubsw m3, m9 ; b1
+ mova m13, m15
+ pmaddubsw m15, m9
+ paddw m17, m2
+ punpcklbw m2, m16, m5 ; 67
+ paddw m18, m14
+ punpckhbw m14, m16, m5
+ vpermq m16, m6, [r5+ssq*0]
+ paddw m19, m3
+ pmaddubsw m3, m2, m10 ; a3
+ paddw m20, m15
+ pmaddubsw m15, m14, m10
+ paddw m17, m3
+ punpcklbw m3, m5, m16 ; 78
+ pmaddubsw m4, m3, m10 ; b3
+ paddw m18, m15
+ punpckhbw m15, m5, m16
+ pmaddubsw m5, m15, m10
+ paddw m19, m4
+ paddw m20, m5
+ REPX {pmulhrsw x, m7}, m17, m18, m19, m20
+ mova [r7+wq*0+ 0], m17
+ mova [r7+wq*0+64], m18
+ mova [r7+wq*1+ 0], m19
+ mova [r7+wq*1+64], m20
+ lea r7, [r7+wq*2]
+ sub hd, 2
+ jg .v_loop
+ add srcq, 64
+ add tmpq, 128
+ movzx hd, r6b
+ sub r6d, 1<<8
+ jg .v_loop0
+ vzeroupper
+ RET
+.h:
+ test myd, 0xf00
+ jz mangle(private_prefix %+ _prep_8tap_8bpc_avx512icl).h2
+.hv:
+ vpbroadcastd m8, [pd_2]
+ vpbroadcastd m9, [pd_32]
+ cmp wd, 4
+ jg .hv_w8
+ movzx mxd, mxb
+ vpbroadcastd m11, [base+subpel_filters+mxq*8+2]
+ movzx mxd, myb
+ shr myd, 16
+ cmp hd, 4
+ cmove myd, mxd
+ vpbroadcastq m3, [base+subpel_filters+1+myq*8]
+ vbroadcasti128 m10, [subpel_h_shufA]
+ lea r6, [ssq*2+1]
+ mov r3d, 0x30
+ sub srcq, r6
+ kmovb k1, r3d
+ vpbroadcastq ym2, [srcq+ssq*0]
+ lea ss3q, [ssq*3]
+ vpbroadcastq m1, [srcq+ssq*1]
+ kaddb k2, k1, k1
+ vpbroadcastq m2{k1}, [srcq+ssq*2]
+ add srcq, ss3q
+ vpbroadcastq m1{k2}, [srcq+ssq*0] ; _ _ 1 3
+ punpcklbw m3, m3
+ vpbroadcastq m2{k2}, [srcq+ssq*1] ; _ 0 2 4
+ psraw m3, 8 ; sign-extend
+ mova m6, [spel_hv_perm4a]
+ kshiftrb k1, k1, 2
+ movu m7, [spel_hv_perm4b]
+ pshufb m1, m10
+ mova m0, m8
+ vpdpbusd m0, m1, m11
+ pshufb m2, m10
+ mova m1, m8
+ vpdpbusd m1, m2, m11
+ pshufd m12, m3, q0000
+ pshufd m13, m3, q1111
+ pshufd m14, m3, q2222
+ packssdw m0, m1 ; _ _ _ 0 1 2 3 4
+ psraw m0, 2
+ vpermb m1, m7, m0 ; 01 12 23 34
+.hv_w4_loop:
+ movq xm3, [srcq+ssq*2]
+ movq xm4, [srcq+ss3q ]
+ lea srcq, [srcq+ssq*4]
+ vpbroadcastq ym3{k1}, [srcq+ssq*0] ; 5 7
+ vpbroadcastq ym4{k1}, [srcq+ssq*1] ; 6 8
+ pshufb ym3, ym10
+ mova ym2, ym8
+ vpdpbusd ym2, ym3, ym11
+ pshufb ym4, ym10
+ mova ym3, ym8
+ vpdpbusd ym3, ym4, ym11
+ mova m4, m9
+ vpdpwssd m4, m1, m12 ; a0 b0 c0 d0
+ packssdw ym2, ym3 ; 5 6 7 8
+ psraw ym2, 2
+ vshufi32x4 m0, m2, q1032 ; _ 2 3 4 5 6 7 8
+ vpermb m2, m6, m0 ; 23 34 45 56
+ vpermb m1, m7, m0 ; 45 56 67 78
+ vpdpwssd m4, m2, m13 ; a1 b1 c1 d1
+ vpdpwssd m4, m1, m14 ; a2 b2 c2 d2
+ psrad m4, 6
+ vpmovdw [tmpq], m4
+ add tmpq, 32
+ sub hd, 4
+ jg .hv_w4_loop
+ RET
+.hv_w8:
+ shr mxd, 16
+ vpbroadcastd m10, [base+subpel_filters+mxq*8+0]
+ vpbroadcastd m11, [base+subpel_filters+mxq*8+4]
+ movzx mxd, myb
+ shr myd, 16
+ cmp hd, 4
+ cmove myd, mxd
+ vpbroadcastq m0, [base+subpel_filters+1+myq*8]
+ lea r6, [ssq*2+3]
+ punpcklbw m0, m0
+ sub srcq, r6
+ psraw m0, 8 ; sign-extend
+ lea ss3q, [ssq*3]
+ pshufd m12, m0, q0000
+ pshufd m13, m0, q1111
+ pshufd m14, m0, q2222
+ cmp wd, 8
+ jg .hv_w16
+ movu xm16, [srcq+ssq*0]
+ vbroadcasti32x4 m19, [subpel_h_shufA]
+ vinserti128 ym16, [srcq+ssq*1], 1
+ vbroadcasti32x4 m21, [subpel_h_shufC]
+ vinserti32x4 m16, [srcq+ssq*2], 2
+ add srcq, ss3q
+ vinserti32x4 m16, [srcq+ssq*0], 3
+ movu xm17, [srcq+ssq*1]
+ vbroadcasti32x4 m20, [subpel_h_shufB]
+ pshufb m3, m16, m19 ; 0 1 2 3 0123
+ mova m2, m8
+ pshufb m0, m16, m21 ; 0 1 2 3 89ab
+ vpdpbusd m2, m3, m10
+ mova m3, m8
+ pshufb xm1, xm17, xm19 ; 3 4 5 6 0123
+ vpdpbusd m3, m0, m11
+ mova xm0, xm8
+ pshufb xm18, xm17, xm21 ; 3 4 5 6 89ab
+ vpdpbusd xm0, xm1, xm10
+ mova xm1, xm8
+ pshufb m16, m20 ; 0 1 2 3 4567
+ vpdpbusd xm1, xm18, xm11
+ pshufb xm17, xm20 ; 3 4 5 6 4567
+ vpdpbusd m2, m16, m11
+ vpdpbusd m3, m16, m10
+ vpdpbusd xm0, xm17, xm11
+ vpdpbusd xm1, xm17, xm10
+ packssdw m2, m3
+ packssdw xm0, xm1
+ psraw m2, 2 ; 0 1 2 3
+ psraw xm0, 2 ; 4
+ valignq m0, m2, 2 ; 1 2 3 4
+ punpcklwd m1, m2, m0 ; 01 12 23 34
+ punpckhwd m2, m0
+.hv_w8_loop:
+ movu xm16, [srcq+ssq*2]
+ vinserti128 ym16, [srcq+ss3q ], 1
+ lea srcq, [srcq+ssq*4]
+ vinserti32x4 m16, [srcq+ssq*0], 2
+ vinserti32x4 m16, [srcq+ssq*1], 3
+ pshufb m6, m16, m19 ; 5 6 7 8 0123
+ mova m5, m8
+ pshufb m3, m16, m21 ; 5 6 7 8 89ab
+ vpdpbusd m5, m6, m10
+ mova m6, m8
+ pshufb m16, m20 ; 5 6 7 8 4567
+ vpdpbusd m6, m3, m11
+ mova m3, m9
+ vpdpwssd m3, m1, m12 ; a0 b0 c0 d0
+ mova m4, m9
+ vpdpwssd m4, m2, m12
+ vpdpbusd m5, m16, m11
+ vpdpbusd m6, m16, m10
+ mova m16, m1
+ packssdw m5, m6
+ mova m6, m2
+ psraw m5, 2 ; 5 6 7 8
+ valignq m2, m5, m0, 6 ; 4 5 6 7
+ mova m0, m5
+ punpcklwd m1, m2, m5 ; 45 56 67 78
+ punpckhwd m2, m5
+ vpdpwssd m3, m1, m14 ; a2 b2 c2 d2
+ vpdpwssd m4, m2, m14
+ vshufi32x4 m16, m1, q1032 ; 23 34 45 56
+ vshufi32x4 m6, m2, q1032
+ vpdpwssd m3, m16, m13 ; a1 b1 c1 d1
+ vpdpwssd m4, m6, m13
+ psrad m3, 6
+ psrad m4, 6
+ packssdw m3, m4
+ mova [tmpq], m3
+ add tmpq, 64
+ sub hd, 4
+ jg .hv_w8_loop
+ vzeroupper
+ RET
+.hv_w16:
+ mova m16, [spel_h_perm16]
+ vpbroadcastd m18, [pb_4]
+ add wd, wd
+ paddb m17, m18, m16
+ lea r6d, [hq+wq*8-256]
+ paddb m18, m17
+.hv_w16_loop0:
+ movu ym19, [srcq+ssq*0]
+ vinserti32x8 m19, [srcq+ssq*1], 1
+ lea r5, [srcq+ssq*2]
+ movu ym20, [r5 +ssq*0]
+ vinserti32x8 m20, [r5 +ssq*1], 1
+ lea r5, [r5 +ssq*2]
+ movu ym21, [r5 +ssq*0]
+ mov r7, tmpq
+ vpermb m3, m16, m19 ; 0 1 0123 89ab
+ mova m2, m8
+ vpermb m4, m18, m19 ; 0 1 89ab ghij
+ vpdpbusd m2, m3, m10
+ mova m3, m8
+ vpermb m5, m16, m20 ; 2 3 0123 89ab
+ vpdpbusd m3, m4, m11
+ mova m4, m8
+ vpermb m0, m18, m20 ; 2 3 89ab ghij
+ vpdpbusd m4, m5, m10
+ mova m5, m8
+ vpermb ym1, ym16, ym21 ; 4 0123 89ab
+ vpdpbusd m5, m0, m11
+ mova ym0, ym8
+ vpermb ym6, ym18, ym21 ; 4 89ab ghij
+ vpdpbusd ym0, ym1, ym10
+ mova ym1, ym8
+ vpermb m19, m17, m19 ; 0 1 4567 cdef
+ vpdpbusd ym1, ym6, ym11
+ vpermb m20, m17, m20 ; 2 3 4567 cdef
+ vpdpbusd m2, m19, m11
+ vpdpbusd m3, m19, m10
+ vpermb ym21, ym17, ym21 ; 4 4567 cdef
+ vpdpbusd m4, m20, m11
+ vpdpbusd m5, m20, m10
+ vpdpbusd ym0, ym21, ym11
+ vpdpbusd ym1, ym21, ym10
+ packssdw m2, m3 ; 0 1
+ packssdw m4, m5 ; 2 3
+ packssdw ym0, ym1 ; 4
+ REPX {psraw x, 2}, m2, m4, ym0
+ vshufi32x4 m3, m2, m4, q1032 ; 1 2
+ vshufi32x4 m0, m4, m0, q1032 ; 3 4
+ punpcklwd m1, m2, m3 ; 01 12
+ punpckhwd m2, m3
+ punpcklwd m3, m4, m0 ; 23 34
+ punpckhwd m4, m0
+.hv_w16_loop:
+ movu ym19, [r5+ssq*1]
+ lea r5, [r5+ssq*2]
+ vinserti32x8 m19, [r5+ssq*0], 1
+ vpermb m6, m16, m19 ; 5 6 0123 89ab
+ mova m5, m8
+ vpermb m20, m18, m19 ; 5 6 89ab ghij
+ vpdpbusd m5, m6, m10
+ mova m6, m8
+ vpermb m19, m17, m19 ; 5 6 4567 cdef
+ vpdpbusd m6, m20, m11
+ mova m20, m9
+ vpdpwssd m20, m1, m12 ; a0 b0
+ mova m21, m9
+ vpdpwssd m21, m2, m12
+ vpdpbusd m5, m19, m11
+ vpdpbusd m6, m19, m10
+ vpdpwssd m20, m3, m13 ; a1 b1
+ vpdpwssd m21, m4, m13
+ packssdw m5, m6
+ mova m1, m3
+ psraw m5, 2 ; 5 6
+ mova m2, m4
+ vshufi32x4 m4, m0, m5, q1032 ; 4 5
+ mova m0, m5
+ punpcklwd m3, m4, m0 ; 45 56
+ punpckhwd m4, m0
+ vpdpwssd m20, m3, m14 ; a2 b2
+ vpdpwssd m21, m4, m14
+ psrad m20, 6
+ psrad m21, 6
+ packssdw m20, m21
+ mova [r7+wq*0], ym20
+ vextracti32x8 [r7+wq*1], m20, 1
+ lea r7, [r7+wq*2]
+ sub hd, 2
+ jg .hv_w16_loop
+ add srcq, 16
+ add tmpq, 32
movzx hd, r6b
- mov srcq, r4
- mov dstq, r7
sub r6d, 1<<8
jg .hv_w16_loop0
vzeroupper
@@ -2353,183 +3424,38 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
mova [tmpq+64*1], m1
%endmacro
-%if WIN64
-DECLARE_REG_TMP 6, 4
-%else
-DECLARE_REG_TMP 6, 7
-%endif
-
-%define PREP_8TAP_FN FN prep_8tap,
-
+PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP, prep_8tap_8bpc
+PREP_8TAP_FN regular_sharp, REGULAR, SHARP, prep_8tap_8bpc
PREP_8TAP_FN sharp, SHARP, SHARP
-PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH
-PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP
-PREP_8TAP_FN smooth, SMOOTH, SMOOTH
-PREP_8TAP_FN sharp_regular, SHARP, REGULAR
-PREP_8TAP_FN regular_sharp, REGULAR, SHARP
-PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR
-PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH
-PREP_8TAP_FN regular, REGULAR, REGULAR
-cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3
+cglobal prep_8tap_8bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my, stride3
imul mxd, mxm, 0x010101
add mxd, t0d ; 8tap_h, mx, 4tap_h
imul myd, mym, 0x010101
add myd, t1d ; 8tap_v, my, 4tap_v
lea r7, [prep_avx512icl]
- movsxd wq, wm
movifnidn hd, hm
test mxd, 0xf00
jnz .h
test myd, 0xf00
- jnz .v
- tzcnt wd, wd
- movzx wd, word [r7+wq*2+table_offset(prep,)]
- add wq, r7
- lea r6, [strideq*3]
-%if WIN64
- pop r7
-%endif
- jmp wq
-.h:
- test myd, 0xf00
- jnz .hv
- vpbroadcastd m4, [pd_2]
- WIN64_SPILL_XMM 10
- cmp wd, 4
- je .h_w4
- tzcnt wd, wd
- shr mxd, 16
- sub srcq, 3
- movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)]
- vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+0]
- vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep_avx512icl+4]
- add wq, r7
- jmp wq
-.h_w4:
- movzx mxd, mxb
- vbroadcasti128 ym5, [subpel_h_shufA]
- mov r3d, 0x4
- dec srcq
- vpbroadcastd ym6, [r7+mxq*8+subpel_filters-prep_avx512icl+2]
- kmovb k1, r3d
- lea stride3q, [strideq*3]
-.h_w4_loop:
- movq xm2, [srcq+strideq*0]
- movq xm3, [srcq+strideq*1]
- vpbroadcastq ym2{k1}, [srcq+strideq*2]
- vpbroadcastq ym3{k1}, [srcq+stride3q ]
- lea srcq, [srcq+strideq*4]
- pshufb ym2, ym5
- pshufb ym3, ym5
- mova ym0, ym4
- vpdpbusd ym0, ym2, ym6
- mova ym1, ym4
- vpdpbusd ym1, ym3, ym6
- packssdw ym0, ym1
- psraw ym0, 2
- mova [tmpq], ym0
- add tmpq, 32
- sub hd, 4
- jg .h_w4_loop
- RET
-.h_w8:
- vbroadcasti128 m5, [subpel_h_shufA]
- vbroadcasti128 m6, [subpel_h_shufB]
- vbroadcasti128 m7, [subpel_h_shufC]
- lea stride3q, [strideq*3]
-.h_w8_loop:
- movu xmm3, [srcq+strideq*0]
- vinserti128 ym3, ymm3, [srcq+strideq*1], 1
- vinserti128 m3, [srcq+strideq*2], 2
- vinserti128 m3, [srcq+stride3q ], 3
- lea srcq, [srcq+strideq*4]
- pshufb m1, m3, m5
- pshufb m2, m3, m6
- mova m0, m4
- vpdpbusd m0, m1, m8
- mova m1, m4
- vpdpbusd m1, m2, m8
- pshufb m3, m7
- vpdpbusd m0, m2, m9
- vpdpbusd m1, m3, m9
- packssdw m0, m1
- psraw m0, 2
- mova [tmpq], m0
- add tmpq, 64
- sub hd, 4
- jg .h_w8_loop
- RET
-.h_w16:
- mova m5, [spel_h_perm16a]
- mova m6, [spel_h_perm16b]
- mova m7, [spel_h_perm16c]
- lea stride3q, [strideq*3]
-.h_w16_loop:
- movu ym0, [srcq+strideq*0]
- movu ym1, [srcq+strideq*2]
- vinserti32x8 m0, [srcq+strideq*1], 1
- vinserti32x8 m1, [srcq+stride3q ], 1
- lea srcq, [srcq+strideq*4]
- PREP_8TAP_H
- add tmpq, 64*2
- sub hd, 4
- jg .h_w16_loop
- RET
-.h_w32:
- mova m5, [spel_h_perm32a]
- mova m6, [spel_h_perm32b]
- mova m7, [spel_h_perm32c]
-.h_w32_loop:
- movu m0, [srcq+strideq*0]
- movu m1, [srcq+strideq*1]
- lea srcq, [srcq+strideq*2]
- PREP_8TAP_H
- add tmpq, 64*2
- sub hd, 2
- jg .h_w32_loop
- RET
-.h_w64:
- xor r6d, r6d
- jmp .h_start
-.h_w128:
- mov r6, -64*1
-.h_start:
- mova m5, [spel_h_perm32a]
- mova m6, [spel_h_perm32b]
- mova m7, [spel_h_perm32c]
- sub srcq, r6
- mov r5, r6
-.h_loop:
- movu m0, [srcq+r6+32*0]
- movu m1, [srcq+r6+32*1]
- PREP_8TAP_H
- add tmpq, 64*2
- add r6, 64
- jle .h_loop
- add srcq, strideq
- mov r6, r5
- dec hd
- jg .h_loop
- RET
+ jz mangle(private_prefix %+ _prep_6tap_8bpc_avx512icl).prep
.v:
movzx mxd, myb ; Select 4-tap/8-tap filter multipliers.
shr myd, 16 ; Note that the code is 8-tap only, having
- tzcnt wd, wd
cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4
cmove myd, mxd ; had a negligible effect on performance.
- ; TODO: Would a 6-tap code path be worth it?
- lea myq, [r7+myq*8+subpel_filters-prep_avx512icl]
- movzx wd, word [r7+wq*2+table_offset(prep, _8tap_v)]
- add wq, r7
- lea stride3q, [strideq*3]
- sub srcq, stride3q
+ tzcnt r5d, wd
+ lea myq, [base+subpel_filters+myq*8]
+ movzx r5d, word [r7+r5*2+table_offset(prep, _8tap_v)]
vpbroadcastd m7, [pw_8192]
vpbroadcastw m8, [myq+0]
+ add r5, r7
vpbroadcastw m9, [myq+2]
+ lea stride3q, [strideq*3]
vpbroadcastw m10, [myq+4]
+ sub srcq, stride3q
vpbroadcastw m11, [myq+6]
- jmp wq
+ jmp r5
.v_w4:
movd xmm0, [srcq+strideq*0]
vpbroadcastd ymm1, [srcq+strideq*2]
@@ -2576,172 +3502,146 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3
vzeroupper
RET
.v_w8:
- mov r3d, 0xf044
- kmovw k1, r3d
- kshiftrw k2, k1, 8
- movq xm0, [srcq+strideq*0]
- vpbroadcastq ym1, [srcq+strideq*1]
- vpbroadcastq m2, [srcq+strideq*2]
- vpbroadcastq m3, [srcq+stride3q ]
+ mova m6, [spel_v_perm8]
+ movq xm1, [srcq+strideq*0]
+ mov r6d, 0x3e
+ movq xm2, [srcq+strideq*1]
+ vpbroadcastq ym3, [srcq+strideq*2]
+ kmovb k1, r6d
+ vpbroadcastq ym4, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
- vpbroadcastq m4, [srcq+strideq*0]
- vpbroadcastq m5, [srcq+strideq*1]
- vpbroadcastq m6, [srcq+strideq*2]
- vmovdqa64 ym0{k1}, ym1
- vmovdqa64 ym1{k1}, ym2
- vmovdqa64 m2{k1}, m3
- vmovdqa64 m3{k1}, m4
- vmovdqa64 m4{k1}, m5
- vmovdqa64 m5{k1}, m6
- punpcklbw ym0, ym1 ; 01 12 __ __
- punpcklbw m2, m3 ; 23 34 23 34
- punpcklbw m4, m5 ; 45 56 45 56
- vmovdqa64 m0{k2}, m2 ; 01 12 23 34
- vmovdqa64 m2{k2}, m4 ; 23 34 45 56
+ vpunpcklqdq m1{k1}, m3, [srcq+strideq*0] {1to8}
+ vpunpcklqdq m2{k1}, m4, [srcq+strideq*1] {1to8}
+ movq xm0, [srcq+strideq*2]
+ kshiftlb k2, k1, 2
+ shufpd m1, m2, 0x30 ; 0 1 2 3 4 5
+ vshufi32x4 m2, m1, m0, q0021 ; 2 3 4 5 6 _
+ vpermb m1, m6, m1 ; 01 12 23 34
+ vpermb m2, m6, m2 ; 23 34 45 56
.v_w8_loop:
- vpbroadcastq m1, [srcq+stride3q ]
+ vpbroadcastq ym3, [srcq+strideq*4]
+ vpunpcklqdq ym0{k1}, ym3, [srcq+stride3q] {1to4}
lea srcq, [srcq+strideq*4]
- vpbroadcastq m3, [srcq+strideq*0]
- vpbroadcastq m5, [srcq+strideq*1]
- pmaddubsw m14, m0, m8
- pmaddubsw m15, m2, m9
- vpblendmq m0{k1}, m6, m1
- vpblendmq m2{k1}, m1, m3
- vpbroadcastq m6, [srcq+strideq*2]
- paddw m14, m15
- punpcklbw m2, m0, m2 ; 67 78 67 78
- vpblendmq m12{k1}, m3, m5
- vpblendmq m13{k1}, m5, m6
- vpblendmq m0{k2}, m4, m2 ; 45 56 67 78
- punpcklbw m4, m12, m13 ; 89 9a 89 9a
- vmovdqa64 m2{k2}, m4 ; 67 78 89 9a
- pmaddubsw m12, m0, m10
- pmaddubsw m13, m2, m11
- paddw m14, m12
- paddw m14, m13
- pmulhrsw m14, m7
- mova [tmpq], m14
+ vpbroadcastq m3, [srcq+strideq*2]
+ vpunpcklqdq m0{k2}, m3, [srcq+strideq*1] {1to8}
+ pmaddubsw m4, m1, m8 ; a0 b0 c0 d0
+ mova m1, m2
+ pmaddubsw m5, m2, m9 ; a1 b1 c1 d1
+ vpermb m2, m6, m0 ; 67 78 89 9a
+ mova xm0, xm3
+ vshufi32x4 m1, m2, q1032 ; 45 56 67 78
+ pmaddubsw m3, m2, m11 ; a3 b3 c3 d3
+ paddw m4, m5
+ pmaddubsw m5, m1, m10 ; a2 b2 c2 d2
+ paddw m4, m3
+ paddw m4, m5
+ pmulhrsw m4, m7
+ mova [tmpq], m4
add tmpq, 64
sub hd, 4
jg .v_w8_loop
RET
.v_w16:
- mov r3d, 0xf0
- kmovb k1, r3d
- vbroadcasti128 m0, [srcq+strideq*0]
- vbroadcasti128 m1, [srcq+strideq*1]
- vbroadcasti128 m2, [srcq+strideq*2]
- vbroadcasti128 m3, [srcq+stride3q ]
+ mova m12, [spel_v_perm16b]
+ vbroadcasti32x4 m1, [srcq+strideq*0]
+ mov r6d, 0x0f
+ vbroadcasti32x4 ym4, [srcq+strideq*1]
+ vbroadcasti32x4 m2, [srcq+strideq*2]
+ kmovb k1, r6d
+ vbroadcasti32x4 ym5, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
- vbroadcasti128 m4, [srcq+strideq*0]
- vbroadcasti128 m5, [srcq+strideq*1]
- vbroadcasti128 m6, [srcq+strideq*2]
- vmovdqa64 m0{k1}, m1
- vmovdqa64 m1{k1}, m2
- vmovdqa64 m2{k1}, m3
- vmovdqa64 m3{k1}, m4
- vmovdqa64 m4{k1}, m5
- vmovdqa64 m5{k1}, m6
- shufpd m0, m2, 0xcc ; 0a_2a 0b_2b 1a_3a 1b_3b
- shufpd m1, m3, 0xcc ; 1a_3a 1b_3b 2a_4a 2b_4b
- shufpd m4, m4, 0x44 ; 4a_-- 4b_-- 5a_-- 5b_--
- shufpd m5, m5, 0x44 ; 5a_-- 5b_-- 6a_-- 6b_--
- punpckhbw m2, m0, m1 ; 23a 23b 34a 34b
- punpcklbw m0, m1 ; 01a 01b 12a 12b
- punpcklbw m4, m5 ; 45a 45b 56a 56b
+ vbroadcasti32x4 m3, [srcq+strideq*0]
+ vbroadcasti32x4 ym6, [srcq+strideq*1]
+ vbroadcasti32x4 m0, [srcq+strideq*2]
+ vshufpd m1{k1}, m4, m2, 0xcc
+ vshufpd m2{k1}, m5, m3, 0xcc
+ vshufpd m3{k1}, m6, m0, 0xcc
+ vpermb m1, m12, m1 ; 01 12
+ vpermb m2, m12, m2 ; 23 34
+ vpermb m3, m12, m3 ; 45 56
.v_w16_loop:
- vbroadcasti128 m3, [srcq+stride3q ]
+ pmaddubsw m4, m1, m8 ; a0 b0
+ mova m1, m3
+ pmaddubsw m13, m2, m9 ; a1 b1
+ vbroadcasti32x4 ym6, [srcq+stride3q ]
+ pmaddubsw m5, m2, m8 ; c0 d0
lea srcq, [srcq+strideq*4]
- vbroadcasti128 m5, [srcq+strideq*0]
- vpblendmq m1{k1}, m6, m3
- vmovdqa64 m3{k1}, m5
- pmaddubsw m12, m0, m8
- pmaddubsw m13, m2, m8
- pmaddubsw m14, m2, m9
- pmaddubsw m15, m4, m9
- pmaddubsw m0, m4, m10
- vbroadcasti128 m2, [srcq+strideq*1]
- vbroadcasti128 m6, [srcq+strideq*2]
- paddw m12, m14
- paddw m13, m15
- paddw m12, m0
- vmovdqa64 m5{k1}, m2
- vmovdqa64 m2{k1}, m6
- mova m0, m4
- shufpd m1, m5, 0xcc ; 6a_8a 6b_8b 7a_9a 7b_9b
- shufpd m3, m2, 0xcc ; 7a_9a 7b_9b 8a_Aa 8b_Ab
- punpcklbw m2, m1, m3 ; 67a 67b 78a 78b
- punpckhbw m4, m1, m3 ; 89a 89b 9Aa 9Ab
- pmaddubsw m14, m2, m10
- pmaddubsw m15, m2, m11
- paddw m13, m14
- paddw m12, m15
- pmaddubsw m14, m4, m11
- paddw m13, m14
- pmulhrsw m12, m7
- pmulhrsw m13, m7
- mova [tmpq+ 0], m12
- mova [tmpq+64], m13
+ pmaddubsw m14, m3, m9 ; c1 d1
+ vbroadcasti32x4 m3, [srcq+strideq*0]
+ vshufpd m0{k1}, m6, m3, 0xcc
+ vbroadcasti32x4 ym6, [srcq+strideq*1]
+ vpermb m2, m12, m0 ; 67 78
+ vbroadcasti32x4 m0, [srcq+strideq*2]
+ vshufpd m3{k1}, m6, m0, 0xcc
+ paddw m4, m13
+ pmaddubsw m13, m1, m10 ; a2 b2
+ vpermb m3, m12, m3 ; 89 9a
+ paddw m5, m14
+ pmaddubsw m14, m2, m10 ; c2 d2
+ pmaddubsw m15, m2, m11 ; a3 b3
+ pmaddubsw m6, m3, m11 ; c3 d3
+ paddw m4, m13
+ paddw m5, m14
+ paddw m4, m15
+ paddw m5, m6
+ pmulhrsw m4, m7
+ pmulhrsw m5, m7
+ mova [tmpq+ 0], m4
+ mova [tmpq+64], m5
add tmpq, 64*2
sub hd, 4
jg .v_w16_loop
RET
.v_w32:
- mova m18, [bilin_v_perm64]
- movu ym0, [srcq+strideq*0]
- movu ym1, [srcq+strideq*1]
- lea srcq, [srcq+strideq*2]
- movu ym2, [srcq+strideq*0]
- movu ym3, [srcq+strideq*1]
- lea srcq, [srcq+strideq*2]
- movu ym4, [srcq+strideq*0]
- movu ym5, [srcq+strideq*1]
- lea srcq, [srcq+strideq*2]
- movu ym6, [srcq+strideq*0]
- vpermq m0, m18, m0
- vpermq m1, m18, m1
- vpermq m2, m18, m2
- vpermq m3, m18, m3
- vpermq m4, m18, m4
- vpermq m5, m18, m5
- vpermq m6, m18, m6
- punpcklbw m0, m1
- punpcklbw m1, m2
- punpcklbw m2, m3
- punpcklbw m3, m4
- punpcklbw m4, m5
- punpcklbw m5, m6
+ movshdup m21, [bilin_v_perm64]
+ movu ym16, [srcq+strideq*0]
+ movu ym17, [srcq+strideq*1]
+ movu ym18, [srcq+strideq*2]
+ add srcq, stride3q
+ movu ym19, [srcq+strideq*0]
+ vpermt2q m16, m21, m19 ; 0 3
+ movu ym20, [srcq+strideq*1]
+ vpermt2q m17, m21, m20 ; 1 4
+ movu ym20, [srcq+strideq*2]
+ add srcq, stride3q
+ vpermt2q m18, m21, m20 ; 2 5
+ movu ym20, [srcq+strideq*0]
+ vpermt2q m19, m21, m20 ; 3 6
+ punpcklbw m0, m16, m17 ; 01
+ punpcklbw m1, m17, m18 ; 12
+ punpcklbw m2, m18, m19 ; 23
+ punpckhbw m3, m16, m17 ; 34
+ punpckhbw m4, m17, m18 ; 45
+ punpckhbw m5, m18, m19 ; 56
.v_w32_loop:
- movu ym12, [srcq+strideq*1]
+ movu ym16, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
- movu ym13, [srcq+strideq*0]
+ movu ym17, [srcq+strideq*0]
pmaddubsw m14, m0, m8
- pmaddubsw m16, m2, m9
- pmaddubsw m15, m1, m8
- pmaddubsw m17, m3, m9
mova m0, m2
+ pmaddubsw m15, m1, m8
mova m1, m3
- vpermq m12, m18, m12
- vpermq m13, m18, m13
- paddw m14, m16
- paddw m15, m17
- pmaddubsw m16, m4, m10
- pmaddubsw m17, m5, m10
- punpcklbw m6, m12
- punpcklbw m12, m13
+ pmaddubsw m2, m9
+ vpermt2q m16, m21, m17 ; 7 8
+ pmaddubsw m3, m9
+ pmaddubsw m12, m4, m10
+ pmaddubsw m13, m5, m10
+ shufpd m19, m16, 0x55 ; 6 7
+ paddw m14, m2
mova m2, m4
+ punpcklbw m4, m19, m16 ; 67
+ paddw m15, m3
mova m3, m5
- paddw m14, m16
- paddw m15, m17
- pmaddubsw m16, m6, m11
- pmaddubsw m17, m12, m11
- mova m4, m6
- mova m5, m12
- paddw m14, m16
- paddw m15, m17
+ punpckhbw m5, m19, m16 ; 78
+ paddw m14, m12
+ paddw m15, m13
+ pmaddubsw m12, m4, m11
+ pmaddubsw m13, m5, m11
+ mova m19, m16
+ paddw m14, m12
+ paddw m15, m13
pmulhrsw m14, m7
pmulhrsw m15, m7
- mova m6, m13
mova [tmpq+ 0], m14
mova [tmpq+64], m15
add tmpq, 64*2
@@ -2750,154 +3650,241 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3
vzeroupper
RET
.v_w64:
- mov wd, 64
- jmp .v_start
.v_w128:
- mov wd, 128
-.v_start:
- WIN64_SPILL_XMM 27
- mova m26, [bilin_v_perm64]
- lea r6d, [hq+wq*2]
- mov r5, srcq
- mov r7, tmpq
+ WIN64_SPILL_XMM 24
+ mova m23, [bilin_v_perm64]
+ add wd, wd
+ lea r6d, [hq+wq]
.v_loop0:
- vpermq m0, m26, [srcq+strideq*0]
- vpermq m1, m26, [srcq+strideq*1]
- lea srcq, [srcq+strideq*2]
- vpermq m2, m26, [srcq+strideq*0]
- vpermq m3, m26, [srcq+strideq*1]
- lea srcq, [srcq+strideq*2]
- vpermq m4, m26, [srcq+strideq*0]
- vpermq m5, m26, [srcq+strideq*1]
- lea srcq, [srcq+strideq*2]
- vpermq m6, m26, [srcq+strideq*0]
- punpckhbw m12, m0, m1
- punpcklbw m0, m1
- punpckhbw m13, m1, m2
- punpcklbw m1, m2
- punpckhbw m14, m2, m3
- punpcklbw m2, m3
- punpckhbw m15, m3, m4
- punpcklbw m3, m4
- punpckhbw m16, m4, m5
- punpcklbw m4, m5
- punpckhbw m17, m5, m6
- punpcklbw m5, m6
+ vpermq m12, m23, [srcq+strideq*0]
+ vpermq m13, m23, [srcq+strideq*1]
+ lea r5, [srcq+strideq*2]
+ vpermq m14, m23, [r5 +strideq*0]
+ vpermq m15, m23, [r5 +strideq*1]
+ lea r5, [r5+strideq*2]
+ vpermq m16, m23, [r5 +strideq*0]
+ vpermq m17, m23, [r5 +strideq*1]
+ lea r5, [r5+strideq*2]
+ vpermq m18, m23, [r5 +strideq*0]
+ mov r7, tmpq
+ punpcklbw m0, m12, m13 ; 01
+ punpckhbw m12, m13
+ punpcklbw m1, m13, m14 ; 12
+ punpckhbw m13, m14
+ punpcklbw m2, m14, m15 ; 23
+ punpckhbw m14, m15
+ punpcklbw m3, m15, m16 ; 34
+ punpckhbw m15, m16
+ punpcklbw m4, m16, m17 ; 45
+ punpckhbw m16, m17
+ punpcklbw m5, m17, m18 ; 56
+ punpckhbw m17, m18
.v_loop:
- vpermq m18, m26, [srcq+strideq*1]
- lea srcq, [srcq+strideq*2]
- vpermq m19, m26, [srcq+strideq*0]
- pmaddubsw m20, m0, m8
- pmaddubsw m21, m12, m8
- pmaddubsw m22, m1, m8
- pmaddubsw m23, m13, m8
+ pmaddubsw m19, m0, m8 ; a0
+ vpermq m6, m23, [r5+strideq*1]
+ pmaddubsw m20, m12, m8
mova m0, m2
+ pmaddubsw m2, m9 ; a1
mova m12, m14
+ pmaddubsw m14, m9
+ lea r5, [r5+strideq*2]
+ pmaddubsw m21, m1, m8 ; b0
+ pmaddubsw m22, m13, m8
mova m1, m3
+ pmaddubsw m3, m9 ; b1
mova m13, m15
- pmaddubsw m2, m9
- pmaddubsw m14, m9
- pmaddubsw m3, m9
pmaddubsw m15, m9
- punpckhbw m24, m6, m18
- punpcklbw m6, m18
- paddw m20, m2
- paddw m21, m14
- paddw m22, m3
- paddw m23, m15
+ paddw m19, m2
mova m2, m4
+ pmaddubsw m4, m10 ; a2
+ paddw m20, m14
mova m14, m16
+ pmaddubsw m16, m10
+ paddw m21, m3
mova m3, m5
+ pmaddubsw m5, m10 ; b2
+ paddw m22, m15
mova m15, m17
- pmaddubsw m4, m10
- pmaddubsw m16, m10
- pmaddubsw m5, m10
pmaddubsw m17, m10
- punpckhbw m25, m18, m19
- punpcklbw m18, m19
- paddw m20, m4
- paddw m21, m16
- paddw m22, m5
- paddw m23, m17
- mova m4, m6
- mova m16, m24
- mova m5, m18
- mova m17, m25
- pmaddubsw m6, m11
- pmaddubsw m24, m11
- pmaddubsw m18, m11
- pmaddubsw m25, m11
- paddw m20, m6
- paddw m21, m24
- paddw m22, m18
- paddw m23, m25
- pmulhrsw m20, m7
- pmulhrsw m21, m7
- pmulhrsw m22, m7
- pmulhrsw m23, m7
- mova m6, m19
- mova [tmpq+wq*0+ 0], m20
- mova [tmpq+wq*0+64], m21
- mova [tmpq+wq*2+ 0], m22
- mova [tmpq+wq*2+64], m23
- lea tmpq, [tmpq+wq*4]
+ paddw m19, m4
+ punpcklbw m4, m18, m6 ; 67
+ paddw m20, m16
+ punpckhbw m16, m18, m6
+ vpermq m18, m23, [r5+strideq*0]
+ paddw m21, m5
+ pmaddubsw m5, m4, m11 ; a3
+ paddw m22, m17
+ pmaddubsw m17, m16, m11
+ paddw m19, m5
+ punpcklbw m5, m6, m18 ; 78
+ paddw m20, m17
+ punpckhbw m17, m6, m18
+ pmaddubsw m6, m5, m11 ; b3
+ paddw m21, m6
+ pmaddubsw m6, m17, m11
+ paddw m22, m6
+ REPX {pmulhrsw x, m7}, m19, m20, m21, m22
+ mova [r7+wq*0+ 0], m19
+ mova [r7+wq*0+64], m20
+ mova [r7+wq*1+ 0], m21
+ mova [r7+wq*1+64], m22
+ lea r7, [r7+wq*2]
sub hd, 2
jg .v_loop
- add r5, 64
- add r7, 128
+ add srcq, 64
+ add tmpq, 128
movzx hd, r6b
- mov srcq, r5
- mov tmpq, r7
sub r6d, 1<<8
jg .v_loop0
RET
-.hv:
- WIN64_SPILL_XMM 16
+.h:
+ RESET_STACK_STATE
+ test myd, 0xf00
+ jnz .hv
+.h2:
+ vpbroadcastd m4, [pd_2]
cmp wd, 4
- je .hv_w4
+ je .h_w4
+ tzcnt wd, wd
shr mxd, 16
sub srcq, 3
- vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep_avx512icl+0]
- vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep_avx512icl+4]
- movzx mxd, myb
- shr myd, 16
- cmp hd, 4
- cmove myd, mxd
- tzcnt wd, wd
- vpbroadcastd m8, [pd_2]
- movzx wd, word [r7+wq*2+table_offset(prep, _8tap_hv)]
- vpbroadcastd m9, [pd_32]
+ movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)]
+ vpbroadcastd m8, [base+subpel_filters+mxq*8+0]
+ vpbroadcastd m9, [base+subpel_filters+mxq*8+4]
add wq, r7
- vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl]
- lea stride3q, [strideq*3]
- sub srcq, stride3q
- punpcklbw m0, m0
- psraw m0, 8 ; sign-extend
- pshufd m12, m0, q0000
- pshufd m13, m0, q1111
- pshufd m14, m0, q2222
- pshufd m15, m0, q3333
jmp wq
-.hv_w4:
+.h_w4:
+ movzx mxd, mxb
+ vbroadcasti128 ym5, [subpel_h_shufA]
+ mov r3d, 0x4
+ dec srcq
+ vpbroadcastd ym6, [base+subpel_filters+mxq*8+2]
+ kmovb k1, r3d
+ lea stride3q, [strideq*3]
+.h_w4_loop:
+ movq xm2, [srcq+strideq*0]
+ movq xm3, [srcq+strideq*1]
+ vpbroadcastq ym2{k1}, [srcq+strideq*2]
+ vpbroadcastq ym3{k1}, [srcq+stride3q ]
+ lea srcq, [srcq+strideq*4]
+ pshufb ym2, ym5
+ pshufb ym3, ym5
+ mova ym0, ym4
+ vpdpbusd ym0, ym2, ym6
+ mova ym1, ym4
+ vpdpbusd ym1, ym3, ym6
+ packssdw ym0, ym1
+ psraw ym0, 2
+ mova [tmpq], ym0
+ add tmpq, 32
+ sub hd, 4
+ jg .h_w4_loop
+ RET
+.h_w8:
+ vbroadcasti128 m5, [subpel_h_shufA]
+ vbroadcasti128 m6, [subpel_h_shufB]
+ vbroadcasti128 m7, [subpel_h_shufC]
+ lea stride3q, [strideq*3]
+.h_w8_loop:
+ movu xmm3, [srcq+strideq*0]
+ vinserti128 ym3, ymm3, [srcq+strideq*1], 1
+ vinserti128 m3, [srcq+strideq*2], 2
+ vinserti128 m3, [srcq+stride3q ], 3
+ lea srcq, [srcq+strideq*4]
+ pshufb m1, m3, m5
+ pshufb m2, m3, m6
+ mova m0, m4
+ vpdpbusd m0, m1, m8
+ mova m1, m4
+ vpdpbusd m1, m2, m8
+ pshufb m3, m7
+ vpdpbusd m0, m2, m9
+ vpdpbusd m1, m3, m9
+ packssdw m0, m1
+ psraw m0, 2
+ mova [tmpq], m0
+ add tmpq, 64
+ sub hd, 4
+ jg .h_w8_loop
+ RET
+.h_w16:
+ mova m5, [spel_h_perm16]
+ vpbroadcastd m7, [pb_4]
+ lea stride3q, [strideq*3]
+ paddb m6, m7, m5
+ paddb m7, m6
+.h_w16_loop:
+ movu ym0, [srcq+strideq*0]
+ movu ym1, [srcq+strideq*2]
+ vinserti32x8 m0, [srcq+strideq*1], 1
+ vinserti32x8 m1, [srcq+stride3q ], 1
+ lea srcq, [srcq+strideq*4]
+ PREP_8TAP_H
+ add tmpq, 64*2
+ sub hd, 4
+ jg .h_w16_loop
+ RET
+.h_w32:
+ mova m5, [spel_h_perm32]
+ vpbroadcastd m7, [pb_4]
+ paddb m6, m7, m5
+ paddb m7, m6
+.h_w32_loop:
+ movu m0, [srcq+strideq*0]
+ movu m1, [srcq+strideq*1]
+ lea srcq, [srcq+strideq*2]
+ PREP_8TAP_H
+ add tmpq, 64*2
+ sub hd, 2
+ jg .h_w32_loop
+ RET
+.h_w64:
+ xor r6d, r6d
+ jmp .h_start
+.h_w128:
+ mov r6, -64*1
+.h_start:
+ mova m5, [spel_h_perm32]
+ vpbroadcastd m7, [pb_4]
+ sub srcq, r6
+ paddb m6, m7, m5
+ paddb m7, m6
+.h_loop0:
+ mov r5, r6
+.h_loop:
+ movu m0, [srcq+r5+32*0]
+ movu m1, [srcq+r5+32*1]
+ PREP_8TAP_H
+ add tmpq, 64*2
+ add r5, 64
+ jle .h_loop
+ add srcq, strideq
+ dec hd
+ jg .h_loop0
+ RET
+.hv:
+ RESET_STACK_STATE
+ vpbroadcastd m8, [pd_2]
+ vpbroadcastd m9, [pd_32]
+ cmp wd, 4
+ jg .hv_w8
movzx mxd, mxb
dec srcq
- vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+2]
+ vpbroadcastd m11, [base+subpel_filters+mxq*8+2]
movzx mxd, myb
shr myd, 16
cmp hd, 4
cmove myd, mxd
- vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl]
+ vpbroadcastq m0, [base+subpel_filters+myq*8]
lea stride3q, [strideq*3]
sub srcq, stride3q
mov r3d, 0x04
kmovb k1, r3d
kshiftlb k2, k1, 2
kshiftlb k3, k1, 4
- vpbroadcastd m10, [pd_2]
- vbroadcasti128 m16, [subpel_h_shufA]
+ vbroadcasti128 m10, [subpel_h_shufA]
punpcklbw m0, m0
psraw m0, 8 ; sign-extend
- vpbroadcastd m11, [pd_32]
pshufd m12, m0, q0000
pshufd m13, m0, q1111
pshufd m14, m0, q2222
@@ -2910,263 +3897,265 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3
vpbroadcastq m3{k2}, [srcq+strideq*0]
vpbroadcastq m2{k3}, [srcq+strideq*1]
vpbroadcastq m3{k3}, [srcq+strideq*2]
- mova m17, [spel_hv_perm4a]
- movu m18, [spel_hv_perm4b]
- mova m0, m10
- mova m1, m10
- pshufb m2, m16
- pshufb m3, m16
- vpdpbusd m0, m2, m8
- vpdpbusd m1, m3, m8
+ mova m6, [spel_hv_perm4a]
+ movu m7, [spel_hv_perm4b]
+ mova m0, m8
+ mova m1, m8
+ pshufb m2, m10
+ pshufb m3, m10
+ vpdpbusd m0, m2, m11
+ vpdpbusd m1, m3, m11
packssdw m0, m1 ; _ 0 1 2 3 4 5 6
psraw m0, 2
- vpermb m1, m17, m0 ; 01 12 23 34
- vpermb m2, m18, m0 ; 23 34 45 56
+ vpermb m1, m6, m0 ; 01 12 23 34
+ vpermb m2, m7, m0 ; 23 34 45 56
.hv_w4_loop:
movq xm3, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
movq xm4, [srcq+strideq*0]
vpbroadcastq ym3{k1}, [srcq+strideq*1]
vpbroadcastq ym4{k1}, [srcq+strideq*2]
- mova ym5, ym10
- mova ym6, ym10
- pshufb ym3, ym16
- pshufb ym4, ym16
- vpdpbusd ym5, ym3, ym8
- vpdpbusd ym6, ym4, ym8
- mova m7, m11
- packssdw ym5, ym6 ; 7 8 9 a _ _ _ _
- psraw ym5, 2
- valignq m0, m5, m0, 4 ; _ 4 5 6 7 8 9 a
- vpdpwssd m7, m1, m12
- vpdpwssd m7, m2, m13
- vpermb m1, m17, m0 ; 45 56 67 78
- vpermb m2, m18, m0 ; 67 78 89 9a
- vpdpwssd m7, m1, m14
- vpdpwssd m7, m2, m15
- psrad m7, 6
- vpmovdw [tmpq], m7
+ mova m5, m9
+ pshufb ym3, ym10
+ vpdpwssd m5, m1, m12 ; a0 b0 c0 d0
+ mova ym1, ym8
+ pshufb ym4, ym10
+ vpdpbusd ym1, ym3, ym11
+ mova ym3, ym8
+ vpdpbusd ym3, ym4, ym11
+ vpdpwssd m5, m2, m13 ; a1 b1 c1 d1
+ packssdw ym1, ym3 ; 7 8 9 a
+ psraw ym1, 2
+ vshufi32x4 m0, m1, q1032 ; _ 4 5 6 7 8 9 a
+ vpermb m1, m6, m0 ; 45 56 67 78
+ vpermb m2, m7, m0 ; 67 78 89 9a
+ vpdpwssd m5, m1, m14 ; a2 b2 c2 d2
+ vpdpwssd m5, m2, m15 ; a3 b3 c3 d3
+ psrad m5, 6
+ vpmovdw [tmpq], m5
add tmpq, 32
sub hd, 4
jg .hv_w4_loop
- vzeroupper
RET
.hv_w8:
- WIN64_SPILL_XMM 24
- vbroadcasti128 m16, [subpel_h_shufA]
- vbroadcasti128 m17, [subpel_h_shufB]
- vbroadcasti128 m18, [subpel_h_shufC]
- vinserti128 ym0, [srcq+strideq*0], 1
- vinserti128 m0, [srcq+strideq*1], 2
- vinserti128 m0, [srcq+strideq*2], 3
- movu xm1, [srcq+stride3q ]
+ shr mxd, 16
+ sub srcq, 3
+ vpbroadcastd m10, [base+subpel_filters+mxq*8+0]
+ vpbroadcastd m11, [base+subpel_filters+mxq*8+4]
+ movzx mxd, myb
+ shr myd, 16
+ cmp hd, 4
+ cmove myd, mxd
+ vpbroadcastq m0, [base+subpel_filters+myq*8]
+ lea stride3q, [strideq*3]
+ sub srcq, stride3q
+ punpcklbw m0, m0
+ psraw m0, 8 ; sign-extend
+ pshufd m12, m0, q0000
+ pshufd m13, m0, q1111
+ pshufd m14, m0, q2222
+ pshufd m15, m0, q3333
+ cmp wd, 8
+ jg .hv_w16
+ vbroadcasti32x4 m17, [srcq+stride3q ]
+ vinserti32x4 m16, m17, [srcq+strideq*0], 0
+ vbroadcasti32x4 m19, [subpel_h_shufA]
+ vinserti32x4 m16, [srcq+strideq*1], 1
+ vbroadcasti32x4 m21, [subpel_h_shufC]
+ vinserti32x4 m16, [srcq+strideq*2], 2
lea srcq, [srcq+strideq*4]
- vinserti128 ym1, [srcq+strideq*0], 1
- vinserti128 m1, [srcq+strideq*1], 2
- vinserti128 m1, [srcq+strideq*2], 3
+ vinserti128 ym17, [srcq+strideq*0], 1
+ vbroadcasti32x4 m20, [subpel_h_shufB]
+ vinserti32x4 m17, [srcq+strideq*1], 2
+ vinserti32x4 m17, [srcq+strideq*2], 3
+ pshufb m3, m16, m19 ; 0 1 2 3 0123
mova m2, m8
- mova m4, m8
+ pshufb m0, m16, m21 ; 0 1 2 3 89ab
+ vpdpbusd m2, m3, m10
mova m3, m8
- mova m5, m8
- pshufb m20, m0, m16
- pshufb m21, m0, m17
- pshufb m22, m0, m18
- pshufb m23, m1, m16
- pshufb m6, m1, m17
- pshufb m7, m1, m18
- vpdpbusd m2, m20, m10
- vpdpbusd m4, m21, m10
- vpdpbusd m2, m21, m11
- vpdpbusd m4, m22, m11
- vpdpbusd m3, m23, m10
- vpdpbusd m5, m6, m10
- vpdpbusd m3, m6, m11
- vpdpbusd m5, m7, m11
- packssdw m2, m4
- packssdw m3, m5
- psraw m2, 2 ; _ 0 1 2
- psraw m3, 2 ; 3 4 5 6
- valignq m0, m3, m2, 2 ; 0 1 2 3
- valignq m1, m3, m2, 4 ; 1 2 3 4
- valignq m2, m3, m2, 6 ; 2 3 4 5
- punpcklwd m4, m0, m1 ; 01a 12a 23a 34a
- punpckhwd m5, m0, m1 ; 01b 12b 23b 34b
- punpcklwd m6, m2, m3 ; 23a 34a 45a 56a
- punpckhwd m7, m2, m3 ; 23b 34b 45b 56b
+ pshufb m1, m17, m19 ; 3 4 5 6 0123
+ vpdpbusd m3, m0, m11
+ mova m0, m8
+ pshufb m4, m17, m21 ; 3 4 5 6 89ab
+ vpdpbusd m0, m1, m10
+ mova m1, m8
+ pshufb m16, m20 ; 0 1 2 3 4567
+ vpdpbusd m1, m4, m11
+ pshufb m17, m20 ; 3 4 5 6 4567
+ vpdpbusd m2, m16, m11
+ vpdpbusd m3, m16, m10
+ vpdpbusd m0, m17, m11
+ vpdpbusd m1, m17, m10
+ packssdw m2, m3
+ packssdw m0, m1
+ psraw m2, 2 ; 0 1 2 3
+ psraw m0, 2 ; 3 4 5 6
+ vshufi32x4 m4, m2, m0, q2132 ; 2 3 4 5
+ vshufi32x4 m5, m2, m0, q1021 ; 1 2 3 4
+ punpcklwd m3, m4, m0 ; 23 34 45 56
+ punpckhwd m4, m0
+ punpcklwd m1, m2, m5 ; 01 12 23 34
+ punpckhwd m2, m5
.hv_w8_loop:
- movu xm19, [srcq+stride3q ]
+ movu xm18, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
- vinserti128 ym19, [srcq+strideq*0], 1
- vinserti128 m19, [srcq+strideq*1], 2
- vinserti128 m19, [srcq+strideq*2], 3
- mova m20, m9
- mova m21, m9
- mova m22, m8
- mova m23, m8
- vpdpwssd m20, m4, m12
- vpdpwssd m21, m5, m12
- vpdpwssd m20, m6, m13
- vpdpwssd m21, m7, m13
- pshufb m0, m19, m16
- pshufb m1, m19, m17
- pshufb m2, m19, m18
- vpdpbusd m22, m0, m10
- vpdpbusd m23, m1, m10
- vpdpbusd m22, m1, m11
- vpdpbusd m23, m2, m11
- packssdw m22, m23
- psraw m22, 2 ; 7 8 9 A
- valignq m0, m22, m3, 2 ; 4 5 6 7
- valignq m1, m22, m3, 4 ; 5 6 7 8
- valignq m2, m22, m3, 6 ; 6 7 8 9
- mova m3, m22
- punpcklwd m4, m0, m1 ; 45a 56a 67a 78a
- punpckhwd m5, m0, m1 ; 45b 56b 67b 78b
- punpcklwd m6, m2, m3 ; 67a 78a 89a 9Aa
- punpckhwd m7, m2, m3 ; 67b 78b 89b 9Ab
- vpdpwssd m20, m4, m14
- vpdpwssd m21, m5, m14
- vpdpwssd m20, m6, m15
- vpdpwssd m21, m7, m15
- psrad m20, 6
- psrad m21, 6
- packssdw m20, m21
- mova [tmpq], m20
+ vinserti128 ym18, [srcq+strideq*0], 1
+ vinserti32x4 m18, [srcq+strideq*1], 2
+ vinserti32x4 m18, [srcq+strideq*2], 3
+ pshufb m17, m18, m19 ; 7 8 9 a 0123
+ mova m16, m8
+ pshufb m5, m18, m21 ; 7 8 9 a 89ab
+ vpdpbusd m16, m17, m10
+ mova m17, m8
+ pshufb m18, m20 ; 7 8 9 a 4567
+ vpdpbusd m17, m5, m11
+ mova m5, m9
+ vpdpwssd m5, m3, m13 ; a1 b1 c1 d1
+ mova m6, m9
+ vpdpwssd m6, m4, m13
+ vpdpbusd m16, m18, m11
+ vpdpbusd m17, m18, m10
+ vpdpwssd m5, m1, m12 ; a0 b0 c0 d0
+ mova m1, m3
+ vpdpwssd m6, m2, m12
+ mova m2, m4
+ packssdw m16, m17
+ psraw m16, 2 ; 7 8 9 a
+ valignq m4, m16, m0, 6 ; 6 7 8 9
+ mova m0, m16
+ punpcklwd m3, m4, m16 ; 67 78 89 9a
+ punpckhwd m4, m16
+ vpdpwssd m5, m3, m15 ; a3 b3 c3 d3
+ vpdpwssd m6, m4, m15
+ vshufi32x4 m1, m3, q1032 ; 45 56 67 78
+ vshufi32x4 m2, m4, q1032
+ vpdpwssd m5, m1, m14 ; a2 b2 c2 d2
+ vpdpwssd m6, m2, m14
+ psrad m5, 6
+ psrad m6, 6
+ packssdw m5, m6
+ mova [tmpq], m5
add tmpq, 64
sub hd, 4
jg .hv_w8_loop
+ vzeroupper
RET
.hv_w16:
- mov wd, 16*2
- jmp .hv_start
-.hv_w32:
- mov wd, 32*2
- jmp .hv_start
-.hv_w64:
- mov wd, 64*2
- jmp .hv_start
-.hv_w128:
- mov wd, 128*2
-.hv_start:
- WIN64_SPILL_XMM 31
- mova m16, [spel_h_perm16a]
- mova m17, [spel_h_perm16b]
- mova m18, [spel_h_perm16c]
+ WIN64_SPILL_XMM 23
+ mova m16, [spel_h_perm16]
+ vpbroadcastd m18, [pb_4]
+ add wd, wd
+ paddb m17, m18, m16
lea r6d, [hq+wq*8-256]
- mov r5, srcq
+ paddb m18, m17
+.hv_w16_loop0:
+ movu ym19, [srcq+strideq*0]
+ vinserti32x8 m19, [srcq+strideq*1], 1
+ lea r5, [srcq+strideq*2]
+ movu ym20, [r5 +strideq*0]
+ vinserti32x8 m20, [r5 +strideq*1], 1
+ lea r5, [r5 +strideq*2]
+ movu ym21, [r5 +strideq*0]
+ vinserti32x8 m21, [r5 +strideq*1], 1
+ lea r5, [r5 +strideq*2]
+ movu ym22, [r5 +strideq*0]
mov r7, tmpq
-.hv_loop0:
- movu ym0, [srcq+strideq*0]
- vinserti32x8 m0, [srcq+strideq*1], 1
- lea srcq, [srcq+strideq*2]
- movu ym1, [srcq+strideq*0]
- vinserti32x8 m1, [srcq+strideq*1], 1
- lea srcq, [srcq+strideq*2]
- movu ym2, [srcq+strideq*0]
- vinserti32x8 m2, [srcq+strideq*1], 1
- lea srcq, [srcq+strideq*2]
- movu ym3, [srcq+strideq*0]
+ vpermb m3, m16, m19 ; 0 1 0123 89ab
+ mova m2, m8
+ vpermb m4, m18, m19 ; 0 1 89ab ghij
+ vpdpbusd m2, m3, m10
+ mova m3, m8
+ vpermb m5, m16, m20 ; 2 3 0123 89ab
+ vpdpbusd m3, m4, m11
mova m4, m8
+ vpermb m6, m18, m20 ; 2 3 89ab ghij
+ vpdpbusd m4, m5, m10
mova m5, m8
+ vpermb m7, m16, m21 ; 4 5 0123 89ab
+ vpdpbusd m5, m6, m11
mova m6, m8
+ vpermb m0, m18, m21 ; 4 5 89ab ghij
+ vpdpbusd m6, m7, m10
mova m7, m8
- vpermb m19, m16, m0
- vpermb m20, m17, m0
- vpermb m21, m18, m0
- vpermb m22, m16, m1
- vpermb m23, m17, m1
- vpermb m24, m18, m1
- vpermb m25, m16, m2
- vpermb m26, m17, m2
- vpermb m27, m18, m2
- vpermb ym28, ym16, ym3
- vpermb ym29, ym17, ym3
- vpermb ym30, ym18, ym3
- mova m0, m8
- mova m1, m8
- mova ym2, ym8
- mova ym3, ym8
- vpdpbusd m4, m19, m10
- vpdpbusd m5, m20, m10
- vpdpbusd m6, m22, m10
- vpdpbusd m7, m23, m10
- vpdpbusd m0, m25, m10
- vpdpbusd m1, m26, m10
- vpdpbusd ym2, ym28, ym10
- vpdpbusd ym3, ym29, ym10
+ vpermb ym1, ym16, ym22 ; 6 0123 89ab
+ vpdpbusd m7, m0, m11
+ mova ym0, ym8
+ vpermb m19, m17, m19 ; 0 1 4567 cdef
+ vpdpbusd ym0, ym1, ym10
+ vpermb ym1, ym18, ym22 ; 6 89ab ghij
+ vpdpbusd m2, m19, m11
+ vpdpbusd m3, m19, m10
+ mova ym19, ym8
+ vpermb m20, m17, m20 ; 2 3 4567 cdef
+ vpdpbusd ym19, ym1, ym11
+ vpermb m21, m17, m21 ; 4 5 4567 cdef
vpdpbusd m4, m20, m11
- vpdpbusd m5, m21, m11
- vpdpbusd m6, m23, m11
- vpdpbusd m7, m24, m11
- vpdpbusd m0, m26, m11
- vpdpbusd m1, m27, m11
- vpdpbusd ym2, ym29, ym11
- vpdpbusd ym3, ym30, ym11
- packssdw m4, m5
- packssdw m6, m7
- packssdw m0, m1
- packssdw ym2, ym3
- psraw m4, 2 ; 0a 0b 1a 1b
- psraw m6, 2 ; 2a 2b 3a 3b
- psraw m0, 2 ; 4a 4b 5a 5b
- psraw ym2, 2 ; 6a 6b __ __
- vshufi32x4 m5, m4, m6, q1032 ; 1a 1b 2a 2b
- vshufi32x4 m7, m6, m0, q1032 ; 3a 3b 4a 4b
- vshufi32x4 m1, m0, m2, q1032 ; 5a 5b 6a 6b
- punpcklwd m2, m4, m5 ; 01a 01c 12a 12c
- punpckhwd m3, m4, m5 ; 01b 01d 12b 12d
- punpcklwd m4, m6, m7 ; 23a 23c 34a 34c
- punpckhwd m5, m6, m7 ; 23b 23d 34b 34d
- punpcklwd m6, m0, m1 ; 45a 45c 56a 56c
- punpckhwd m7, m0, m1 ; 45b 45d 56b 56d
-.hv_loop:
- movu ym19, [srcq+strideq*1]
- lea srcq, [srcq+strideq*2]
- vinserti32x8 m19, [srcq+strideq*0], 1
+ vpdpbusd m5, m20, m10
+ vpermb ym22, ym17, ym22 ; 6 4567 cdef
+ vpdpbusd m6, m21, m11
+ vpdpbusd m7, m21, m10
+ packssdw m2, m3 ; 0 1
+ vpdpbusd ym0, ym22, ym11
+ packssdw m4, m5 ; 2 3
+ vpdpbusd ym19, ym22, ym10
+ packssdw m6, m7 ; 4 5
+ packssdw ym0, ym19 ; 6
+ REPX {psraw x, 2}, m2, m4, m6, ym0
+ vshufi32x4 m3, m2, m4, q1032 ; 1 2
+ vshufi32x4 m5, m4, m6, q1032 ; 3 4
+ vshufi32x4 m0, m6, m0, q1032 ; 5 6
+ punpcklwd m1, m2, m3 ; 01 12
+ punpckhwd m2, m3
+ punpcklwd m3, m4, m5 ; 23 34
+ punpckhwd m4, m5
+ punpcklwd m5, m6, m0 ; 45 56
+ punpckhwd m6, m0
+.hv_w16_loop:
+ movu ym19, [r5+strideq*1]
+ lea r5, [r5+strideq*2]
+ vinserti32x8 m19, [r5+strideq*0], 1
mova m20, m9
+ vpdpwssd m20, m1, m12 ; a0
+ vpermb m1, m16, m19
mova m21, m9
+ vpdpwssd m21, m2, m12 ; b0
+ vpermb m2, m17, m19
mova m22, m8
- mova m23, m8
- vpdpwssd m20, m2, m12
- vpdpwssd m21, m3, m12
- vpdpwssd m20, m4, m13
- vpdpwssd m21, m5, m13
- vpermb m24, m16, m19
- vpermb m25, m17, m19
- vpermb m26, m18, m19
- vpdpbusd m22, m24, m10
- vpdpbusd m23, m25, m10
- vpdpbusd m22, m25, m11
- vpdpbusd m23, m26, m11
- packssdw m22, m23
- psraw m22, 2 ; 7a 7b 8a 8b
- vshufi32x4 m0, m1, m22, q1032 ; 6a 6b 7a 7b
+ vpdpbusd m22, m1, m10
+ mova m1, m8
+ vpermb m19, m18, m19
+ vpdpbusd m1, m2, m10
+ vpdpwssd m20, m3, m13 ; a1
+ vpdpwssd m21, m4, m13 ; b1
+ vpdpbusd m22, m2, m11
mova m2, m4
- mova m3, m5
- mova m1, m22
+ vpdpbusd m1, m19, m11
mova m4, m6
- mova m5, m7
- punpcklwd m6, m0, m1 ; 67a 67c 78a 78c
- punpckhwd m7, m0, m1 ; 67b 67d 78b 78d
- vpdpwssd m20, m4, m14
- vpdpwssd m21, m5, m14
- vpdpwssd m20, m6, m15
- vpdpwssd m21, m7, m15
+ vpdpwssd m20, m5, m14 ; a2
+ vpdpwssd m21, m6, m14 ; b2
+ packssdw m22, m1
+ mova m1, m3
+ psraw m22, 2 ; 7 8
+ mova m3, m5
+ vshufi32x4 m6, m0, m22, q1032 ; 6 7
+ mova m0, m22
+ punpcklwd m5, m6, m0 ; 67 78
+ punpckhwd m6, m0
+ vpdpwssd m20, m5, m15 ; a3
+ vpdpwssd m21, m6, m15 ; b3
psrad m20, 6
psrad m21, 6
packssdw m20, m21
- mova [tmpq+wq*0], ym20
- vextracti32x8 [tmpq+wq*1], m20, 1
- lea tmpq, [tmpq+wq*2]
+ mova [r7+wq*0], ym20
+ vextracti32x8 [r7+wq*1], m20, 1
+ lea r7, [r7+wq*2]
sub hd, 2
- jg .hv_loop
- add r5, 16
- add r7, 32
+ jg .hv_w16_loop
+ add srcq, 16
+ add tmpq, 32
movzx hd, r6b
- mov srcq, r5
- mov tmpq, r7
sub r6d, 1<<8
- jg .hv_loop0
+ jg .hv_w16_loop0
RET
cglobal warp_affine_8x8t_8bpc, 4, 7, 22, tmp, ts
generated by cgit v1.2.3 (git 2.39.1) at 2024-09-15 19:57:15 +0000