Source code
Revision control
Copy as Markdown
Other Tools
; Copyright © 2020, VideoLAN and dav1d authors
; Copyright © 2020, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
; list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
; this list of conditions and the following disclaimer in the documentation
; and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%include "config.asm"
%include "ext/x86/x86inc.asm"
%if ARCH_X86_64
SECTION_RODATA 64
obmc_masks:
pw_512: times 2 dw 512
; 2
db 45, 19, 64, 0
; 4
db 39, 25, 50, 14, 59, 5, 64, 0
; 8
db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0
; 16
db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10
db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0
; 32
db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20
db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9
db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2
db 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0
warp_8x8_permA: db 4, 5, 6, 7, 16, 17, 18, 19, 5, 6, 7, 8, 17, 18, 19, 20
db 6, 7, 8, 9, 18, 19, 20, 21, 7, 8, 9, 10, 19, 20, 21, 22
db 8, 9, 10, 11, 20, 21, 22, 23, 9, 10, 11, 12, 21, 22, 23, 24
db 10, 11, 12, 13, 22, 23, 24, 25, 11, 12, 13, 14, 23, 24, 25, 26
warp_8x8_permB: db 0, 1, 2, 3, 20, 21, 22, 23, 1, 2, 3, 4, 21, 22, 23, 24
db 2, 3, 4, 5, 22, 23, 24, 25, 3, 4, 5, 6, 23, 24, 25, 26
db 4, 5, 6, 7, 24, 25, 26, 27, 5, 6, 7, 8, 25, 26, 27, 28
db 6, 7, 8, 9, 26, 27, 28, 29, 7, 8, 9, 10, 27, 28, 29, 30
warp_8x8_permC: db -1, 0, -1, 1, -1, 8, -1, 9, -1, 4, -1, 5, -1, 12, -1, 13
warp_8x8_permD: db -1, 2, -1, 3, -1, 10, -1, 11, -1, 6, -1, 7, -1, 14, -1, 15
pd_0to7: dd 0, 1, 2, 3, 4, 5, 6, 7
warp_8x8_hpack: db 3, 11, 3, 11, 35, 43, 35, 43
pd_16384: dd 16384
pd_262144: dd 262144
warp_8x8_end: db 0, 4, 16, 20, 32, 36, 48, 52, 2, 6, 18, 22, 34, 38, 50, 54
warp_8x8t_end: db 2, 3, 10, 11, 18, 19, 26, 27, 34, 35, 42, 43, 50, 51, 58, 59
db 6, 7, 14, 15, 22, 23, 30, 31, 38, 39, 46, 47, 54, 55, 62, 63
bidir_sctr_w4: dd 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
wm_420_perm4: db 1, 3, 9, 11, 5, 7, 13, 15, 17, 19, 25, 27, 21, 23, 29, 31
db 33, 35, 41, 43, 37, 39, 45, 47, 49, 51, 57, 59, 53, 55, 61, 63
db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30
db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62
wm_420_perm8: db 1, 3, 17, 19, 5, 7, 21, 23, 9, 11, 25, 27, 13, 15, 29, 31
db 33, 35, 49, 51, 37, 39, 53, 55, 41, 43, 57, 59, 45, 47, 61, 63
db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30
db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62
wm_420_perm16: db 1, 3, 33, 35, 5, 7, 37, 39, 9, 11, 41, 43, 13, 15, 45, 47
db 17, 19, 49, 51, 21, 23, 53, 55, 25, 27, 57, 59, 29, 31, 61, 63
db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46
db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62
wm_420_mask: db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63
db 67, 71, 75, 79, 83, 87, 91, 95, 99,103,107,111,115,119,123,127
db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61
db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125
wm_422_mask: db 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62
db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61
db 66, 70, 74, 78, 82, 86, 90, 94, 98,102,106,110,114,118,122,126
db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125
wm_444_mask: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63
db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62
bilin_h_perm16: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
db 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16
db 32, 33, 33, 34, 34, 35, 35, 36, 36, 37, 37, 38, 38, 39, 39, 40
db 40, 41, 41, 42, 42, 43, 43, 44, 44, 45, 45, 46, 46, 47, 47, 48
bilin_h_perm32: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8
db 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16
db 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24
db 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32
bilin_v_perm8: db 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23
db 16, 80, 17, 81, 18, 82, 19, 83, 20, 84, 21, 85, 22, 86, 23, 87
db 80, 32, 81, 33, 82, 34, 83, 35, 84, 36, 85, 37, 86, 38, 87, 39
db 32, 64, 33, 65, 34, 66, 35, 67, 36, 68, 37, 69, 38, 70, 39, 71
bilin_v_perm16: db 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23
db 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31
db 16, 64, 17, 65, 18, 66, 19, 67, 20, 68, 21, 69, 22, 70, 23, 71
db 24, 72, 25, 73, 26, 74, 27, 75, 28, 76, 29, 77, 30, 78, 31, 79
bilin_v_perm32: db 0, 64, 1, 65, 2, 66, 3, 67, 4, 68, 5, 69, 6, 70, 7, 71
db 8, 72, 9, 73, 10, 74, 11, 75, 12, 76, 13, 77, 14, 78, 15, 79
db 16, 80, 17, 81, 18, 82, 19, 83, 20, 84, 21, 85, 22, 86, 23, 87
db 24, 88, 25, 89, 26, 90, 27, 91, 28, 92, 29, 93, 30, 94, 31, 95
bilin_v_perm64: dd 0, 0, 4, 8, 1, 1, 5, 9, 2, 2, 6, 10, 3, 3, 7, 11
spel_h_perm16: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38
db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46
spel_h_perm32: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30
spel_v_perm8: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
db 8, 16, 9, 17, 10, 18, 11, 19, 12, 20, 13, 21, 14, 22, 15, 23
db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31
db 24, 32, 25, 33, 26, 34, 27, 35, 28, 36, 29, 37, 30, 38, 31, 39
spel_v_perm16a: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7
db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
db 40, 16, 41, 17, 42, 18, 43, 19, 44, 20, 45, 21, 46, 22, 47, 23
db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31
spel_v_perm16b: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7
db 40, 16, 41, 17, 42, 18, 43, 19, 44, 20, 45, 21, 46, 22, 47, 23
db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31
spel_v_perm32: db 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39
db 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47
db 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55
db 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63
spel_hv_perm4a: db 8, 9, 16, 17, 10, 11, 18, 19, 12, 13, 20, 21, 14, 15, 22, 23
db 16, 17, 24, 25, 18, 19, 26, 27, 20, 21, 28, 29, 22, 23, 30, 31
spel_hv_perm4b: db 24, 25, 32, 33, 26, 27, 34, 35, 28, 29, 36, 37, 30, 31, 38, 39
db 32, 33, 40, 41, 34, 35, 42, 43, 36, 37, 44, 45, 38, 39, 46, 47
spel_hv_perm4c: db 40, 41, 48, 49, 42, 43, 50, 51, 44, 45, 52, 53, 46, 47, 54, 55
db 48, 49, 56, 57, 50, 51, 58, 59, 52, 53, 60, 61, 54, 55, 62, 63
spel_hv_perm4d: db 18, 19, 0, 1, 22, 23, 4, 5, 26, 27, 8, 9, 30, 31, 12, 13
db 0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29
spel_hv_perm8a: db 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23
db 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31
db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39
db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47
spel_hv_perm8b: db 34, 35, 0, 1, 38, 39, 4, 5, 42, 43, 8, 9, 46, 47, 12, 13
db 50, 51, 16, 17, 54, 55, 20, 21, 58, 59, 24, 25, 62, 63, 28, 29
db 0, 1, 32, 33, 4, 5, 36, 37, 8, 9, 40, 41, 12, 13, 44, 45
db 16, 17, 48, 49, 20, 21, 52, 53, 24, 25, 56, 57, 28, 29, 60, 61
spel_hv_perm16a:db 0, 1, 2, 3, 32, 33, 34, 35, 1, 2, 3, 4, 33, 34, 35, 36
db 2, 3, 4, 5, 34, 35, 36, 37, 3, 4, 5, 6, 35, 36, 37, 38
db 8, 9, 10, 11, 40, 41, 42, 43, 9, 10, 11, 12, 41, 42, 43, 44
db 10, 11, 12, 13, 42, 43, 44, 45, 11, 12, 13, 14, 43, 44, 45, 46
spel_hv_perm16b:db 0, 1, 2, 3, 1, 2, 3, 4, 4, 5, 6, 7, 5, 6, 7, 8
db 2, 3, 4, 5, 3, 4, 5, 6, 6, 7, 8, 9, 7, 8, 9, 10
db 8, 9, 10, 11, 9, 10, 11, 12, 12, 13, 14, 15, 13, 14, 15, 16
db 10, 11, 12, 13, 11, 12, 13, 14, 14, 15, 16, 17, 15, 16, 17, 18
spel_hv_end16: db 1, 3, 17, 19, 5, 7, 21, 23, 33, 35, 49, 51, 37, 39, 53, 55
db 9, 11, 25, 27, 13, 15, 29, 31, 41, 43, 57, 59, 45, 47, 61, 63
spel_hv_end: db 1, 3, 5, 7, 17, 19, 21, 23, 33, 35, 37, 39, 49, 51, 53, 55
deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11
subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12
db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14
subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6
subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10
subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
bilin_h_shuf4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12
bilin_v_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11
blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3
rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
resize_permA: dd 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
resize_permB: dd 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
resize_permC: dd 0, 4, 8, 12
resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7
pb_02461357: db 0, 2, 4, 6, 1, 3, 5, 7
wm_420_perm64: dq 0xfedcba9876543210
wm_sign: dd 0x40804080, 0xc0c0c0c0, 0x40404040
pb_8x0_8x8: times 8 db 0
times 8 db 8
pb_4: times 4 db 4
pb_32: times 4 db 32
pb_127: times 4 db 127
pw_m128 times 2 dw -128
pw_m256: times 2 dw -256
pw_1024: times 2 dw 1024
pw_2048: times 2 dw 2048
pw_6903: times 2 dw 6903
pw_8192: times 2 dw 8192
pd_32: dd 32
pd_34: dd 34
pd_63: dd 63
pd_512: dd 512
%define pb_m64 (wm_sign+4)
%define pb_64 (wm_sign+8)
%define pd_2 (pd_0to7+8)
cextern mc_subpel_filters
%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)
cextern mc_warp_filter
cextern resize_filter
%macro BASE_JMP_TABLE 3-*
%xdefine %1_%2_table (%%table - %3)
%xdefine %%base %1_%2
%%table:
%rep %0 - 2
dw %%base %+ _w%3 - %%base
%rotate 1
%endrep
%endmacro
%macro HV_JMP_TABLE 5-*
%xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3)
%xdefine %%base %1_%3
%assign %%types %4
%if %%types & 1
%xdefine %1_%2_h_%3_table (%%h - %5)
%%h:
%rep %0 - 4
dw %%prefix %+ .h_w%5 - %%base
%rotate 1
%endrep
%rotate 4
%endif
%if %%types & 2
%xdefine %1_%2_v_%3_table (%%v - %5)
%%v:
%rep %0 - 4
dw %%prefix %+ .v_w%5 - %%base
%rotate 1
%endrep
%rotate 4
%endif
%if %%types & 4
%xdefine %1_%2_hv_%3_table (%%hv - %5)
%%hv:
%rep %0 - 4
dw %%prefix %+ .hv_w%5 - %%base
%rotate 1
%endrep
%endif
%endmacro
%macro BIDIR_JMP_TABLE 2-*
%xdefine %1_%2_table (%%table - 2*%3)
%xdefine %%base %1_%2_table
%xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2)
%%table:
%rep %0 - 2
dd %%prefix %+ .w%3 - %%base
%rotate 1
%endrep
%endmacro
%xdefine put_avx512icl mangle(private_prefix %+ _put_bilin_8bpc_avx512icl.put)
%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_8bpc_avx512icl.prep)
%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX
BASE_JMP_TABLE put, avx512icl, 2, 4, 8, 16, 32, 64, 128
BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE put, bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE put, 6tap, avx512icl, 2, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE put, 8tap, avx512icl, 3, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE prep, 6tap, avx512icl, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE prep, 8tap, avx512icl, 3, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_mask_420, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_mask_422, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_mask_444, avx512icl, 4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE blend, avx512icl, 4, 8, 16, 32
BIDIR_JMP_TABLE blend_v, avx512icl, 2, 4, 8, 16, 32
BIDIR_JMP_TABLE blend_h, avx512icl, 2, 4, 8, 16, 32, 64, 128
SECTION .text
%macro WRAP_YMM 1+
INIT_YMM cpuname
%1
INIT_ZMM cpuname
%endmacro
INIT_ZMM avx512icl
cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
movifnidn mxyd, r6m ; mx
lea r7, [put_avx512icl]
tzcnt wd, wm
movifnidn hd, hm
test mxyd, mxyd
jnz .h
mov mxyd, r7m ; my
test mxyd, mxyd
jnz .v
.put:
movzx wd, word [r7+wq*2+table_offset(put,)]
add wq, r7
jmp wq
.put_w2:
movzx r6d, word [srcq+ssq*0]
movzx r7d, word [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mov [dstq+dsq*0], r6w
mov [dstq+dsq*1], r7w
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w2
RET
.put_w4:
mov r6d, [srcq+ssq*0]
mov r7d, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mov [dstq+dsq*0], r6d
mov [dstq+dsq*1], r7d
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w4
RET
.put_w8:
mov r6, [srcq+ssq*0]
mov r7, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mov [dstq+dsq*0], r6
mov [dstq+dsq*1], r7
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w8
RET
.put_w16:
movu xmm0, [srcq+ssq*0]
movu xmm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mova [dstq+dsq*0], xmm0
mova [dstq+dsq*1], xmm1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w16
RET
.put_w32:
movu ym0, [srcq+ssq*0]
movu ym1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mova [dstq+dsq*0], ym0
mova [dstq+dsq*1], ym1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w32
RET
.put_w64:
movu m0, [srcq+ssq*0]
movu m1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
mova [dstq+dsq*0], m0
mova [dstq+dsq*1], m1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w64
RET
.put_w128:
movu m0, [srcq+ssq*0+64*0]
movu m1, [srcq+ssq*0+64*1]
movu m2, [srcq+ssq*1+64*0]
movu m3, [srcq+ssq*1+64*1]
lea srcq, [srcq+ssq*2]
mova [dstq+dsq*0+64*0], m0
mova [dstq+dsq*0+64*1], m1
mova [dstq+dsq*1+64*0], m2
mova [dstq+dsq*1+64*1], m3
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .put_w128
RET
.h:
; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4
; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4
imul mxyd, 255
vbroadcasti128 m4, [bilin_h_perm16]
add mxyd, 16
vpbroadcastw m5, mxyd
mov mxyd, r7m ; my
test mxyd, mxyd
jnz .hv
movzx wd, word [r7+wq*2+table_offset(put, _bilin_h)]
vpbroadcastd m3, [pw_2048]
add wq, r7
jmp wq
.h_w2:
movd xmm0, [srcq+ssq*0]
pinsrd xmm0, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
pshufb xmm0, xm4
pmaddubsw xmm0, xm5
pmulhrsw xmm0, xm3
packuswb xmm0, xmm0
pextrw [dstq+dsq*0], xmm0, 0
pextrw [dstq+dsq*1], xmm0, 2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w2
RET
.h_w4:
mova xmm4, [bilin_h_shuf4]
.h_w4_loop:
movq xmm0, [srcq+ssq*0]
movhps xmm0, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pshufb xmm0, xmm4
pmaddubsw xmm0, xm5
pmulhrsw xmm0, xm3
packuswb xmm0, xmm0
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w4_loop
RET
.h_w8:
movu xm0, [srcq+ssq*0]
vinserti32x4 ym0, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
pshufb ym0, ym4
pmaddubsw ym0, ym5
pmulhrsw ym0, ym3
vpmovuswb xm0, ym0
movq [dstq+dsq*0], xm0
movhps [dstq+dsq*1], xm0
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w8
RET
.h_w16:
mova m4, [bilin_h_perm16]
.h_w16_loop:
movu ym0, [srcq+ssq*0]
vinserti32x8 m0, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
vpermb m0, m4, m0
pmaddubsw m0, m5
pmulhrsw m0, m3
vpmovuswb ym0, m0
mova [dstq+dsq*0], xm0
vextracti128 [dstq+dsq*1], ym0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w16_loop
RET
.h_w32:
movu ym0, [srcq+ssq*0+8*0]
vinserti32x8 m0, [srcq+ssq*1+8*0], 1
movu ym1, [srcq+ssq*0+8*1]
vinserti32x8 m1, [srcq+ssq*1+8*1], 1
lea srcq, [srcq+ssq*2]
pshufb m0, m4
pshufb m1, m4
pmaddubsw m0, m5
pmaddubsw m1, m5
pmulhrsw m0, m3
pmulhrsw m1, m3
packuswb m0, m1
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w32
RET
.h_w64:
movu m0, [srcq+8*0]
movu m1, [srcq+8*1]
pshufb m0, m4
pshufb m1, m4
pmaddubsw m0, m5
pmaddubsw m1, m5
pmulhrsw m0, m3
pmulhrsw m1, m3
packuswb m0, m1
add srcq, ssq
mova [dstq], m0
add dstq, dsq
dec hd
jg .h_w64
RET
.h_w128:
movu m0, [srcq+8*0]
movu m2, [srcq+8*1]
movu m1, [srcq+8*8]
movu m6, [srcq+8*9]
add srcq, ssq
REPX {pshufb x, m4}, m0, m2, m1, m6
REPX {pmaddubsw x, m5}, m0, m2, m1, m6
REPX {pmulhrsw x, m3}, m0, m2, m1, m6
packuswb m0, m2
packuswb m1, m6
mova [dstq+64*0], m0
mova [dstq+64*1], m1
add dstq, dsq
dec hd
jg .h_w128
RET
.v:
movzx wd, word [r7+wq*2+table_offset(put, _bilin_v)]
imul mxyd, 255
vpbroadcastd m5, [pw_2048]
add mxyd, 16
add wq, r7
vpbroadcastw m4, mxyd
jmp wq
.v_w2:
movd xmm0, [srcq+ssq*0]
.v_w2_loop:
pinsrw xmm1, xmm0, [srcq+ssq*1], 1 ; 0 1
lea srcq, [srcq+ssq*2]
pinsrw xmm0, xmm1, [srcq+ssq*0], 0 ; 2 1
pshuflw xmm1, xmm1, q2301 ; 1 0
punpcklbw xmm1, xmm0
pmaddubsw xmm1, xm4
pmulhrsw xmm1, xm5
packuswb xmm1, xmm1
pextrw [dstq+dsq*0], xmm1, 1
pextrw [dstq+dsq*1], xmm1, 0
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w2_loop
RET
.v_w4:
movd xmm0, [srcq+ssq*0]
.v_w4_loop:
vpbroadcastd xmm2, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vpblendd xmm1, xmm2, xmm0, 0x01 ; 0 1
vpbroadcastd xmm0, [srcq+ssq*0]
vpblendd xmm2, xmm0, 0x02 ; 1 2
punpcklbw xmm1, xmm2
pmaddubsw xmm1, xm4
pmulhrsw xmm1, xm5
packuswb xmm1, xmm1
movd [dstq+dsq*0], xmm1
pextrd [dstq+dsq*1], xmm1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w4_loop
RET
.v_w8:
movq xmm0, [srcq+ssq*0]
.v_w8_loop:
movq xmm2, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
punpcklbw xmm1, xmm0, xmm2
movq xmm0, [srcq+ssq*0]
punpcklbw xmm2, xmm0
pmaddubsw xmm1, xm4
pmaddubsw xmm2, xm4
pmulhrsw xmm1, xm5
pmulhrsw xmm2, xm5
packuswb xmm1, xmm2
movq [dstq+dsq*0], xmm1
movhps [dstq+dsq*1], xmm1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w8_loop
RET
.v_w16:
movu xmm0, [srcq+ssq*0]
.v_w16_loop:
vbroadcasti128 ymm3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vpblendd ymm2, ymm3, ymm0, 0x0f ; 0 1
vbroadcasti128 ymm0, [srcq+ssq*0]
vpblendd ymm3, ymm0, 0xf0 ; 1 2
punpcklbw ymm1, ymm2, ymm3
punpckhbw ymm2, ymm3
pmaddubsw ymm1, ym4
pmaddubsw ymm2, ym4
pmulhrsw ymm1, ym5
pmulhrsw ymm2, ym5
packuswb ymm1, ymm2
mova [dstq+dsq*0], xmm1
vextracti128 [dstq+dsq*1], ymm1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w16_loop
vzeroupper
RET
.v_w32:
movu ym0, [srcq+ssq*0]
kxnorb k1, k1, k1
.v_w32_loop:
vbroadcasti32x8 m3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vpblendmd m2{k1}, m3, m0 ; 0 1
vbroadcasti32x8 m0, [srcq+ssq*0]
vpblendmd m3{k1}, m0, m3 ; 1 2
punpcklbw m1, m2, m3
punpckhbw m2, m3
pmaddubsw m1, m4
pmaddubsw m2, m4
pmulhrsw m1, m5
pmulhrsw m2, m5
packuswb m1, m2
mova [dstq+dsq*0], ym1
vextracti32x8 [dstq+dsq*1], m1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w32_loop
RET
.v_w64:
movu m0, [srcq+ssq*0]
.v_w64_loop:
movu m3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
punpcklbw m1, m0, m3
punpckhbw m6, m0, m3
movu m0, [srcq+ssq*0]
pmaddubsw m1, m4
pmaddubsw m6, m4
punpcklbw m2, m3, m0
punpckhbw m3, m0
pmaddubsw m2, m4
pmaddubsw m3, m4
REPX {pmulhrsw x, m5}, m1, m6, m2, m3
packuswb m1, m6
packuswb m2, m3
mova [dstq+dsq*0], m1
mova [dstq+dsq*1], m2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w64_loop
RET
.v_w128:
movu m0, [srcq+64*0]
movu m1, [srcq+64*1]
.v_w128_loop:
add srcq, ssq
movu m2, [srcq+64*0]
movu m3, [srcq+64*1]
punpcklbw m6, m0, m2
pmaddubsw m6, m4
punpckhbw m0, m2
pmaddubsw m0, m4
punpcklbw m7, m1, m3
pmaddubsw m7, m4
punpckhbw m1, m3
pmaddubsw m1, m4
REPX {pmulhrsw x, m5}, m6, m0, m7, m1
packuswb m6, m0
mova m0, m2
packuswb m7, m1
mova m1, m3
mova [dstq+64*0], m6
mova [dstq+64*1], m7
add dstq, dsq
dec hd
jg .v_w128_loop
RET
.hv:
; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8
; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4
movzx wd, word [r7+wq*2+table_offset(put, _bilin_hv)]
WIN64_SPILL_XMM 8
shl mxyd, 11 ; can't shift by 12 due to signed overflow
vpbroadcastd m7, [pw_2048]
add wq, r7
vpbroadcastw m6, mxyd
jmp wq
.hv_w2:
vpbroadcastd xmm0, [srcq+ssq*0]
pshufb xmm0, xm4
pmaddubsw xmm0, xm5
.hv_w2_loop:
movd xmm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pinsrd xmm1, [srcq+ssq*0], 1
pshufb xmm1, xm4
pmaddubsw xmm1, xm5 ; 1 _ 2 _
shufps xmm2, xmm0, xmm1, q1032 ; 0 _ 1 _
mova xmm0, xmm1
psubw xmm1, xmm2
paddw xmm1, xmm1
pmulhw xmm1, xm6
paddw xmm1, xmm2
pmulhrsw xmm1, xm7
packuswb xmm1, xmm1
pextrw [dstq+dsq*0], xmm1, 0
pextrw [dstq+dsq*1], xmm1, 2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w2_loop
RET
.hv_w4:
mova xmm4, [bilin_h_shuf4]
movddup xmm0, [srcq+ssq*0]
pshufb xmm0, xmm4
pmaddubsw xmm0, xm5
.hv_w4_loop:
movq xmm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
movhps xmm1, [srcq+ssq*0]
pshufb xmm1, xmm4
pmaddubsw xmm1, xm5 ; 1 2
shufps xmm2, xmm0, xmm1, q1032 ; 0 1
mova xmm0, xmm1
psubw xmm1, xmm2
paddw xmm1, xmm1
pmulhw xmm1, xm6
paddw xmm1, xmm2
pmulhrsw xmm1, xm7
packuswb xmm1, xmm1
movd [dstq+dsq*0], xmm1
pextrd [dstq+dsq*1], xmm1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w4_loop
RET
.hv_w8:
vbroadcasti128 ym0, [srcq+ssq*0]
pshufb ym0, ym4
pmaddubsw ym0, ym5
.hv_w8_loop:
movu xm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vinserti128 ym1, [srcq+ssq*0], 1
pshufb ym1, ym4
pmaddubsw ym1, ym5 ; 1 2
valignq ym2, ym1, ym0, 2
mova ym0, ym1
psubw ym1, ym2
paddw ym1, ym1
pmulhw ym1, ym6
paddw ym1, ym2
pmulhrsw ym1, ym7
vpmovuswb xm1, ym1
movq [dstq+dsq*0], xm1
movhps [dstq+dsq*1], xm1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w8_loop
RET
.hv_w16:
vbroadcasti32x8 m0, [srcq+ssq*0]
mova m4, [bilin_h_perm16]
vpermb m0, m4, m0
pmaddubsw m0, m5
.hv_w16_loop:
movu ym1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vinserti32x8 m1, [srcq+ssq*0], 1
vpermb m1, m4, m1
pmaddubsw m1, m5 ; 1 2
valignq m2, m1, m0, 4 ; 0 1
mova m0, m1
psubw m1, m2
paddw m1, m1
pmulhw m1, m6
paddw m1, m2
pmulhrsw m1, m7
vpmovuswb ym1, m1
mova [dstq+dsq*0], xm1
vextracti32x4 [dstq+dsq*1], ym1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w16_loop
RET
.hv_w32:
mova m4, [bilin_h_perm32]
vpermb m0, m4, [srcq+ssq*0]
pmovzxbq m8, [pb_02461357]
pmaddubsw m0, m5
.hv_w32_loop:
vpermb m2, m4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vpermb m3, m4, [srcq+ssq*0]
pmaddubsw m2, m5
psubw m1, m2, m0
paddw m1, m1
pmulhw m1, m6
paddw m1, m0
pmaddubsw m0, m3, m5
psubw m3, m0, m2
paddw m3, m3
pmulhw m3, m6
paddw m3, m2
pmulhrsw m1, m7
pmulhrsw m3, m7
packuswb m1, m3
vpermq m1, m8, m1
mova [dstq+dsq*0], ym1
vextracti32x8 [dstq+dsq*1], m1, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w32_loop
RET
.hv_w64:
movu m0, [srcq+8*0]
movu m1, [srcq+8*1]
pshufb m0, m4
pshufb m1, m4
pmaddubsw m0, m5
pmaddubsw m1, m5
.hv_w64_loop:
add srcq, ssq
movu m2, [srcq+8*0]
movu m3, [srcq+8*1]
pshufb m2, m4
pshufb m3, m4
pmaddubsw m2, m5
pmaddubsw m3, m5
psubw m8, m2, m0
psubw m9, m3, m1
paddw m8, m8
pmulhw m8, m6
paddw m9, m9
pmulhw m9, m6
paddw m8, m0
pmulhrsw m8, m7
paddw m9, m1
pmulhrsw m9, m7
mova m0, m2
mova m1, m3
packuswb m8, m9
mova [dstq], m8
add dstq, dsq
dec hd
jg .hv_w64_loop
RET
.hv_w128:
movu m0, [srcq+8*0]
movu m1, [srcq+8*1]
movu m2, [srcq+8*8]
movu m3, [srcq+8*9]
REPX {pshufb x, m4}, m0, m1, m2, m3
REPX {pmaddubsw x, m5}, m0, m1, m2, m3
.hv_w128_loop:
add srcq, ssq
movu m8, [srcq+8*0]
movu m9, [srcq+8*1]
movu m10, [srcq+8*8]
movu m11, [srcq+8*9]
REPX {pshufb x, m4}, m8, m9, m10, m11
REPX {pmaddubsw x, m5}, m8, m9, m10, m11
psubw m12, m8, m0
psubw m13, m9, m1
psubw m14, m10, m2
psubw m15, m11, m3
paddw m12, m12
pmulhw m12, m6
paddw m13, m13
pmulhw m13, m6
paddw m14, m14
pmulhw m14, m6
paddw m15, m15
pmulhw m15, m6
paddw m12, m0
pmulhrsw m12, m7
paddw m13, m1
pmulhrsw m13, m7
paddw m14, m2
pmulhrsw m14, m7
paddw m15, m3
pmulhrsw m15, m7
mova m0, m8
mova m1, m9
mova m2, m10
mova m3, m11
packuswb m12, m13
packuswb m14, m15
mova [dstq+64*0], m12
mova [dstq+64*1], m14
add dstq, dsq
dec hd
jg .hv_w128_loop
RET
DECLARE_REG_TMP 3, 5, 6
cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
movifnidn mxyd, r5m ; mx
lea t2, [prep_avx512icl]
tzcnt wd, wm
movifnidn hd, hm
test mxyd, mxyd
jnz .h
mov mxyd, r6m ; my
test mxyd, mxyd
jnz .v
.prep:
movzx wd, word [t2+wq*2+table_offset(prep,)]
add wq, t2
lea stride3q, [strideq*3]
jmp wq
.prep_w4:
movd xmm0, [srcq+strideq*0]
pinsrd xmm0, [srcq+strideq*1], 1
pinsrd xmm0, [srcq+strideq*2], 2
pinsrd xmm0, [srcq+stride3q ], 3
lea srcq, [srcq+strideq*4]
pmovzxbw ym0, xmm0
psllw ym0, 4
mova [tmpq], ym0
add tmpq, 32
sub hd, 4
jg .prep_w4
RET
.prep_w8:
movq xmm0, [srcq+strideq*0]
movq xmm1, [srcq+strideq*1]
vinserti128 ym0, ymm0, [srcq+strideq*2], 1
vinserti128 ym1, ymm1, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
punpcklqdq ym0, ym1
pmovzxbw m0, ym0
psllw m0, 4
mova [tmpq], m0
add tmpq, 32*2
sub hd, 4
jg .prep_w8
RET
.prep_w16:
movu xmm0, [srcq+strideq*0]
vinserti128 ym0, ymm0, [srcq+strideq*1], 1
movu xmm1, [srcq+strideq*2]
vinserti128 ym1, ymm1, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
pmovzxbw m0, ym0
pmovzxbw m1, ym1
psllw m0, 4
psllw m1, 4
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
add tmpq, 32*4
sub hd, 4
jg .prep_w16
RET
.prep_w32:
pmovzxbw m0, [srcq+strideq*0]
pmovzxbw m1, [srcq+strideq*1]
pmovzxbw m2, [srcq+strideq*2]
pmovzxbw m3, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
REPX {psllw x, 4}, m0, m1, m2, m3
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
sub hd, 4
jg .prep_w32
RET
.prep_w64:
pmovzxbw m0, [srcq+strideq*0+32*0]
pmovzxbw m1, [srcq+strideq*0+32*1]
pmovzxbw m2, [srcq+strideq*1+32*0]
pmovzxbw m3, [srcq+strideq*1+32*1]
lea srcq, [srcq+strideq*2]
REPX {psllw x, 4}, m0, m1, m2, m3
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
sub hd, 2
jg .prep_w64
RET
.prep_w128:
pmovzxbw m0, [srcq+32*0]
pmovzxbw m1, [srcq+32*1]
pmovzxbw m2, [srcq+32*2]
pmovzxbw m3, [srcq+32*3]
REPX {psllw x, 4}, m0, m1, m2, m3
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
add srcq, strideq
dec hd
jg .prep_w128
RET
.h:
; 16 * src[x] + (mx * (src[x + 1] - src[x]))
; = (16 - mx) * src[x] + mx * src[x + 1]
imul mxyd, 255
add mxyd, 16
vpbroadcastw m5, mxyd
mov mxyd, r6m ; my
test mxyd, mxyd
jnz .hv
movzx wd, word [t2+wq*2+table_offset(prep, _bilin_h)]
add wq, t2
lea stride3q, [strideq*3]
jmp wq
.h_w4:
vbroadcasti32x4 ym4, [bilin_h_shuf4]
.h_w4_loop:
movq xmm0, [srcq+strideq*0]
movq xmm1, [srcq+strideq*1]
vinserti32x4 ym0, ymm0, [srcq+strideq*2], 1
vinserti32x4 ym1, ymm1, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
punpcklqdq ym0, ym1
pshufb ym0, ym4
pmaddubsw ym0, ym5
mova [tmpq], ym0
add tmpq, 32
sub hd, 4
jg .h_w4_loop
RET
.h_w8:
vbroadcasti32x4 m4, [bilin_h_perm16]
.h_w8_loop:
movu xmm0, [srcq+strideq*0]
vinserti32x4 ym0, ymm0, [srcq+strideq*1], 1
vinserti32x4 m0, [srcq+strideq*2], 2
vinserti32x4 m0, [srcq+stride3q ], 3
lea srcq, [srcq+strideq*4]
pshufb m0, m4
pmaddubsw m0, m5
mova [tmpq], m0
add tmpq, 64
sub hd, 4
jg .h_w8_loop
RET
.h_w16:
mova m4, [bilin_h_perm16]
.h_w16_loop:
movu ym0, [srcq+strideq*0]
vinserti32x8 m0, [srcq+strideq*1], 1
movu ym1, [srcq+strideq*2]
vinserti32x8 m1, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
vpermb m0, m4, m0
vpermb m1, m4, m1
pmaddubsw m0, m5
pmaddubsw m1, m5
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
add tmpq, 64*2
sub hd, 4
jg .h_w16_loop
RET
.h_w32:
mova m4, [bilin_h_perm32]
.h_w32_loop:
vpermb m0, m4, [srcq+strideq*0]
vpermb m1, m4, [srcq+strideq*1]
vpermb m2, m4, [srcq+strideq*2]
vpermb m3, m4, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
pmaddubsw m0, m5
pmaddubsw m1, m5
pmaddubsw m2, m5
pmaddubsw m3, m5
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
sub hd, 4
jg .h_w32_loop
RET
.h_w64:
mova m4, [bilin_h_perm32]
.h_w64_loop:
vpermb m0, m4, [srcq+strideq*0+32*0]
vpermb m1, m4, [srcq+strideq*0+32*1]
vpermb m2, m4, [srcq+strideq*1+32*0]
vpermb m3, m4, [srcq+strideq*1+32*1]
lea srcq, [srcq+strideq*2]
pmaddubsw m0, m5
pmaddubsw m1, m5
pmaddubsw m2, m5
pmaddubsw m3, m5
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
sub hd, 2
jg .h_w64_loop
RET
.h_w128:
mova m4, [bilin_h_perm32]
.h_w128_loop:
vpermb m0, m4, [srcq+32*0]
vpermb m1, m4, [srcq+32*1]
vpermb m2, m4, [srcq+32*2]
vpermb m3, m4, [srcq+32*3]
pmaddubsw m0, m5
pmaddubsw m1, m5
pmaddubsw m2, m5
pmaddubsw m3, m5
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
mova [tmpq+64*2], m2
mova [tmpq+64*3], m3
add tmpq, 64*4
add srcq, strideq
dec hd
jg .h_w128_loop
RET
.v:
WIN64_SPILL_XMM 7
movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)]
imul mxyd, 255
add mxyd, 16
add wq, t2
lea stride3q, [strideq*3]
vpbroadcastw m6, mxyd
jmp wq
.v_w4:
vpbroadcastd xm0, [srcq+strideq*0]
mov r3d, 0x29
vbroadcasti32x4 ym3, [bilin_v_shuf4]
kmovb k1, r3d
.v_w4_loop:
vpblendmd xm1{k1}, xm0, [srcq+strideq*1] {1to4} ; __01 ____
vpbroadcastd ym2, [srcq+strideq*2]
vpbroadcastd ym2{k1}, [srcq+stride3q ] ; __2_ 23__
lea srcq, [srcq+strideq*4]
vpbroadcastd ym0, [srcq+strideq*0]
punpckhqdq ym2{k1}, ym1, ym0 ; 012_ 234_
pshufb ym2, ym3
pmaddubsw ym2, ym6
mova [tmpq], ym2
add tmpq, 32
sub hd, 4
jg .v_w4_loop
RET
.v_w8:
mova m5, [bilin_v_perm8]
vbroadcasti32x4 ym0, [srcq+strideq*0]
.v_w8_loop:
vinserti32x4 ym1, ym0, [srcq+strideq*1], 1
vpbroadcastq ym0, [srcq+strideq*2]
vinserti32x4 m1, [srcq+stride3q ], 2
lea srcq, [srcq+strideq*4]
vinserti32x4 ym0, [srcq+strideq*0], 0
vpermt2b m1, m5, m0
pmaddubsw m1, m6
mova [tmpq], m1
add tmpq, 64
sub hd, 4
jg .v_w8_loop
RET
.v_w16:
mova m5, [bilin_v_perm16]
movu xm0, [srcq+strideq*0]
.v_w16_loop:
movu xm2, [srcq+strideq*2]
vinserti32x4 ym1, ym0, [srcq+strideq*1], 1
vpermt2b m1, m5, m2
vinserti32x4 ym2, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
movu xm0, [srcq+strideq*0]
vpermt2b m2, m5, m0
pmaddubsw m1, m6
pmaddubsw m2, m6
mova [tmpq+64*0], m1
mova [tmpq+64*1], m2
add tmpq, 64*2
sub hd, 4
jg .v_w16_loop
RET
.v_w32:
mova m5, [bilin_v_perm32]
movu ym0, [srcq+strideq*0]
.v_w32_loop:
movu ym2, [srcq+strideq*1]
movu ym3, [srcq+strideq*2]
movu ym4, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
vpermt2b m0, m5, m2
vpermt2b m2, m5, m3
vpermt2b m3, m5, m4
pmaddubsw m1, m0, m6
movu ym0, [srcq+strideq*0]
vpermt2b m4, m5, m0
pmaddubsw m2, m6
pmaddubsw m3, m6
pmaddubsw m4, m6
mova [tmpq+64*0], m1
mova [tmpq+64*1], m2
mova [tmpq+64*2], m3
mova [tmpq+64*3], m4
add tmpq, 64*4
sub hd, 4
jg .v_w32_loop
RET
.v_w64:
mova m5, [bilin_v_perm64]
vpermq m0, m5, [srcq+strideq*0]
.v_w64_loop:
vpermq m1, m5, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
punpcklbw m4, m0, m1
punpckhbw m2, m0, m1
vpermq m0, m5, [srcq+strideq*0]
punpcklbw m3, m1, m0
punpckhbw m1, m0
pmaddubsw m4, m6
pmaddubsw m2, m6
pmaddubsw m3, m6
pmaddubsw m1, m6
mova [tmpq+64*0], m4
mova [tmpq+64*1], m2
mova [tmpq+64*2], m3
mova [tmpq+64*3], m1
add tmpq, 64*4
sub hd, 2
jg .v_w64_loop
RET
.v_w128:
mova m5, [bilin_v_perm64]
vpermq m0, m5, [srcq+strideq*0+ 0]
vpermq m1, m5, [srcq+strideq*0+64]
.v_w128_loop:
vpermq m2, m5, [srcq+strideq*1+ 0]
vpermq m3, m5, [srcq+strideq*1+64]
lea srcq, [srcq+strideq*2]
punpcklbw m4, m0, m2
punpckhbw m0, m2
pmaddubsw m4, m6
pmaddubsw m0, m6
mova [tmpq+64*0], m4
mova [tmpq+64*1], m0
punpcklbw m4, m1, m3
punpckhbw m1, m3
pmaddubsw m4, m6
pmaddubsw m1, m6
mova [tmpq+64*2], m4
mova [tmpq+64*3], m1
vpermq m0, m5, [srcq+strideq*0+ 0]
vpermq m1, m5, [srcq+strideq*0+64]
punpcklbw m4, m2, m0
punpckhbw m2, m0
pmaddubsw m4, m6
pmaddubsw m2, m6
mova [tmpq+64*4], m4
mova [tmpq+64*5], m2
punpcklbw m4, m3, m1
punpckhbw m3, m1
pmaddubsw m4, m6
pmaddubsw m3, m6
mova [tmpq+64*6], m4
mova [tmpq+64*7], m3
add tmpq, 64*8
sub hd, 2
jg .v_w128_loop
RET
.hv:
; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4
; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4)
WIN64_SPILL_XMM 7
movzx wd, word [t2+wq*2+table_offset(prep, _bilin_hv)]
shl mxyd, 11
vpbroadcastw m6, mxyd
add wq, t2
lea stride3q, [strideq*3]
jmp wq
.hv_w4:
vbroadcasti32x4 ym4, [bilin_h_shuf4]
vpbroadcastq ym0, [srcq+strideq*0]
pshufb ym0, ym4
pmaddubsw ym0, ym5
.hv_w4_loop:
movq xmm1, [srcq+strideq*1]
movq xmm2, [srcq+strideq*2]
vinserti32x4 ym1, ymm1, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
vinserti32x4 ym2, ymm2, [srcq+strideq*0], 1
punpcklqdq ym1, ym2
pshufb ym1, ym4
pmaddubsw ym1, ym5 ; 1 2 3 4
valignq ym2, ym1, ym0, 3 ; 0 1 2 3
mova ym0, ym1
psubw ym1, ym2
pmulhrsw ym1, ym6
paddw ym1, ym2
mova [tmpq], ym1
add tmpq, 32
sub hd, 4
jg .hv_w4_loop
RET
.hv_w8:
vbroadcasti32x4 m4, [bilin_h_perm16]
vbroadcasti32x4 m0, [srcq+strideq*0]
pshufb m0, m4
pmaddubsw m0, m5
.hv_w8_loop:
movu xmm1, [srcq+strideq*1]
vinserti128 ym1, ymm1, [srcq+strideq*2], 1
vinserti128 m1, [srcq+stride3q ], 2
lea srcq, [srcq+strideq*4]
vinserti128 m1, [srcq+strideq*0], 3
pshufb m1, m4
pmaddubsw m1, m5 ; 1 2 3 4
valignq m2, m1, m0, 6 ; 0 1 2 3
mova m0, m1
psubw m1, m2
pmulhrsw m1, m6
paddw m1, m2
mova [tmpq], m1
add tmpq, 64
sub hd, 4
jg .hv_w8_loop
RET
.hv_w16:
mova m4, [bilin_h_perm16]
vbroadcasti32x8 m0, [srcq+strideq*0]
vpermb m0, m4, m0
pmaddubsw m0, m5
.hv_w16_loop:
movu ym1, [srcq+strideq*1]
vinserti32x8 m1, [srcq+strideq*2], 1
movu ym2, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
vinserti32x8 m2, [srcq+strideq*0], 1
vpermb m1, m4, m1
vpermb m2, m4, m2
pmaddubsw m1, m5 ; 1 2
vshufi32x4 m3, m0, m1, q1032 ; 0 1
pmaddubsw m0, m2, m5 ; 3 4
vshufi32x4 m2, m1, m0, q1032 ; 2 3
psubw m1, m3
pmulhrsw m1, m6
paddw m1, m3
psubw m3, m0, m2
pmulhrsw m3, m6
paddw m3, m2
mova [tmpq+64*0], m1
mova [tmpq+64*1], m3
add tmpq, 64*2
sub hd, 4
jg .hv_w16_loop
RET
.hv_w32:
mova m4, [bilin_h_perm32]
vpermb m0, m4, [srcq+strideq*0]
pmaddubsw m0, m5
.hv_w32_loop:
vpermb m1, m4, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
vpermb m2, m4, [srcq+strideq*0]
pmaddubsw m1, m5
psubw m3, m1, m0
pmulhrsw m3, m6
paddw m3, m0
pmaddubsw m0, m2, m5
psubw m2, m0, m1
pmulhrsw m2, m6
paddw m2, m1
mova [tmpq+64*0], m3
mova [tmpq+64*1], m2
add tmpq, 64*2
sub hd, 2
jg .hv_w32_loop
RET
.hv_w64:
mova m4, [bilin_h_perm32]
vpermb m0, m4, [srcq+32*0]
vpermb m1, m4, [srcq+32*1]
pmaddubsw m0, m5
pmaddubsw m1, m5
.hv_w64_loop:
add srcq, strideq
vpermb m2, m4, [srcq+32*0]
vpermb m3, m4, [srcq+32*1]
pmaddubsw m2, m5
pmaddubsw m3, m5
psubw m7, m2, m0
psubw m8, m3, m1
pmulhrsw m7, m6
pmulhrsw m8, m6
paddw m7, m0
mova m0, m2
paddw m8, m1
mova m1, m3
mova [tmpq+64*0], m7
mova [tmpq+64*1], m8
add tmpq, 64*2
dec hd
jg .hv_w64_loop
RET
.hv_w128:
mova m4, [bilin_h_perm32]
vpermb m0, m4, [srcq+32*0]
vpermb m1, m4, [srcq+32*1]
vpermb m2, m4, [srcq+32*2]
vpermb m3, m4, [srcq+32*3]
REPX {pmaddubsw x, m5}, m0, m1, m2, m3
.hv_w128_loop:
add srcq, strideq
vpermb m7, m4, [srcq+32*0]
vpermb m8, m4, [srcq+32*1]
vpermb m9, m4, [srcq+32*2]
vpermb m10, m4, [srcq+32*3]
REPX {pmaddubsw x, m5}, m7, m8, m9, m10
psubw m11, m7, m0
psubw m12, m8, m1
psubw m13, m9, m2
psubw m14, m10, m3
REPX {pmulhrsw x, m6}, m11, m12, m13, m14
paddw m11, m0
mova m0, m7
paddw m12, m1
mova m1, m8
paddw m13, m2
mova m2, m9
paddw m14, m3
mova m3, m10
mova [tmpq+64*0], m11
mova [tmpq+64*1], m12
mova [tmpq+64*2], m13
mova [tmpq+64*3], m14
add tmpq, 64*4
dec hd
jg .hv_w128_loop
RET
; int8_t subpel_filters[5][15][8]
%assign FILTER_REGULAR (0*15 << 16) | 3*15
%assign FILTER_SMOOTH (1*15 << 16) | 4*15
%assign FILTER_SHARP (2*15 << 16) | 3*15
%macro FN 4-5 ; fn, type, type_h, type_v, jmp_to
cglobal %1_%2_8bpc
mov t0d, FILTER_%3
%ifidn %3, %4
mov t1d, t0d
%else
mov t1d, FILTER_%4
%endif
%if %0 == 5 ; skip the jump in the last filter
jmp mangle(private_prefix %+ _%5 %+ SUFFIX)
%endif
%endmacro
%macro PUT_8TAP_H 4-5 0 ; dst/src, tmp[1-3], vpermb
%if %5
vpermb m%2, m6, m%1
vpermb m%3, m7, m%1
vpermb m%4, m8, m%1
%else
%if %2 < %4 ; reuse a previous value if possible
pshufb m%2, m%1, m6
%endif
pshufb m%3, m%1, m7
pshufb m%4, m%1, m8
%endif
mova m%1, m5
vpdpbusd m%1, m%2, m9
mova m%2, m5
vpdpbusd m%2, m%3, m9
vpdpbusd m%1, m%3, m10
vpdpbusd m%2, m%4, m10
packusdw m%1, m%2
psrlw m%1, 6
%endmacro
%if WIN64
DECLARE_REG_TMP 4, 5
%else
DECLARE_REG_TMP 7, 8
%endif
; Due to the use of vpdpbusd (which does 4 pixels per instruction) in
; the horizontal filter, 6-tap is only used for the vertical filter.
%define PUT_8TAP_FN FN put_8tap,
PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH, put_6tap_8bpc
PUT_8TAP_FN sharp_regular, SHARP, REGULAR, put_6tap_8bpc
PUT_8TAP_FN smooth, SMOOTH, SMOOTH, put_6tap_8bpc
PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR, put_6tap_8bpc
PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH, put_6tap_8bpc
PUT_8TAP_FN regular, REGULAR, REGULAR
cglobal put_6tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ns
%define base r8-put_avx512icl
imul mxd, mxm, 0x010101
add mxd, t0d ; 6tap_h, mx, 4tap_h
imul myd, mym, 0x010101
add myd, t1d ; 6tap_v, my, 4tap_v
lea r8, [put_avx512icl]
movsxd wq, wm
movifnidn hd, hm
test mxd, 0xf00
jnz .h
test myd, 0xf00
jnz .v
.put:
tzcnt wd, wd
movzx wd, word [r8+wq*2+table_offset(put,)]
add wq, r8
lea r6, [ssq*3]
lea r7, [dsq*3]
%if WIN64
pop r8
%endif
jmp wq
.v:
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
tzcnt r6d, wd
movzx r6d, word [r8+r6*2+table_offset(put, _6tap_v)]
vpbroadcastd m6, [pw_512]
lea myq, [base+subpel_filters+1+myq*8]
vpbroadcastw m7, [myq+0]
add r6, r8
vpbroadcastw m8, [myq+2]
mov nsq, ssq
vpbroadcastw m9, [myq+4]
neg nsq
jmp r6
.v_w2:
movd xmm2, [srcq+nsq*2]
pinsrw xmm2, [srcq+nsq*1], 2
pinsrw xmm2, [srcq+ssq*0], 4
pinsrw xmm2, [srcq+ssq*1], 6 ; 0 1 2 3
lea srcq, [srcq+ssq*2]
vpbroadcastd xmm0, [srcq+ssq*0]
palignr xmm3, xmm0, xmm2, 4 ; 1 2 3 4
punpcklbw xmm1, xmm2, xmm3 ; 01 12
punpckhbw xmm2, xmm3 ; 23 34
.v_w2_loop:
vpbroadcastd xmm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pmaddubsw xmm3, xmm1, xm7 ; a0 b0
mova xmm1, xmm2
pmaddubsw xmm2, xm8 ; a1 b1
paddw xmm3, xmm2
vpblendd xmm2, xmm0, xmm4, 0x02 ; 4 5
vpbroadcastd xmm0, [srcq+ssq*0]
vpblendd xmm4, xmm0, 0x02 ; 5 6
punpcklbw xmm2, xmm4 ; 67 78
pmaddubsw xmm4, xmm2, xm9 ; a3 b3
paddw xmm3, xmm4
pmulhrsw xmm3, xm6
packuswb xmm3, xmm3
pextrw [dstq+dsq*0], xmm3, 0
pextrw [dstq+dsq*1], xmm3, 2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w2_loop
RET
.v_w4:
movd xmm2, [srcq+nsq*2]
pinsrd xmm2, [srcq+nsq*1], 1
pinsrd xmm2, [srcq+ssq*0], 2
pinsrd xmm2, [srcq+ssq*1], 3 ; 0 1 2 3
lea srcq, [srcq+ssq*2]
vpbroadcastd xmm0, [srcq+ssq*0]
palignr xmm3, xmm0, xmm2, 4 ; 1 2 3 4
punpcklbw xmm1, xmm2, xmm3 ; 01 12
punpckhbw xmm2, xmm3 ; 23 34
.v_w4_loop:
vpbroadcastd xmm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pmaddubsw xmm3, xmm1, xm7 ; a0 b0
mova xmm1, xmm2
pmaddubsw xmm2, xm8 ; a1 b1
paddw xmm3, xmm2
vpblendd xmm2, xmm0, xmm4, 0x02 ; 4 5
vpbroadcastd xmm0, [srcq+ssq*0]
vpblendd xmm4, xmm0, 0x02 ; 5 6
punpcklbw xmm2, xmm4 ; 45 56
pmaddubsw xmm4, xmm2, xm9 ; a2 b2
paddw xmm3, xmm4
pmulhrsw xmm3, xm6
packuswb xmm3, xmm3
movd [dstq+dsq*0], xmm3
pextrd [dstq+dsq*1], xmm3, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w4_loop
RET
.v_w8:
movq xmm1, [srcq+nsq*2]
vpbroadcastq ymm3, [srcq+nsq*1]
vpbroadcastq ymm2, [srcq+ssq*0]
vpbroadcastq ymm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vpbroadcastq ymm0, [srcq+ssq*0]
vpblendd ymm1, ymm3, 0x30
vpblendd ymm3, ymm2, 0x30
punpcklbw ymm1, ymm3 ; 01 12
vpblendd ymm2, ymm4, 0x30
vpblendd ymm4, ymm0, 0x30
punpcklbw ymm2, ymm4 ; 23 34
.v_w8_loop:
vpbroadcastq ymm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pmaddubsw ymm3, ymm1, ym7 ; a0 b0
mova ymm1, ymm2
pmaddubsw ymm2, ym8 ; a1 b1
paddw ymm3, ymm2
vpblendd ymm2, ymm0, ymm4, 0x30
vpbroadcastq ymm0, [srcq+ssq*0]
vpblendd ymm4, ymm0, 0x30
punpcklbw ymm2, ymm4 ; 45 56
pmaddubsw ymm4, ymm2, ym9 ; a2 b2
paddw ymm3, ymm4
pmulhrsw ymm3, ym6
vextracti128 xmm4, ymm3, 1
packuswb xmm3, xmm4
movq [dstq+dsq*0], xmm3
movhps [dstq+dsq*1], xmm3
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w8_loop
vzeroupper
RET
.v_w16:
mova m5, [spel_v_perm16a]
vbroadcasti32x4 m1, [srcq+nsq*2]
vbroadcasti32x4 ym3, [srcq+nsq*1]
mov r6d, 0x0f
vbroadcasti32x4 m2, [srcq+ssq*0]
kmovb k1, r6d
vbroadcasti32x4 ym4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vbroadcasti32x4 m0, [srcq+ssq*0]
vshufpd m1{k1}, m3, m2, 0xcc
vshufpd m2{k1}, m4, m0, 0xcc
vpermb m1, m5, m1 ; 01 12
vpermb m2, m5, m2 ; 23 34
.v_w16_loop:
vbroadcasti32x4 ym4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pmaddubsw m3, m1, m7 ; a0 b0
mova m1, m2
pmaddubsw m2, m8 ; a1 b1
paddw m3, m2
mova m2, m0
vbroadcasti32x4 m0, [srcq+ssq*0]
vshufpd m2{k1}, m4, m0, 0xcc
vpermb m2, m5, m2 ; 45 56
pmaddubsw m4, m2, m9 ; a2 b2
paddw m3, m4
pmulhrsw m3, m6
vextracti32x8 ym4, m3, 1
packuswb ym3, ym4
mova [dstq+dsq*0], xm3
vextracti32x4 [dstq+dsq*1], ym3, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w16_loop
RET
.v_w32:
mova m10, [spel_v_perm32]
pmovzxbq m5, [pb_02461357]
vpshrdw m11, m10, m10, 8
movu ym0, [srcq+nsq*2]
vinserti32x8 m0, [srcq+nsq*1], 1
vpermb m1, m10, m0 ; 01
vinserti32x8 m0, [srcq+ssq*0], 0
vpermb m2, m11, m0 ; 12
vinserti32x8 m0, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
vpermb m3, m10, m0 ; 23
vinserti32x8 m0, [srcq+ssq*0], 0
vpermb m4, m11, m0 ; 34
.v_w32_loop:
vinserti32x8 m0, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
pmaddubsw m12, m1, m7
mova m1, m3
pmaddubsw m13, m2, m7
mova m2, m4
pmaddubsw m14, m3, m8
vpermb m3, m10, m0 ; 45
vinserti32x8 m0, [srcq+ssq*0], 0
pmaddubsw m15, m4, m8
vpermb m4, m11, m0 ; 56
paddw m12, m14
pmaddubsw m14, m3, m9
paddw m13, m15
pmaddubsw m15, m4, m9
paddw m12, m14
paddw m13, m15
pmulhrsw m12, m6
pmulhrsw m13, m6
packuswb m12, m13
vpermq m12, m5, m12
mova [dstq+dsq*0], ym12
vextracti32x8 [dstq+dsq*1], m12, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w32_loop
RET
.v_w64:
.v_w128:
lea r6d, [hq+wq*4-256]
.v_loop0:
movu m2, [srcq+nsq*2]
movu m4, [srcq+nsq*1]
lea r4, [srcq+ssq*2]
movu m11, [srcq+ssq*0]
movu m13, [srcq+ssq*1]
mov r7, dstq
movu m0, [r4 +ssq*0]
punpcklbw m1, m2, m4 ; 01l
punpckhbw m2, m4 ; 01h
punpcklbw m3, m4, m11 ; 12l
punpckhbw m4, m11 ; 12h
punpcklbw m10, m11, m13 ; 23l
punpckhbw m11, m13 ; 23h
punpcklbw m12, m13, m0 ; 34l
punpckhbw m13, m0 ; 34h
.v_loop:
movu m5, [r4+ssq*1]
pmaddubsw m14, m1, m7 ; a0l
mova m1, m10
pmaddubsw m10, m8 ; a1l
lea r4, [r4+ssq*2]
pmaddubsw m15, m2, m7 ; a0h
mova m2, m11
pmaddubsw m11, m8 ; a1h
paddw m14, m10
punpcklbw m10, m0, m5 ; 45l
paddw m15, m11
punpckhbw m11, m0, m5 ; 45h
pmaddubsw m0, m10, m9 ; a2l
paddw m14, m0
pmaddubsw m0, m11, m9 ; a2h
paddw m15, m0
movu m0, [r4+ssq*0]
pmulhrsw m14, m6
pmulhrsw m15, m6
packuswb m14, m15
pmaddubsw m15, m3, m7 ; b0l
mova m3, m12
pmaddubsw m12, m8 ; b1l
mova [r7+dsq*0], m14
pmaddubsw m14, m4, m7 ; b0h
mova m4, m13
pmaddubsw m13, m8 ; b1h
paddw m15, m12
punpcklbw m12, m5, m0 ; 56l
paddw m14, m13
punpckhbw m13, m5, m0 ; 56h
pmaddubsw m5, m12, m9 ; b2l
paddw m15, m5
pmaddubsw m5, m13, m9 ; b2h
paddw m14, m5
pmulhrsw m15, m6
pmulhrsw m14, m6
packuswb m15, m14
mova [r7+dsq*1], m15
lea r7, [r7+dsq*2]
sub hd, 2
jg .v_loop
add srcq, 64
add dstq, 64
movzx hd, r6b
sub r6d, 256
jg .v_loop0
RET
.h:
test myd, 0xf00
jz mangle(private_prefix %+ _put_8tap_8bpc_avx512icl).h2
.hv:
vpbroadcastd m9, [pd_34]
mova xm10, [spel_hv_end]
pxor xm0, xm0
cmp wd, 4
jg .hv_w8
movzx mxd, mxb
dec srcq
vpbroadcastd m7, [base+subpel_filters+mxq*8+2]
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
vpbroadcastq ym1, [base+subpel_filters+1+myq*8]
mov nsq, ssq
punpcklbw ym0, ym1
neg nsq
psraw ym0, 2 ; << 6
pshufd ym11, ym0, q0000
pshufd ym12, ym0, q1111
pshufd ym13, ym0, q2222
cmp wd, 4
je .hv_w4
vbroadcasti128 ym5, [subpel_h_shuf4]
movq xmm0, [srcq+nsq*2]
movhps xmm0, [srcq+nsq*1]
movq xmm2, [srcq+ssq*0]
movhps xmm2, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vpbroadcastq ymm1, [srcq+ssq*0]
vpblendd ymm0, ymm1, 0x30
pshufb xmm2, xm5 ; 2 3
pshufb ymm0, ym5 ; 0 1 4
mova xmm1, xm9
vpdpbusd xmm1, xmm2, xm7
mova ymm2, ym9
vpdpbusd ymm2, ymm0, ym7
packssdw ymm2, ymm1
psraw ymm2, 2
vextracti128 xmm0, ymm2, 1
vzeroupper
palignr xmm0, xmm2, 4
punpcklwd xmm1, xmm2, xmm0 ; 01 12
punpckhwd xmm2, xmm0 ; 23 34
.hv_w2_loop:
movq xmm3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
movhps xmm3, [srcq+ssq*0]
pmaddwd xmm4, xmm1, xm11 ; a0 b0
mova xmm1, xmm2
vpdpwssd xmm4, xmm2, xm12 ; a1 b1
pshufb xmm3, xm5
mova xmm2, xm9
vpdpbusd xmm2, xmm3, xm7
packssdw xmm3, xmm2, xmm2
psraw xmm3, 2
palignr xmm2, xmm3, xmm0, 12
mova xmm0, xmm3
punpcklwd xmm2, xmm3 ; 45 56
vpdpwssd xmm4, xmm2, xm13 ; a2 b2
packuswb xmm4, xmm4
pshufb xmm4, xm10
pextrw [dstq+dsq*0], xmm4, 0
pextrw [dstq+dsq*1], xmm4, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w2_loop
RET
.hv_w4:
movq xm2, [srcq+nsq*2]
vpbroadcastq ym1, [srcq+nsq*1]
vinserti32x4 ym2, [srcq+ssq*0], 1
vinserti32x4 m1, [srcq+ssq*1], 2 ; _ 1 3
lea srcq, [srcq+ssq*2]
vbroadcasti32x4 m5, [subpel_h_shufA]
vinserti32x4 m2, [srcq+ssq*0], 2 ; 0 2 4
pshufb m1, m5
mova m0, m9
pshufb m2, m5
mova m3, m9
vpdpbusd m0, m1, m7
mova ym1, [spel_hv_perm4a]
vpdpbusd m3, m2, m7
mova ym2, [spel_hv_perm4b]
mov r6d, 0x5555
mova ym6, [spel_hv_perm4d]
packssdw m0, m3
kmovw k1, r6d
psraw m0, 2 ; _ 0 1 2 3 4 5 6
vpermb ym1, ym1, ym0 ; 01 12
vpermb m2, m2, m0 ; 23 34
.hv_w4_loop:
movq xm3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vinserti32x4 ym3, [srcq+ssq*0], 1
pmaddwd ym4, ym1, ym11 ; a0 b0
mova ym1, ym2
pshufb ym3, ym5
mova ym0, ym9
vpdpbusd ym0, ym3, ym7
vpdpwssd ym4, ym2, ym12 ; a1 b1
vpsraw ym2{k1}, ym0, 2 ; 5 6
vpermb ym2, ym6, ym2 ; 45 56
vpdpwssd ym4, ym2, ym13 ; a2 b2
packuswb ym4, ym4
vpermb ym4, ym10, ym4
movd [dstq+dsq*0], xm4
pextrd [dstq+dsq*1], xm4, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w4_loop
RET
.hv_w8:
shr mxd, 16
sub srcq, 3
vpbroadcastd m11, [base+subpel_filters+mxq*8+0]
vpbroadcastd m12, [base+subpel_filters+mxq*8+4]
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
vpbroadcastq m1, [base+subpel_filters+1+myq*8]
mov nsq, ssq
punpcklbw m0, m1
neg nsq
psraw m0, 2 ; << 6
pshufd m13, m0, q0000
pshufd m14, m0, q1111
pshufd m15, m0, q2222
cmp wd, 8
jne .hv_w16
movu xm0, [srcq+nsq*2]
vinserti32x4 ym0, [srcq+nsq*1], 1
vbroadcasti32x4 m1, [subpel_h_shufA]
vinserti32x4 m0, [srcq+ssq*0], 2
vbroadcasti32x4 m4, [subpel_h_shufB]
vinserti32x4 m0, [srcq+ssq*1], 3
lea srcq, [srcq+ssq*2]
vbroadcasti32x4 m7, [subpel_h_shufC]
vbroadcasti32x4 ym5, [srcq+ssq*0]
vbroadcasti32x8 m6, [subpel_h_shufA]
pshufb m1, m0, m1 ; 0 1 2 3 0123
mova m2, m9
vpdpbusd m2, m1, m11
pshufb m4, m0, m4 ; 0 1 2 3 4567
mova m1, m9
vpdpbusd m1, m4, m11
pshufb m0, m7 ; 0 1 2 3 89ab
pshufb ym7, ym5, ym6 ; 4 0123 4567
mova ym3, ym9
vpdpbusd ym3, ym7, ym11
vbroadcasti32x8 m7, [subpel_h_shufB]
vpdpbusd m2, m4, m12
mova m4, [spel_hv_perm8a]
pshufb ym5, ym7 ; 4 4567 89ab
vpdpbusd m1, m0, m12
vpaddd m0, m4, [pb_32] {1to16}
vpdpbusd ym3, ym5, ym12
mova m5, [spel_hv_perm8b]
mov r6, 0x55555555ff00
packssdw m2, m1
vpmovsdw xm3, ym3
kmovq k1, r6
psraw m2, 2 ; 0 1 2 3
psraw xm3, 2 ; 4
vpermb m1, m4, m2 ; 01 12
kshiftrq k2, k1, 16
vpermt2b m2, m0, m3 ; 23 34
.hv_w8_loop:
vbroadcasti32x4 ym3, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vbroadcasti32x4 m3{k1}, [srcq+ssq*0]
pmaddwd m0, m1, m13 ; a0 b0
pshufb m1, m3, m6 ; 5 6 0123 4567
mova m4, m9
vpdpbusd m4, m1, m11
pshufb m3, m7 ; 5 6 4567 89ab
vpdpwssd m0, m2, m14 ; a1 b1
mova m1, m2
vpdpbusd m4, m3, m12
psraw m2{k2}, m4, 2 ; 53 64
vpermb m2, m5, m2 ; 45 56
vpdpwssd m0, m2, m15 ; a2 b2
packuswb m0, m0
vpermb m0, m10, m0
movq [dstq+dsq*0], xm0
movhps [dstq+dsq*1], xm0
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w8_loop
RET
.hv_w16:
movu m19, [spel_hv_perm16a]
vpbroadcastd m7, [pb_4]
lea r6d, [wq*2-32]
mova m6, [spel_hv_perm16b]
paddb m20, m7, m19
lea r6d, [hq+r6*8]
paddb m21, m7, m20
mova ym10, [spel_hv_end16]
paddb m7, m6
.hv_w16_loop0:
movu ym16, [srcq+nsq*2]
vinserti32x8 m16, [srcq+nsq*1], 1
lea r4, [srcq+ssq*2]
movu ym17, [srcq+ssq*0]
vinserti32x8 m17, [srcq+ssq*1], 1
mov r7, dstq
movu ym18, [r4 +ssq*0]
vpermb m2, m19, m16 ; 0 1 0123 89ab
mova m1, m9
vpermb m3, m21, m16 ; 0 1 89ab ghij
vpdpbusd m1, m2, m11
mova m2, m9
vpermb m4, m19, m17 ; 2 3 0123 89ab
vpdpbusd m2, m3, m12
mova m3, m9
vpermb m5, m21, m17 ; 2 3 89ab ghij
vpdpbusd m3, m4, m11
mova m4, m9
vpermb m0, m6, m18 ; 4 0145 2367 89cd abef
vpdpbusd m4, m5, m12
mova m5, m9
vpermb m16, m20, m16 ; 0 1 4567 cdef
vpdpbusd m5, m0, m11
vpermb m17, m20, m17 ; 2 3 4567 cdef
vpdpbusd m1, m16, m12
vpermb m18, m7, m18 ; 4 4589 67ab cdgh efij
vpdpbusd m2, m16, m11
vpdpbusd m3, m17, m12
vpdpbusd m4, m17, m11
vpdpbusd m5, m18, m12
packssdw m1, m2 ; 01
packssdw m3, m4 ; 23
REPX {psraw x, 2}, m1, m3, m5
vpshrdd m2, m1, m3, 16 ; 12
vpshrdd m4, m3, m5, 16 ; 34
.hv_w16_loop:
movu ym18, [r4+ssq*1]
lea r4, [r4+ssq*2]
vinserti32x8 m18, [r4+ssq*0], 1
pmaddwd m16, m1, m13 ; a0
vpermb m1, m19, m18 ; 5 6 0123 89ab
pmaddwd m17, m2, m13 ; b0
vpermb m2, m20, m18 ; 5 6 4567 cdef
mova m0, m9
vpdpbusd m0, m1, m11
vpermb m18, m21, m18
mova m1, m9
vpdpbusd m1, m2, m11
vpdpwssd m16, m3, m14 ; a1
vpdpwssd m17, m4, m14 ; b1
vpdpbusd m0, m2, m12
mova m2, m4
vpdpbusd m1, m18, m12
packssdw m0, m1
mova m1, m3
psraw m4, m0, 2 ; 5 6
vpshrdd m3, m2, m4, 16 ; 4 5
vpdpwssd m17, m4, m15 ; b2
vpdpwssd m16, m3, m15 ; a2
packuswb m16, m17
vpermb m16, m10, m16
mova [r7+dsq*0], xm16
vextracti128 [r7+dsq*1], ym16, 1
lea r7, [r7+dsq*2]
sub hd, 2
jg .hv_w16_loop
add srcq, 16
add dstq, 16
movzx hd, r6b
sub r6d, 1<<8
jg .hv_w16_loop0
vzeroupper
RET
PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP, put_8tap_8bpc
PUT_8TAP_FN regular_sharp, REGULAR, SHARP, put_8tap_8bpc
PUT_8TAP_FN sharp, SHARP, SHARP
cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
imul mxd, mxm, 0x010101
add mxd, t0d ; 8tap_h, mx, 4tap_h
imul myd, mym, 0x010101
add myd, t1d ; 8tap_v, my, 4tap_v
lea r8, [put_avx512icl]
movsxd wq, wm
movifnidn hd, hm
test mxd, 0xf00
jnz .h
test myd, 0xf00
jz mangle(private_prefix %+ _put_6tap_8bpc_avx512icl).put
.v:
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
tzcnt r6d, wd
lea myq, [base+subpel_filters+myq*8]
movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)]
vpbroadcastd m7, [pw_512]
vpbroadcastw m8, [myq+0]
add r6, r8
vpbroadcastw m9, [myq+2]
lea ss3q, [ssq*3]
vpbroadcastw m10, [myq+4]
sub srcq, ss3q
vpbroadcastw m11, [myq+6]
jmp r6
.v_w2:
movd xmm2, [srcq+ssq*0]
pinsrw xmm2, [srcq+ssq*1], 2
pinsrw xmm2, [srcq+ssq*2], 4
add srcq, ss3q
pinsrw xmm2, [srcq+ssq*0], 6 ; 0 1 2 3
movd xmm3, [srcq+ssq*1]
vpbroadcastd xmm1, [srcq+ssq*2]
add srcq, ss3q
vpbroadcastd xmm0, [srcq+ssq*0]
vpblendd xmm3, xmm3, xmm1, 0x02 ; 4 5
vpblendd xmm1, xmm1, xmm0, 0x02 ; 5 6
palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4
punpcklbw xmm3, xmm1 ; 45 56
punpcklbw xmm1, xmm2, xmm4 ; 01 12
punpckhbw xmm2, xmm4 ; 23 34
.v_w2_loop:
pmaddubsw xmm5, xmm1, xm8 ; a0 b0
mova xmm1, xmm2
pmaddubsw xmm2, xm9 ; a1 b1
paddw xmm5, xmm2
mova xmm2, xmm3
pmaddubsw xmm3, xm10 ; a2 b2
paddw xmm5, xmm3
vpbroadcastd xmm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7
vpbroadcastd xmm0, [srcq+ssq*0]
vpblendd xmm4, xmm4, xmm0, 0x02 ; 7 8
punpcklbw xmm3, xmm4 ; 67 78
pmaddubsw xmm4, xmm3, xm11 ; a3 b3
paddw xmm5, xmm4
pmulhrsw xmm5, xm7
packuswb xmm5, xmm5
pextrw [dstq+dsq*0], xmm5, 0
pextrw [dstq+dsq*1], xmm5, 2
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w2_loop
RET
.v_w4:
movd xmm2, [srcq+ssq*0]
pinsrd xmm2, [srcq+ssq*1], 1
pinsrd xmm2, [srcq+ssq*2], 2
add srcq, ss3q
pinsrd xmm2, [srcq+ssq*0], 3 ; 0 1 2 3
movd xmm3, [srcq+ssq*1]
vpbroadcastd xmm1, [srcq+ssq*2]
add srcq, ss3q
vpbroadcastd xmm0, [srcq+ssq*0]
vpblendd xmm3, xmm3, xmm1, 0x02 ; 4 5
vpblendd xmm1, xmm1, xmm0, 0x02 ; 5 6
palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4
punpcklbw xmm3, xmm1 ; 45 56
punpcklbw xmm1, xmm2, xmm4 ; 01 12
punpckhbw xmm2, xmm4 ; 23 34
.v_w4_loop:
vpbroadcastd xmm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pmaddubsw xmm5, xmm1, xm8 ; a0 b0
mova xmm1, xmm2
pmaddubsw xmm2, xm9 ; a1 b1
paddw xmm5, xmm2
mova xmm2, xmm3
pmaddubsw xmm3, xm10 ; a2 b2
paddw xmm5, xmm3
vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7
vpbroadcastd xmm0, [srcq+ssq*0]
vpblendd xmm4, xmm4, xmm0, 0x02 ; 7 8
punpcklbw xmm3, xmm4 ; 67 78
pmaddubsw xmm4, xmm3, xm11 ; a3 b3
paddw xmm5, xmm4
pmulhrsw xmm5, xm7
packuswb xmm5, xmm5
movd [dstq+dsq*0], xmm5
pextrd [dstq+dsq*1], xmm5, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w4_loop
RET
.v_w8:
movq xmm1, [srcq+ssq*0]
vpbroadcastq ymm0, [srcq+ssq*1]
vpbroadcastq ymm2, [srcq+ssq*2]
add srcq, ss3q
vpbroadcastq ymm5, [srcq+ssq*0]
vpbroadcastq ymm3, [srcq+ssq*1]
vpbroadcastq ymm4, [srcq+ssq*2]
add srcq, ss3q
vpblendd ymm1, ymm0, 0x30
vpblendd ymm0, ymm2, 0x30
punpcklbw ymm1, ymm0 ; 01 12
vpbroadcastq ymm0, [srcq+ssq*0]
vpblendd ymm2, ymm5, 0x30
vpblendd ymm5, ymm3, 0x30
punpcklbw ymm2, ymm5 ; 23 34
vpblendd ymm3, ymm4, 0x30
vpblendd ymm4, ymm0, 0x30
punpcklbw ymm3, ymm4 ; 45 56
.v_w8_loop:
vpbroadcastq ymm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pmaddubsw ymm5, ymm1, ym8 ; a0 b0
mova ymm1, ymm2
pmaddubsw ymm2, ym9 ; a1 b1
paddw ymm5, ymm2
mova ymm2, ymm3
pmaddubsw ymm3, ym10 ; a2 b2
paddw ymm5, ymm3
vpblendd ymm3, ymm0, ymm4, 0x30
vpbroadcastq ymm0, [srcq+ssq*0]
vpblendd ymm4, ymm4, ymm0, 0x30
punpcklbw ymm3, ymm4 ; 67 78
pmaddubsw ymm4, ymm3, ym11 ; a3 b3
paddw ymm5, ymm4
pmulhrsw ymm5, ym7
vextracti128 xmm4, ymm5, 1
packuswb xmm5, xmm4
movq [dstq+dsq*0], xmm5
movhps [dstq+dsq*1], xmm5
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w8_loop
vzeroupper
RET
.v_w16:
mova m12, [spel_v_perm16a]
vbroadcasti32x4 m1, [srcq+ssq*0]
vbroadcasti32x4 ym4, [srcq+ssq*1]
mov r6d, 0x0f
vbroadcasti32x4 m2, [srcq+ssq*2]
add srcq, ss3q
vbroadcasti32x4 ym5, [srcq+ssq*0]
kmovb k1, r6d
vbroadcasti32x4 m3, [srcq+ssq*1]
vbroadcasti32x4 ym6, [srcq+ssq*2]
add srcq, ss3q
vbroadcasti32x4 m0, [srcq+ssq*0]
vshufpd m1{k1}, m4, m2, 0xcc
vshufpd m2{k1}, m5, m3, 0xcc
vshufpd m3{k1}, m6, m0, 0xcc
vpermb m1, m12, m1 ; 01 12
vpermb m2, m12, m2 ; 23 34
vpermb m3, m12, m3 ; 45 56
.v_w16_loop:
pmaddubsw m4, m1, m8 ; a0 b0
mova m1, m2
pmaddubsw m5, m2, m9 ; a1 b1
mova m2, m3
pmaddubsw m6, m3, m10 ; a2 b2
mova m3, m0
paddw m4, m5
vbroadcasti32x4 ym5, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vbroadcasti32x4 m0, [srcq+ssq*0]
vshufpd m3{k1}, m5, m0, 0xcc
vpermb m3, m12, m3 ; 67 78
pmaddubsw m5, m3, m11 ; a3 b3
paddw m4, m6
paddw m4, m5
pmulhrsw m4, m7
vextracti32x8 ym5, m4, 1
packuswb ym4, ym5
mova [dstq+dsq*0], xm4
vextracti32x4 [dstq+dsq*1], ym4, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w16_loop
RET
.v_w32:
mova m12, [spel_v_perm32]
pmovzxbq m14, [pb_02461357]
vpshrdw m13, m12, m12, 8
movu ym0, [srcq+ssq*0]
vinserti32x8 m0, [srcq+ssq*1], 1
vpermb m1, m12, m0 ; 01
vinserti32x8 m0, [srcq+ssq*2], 0
add srcq, ss3q
vpermb m2, m13, m0 ; 12
vinserti32x8 m0, [srcq+ssq*0], 1
vpermb m3, m12, m0 ; 23
vinserti32x8 m0, [srcq+ssq*1], 0
vpermb m4, m13, m0 ; 34
vinserti32x8 m0, [srcq+ssq*2], 1
add srcq, ss3q
vpermb m5, m12, m0 ; 45
vinserti32x8 m0, [srcq+ssq*0], 0
vpermb m6, m13, m0 ; 56
.v_w32_loop:
vinserti32x8 m0, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
pmaddubsw m15, m1, m8
mova m1, m3
pmaddubsw m16, m2, m8
mova m2, m4
pmaddubsw m17, m3, m9
mova m3, m5
pmaddubsw m18, m4, m9
mova m4, m6
pmaddubsw m19, m5, m10
vpermb m5, m12, m0 ; 67
vinserti32x8 m0, [srcq+ssq*0], 0
pmaddubsw m20, m6, m10
vpermb m6, m13, m0 ; 78
paddw m15, m17
pmaddubsw m17, m5, m11
paddw m16, m18
pmaddubsw m18, m6, m11
paddw m15, m19
paddw m16, m20
paddw m15, m17
paddw m16, m18
pmulhrsw m15, m7
pmulhrsw m16, m7
packuswb m15, m16
vpermq m15, m14, m15
mova [dstq+dsq*0], ym15
vextracti32x8 [dstq+dsq*1], m15, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_w32_loop
vzeroupper
RET
.v_w64:
.v_w128:
lea r6d, [hq+wq*4-256]
mov r4, srcq
mov r7, dstq
.v_loop0:
movu m2, [srcq+ssq*0]
movu m4, [srcq+ssq*1]
movu m6, [srcq+ssq*2]
add srcq, ss3q
movu m13, [srcq+ssq*0]
movu m15, [srcq+ssq*1]
movu m17, [srcq+ssq*2]
add srcq, ss3q
movu m0, [srcq+ssq*0]
punpcklbw m1, m2, m4 ; 01l
punpckhbw m2, m4 ; 01h
punpcklbw m3, m4, m6 ; 12l
punpckhbw m4, m6 ; 12h
punpcklbw m5, m6, m13 ; 23l
punpckhbw m6, m13 ; 23h
punpcklbw m12, m13, m15 ; 34l
punpckhbw m13, m15 ; 34h
punpcklbw m14, m15, m17 ; 45l
punpckhbw m15, m17 ; 45h
punpcklbw m16, m17, m0 ; 56l
punpckhbw m17, m0 ; 56h
.v_loop:
pmaddubsw m18, m1, m8 ; a0l
mova m1, m5
pmaddubsw m19, m2, m8 ; a0h
mova m2, m6
pmaddubsw m20, m3, m8 ; b0l
mova m3, m12
pmaddubsw m21, m4, m8 ; b0h
mova m4, m13
pmaddubsw m5, m9 ; a1l
pmaddubsw m6, m9 ; a1h
pmaddubsw m12, m9 ; b1l
pmaddubsw m13, m9 ; b1h
paddw m18, m5
mova m5, m14
pmaddubsw m14, m10 ; a2l
paddw m19, m6
mova m6, m15
pmaddubsw m15, m10 ; a2h
paddw m20, m12
mova m12, m16
pmaddubsw m16, m10 ; b2l
paddw m21, m13
mova m13, m17
pmaddubsw m17, m10 ; b2h
paddw m18, m14
paddw m19, m15
paddw m20, m16
paddw m21, m17
movu m17, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
punpcklbw m14, m0, m17 ; 67l
punpckhbw m15, m0, m17 ; 67h
pmaddubsw m16, m14, m11 ; a3l
pmaddubsw m0, m15, m11 ; a3h
paddw m18, m16
paddw m19, m0
movu m0, [srcq+ssq*0]
punpcklbw m16, m17, m0 ; 78l
punpckhbw m17, m0 ; 78h
pmulhrsw m18, m7
pmulhrsw m19, m7
packuswb m18, m19
mova [dstq+dsq*0], m18
pmaddubsw m18, m16, m11 ; b3l
pmaddubsw m19, m17, m11 ; b3h
paddw m18, m20
paddw m19, m21
pmulhrsw m18, m7
pmulhrsw m19, m7
packuswb m18, m19
mova [dstq+dsq*1], m18
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .v_loop
add r4, 64
add r7, 64
movzx hd, r6b
mov srcq, r4
mov dstq, r7
sub r6d, 256
jg .v_loop0
vzeroupper
RET
.h:
test myd, 0xf00
jnz .hv
.h2:
vpbroadcastd m5, [pd_34] ; 2 + (8 << 2)
cmp wd, 4
jl .h_w2
vbroadcasti128 m6, [subpel_h_shufA]
je .h_w4
tzcnt wd, wd
vbroadcasti128 m7, [subpel_h_shufB]
vbroadcasti128 m8, [subpel_h_shufC]
shr mxd, 16
sub srcq, 3
movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)]
vpbroadcastd m9, [base+mxq*8+subpel_filters+0]
vpbroadcastd m10, [base+mxq*8+subpel_filters+4]
add wq, r8
jmp wq
.h_w2:
movzx mxd, mxb
dec srcq
mova xmm4, [subpel_h_shuf4]
vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2]
.h_w2_loop:
movq xmm0, [srcq+ssq*0]
movhps xmm0, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pshufb xmm0, xmm4
mova xmm1, xm5
vpdpbusd xmm1, xmm0, xmm3
packssdw xmm0, xmm1, xmm1
psraw xmm0, 6
packuswb xmm0, xm0
pextrw [dstq+dsq*0], xmm0, 0
pextrw [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w2_loop
RET
.h_w4:
movzx mxd, mxb
dec srcq
vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2]
.h_w4_loop:
movq xmm0, [srcq+ssq*0]
movq xmm1, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
pshufb xmm0, xm6
pshufb xmm1, xm6
mova xmm2, xm5
vpdpbusd xmm2, xmm0, xmm3
mova xmm0, xm5
vpdpbusd xmm0, xmm1, xmm3
packssdw xmm0, xmm2, xmm0
psraw xmm0, 6
packuswb xmm0, xmm0
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w4_loop
RET
.h_w8:
movu xm0, [srcq+ssq*0]
vinserti32x4 ym0, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
WRAP_YMM PUT_8TAP_H 0, 1, 2, 3
vpmovuswb xm0, ym0
movq [dstq+dsq*0], xm0
movhps [dstq+dsq*1], xm0
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w8
RET
.h_w16:
mova m6, [spel_h_perm16]
vpbroadcastd m8, [pb_4]
paddb m7, m8, m6
paddb m8, m7
.h_w16_loop:
movu ym0, [srcq+ssq*0]
vinserti32x8 m0, [srcq+ssq*1], 1
lea srcq, [srcq+ssq*2]
PUT_8TAP_H 0, 1, 2, 3, 1
vpmovuswb ym0, m0
mova [dstq+dsq*0], xm0
vextracti128 [dstq+dsq*1], ym0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w16_loop
RET
.h_w32:
movu ym0, [srcq+ssq*0+8*0]
vinserti32x8 m0, [srcq+ssq*1+8*0], 1
movu ym1, [srcq+ssq*0+8*1]
vinserti32x8 m1, [srcq+ssq*1+8*1], 1
lea srcq, [srcq+ssq*2]
PUT_8TAP_H 0, 2, 3, 4
PUT_8TAP_H 1, 4, 3, 2
packuswb m0, m1
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .h_w32
RET
.h_w64:
movu m0, [srcq+8*0]
movu m1, [srcq+8*1]
add srcq, ssq
PUT_8TAP_H 0, 2, 3, 4
PUT_8TAP_H 1, 4, 3, 2
packuswb m0, m1
mova [dstq], m0
add dstq, dsq
dec hd
jg .h_w64
RET
.h_w128:
movu m0, [srcq+8*0]
movu m2, [srcq+8*1]
movu m1, [srcq+8*8]
movu m3, [srcq+8*9]
add srcq, ssq
PUT_8TAP_H 0, 4, 11, 12
PUT_8TAP_H 2, 12, 11, 4
PUT_8TAP_H 1, 4, 11, 12
PUT_8TAP_H 3, 12, 11, 4
packuswb m0, m2
packuswb m1, m3
mova [dstq+64*0], m0
mova [dstq+64*1], m1
add dstq, dsq
dec hd
jg .h_w128
RET
.hv:
vpbroadcastd m9, [pd_34]
pxor xm0, xm0
cmp wd, 4
jg .hv_w8
movzx mxd, mxb
dec srcq
vpbroadcastd m7, [base+subpel_filters+mxq*8+2]
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
vpbroadcastq ym1, [base+subpel_filters+myq*8]
lea ss3q, [ssq*3]
mov r6, srcq
punpcklbw ym0, ym1
sub r6, ss3q
psraw ym0, 2 ; << 6
mova xm14, [spel_hv_end]
pshufd ym10, ym0, q0000
pshufd ym11, ym0, q1111
pshufd ym12, ym0, q2222
pshufd ym13, ym0, q3333
cmp wd, 4
je .hv_w4
vbroadcasti128 ym6, [subpel_h_shuf4]
movq xmm2, [r6+ssq*0]
movhps xmm2, [r6+ssq*1]
movq xmm0, [r6+ssq*2]
movhps xmm0, [srcq+ssq*0]
vpbroadcastq ymm3, [srcq+ssq*1]
vpbroadcastq ymm4, [srcq+ssq*2]
add srcq, ss3q
vpbroadcastq ymm1, [srcq+ssq*0]
vpblendd ymm2, ymm3, 0x30
vpblendd ymm0, ymm1, 0x30 ; 2 3 6 _
vpblendd ymm2, ymm4, 0xc0 ; 0 1 4 5
pshufb ymm2, ym6
pshufb ymm0, ym6
mova ymm1, ym9
vpdpbusd ymm1, ymm2, ym7
mova ymm2, ym9
vpdpbusd ymm2, ymm0, ym7
packssdw ymm2, ymm1, ymm2
psraw ymm2, 2
vextracti128 xmm3, ymm2, 1
palignr xmm4, xmm3, xmm2, 4
punpcklwd xmm1, xmm2, xmm4 ; 01 12
punpckhwd xmm2, xmm4 ; 23 34
pshufd xmm0, xmm3, q2121
punpcklwd xmm3, xmm0 ; 45 56
.hv_w2_loop:
movq xmm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
movhps xmm4, [srcq+ssq*0]
pmaddwd xmm5, xmm1, xm10 ; a0 b0
mova xmm1, xmm2
vpdpwssd xmm5, xmm2, xm11 ; a1 b1
pshufb xmm4, xm6
mova xmm2, xmm3
vpdpwssd xmm5, xmm3, xm12 ; a2 b2
mova xmm3, xm9
vpdpbusd xmm3, xmm4, xm7
packssdw xmm4, xmm3, xmm3
psraw xmm4, 2
palignr xmm3, xmm4, xmm0, 12
mova xmm0, xmm4
punpcklwd xmm3, xmm4 ; 67 78
vpdpwssd xmm5, xmm3, xm13 ; a3 b3
packuswb xmm5, xmm5
pshufb xmm5, xm14
pextrw [dstq+dsq*0], xmm5, 0
pextrw [dstq+dsq*1], xmm5, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w2_loop
vzeroupper
RET
.hv_w4:
movq xmm1, [r6+ssq*0]
vpbroadcastq ym2, [r6+ssq*1]
vinserti32x4 ym1, ymm1, [r6+ssq*2], 1
vinserti32x4 m2, [srcq+ssq*0], 2
vinserti32x4 m1, [srcq+ssq*1], 2
vinserti32x4 m2, [srcq+ssq*2], 3 ; _ 1 3 5
vbroadcasti32x4 m6, [subpel_h_shufA]
add srcq, ss3q
vinserti32x4 m1, [srcq+ssq*0], 3 ; 0 2 4 6
pshufb m2, m6
pshufb m1, m6
mova m0, m9
vpdpbusd m0, m2, m7
mova m4, m9
vpdpbusd m4, m1, m7
mova ym1, [spel_hv_perm4a]
mova ym2, [spel_hv_perm4b]
mova ym3, [spel_hv_perm4c]
packssdw m0, m4
psraw m0, 2 ; _ 0 1 2 3 4 5 6
mov r6d, 0x5555
vpermb ym1, ym1, ym0 ; 01 12
vpermb m2, m2, m0 ; 23 34
vpermb m3, m3, m0 ; 45 56
kmovw k1, r6d
mova ym15, [spel_hv_perm4d]
.hv_w4_loop:
movq xmm4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vinserti32x4 ym4, ymm4, [srcq+ssq*0], 1
pmaddwd ym5, ym1, ym10 ; a0 b0
mova ym1, ym2
pshufb ym4, ym6
mova ym0, ym9
vpdpbusd ym0, ym4, ym7
vpdpwssd ym5, ym2, ym11 ; a1 b1
mova ym2, ym3
vpdpwssd ym5, ym3, ym12 ; a2 b2
vpsraw ym3{k1}, ym0, 2 ; 7 8
vpermb ym3, ym15, ym3 ; 67 78
vpdpwssd ym5, ym3, ym13 ; a3 b3
packuswb ym5, ym5
vpermb ym5, ym14, ym5
movd [dstq+dsq*0], xm5
pextrd [dstq+dsq*1], xm5, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w4_loop
RET
.hv_w8:
shr mxd, 16
sub srcq, 3
vpbroadcastd m10, [base+subpel_filters+mxq*8+0]
vpbroadcastd m11, [base+subpel_filters+mxq*8+4]
movzx mxd, myb
shr myd, 16
cmp hd, 6
cmovs myd, mxd
vpbroadcastq m1, [base+subpel_filters+myq*8]
punpcklbw m0, m1
lea ss3q, [ssq*3]
psraw m0, 2 ; << 6
pshufd m12, m0, q0000
pshufd m13, m0, q1111
pshufd m14, m0, q2222
pshufd m15, m0, q3333
cmp wd, 8
jne .hv_w16
mov r6, srcq
sub r6, ss3q
movu xmm1, [r6+ssq*0]
vinserti128 ymm1, [r6+ssq*1], 1
movu xmm2, [srcq+ssq*1]
vinserti32x4 m6, zmm1, [r6+ssq*2], 2
vinserti128 ymm2, [srcq+ssq*2], 1
vinserti32x4 m6, [srcq+ssq*0], 3 ; 0 1 2 3
add srcq, ss3q
vbroadcasti32x4 m4, [subpel_h_shufA]
vinserti32x4 m0, zmm2, [srcq+ssq*0], 2 ; 4 5 6 _
vbroadcasti32x4 m7, [subpel_h_shufB]
vbroadcasti32x4 m8, [subpel_h_shufC]
pshufb m1, m6, m4 ; 0 1 2 3 0123
mova m2, m9
vpdpbusd m2, m1, m10
pshufb m5, m6, m7 ; 0 1 2 3 4567
mova m1, m9
vpdpbusd m1, m5, m10
pshufb m4, m0, m4 ; 4 5 6 _ 0123
mova m3, m9
vpdpbusd m3, m4, m10
pshufb m7, m0, m7 ; 4 5 6 _ 4567
mova m4, m9
vpdpbusd m4, m7, m10
pshufb m6, m8
vpdpbusd m2, m5, m11
vpdpbusd m1, m6, m11
pshufb m6, m0, m8
vpdpbusd m3, m7, m11
vpdpbusd m4, m6, m11
mova m5, [spel_hv_perm8a]
vpaddd m0, m5, [pb_32] {1to16}
mov r6, 0x55555555ff00
packssdw m2, m1
packssdw m3, m4
mova m8, [spel_hv_perm8b]
psraw m2, 2 ; 0 1 2 3
psraw m3, 2 ; 4 5 6 _
vpermb m1, m5, m2 ; 01 12
vbroadcasti32x8 m6, [subpel_h_shufA]
kmovq k1, r6
vpermt2b m2, m0, m3 ; 23 34
vbroadcasti32x8 m7, [subpel_h_shufB]
kshiftrq k2, k1, 16
mova xm16, [spel_hv_end]
vpermb m3, m5, m3 ; 45 56
.hv_w8_loop:
vbroadcasti32x4 ym4, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
vbroadcasti32x4 m4{k1}, [srcq+ssq*0]
pmaddwd m0, m1, m12 ; a0 b0
pshufb m1, m4, m6 ; 7 8 0123 4567
mova m5, m9
vpdpbusd m5, m1, m10
pshufb m4, m7 ; 7 8 4567 89ab
vpdpwssd m0, m2, m13 ; a1 b1
mova m1, m2
vpdpbusd m5, m4, m11
mova m2, m3
vpdpwssd m0, m3, m14 ; a2 b2
psraw m3{k2}, m5, 2 ; 75 86
vpermb m3, m8, m3 ; 67 78
vpdpwssd m0, m3, m15 ; a3 b3
packuswb m0, m0
vpermb zmm1, m16, m0
movq [dstq+dsq*0], xmm1
movhps [dstq+dsq*1], xmm1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .hv_w8_loop
vzeroupper
RET
.hv_w16:
WIN64_SPILL_XMM 23
movu m22, [spel_hv_perm16a]
sub srcq, ss3q
vpbroadcastd m8, [pb_4]
lea r6d, [wq*2-32]
mova m7, [spel_hv_perm16b]
paddb m20, m8, m22
mova ym16, [spel_hv_end16]
paddb m21, m8, m20
lea r6d, [hq+r6*8]
paddb m8, m7
.hv_w16_loop0:
movu ym17, [srcq+ssq*0]
vinserti32x8 m17, [srcq+ssq*1], 1 ; 0 1
lea r4, [srcq+ss3q]
movu ym18, [srcq+ssq*2]
vinserti32x8 m18, [r4 +ssq*0], 1 ; 2 3
mov r7, dstq
movu ym19, [r4 +ssq*1]
vinserti32x8 m19, [r4 +ssq*2], 1 ; 4 5
add r4, ss3q
vpermb m2, m22, m17 ; 0 1 0123 89ab
mova m1, m9
vpermb m3, m21, m17 ; 0 1 89ab ghij
vpdpbusd m1, m2, m10
mova m2, m9
vpermb m4, m22, m18 ; 2 3 0123 89ab
vpdpbusd m2, m3, m11
mova m3, m9
vpermb m5, m21, m18 ; 2 3 89ab ghij
vpdpbusd m3, m4, m10
mova m4, m9
vpermb m6, m22, m19 ; 4 5 0123 89ab
vpdpbusd m4, m5, m11
mova m5, m9
vpermb m17, m20, m17 ; 0 1 4567 cdef
vpdpbusd m5, m6, m10
mova m6, m9
vpermb m0, m21, m19 ; 4 5 89ab ghij
vpdpbusd m1, m17, m11
vpdpbusd m2, m17, m10
movu ym17, [r4+ssq*0] ; 6
vpermb m18, m20, m18 ; 2 3 4567 cdef
vpdpbusd m6, m0, m11
vpermb m0, m7, m17 ; 6 0145 2367 89cd abef
vpdpbusd m3, m18, m11
vpermb m19, m20, m19 ; 4 5 4567 cdef
vpdpbusd m4, m18, m10
mova m18, m9
vpermb m17, m8, m17 ; 6 4589 67ab cdgh efij
vpdpbusd m18, m0, m10
packssdw m1, m2
vpdpbusd m5, m19, m11
vpdpbusd m6, m19, m10
packssdw m3, m4
vpdpbusd m18, m17, m11
psraw m1, 2 ; 01
psraw m3, 2 ; 23
packssdw m5, m6
vpshrdd m2, m1, m3, 16 ; 12
psraw m5, 2 ; 45
vpshrdd m4, m3, m5, 16 ; 34
psraw m18, 2
vpshrdd m6, m5, m18, 16 ; 56
.hv_w16_loop:
movu ym19, [r4+ssq*1]
lea r4, [r4+ssq*2]
vinserti32x8 m19, [r4+ssq*0], 1
pmaddwd m17, m1, m12 ; a0
vpermb m1, m22, m19 ; 7 8 0123 89ab
pmaddwd m18, m2, m12 ; b0
mova m0, m9
vpermb m2, m21, m19 ; 7 8 89ab ghij
vpdpbusd m0, m1, m10
mova m1, m9
vpermb m19, m20, m19 ; 7 8 4567 cdef
vpdpbusd m1, m2, m11
mova m2, m4
vpdpwssd m17, m3, m13 ; a1
vpdpwssd m18, m4, m13 ; b1
mova m4, m6
vpdpbusd m0, m19, m11
vpdpbusd m1, m19, m10
vpdpwssd m17, m5, m14 ; a2
vpdpwssd m18, m6, m14 ; b2
packssdw m0, m1
mova m1, m3
psraw m6, m0, 2 ; 78
mova m3, m5
vpshrdd m5, m4, m6, 16 ; 67
vpdpwssd m18, m6, m15 ; b3
vpdpwssd m17, m5, m15 ; a3
packuswb m17, m18
vpermb m17, m16, m17
mova [r7+dsq*0], xm17
vextracti128 [r7+dsq*1], ym17, 1
lea r7, [r7+dsq*2]
sub hd, 2
jg .hv_w16_loop
add srcq, 16
add dstq, 16
movzx hd, r6b
sub r6d, 1<<8
jg .hv_w16_loop0
RET
%if WIN64
DECLARE_REG_TMP 6, 4
%else
DECLARE_REG_TMP 6, 7
%endif
%define PREP_8TAP_FN FN prep_8tap,
PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH, prep_6tap_8bpc
PREP_8TAP_FN sharp_regular, SHARP, REGULAR, prep_6tap_8bpc
PREP_8TAP_FN smooth, SMOOTH, SMOOTH, prep_6tap_8bpc
PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR, prep_6tap_8bpc
PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH, prep_6tap_8bpc
PREP_8TAP_FN regular, REGULAR, REGULAR
cglobal prep_6tap_8bpc, 4, 8, 0, tmp, src, ss, w, h, mx, my, ss3
%define base r7-prep_avx512icl
imul mxd, mxm, 0x010101
add mxd, t0d ; 6tap_h, mx, 4tap_h
imul myd, mym, 0x010101
add myd, t1d ; 6tap_v, my, 4tap_v
lea r7, [prep_avx512icl]
movifnidn hd, hm
test mxd, 0xf00
jnz .h
test myd, 0xf00
jnz .v
.prep:
tzcnt wd, wd
movzx wd, word [r7+wq*2+table_offset(prep,)]
add wq, r7
lea r6, [ssq*3]
%if WIN64
pop r7
%endif
jmp wq
.v:
movzx mxd, myb
shr myd, 16
cmp hd, 4
cmove myd, mxd
tzcnt r5d, wd
lea myq, [base+subpel_filters+1+myq*8]
movzx r5d, word [r7+r5*2+table_offset(prep, _6tap_v)]
vpbroadcastd m7, [pw_8192]
sub srcq, ssq
vpbroadcastw m8, [myq+0]
add r5, r7
vpbroadcastw m9, [myq+2]
lea ss3q, [ssq*3]
vpbroadcastw m10, [myq+4]
sub srcq, ssq
jmp r5
.v_w4:
movd xmm2, [srcq+ssq*0]
pinsrd xmm2, [srcq+ssq*1], 1
vpbroadcastd ymm1, [srcq+ssq*2]
add srcq, ss3q
vpbroadcastd ymm3, [srcq+ssq*0]
vpbroadcastd ymm0, [srcq+ssq*1]
vbroadcasti128 ymm5, [deint_shuf4]
vpblendd ymm1, ymm2, 0xeb
punpcklqdq ymm3, ymm0
vpblendd ymm1, ymm3, 0x60 ; 0 1 2 _ 2 3 4 _
pshufb ymm1, ymm5 ; 01 12 23 34
.v_w4_loop:
pinsrd xmm0, [srcq+ssq*2], 1
vpbroadcastd ymm2, [srcq+ss3q ]
lea srcq, [srcq+ssq*4]
vpbroadcastd ymm3, [srcq+ssq*0]
vpblendd ymm2, ymm0, 0xeb
vpbroadcastd ymm0, [srcq+ssq*1]
punpcklqdq ymm3, ymm0
vpblendd ymm2, ymm3, 0x60 ; 4 5 6 _ 6 7 8 _
pshufb ymm2, ymm5 ; 45 56 67 78
pmaddubsw ymm3, ymm1, ym8 ; a0 b0 c0 d0
vperm2i128 ymm1, ymm2, 0x21 ; 23 34 45 56
pmaddubsw ymm4, ymm2, ym10 ; a2 b2 c2 d2
pmaddubsw ymm1, ym9 ; a1 b1 c1 d1
paddw ymm3, ymm4
paddw ymm3, ymm1
pmulhrsw ymm3, ym7
mova ymm1, ymm2
mova [tmpq], ymm3
add tmpq, 32
sub hd, 4
jg .v_w4_loop
vzeroupper
RET
.v_w8:
mova m6, [spel_v_perm8]
movq xm1, [srcq+ssq*0]
mov r6d, 0x3e
movq xm2, [srcq+ssq*1]
kmovb k1, r6d
vpbroadcastq ym3, [srcq+ssq*2]
add srcq, ss3q
vpunpcklqdq ym2, [srcq+ssq*0] {1to4}
vpunpcklqdq m1{k1}, m3, [srcq+ssq*1] {1to8}
movq xm0, [srcq+ssq*1]
kshiftlb k2, k1, 2
shufpd m1, m2, 0x18 ; 0 1 2 3 4
vpermb m1, m6, m1 ; 01 12 23 34
.v_w8_loop:
vpbroadcastq ym3, [srcq+ss3q ]
vpunpcklqdq ym0{k1}, ym3, [srcq+ssq*2] {1to4}
lea srcq, [srcq+ssq*4]
vpbroadcastq m3, [srcq+ssq*1]
vpunpcklqdq m0{k2}, m3, [srcq+ssq*0] {1to8}
pmaddubsw m4, m1, m8 ; a0 b0 c0 d0
vpermb m2, m6, m0 ; 45 56 67 78
mova xm0, xm3
vshufi32x4 m1, m2, q1032 ; 23 34 45 56
pmaddubsw m3, m2, m10 ; a3 b3 c3 d3
pmaddubsw m5, m1, m9 ; a2 b2 c2 d2
mova m1, m2
paddw m4, m3
paddw m4, m5
pmulhrsw m4, m7
mova [tmpq], m4
add tmpq, 64
sub hd, 4
jg .v_w8_loop
RET
.v_w16:
mova m11, [spel_v_perm16b]
vbroadcasti32x4 m1, [srcq+ssq*0]
mov r6d, 0x0f
vbroadcasti32x4 ym3, [srcq+ssq*1]
vbroadcasti32x4 m2, [srcq+ssq*2]
kmovb k1, r6d
add srcq, ss3q
vbroadcasti32x4 ym4, [srcq+ssq*0]
vbroadcasti32x4 m0, [srcq+ssq*1]
vshufpd m1{k1}, m3, m2, 0xcc
vshufpd m2{k1}, m4, m0, 0xcc
vpermb m1, m11, m1 ; 01 12
vpermb m2, m11, m2 ; 23 34
.v_w16_loop:
pmaddubsw m3, m1, m8 ; a0 b0
pmaddubsw m5, m2, m9 ; a1 b1
vbroadcasti32x4 ym6, [srcq+ssq*2]
pmaddubsw m4, m2, m8 ; c0 d0
vbroadcasti32x4 m2, [srcq+ss3q ]
lea srcq, [srcq+ssq*4]
vshufpd m0{k1}, m6, m2, 0xcc
vbroadcasti32x4 ym6, [srcq+ssq*0]
vpermb m1, m11, m0 ; 45 56
vbroadcasti32x4 m0, [srcq+ssq*1]
vshufpd m2{k1}, m6, m0, 0xcc
pmaddubsw m6, m1, m9 ; c1 d1
vpermb m2, m11, m2 ; 67 78
paddw m3, m5
pmaddubsw m5, m1, m10 ; a2 b2
paddw m4, m6
pmaddubsw m6, m2, m10 ; c2 d2
paddw m3, m5
paddw m4, m6
pmulhrsw m3, m7
pmulhrsw m4, m7
mova [tmpq+ 0], m3
mova [tmpq+64], m4
add tmpq, 64*2
sub hd, 4
jg .v_w16_loop
RET
.v_w32:
movshdup m6, [bilin_v_perm64]
movu ym16, [srcq+ssq*0]
movu ym17, [srcq+ssq*1]
movu ym18, [srcq+ssq*2]
add srcq, ss3q
movu ym19, [srcq+ssq*0]
add srcq, ssq
movu ym20, [srcq+ssq*0]
vpermt2q m16, m6, m18 ; 0 2
vpermt2q m17, m6, m19 ; 1 3
vpermt2q m18, m6, m20 ; 2 4
punpcklbw m0, m16, m17 ; 01
punpcklbw m1, m17, m18 ; 12
punpckhbw m2, m16, m17 ; 23
punpckhbw m3, m17, m18 ; 34
.v_w32_loop:
movu ym16, [srcq+ssq*1]
lea srcq, [srcq+ssq*2]
movu ym17, [srcq+ssq*0]
pmaddubsw m4, m0, m8 ; a0
mova m0, m2
pmaddubsw m2, m9 ; a1
vpermt2q m16, m6, m17 ; 5 6
pmaddubsw m5, m1, m8 ; b0
mova m1, m3
pmaddubsw m3, m9 ; b1
shufpd m18, m16, 0x55 ; 4 5
paddw m4, m2
punpcklbw m2, m18, m16 ; 45
paddw m5, m3
punpckhbw m3, m18, m16 ; 56
mova m18, m16
pmaddubsw m16, m2, m10 ; a2
pmaddubsw m17, m3, m10 ; b2
paddw m4, m16
paddw m5, m17
pmulhrsw m4, m7
pmulhrsw m5, m7
mova [tmpq+ 0], m4
mova [tmpq+64], m5
add tmpq, 64*2
sub hd, 2
jg .v_w32_loop
vzeroupper
RET
.v_w64:
.v_w128:
mova m6, [bilin_v_perm64]
add wd, wd
lea r6d, [hq+wq]
.v_loop0:
vpermq m12, m6, [srcq+ssq*0]
vpermq m13, m6, [srcq+ssq*1]
lea r5, [srcq+ssq*2]
vpermq m14, m6, [r5 +ssq*0]
vpermq m15, m6, [r5 +ssq*1]
lea r5, [r5+ssq*2]
vpermq m16, m6, [r5 +ssq*0]
mov r7, tmpq
punpcklbw m0, m12, m13 ; 01
punpckhbw m12, m13
punpcklbw m1, m13, m14 ; 12
punpckhbw m13, m14
punpcklbw m2, m14, m15 ; 23
punpckhbw m14, m15
punpcklbw m3, m15, m16 ; 34
punpckhbw m15, m16
.v_loop:
pmaddubsw m17, m0, m8 ; a0
vpermq m5, m6, [r5+ssq*1]
pmaddubsw m18, m12, m8
mova m0, m2
pmaddubsw m2, m9 ; a1
mova m12, m14
pmaddubsw m14, m9
lea r5, [r5+ssq*2]
pmaddubsw m19, m1, m8 ; b0
pmaddubsw m20, m13, m8
mova m1, m3
pmaddubsw m3, m9 ; b1
mova m13, m15
pmaddubsw m15, m9
paddw m17, m2
punpcklbw m2, m16, m5 ; 67
paddw m18, m14
punpckhbw m14, m16, m5
vpermq m16, m6, [r5+ssq*0]
paddw m19, m3
pmaddubsw m3, m2, m10 ; a3
paddw m20, m15
pmaddubsw m15, m14, m10
paddw m17, m3
punpcklbw m3, m5, m16 ; 78
pmaddubsw m4, m3, m10 ; b3
paddw m18, m15
punpckhbw m15, m5, m16
pmaddubsw m5, m15, m10
paddw m19, m4
paddw m20, m5
REPX {pmulhrsw x, m7}, m17, m18, m19, m20
mova [r7+wq*0+ 0], m17
mova [r7+wq*0+64], m18
mova [r7+wq*1+ 0], m19
mova [r7+wq*1+64], m20
lea r7, [r7+wq*2]
sub hd, 2
jg .v_loop
add srcq, 64
add tmpq, 128
movzx hd, r6b
sub r6d, 1<<8
jg .v_loop0
vzeroupper
RET
.h:
test myd, 0xf00
jz mangle(private_prefix %+ _prep_8tap_8bpc_avx512icl).h2
.hv:
vpbroadcastd m8, [pd_2]
vpbroadcastd m9, [pd_32]
cmp wd, 4
jg .hv_w8
movzx mxd, mxb
vpbroadcastd m11, [base+subpel_filters+mxq*8+2]
movzx mxd, myb
shr myd, 16
cmp hd, 4
cmove myd, mxd
vpbroadcastq m3, [base+subpel_filters+1+myq*8]
vbroadcasti128 m10, [subpel_h_shufA]
lea r6, [ssq*2+1]
mov r3d, 0x30
sub srcq, r6
kmovb k1, r3d
vpbroadcastq ym2, [srcq+ssq*0]
lea ss3q, [ssq*3]
vpbroadcastq m1, [srcq+ssq*1]
kaddb k2, k1, k1
vpbroadcastq m2{k1}, [srcq+ssq*2]
add srcq, ss3q
vpbroadcastq m1{k2}, [srcq+ssq*0] ; _ _ 1 3
punpcklbw m3, m3
vpbroadcastq m2{k2}, [srcq+ssq*1] ; _ 0 2 4
psraw m3, 8 ; sign-extend
mova m6, [spel_hv_perm4a]
kshiftrb k1, k1, 2
movu m7, [spel_hv_perm4b]
pshufb m1, m10
mova m0, m8
vpdpbusd m0, m1, m11
pshufb m2, m10
mova m1, m8
vpdpbusd m1, m2, m11
pshufd m12, m3, q0000
pshufd m13, m3, q1111
pshufd m14, m3, q2222
packssdw m0, m1 ; _ _ _ 0 1 2 3 4
psraw m0, 2
vpermb m1, m7, m0 ; 01 12 23 34
.hv_w4_loop:
movq xm3, [srcq+ssq*2]
movq xm4, [srcq+ss3q ]
lea srcq, [srcq+ssq*4]
vpbroadcastq ym3{k1}, [srcq+ssq*0] ; 5 7
vpbroadcastq ym4{k1}, [srcq+ssq*1] ; 6 8
pshufb ym3, ym10
mova ym2, ym8
vpdpbusd ym2, ym3, ym11
pshufb ym4, ym10
mova ym3, ym8
vpdpbusd ym3, ym4, ym11
mova m4, m9
vpdpwssd m4, m1, m12 ; a0 b0 c0 d0
packssdw ym2, ym3 ; 5 6 7 8
psraw ym2, 2
vshufi32x4 m0, m2, q1032 ; _ 2 3 4 5 6 7 8
vpermb m2, m6, m0 ; 23 34 45 56
vpermb m1, m7, m0 ; 45 56 67 78
vpdpwssd m4, m2, m13 ; a1 b1 c1 d1
vpdpwssd m4, m1, m14 ; a2 b2 c2 d2
psrad m4, 6
vpmovdw [tmpq], m4
add tmpq, 32
sub hd, 4
jg .hv_w4_loop
RET
.hv_w8:
shr mxd, 16
vpbroadcastd m10, [base+subpel_filters+mxq*8+0]
vpbroadcastd m11, [base+subpel_filters+mxq*8+4]
movzx mxd, myb
shr myd, 16
cmp hd, 4
cmove myd, mxd
vpbroadcastq m0, [base+subpel_filters+1+myq*8]
lea r6, [ssq*2+3]
punpcklbw m0, m0
sub srcq, r6
psraw m0, 8 ; sign-extend
lea ss3q, [ssq*3]
pshufd m12, m0, q0000
pshufd m13, m0, q1111
pshufd m14, m0, q2222
cmp wd, 8
jg .hv_w16
movu xm16, [srcq+ssq*0]
vbroadcasti32x4 m19, [subpel_h_shufA]
vinserti128 ym16, [srcq+ssq*1], 1
vbroadcasti32x4 m21, [subpel_h_shufC]
vinserti32x4 m16, [srcq+ssq*2], 2
add srcq, ss3q
vinserti32x4 m16, [srcq+ssq*0], 3
movu xm17, [srcq+ssq*1]
vbroadcasti32x4 m20, [subpel_h_shufB]
pshufb m3, m16, m19 ; 0 1 2 3 0123
mova m2, m8
pshufb m0, m16, m21 ; 0 1 2 3 89ab
vpdpbusd m2, m3, m10
mova m3, m8
pshufb xm1, xm17, xm19 ; 3 4 5 6 0123
vpdpbusd m3, m0, m11
mova xm0, xm8
pshufb xm18, xm17, xm21 ; 3 4 5 6 89ab
vpdpbusd xm0, xm1, xm10
mova xm1, xm8
pshufb m16, m20 ; 0 1 2 3 4567
vpdpbusd xm1, xm18, xm11
pshufb xm17, xm20 ; 3 4 5 6 4567
vpdpbusd m2, m16, m11
vpdpbusd m3, m16, m10
vpdpbusd xm0, xm17, xm11
vpdpbusd xm1, xm17, xm10
packssdw m2, m3
packssdw xm0, xm1
psraw m2, 2 ; 0 1 2 3
psraw xm0, 2 ; 4
valignq m0, m2, 2 ; 1 2 3 4
punpcklwd m1, m2, m0 ; 01 12 23 34
punpckhwd m2, m0
.hv_w8_loop:
movu xm16, [srcq+ssq*2]
vinserti128 ym16, [srcq+ss3q ], 1
lea srcq, [srcq+ssq*4]
vinserti32x4 m16, [srcq+ssq*0], 2
vinserti32x4 m16, [srcq+ssq*1], 3
pshufb m6, m16, m19 ; 5 6 7 8 0123
mova m5, m8
pshufb m3, m16, m21 ; 5 6 7 8 89ab
vpdpbusd m5, m6, m10
mova m6, m8
pshufb m16, m20 ; 5 6 7 8 4567
vpdpbusd m6, m3, m11
mova m3, m9
vpdpwssd m3, m1, m12 ; a0 b0 c0 d0
mova m4, m9
vpdpwssd m4, m2, m12
vpdpbusd m5, m16, m11
vpdpbusd m6, m16, m10
mova m16, m1
packssdw m5, m6
mova m6, m2
psraw m5, 2 ; 5 6 7 8
valignq m2, m5, m0, 6 ; 4 5 6 7
mova m0, m5
punpcklwd m1, m2, m5 ; 45 56 67 78
punpckhwd m2, m5
vpdpwssd m3, m1, m14 ; a2 b2 c2 d2
vpdpwssd m4, m2, m14
vshufi32x4 m16, m1, q1032 ; 23 34 45 56
vshufi32x4 m6, m2, q1032
vpdpwssd m3, m16, m13 ; a1 b1 c1 d1
vpdpwssd m4, m6, m13
psrad m3, 6
psrad m4, 6
packssdw m3, m4
mova [tmpq], m3
add tmpq, 64
sub hd, 4
jg .hv_w8_loop
vzeroupper
RET
.hv_w16:
mova m16, [spel_h_perm16]
vpbroadcastd m18, [pb_4]
add wd, wd
paddb m17, m18, m16
lea r6d, [hq+wq*8-256]
paddb m18, m17
.hv_w16_loop0:
movu ym19, [srcq+ssq*0]
vinserti32x8 m19, [srcq+ssq*1], 1
lea r5, [srcq+ssq*2]
movu ym20, [r5 +ssq*0]
vinserti32x8 m20, [r5 +ssq*1], 1
lea r5, [r5 +ssq*2]
movu ym21, [r5 +ssq*0]
mov r7, tmpq
vpermb m3, m16, m19 ; 0 1 0123 89ab
mova m2, m8
vpermb m4, m18, m19 ; 0 1 89ab ghij
vpdpbusd m2, m3, m10
mova m3, m8
vpermb m5, m16, m20 ; 2 3 0123 89ab
vpdpbusd m3, m4, m11
mova m4, m8
vpermb m0, m18, m20 ; 2 3 89ab ghij
vpdpbusd m4, m5, m10
mova m5, m8
vpermb ym1, ym16, ym21 ; 4 0123 89ab
vpdpbusd m5, m0, m11
mova ym0, ym8
vpermb ym6, ym18, ym21 ; 4 89ab ghij
vpdpbusd ym0, ym1, ym10
mova ym1, ym8
vpermb m19, m17, m19 ; 0 1 4567 cdef
vpdpbusd ym1, ym6, ym11
vpermb m20, m17, m20 ; 2 3 4567 cdef
vpdpbusd m2, m19, m11
vpdpbusd m3, m19, m10
vpermb ym21, ym17, ym21 ; 4 4567 cdef
vpdpbusd m4, m20, m11
vpdpbusd m5, m20, m10
vpdpbusd ym0, ym21, ym11
vpdpbusd ym1, ym21, ym10
packssdw m2, m3 ; 0 1
packssdw m4, m5 ; 2 3
packssdw ym0, ym1 ; 4
REPX {psraw x, 2}, m2, m4, ym0
vshufi32x4 m3, m2, m4, q1032 ; 1 2
vshufi32x4 m0, m4, m0, q1032 ; 3 4
punpcklwd m1, m2, m3 ; 01 12
punpckhwd m2, m3
punpcklwd m3, m4, m0 ; 23 34
punpckhwd m4, m0
.hv_w16_loop:
movu ym19, [r5+ssq*1]
lea r5, [r5+ssq*2]
vinserti32x8 m19, [r5+ssq*0], 1
vpermb m6, m16, m19 ; 5 6 0123 89ab
mova m5, m8
vpermb m20, m18, m19 ; 5 6 89ab ghij
vpdpbusd m5, m6, m10
mova m6, m8
vpermb m19, m17, m19 ; 5 6 4567 cdef
vpdpbusd m6, m20, m11
mova m20, m9
vpdpwssd m20, m1, m12 ; a0 b0
mova m21, m9
vpdpwssd m21, m2, m12
vpdpbusd m5, m19, m11
vpdpbusd m6, m19, m10
vpdpwssd m20, m3, m13 ; a1 b1
vpdpwssd m21, m4, m13
packssdw m5, m6
mova m1, m3
psraw m5, 2 ; 5 6
mova m2, m4
vshufi32x4 m4, m0, m5, q1032 ; 4 5
mova m0, m5
punpcklwd m3, m4, m0 ; 45 56
punpckhwd m4, m0
vpdpwssd m20, m3, m14 ; a2 b2
vpdpwssd m21, m4, m14
psrad m20, 6
psrad m21, 6
packssdw m20, m21
mova [r7+wq*0], ym20
vextracti32x8 [r7+wq*1], m20, 1
lea r7, [r7+wq*2]
sub hd, 2
jg .hv_w16_loop
add srcq, 16
add tmpq, 32
movzx hd, r6b
sub r6d, 1<<8
jg .hv_w16_loop0
vzeroupper
RET
%macro PREP_8TAP_H 0
vpermb m10, m5, m0
vpermb m11, m5, m1
vpermb m12, m6, m0
vpermb m13, m6, m1
vpermb m14, m7, m0
vpermb m15, m7, m1
mova m0, m4
vpdpbusd m0, m10, m8
mova m2, m4
vpdpbusd m2, m12, m8
mova m1, m4
vpdpbusd m1, m11, m8
mova m3, m4
vpdpbusd m3, m13, m8
vpdpbusd m0, m12, m9
vpdpbusd m2, m14, m9
vpdpbusd m1, m13, m9
vpdpbusd m3, m15, m9
packssdw m0, m2
packssdw m1, m3
psraw m0, 2
psraw m1, 2
mova [tmpq+64*0], m0
mova [tmpq+64*1], m1
%endmacro
PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP, prep_8tap_8bpc
PREP_8TAP_FN regular_sharp, REGULAR, SHARP, prep_8tap_8bpc
PREP_8TAP_FN sharp, SHARP, SHARP
cglobal prep_8tap_8bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my, stride3
imul mxd, mxm, 0x010101
add mxd, t0d ; 8tap_h, mx, 4tap_h
imul myd, mym, 0x010101
add myd, t1d ; 8tap_v, my, 4tap_v
lea r7, [prep_avx512icl]
movifnidn hd, hm
test mxd, 0xf00
jnz .h
test myd, 0xf00
jz mangle(private_prefix %+ _prep_6tap_8bpc_avx512icl).prep
.v:
movzx mxd, myb ; Select 4-tap/8-tap filter multipliers.
shr myd, 16 ; Note that the code is 8-tap only, having
cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4
cmove myd, mxd ; had a negligible effect on performance.
tzcnt r5d, wd
lea myq, [base+subpel_filters+myq*8]
movzx r5d, word [r7+r5*2+table_offset(prep, _8tap_v)]
vpbroadcastd m7, [pw_8192]
vpbroadcastw m8, [myq+0]
add r5, r7
vpbroadcastw m9, [myq+2]
lea stride3q, [strideq*3]
vpbroadcastw m10, [myq+4]
sub srcq, stride3q
vpbroadcastw m11, [myq+6]
jmp r5
.v_w4:
movd xmm0, [srcq+strideq*0]
vpbroadcastd ymm1, [srcq+strideq*2]
vpbroadcastd xmm2, [srcq+strideq*1]
vpbroadcastd ymm3, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
vpblendd ymm1, ymm0, 0x01 ; 0 2 2 _ 2 _ _ _
vpblendd ymm3, ymm2, 0x03 ; 1 1 3 3 3 3 _ _
vpbroadcastd ymm0, [srcq+strideq*0]
vpbroadcastd ymm2, [srcq+strideq*1]
vpblendd ymm1, ymm0, 0x68 ; 0 2 2 4 2 4 4 _
vpbroadcastd ymm0, [srcq+strideq*2]
vbroadcasti128 ymm5, [deint_shuf4]
vpblendd ymm3, ymm2, 0xc0 ; 1 1 3 3 3 3 5 5
vpblendd ymm2, ymm3, ymm1, 0x55 ; 0 1 2 3 2 3 4 5
vpblendd ymm3, ymm1, 0xaa ; 1 2 3 4 3 4 5 _
punpcklbw ymm1, ymm2, ymm3 ; 01 12 23 34
vpblendd ymm3, ymm0, 0x80 ; 1 2 3 4 3 4 5 6
punpckhbw ymm2, ymm3 ; 23 34 45 56
.v_w4_loop:
pinsrd xmm0, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
vpbroadcastd ymm3, [srcq+strideq*0]
vpbroadcastd ymm4, [srcq+strideq*1]
vpblendd ymm3, ymm4, 0x20 ; _ _ 8 _ 8 9 _ _
vpblendd ymm3, ymm0, 0x03 ; 6 7 8 _ 8 9 _ _
vpbroadcastd ymm0, [srcq+strideq*2]
vpblendd ymm3, ymm0, 0x40 ; 6 7 8 _ 8 9 a _
pshufb ymm3, ymm5 ; 67 78 89 9a
pmaddubsw ymm4, ymm1, ym8
vperm2i128 ymm1, ymm2, ymm3, 0x21 ; 45 56 67 78
pmaddubsw ymm2, ym9
paddw ymm4, ymm2
mova ymm2, ymm3
pmaddubsw ymm3, ym11
paddw ymm3, ymm4
pmaddubsw ymm4, ymm1, ym10
paddw ymm3, ymm4
pmulhrsw ymm3, ym7
mova [tmpq], ymm3
add tmpq, 32
sub hd, 4
jg .v_w4_loop
vzeroupper
RET
.v_w8:
mova m6, [spel_v_perm8]
movq xm1, [srcq+strideq*0]
mov r6d, 0x3e
movq xm2, [srcq+strideq*1]
vpbroadcastq ym3, [srcq+strideq*2]
kmovb k1, r6d
vpbroadcastq ym4, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
vpunpcklqdq m1{k1}, m3, [srcq+strideq*0] {1to8}
vpunpcklqdq m2{k1}, m4, [srcq+strideq*1] {1to8}
movq xm0, [srcq+strideq*2]
kshiftlb k2, k1, 2
shufpd m1, m2, 0x30 ; 0 1 2 3 4 5
vshufi32x4 m2, m1, m0, q0021 ; 2 3 4 5 6 _
vpermb m1, m6, m1 ; 01 12 23 34
vpermb m2, m6, m2 ; 23 34 45 56
.v_w8_loop:
vpbroadcastq ym3, [srcq+strideq*4]
vpunpcklqdq ym0{k1}, ym3, [srcq+stride3q] {1to4}
lea srcq, [srcq+strideq*4]
vpbroadcastq m3, [srcq+strideq*2]
vpunpcklqdq m0{k2}, m3, [srcq+strideq*1] {1to8}
pmaddubsw m4, m1, m8 ; a0 b0 c0 d0
mova m1, m2
pmaddubsw m5, m2, m9 ; a1 b1 c1 d1
vpermb m2, m6, m0 ; 67 78 89 9a
mova xm0, xm3
vshufi32x4 m1, m2, q1032 ; 45 56 67 78
pmaddubsw m3, m2, m11 ; a3 b3 c3 d3
paddw m4, m5
pmaddubsw m5, m1, m10 ; a2 b2 c2 d2
paddw m4, m3
paddw m4, m5
pmulhrsw m4, m7
mova [tmpq], m4
add tmpq, 64
sub hd, 4
jg .v_w8_loop
RET
.v_w16:
mova m12, [spel_v_perm16b]
vbroadcasti32x4 m1, [srcq+strideq*0]
mov r6d, 0x0f
vbroadcasti32x4 ym4, [srcq+strideq*1]
vbroadcasti32x4 m2, [srcq+strideq*2]
kmovb k1, r6d
vbroadcasti32x4 ym5, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
vbroadcasti32x4 m3, [srcq+strideq*0]
vbroadcasti32x4 ym6, [srcq+strideq*1]
vbroadcasti32x4 m0, [srcq+strideq*2]
vshufpd m1{k1}, m4, m2, 0xcc
vshufpd m2{k1}, m5, m3, 0xcc
vshufpd m3{k1}, m6, m0, 0xcc
vpermb m1, m12, m1 ; 01 12
vpermb m2, m12, m2 ; 23 34
vpermb m3, m12, m3 ; 45 56
.v_w16_loop:
pmaddubsw m4, m1, m8 ; a0 b0
mova m1, m3
pmaddubsw m13, m2, m9 ; a1 b1
vbroadcasti32x4 ym6, [srcq+stride3q ]
pmaddubsw m5, m2, m8 ; c0 d0
lea srcq, [srcq+strideq*4]
pmaddubsw m14, m3, m9 ; c1 d1
vbroadcasti32x4 m3, [srcq+strideq*0]
vshufpd m0{k1}, m6, m3, 0xcc
vbroadcasti32x4 ym6, [srcq+strideq*1]
vpermb m2, m12, m0 ; 67 78
vbroadcasti32x4 m0, [srcq+strideq*2]
vshufpd m3{k1}, m6, m0, 0xcc
paddw m4, m13
pmaddubsw m13, m1, m10 ; a2 b2
vpermb m3, m12, m3 ; 89 9a
paddw m5, m14
pmaddubsw m14, m2, m10 ; c2 d2
pmaddubsw m15, m2, m11 ; a3 b3
pmaddubsw m6, m3, m11 ; c3 d3
paddw m4, m13
paddw m5, m14
paddw m4, m15
paddw m5, m6
pmulhrsw m4, m7
pmulhrsw m5, m7
mova [tmpq+ 0], m4
mova [tmpq+64], m5
add tmpq, 64*2
sub hd, 4
jg .v_w16_loop
RET
.v_w32:
movshdup m21, [bilin_v_perm64]
movu ym16, [srcq+strideq*0]
movu ym17, [srcq+strideq*1]
movu ym18, [srcq+strideq*2]
add srcq, stride3q
movu ym19, [srcq+strideq*0]
vpermt2q m16, m21, m19 ; 0 3
movu ym20, [srcq+strideq*1]
vpermt2q m17, m21, m20 ; 1 4
movu ym20, [srcq+strideq*2]
add srcq, stride3q
vpermt2q m18, m21, m20 ; 2 5
movu ym20, [srcq+strideq*0]
vpermt2q m19, m21, m20 ; 3 6
punpcklbw m0, m16, m17 ; 01
punpcklbw m1, m17, m18 ; 12
punpcklbw m2, m18, m19 ; 23
punpckhbw m3, m16, m17 ; 34
punpckhbw m4, m17, m18 ; 45
punpckhbw m5, m18, m19 ; 56
.v_w32_loop:
movu ym16, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
movu ym17, [srcq+strideq*0]
pmaddubsw m14, m0, m8
mova m0, m2
pmaddubsw m15, m1, m8
mova m1, m3
pmaddubsw m2, m9
vpermt2q m16, m21, m17 ; 7 8
pmaddubsw m3, m9
pmaddubsw m12, m4, m10
pmaddubsw m13, m5, m10
shufpd m19, m16, 0x55 ; 6 7
paddw m14, m2
mova m2, m4
punpcklbw m4, m19, m16 ; 67
paddw m15, m3
mova m3, m5
punpckhbw m5, m19, m16 ; 78
paddw m14, m12
paddw m15, m13
pmaddubsw m12, m4, m11
pmaddubsw m13, m5, m11
mova m19, m16
paddw m14, m12
paddw m15, m13
pmulhrsw m14, m7
pmulhrsw m15, m7
mova [tmpq+ 0], m14
mova [tmpq+64], m15
add tmpq, 64*2
sub hd, 2
jg .v_w32_loop
vzeroupper
RET
.v_w64:
.v_w128:
WIN64_SPILL_XMM 24
mova m23, [bilin_v_perm64]
add wd, wd
lea r6d, [hq+wq]
.v_loop0:
vpermq m12, m23, [srcq+strideq*0]
vpermq m13, m23, [srcq+strideq*1]
lea r5, [srcq+strideq*2]
vpermq m14, m23, [r5 +strideq*0]
vpermq m15, m23, [r5 +strideq*1]
lea r5, [r5+strideq*2]
vpermq m16, m23, [r5 +strideq*0]
vpermq m17, m23, [r5 +strideq*1]
lea r5, [r5+strideq*2]
vpermq m18, m23, [r5 +strideq*0]
mov r7, tmpq
punpcklbw m0, m12, m13 ; 01
punpckhbw m12, m13
punpcklbw m1, m13, m14 ; 12
punpckhbw m13, m14
punpcklbw m2, m14, m15 ; 23
punpckhbw m14, m15
punpcklbw m3, m15, m16 ; 34
punpckhbw m15, m16
punpcklbw m4, m16, m17 ; 45
punpckhbw m16, m17
punpcklbw m5, m17, m18 ; 56
punpckhbw m17, m18
.v_loop:
pmaddubsw m19, m0, m8 ; a0
vpermq m6, m23, [r5+strideq*1]
pmaddubsw m20, m12, m8
mova m0, m2
pmaddubsw m2, m9 ; a1
mova m12, m14
pmaddubsw m14, m9
lea r5, [r5+strideq*2]
pmaddubsw m21, m1, m8 ; b0
pmaddubsw m22, m13, m8
mova m1, m3
pmaddubsw m3, m9 ; b1
mova m13, m15
pmaddubsw m15, m9
paddw m19, m2
mova m2, m4
pmaddubsw m4, m10 ; a2
paddw m20, m14
mova m14, m16
pmaddubsw m16, m10
paddw m21, m3
mova m3, m5
pmaddubsw m5, m10 ; b2
paddw m22, m15
mova m15, m17
pmaddubsw m17, m10
paddw m19, m4
punpcklbw m4, m18, m6 ; 67
paddw m20, m16
punpckhbw m16, m18, m6
vpermq m18, m23, [r5+strideq*0]
paddw m21, m5
pmaddubsw m5, m4, m11 ; a3
paddw m22, m17
pmaddubsw m17, m16, m11
paddw m19, m5
punpcklbw m5, m6, m18 ; 78
paddw m20, m17
punpckhbw m17, m6, m18
pmaddubsw m6, m5, m11 ; b3
paddw m21, m6
pmaddubsw m6, m17, m11
paddw m22, m6
REPX {pmulhrsw x, m7}, m19, m20, m21, m22
mova [r7+wq*0+ 0], m19
mova [r7+wq*0+64], m20
mova [r7+wq*1+ 0], m21
mova [r7+wq*1+64], m22
lea r7, [r7+wq*2]
sub hd, 2
jg .v_loop
add srcq, 64
add tmpq, 128
movzx hd, r6b
sub r6d, 1<<8
jg .v_loop0
RET
.h:
RESET_STACK_STATE
test myd, 0xf00
jnz .hv
.h2:
vpbroadcastd m4, [pd_2]
cmp wd, 4
je .h_w4
tzcnt wd, wd
shr mxd, 16
sub srcq, 3
movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)]
vpbroadcastd m8, [base+subpel_filters+mxq*8+0]
vpbroadcastd m9, [base+subpel_filters+mxq*8+4]
add wq, r7
jmp wq
.h_w4:
movzx mxd, mxb
vbroadcasti128 ym5, [subpel_h_shufA]
mov r3d, 0x4
dec srcq
vpbroadcastd ym6, [base+subpel_filters+mxq*8+2]
kmovb k1, r3d
lea stride3q, [strideq*3]
.h_w4_loop:
movq xm2, [srcq+strideq*0]
movq xm3, [srcq+strideq*1]
vpbroadcastq ym2{k1}, [srcq+strideq*2]
vpbroadcastq ym3{k1}, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
pshufb ym2, ym5
pshufb ym3, ym5
mova ym0, ym4
vpdpbusd ym0, ym2, ym6
mova ym1, ym4
vpdpbusd ym1, ym3, ym6
packssdw ym0, ym1
psraw ym0, 2
mova [tmpq], ym0
add tmpq, 32
sub hd, 4
jg .h_w4_loop
RET
.h_w8:
vbroadcasti128 m5, [subpel_h_shufA]
vbroadcasti128 m6, [subpel_h_shufB]
vbroadcasti128 m7, [subpel_h_shufC]
lea stride3q, [strideq*3]
.h_w8_loop:
movu xmm3, [srcq+strideq*0]
vinserti128 ym3, ymm3, [srcq+strideq*1], 1
vinserti128 m3, [srcq+strideq*2], 2
vinserti128 m3, [srcq+stride3q ], 3
lea srcq, [srcq+strideq*4]
pshufb m1, m3, m5
pshufb m2, m3, m6
mova m0, m4
vpdpbusd m0, m1, m8
mova m1, m4
vpdpbusd m1, m2, m8
pshufb m3, m7
vpdpbusd m0, m2, m9
vpdpbusd m1, m3, m9
packssdw m0, m1
psraw m0, 2
mova [tmpq], m0
add tmpq, 64
sub hd, 4
jg .h_w8_loop
RET
.h_w16:
mova m5, [spel_h_perm16]
vpbroadcastd m7, [pb_4]
lea stride3q, [strideq*3]
paddb m6, m7, m5
paddb m7, m6
.h_w16_loop:
movu ym0, [srcq+strideq*0]
movu ym1, [srcq+strideq*2]
vinserti32x8 m0, [srcq+strideq*1], 1
vinserti32x8 m1, [srcq+stride3q ], 1
lea srcq, [srcq+strideq*4]
PREP_8TAP_H
add tmpq, 64*2
sub hd, 4
jg .h_w16_loop
RET
.h_w32:
mova m5, [spel_h_perm32]
vpbroadcastd m7, [pb_4]
paddb m6, m7, m5
paddb m7, m6
.h_w32_loop:
movu m0, [srcq+strideq*0]
movu m1, [srcq+strideq*1]
lea srcq, [srcq+strideq*2]
PREP_8TAP_H
add tmpq, 64*2
sub hd, 2
jg .h_w32_loop
RET
.h_w64:
xor r6d, r6d
jmp .h_start
.h_w128:
mov r6, -64*1
.h_start:
mova m5, [spel_h_perm32]
vpbroadcastd m7, [pb_4]
sub srcq, r6
paddb m6, m7, m5
paddb m7, m6
.h_loop0:
mov r5, r6
.h_loop:
movu m0, [srcq+r5+32*0]
movu m1, [srcq+r5+32*1]
PREP_8TAP_H
add tmpq, 64*2
add r5, 64
jle .h_loop
add srcq, strideq
dec hd
jg .h_loop0
RET
.hv:
RESET_STACK_STATE
vpbroadcastd m8, [pd_2]
vpbroadcastd m9, [pd_32]
cmp wd, 4
jg .hv_w8
movzx mxd, mxb
dec srcq
vpbroadcastd m11, [base+subpel_filters+mxq*8+2]
movzx mxd, myb
shr myd, 16
cmp hd, 4
cmove myd, mxd
vpbroadcastq m0, [base+subpel_filters+myq*8]
lea stride3q, [strideq*3]
sub srcq, stride3q
mov r3d, 0x04
kmovb k1, r3d
kshiftlb k2, k1, 2
kshiftlb k3, k1, 4
vbroadcasti128 m10, [subpel_h_shufA]
punpcklbw m0, m0
psraw m0, 8 ; sign-extend
pshufd m12, m0, q0000
pshufd m13, m0, q1111
pshufd m14, m0, q2222
pshufd m15, m0, q3333
movq xm3, [srcq+strideq*0]
vpbroadcastq ym2, [srcq+strideq*1]
vpbroadcastq ym3{k1}, [srcq+strideq*2]
vpbroadcastq m2{k2}, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
vpbroadcastq m3{k2}, [srcq+strideq*0]
vpbroadcastq m2{k3}, [srcq+strideq*1]
vpbroadcastq m3{k3}, [srcq+strideq*2]
mova m6, [spel_hv_perm4a]
movu m7, [spel_hv_perm4b]
mova m0, m8
mova m1, m8
pshufb m2, m10
pshufb m3, m10
vpdpbusd m0, m2, m11
vpdpbusd m1, m3, m11
packssdw m0, m1 ; _ 0 1 2 3 4 5 6
psraw m0, 2
vpermb m1, m6, m0 ; 01 12 23 34
vpermb m2, m7, m0 ; 23 34 45 56
.hv_w4_loop:
movq xm3, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
movq xm4, [srcq+strideq*0]
vpbroadcastq ym3{k1}, [srcq+strideq*1]
vpbroadcastq ym4{k1}, [srcq+strideq*2]
mova m5, m9
pshufb ym3, ym10
vpdpwssd m5, m1, m12 ; a0 b0 c0 d0
mova ym1, ym8
pshufb ym4, ym10
vpdpbusd ym1, ym3, ym11
mova ym3, ym8
vpdpbusd ym3, ym4, ym11
vpdpwssd m5, m2, m13 ; a1 b1 c1 d1
packssdw ym1, ym3 ; 7 8 9 a
psraw ym1, 2
vshufi32x4 m0, m1, q1032 ; _ 4 5 6 7 8 9 a
vpermb m1, m6, m0 ; 45 56 67 78
vpermb m2, m7, m0 ; 67 78 89 9a
vpdpwssd m5, m1, m14 ; a2 b2 c2 d2
vpdpwssd m5, m2, m15 ; a3 b3 c3 d3
psrad m5, 6
vpmovdw [tmpq], m5
add tmpq, 32
sub hd, 4
jg .hv_w4_loop
RET
.hv_w8:
shr mxd, 16
sub srcq, 3
vpbroadcastd m10, [base+subpel_filters+mxq*8+0]
vpbroadcastd m11, [base+subpel_filters+mxq*8+4]
movzx mxd, myb
shr myd, 16
cmp hd, 4
cmove myd, mxd
vpbroadcastq m0, [base+subpel_filters+myq*8]
lea stride3q, [strideq*3]
sub srcq, stride3q
punpcklbw m0, m0
psraw m0, 8 ; sign-extend
pshufd m12, m0, q0000
pshufd m13, m0, q1111
pshufd m14, m0, q2222
pshufd m15, m0, q3333
cmp wd, 8
jg .hv_w16
vbroadcasti32x4 m17, [srcq+stride3q ]
vinserti32x4 m16, m17, [srcq+strideq*0], 0
vbroadcasti32x4 m19, [subpel_h_shufA]
vinserti32x4 m16, [srcq+strideq*1], 1
vbroadcasti32x4 m21, [subpel_h_shufC]
vinserti32x4 m16, [srcq+strideq*2], 2
lea srcq, [srcq+strideq*4]
vinserti128 ym17, [srcq+strideq*0], 1
vbroadcasti32x4 m20, [subpel_h_shufB]
vinserti32x4 m17, [srcq+strideq*1], 2
vinserti32x4 m17, [srcq+strideq*2], 3
pshufb m3, m16, m19 ; 0 1 2 3 0123
mova m2, m8
pshufb m0, m16, m21 ; 0 1 2 3 89ab
vpdpbusd m2, m3, m10
mova m3, m8
pshufb m1, m17, m19 ; 3 4 5 6 0123
vpdpbusd m3, m0, m11
mova m0, m8
pshufb m4, m17, m21 ; 3 4 5 6 89ab
vpdpbusd m0, m1, m10
mova m1, m8
pshufb m16, m20 ; 0 1 2 3 4567
vpdpbusd m1, m4, m11
pshufb m17, m20 ; 3 4 5 6 4567
vpdpbusd m2, m16, m11
vpdpbusd m3, m16, m10
vpdpbusd m0, m17, m11
vpdpbusd m1, m17, m10
packssdw m2, m3
packssdw m0, m1
psraw m2, 2 ; 0 1 2 3
psraw m0, 2 ; 3 4 5 6
vshufi32x4 m4, m2, m0, q2132 ; 2 3 4 5
vshufi32x4 m5, m2, m0, q1021 ; 1 2 3 4
punpcklwd m3, m4, m0 ; 23 34 45 56
punpckhwd m4, m0
punpcklwd m1, m2, m5 ; 01 12 23 34
punpckhwd m2, m5
.hv_w8_loop:
movu xm18, [srcq+stride3q ]
lea srcq, [srcq+strideq*4]
vinserti128 ym18, [srcq+strideq*0], 1
vinserti32x4 m18, [srcq+strideq*1], 2
vinserti32x4 m18, [srcq+strideq*2], 3
pshufb m17, m18, m19 ; 7 8 9 a 0123
mova m16, m8
pshufb m5, m18, m21 ; 7 8 9 a 89ab
vpdpbusd m16, m17, m10
mova m17, m8
pshufb m18, m20 ; 7 8 9 a 4567
vpdpbusd m17, m5, m11
mova m5, m9
vpdpwssd m5, m3, m13 ; a1 b1 c1 d1
mova m6, m9
vpdpwssd m6, m4, m13
vpdpbusd m16, m18, m11
vpdpbusd m17, m18, m10
vpdpwssd m5, m1, m12 ; a0 b0 c0 d0
mova m1, m3
vpdpwssd m6, m2, m12
mova m2, m4
packssdw m16, m17
psraw m16, 2 ; 7 8 9 a
valignq m4, m16, m0, 6 ; 6 7 8 9
mova m0, m16
punpcklwd m3, m4, m16 ; 67 78 89 9a
punpckhwd m4, m16
vpdpwssd m5, m3, m15 ; a3 b3 c3 d3
vpdpwssd m6, m4, m15
vshufi32x4 m1, m3, q1032 ; 45 56 67 78
vshufi32x4 m2, m4, q1032
vpdpwssd m5, m1, m14 ; a2 b2 c2 d2
vpdpwssd m6, m2, m14
psrad m5, 6
psrad m6, 6
packssdw m5, m6
mova [tmpq], m5
add tmpq, 64
sub hd, 4
jg .hv_w8_loop
vzeroupper
RET
.hv_w16:
WIN64_SPILL_XMM 23
mova m16, [spel_h_perm16]
vpbroadcastd m18, [pb_4]
add wd, wd
paddb m17, m18, m16
lea r6d, [hq+wq*8-256]
paddb m18, m17
.hv_w16_loop0:
movu ym19, [srcq+strideq*0]
vinserti32x8 m19, [srcq+strideq*1], 1
lea r5, [srcq+strideq*2]
movu ym20, [r5 +strideq*0]
vinserti32x8 m20, [r5 +strideq*1], 1
lea r5, [r5 +strideq*2]
movu ym21, [r5 +strideq*0]
vinserti32x8 m21, [r5 +strideq*1], 1
lea r5, [r5 +strideq*2]
movu ym22, [r5 +strideq*0]
mov r7, tmpq
vpermb m3, m16, m19 ; 0 1 0123 89ab
mova m2, m8
vpermb m4, m18, m19 ; 0 1 89ab ghij
vpdpbusd m2, m3, m10
mova m3, m8
vpermb m5, m16, m20 ; 2 3 0123 89ab
vpdpbusd m3, m4, m11
mova m4, m8
vpermb m6, m18, m20 ; 2 3 89ab ghij
vpdpbusd m4, m5, m10
mova m5, m8
vpermb m7, m16, m21 ; 4 5 0123 89ab
vpdpbusd m5, m6, m11
mova m6, m8
vpermb m0, m18, m21 ; 4 5 89ab ghij
vpdpbusd m6, m7, m10
mova m7, m8
vpermb ym1, ym16, ym22 ; 6 0123 89ab
vpdpbusd m7, m0, m11
mova ym0, ym8
vpermb m19, m17, m19 ; 0 1 4567 cdef
vpdpbusd ym0, ym1, ym10
vpermb ym1, ym18, ym22 ; 6 89ab ghij
vpdpbusd m2, m19, m11
vpdpbusd m3, m19, m10
mova ym19, ym8
vpermb m20, m17, m20 ; 2 3 4567 cdef
vpdpbusd ym19, ym1, ym11
vpermb m21, m17, m21 ; 4 5 4567 cdef
vpdpbusd m4, m20, m11
vpdpbusd m5, m20, m10
vpermb ym22, ym17, ym22 ; 6 4567 cdef
vpdpbusd m6, m21, m11
vpdpbusd m7, m21, m10
packssdw m2, m3 ; 0 1
vpdpbusd ym0, ym22, ym11
packssdw m4, m5 ; 2 3
vpdpbusd ym19, ym22, ym10
packssdw m6, m7 ; 4 5
packssdw ym0, ym19 ; 6
REPX {psraw x, 2}, m2, m4, m6, ym0
vshufi32x4 m3, m2, m4, q1032 ; 1 2
vshufi32x4 m5, m4, m6, q1032 ; 3 4
vshufi32x4 m0, m6, m0, q1032 ; 5 6
punpcklwd m1, m2, m3 ; 01 12
punpckhwd m2, m3
punpcklwd m3, m4, m5 ; 23 34
punpckhwd m4, m5
punpcklwd m5, m6, m0 ; 45 56
punpckhwd m6, m0
.hv_w16_loop:
movu ym19, [r5+strideq*1]
lea r5, [r5+strideq*2]
vinserti32x8 m19, [r5+strideq*0], 1
mova m20, m9
vpdpwssd m20, m1, m12 ; a0
vpermb m1, m16, m19
mova m21, m9
vpdpwssd m21, m2, m12 ; b0
vpermb m2, m17, m19
mova m22, m8
vpdpbusd m22, m1, m10
mova m1, m8
vpermb m19, m18, m19
vpdpbusd m1, m2, m10
vpdpwssd m20, m3, m13 ; a1
vpdpwssd m21, m4, m13 ; b1
vpdpbusd m22, m2, m11
mova m2, m4
vpdpbusd m1, m19, m11
mova m4, m6
vpdpwssd m20, m5, m14 ; a2
vpdpwssd m21, m6, m14 ; b2
packssdw m22, m1
mova m1, m3
psraw m22, 2 ; 7 8
mova m3, m5
vshufi32x4 m6, m0, m22, q1032 ; 6 7
mova m0, m22
punpcklwd m5, m6, m0 ; 67 78
punpckhwd m6, m0
vpdpwssd m20, m5, m15 ; a3
vpdpwssd m21, m6, m15 ; b3
psrad m20, 6
psrad m21, 6
packssdw m20, m21
mova [r7+wq*0], ym20
vextracti32x8 [r7+wq*1], m20, 1
lea r7, [r7+wq*2]
sub hd, 2
jg .hv_w16_loop
add srcq, 16
add tmpq, 32
movzx hd, r6b
sub r6d, 1<<8
jg .hv_w16_loop0
RET
cglobal warp_affine_8x8t_8bpc, 4, 7, 22, tmp, ts
vpbroadcastd m9, [pd_16384]
mova ym15, [warp_8x8t_end]
call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx512icl).main
jmp .start
.loop:
call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx512icl).main2
lea tmpq, [tmpq+tsq*4]
.start:
paddd m16, m16
vpermb m16, m15, m16
mova [tmpq+tsq*0], xm16
vextracti128 [tmpq+tsq*2], ym16, 1
sub r6d, 0x1800
jg .loop
RET
cglobal warp_affine_8x8_8bpc, 4, 7, 22, dst, ds, src, ss, abcd, filter
vpbroadcastd m9, [pd_262144]
mova xm15, [warp_8x8_end]
call .main
jmp .start
.loop:
call .main2
lea dstq, [dstq+dsq*2]
.start:
psrad m16, 19
packuswb m16, m16
vpermb m16, m15, m16
movq [dstq+dsq*0], xm16
movhps [dstq+dsq*1], xm16
sub r6d, 0x1800
jg .loop
RET
ALIGN function_align
.main:
vpbroadcastd m1, [pd_512]
%if WIN64
mov abcdq, r5mp
vpaddd ym18, ym1, r6m {1to8} ; mx
%else
add r5d, 512
vpbroadcastd ym18, r5d
%endif
vpaddd ym20, ym1, r7m {1to8} ; my
mova ym16, [pd_0to7]
vpbroadcastd ym19, [abcdq+4*0]
vpbroadcastd ym21, [abcdq+4*1]
lea r4, [ssq*3+3]
mova m10, [warp_8x8_permA]
mov r6d, 0x5555
mova m11, [warp_8x8_permB]
lea filterq, [mc_warp_filter+64*8]
vpbroadcastq m12, [warp_8x8_hpack]
sub srcq, r4 ; src -= src_stride*3 + 3
vbroadcasti32x4 m13, [warp_8x8_permC]
kxnorb k2, k2, k2
vbroadcasti32x4 m14, [warp_8x8_permD]
vpdpwssd ym18, ym19, ym16 ; alpha
vpdpwssd ym20, ym21, ym16 ; gamma
vbroadcasti32x4 m0, [srcq]
psrad ym19, 16 ; beta
psrad ym21, 16 ; delta
kmovw k1, r6d
psrad ym16, ym18, 10
kmovb k3, k2
paddd ym18, ym19
vpgatherdq m2{k2}, [filterq+ym16*8] ; filter_x0
psrld m1, 8 ; pd_2
pshufb m0, m11
paddd m8, m1, m1 ; pd_4
vpdpbusd m1, m0, m2
call .h
psllq m2, m1, 45
pslld m1, 13
paddd m1, m2
vpshrdq m1, m0, 48 ; 01 12
call .h
vpshrdq m2, m1, m0, 48 ; 23 34
call .h
vpshrdq m3, m2, m0, 48 ; 45 56
.main2:
call .h
psrad ym17, ym20, 10
kmovb k2, k3
paddd ym20, ym21
vpgatherdq m7{k3}, [filterq+ym17*8] ; filter_y0
psrad ym16, ym20, 10
kmovb k3, k2
paddd ym20, ym21
vpgatherdq m17{k2}, [filterq+ym16*8] ; filter_y1
shufps m5, m7, m17, q2020 ; a0 a1 a2 a3 b0 b1 b2 b3 A0 A1 A2 A3 B0 B1 B2 B3
mova m16, m9
pshufb m4, m5, m13 ; a0 a1 A0 A1 b0 b1 B0 B1
vpdpwssd m16, m1, m4
pshufb m5, m14 ; a2 a3 A2 A3 b2 b3 B2 B3
mova m1, m2
vpdpwssd m16, m2, m5
shufps m5, m7, m17, q3131 ; a4 a5 a6 a7 b4 b5 b6 b7 A4 A5 A6 A7 B4 B5 B6 B7
mova m2, m3
pshufb m4, m5, m13 ; a4 a5 A4 A5 b4 b5 B4 B5
vpdpwssd m16, m3, m4
vpshrdq m3, m0, 48 ; 67 78
pshufb m5, m14 ; a6 a7 A6 A7 b6 b7 B6 B7
vpdpwssd m16, m3, m5
ret
ALIGN function_align
.h:
movu xm5, [srcq+ssq*1]
psrad ym16, ym18, 10
lea srcq, [srcq+ssq*2]
vinserti32x4 ym5, [srcq+ssq*0], 1
kmovb k2, k3
paddd ym18, ym19
vpgatherdq m6{k3}, [filterq+ym16*8] ; filter_x1
psrad ym17, ym18, 10
kmovb k3, k2
paddd ym18, ym19
vpgatherdq m16{k2}, [filterq+ym17*8] ; filter_x2
mova m0, m8
vpermb m4, m10, m5 ; a4 b0 a5 b1 a6 b2 a7 b3 a8 b4 a9 b5 aa b6 ab b7
vpshldq m17, m16, m6, 32 ; a4 a5 a6 a7 b0 b1 b2 b3
vpdpbusd m0, m4, m17
vpermb m5, m11, m5 ; a0 b4 a1 b5 a2 b6 a3 b7 a4 b8 a5 b9 a6 ba a7 bb
vmovdqa32 m16{k1}, m6 ; a0 a1 a2 a3 b4 b5 b6 b7
vpdpbusd m0, m5, m16
vpmultishiftqb m0, m12, m0 ; 1 1 2 2 (>> 3)
ret
%macro BIDIR_FN 1 ; op
lea stride3q, [strideq*3]
jmp wq
.w4:
cmp hd, 8
jg .w4_h16
WRAP_YMM %1 0
vextracti32x4 xm1, ym0, 1
movd [dstq ], xm0
pextrd [dstq+strideq*1], xm0, 1
movd [dstq+strideq*2], xm1
pextrd [dstq+stride3q ], xm1, 1
jl .w4_ret
lea dstq, [dstq+strideq*4]
pextrd [dstq ], xm0, 2
pextrd [dstq+strideq*1], xm0, 3
pextrd [dstq+strideq*2], xm1, 2
pextrd [dstq+stride3q ], xm1, 3
.w4_ret:
RET
.w4_h16:
vpbroadcastd m7, strided
pmulld m7, [bidir_sctr_w4]
%1 0
kxnorw k1, k1, k1
vpscatterdd [dstq+m7]{k1}, m0
RET
.w8:
cmp hd, 4
jne .w8_h8
WRAP_YMM %1 0
vextracti32x4 xm1, ym0, 1
movq [dstq ], xm0
movq [dstq+strideq*1], xm1
movhps [dstq+strideq*2], xm0
movhps [dstq+stride3q ], xm1
RET
.w8_loop:
%1_INC_PTR 2
lea dstq, [dstq+strideq*4]
.w8_h8:
%1 0
vextracti32x4 xm1, ym0, 1
vextracti32x4 xm2, m0, 2
vextracti32x4 xm3, m0, 3
movq [dstq ], xm0
movq [dstq+strideq*1], xm1
movq [dstq+strideq*2], xm2
movq [dstq+stride3q ], xm3
lea dstq, [dstq+strideq*4]
movhps [dstq ], xm0
movhps [dstq+strideq*1], xm1
movhps [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm3
sub hd, 8
jg .w8_loop
RET
.w16_loop:
%1_INC_PTR 2
lea dstq, [dstq+strideq*4]
.w16:
%1 0
vpermq m0, m0, q3120
mova [dstq ], xm0
vextracti32x4 [dstq+strideq*1], m0, 2
vextracti32x4 [dstq+strideq*2], ym0, 1
vextracti32x4 [dstq+stride3q ], m0, 3
sub hd, 4
jg .w16_loop
RET
.w32:
pmovzxbq m7, [pb_02461357]
.w32_loop:
%1 0
%1_INC_PTR 2
vpermq m0, m7, m0
mova [dstq+strideq*0], ym0
vextracti32x8 [dstq+strideq*1], m0, 1
lea dstq, [dstq+strideq*2]
sub hd, 2
jg .w32_loop
RET
.w64:
pmovzxbq m7, [pb_02461357]
.w64_loop:
%1 0
%1_INC_PTR 2
vpermq m0, m7, m0
mova [dstq], m0
add dstq, strideq
dec hd
jg .w64_loop
RET
.w128:
pmovzxbq m7, [pb_02461357]
.w128_loop:
%1 0
vpermq m6, m7, m0
%1 2
mova [dstq+64*0], m6
%1_INC_PTR 4
vpermq m6, m7, m0
mova [dstq+64*1], m6
add dstq, strideq
dec hd
jg .w128_loop
RET
%endmacro
%macro AVG 1 ; src_offset
mova m0, [tmp1q+(%1+0)*mmsize]
paddw m0, [tmp2q+(%1+0)*mmsize]
mova m1, [tmp1q+(%1+1)*mmsize]
paddw m1, [tmp2q+(%1+1)*mmsize]
pmulhrsw m0, m4
pmulhrsw m1, m4
packuswb m0, m1
%endmacro
%macro AVG_INC_PTR 1
add tmp1q, %1*mmsize
add tmp2q, %1*mmsize
%endmacro
cglobal avg_8bpc, 4, 7, 5, dst, stride, tmp1, tmp2, w, h, stride3
%define base r6-avg_avx512icl_table
lea r6, [avg_avx512icl_table]
tzcnt wd, wm
movifnidn hd, hm
movsxd wq, dword [r6+wq*4]
vpbroadcastd m4, [base+pw_1024]
add wq, r6
BIDIR_FN AVG
%macro W_AVG 1 ; src_offset
; (a * weight + b * (16 - weight) + 128) >> 8
; = ((a - b) * weight + (b << 4) + 128) >> 8
; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4
; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4
mova m0, [tmp1q+(%1+0)*mmsize]
psubw m2, m0, [tmp2q+(%1+0)*mmsize]
mova m1, [tmp1q+(%1+1)*mmsize]
psubw m3, m1, [tmp2q+(%1+1)*mmsize]
pmulhw m2, m4
pmulhw m3, m4
paddw m0, m2
paddw m1, m3
pmulhrsw m0, m5
pmulhrsw m1, m5
packuswb m0, m1
%endmacro
%define W_AVG_INC_PTR AVG_INC_PTR
cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3
%define base r6-w_avg_avx512icl_table
lea r6, [w_avg_avx512icl_table]
tzcnt wd, wm
movifnidn hd, hm
vpbroadcastw m4, r6m ; weight
movsxd wq, dword [r6+wq*4]
vpbroadcastd m5, [base+pw_2048]
psllw m4, 12 ; (weight-16) << 12 when interpreted as signed
add wq, r6
cmp dword r6m, 7
jg .weight_gt7
mov r6, tmp1q
pxor m0, m0
mov tmp1q, tmp2q
psubw m4, m0, m4 ; -weight
mov tmp2q, r6
.weight_gt7:
BIDIR_FN W_AVG
%macro MASK 1 ; src_offset
; (a * m + b * (64 - m) + 512) >> 10
; = ((a - b) * m + (b << 6) + 512) >> 10
; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4
%if mmsize == 64
vpermq m3, m8, [maskq+%1*32]
%else
vpermq m3, [maskq+%1*16], q3120
%endif
mova m0, [tmp2q+(%1+0)*mmsize]
psubw m1, m0, [tmp1q+(%1+0)*mmsize]
psubb m3, m4, m3
paddw m1, m1 ; (b - a) << 1
paddb m3, m3
punpcklbw m2, m4, m3 ; -m << 9
pmulhw m1, m2
paddw m0, m1
mova m1, [tmp2q+(%1+1)*mmsize]
psubw m2, m1, [tmp1q+(%1+1)*mmsize]
paddw m2, m2
punpckhbw m3, m4, m3
pmulhw m2, m3
paddw m1, m2
pmulhrsw m0, m5
pmulhrsw m1, m5
packuswb m0, m1
%endmacro
%macro MASK_INC_PTR 1
add maskq, %1*32
add tmp2q, %1*64
add tmp1q, %1*64
%endmacro
cglobal mask_8bpc, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-mask_avx512icl_table
lea r7, [mask_avx512icl_table]
tzcnt wd, wm
movifnidn hd, hm
mov maskq, maskmp
movsxd wq, dword [r7+wq*4]
pxor m4, m4
mova m8, [base+bilin_v_perm64]
vpbroadcastd m5, [base+pw_2048]
add wq, r7
BIDIR_FN MASK
%macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4
mova m%1, [tmp1q+mmsize*%3]
mova m1, [tmp2q+mmsize*%3]
psubw m1, m%1
pabsw m%2, m1
psubusw m%2, m6, m%2
psrlw m%2, 8 ; 64 - m
psllw m2, m%2, 10
pmulhw m1, m2
paddw m%1, m1
mova m1, [tmp1q+mmsize*%4]
mova m2, [tmp2q+mmsize*%4]
psubw m2, m1
pabsw m3, m2
psubusw m3, m6, m3
vpshldw m%2, m3, 8
psllw m3, m%2, 10
%if %5
psubb m%2, m5, m%2
%endif
pmulhw m2, m3
paddw m1, m2
pmulhrsw m%1, m7
pmulhrsw m1, m7
packuswb m%1, m1
%endmacro
cglobal w_mask_420_8bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_420_avx512icl_table
lea r7, [w_mask_420_avx512icl_table]
tzcnt wd, wm
mov r6d, r7m ; sign
movifnidn hd, hm
movsxd wq, [r7+wq*4]
vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
vpbroadcastd m7, [base+pw_2048]
vpbroadcastd m9, [base+pb_m64] ; -1 << 6
mova ym10, [base+wm_420_mask+32]
vpbroadcastd m8, [base+wm_sign+r6*8] ; (258 - sign) << 6
add wq, r7
mov maskq, maskmp
lea stride3q, [strideq*3]
jmp wq
.w4:
mova m5, [wm_420_perm4]
cmp hd, 8
jg .w4_h16
WRAP_YMM W_MASK 0, 4, 0, 1
vinserti128 ym5, [wm_420_perm4+32], 1
vpermb ym4, ym5, ym4
vpdpbusd ym8, ym4, ym9
vextracti32x4 xm1, m0, 1
movd [dstq+strideq*0], xm0
pextrd [dstq+strideq*1], xm0, 1
movd [dstq+strideq*2], xm1
pextrd [dstq+stride3q ], xm1, 1
jl .w4_end
lea dstq, [dstq+strideq*4]
pextrd [dstq+strideq*0], xm0, 2
pextrd [dstq+strideq*1], xm0, 3
pextrd [dstq+strideq*2], xm1, 2
pextrd [dstq+stride3q ], xm1, 3
.w4_end:
vpermb ym8, ym10, ym8
movq [maskq], xm8
RET
.w4_h16:
vpbroadcastd m11, strided
pmulld m11, [bidir_sctr_w4]
W_MASK 0, 4, 0, 1
vpermb m4, m5, m4
vpdpbusd m8, m4, m9
kxnorw k1, k1, k1
vpermb m8, m10, m8
mova [maskq], xm8
vpscatterdd [dstq+m11]{k1}, m0
RET
.w8:
mova m5, [wm_420_perm8]
cmp hd, 4
jne .w8_h8
WRAP_YMM W_MASK 0, 4, 0, 1
vinserti128 ym5, [wm_420_perm8+32], 1
vpermb ym4, ym5, ym4
vpdpbusd ym8, ym4, ym9
vpermb m8, m10, m8
mova [maskq], xm8
vextracti32x4 xm1, ym0, 1
movq [dstq+strideq*0], xm0
movq [dstq+strideq*1], xm1
movhps [dstq+strideq*2], xm0
movhps [dstq+stride3q ], xm1
RET
.w8_loop:
add tmp1q, 128
add tmp2q, 128
add maskq, 16
lea dstq, [dstq+strideq*4]
.w8_h8:
W_MASK 0, 4, 0, 1
vpermb m4, m5, m4
mova m1, m8
vpdpbusd m1, m4, m9
vpermb m1, m10, m1
mova [maskq], xm1
vextracti32x4 xm1, ym0, 1
vextracti32x4 xm2, m0, 2
vextracti32x4 xm3, m0, 3
movq [dstq+strideq*0], xm0
movq [dstq+strideq*1], xm1
movq [dstq+strideq*2], xm2
movq [dstq+stride3q ], xm3
lea dstq, [dstq+strideq*4]
movhps [dstq+strideq*0], xm0
movhps [dstq+strideq*1], xm1
movhps [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm3
sub hd, 8
jg .w8_loop
RET
.w16:
mova m5, [wm_420_perm16]
.w16_loop:
W_MASK 0, 4, 0, 1
vpermb m4, m5, m4
mova m1, m8
vpdpbusd m1, m4, m9
add tmp1q, 128
add tmp2q, 128
vpermb m1, m10, m1
vpermq m0, m0, q3120
mova [maskq], xm1
add maskq, 16
mova [dstq+strideq*0], xm0
vextracti32x4 [dstq+strideq*1], m0, 2
vextracti32x4 [dstq+strideq*2], ym0, 1
vextracti32x4 [dstq+stride3q ], m0, 3
lea dstq, [dstq+strideq*4]
sub hd, 4
jg .w16_loop
RET
.w32:
pmovzxbq m5, [pb_02461357]
.w32_loop:
W_MASK 0, 4, 0, 1
mova m1, m8
vpdpbusd m1, m4, m9
add tmp1q, 128
add tmp2q, 128
vpermb m1, m10, m1
vpermq m0, m5, m0
mova [maskq], xm1
add maskq, 16
mova [dstq+strideq*0], ym0
vextracti32x8 [dstq+strideq*1], m0, 1
lea dstq, [dstq+strideq*2]
sub hd, 2
jg .w32_loop
RET
.w64:
pmovzxbq m12, [wm_420_perm64] ; 0, 2, 4, 6, 8, 10, 12, 14
psrlq m13, m12, 4 ; 1, 3, 5, 7, 9, 11, 13, 15
.w64_loop:
W_MASK 0, 4, 0, 2
W_MASK 11, 5, 1, 3
mova m2, m8
vpdpbusd m2, m4, m9
mova m3, m8
vpdpbusd m3, m5, m9
add tmp1q, 256
add tmp2q, 256
vpermt2b m2, m10, m3
mova m1, m0
vpermt2q m0, m12, m11
vpermt2q m1, m13, m11
mova [maskq], ym2
add maskq, 32
mova [dstq+strideq*0], m0
mova [dstq+strideq*1], m1
lea dstq, [dstq+strideq*2]
sub hd, 2
jg .w64_loop
RET
.w128:
pmovzxbq m14, [wm_420_perm64]
mova m10, [wm_420_mask]
psrlq m15, m14, 4
.w128_loop:
W_MASK 0, 12, 0, 4
W_MASK 11, 13, 1, 5
mova m4, m8
vpdpbusd m4, m12, m9
mova m5, m8
vpdpbusd m5, m13, m9
mova m1, m0
vpermt2q m0, m14, m11
vpermt2q m1, m15, m11
mova [dstq+strideq*0+64*0], m0
mova [dstq+strideq*1+64*0], m1
W_MASK 0, 12, 2, 6
W_MASK 11, 13, 3, 7
vprold m4, 16
vprold m5, 16
vpdpbusd m4, m12, m9
vpdpbusd m5, m13, m9
add tmp1q, 512
add tmp2q, 512
vpermt2b m4, m10, m5
mova m1, m0
vpermt2q m0, m14, m11
vpermt2q m1, m15, m11
mova [maskq], m4
add maskq, 64
mova [dstq+strideq*0+64*1], m0
mova [dstq+strideq*1+64*1], m1
lea dstq, [dstq+strideq*2]
sub hd, 2
jg .w128_loop
RET
cglobal w_mask_422_8bpc, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_422_avx512icl_table
lea r7, [w_mask_422_avx512icl_table]
tzcnt wd, wm
mov r6d, r7m ; sign
movifnidn hd, hm
movsxd wq, dword [r7+wq*4]
vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
vpbroadcastd m7, [base+pw_2048]
vpbroadcastd m9, [base+pw_m128]
mova m10, [base+wm_422_mask]
vpbroadcastd m11, [base+pb_127]
add wq, r7
vpbroadcastd m8, [base+wm_sign+4+r6*4]
mov maskq, maskmp
lea stride3q, [strideq*3]
jmp wq
.w4:
cmp hd, 8
jg .w4_h16
WRAP_YMM W_MASK 0, 4, 0, 1
movhps xm10, [wm_422_mask+16]
vpdpwssd ym8, ym4, ym9
vpermb ym8, ym10, ym8
vextracti32x4 xm1, m0, 1
movd [dstq+strideq*0], xm0
pextrd [dstq+strideq*1], xm0, 1
movd [dstq+strideq*2], xm1
pextrd [dstq+stride3q ], xm1, 1
jl .w4_end
lea dstq, [dstq+strideq*4]
pextrd [dstq+strideq*0], xm0, 2
pextrd [dstq+strideq*1], xm0, 3
pextrd [dstq+strideq*2], xm1, 2
pextrd [dstq+stride3q ], xm1, 3
.w4_end:
pand xm8, xm11
mova [maskq], xm8
RET
.w4_h16:
vpbroadcastd m5, strided
pmulld m5, [bidir_sctr_w4]
W_MASK 0, 4, 0, 1
vpdpwssd m8, m4, m9
kxnorw k1, k1, k1
vpermb m8, m10, m8
pand ym8, ym11
mova [maskq], ym8
vpscatterdd [dstq+m5]{k1}, m0
RET
.w8:
cmp hd, 4
jne .w8_h8
WRAP_YMM W_MASK 0, 4, 0, 1
movhps xm10, [wm_422_mask+16]
vpdpwssd ym8, ym4, ym9
vpermb ym8, ym10, ym8
pand xm8, xm11
mova [maskq], xm8
vextracti32x4 xm1, ym0, 1
movq [dstq+strideq*0], xm0
movq [dstq+strideq*1], xm1
movhps [dstq+strideq*2], xm0
movhps [dstq+stride3q ], xm1
RET
.w8_loop:
add tmp1q, 128
add tmp2q, 128
add maskq, 32
lea dstq, [dstq+strideq*4]
.w8_h8:
W_MASK 0, 4, 0, 1
mova m1, m8
vpdpwssd m1, m4, m9
vpermb m1, m10, m1
pand ym1, ym11
mova [maskq], ym1
vextracti32x4 xm1, ym0, 1
vextracti32x4 xm2, m0, 2
vextracti32x4 xm3, m0, 3
movq [dstq+strideq*0], xm0
movq [dstq+strideq*1], xm1
movq [dstq+strideq*2], xm2
movq [dstq+stride3q ], xm3
lea dstq, [dstq+strideq*4]
movhps [dstq+strideq*0], xm0
movhps [dstq+strideq*1], xm1
movhps [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm3
sub hd, 8
jg .w8_loop
RET
.w16_loop:
add tmp1q, 128
add tmp2q, 128
add maskq, 32
lea dstq, [dstq+strideq*4]
.w16:
W_MASK 0, 4, 0, 1
mova m1, m8
vpdpwssd m1, m4, m9
vpermb m1, m10, m1
vpermq m0, m0, q3120
pand ym1, ym11
mova [maskq], ym1
mova [dstq+strideq*0], xm0
vextracti32x4 [dstq+strideq*1], m0, 2
vextracti32x4 [dstq+strideq*2], ym0, 1
vextracti32x4 [dstq+stride3q ], m0, 3
sub hd, 4
jg .w16_loop
RET
.w32:
pmovzxbq m5, [pb_02461357]
.w32_loop:
W_MASK 0, 4, 0, 1
mova m1, m8
vpdpwssd m1, m4, m9
add tmp1q, 128
add tmp2q, 128
vpermb m1, m10, m1
vpermq m0, m5, m0
pand ym1, ym11
mova [maskq], ym1
add maskq, 32
mova [dstq+strideq*0], ym0
vextracti32x8 [dstq+strideq*1], m0, 1
lea dstq, [dstq+strideq*2]
sub hd, 2
jg .w32_loop
RET
.w64:
pmovzxbq m5, [pb_02461357]
.w64_loop:
W_MASK 0, 4, 0, 1
mova m1, m8
vpdpwssd m1, m4, m9
add tmp1q, 128
add tmp2q, 128
vpermb m1, m10, m1
vpermq m0, m5, m0
pand ym1, ym11
mova [maskq], ym1
add maskq, 32
mova [dstq], m0
add dstq, strideq
dec hd
jg .w64_loop
RET
.w128:
pmovzxbq m13, [pb_02461357]
.w128_loop:
W_MASK 0, 4, 0, 1
W_MASK 12, 5, 2, 3
mova m2, m8
vpdpwssd m2, m4, m9
mova m3, m8
vpdpwssd m3, m5, m9
add tmp1q, 256
add tmp2q, 256
vpermt2b m2, m10, m3
vpermq m0, m13, m0
vpermq m1, m13, m12
pand m2, m11
mova [maskq], m2
add maskq, 64
mova [dstq+64*0], m0
mova [dstq+64*1], m1
add dstq, strideq
dec hd
jg .w128_loop
RET
cglobal w_mask_444_8bpc, 4, 8, 12, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_444_avx512icl_table
lea r7, [w_mask_444_avx512icl_table]
tzcnt wd, wm
movifnidn hd, hm
movsxd wq, dword [r7+wq*4]
vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
vpbroadcastd m5, [base+pb_64]
vpbroadcastd m7, [base+pw_2048]
mova m8, [base+wm_444_mask]
add wq, r7
mov maskq, maskmp
lea stride3q, [strideq*3]
jmp wq
.w4:
cmp hd, 8
jg .w4_h16
WRAP_YMM W_MASK 0, 4, 0, 1, 1
vinserti128 ym8, [wm_444_mask+32], 1
vpermb ym4, ym8, ym4
mova [maskq], ym4
vextracti32x4 xm1, m0, 1
movd [dstq+strideq*0], xm0
pextrd [dstq+strideq*1], xm0, 1
movd [dstq+strideq*2], xm1
pextrd [dstq+stride3q ], xm1, 1
jl .w4_end
lea dstq, [dstq+strideq*4]
pextrd [dstq+strideq*0], xm0, 2
pextrd [dstq+strideq*1], xm0, 3
pextrd [dstq+strideq*2], xm1, 2
pextrd [dstq+stride3q ], xm1, 3
.w4_end:
RET
.w4_h16:
vpbroadcastd m9, strided
pmulld m9, [bidir_sctr_w4]
W_MASK 0, 4, 0, 1, 1
vpermb m4, m8, m4
kxnorw k1, k1, k1
mova [maskq], m4
vpscatterdd [dstq+m9]{k1}, m0
RET
.w8:
cmp hd, 4
jne .w8_h8
WRAP_YMM W_MASK 0, 4, 0, 1, 1
vinserti128 ym8, [wm_444_mask+32], 1
vpermb ym4, ym8, ym4
mova [maskq], ym4
vextracti32x4 xm1, ym0, 1
movq [dstq+strideq*0], xm0
movq [dstq+strideq*1], xm1
movhps [dstq+strideq*2], xm0
movhps [dstq+stride3q ], xm1
RET
.w8_loop:
add tmp1q, 128
add tmp2q, 128
add maskq, 64
lea dstq, [dstq+strideq*4]
.w8_h8:
W_MASK 0, 4, 0, 1, 1
vpermb m4, m8, m4
mova [maskq], m4
vextracti32x4 xm1, ym0, 1
vextracti32x4 xm2, m0, 2
vextracti32x4 xm3, m0, 3
movq [dstq+strideq*0], xm0
movq [dstq+strideq*1], xm1
movq [dstq+strideq*2], xm2
movq [dstq+stride3q ], xm3
lea dstq, [dstq+strideq*4]
movhps [dstq+strideq*0], xm0
movhps [dstq+strideq*1], xm1
movhps [dstq+strideq*2], xm2
movhps [dstq+stride3q ], xm3
sub hd, 8
jg .w8_loop
RET
.w16_loop:
add tmp1q, 128
add tmp2q, 128
add maskq, 64
lea dstq, [dstq+strideq*4]
.w16:
W_MASK 0, 4, 0, 1, 1
vpermb m4, m8, m4
vpermq m0, m0, q3120
mova [maskq], m4
mova [dstq+strideq*0], xm0
vextracti32x4 [dstq+strideq*1], m0, 2
vextracti32x4 [dstq+strideq*2], ym0, 1
vextracti32x4 [dstq+stride3q ], m0, 3
sub hd, 4
jg .w16_loop
RET
.w32:
pmovzxbq m9, [pb_02461357]
.w32_loop:
W_MASK 0, 4, 0, 1, 1
vpermb m4, m8, m4
add tmp1q, 128
add tmp2q, 128
vpermq m0, m9, m0
mova [maskq], m4
add maskq, 64
mova [dstq+strideq*0], ym0
vextracti32x8 [dstq+strideq*1], m0, 1
lea dstq, [dstq+strideq*2]
sub hd, 2
jg .w32_loop
RET
.w64:
pmovzxbq m9, [pb_02461357]
.w64_loop:
W_MASK 0, 4, 0, 1, 1
vpermb m4, m8, m4
add tmp1q, 128
add tmp2q, 128
vpermq m0, m9, m0
mova [maskq], m4
add maskq, 64
mova [dstq], m0
add dstq, strideq
dec hd
jg .w64_loop
RET
.w128:
pmovzxbq m11, [pb_02461357]
.w128_loop:
W_MASK 0, 4, 0, 1, 1
W_MASK 10, 9, 2, 3, 1
vpermb m4, m8, m4
vpermb m9, m8, m9
add tmp1q, 256
add tmp2q, 256
vpermq m0, m11, m0
vpermq m10, m11, m10
mova [maskq+64*0], m4
mova [maskq+64*1], m9
add maskq, 128
mova [dstq+64*0], m0
mova [dstq+64*1], m10
add dstq, strideq
dec hd
jg .w128_loop
RET
cglobal blend_8bpc, 3, 7, 8, dst, ds, tmp, w, h, mask
%define base r6-blend_avx512icl_table
lea r6, [blend_avx512icl_table]
tzcnt wd, wm
movifnidn maskq, maskmp
movifnidn hd, hm
movsxd wq, [r6+wq*4]
vpbroadcastd m6, [base+pb_64]
vpbroadcastd m7, [base+pw_512]
sub tmpq, maskq
add wq, r6
lea r6, [dsq*3]
jmp wq
.w4:
movd xmm0, [dstq+dsq*0]
pinsrd xmm0, [dstq+dsq*1], 1
vpbroadcastd xmm1, [dstq+dsq*2]
pinsrd xmm1, [dstq+r6 ], 3
mova xmm4, [maskq]
mova xmm5, [maskq+tmpq]
add maskq, 4*4
psubb xmm3, xm6, xmm4
punpcklbw xmm0, xmm5
punpcklbw xmm2, xmm3, xmm4
punpckhbw xmm1, xmm5
punpckhbw xmm3, xmm4
pmaddubsw xmm0, xmm2
pmaddubsw xmm1, xmm3
pmulhrsw xmm0, xm7
pmulhrsw xmm1, xm7
packuswb xmm0, xmm1
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 1
pextrd [dstq+dsq*2], xmm0, 2
pextrd [dstq+r6 ], xmm0, 3
lea dstq, [dstq+dsq*4]
sub hd, 4
jg .w4
RET
.w8:
movq xmm0, [dstq+dsq*0]
vpbroadcastq xmm1, [dstq+dsq*1]
vpbroadcastq ymm2, [dstq+dsq*2]
vpbroadcastq ymm3, [dstq+r6 ]
mova ymm4, [maskq]
mova ymm5, [maskq+tmpq]
add maskq, 8*4
vpblendd ymm0, ymm2, 0x30
vpblendd ymm1, ymm3, 0xc0
psubb ymm3, ym6, ymm4
punpcklbw ymm0, ymm5
punpcklbw ymm2, ymm3, ymm4
punpckhbw ymm1, ymm5
punpckhbw ymm3, ymm4
pmaddubsw ymm0, ymm2
pmaddubsw ymm1, ymm3
pmulhrsw ymm0, ym7
pmulhrsw ymm1, ym7
packuswb ymm0, ymm1
vextracti128 xmm1, ymm0, 1
movq [dstq+dsq*0], xmm0
movhps [dstq+dsq*1], xmm0
movq [dstq+dsq*2], xmm1
movhps [dstq+r6 ], xmm1
lea dstq, [dstq+dsq*4]
sub hd, 4
jg .w8
vzeroupper
RET
.w16:
mova xm1, [dstq+dsq*0]
vinserti32x4 ym1, [dstq+dsq*1], 1
vinserti32x4 m1, [dstq+dsq*2], 2
mova m4, [maskq]
vinserti32x4 m1, [dstq+r6 ], 3
mova m5, [maskq+tmpq]
add maskq, 16*4
psubb m3, m6, m4
punpcklbw m0, m1, m5
punpcklbw m2, m3, m4
punpckhbw m1, m5
punpckhbw m3, m4
pmaddubsw m0, m2
pmaddubsw m1, m3
pmulhrsw m0, m7
pmulhrsw m1, m7
packuswb m0, m1
mova [dstq+dsq*0], xm0
vextracti32x4 [dstq+dsq*1], ym0, 1
vextracti32x4 [dstq+dsq*2], m0, 2
vextracti32x4 [dstq+r6 ], m0, 3
lea dstq, [dstq+dsq*4]
sub hd, 4
jg .w16
RET
.w32:
mova ym1, [dstq+dsq*0]
vinserti32x8 m1, [dstq+dsq*1], 1
mova m4, [maskq]
mova m5, [maskq+tmpq]
add maskq, 32*2
psubb m3, m6, m4
punpcklbw m0, m1, m5
punpcklbw m2, m3, m4
punpckhbw m1, m5
punpckhbw m3, m4
pmaddubsw m0, m2
pmaddubsw m1, m3
pmulhrsw m0, m7
pmulhrsw m1, m7
packuswb m0, m1
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w32
RET
cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask
%define base r5-blend_v_avx512icl_table
lea r5, [blend_v_avx512icl_table]
tzcnt wd, wm
movifnidn hd, hm
movsxd wq, [r5+wq*4]
vpbroadcastd m5, [base+pw_512]
add wq, r5
add maskq, obmc_masks-blend_v_avx512icl_table
jmp wq
.w2:
vpbroadcastd xmm2, [maskq+2*2]
.w2_s0_loop:
movd xmm0, [dstq+dsq*0]
pinsrw xmm0, [dstq+dsq*1], 1
movd xmm1, [tmpq]
add tmpq, 2*2
punpcklbw xmm0, xmm1
pmaddubsw xmm0, xmm2
pmulhrsw xmm0, xm5
packuswb xmm0, xmm0
pextrw [dstq+dsq*0], xmm0, 0
pextrw [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w2_s0_loop
RET
.w4:
vpbroadcastq xmm2, [maskq+4*2]
.w4_loop:
movd xmm0, [dstq+dsq*0]
pinsrd xmm0, [dstq+dsq*1], 1
movq xmm1, [tmpq]
add tmpq, 4*2
punpcklbw xmm0, xmm1
pmaddubsw xmm0, xmm2
pmulhrsw xmm0, xm5
packuswb xmm0, xmm0
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w4_loop
RET
.w8:
mova xmm3, [maskq+8*2]
.w8_loop:
movq xmm0, [dstq+dsq*0]
vpbroadcastq xmm1, [dstq+dsq*1]
mova xmm2, [tmpq]
add tmpq, 8*2
punpcklbw xmm0, xmm2
punpckhbw xmm1, xmm2
pmaddubsw xmm0, xmm3
pmaddubsw xmm1, xmm3
pmulhrsw xmm0, xm5
pmulhrsw xmm1, xm5
packuswb xmm0, xmm1
movq [dstq+dsq*0], xmm0
movhps [dstq+dsq*1], xmm0
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w8_loop
RET
.w16:
vbroadcasti32x4 ym3, [maskq+16*2]
vbroadcasti32x4 ym4, [maskq+16*3]
.w16_loop:
mova xm1, [dstq+dsq*0]
vinserti32x4 ym1, [dstq+dsq*1], 1
mova ym2, [tmpq]
add tmpq, 16*2
punpcklbw ym0, ym1, ym2
punpckhbw ym1, ym2
pmaddubsw ym0, ym3
pmaddubsw ym1, ym4
pmulhrsw ym0, ym5
pmulhrsw ym1, ym5
packuswb ym0, ym1
mova [dstq+dsq*0], xm0
vextracti32x4 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w16_loop
RET
.w32:
mova m4, [maskq+32*2]
vshufi32x4 m3, m4, m4, q2020
vshufi32x4 m4, m4, q3131
.w32_loop:
mova ym1, [dstq+dsq*0]
vinserti32x8 m1, [dstq+dsq*1], 1
mova m2, [tmpq]
add tmpq, 32*2
punpcklbw m0, m1, m2
punpckhbw m1, m2
pmaddubsw m0, m3
pmaddubsw m1, m4
pmulhrsw m0, m5
pmulhrsw m1, m5
packuswb m0, m1
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
sub hd, 2
jg .w32_loop
RET
cglobal blend_h_8bpc, 3, 7, 6, dst, ds, tmp, w, h, mask
%define base r6-blend_h_avx512icl_table
lea r6, [blend_h_avx512icl_table]
tzcnt wd, wm
mov hd, hm
movsxd wq, [r6+wq*4]
lea maskq, [base+obmc_masks+hq*2]
vpbroadcastd m5, [base+pw_512]
lea hd, [hq*3]
add wq, r6
shr hd, 2 ; h * 3/4
lea maskq, [maskq+hq*2]
neg hq
jmp wq
.w2:
movd xmm0, [dstq+dsq*0]
pinsrw xmm0, [dstq+dsq*1], 1
movd xmm2, [maskq+hq*2]
movd xmm1, [tmpq]
add tmpq, 2*2
punpcklwd xmm2, xmm2
punpcklbw xmm0, xmm1
pmaddubsw xmm0, xmm2
pmulhrsw xmm0, xm5
packuswb xmm0, xmm0
pextrw [dstq+dsq*0], xmm0, 0
pextrw [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w2
RET
.w4:
mova xmm3, [blend_shuf]
.w4_loop:
movd xmm0, [dstq+dsq*0]
pinsrd xmm0, [dstq+dsq*1], 1
movd xmm2, [maskq+hq*2]
movq xmm1, [tmpq]
add tmpq, 4*2
pshufb xmm2, xmm3
punpcklbw xmm0, xmm1
pmaddubsw xmm0, xmm2
pmulhrsw xmm0, xm5
packuswb xmm0, xmm0
movd [dstq+dsq*0], xmm0
pextrd [dstq+dsq*1], xmm0, 1
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w4_loop
RET
.w8:
vbroadcasti128 ymm4, [blend_shuf]
shufpd ymm4, ymm4, 0x03
.w8_loop:
vpbroadcastq ymm1, [dstq+dsq*0]
movq xmm0, [dstq+dsq*1]
vpblendd ymm0, ymm1, 0x30
vpbroadcastd ymm3, [maskq+hq*2]
movq xmm1, [tmpq+8*1]
vinserti128 ymm1, [tmpq+8*0], 1
add tmpq, 8*2
pshufb ymm3, ymm4
punpcklbw ymm0, ymm1
pmaddubsw ymm0, ymm3
pmulhrsw ymm0, ym5
vextracti128 xmm1, ymm0, 1
packuswb xmm0, xmm1
movhps [dstq+dsq*0], xmm0
movq [dstq+dsq*1], xmm0
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w8_loop
vzeroupper
RET
.w16:
vbroadcasti32x4 ym4, [blend_shuf]
shufpd ym4, ym4, 0x0c
.w16_loop:
mova xm1, [dstq+dsq*0]
vinserti32x4 ym1, [dstq+dsq*1], 1
vpbroadcastd ym3, [maskq+hq*2]
mova ym2, [tmpq]
add tmpq, 16*2
pshufb ym3, ym4
punpcklbw ym0, ym1, ym2
punpckhbw ym1, ym2
pmaddubsw ym0, ym3
pmaddubsw ym1, ym3
pmulhrsw ym0, ym5
pmulhrsw ym1, ym5
packuswb ym0, ym1
mova [dstq+dsq*0], xm0
vextracti32x4 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w16_loop
RET
.w32:
vbroadcasti32x4 m4, [blend_shuf]
shufpd m4, m4, 0xf0
.w32_loop:
mova ym1, [dstq+dsq*0]
vinserti32x8 m1, [dstq+dsq*1], 1
vpbroadcastd m3, [maskq+hq*2]
mova m2, [tmpq]
add tmpq, 32*2
pshufb m3, m4
punpcklbw m0, m1, m2
punpckhbw m1, m2
pmaddubsw m0, m3
pmaddubsw m1, m3
pmulhrsw m0, m5
pmulhrsw m1, m5
packuswb m0, m1
mova [dstq+dsq*0], ym0
vextracti32x8 [dstq+dsq*1], m0, 1
lea dstq, [dstq+dsq*2]
add hq, 2
jl .w32_loop
RET
.w64:
vpbroadcastw m3, [maskq+hq*2]
mova m1, [dstq]
mova m2, [tmpq]
add tmpq, 32*2
punpcklbw m0, m1, m2
punpckhbw m1, m2
pmaddubsw m0, m3
pmaddubsw m1, m3
pmulhrsw m0, m5
pmulhrsw m1, m5
packuswb m0, m1
mova [dstq], m0
add dstq, dsq
inc hq
jl .w64
RET
.w128:
vpbroadcastw m6, [maskq+hq*2]
mova m2, [dstq+64*0]
mova m1, [tmpq+64*0]
mova m3, [dstq+64*1]
mova m4, [tmpq+64*1]
add tmpq, 64*2
punpcklbw m0, m2, m1
punpckhbw m2, m1
pmaddubsw m0, m6
pmaddubsw m2, m6
punpcklbw m1, m3, m4
punpckhbw m3, m4
pmaddubsw m1, m6
pmaddubsw m3, m6
REPX {pmulhrsw x, m5}, m0, m2, m1, m3
packuswb m0, m2
packuswb m1, m3
mova [dstq+64*0], m0
mova [dstq+64*1], m1
add dstq, dsq
inc hq
jl .w128
RET
cglobal resize_8bpc, 6, 12, 19, dst, dst_stride, src, src_stride, \
dst_w, h, src_w, dx, mx0
sub dword mx0m, 4<<14
sub dword src_wm, 8
mov r6, ~0
vpbroadcastd m5, dxm
vpbroadcastd m8, mx0m
vpbroadcastd m6, src_wm
kmovq k3, r6
DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x
LEA r7, $$
%define base r7-$$
vpbroadcastd m3, [base+pw_m256]
vpbroadcastd m7, [base+pd_63]
vbroadcasti32x4 m15, [base+pb_8x0_8x8]
vpdpwssd m8, m5, [base+rescale_mul] ; mx+dx*[0-15]
pslld m5, 4 ; dx*16
pslld m6, 14
pxor m2, m2
mova m16, [base+resize_permA]
mova m17, [base+resize_permB]
mova xm18, [base+resize_permC]
.loop_y:
xor xd, xd
mova m4, m8 ; per-line working version of mx
.loop_x:
pmaxsd m0, m4, m2
psrad m9, m4, 8 ; filter offset (unmasked)
pminsd m0, m6 ; iclip(mx, 0, src_w-8)
psubd m1, m4, m0 ; pshufb offset
psrad m0, 14 ; clipped src_x offset
psrad m1, 14 ; pshufb edge_emu offset
vptestmd k4, m1, m1
pand m9, m7 ; filter offset (masked)
ktestw k4, k4
jz .load
vextracti32x8 ym12, m0, 1
vextracti32x8 ym13, m1, 1
kmovq k1, k3
kmovq k2, k3
vpgatherdq m10{k1}, [srcq+ym0]
vpgatherdq m11{k2}, [srcq+ym12]
kmovq k1, k3
kmovq k2, k3
vpgatherdq m14{k1}, [base+resize_shuf+4+ym1]
vpgatherdq m0{k2}, [base+resize_shuf+4+ym13]
mova m12, m16
mova m13, m17
paddb m14, m15
paddb m0, m15
pshufb m10, m14
pshufb m11, m0
vpermi2d m12, m10, m11
vpermi2d m13, m10, m11
jmp .filter
.load:
kmovq k1, k3
kmovq k2, k3
vpgatherdd m12{k1}, [srcq+m0+0]
vpgatherdd m13{k2}, [srcq+m0+4]
.filter:
kmovq k1, k3
kmovq k2, k3
vpgatherdd m10{k1}, [base+resize_filter+m9*8+0]
vpgatherdd m11{k2}, [base+resize_filter+m9*8+4]
mova m14, m2
vpdpbusd m14, m12, m10
vpdpbusd m14, m13, m11
packssdw m14, m14
pmulhrsw m14, m3
packuswb m14, m14
vpermd m14, m18, m14
mova [dstq+xq], xm14
paddd m4, m5
add xd, 16
cmp xd, dst_wd
jl .loop_x
add dstq, dst_strideq
add srcq, src_strideq
dec hd
jg .loop_y
RET
%endif ; ARCH_X86_64