; Copyright © 2018-2026, VideoLAN and dav2d authors
; Copyright © 2018-2026, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"

SECTION_RODATA 32

warp_8x8_shufA: db  0,  2,  4,  6,  1,  3,  5,  7,  1,  3,  5,  7,  2,  4,  6,  8
                db  4,  6,  8, 10,  5,  7,  9, 11,  5,  7,  9, 11,  6,  8, 10, 12
warp_8x8_shufB: db  2,  4,  6,  8,  3,  5,  7,  9,  3,  5,  7,  9,  4,  6,  8, 10
                db  6,  8, 10, 12,  7,  9, 11, 13,  7,  9, 11, 13,  8, 10, 12, 14
subpel_h_shuf4: db  0,  1,  2,  3,  1,  2,  3,  4,  8,  9, 10, 11,  9, 10, 11, 12
                db  2,  3,  4,  5,  3,  4,  5,  6, 10, 11, 12, 13, 11, 12, 13, 14
pw_4x84a10_4x42a5: times 4 dw 84, 10
                   times 4 dw 42,  5
subpel_h_shufA: db  0,  1,  2,  3,  1,  2,  3,  4,  2,  3,  4,  5,  3,  4,  5,  6
subpel_h_shufB: db  4,  5,  6,  7,  5,  6,  7,  8,  6,  7,  8,  9,  7,  8,  9, 10
subpel_h_shufC: db  8,  9, 10, 11,  9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
subpel_v_shuf4: db  0,  4,  8, 12,  1,  5,  9, 13,  2,  6, 10, 14,  3,  7, 11, 15
subpel_s_shuf2: db  0,  1,  2,  3,  0,  1,  2,  3,  8,  9, 10, 11,  8,  9, 10, 11
subpel_s_shuf8: db  0,  1,  8,  9,  2,  3, 10, 11,  4,  5, 12, 13,  6,  7, 14, 15
bilin_h_shuf4:  db  0,  1,  1,  2,  2,  3,  3,  4,  8,  9,  9, 10, 10, 11, 11, 12
bilin_v_shuf4:  db  4,  0,  5,  1,  6,  2,  7,  3,  8,  4,  9,  5, 10,  6, 11,  7
deint_shuf4:    db  0,  4,  1,  5,  2,  6,  3,  7,  4,  8,  5,  9,  6, 10,  7, 11
blend_shuf:     db  0,  1,  0,  1,  0,  1,  0,  1,  2,  3,  2,  3,  2,  3,  2,  3
pb_8x0_8x8:     db  0,  0,  0,  0,  0,  0,  0,  0,  8,  8,  8,  8,  8,  8,  8,  8
bdct_lb_dw:     db  0,  0,  0,  0,  4,  4,  4,  4,  8,  8,  8,  8, 12, 12, 12, 12
wswap:          db  2,  3,  0,  1,  6,  7,  4,  5, 10, 11,  8,  9, 14, 15, 12, 13
morph_A:        db  0, -1,  4, -1,  1, -1,  5, -1,  2, -1,  6, -1,  3, -1,  7, -1
morph_B:        db  8, -1, 12, -1,  9, -1, 13, -1, 10, -1, 14, -1, 11, -1, 15, -1
rescale_mul:    dd  0,  1,  2,  3,  4,  5,  6,  7

wm_420_sign:    dd 0x01020102, 0x01010101
wm_422_sign:    dd 0x80808080, 0x7f7f7f7f

pw_3x42a5_1x84a10: times 3 dw 42,  5
                   times 1 dw 84, 10
shuf_left_1w: db 0, 1, 0, 1
shuf_right_1w: db 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 15

pb_1_m1: times 2 db 1, -1
pb_64:   times 4 db 64
pw_m256: times 2 dw -256
pw_1:    times 2 dw 1
pw_15:   times 2 dw 15
pw_32:   times 2 dw 32
pw_34:   times 2 dw 34
pw_258:  times 2 dw 258
pw_512:  times 2 dw 512
pw_1024: times 2 dw 1024
pw_2048: times 2 dw 2048
pw_6903: times 2 dw 6903
pw_8192: times 2 dw 8192
pd_32:           dd 32
pd_63:           dd 63
pd_64:           dd 64
pd_512:          dd 512
pd_32768:        dd 32768
pd_0x3ff:        dd 0x3ff
pd_0x4000:       dd 0x4000
pq_16:           dq 16
pq_0x40000000:   dq 0x40000000

sadrefinemv_idx2off: db -2, -2, -2, -1, -2, +0, -2, +1, -2, +2
                     db -1, -2, -1, -1, -1, +0, -1, +1, -1, +2
                     db +0, -2, +0, -1,         +0, +1, +0, +2
                     db +1, -2, +1, -1, +1, +0, +1, +1, +1, +2
                     db +2, -2, +2, -1, +2, +0, +2, +1, +2, +2

cextern mc_subpel_filters
cextern mc_warp_filter2
cextern z_filter_s

%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)

%macro BASE_JMP_TABLE 3-*
    %xdefine %1_%2_table (%%table - %3)
    %xdefine %%base %1_%2
    %%table:
    %rep %0 - 2
        dw %%base %+ _w%3 - %%base
        %rotate 1
    %endrep
%endmacro

%macro HV_JMP_TABLE 5-*
    %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3)
    %xdefine %%base %1_%3
    %assign %%types %4
    %if %%types & 1
        %xdefine %1_%2_h_%3_table  (%%h  - %5)
        %%h:
        %rep %0 - 4
            dw %%prefix %+ .h_w%5 - %%base
            %rotate 1
        %endrep
        %rotate 4
    %endif
    %if %%types & 2
        %xdefine %1_%2_v_%3_table  (%%v  - %5)
        %%v:
        %rep %0 - 4
            dw %%prefix %+ .v_w%5 - %%base
            %rotate 1
        %endrep
        %rotate 4
    %endif
    %if %%types & 4
        %xdefine %1_%2_hv_%3_table (%%hv - %5)
        %%hv:
        %rep %0 - 4
            dw %%prefix %+ .hv_w%5 - %%base
            %rotate 1
        %endrep
    %endif
%endmacro

%macro BIDIR_JMP_TABLE 2-*
    %xdefine %1_%2_table (%%table - 2*%3)
    %xdefine %%base %1_%2_table
    %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2)
    %%table:
    %rep %0 - 2
        dd %%prefix %+ .w%3 - %%base
        %rotate 1
    %endrep
%endmacro

%macro SCALED_JMP_TABLE 2-*
    %xdefine %1_%2_table (%%table - %3)
    %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2)
%%table:
    %rep %0 - 2
        dw %%base %+ .w%3 - %%base
        %rotate 1
    %endrep
    %rotate 2
%%dy_1024:
    %xdefine %1_%2_dy1_table (%%dy_1024 - %3)
    %rep %0 - 2
        dw %%base %+ .dy1_w%3 - %%base
        %rotate 1
    %endrep
    %rotate 2
%%dy_2048:
    %xdefine %1_%2_dy2_table (%%dy_2048 - %3)
    %rep %0 - 2
        dw %%base %+ .dy2_w%3 - %%base
        %rotate 1
    %endrep
%endmacro

%xdefine put_avx2 mangle(private_prefix %+ _put_bilin_8bpc_avx2.put)
%xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_8bpc_avx2.prep)

%define table_offset(type, fn) type %+ fn %+ _avx2_table - type %+ _avx2

BASE_JMP_TABLE   put,               avx2,    2, 4, 8, 16, 32, 64
BASE_JMP_TABLE   prep,              avx2,       4, 8, 16, 32, 64
HV_JMP_TABLE     put,  bilin,       avx2, 7, 2, 4, 8, 16, 32, 64
HV_JMP_TABLE     prep, bilin,       avx2, 7,    4, 8, 16, 32, 64
HV_JMP_TABLE     put,  6tap,        avx2, 3, 2, 4, 8, 16, 32, 64
HV_JMP_TABLE     put,  8tap_sharp,  avx2, 3, 2, 4, 8, 16, 32, 64
HV_JMP_TABLE     prep, 6tap,        avx2, 1,    4, 8, 16, 32, 64
HV_JMP_TABLE     prep, 8tap_sharp,  avx2, 1,    4, 8, 16, 32, 64
SCALED_JMP_TABLE put_8tap_scaled,   avx2,    2, 4, 8, 16, 32, 64
SCALED_JMP_TABLE prep_8tap_scaled,  avx2,       4, 8, 16, 32, 64
BIDIR_JMP_TABLE  avg,               avx2,       4, 8, 16, 32, 64
BIDIR_JMP_TABLE  w_avg,             avx2,       4, 8, 16, 32, 64
BIDIR_JMP_TABLE  mask,              avx2,       4, 8, 16, 32, 64
BIDIR_JMP_TABLE  w_mask_420,        avx2,       4, 8, 16, 32, 64
BIDIR_JMP_TABLE  w_mask_422,        avx2,       4, 8, 16, 32, 64
BIDIR_JMP_TABLE  w_mask_444,        avx2,       4, 8, 16, 32, 64
BIDIR_JMP_TABLE  blend,             avx2,       4, 8, 16, 32, 64
BIDIR_JMP_TABLE  morph,             avx2,       4, 8, 16, 32, 64

SECTION .text

INIT_XMM avx2
cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
    movifnidn          mxyd, r6m ; mx
    lea                  r7, [put_avx2]
    tzcnt                wd, wm
    movifnidn            hd, hm
    test               mxyd, mxyd
    jnz .h
    mov                mxyd, r7m ; my
    test               mxyd, mxyd
    jnz .v
.put:
    movzx                wd, word [r7+wq*2+table_offset(put,)]
    add                  wq, r7
    jmp                  wq
.put_w2:
    movzx               r6d, word [srcq+ssq*0]
    movzx               r7d, word [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mov        [dstq+dsq*0], r6w
    mov        [dstq+dsq*1], r7w
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w2
    RET
.put_w4:
    mov                 r6d, [srcq+ssq*0]
    mov                 r7d, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mov        [dstq+dsq*0], r6d
    mov        [dstq+dsq*1], r7d
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w4
    RET
.put_w8:
    mov                  r6, [srcq+ssq*0]
    mov                  r7, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mov        [dstq+dsq*0], r6
    mov        [dstq+dsq*1], r7
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w8
    RET
.put_w16:
    movu                 m0, [srcq+ssq*0]
    movu                 m1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    movu       [dstq+dsq*0], m0
    movu       [dstq+dsq*1], m1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w16
    RET
INIT_YMM avx2
.put_w32:
    movu                 m0, [srcq+ssq*0]
    movu                 m1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    movu       [dstq+dsq*0], m0
    movu       [dstq+dsq*1], m1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w32
    RET
.put_w64:
    movu                 m0, [srcq+ssq*0+32*0]
    movu                 m1, [srcq+ssq*0+32*1]
    movu                 m2, [srcq+ssq*1+32*0]
    movu                 m3, [srcq+ssq*1+32*1]
    lea                srcq, [srcq+ssq*2]
    mova  [dstq+dsq*0+32*0], m0
    mova  [dstq+dsq*0+32*1], m1
    mova  [dstq+dsq*1+32*0], m2
    mova  [dstq+dsq*1+32*1], m3
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w64
    RET
.h:
    ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4
    ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4
    imul               mxyd, 255
    vbroadcasti128       m4, [z_filter_s+2]
    add                mxyd, 16
    movd                xm5, mxyd
    mov                mxyd, r7m ; my
    vpbroadcastw         m5, xm5
    test               mxyd, mxyd
    jnz .hv
    movzx                wd, word [r7+wq*2+table_offset(put, _bilin_h)]
    vpbroadcastd         m3, [pw_2048]
    add                  wq, r7
    jmp                  wq
.h_w2:
    movd                xm0, [srcq+ssq*0]
    pinsrd              xm0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    pshufb              xm0, xm4
    pmaddubsw           xm0, xm5
    pmulhrsw            xm0, xm3
    packuswb            xm0, xm0
    pextrw     [dstq+dsq*0], xm0, 0
    pextrw     [dstq+dsq*1], xm0, 2
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w2
    RET
.h_w4:
    mova                xm4, [bilin_h_shuf4]
.h_w4_loop:
    movq                xm0, [srcq+ssq*0]
    movhps              xm0, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pshufb              xm0, xm4
    pmaddubsw           xm0, xm5
    pmulhrsw            xm0, xm3
    packuswb            xm0, xm0
    movd       [dstq+dsq*0], xm0
    pextrd     [dstq+dsq*1], xm0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w4_loop
    RET
.h_w8:
    movu                xm0, [srcq+ssq*0]
    movu                xm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pshufb              xm0, xm4
    pshufb              xm1, xm4
    pmaddubsw           xm0, xm5
    pmaddubsw           xm1, xm5
    pmulhrsw            xm0, xm3
    pmulhrsw            xm1, xm3
    packuswb            xm0, xm1
    movq       [dstq+dsq*0], xm0
    movhps     [dstq+dsq*1], xm0
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w8
    RET
.h_w16:
    movu                xm0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*1+8*0], 1
    movu                xm1, [srcq+ssq*0+8*1]
    vinserti128          m1, [srcq+ssq*1+8*1], 1
    lea                srcq, [srcq+ssq*2]
    pshufb               m0, m4
    pshufb               m1, m4
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    pmulhrsw             m0, m3
    pmulhrsw             m1, m3
    packuswb             m0, m1
    movu         [dstq+dsq*0], xm0
    vextracti128 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w16
    RET
.h_w32:
    movu                 m0, [srcq+8*0]
    movu                 m1, [srcq+8*1]
    add                srcq, ssq
    pshufb               m0, m4
    pshufb               m1, m4
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    pmulhrsw             m0, m3
    pmulhrsw             m1, m3
    packuswb             m0, m1
    movu             [dstq], m0
    add                dstq, dsq
    dec                  hd
    jg .h_w32
    RET
.h_w64:
    movu                 m0, [srcq+8*0]
    movu                 m1, [srcq+8*1]
    pshufb               m0, m4
    pshufb               m1, m4
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    pmulhrsw             m0, m3
    pmulhrsw             m1, m3
    packuswb             m0, m1
    movu                 m1, [srcq+8*4]
    movu                 m2, [srcq+8*5]
    add                srcq, ssq
    pshufb               m1, m4
    pshufb               m2, m4
    pmaddubsw            m1, m5
    pmaddubsw            m2, m5
    pmulhrsw             m1, m3
    pmulhrsw             m2, m3
    packuswb             m1, m2
    mova        [dstq+32*0], m0
    mova        [dstq+32*1], m1
    add                dstq, dsq
    dec                  hd
    jg .h_w64
    RET
.v:
    movzx                wd, word [r7+wq*2+table_offset(put, _bilin_v)]
    imul               mxyd, 255
    vpbroadcastd         m5, [pw_2048]
    add                mxyd, 16
    add                  wq, r7
    movd                xm4, mxyd
    vpbroadcastw         m4, xm4
    jmp                  wq
.v_w2:
    movd                xm0,      [srcq+ssq*0]
.v_w2_loop:
    pinsrw              xm1, xm0, [srcq+ssq*1], 1 ; 0 1
    lea                srcq,      [srcq+ssq*2]
    pinsrw              xm0, xm1, [srcq+ssq*0], 0 ; 2 1
    pshuflw             xm1, xm1, q2301           ; 1 0
    punpcklbw           xm1, xm0
    pmaddubsw           xm1, xm4
    pmulhrsw            xm1, xm5
    packuswb            xm1, xm1
    pextrw     [dstq+dsq*0], xm1, 1
    pextrw     [dstq+dsq*1], xm1, 0
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w2_loop
    RET
.v_w4:
    movd                xm0, [srcq+ssq*0]
.v_w4_loop:
    vpbroadcastd        xm2, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpblendd            xm1, xm2, xm0, 0x01 ; 0 1
    vpbroadcastd        xm0, [srcq+ssq*0]
    vpblendd            xm2, xm0, 0x02      ; 1 2
    punpcklbw           xm1, xm2
    pmaddubsw           xm1, xm4
    pmulhrsw            xm1, xm5
    packuswb            xm1, xm1
    movd       [dstq+dsq*0], xm1
    pextrd     [dstq+dsq*1], xm1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w4_loop
    RET
.v_w8:
    movq                xm0, [srcq+ssq*0]
.v_w8_loop:
    movq                xm2, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    punpcklbw           xm1, xm0, xm2
    movq                xm0, [srcq+ssq*0]
    punpcklbw           xm2, xm0
    pmaddubsw           xm1, xm4
    pmaddubsw           xm2, xm4
    pmulhrsw            xm1, xm5
    pmulhrsw            xm2, xm5
    packuswb            xm1, xm2
    movq       [dstq+dsq*0], xm1
    movhps     [dstq+dsq*1], xm1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w8_loop
    RET
.v_w16:
    movu                xm0, [srcq+ssq*0]
.v_w16_loop:
    vbroadcasti128       m3, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpblendd             m2, m3, m0, 0x0f ; 0 1
    vbroadcasti128       m0, [srcq+ssq*0]
    vpblendd             m3, m0, 0xf0     ; 1 2
    punpcklbw            m1, m2, m3
    punpckhbw            m2, m3
    pmaddubsw            m1, m4
    pmaddubsw            m2, m4
    pmulhrsw             m1, m5
    pmulhrsw             m2, m5
    packuswb             m1, m2
    movu         [dstq+dsq*0], xm1
    vextracti128 [dstq+dsq*1], m1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w16_loop
    RET
.v_w32:
    movu                 m0, [srcq+ssq*0]
.v_w32_loop:
    movu                 m3, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    punpcklbw            m1, m0, m3
    punpckhbw            m2, m0, m3
    movu                 m0, [srcq+ssq*0]
    pmaddubsw            m1, m4
    pmaddubsw            m2, m4
    pmulhrsw             m1, m5
    pmulhrsw             m2, m5
    packuswb             m1, m2
    punpcklbw            m2, m3, m0
    punpckhbw            m3, m0
    pmaddubsw            m2, m4
    pmaddubsw            m3, m4
    pmulhrsw             m2, m5
    pmulhrsw             m3, m5
    packuswb             m2, m3
    movu       [dstq+dsq*0], m1
    movu       [dstq+dsq*1], m2
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w32_loop
    RET
.v_w64:
    movu                 m0, [srcq+32*0]
    movu                 m1, [srcq+32*1]
.v_w64_loop:
    add                srcq, ssq
    movu                 m3, [srcq+32*0]
    punpcklbw            m2, m0, m3
    punpckhbw            m0, m3
    pmaddubsw            m2, m4
    pmaddubsw            m0, m4
    pmulhrsw             m2, m5
    pmulhrsw             m0, m5
    packuswb             m2, m0
    mova                 m0, m3
    movu                 m3, [srcq+32*1]
    mova        [dstq+32*0], m2
    punpcklbw            m2, m1, m3
    punpckhbw            m1, m3
    pmaddubsw            m2, m4
    pmaddubsw            m1, m4
    pmulhrsw             m2, m5
    pmulhrsw             m1, m5
    packuswb             m2, m1
    mova                 m1, m3
    mova        [dstq+32*1], m2
    add                dstq, dsq
    dec                  hd
    jg .v_w64_loop
    RET
.hv:
    ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8
    ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4
    movzx                wd, word [r7+wq*2+table_offset(put, _bilin_hv)]
    WIN64_SPILL_XMM       8
    shl                mxyd, 11 ; can't shift by 12 due to signed overflow
    vpbroadcastd         m7, [pw_15]
    movd                xm6, mxyd
    add                  wq, r7
    paddb                m5, m5
    vpbroadcastw         m6, xm6
    jmp                  wq
.hv_w2:
    vpbroadcastd        xm0, [srcq+ssq*0]
    pshufb              xm0, xm4
    pmaddubsw           xm0, xm5
.hv_w2_loop:
    movd                xm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pinsrd              xm1, [srcq+ssq*0], 1
    pshufb              xm1, xm4
    pmaddubsw           xm1, xm5             ; 1 _ 2 _
    shufps              xm2, xm0, xm1, q1032 ; 0 _ 1 _
    mova                xm0, xm1
    psubw               xm1, xm2
    pmulhw              xm1, xm6
    pavgw               xm2, xm7
    paddw               xm1, xm2
    psrlw               xm1, 4
    packuswb            xm1, xm1
    pextrw     [dstq+dsq*0], xm1, 0
    pextrw     [dstq+dsq*1], xm1, 2
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w2_loop
    RET
.hv_w4:
    mova                xm4, [bilin_h_shuf4]
    movddup             xm0, [srcq+ssq*0]
    pshufb              xm0, xm4
    pmaddubsw           xm0, xm5
.hv_w4_loop:
    movq                xm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    movhps              xm1, [srcq+ssq*0]
    pshufb              xm1, xm4
    pmaddubsw           xm1, xm5             ; 1 2
    shufps              xm2, xm0, xm1, q1032 ; 0 1
    mova                xm0, xm1
    psubw               xm1, xm2
    pmulhw              xm1, xm6
    pavgw               xm2, xm7
    paddw               xm1, xm2
    psrlw               xm1, 4
    packuswb            xm1, xm1
    movd       [dstq+dsq*0], xm1
    pextrd     [dstq+dsq*1], xm1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w4_loop
    RET
.hv_w8:
    vbroadcasti128       m0, [srcq+ssq*0]
    pshufb               m0, m4
    pmaddubsw            m0, m5
.hv_w8_loop:
    movu                xm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vinserti128          m1, [srcq+ssq*0], 1
    pshufb               m1, m4
    pmaddubsw            m1, m5           ; 1 2
    vperm2i128           m2, m0, m1, 0x21 ; 0 1
    mova                 m0, m1
    psubw                m1, m2
    pmulhw               m1, m6
    pavgw                m2, m7
    paddw                m1, m2
    psrlw                m1, 4
    vextracti128        xm2, m1, 1
    packuswb            xm1, xm2
    movq       [dstq+dsq*0], xm1
    movhps     [dstq+dsq*1], xm1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w8_loop
    RET
.hv_w16:
    movu                 m0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*0+8*1], 1
    pshufb               m0, m4
    pmaddubsw            m0, m5
.hv_w16_loop:
    movu                xm2, [srcq+ssq*1+8*0]
    vinserti128          m2, [srcq+ssq*1+8*1], 1
    lea                srcq, [srcq+ssq*2]
    movu                xm3, [srcq+ssq*0+8*0]
    vinserti128          m3, [srcq+ssq*0+8*1], 1
    pshufb               m2, m4
    pshufb               m3, m4
    pmaddubsw            m2, m5
    psubw                m1, m2, m0
    pmulhw               m1, m6
    pavgw                m0, m7
    paddw                m1, m0
    pmaddubsw            m0, m3, m5
    psubw                m3, m0, m2
    pmulhw               m3, m6
    pavgw                m2, m7
    paddw                m3, m2
    psrlw                m1, 4
    psrlw                m3, 4
    packuswb             m1, m3
    vpermq               m1, m1, q3120
    movu         [dstq+dsq*0], xm1
    vextracti128 [dstq+dsq*1], m1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w16_loop
    RET
.hv_w64:
    lea                 r6d, [hq+(1<<16)]
.hv_w32_start:
    mov                  r4, srcq
    mov                  r7, dstq
.hv_w32:
%if WIN64
    movaps              r4m, xmm8
%endif
.hv_w32_loop0:
    movu                 m0, [srcq+8*0]
    movu                 m1, [srcq+8*1]
    pshufb               m0, m4
    pshufb               m1, m4
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
.hv_w32_loop:
    add                srcq, ssq
    movu                 m2, [srcq+8*0]
    movu                 m3, [srcq+8*1]
    pshufb               m2, m4
    pshufb               m3, m4
    pmaddubsw            m2, m5
    pmaddubsw            m3, m5
    psubw                m8, m2, m0
    pmulhw               m8, m6
    pavgw                m0, m7
    paddw                m8, m0
    mova                 m0, m2
    psubw                m2, m3, m1
    pmulhw               m2, m6
    pavgw                m1, m7
    paddw                m2, m1
    mova                 m1, m3
    psrlw                m8, 4
    psrlw                m2, 4
    packuswb             m8, m2
    movu             [dstq], m8
    add                dstq, dsq
    dec                  hd
    jg .hv_w32_loop
    add                  r4, 32
    add                  r7, 32
    movzx                hd, r6b
    mov                srcq, r4
    mov                dstq, r7
    sub                 r6d, 1<<16
    jg .hv_w32_loop0
%if WIN64
    movaps             xmm8, r4m
%endif
    RET

INIT_XMM avx2
cglobal prep_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
    movifnidn          mxyd, r6m ; mx
    lea                  r7, [prep_avx2]
    tzcnt                wd, wm
    movifnidn            hd, hm
    test               mxyd, mxyd
    jnz .h
    mov                mxyd, r7m ; my
    test               mxyd, mxyd
    jnz .v
.prep:
    movzx                wd, word [r7+wq*2+table_offset(prep,)]
    lea                  r6, [ssq*3]
    add                  wq, r7
    lea                  r7, [dsq*3]
    jmp                  wq
.prep_w4:
    movd                xm0, [srcq+ssq*0]
    pinsrd              xm0, [srcq+ssq*1], 1
    movd                xm1, [srcq+ssq*2]
    pinsrd              xm1, [srcq+r6   ], 1
    lea                srcq, [srcq+ssq*4]
    pmovzxbw            xm0, xm0
    pmovzxbw            xm1, xm1
    psllw               xm0, 4
    psllw               xm1, 4
    movq       [dstq+dsq*0], xm0
    movhps     [dstq+dsq*2], xm0
    movq       [dstq+dsq*4], xm1
    movhps     [dstq+r7 *2], xm1
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .prep_w4
    RET
INIT_YMM avx2
.prep_w8:
    movq                xm0, [srcq+ssq*0]
    movhps              xm0, [srcq+ssq*1]
    movq                xm1, [srcq+ssq*2]
    movhps              xm1, [srcq+r6   ]
    lea                srcq, [srcq+ssq*4]
    pmovzxbw             m0, xm0
    pmovzxbw             m1, xm1
    psllw                m0, 4
    psllw                m1, 4
    mova         [dstq+dsq*0], xm0
    vextracti128 [dstq+dsq*2], m0, 1
    mova         [dstq+dsq*4], xm1
    vextracti128 [dstq+r7 *2], m1, 1
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .prep_w8
    RET
.prep_w16:
    pmovzxbw             m0, [srcq+ssq*0]
    pmovzxbw             m1, [srcq+ssq*1]
    pmovzxbw             m2, [srcq+ssq*2]
    pmovzxbw             m3, [srcq+r6   ]
    lea                srcq, [srcq+ssq*4]
    REPX       {psllw x, 4}, m0, m1, m2, m3
    mova       [dstq+dsq*0], m0
    mova       [dstq+dsq*2], m1
    mova       [dstq+dsq*4], m2
    mova       [dstq+r7 *2], m3
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .prep_w16
    RET
.prep_w32:
    pmovzxbw             m0, [srcq+ssq*0+16*0]
    pmovzxbw             m1, [srcq+ssq*0+16*1]
    pmovzxbw             m2, [srcq+ssq*1+16*0]
    pmovzxbw             m3, [srcq+ssq*1+16*1]
    lea                srcq, [srcq+ssq*2]
    REPX       {psllw x, 4}, m0, m1, m2, m3
    mova  [dstq+dsq*0+32*0], m0
    mova  [dstq+dsq*0+32*1], m1
    mova  [dstq+dsq*2+32*0], m2
    mova  [dstq+dsq*2+32*1], m3
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 2
    jg .prep_w32
    RET
.prep_w64:
    pmovzxbw             m0, [srcq+16*0]
    pmovzxbw             m1, [srcq+16*1]
    pmovzxbw             m2, [srcq+16*2]
    pmovzxbw             m3, [srcq+16*3]
    add                srcq, ssq
    REPX       {psllw x, 4}, m0, m1, m2, m3
    mova        [dstq+32*0], m0
    mova        [dstq+32*1], m1
    mova        [dstq+32*2], m2
    mova        [dstq+32*3], m3
    lea                dstq, [dstq+dsq*2]
    dec                  hd
    jg .prep_w64
    RET
.h:
    ; 16 * src[x] + (mx * (src[x + 1] - src[x]))
    ; = (16 - mx) * src[x] + mx * src[x + 1]
    imul               mxyd, 255
    vbroadcasti128       m4, [z_filter_s+2]
    add                mxyd, 16
    movd                xm5, mxyd
    mov                mxyd, r7m ; my
    vpbroadcastw         m5, xm5
    test               mxyd, mxyd
    jnz .hv
    movzx                wd, word [r7+wq*2+table_offset(prep, _bilin_h)]
    lea                  r6, [ssq*3]
    add                  wq, r7
    lea                  r7, [dsq*3]
    jmp                  wq
.h_w4:
    vbroadcasti128       m4, [bilin_h_shuf4]
.h_w4_loop:
    movq                xm0, [srcq+ssq*0]
    movhps              xm0, [srcq+ssq*1]
    movq                xm1, [srcq+ssq*2]
    movhps              xm1, [srcq+r6   ]
    lea                srcq, [srcq+ssq*4]
    pshufb              xm0, xm4
    pshufb              xm1, xm4
    pmaddubsw           xm0, xm5
    pmaddubsw           xm1, xm5
    movq       [dstq+dsq*0], xm0
    movhps     [dstq+dsq*2], xm0
    movq       [dstq+dsq*4], xm1
    movhps     [dstq+r7 *2], xm1
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .h_w4_loop
    RET
.h_w8:
    movu                xm0, [srcq+ssq*0]
    vinserti128          m0, [srcq+ssq*1], 1
    movu                xm1, [srcq+ssq*2]
    vinserti128          m1, [srcq+r6   ], 1
    lea                srcq, [srcq+ssq*4]
    pshufb               m0, m4
    pshufb               m1, m4
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    mova         [dstq+dsq*0], xm0
    vextracti128 [dstq+dsq*2], m0, 1
    mova         [dstq+dsq*4], xm1
    vextracti128 [dstq+r7 *2], m1, 1
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .h_w8
    RET
.h_w16:
    movu                xm0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*0+8*1], 1
    movu                xm1, [srcq+ssq*1+8*0]
    vinserti128          m1, [srcq+ssq*1+8*1], 1
    movu                xm2, [srcq+ssq*2+8*0]
    vinserti128          m2, [srcq+ssq*2+8*1], 1
    movu                xm3, [srcq+r6   +8*0]
    vinserti128          m3, [srcq+r6   +8*1], 1
    lea                srcq, [srcq+ssq*4]
    REPX  {pshufb    x, m4}, m0, m1, m2, m3
    REPX  {pmaddubsw x, m5}, m0, m1, m2, m3
    mova       [dstq+dsq*0], m0
    mova       [dstq+dsq*2], m1
    mova       [dstq+dsq*4], m2
    mova       [dstq+r7 *2], m3
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .h_w16
    RET
.h_w32:
    movu                xm0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*0+8*1], 1
    movu                xm1, [srcq+ssq*0+8*2]
    vinserti128          m1, [srcq+ssq*0+8*3], 1
    movu                xm2, [srcq+ssq*1+8*0]
    vinserti128          m2, [srcq+ssq*1+8*1], 1
    movu                xm3, [srcq+ssq*1+8*2]
    vinserti128          m3, [srcq+ssq*1+8*3], 1
    lea                srcq, [srcq+ssq*2]
    REPX  {pshufb    x, m4}, m0, m1, m2, m3
    REPX  {pmaddubsw x, m5}, m0, m1, m2, m3
    mova  [dstq+dsq*0+32*0], m0
    mova  [dstq+dsq*0+32*1], m1
    mova  [dstq+dsq*2+32*0], m2
    mova  [dstq+dsq*2+32*1], m3
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 2
    jg .h_w32
    RET
.h_w64:
    movu                xm0, [srcq+8*0]
    vinserti128          m0, [srcq+8*1], 1
    movu                xm1, [srcq+8*2]
    vinserti128          m1, [srcq+8*3], 1
    movu                xm2, [srcq+8*4]
    vinserti128          m2, [srcq+8*5], 1
    movu                xm3, [srcq+8*6]
    vinserti128          m3, [srcq+8*7], 1
    add                srcq, ssq
    REPX  {pshufb    x, m4}, m0, m1, m2, m3
    REPX  {pmaddubsw x, m5}, m0, m1, m2, m3
    mova        [dstq+32*0], m0
    mova        [dstq+32*1], m1
    mova        [dstq+32*2], m2
    mova        [dstq+32*3], m3
    lea                dstq, [dstq+dsq*2]
    dec                  hd
    jg .h_w64
    RET
.v:
    movzx                wd, word [r7+wq*2+table_offset(prep, _bilin_v)]
    imul               mxyd, 255
    add                mxyd, 16
    add                  wq, r7
    lea                  r7, [dsq*3]
    movd                xm5, mxyd
    lea                  r6, [ssq*3]
    vpbroadcastw         m5, xm5
    jmp                  wq
.v_w4:
    movd                xm0, [srcq+ssq*0]
.v_w4_loop:
    movd                xm2, [srcq+ssq*1]
    vpbroadcastd        xm3, [srcq+ssq*2]
    pinsrd              xm2, [srcq+r6   ], 1 ; 1 3
    lea                srcq, [srcq+ssq*4]
    vpblendd            xm1, xm0, xm3, 0x02  ; 0 2
    vpbroadcastd        xm0, [srcq+ssq*0]
    vpblendd            xm3, xm0, 0x02       ; 2 4
    punpcklbw           xm1, xm2
    punpcklbw           xm2, xm3
    pmaddubsw           xm1, xm5
    pmaddubsw           xm2, xm5
    movq       [dstq+dsq*0], xm1
    movq       [dstq+dsq*2], xm2
    movhps     [dstq+dsq*4], xm1
    movhps     [dstq+r7 *2], xm2
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .v_w4_loop
    RET
.v_w8:
    movq                xm0, [srcq+ssq*0]
.v_w8_loop:
    movq                xm2, [srcq+ssq*1]
    vpbroadcastq         m3, [srcq+ssq*2]
    vpbroadcastq         m4, [srcq+r6   ]
    lea                srcq, [srcq+ssq*4]
    vpblendd             m1, m0, m3, 0x30 ; 0 2
    vpbroadcastq         m0, [srcq+ssq*0]
    vpblendd             m2, m4, 0x30     ; 1 3
    vpblendd             m3, m0, 0x30     ; 2 4
    punpcklbw            m1, m2
    punpcklbw            m2, m3
    pmaddubsw            m1, m5
    pmaddubsw            m2, m5
    mova         [dstq+dsq*0], xm1
    mova         [dstq+dsq*2], xm2
    vextracti128 [dstq+dsq*4], m1, 1
    vextracti128 [dstq+r7 *2], m2, 1
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .v_w8_loop
    RET
.v_w16:
    vbroadcasti128       m0, [srcq+ssq*0]
.v_w16_loop:
    vbroadcasti128       m3, [srcq+ssq*1]
    vbroadcasti128       m4, [srcq+ssq*2]
    vbroadcasti128       m1, [srcq+r6   ]
    lea                srcq, [srcq+ssq*4]
    shufpd               m2, m0, m4, 0x0c ; 0 2
    vbroadcasti128       m0, [srcq+ssq*0]
    shufpd               m3, m1, 0x0c     ; 1 3
    punpcklbw            m1, m2, m3
    punpckhbw            m2, m3
    pmaddubsw            m1, m5
    pmaddubsw            m2, m5
    shufpd               m4, m0, 0x0c     ; 2 4
    mova       [dstq+dsq*0], m1
    mova       [dstq+dsq*4], m2
    punpcklbw            m1, m3, m4
    punpckhbw            m3, m4
    pmaddubsw            m1, m5
    pmaddubsw            m3, m5
    mova       [dstq+dsq*2], m1
    mova       [dstq+r7 *2], m3
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .v_w16_loop
    RET
.v_w32:
    vpermq               m0, [srcq+ssq*0], q3120
.v_w32_loop:
    vpermq               m4, [srcq+ssq*1], q3120
    lea                srcq, [srcq+ssq*2]
    punpcklbw            m1, m0, m4
    punpckhbw            m2, m0, m4
    vpermq               m0, [srcq+ssq*0], q3120
    punpcklbw            m3, m4, m0
    punpckhbw            m4, m0
    REPX  {pmaddubsw x, m5}, m1, m2, m3, m4
    mova  [dstq+dsq*0+32*0], m1
    mova  [dstq+dsq*0+32*1], m2
    mova  [dstq+dsq*2+32*0], m3
    mova  [dstq+dsq*2+32*1], m4
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 2
    jg .v_w32_loop
    RET
.v_w64:
    vpermq               m0, [srcq+ssq*0+32*0], q3120
    vpermq               m1, [srcq+ssq*0+32*1], q3120
.v_w64_loop:
    vpermq               m2, [srcq+ssq*1+32*0], q3120
    vpermq               m3, [srcq+ssq*1+32*1], q3120
    lea                srcq, [srcq+ssq*2]
    punpcklbw            m4, m0, m2
    punpckhbw            m0, m2
    pmaddubsw            m4, m5
    pmaddubsw            m0, m5
    mova  [dstq+dsq*0+32*0], m4
    mova  [dstq+dsq*0+32*1], m0
    punpcklbw            m4, m1, m3
    punpckhbw            m1, m3
    pmaddubsw            m4, m5
    pmaddubsw            m1, m5
    mova  [dstq+dsq*0+32*2], m4
    mova  [dstq+dsq*0+32*3], m1
    vpermq               m0, [srcq+ssq*0+32*0], q3120
    vpermq               m1, [srcq+ssq*0+32*1], q3120
    punpcklbw            m4, m2, m0
    punpckhbw            m2, m0
    pmaddubsw            m4, m5
    pmaddubsw            m2, m5
    mova  [dstq+dsq*2+32*0], m4
    mova  [dstq+dsq*2+32*1], m2
    punpcklbw            m4, m3, m1
    punpckhbw            m3, m1
    pmaddubsw            m4, m5
    pmaddubsw            m3, m5
    mova  [dstq+dsq*2+32*2], m4
    mova  [dstq+dsq*2+32*3], m3
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 2
    jg .v_w64_loop
    RET
.hv:
    ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4
    ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4)
    WIN64_SPILL_XMM       7
    movzx                wd, word [r7+wq*2+table_offset(prep, _bilin_hv)]
    shl                mxyd, 11
    movd                xm6, mxyd
    lea                  r6, [ssq*3]
    vpbroadcastw         m6, xm6
    add                  wq, r7
    lea                  r7, [dsq*3]
    jmp                  wq
.hv_w4:
    vbroadcasti128       m4, [bilin_h_shuf4]
    vpbroadcastq         m0, [srcq+ssq*0]
    pshufb               m0, m4
    pmaddubsw            m0, m5
.hv_w4_loop:
    movq                xm1, [srcq+ssq*1]
    movhps              xm1, [srcq+ssq*2]
    movq                xm2, [srcq+r6   ]
    lea                srcq, [srcq+ssq*4]
    movhps              xm2, [srcq+ssq*0]
    vinserti128          m1, xm2, 1
    pshufb               m1, m4
    pmaddubsw            m1, m5        ; 1 2 3 4
    vpblendd             m2, m1, m0, 0xc0
    vpermq               m2, m2, q2103 ; 0 1 2 3
    mova                 m0, m1
    psubw                m1, m2
    pmulhrsw             m1, m6
    paddw                m1, m2
    vextracti128        xm2, m1, 1
    movq       [dstq+dsq*0], xm1
    movhps     [dstq+dsq*2], xm1
    movq       [dstq+dsq*4], xm2
    movhps     [dstq+r7 *2], xm2
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .hv_w4_loop
    RET
.hv_w8:
    vbroadcasti128       m0, [srcq+ssq*0]
    pshufb               m0, m4
    pmaddubsw            m0, m5
.hv_w8_loop:
    movu                xm1, [srcq+ssq*1]
    vinserti128          m1, [srcq+ssq*2], 1
    movu                xm2, [srcq+r6   ]
    lea                srcq, [srcq+ssq*4]
    vinserti128          m2, [srcq+ssq*0], 1
    pshufb               m1, m4
    pshufb               m2, m4
    pmaddubsw            m1, m5           ; 1 2
    vperm2i128           m3, m0, m1, 0x21 ; 0 1
    pmaddubsw            m0, m2, m5       ; 3 4
    vperm2i128           m2, m1, m0, 0x21 ; 2 3
    psubw                m1, m3
    pmulhrsw             m1, m6
    paddw                m1, m3
    psubw                m3, m0, m2
    pmulhrsw             m3, m6
    paddw                m3, m2
    mova         [dstq+dsq*0], xm1
    vextracti128 [dstq+dsq*2], m1, 1
    mova         [dstq+dsq*4], xm3
    vextracti128 [dstq+r7 *2], m3, 1
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .hv_w8_loop
    RET
.hv_w16:
    movu                xm0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*0+8*1], 1
    pshufb               m0, m4
    pmaddubsw            m0, m5
.hv_w16_loop:
    movu                xm1, [srcq+ssq*1+8*0]
    vinserti128          m1, [srcq+ssq*1+8*1], 1
    lea                srcq, [srcq+ssq*2]
    movu                xm2, [srcq+ssq*0+8*0]
    vinserti128          m2, [srcq+ssq*0+8*1], 1
    pshufb               m1, m4
    pshufb               m2, m4
    pmaddubsw            m1, m5
    psubw                m3, m1, m0
    pmulhrsw             m3, m6
    paddw                m3, m0
    pmaddubsw            m0, m2, m5
    psubw                m2, m0, m1
    pmulhrsw             m2, m6
    paddw                m2, m1
    mova       [dstq+dsq*0], m3
    mova       [dstq+dsq*2], m2
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 2
    jg .hv_w16_loop
    RET
.hv_w32:
    movu                xm0, [srcq+8*0]
    vinserti128          m0, [srcq+8*1], 1
    movu                xm1, [srcq+8*2]
    vinserti128          m1, [srcq+8*3], 1
    pshufb               m0, m4
    pshufb               m1, m4
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
.hv_w32_loop:
    add                srcq, ssq
    movu                xm2, [srcq+8*0]
    vinserti128          m2, [srcq+8*1], 1
    pshufb               m2, m4
    pmaddubsw            m2, m5
    psubw                m3, m2, m0
    pmulhrsw             m3, m6
    paddw                m3, m0
    mova                 m0, m2
    movu                xm2, [srcq+8*2]
    vinserti128          m2, [srcq+8*3], 1
    pshufb               m2, m4
    pmaddubsw            m2, m5
    mova        [dstq+32*0], m3
    psubw                m3, m2, m1
    pmulhrsw             m3, m6
    paddw                m3, m1
    mova                 m1, m2
    mova        [dstq+32*1], m3
    lea                dstq, [dstq+dsq*2]
    dec                  hd
    jg .hv_w32_loop
    RET
.hv_w64:
    lea                 r6d, [hq+(3<<8)]
.hv_w64_loop0:
    movu                xm0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*0+8*1], 1
    mov                  r4, srcq
    mov                  r7, dstq
    pshufb               m0, m4
    pmaddubsw            m0, m5
.hv_w64_loop:
    movu                xm1, [r4+ssq*1+8*0]
    vinserti128          m1, [r4+ssq*1+8*1], 1
    lea                  r4, [r4+ssq*2]
    movu                xm2, [r4+ssq*0+8*0]
    vinserti128          m2, [r4+ssq*0+8*1], 1
    pshufb               m1, m4
    pshufb               m2, m4
    pmaddubsw            m1, m5
    psubw                m3, m1, m0
    pmulhrsw             m3, m6
    paddw                m3, m0
    pmaddubsw            m0, m2, m5
    psubw                m2, m0, m1
    pmulhrsw             m2, m6
    paddw                m2, m1
    mova         [r7+dsq*0], m3
    mova         [r7+dsq*2], m2
    lea                  r7, [r7+dsq*4]
    sub                  hd, 2
    jg .hv_w64_loop
    add                srcq, 16
    add                dstq, 32
    movzx                hd, r6b
    sub                 r6d, 1<<8
    jg .hv_w64_loop0
    RET

; int8_t subpel_filters[5][15][8]
%assign FILTER_REGULAR (0*15 << 16) | 3*15
%assign FILTER_SMOOTH  (1*15 << 16) | 4*15
%assign FILTER_SHARP   (2*15 << 16) | 3*15

%macro FN 3-4 ; fn, name, type, jmp_to
cglobal %1_%2_8bpc
    mov                 t0d, FILTER_%3
%if %0 == 4 ; skip the jump in the last filter
    jmp mangle(private_prefix %+ _%4_avx2)
%endif
%endmacro

%if WIN64
DECLARE_REG_TMP 5
%else
DECLARE_REG_TMP 8
%endif

%define PUT_8TAP_FN FN put_8tap,
PUT_8TAP_FN smooth,  SMOOTH,  put_6tap_8bpc
PUT_8TAP_FN regular, REGULAR

cglobal put_6tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ns
    imul                mxd, mxm, 0x010101
    imul                myd, mym, 0x010101
    mov                  wd, wm
    add                 mxd, t0d ; 6tap_h, mx, 4tap_h
    add                 myd, t0d ; 6tap_v, my, 4tap_v
    movifnidn            hd, hm
    lea                  r8, [put_avx2]
    test                mxd, 0xf00
    jnz .h
    test                myd, 0xf00
    jnz .v
.put:
    tzcnt                wd, wd
    movzx                wd, word [r8+wq*2+table_offset(put,)]
    add                  wq, r8
    lea                  r6, [ssq*3]
    lea                  r7, [dsq*3]
%if WIN64
    pop                  r8
%endif
    jmp                  wq
.h_w2:
    movzx               mxd, mxb
    lea                srcq, [srcq-1]
    vpbroadcastd        xm4, [r8+mxq*8+subpel_filters-put_avx2+2]
    je .h_w4
    mova                xm3, [subpel_h_shuf4]
.h_w2_loop:
    movq                xm0, [srcq+ssq*0]
    movhps              xm0, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pshufb              xm0, xm3
    pmaddubsw           xm0, xm4
    phaddw              xm0, xm0
    paddw               xm0, xm5
    psraw               xm0, 6
    packuswb            xm0, xm0
    pextrw     [dstq+dsq*0], xm0, 0
    pextrw     [dstq+dsq*1], xm0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w2_loop
    RET
.h_w4:
    mova                xm3, [subpel_h_shufA]
.h_w4_loop:
    movq                xm0, [srcq+ssq*0]
    movq                xm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pshufb              xm0, xm3
    pshufb              xm1, xm3
    pmaddubsw           xm0, xm4
    pmaddubsw           xm1, xm4
    phaddw              xm0, xm1
    paddw               xm0, xm5
    psraw               xm0, 6
    packuswb            xm0, xm0
    movd       [dstq+dsq*0], xm0
    pextrd     [dstq+dsq*1], xm0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w4_loop
    RET
.h:
    test                myd, 0xf00
    jnz .hv
    vpbroadcastd         m5, [pw_34] ; 2 + (8 << 2)
    cmp                  wd, 4
    jle .h_w2
    WIN64_SPILL_XMM      11
    tzcnt                wd, wd
    vbroadcasti128       m4, [z_filter_s+ 2] ; 01
    shr                 mxd, 16
    vbroadcasti128       m6, [z_filter_s+ 6] ; 23
    sub                srcq, 2
    vbroadcasti128       m7, [z_filter_s+10] ; 45
    lea                 mxq, [r8+mxq*8+subpel_filters+1-put_avx2]
    movzx                wd, word [r8+wq*2+table_offset(put, _6tap_h)]
    vpbroadcastw         m8, [mxq+0]
    vpbroadcastw         m9, [mxq+2]
    add                  wq, r8
    vpbroadcastw        m10, [mxq+4]
    jmp                  wq
.h_w8:
%macro PUT_6TAP_H 3 ; dst/src, tmp[1-2]
    pshufb              m%2, m%1, m4
    pmaddubsw           m%2, m8
    pshufb              m%3, m%1, m6
    pmaddubsw           m%3, m9
    pshufb              m%1, m7
    pmaddubsw           m%1, m10
    paddw               m%2, m5
    paddw               m%1, m%3
    paddw               m%1, m%2
    psraw               m%1, 6
%endmacro
    movu                xm0, [srcq+ssq*0]
    vinserti128          m0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    PUT_6TAP_H            0, 1, 2
    vextracti128        xm1, m0, 1
    packuswb            xm0, xm1
    movq       [dstq+dsq*0], xm0
    movhps     [dstq+dsq*1], xm0
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w8
    RET
.h_w16:
    movu                xm0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*1+8*0], 1
    movu                xm1, [srcq+ssq*0+8*1]
    vinserti128          m1, [srcq+ssq*1+8*1], 1
    PUT_6TAP_H            0, 2, 3
    lea                srcq, [srcq+ssq*2]
    PUT_6TAP_H            1, 2, 3
    packuswb             m0, m1
    movu         [dstq+dsq*0], xm0
    vextracti128 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w16
    RET
.h_w32:
    xor                 r6d, r6d
    jmp .h_start
.h_w64:
    mov                  r6, -32*1
    sub                srcq, r6
    sub                dstq, r6
.h_start:
    mov                  r4, r6
.h_loop:
    movu                 m0, [srcq+r6+8*0]
    movu                 m1, [srcq+r6+8*1]
    PUT_6TAP_H            0, 2, 3
    PUT_6TAP_H            1, 2, 3
    packuswb             m0, m1
    movu          [dstq+r6], m0
    add                  r6, 32
    jle .h_loop
    add                srcq, ssq
    add                dstq, dsq
    mov                  r6, r4
    dec                  hd
    jg .h_loop
    RET
.v:
    WIN64_SPILL_XMM       9, 12
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    tzcnt               r6d, wd
    movzx               r6d, word [r8+r6*2+table_offset(put, _6tap_v)]
    vpbroadcastd         m8, [pw_512]
    lea                 myq, [r8+myq*8+subpel_filters+1-put_avx2]
    vpbroadcastw         m5, [myq+0]
    vpbroadcastw         m6, [myq+2]
    vpbroadcastw         m7, [myq+4]
    add                  r6, r8
    mov                 nsq, ssq
    neg                 nsq
    jmp                  r6
.v_w2:
    movd                xm2, [srcq+nsq*2]
    pinsrw              xm2, [srcq+nsq*1], 2
    pinsrw              xm2, [srcq+ssq*0], 4
    pinsrw              xm2, [srcq+ssq*1], 6 ; 0 1 2 3
    lea                srcq, [srcq+ssq*2]
    vpbroadcastd        xm0, [srcq+ssq*0]
    palignr             xm3, xm0, xm2, 4     ; 1 2 3 4
    punpcklbw           xm1, xm2, xm3        ; 01 12
    punpckhbw           xm2, xm3             ; 23 34
.v_w2_loop:
    vpbroadcastd        xm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pmaddubsw           xm3, xm1, xm5        ; a0 b0
    mova                xm1, xm2
    pmaddubsw           xm2, xm6             ; a1 b1
    paddw               xm3, xm2
    vpblendd            xm2, xm0, xm4, 0x02  ; 4 5
    vpbroadcastd        xm0, [srcq+ssq*0]
    vpblendd            xm4, xm0, 0x02       ; 5 6
    punpcklbw           xm2, xm4             ; 67 78
    pmaddubsw           xm4, xm2, xm7        ; a3 b3
    paddw               xm3, xm4
    pmulhrsw            xm3, xm8
    packuswb            xm3, xm3
    pextrw     [dstq+dsq*0], xm3, 0
    pextrw     [dstq+dsq*1], xm3, 2
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w2_loop
    RET
.v_w4:
    movd                xm2, [srcq+nsq*2]
    pinsrd              xm2, [srcq+nsq*1], 1
    pinsrd              xm2, [srcq+ssq*0], 2
    pinsrd              xm2, [srcq+ssq*1], 3 ; 0 1 2 3
    lea                srcq, [srcq+ssq*2]
    vpbroadcastd        xm0, [srcq+ssq*0]
    palignr             xm3, xm0, xm2, 4     ; 1 2 3 4
    punpcklbw           xm1, xm2, xm3        ; 01 12
    punpckhbw           xm2, xm3             ; 23 34
.v_w4_loop:
    vpbroadcastd        xm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pmaddubsw           xm3, xm1, xm5        ; a0 b0
    mova                xm1, xm2
    pmaddubsw           xm2, xm6             ; a1 b1
    paddw               xm3, xm2
    vpblendd            xm2, xm0, xm4, 0x02  ; 4 5
    vpbroadcastd        xm0, [srcq+ssq*0]
    vpblendd            xm4, xm0, 0x02       ; 5 6
    punpcklbw           xm2, xm4             ; 45 56
    pmaddubsw           xm4, xm2, xm7        ; a2 b2
    paddw               xm3, xm4
    pmulhrsw            xm3, xm8
    packuswb            xm3, xm3
    movd       [dstq+dsq*0], xm3
    pextrd     [dstq+dsq*1], xm3, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w4_loop
    RET
.v_w8:
    movq                xm1, [srcq+nsq*2]
    vpbroadcastq         m3, [srcq+nsq*1]
    vpbroadcastq         m2, [srcq+ssq*0]
    vpbroadcastq         m4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpbroadcastq         m0, [srcq+ssq*0]
    vpblendd             m1, m3, 0x30
    vpblendd             m3, m2, 0x30
    punpcklbw            m1, m3      ; 01 12
    vpblendd             m2, m4, 0x30
    vpblendd             m4, m0, 0x30
    punpcklbw            m2, m4      ; 23 34
.v_w8_loop:
    vpbroadcastq         m4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pmaddubsw            m3, m1, m5  ; a0 b0
    mova                 m1, m2
    pmaddubsw            m2, m6      ; a1 b1
    paddw                m3, m2
    vpblendd             m2, m0, m4, 0x30
    vpbroadcastq         m0, [srcq+ssq*0]
    vpblendd             m4, m0, 0x30
    punpcklbw            m2, m4      ; 45 56
    pmaddubsw            m4, m2, m7  ; a2 b2
    paddw                m3, m4
    pmulhrsw             m3, m8
    vextracti128        xm4, m3, 1
    packuswb            xm3, xm4
    movq       [dstq+dsq*0], xm3
    movhps     [dstq+dsq*1], xm3
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w8_loop
    RET
.v_w16:
.v_w32:
.v_w64:
    lea                 r6d, [wq*8-128]
    WIN64_PUSH_XMM       12
    lea                 r6d, [hq+r6*2]
.v_w16_loop0:
    vbroadcasti128       m3, [srcq+nsq*2]
    vbroadcasti128       m4, [srcq+nsq*1]
    lea                  r4, [srcq+ssq*2]
    vbroadcasti128       m0, [srcq+ssq*0]
    vbroadcasti128       m1, [srcq+ssq*1]
    mov                  r7, dstq
    vbroadcasti128       m2, [r4+ssq*0]
    shufpd               m3, m0, 0x0c
    shufpd               m4, m1, 0x0c
    punpcklbw            m1, m3, m4 ; 01
    punpckhbw            m3, m4     ; 23
    shufpd               m0, m2, 0x0c
    punpcklbw            m2, m4, m0 ; 12
    punpckhbw            m4, m0     ; 34
.v_w16_loop:
    vbroadcasti128       m9, [r4+ssq*1]
    pmaddubsw           m10, m1, m5  ; a0
    lea                  r4, [r4+ssq*2]
    pmaddubsw           m11, m2, m5  ; b0
    mova                 m1, m3
    pmaddubsw            m3, m6      ; a1
    mova                 m2, m4
    pmaddubsw            m4, m6      ; b1
    paddw               m10, m3
    vbroadcasti128       m3, [r4+ssq*0]
    paddw               m11, m4
    shufpd               m4, m0, m9, 0x0d
    shufpd               m0, m9, m3, 0x0c
    punpcklbw            m3, m4, m0  ; 45
    punpckhbw            m4, m0      ; 56
    pmaddubsw            m9, m3, m7  ; a2
    paddw               m10, m9
    pmaddubsw            m9, m4, m7  ; b2
    paddw               m11, m9
    pmulhrsw            m10, m8
    pmulhrsw            m11, m8
    packuswb            m10, m11
    vpermq              m10, m10, q3120
    movu         [r7+dsq*0], xm10
    vextracti128 [r7+dsq*1], m10, 1
    lea                  r7, [r7+dsq*2]
    sub                  hd, 2
    jg .v_w16_loop
    add                srcq, 16
    add                dstq, 16
    movzx                hd, r6b
    sub                 r6d, 1<<8
    jg .v_w16_loop0
    RET
.hv:
    WIN64_SPILL_XMM      12, 16
    cmp                  wd, 4
    jg .hv_w8
    movzx               mxd, mxb
    dec                srcq
    vpbroadcastd         m6, [r8+mxq*8+subpel_filters-put_avx2+2]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    vpbroadcastq         m0, [r8+myq*8+subpel_filters+1-put_avx2]
    vpbroadcastd         m7, [pw_8192]
    punpcklbw            m0, m0
    vpbroadcastd         m8, [pd_512]
    psraw                m0, 8 ; sign-extend
    mov                 nsq, ssq
    pshufd               m9, m0, q0000
    neg                 nsq
    pshufd              m10, m0, q1111
    pshufd              m11, m0, q2222
    cmp                  wd, 4
    je .hv_w4
    vbroadcasti128       m5, [subpel_h_shuf4]
    movq                xm2, [srcq+nsq*2]
    movhps              xm2, [srcq+nsq*1]
    movq                xm0, [srcq+ssq*0]
    movhps              xm0, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpbroadcastq         m1, [srcq+ssq*0]
    vpblendd             m2, m1, 0x30
    pshufb               m2, m5
    pshufb              xm0, xm5
    pmaddubsw            m2, m6
    pmaddubsw           xm0, xm6
    phaddw               m2, m0
    pmulhrsw             m2, m7
    vextracti128        xm0, m2, 1
    palignr             xm0, xm2, 4
    punpcklwd           xm1, xm2, xm0  ; 01 12
    punpckhwd           xm2, xm0       ; 23 34
.hv_w2_loop:
    movq                xm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    movhps              xm4, [srcq+ssq*0]
    pshufb              xm4, xm5
    pmaddubsw           xm4, xm6
    pmaddwd             xm3, xm9, xm1  ; a0 b0
    mova                xm1, xm2
    pmaddwd             xm2, xm10      ; a1 b1
    phaddw              xm4, xm4
    paddd               xm3, xm2
    pmulhrsw            xm4, xm7
    palignr             xm2, xm4, xm0, 12
    mova                xm0, xm4
    punpcklwd           xm2, xm4       ; 45 56
    pmaddwd             xm4, xm11, xm2 ; a2 b2
    paddd               xm3, xm8
    paddd               xm3, xm4
    psrad               xm3, 10
    packssdw            xm3, xm3
    packuswb            xm3, xm3
    pextrw     [dstq+dsq*0], xm3, 0
    pextrw     [dstq+dsq*1], xm3, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w2_loop
    RET
.hv_w4:
    mova                 m5, [subpel_h_shuf4]
    vpbroadcastq         m2, [srcq+nsq*2]
    vpbroadcastq         m4, [srcq+nsq*1]
    vpbroadcastq         m1, [srcq+ssq*0]
    vpbroadcastq         m3, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpbroadcastq         m0, [srcq+ssq*0]
    vpblendd             m2, m4, 0xcc ; 0 1
    vpblendd             m1, m3, 0xcc ; 2 3
    pshufb               m2, m5
    pshufb               m1, m5
    pshufb               m0, m5
    pmaddubsw            m2, m6
    pmaddubsw            m1, m6
    pmaddubsw            m0, m6
    phaddw               m2, m1
    phaddw               m0, m0
    pmulhrsw             m2, m7
    pmulhrsw             m0, m7
    palignr              m3, m0, m2, 4
    punpcklwd            m1, m2, m3   ; 01 12
    punpckhwd            m2, m3       ; 23 34
.hv_w4_loop:
    vpbroadcastq         m4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pmaddwd              m3, m9, m1   ; a0 b0
    mova                 m1, m2
    pmaddwd              m2, m10      ; a1 b1
    paddd                m3, m2
    vpbroadcastq         m2, [srcq+ssq*0]
    vpblendd             m4, m2, 0xcc ; 5 6
    pshufb               m4, m5
    pmaddubsw            m4, m6
    phaddw               m4, m4
    pmulhrsw             m4, m7
    palignr              m2, m4, m0, 12
    mova                 m0, m4
    punpcklwd            m2, m4       ; 45 56
    pmaddwd              m4, m11, m2  ; a2 b2
    paddd                m3, m8
    paddd                m3, m4
    psrad                m3, 10
    vextracti128        xm4, m3, 1
    packssdw            xm3, xm4
    packuswb            xm3, xm3
    pshuflw             xm3, xm3, q3120
    movd       [dstq+dsq*0], xm3
    pextrd     [dstq+dsq*1], xm3, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w4_loop
    RET
.hv_w8:
    shr                 mxd, 16
    sub                srcq, 2
    lea                 mxq, [r8+mxq*8+subpel_filters+1-put_avx2]
    WIN64_PUSH_XMM       16
    vpbroadcastw        m10, [mxq+0]
    vpbroadcastw        m11, [mxq+2]
    vpbroadcastw        m12, [mxq+4]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    vpbroadcastq         m0, [r8+myq*8+subpel_filters+1-put_avx2]
    lea                 r6d, [wq*8-64]
    vbroadcasti128       m8, [z_filter_s+ 6]
    punpcklbw            m0, m0
    vbroadcasti128       m9, [z_filter_s+10]
    psraw                m0, 8 ; sign-extend
    mov                 nsq, ssq
    pshufd              m13, m0, q0000
    neg                 nsq
    pshufd              m14, m0, q1111
    lea                 r6d, [hq+r6*4]
    pshufd              m15, m0, q2222
.hv_w8_loop0:
    vbroadcasti128       m7, [z_filter_s+2]
    movu                xm3, [srcq+nsq*2]
    lea                  r4, [srcq+ssq*2]
    movu                xm4, [srcq+nsq*1]
    vbroadcasti128       m0, [srcq+ssq*0]
    mov                  r7, dstq
    vinserti128          m4, [srcq+ssq*1], 1 ; 1 3
    vpblendd             m3, m0, 0xf0        ; 0 2
    vinserti128          m0, [r4+ssq*0], 1   ; 2 4
    vpbroadcastd         m5, [pw_8192]
%macro HV_H_6TAP_W8 6 ; src/dst, tmp[1-2], shuf[1-3]
    pshufb               %2, %1, %4
    pmaddubsw            %2, m10
    pshufb               %3, %1, %5
    pmaddubsw            %3, m11
    pshufb               %1, %6
    pmaddubsw            %1, m12
    paddw                %2, %3
    paddw                %1, %2
%endmacro
    HV_H_6TAP_W8         m3, m1, m2, m7, m8, m9
    HV_H_6TAP_W8         m4, m1, m2, m7, m8, m9
    HV_H_6TAP_W8         m0, m1, m2, m7, m8, m9
    vpermq               m3, m3, q3120
    vpermq               m4, m4, q3120
    vpermq               m0, m0, q3120
    pmulhrsw             m3, m5
    pmulhrsw             m4, m5
    pmulhrsw             m0, m5
    punpcklwd            m1, m3, m4  ; 01
    punpckhwd            m3, m4      ; 23
    punpcklwd            m2, m4, m0  ; 12
    punpckhwd            m4, m0      ; 34
.hv_w8_loop:
    movu                xm7, [r4+ssq*1]
    lea                  r4, [r4+ssq*2]
    vinserti128          m7, [r4+ssq*0], 1 ; 5 6
    pmaddwd              m5, m13, m1 ; a0
    mova                 m1, m3
    pmaddwd              m6, m13, m2 ; b0
    mova                 m2, m4
    pmaddwd              m3, m14     ; a1
    pmaddwd              m4, m14     ; b1
    paddd                m5, m3
    vbroadcasti128       m3, [z_filter_s+2]
    paddd                m6, m4
    HV_H_6TAP_W8         m7, m3, m4, m3, m8, m9
    vpbroadcastd         m3, [pw_8192]
    vpbroadcastd         m4, [pd_512]
    pmulhrsw             m7, m3
    paddd                m5, m4
    paddd                m6, m4
    mova                 m4, m0
    vpermq               m0, m7, q3120
    shufpd               m4, m0, 0x05
    punpcklwd            m3, m4, m0  ; 45
    pmaddwd              m7, m15, m3 ; a2
    punpckhwd            m4, m0      ; 67
    paddd                m5, m7
    pmaddwd              m7, m15, m4 ; b2
    paddd                m6, m7
    psrad                m5, 10
    psrad                m6, 10
    packssdw             m5, m6
    vextracti128        xm6, m5, 1
    packuswb            xm5, xm6
    pshufd              xm5, xm5, q3120
    movq         [r7+dsq*0], xm5
    movhps       [r7+dsq*1], xm5
    lea                  r7, [r7+dsq*2]
    sub                  hd, 2
    jg .hv_w8_loop
    add                srcq, 8
    add                dstq, 8
    movzx                hd, r6b
    sub                 r6d, 1<<8
    jg .hv_w8_loop0
    RET

cglobal put_8tap_sharp_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
    imul                mxd, mxm, 0x010101
    imul                myd, mym, 0x010101
    mov                  wd, wm
    movifnidn            hd, hm
    add                 mxd, FILTER_SHARP ; 8tap_h, mx, 4tap_h
    add                 myd, FILTER_SHARP ; 8tap_v, my, 4tap_v
    lea                  r8, [put_avx2]
    test                mxd, 0xf00
    jnz .h
    test                myd, 0xf00
    jz mangle(private_prefix %+ _put_6tap_8bpc_avx2).put
.v:
    WIN64_SPILL_XMM      12, 15
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    tzcnt               r6d, wd
    movzx               r6d, word [r8+r6*2+table_offset(put, _8tap_sharp_v)]
    vpbroadcastd         m7, [pw_512]
    lea                 myq, [r8+myq*8+subpel_filters-put_avx2]
    vpbroadcastw         m8, [myq+0]
    vpbroadcastw         m9, [myq+2]
    vpbroadcastw        m10, [myq+4]
    vpbroadcastw        m11, [myq+6]
    add                  r6, r8
    lea                ss3q, [ssq*3]
    sub                srcq, ss3q
    jmp                  r6
.v_w2:
    movd                xm2, [srcq+ssq*0]
    pinsrw              xm2, [srcq+ssq*1], 2
    pinsrw              xm2, [srcq+ssq*2], 4
    add                srcq, ss3q
    pinsrw              xm2, [srcq+ssq*0], 6 ; 0 1 2 3
    movd                xm3, [srcq+ssq*1]
    vpbroadcastd        xm1, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastd        xm0, [srcq+ssq*0]
    vpblendd            xm3, xm1, 0x02       ; 4 5
    vpblendd            xm1, xm0, 0x02       ; 5 6
    palignr             xm4, xm3, xm2, 4     ; 1 2 3 4
    punpcklbw           xm3, xm1             ; 45 56
    punpcklbw           xm1, xm2, xm4        ; 01 12
    punpckhbw           xm2, xm4             ; 23 34
.v_w2_loop:
    pmaddubsw           xm5, xm1, xm8        ; a0 b0
    mova                xm1, xm2
    pmaddubsw           xm2, xm9             ; a1 b1
    paddw               xm5, xm2
    mova                xm2, xm3
    pmaddubsw           xm3, xm10            ; a2 b2
    paddw               xm5, xm3
    vpbroadcastd        xm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpblendd            xm3, xm0, xm4, 0x02  ; 6 7
    vpbroadcastd        xm0, [srcq+ssq*0]
    vpblendd            xm4, xm0, 0x02       ; 7 8
    punpcklbw           xm3, xm4             ; 67 78
    pmaddubsw           xm4, xm3, xm11       ; a3 b3
    paddw               xm5, xm4
    pmulhrsw            xm5, xm7
    packuswb            xm5, xm5
    pextrw     [dstq+dsq*0], xm5, 0
    pextrw     [dstq+dsq*1], xm5, 2
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w2_loop
    RET
.v_w4:
    movd                xm2, [srcq+ssq*0]
    pinsrd              xm2, [srcq+ssq*1], 1
    pinsrd              xm2, [srcq+ssq*2], 2
    add                srcq, ss3q
    pinsrd              xm2, [srcq+ssq*0], 3 ; 0 1 2 3
    movd                xm3, [srcq+ssq*1]
    vpbroadcastd        xm1, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastd        xm0, [srcq+ssq*0]
    vpblendd            xm3, xm1, 0x02       ; 4 5
    vpblendd            xm1, xm0, 0x02       ; 5 6
    palignr             xm4, xm3, xm2, 4     ; 1 2 3 4
    punpcklbw           xm3, xm1             ; 45 56
    punpcklbw           xm1, xm2, xm4        ; 01 12
    punpckhbw           xm2, xm4             ; 23 34
.v_w4_loop:
    pmaddubsw           xm5, xm1, xm8        ; a0 b0
    mova                xm1, xm2
    pmaddubsw           xm2, xm9             ; a1 b1
    paddw               xm5, xm2
    mova                xm2, xm3
    pmaddubsw           xm3, xm10            ; a2 b2
    paddw               xm5, xm3
    vpbroadcastd        xm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpblendd            xm3, xm0, xm4, 0x02  ; 6 7
    vpbroadcastd        xm0, [srcq+ssq*0]
    vpblendd            xm4, xm0, 0x02       ; 7 8
    punpcklbw           xm3, xm4             ; 67 78
    pmaddubsw           xm4, xm3, xm11       ; a3 b3
    paddw               xm5, xm4
    pmulhrsw            xm5, xm7
    packuswb            xm5, xm5
    movd       [dstq+dsq*0], xm5
    pextrd     [dstq+dsq*1], xm5, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w4_loop
    RET
.v_w8:
    movq                xm1, [srcq+ssq*0]
    vpbroadcastq         m4, [srcq+ssq*1]
    vpbroadcastq         m2, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastq         m5, [srcq+ssq*0]
    vpbroadcastq         m3, [srcq+ssq*1]
    vpbroadcastq         m6, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastq         m0, [srcq+ssq*0]
    vpblendd             m1, m4, 0x30
    vpblendd             m4, m2, 0x30
    punpcklbw            m1, m4      ; 01 12
    vpblendd             m2, m5, 0x30
    vpblendd             m5, m3, 0x30
    punpcklbw            m2, m5      ; 23 34
    vpblendd             m3, m6, 0x30
    vpblendd             m6, m0, 0x30
    punpcklbw            m3, m6      ; 45 56
.v_w8_loop:
    vpbroadcastq         m4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pmaddubsw            m5, m1, m8  ; a0 b0
    mova                 m1, m2
    pmaddubsw            m2, m9      ; a1 b1
    paddw                m5, m2
    mova                 m2, m3
    pmaddubsw            m3, m10     ; a2 b2
    paddw                m5, m3
    vpblendd             m3, m0, m4, 0x30
    vpbroadcastq         m0, [srcq+ssq*0]
    vpblendd             m4, m0, 0x30
    punpcklbw            m3, m4      ; 67 78
    pmaddubsw            m4, m3, m11 ; a3 b3
    paddw                m5, m4
    pmulhrsw             m5, m7
    vextracti128        xm4, m5, 1
    packuswb            xm5, xm4
    movq       [dstq+dsq*0], xm5
    movhps     [dstq+dsq*1], xm5
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w8_loop
    RET
.v_w16:
.v_w32:
.v_w64:
    lea                 r6d, [wq*8-128]
    WIN64_PUSH_XMM       15
    lea                 r6d, [hq+r6*2]
.v_w16_loop0:
    vbroadcasti128       m4, [srcq+ssq*0]
    vbroadcasti128       m5, [srcq+ssq*1]
    lea                  r4, [srcq+ss3q]
    vbroadcasti128       m6, [srcq+ssq*2]
    vbroadcasti128       m0, [r4+ssq*0]
    mov                  r7, dstq
    vbroadcasti128       m1, [r4+ssq*1]
    vbroadcasti128       m2, [r4+ssq*2]
    add                  r4, ss3q
    vbroadcasti128       m3, [r4+ssq*0]
    shufpd               m4, m0, 0x0c
    shufpd               m5, m1, 0x0c
    punpcklbw            m1, m4, m5 ; 01
    punpckhbw            m4, m5     ; 34
    shufpd               m6, m2, 0x0c
    punpcklbw            m2, m5, m6 ; 12
    punpckhbw            m5, m6     ; 45
    shufpd               m0, m3, 0x0c
    punpcklbw            m3, m6, m0 ; 23
    punpckhbw            m6, m0     ; 56
.v_w16_loop:
    vbroadcasti128      m12, [r4+ssq*1]
    lea                  r4, [r4+ssq*2]
    pmaddubsw           m13, m1, m8  ; a0
    pmaddubsw           m14, m2, m8  ; b0
    mova                 m1, m3
    mova                 m2, m4
    pmaddubsw            m3, m9      ; a1
    pmaddubsw            m4, m9      ; b1
    paddw               m13, m3
    paddw               m14, m4
    mova                 m3, m5
    mova                 m4, m6
    pmaddubsw            m5, m10     ; a2
    pmaddubsw            m6, m10     ; b2
    paddw               m13, m5
    vbroadcasti128       m5, [r4+ssq*0]
    paddw               m14, m6
    shufpd               m6, m0, m12, 0x0d
    shufpd               m0, m12, m5, 0x0c
    punpcklbw            m5, m6, m0  ; 67
    punpckhbw            m6, m0      ; 78
    pmaddubsw           m12, m5, m11 ; a3
    paddw               m13, m12
    pmaddubsw           m12, m6, m11 ; b3
    paddw               m14, m12
    pmulhrsw            m13, m7
    pmulhrsw            m14, m7
    packuswb            m13, m14
    vpermq              m13, m13, q3120
    movu         [r7+dsq*0], xm13
    vextracti128 [r7+dsq*1], m13, 1
    lea                  r7, [r7+dsq*2]
    sub                  hd, 2
    jg .v_w16_loop
    add                srcq, 16
    add                dstq, 16
    movzx                hd, r6b
    sub                 r6d, 1<<8
    jg .v_w16_loop0
    RET
.h:
.h_w2:
.h_w4:
    test                myd, 0xf00
    jnz .hv
    vpbroadcastd         m5, [pw_34] ; 2 + (8 << 2)
    cmp                  wd, 4
    jle mangle(private_prefix %+ _put_6tap_8bpc_avx2).h_w2
    WIN64_SPILL_XMM      11
    tzcnt                wd, wd
    vbroadcasti128       m6, [subpel_h_shufA]
    shr                 mxd, 16
    vbroadcasti128       m7, [subpel_h_shufB]
    sub                srcq, 3
    vbroadcasti128       m8, [subpel_h_shufC]
    movzx                wd, word [r8+wq*2+table_offset(put, _8tap_sharp_h)]
    vpbroadcastd         m9, [r8+mxq*8+subpel_filters-put_avx2+0]
    vpbroadcastd        m10, [r8+mxq*8+subpel_filters-put_avx2+4]
    add                  wq, r8
    jmp                  wq
.h_w8:
%macro PUT_8TAP_H 4 ; dst/src, tmp[1-3]
    pshufb              m%2, m%1, m7
    pshufb              m%3, m%1, m8
    pshufb              m%1, m6
    pmaddubsw           m%4, m%2, m9
    pmaddubsw           m%2, m10
    pmaddubsw           m%3, m10
    pmaddubsw           m%1, m9
    paddw               m%3, m%4
    paddw               m%1, m%2
    phaddw              m%1, m%3
    paddw               m%1, m5
    psraw               m%1, 6
%endmacro
    movu                xm0, [srcq+ssq*0]
    vinserti128          m0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    PUT_8TAP_H            0, 1, 2, 3
    vextracti128        xm1, m0, 1
    packuswb            xm0, xm1
    movq       [dstq+dsq*0], xm0
    movhps     [dstq+dsq*1], xm0
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w8
    RET
.h_w16:
    movu                xm0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*1+8*0], 1
    movu                xm1, [srcq+ssq*0+8*1]
    vinserti128          m1, [srcq+ssq*1+8*1], 1
    PUT_8TAP_H            0, 2, 3, 4
    lea                srcq, [srcq+ssq*2]
    PUT_8TAP_H            1, 2, 3, 4
    packuswb             m0, m1
    movu         [dstq+dsq*0], xm0
    vextracti128 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w16
    RET
.h_w32:
    xor                 r6d, r6d
    jmp .h_start
.h_w64:
    mov                  r6, -32*1
    sub                srcq, r6
    sub                dstq, r6
.h_start:
    mov                  r4, r6
.h_loop:
    movu                 m0, [srcq+r6+8*0]
    movu                 m1, [srcq+r6+8*1]
    PUT_8TAP_H            0, 2, 3, 4
    PUT_8TAP_H            1, 2, 3, 4
    packuswb             m0, m1
    movu          [dstq+r6], m0
    add                  r6, 32
    jle .h_loop
    add                srcq, ssq
    add                dstq, dsq
    mov                  r6, r4
    dec                  hd
    jg .h_loop
    RET
.hv:
    WIN64_SPILL_XMM      14, 16
    cmp                  wd, 4
    jg .hv_w8
    movzx               mxd, mxb
    dec                srcq
    vpbroadcastd         m7, [r8+mxq*8+subpel_filters-put_avx2+2]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    vpbroadcastq         m0, [r8+myq*8+subpel_filters-put_avx2]
    lea                ss3q, [ssq*3]
    sub                srcq, ss3q
    punpcklbw            m0, m0
    psraw                m0, 8 ; sign-extend
    vpbroadcastd         m8, [pw_8192]
    vpbroadcastd         m9, [pd_512]
    pshufd              m10, m0, q0000
    pshufd              m11, m0, q1111
    pshufd              m12, m0, q2222
    pshufd              m13, m0, q3333
    cmp                  wd, 4
    je .hv_w4
    vbroadcasti128       m6, [subpel_h_shuf4]
    movq                xm2, [srcq+ssq*0]
    movhps              xm2, [srcq+ssq*1]
    movq                xm0, [srcq+ssq*2]
    add                srcq, ss3q
    movhps              xm0, [srcq+ssq*0]
    vpbroadcastq         m3, [srcq+ssq*1]
    vpbroadcastq         m4, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastq         m1, [srcq+ssq*0]
    vpblendd             m2, m3, 0x30
    vpblendd             m0, m1, 0x30
    vpblendd             m2, m4, 0xc0
    pshufb               m2, m6
    pshufb               m0, m6
    pmaddubsw            m2, m7
    pmaddubsw            m0, m7
    phaddw               m2, m0
    pmulhrsw             m2, m8
    vextracti128        xm3, m2, 1
    palignr             xm4, xm3, xm2, 4
    punpcklwd           xm1, xm2, xm4  ; 01 12
    punpckhwd           xm2, xm4       ; 23 34
    pshufd              xm0, xm3, q2121
    punpcklwd           xm3, xm0       ; 45 56
.hv_w2_loop:
    movq                xm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    movhps              xm4, [srcq+ssq*0]
    pshufb              xm4, xm6
    pmaddubsw           xm4, xm7
    pmaddwd             xm5, xm1, xm10 ; a0 b0
    mova                xm1, xm2
    pmaddwd             xm2, xm11      ; a1 b1
    paddd               xm5, xm2
    mova                xm2, xm3
    pmaddwd             xm3, xm12      ; a2 b2
    phaddw              xm4, xm4
    pmulhrsw            xm4, xm8
    paddd               xm5, xm3
    palignr             xm3, xm4, xm0, 12
    mova                xm0, xm4
    punpcklwd           xm3, xm0       ; 67 78
    pmaddwd             xm4, xm3, xm13 ; a3 b3
    paddd               xm5, xm9
    paddd               xm5, xm4
    psrad               xm5, 10
    packssdw            xm5, xm5
    packuswb            xm5, xm5
    pextrw     [dstq+dsq*0], xm5, 0
    pextrw     [dstq+dsq*1], xm5, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w2_loop
    RET
.hv_w4:
    mova                 m6, [subpel_h_shuf4]
    vpbroadcastq         m2, [srcq+ssq*0]
    vpbroadcastq         m4, [srcq+ssq*1]
    vpbroadcastq         m0, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastq         m5, [srcq+ssq*0]
    vpbroadcastq         m3, [srcq+ssq*1]
    vpblendd             m2, m4, 0xcc ; 0 1
    vpbroadcastq         m4, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastq         m1, [srcq+ssq*0]
    vpblendd             m0, m5, 0xcc ; 2 3
    vpblendd             m3, m4, 0xcc ; 4 5
    pshufb               m2, m6
    pshufb               m0, m6
    pshufb               m3, m6
    pshufb               m1, m6
    pmaddubsw            m2, m7
    pmaddubsw            m0, m7
    pmaddubsw            m3, m7
    pmaddubsw            m1, m7
    phaddw               m2, m0
    phaddw               m3, m1
    pmulhrsw             m2, m8
    pmulhrsw             m3, m8
    palignr              m4, m3, m2, 4
    punpcklwd            m1, m2, m4   ; 01 12
    punpckhwd            m2, m4       ; 23 34
    pshufd               m0, m3, q2121
    punpcklwd            m3, m0       ; 45 56
.hv_w4_loop:
    vpbroadcastq         m4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pmaddwd              m5, m1, m10  ; a0 b0
    mova                 m1, m2
    pmaddwd              m2, m11      ; a1 b1
    paddd                m5, m2
    mova                 m2, m3
    pmaddwd              m3, m12      ; a2 b2
    paddd                m5, m3
    vpbroadcastq         m3, [srcq+ssq*0]
    vpblendd             m4, m3, 0xcc ; 7 8
    pshufb               m4, m6
    pmaddubsw            m4, m7
    phaddw               m4, m4
    pmulhrsw             m4, m8
    palignr              m3, m4, m0, 12
    mova                 m0, m4
    punpcklwd            m3, m0       ; 67 78
    pmaddwd              m4, m3, m13  ; a3 b3
    paddd                m5, m9
    paddd                m5, m4
    psrad                m5, 10
    vextracti128        xm4, m5, 1
    packssdw            xm5, xm4
    packuswb            xm5, xm5
    pshuflw             xm5, xm5, q3120
    movd       [dstq+dsq*0], xm5
    pextrd     [dstq+dsq*1], xm5, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w4_loop
    RET
.hv_w8:
    WIN64_PUSH_XMM       16
    shr                 mxd, 16
    sub                srcq, 3
    vpbroadcastd        m10, [r8+mxq*8+subpel_filters-put_avx2+0]
    vpbroadcastd        m11, [r8+mxq*8+subpel_filters-put_avx2+4]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    vpbroadcastq         m0, [r8+myq*8+subpel_filters-put_avx2]
    lea                ss3q, [ssq*3]
    sub                srcq, ss3q
    punpcklbw            m0, m0
    psraw                m0, 8 ; sign-extend
    pshufd              m12, m0, q0000
    pshufd              m13, m0, q1111
    pshufd              m14, m0, q2222
    pshufd              m15, m0, q3333
    lea                 r6d, [wq*8-64]
    lea                 r6d, [hq+r6*4]
.hv_w8_loop0:
    vbroadcasti128       m7, [subpel_h_shufA]
    movu                xm4, [srcq+ssq*0]
    lea                  r4, [srcq+ss3q]
    vbroadcasti128       m8, [subpel_h_shufB]
    movu                xm5, [srcq+ssq*1]
    mov                  r7, dstq
    vbroadcasti128       m9, [subpel_h_shufC]
    movu                xm6, [srcq+ssq*2]
    vbroadcasti128       m0, [r4+ssq*0]
    vpblendd             m4, m0, 0xf0      ; 0 3
    vinserti128          m5, [r4+ssq*1], 1 ; 1 4
    vinserti128          m6, [r4+ssq*2], 1 ; 2 5
    add                  r4, ss3q
    vinserti128          m0, [r4+ssq*0], 1 ; 3 6
%macro HV_H_8TAP_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3]
    pshufb               %3, %1, %6
    pshufb               %4, %1, %7
    pshufb               %1, %5
    pmaddubsw            %2, %3, m10
    pmaddubsw            %4, m11
    pmaddubsw            %3, m11
    pmaddubsw            %1, m10
    paddw                %2, %4
    paddw                %1, %3
    phaddw               %1, %2
%endmacro
    HV_H_8TAP_W8         m4, m1, m2, m3, m7, m8, m9
    HV_H_8TAP_W8         m5, m1, m2, m3, m7, m8, m9
    HV_H_8TAP_W8         m6, m1, m2, m3, m7, m8, m9
    HV_H_8TAP_W8         m0, m1, m2, m3, m7, m8, m9
    vpbroadcastd         m7, [pw_8192]
    vpermq               m4, m4, q3120
    vpermq               m5, m5, q3120
    vpermq               m6, m6, q3120
    pmulhrsw             m0, m7
    pmulhrsw             m4, m7
    pmulhrsw             m5, m7
    pmulhrsw             m6, m7
    vpermq               m7, m0, q3120
    punpcklwd            m1, m4, m5  ; 01
    punpckhwd            m4, m5      ; 34
    punpcklwd            m2, m5, m6  ; 12
    punpckhwd            m5, m6      ; 45
    punpcklwd            m3, m6, m7  ; 23
    punpckhwd            m6, m7      ; 56
.hv_w8_loop:
    vextracti128        r6m, m0, 1 ; not enough registers
    movu                xm0, [r4+ssq*1]
    lea                  r4, [r4+ssq*2]
    vinserti128          m0, [r4+ssq*0], 1 ; 7 8
    pmaddwd              m8, m1, m12 ; a0
    pmaddwd              m9, m2, m12 ; b0
    mova                 m1, m3
    mova                 m2, m4
    pmaddwd              m3, m13     ; a1
    pmaddwd              m4, m13     ; b1
    paddd                m8, m3
    paddd                m9, m4
    mova                 m3, m5
    mova                 m4, m6
    pmaddwd              m5, m14     ; a2
    pmaddwd              m6, m14     ; b2
    paddd                m8, m5
    paddd                m9, m6
    vbroadcasti128       m6, [subpel_h_shufB]
    vbroadcasti128       m7, [subpel_h_shufC]
    vbroadcasti128       m5, [subpel_h_shufA]
    HV_H_8TAP_W8         m0, m5, m6, m7, m5, m6, m7
    vpbroadcastd         m5, [pw_8192]
    vpbroadcastd         m7, [pd_512]
    vbroadcasti128       m6, r6m
    pmulhrsw             m0, m5
    paddd                m8, m7
    paddd                m9, m7
    vpermq               m7, m0, q3120 ; 7 8
    shufpd               m6, m7, 0x04  ; 6 7
    punpcklwd            m5, m6, m7  ; 67
    punpckhwd            m6, m7      ; 78
    pmaddwd              m7, m5, m15 ; a3
    paddd                m8, m7
    pmaddwd              m7, m6, m15 ; b3
    paddd                m7, m9
    psrad                m8, 10
    psrad                m7, 10
    packssdw             m8, m7
    vextracti128        xm7, m8, 1
    packuswb            xm8, xm7
    pshufd              xm7, xm8, q3120
    movq         [r7+dsq*0], xm7
    movhps       [r7+dsq*1], xm7
    lea                  r7, [r7+dsq*2]
    sub                  hd, 2
    jg .hv_w8_loop
    add                srcq, 8
    add                dstq, 8
    movzx                hd, r6b
    sub                 r6d, 1<<8
    jg .hv_w8_loop0
    RET

%define PREP_8TAP_FN FN prep_8tap,
PREP_8TAP_FN smooth,  SMOOTH,  prep_6tap_8bpc
PREP_8TAP_FN regular, REGULAR

cglobal prep_6tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ns
    imul                mxd, mxm, 0x010101
    imul                myd, mym, 0x010101
    mov                  wd, wm
    add                 mxd, t0d ; 6tap_h, mx, 4tap_h
    add                 myd, t0d ; 6tap_v, my, 4tap_v
    movifnidn            hd, hm
    lea                  r8, [prep_avx2]
    test                mxd, 0xf00
    jnz .h
    test                myd, 0xf00
    jnz .v
.prep:
    tzcnt                wd, wd
    movzx                wd, word [r8+wq*2+table_offset(prep,)]
    lea                  r6, [ssq*3]
    add                  wq, r8
    lea                  r7, [dsq*3]
%if WIN64
    pop                  r8
%endif
    jmp                  wq
.v:
    WIN64_SPILL_XMM      10, 12
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 4
    cmove               myd, mxd
    lea                 myq, [r8+myq*8+subpel_filters+1-prep_avx2]
    vpbroadcastd         m9, [pw_8192]
    lea                  r6, [dsq*3]
    vpbroadcastw         m6, [myq+0]
    mov                 nsq, ssq
    vpbroadcastw         m7, [myq+2]
    neg                 nsq
    vpbroadcastw         m8, [myq+4]
    cmp                  wd, 8
    jg .v_w16
    je .v_w8
.v_w4:
    movd                xm2, [srcq+nsq*2]
    pinsrd              xm2, [srcq+nsq*1], 1
    vpbroadcastd         m1, [srcq+ssq*0]
    vpbroadcastd         m3, [srcq+ssq*1]
    vpbroadcastd         m0, [srcq+ssq*2]
    vbroadcasti128       m5, [deint_shuf4]
    vpblendd             m1, m2, 0xeb
    punpcklqdq           m3, m0
    vpblendd             m1, m3, 0x60 ; 0 1 2 _   2 3 4 _
    pshufb               m1, m5       ; 01  12    23  34
.v_w4_loop:
    lea                srcq, [srcq+ssq*4]
    pinsrd              xm0, [srcq+nsq*1], 1
    vpbroadcastd         m2, [srcq+ssq*0]
    vpbroadcastd         m3, [srcq+ssq*1]
    vpblendd             m2, m0, 0xeb
    vpbroadcastd         m0, [srcq+ssq*2]
    punpcklqdq           m3, m0
    vpblendd             m2, m3, 0x60 ; 4 5 6 _   6 7 8 _
    pshufb               m2, m5       ; 45  56    67  78
    pmaddubsw            m3, m1, m6   ; a0  b0    c0  d0
    vperm2i128           m1, m2, 0x21 ; 23  34    45  56
    pmaddubsw            m4, m2, m8   ; a2  b2    c2  d2
    pmaddubsw            m1, m7       ; a1  b1    c1  d1
    paddw                m3, m4
    paddw                m3, m1
    pmulhrsw             m3, m9
    mova                 m1, m2
    vextracti128        xm2, m3, 1
    movq       [dstq+dsq*0], xm3
    movhps     [dstq+dsq*2], xm3
    movq       [dstq+dsq*4], xm2
    movhps     [dstq+r6 *2], xm2
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .v_w4_loop
    RET
.v_w8:
    movq                xm1, [srcq+nsq*2]
    vpbroadcastq         m3, [srcq+nsq*1]
    vpbroadcastq         m2, [srcq+ssq*0]
    vpbroadcastq         m4, [srcq+ssq*1]
    vpbroadcastq         m0, [srcq+ssq*2]
    vpblendd             m1, m3, 0x30
    vpblendd             m3, m2, 0x30
    punpcklbw            m1, m3     ; 01 12
    vpblendd             m2, m4, 0x30
    vpblendd             m4, m0, 0x30
    punpcklbw            m2, m4     ; 23 34
.v_w8_loop:
    lea                srcq, [srcq+ssq*4]
    pmaddubsw            m1, m6     ; a0
    vpbroadcastq         m3, [srcq+nsq*1]
    pmaddubsw            m4, m2, m7 ; a1
    pmaddubsw            m5, m2, m6 ; b0
    vpbroadcastq         m2, [srcq+ssq*0]
    vpblendd             m0, m3, 0x30
    vpblendd             m3, m2, 0x30
    paddw                m4, m1
    punpcklbw            m1, m0, m3 ; 45 56
    vpbroadcastq         m3, [srcq+ssq*1]
    vpbroadcastq         m0, [srcq+ssq*2]
    vpblendd             m2, m3, 0x30
    vpblendd             m3, m0, 0x30
    punpcklbw            m2, m3     ; 67 78
    pmaddubsw            m3, m1, m7 ; b1
    paddw                m5, m3
    pmaddubsw            m3, m1, m8 ; a2
    paddw                m4, m3
    pmaddubsw            m3, m2, m8 ; b2
    paddw                m5, m3
    pmulhrsw             m4, m9
    pmulhrsw             m5, m9
    mova         [dstq+dsq*0], xm4
    vextracti128 [dstq+dsq*2], m4, 1
    mova         [dstq+dsq*4], xm5
    vextracti128 [dstq+r6 *2], m5, 1
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .v_w8_loop
    RET
.v_w16:
    lea                 r6d, [wq*2-32]
    lea                srcq, [srcq+nsq*2]
    WIN64_PUSH_XMM       12
    lea                 r6d, [hq+r6*8]
.v_w16_loop0:
    vbroadcasti128       m3, [srcq+ssq*0]
    lea                  r4, [srcq+ssq*2]
    vbroadcasti128       m4, [srcq+ssq*1]
    mov                  r7, dstq
    vbroadcasti128       m0, [r4+ssq*0]
    vbroadcasti128       m1, [r4+ssq*1]
    lea                  r4, [r4+ssq*2]
    vbroadcasti128       m2, [r4+ssq*0]
    shufpd               m3, m0, 0x0c
    shufpd               m4, m1, 0x0c
    punpcklbw            m1, m3, m4 ; 01
    punpckhbw            m3, m4     ; 23
    shufpd               m0, m2, 0x0c
    punpcklbw            m2, m4, m0 ; 12
    punpckhbw            m4, m0     ; 34
.v_w16_loop:
    vbroadcasti128       m5, [r4+ssq*1]
    pmaddubsw           m10, m1, m6 ; a0
    lea                  r4, [r4+ssq*2]
    pmaddubsw           m11, m2, m6 ; b0
    mova                 m1, m3
    pmaddubsw            m3, m7     ; a1
    mova                 m2, m4
    pmaddubsw            m4, m7     ; b1
    paddw               m10, m3
    vbroadcasti128       m3, [r4+ssq*0]
    paddw               m11, m4
    shufpd               m4, m0, m5, 0x0d
    shufpd               m0, m5, m3, 0x0c
    punpcklbw            m3, m4, m0 ; 45
    punpckhbw            m4, m0     ; 56
    pmaddubsw            m5, m3, m8 ; a2
    paddw               m10, m5
    pmaddubsw            m5, m4, m8 ; b2
    paddw               m11, m5
    pmulhrsw            m10, m9
    pmulhrsw            m11, m9
    mova         [r7+dsq*0], m10
    mova         [r7+dsq*2], m11
    lea                  r7, [r7+dsq*4]
    sub                  hd, 2
    jg .v_w16_loop
    add                srcq, 16
    add                dstq, 32
    movzx                hd, r6b
    sub                 r6d, 1<<8
    jg .v_w16_loop0
    RET
.h_w4:
    RESET_STACK_STATE
    movzx               mxd, mxb
    vbroadcasti128       m3, [subpel_h_shufA]
    dec                srcq
    vpbroadcastd         m5, [r8+mxq*8+subpel_filters-prep_avx2+2]
    lea                  r7, [ssq*3]
    lea                  r6, [dsq*3]
.h_w4_loop:
    movq                xm0, [srcq+ssq*0]
    vpbroadcastq         m2, [srcq+ssq*1]
    movq                xm1, [srcq+ssq*2]
    vpblendd             m0, m2, 0x30
    vpbroadcastq         m2, [srcq+r7   ]
    lea                srcq, [srcq+ssq*4]
    vpblendd             m1, m2, 0x30
    pshufb               m0, m3
    pshufb               m1, m3
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    phaddw               m0, m1
    pmulhrsw             m0, m4
    vextracti128        xm1, m0, 1
    movq       [dstq+dsq*0], xm0
    movq       [dstq+dsq*2], xm1
    movhps     [dstq+dsq*4], xm0
    movhps     [dstq+r6 *2], xm1
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .h_w4_loop
    RET
.h:
    test                myd, 0xf00
    jnz .hv
    vpbroadcastd         m4, [pw_8192]
    cmp                  wd, 4
    je .h_w4
    WIN64_SPILL_XMM      10
    tzcnt                wd, wd
    vbroadcasti128       m3, [z_filter_s+ 2]
    shr                 mxd, 16
    vbroadcasti128       m5, [z_filter_s+ 6]
    sub                srcq, 2
    vbroadcasti128       m6, [z_filter_s+10]
    lea                 mxq, [r8+mxq*8+subpel_filters+1-prep_avx2]
    movzx                wd, word [r8+wq*2+table_offset(prep, _6tap_h)]
    vpbroadcastw         m7, [mxq+0]
    vpbroadcastw         m8, [mxq+2]
    add                  wq, r8
    vpbroadcastw         m9, [mxq+4]
    jmp                  wq
.h_w8:
    movu                xm0, [srcq+ssq*0]
    vinserti128          m0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
%macro PREP_6TAP_H 0
    pshufb               m1, m0, m3
    pmaddubsw            m1, m7
    pshufb               m2, m0, m5
    pmaddubsw            m2, m8
    pshufb               m0, m6
    pmaddubsw            m0, m9
    paddw                m1, m2
    paddw                m1, m0
    pmulhrsw             m1, m4
%endmacro
    PREP_6TAP_H
    mova         [dstq+dsq*0], xm1
    vextracti128 [dstq+dsq*2], m1, 1
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 2
    jg .h_w8
    RET
.h_w16:
    movu                xm0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*0+8*1], 1
    PREP_6TAP_H
    movu                xm0, [srcq+ssq*1+8*0]
    vinserti128          m0, [srcq+ssq*1+8*1], 1
    lea                srcq, [srcq+ssq*2]
    mova       [dstq+dsq*0], m1
    PREP_6TAP_H
    mova       [dstq+dsq*2], m1
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 2
    jg .h_w16
    RET
.h_w32:
    xor                 r6d, r6d
    jmp .h_start
.h_w64:
    mov                  r6, -32*1
    add                dstq, 32*2
    sub                srcq, r6
.h_start:
    mov                  r4, r6
.h_loop:
    movu                xm0, [srcq+r6+8*0]
    vinserti128          m0, [srcq+r6+8*1], 1
    PREP_6TAP_H
    movu                xm0, [srcq+r6+8*2]
    vinserti128          m0, [srcq+r6+8*3], 1
    mova   [dstq+r6*2+32*0], m1
    PREP_6TAP_H
    mova   [dstq+r6*2+32*1], m1
    add                  r6, 32
    jle .h_loop
    add                srcq, ssq
    mov                  r6, r4
    lea                dstq, [dstq+dsq*2]
    dec                  hd
    jg .h_loop
    RET
.hv:
    WIN64_SPILL_XMM      14, 16
    cmp                  wd, 4
    jne .hv_w8
.hv_w4:
    movzx               mxd, mxb
    dec                srcq
    vpbroadcastd         m7, [r8+mxq*8+subpel_filters-prep_avx2+2]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 4
    cmove               myd, mxd
    mova                 m6, [subpel_h_shuf4]
    vpbroadcastq         m0, [r8+myq*8+subpel_filters+1-prep_avx2]
    mov                 nsq, ssq
    pmovzxbd            m13, [deint_shuf4]
    neg                 nsq
    vpbroadcastd         m8, [pw_8192]
    vpbroadcastd         m9, [pd_32]
    punpcklbw            m0, m0
    vpbroadcastq         m2, [srcq+nsq*2]
    psraw                m0, 8 ; sign-extend
    vpbroadcastq         m4, [srcq+nsq*1]
    pshufd              m10, m0, q0000
    vpbroadcastq         m1, [srcq+ssq*0]
    pshufd              m11, m0, q1111
    vpbroadcastq         m3, [srcq+ssq*1]
    pshufd              m12, m0, q2222
    vpbroadcastq         m0, [srcq+ssq*2]
    vpblendd             m2, m4, 0xcc ; 0 1
    vpblendd             m1, m3, 0xcc ; 2 3
    pshufb               m2, m6
    pshufb               m1, m6
    pshufb               m0, m6
    pmaddubsw            m2, m7
    pmaddubsw            m1, m7
    pmaddubsw            m0, m7
    phaddw               m2, m1       ; 0 1 2 3
    phaddw               m0, m0       ; 4
    pmulhrsw             m2, m8
    pmulhrsw             m0, m8
    lea                  r6, [dsq*3]
    palignr              m0, m2, 4
    punpcklwd            m1, m2, m0   ; 01 12
    punpckhwd            m2, m0       ; 23 34
.hv_w4_loop:
    pmaddwd              m4, m10, m1  ; a0 b0
    lea                srcq, [srcq+ssq*4]
    pmaddwd              m5, m2, m10  ; c0 d0
    vpbroadcastq         m1, [srcq+nsq*1]
    pmaddwd              m2, m11      ; a1 b1
    vpbroadcastq         m3, [srcq+ssq*0]
    paddd                m4, m2
    vpbroadcastq         m2, [srcq+ssq*1]
    vpblendd             m1, m3, 0xcc ; 5 6
    vpbroadcastq         m3, [srcq+ssq*2]
    vpblendd             m2, m3, 0xcc ; 7 8
    pshufb               m1, m6
    pshufb               m2, m6
    pmaddubsw            m1, m7
    pmaddubsw            m2, m7
    phaddw               m1, m2       ; 5 6 7 8
    pmulhrsw             m1, m8
    paddd                m5, m9
    paddd                m4, m9
    palignr              m2, m1, m0, 12
    mova                 m0, m1
    punpcklwd            m1, m2, m0   ; 45 56
    punpckhwd            m2, m0       ; 67 78
    pmaddwd              m3, m11, m1  ; c1 d1
    paddd                m5, m3
    pmaddwd              m3, m12, m1  ; a2 b2
    paddd                m4, m3
    pmaddwd              m3, m12, m2  ; c2 d2
    paddd                m5, m3
    psrad                m4, 6
    psrad                m5, 6
    packssdw             m4, m5
    vpermd               m4, m13, m4
    vextracti128        xm5, m4, 1
    movq       [dstq+dsq*0], xm4
    movhps     [dstq+dsq*2], xm4
    movq       [dstq+dsq*4], xm5
    movhps     [dstq+r6 *2], xm5
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .hv_w4_loop
    RET
.hv_w8:
    shr                 mxd, 16
    lea                 mxq, [r8+mxq*8+subpel_filters+1-prep_avx2]
    WIN64_PUSH_XMM       16
    vpbroadcastw        m10, [mxq+0]
    vpbroadcastw        m11, [mxq+2]
    vpbroadcastw        m12, [mxq+4]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    vpbroadcastq         m0, [r8+myq*8+subpel_filters+1-prep_avx2]
    lea                  r7, [ssq*2+2]
    vbroadcasti128       m8, [z_filter_s+ 6]
    punpcklbw            m0, m0
    vbroadcasti128       m9, [z_filter_s+10]
    psraw                m0, 8 ; sign-extend
    lea                 r6d, [wq*8-64]
    pshufd              m13, m0, q0000
    sub                srcq, r7
    pshufd              m14, m0, q1111
    lea                 r6d, [hq+r6*4]
    pshufd              m15, m0, q2222
.hv_w8_loop0:
    vbroadcasti128       m7, [z_filter_s+2]
    movu                xm3, [srcq+ssq*0]
    lea                  r4, [srcq+ssq*2]
    movu                xm4, [srcq+ssq*1]
    vbroadcasti128       m0, [r4+ssq*0]
    mov                  r7, dstq
    vinserti128          m4, [r4+ssq*1], 1 ; 1 3
    lea                  r4, [r4+ssq*2]
    vpblendd             m3, m0, 0xf0      ; 0 2
    vinserti128          m0, [r4+ssq*0], 1 ; 2 4
    vpbroadcastd         m5, [pw_8192]
    HV_H_6TAP_W8         m3, m1, m2, m7, m8, m9
    HV_H_6TAP_W8         m4, m1, m2, m7, m8, m9
    HV_H_6TAP_W8         m0, m1, m2, m7, m8, m9
    vpermq               m3, m3, q3120
    vpermq               m4, m4, q3120
    vpermq               m0, m0, q3120
    pmulhrsw             m3, m5
    pmulhrsw             m4, m5
    pmulhrsw             m0, m5
    punpcklwd            m1, m3, m4  ; 01
    punpckhwd            m3, m4      ; 23
    punpcklwd            m2, m4, m0  ; 12
    punpckhwd            m4, m0      ; 34
.hv_w8_loop:
    movu                xm7, [r4+ssq*1]
    lea                  r4, [r4+ssq*2]
    vinserti128          m7, [r4+ssq*0], 1 ; 5 6
    pmaddwd              m5, m13, m1 ; a0
    mova                 m1, m3
    pmaddwd              m6, m13, m2 ; b0
    mova                 m2, m4
    pmaddwd              m3, m14     ; a1
    pmaddwd              m4, m14     ; b1
    paddd                m5, m3
    vbroadcasti128       m3, [z_filter_s+2]
    paddd                m6, m4
    HV_H_6TAP_W8         m7, m3, m4, m3, m8, m9
    vpbroadcastd         m3, [pw_8192]
    vpbroadcastd         m4, [pd_32]
    pmulhrsw             m7, m3
    paddd                m5, m4
    paddd                m6, m4
    mova                 m4, m0
    vpermq               m0, m7, q3120
    shufpd               m4, m0, 0x05
    punpcklwd            m3, m4, m0  ; 45
    pmaddwd              m7, m15, m3 ; a2
    punpckhwd            m4, m0      ; 67
    paddd                m5, m7
    pmaddwd              m7, m15, m4 ; b2
    paddd                m6, m7
    psrad                m5, 6
    psrad                m6, 6
    packssdw             m5, m6
    vpermq               m5, m5, q3120
    mova         [r7+dsq*0], xm5
    vextracti128 [r7+dsq*2], m5, 1
    lea                  r7, [r7+dsq*4]
    sub                  hd, 2
    jg .hv_w8_loop
    add                srcq, 8
    add                dstq, 16
    movzx                hd, r6b
    sub                 r6d, 1<<8
    jg .hv_w8_loop0
    RET

cglobal prep_8tap_sharp_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my
    imul                mxd, mxm, 0x010101
    imul                myd, mym, 0x010101
    mov                  wd, wm
    movifnidn            hd, hm
    add                 mxd, FILTER_SHARP ; 8tap_h, mx, 4tap_h
    add                 myd, FILTER_SHARP ; 8tap_v, my, 4tap_v
    lea                  r8, [prep_avx2]
    test                mxd, 0xf00
    jnz .h
    test                myd, 0xf00
    jz mangle(private_prefix %+ _prep_6tap_8bpc_avx2).prep
.v:
    WIN64_SPILL_XMM      12, 15
    movzx               mxd, myb ; Select 4-tap/8-tap filter multipliers.
    shr                 myd, 16  ; Note that the code is 8-tap only, having
    cmp                  hd, 4   ; a separate 4-tap code path for (4|8|16)x4
    cmove               myd, mxd ; had a negligible effect on performance.
    lea                 myq, [r8+myq*8+subpel_filters-prep_avx2]
    lea                  r8, [ssq*3]
    sub                srcq, r8
    vpbroadcastd         m7, [pw_8192]
    vpbroadcastw         m8, [myq+0]
    vpbroadcastw         m9, [myq+2]
    vpbroadcastw        m10, [myq+4]
    vpbroadcastw        m11, [myq+6]
    cmp                  wd, 8
    jg .v_w16
    lea                  r6, [dsq*3]
    je .v_w8
.v_w4:
    movd                xm0, [srcq+ssq*0]
    vpbroadcastd         m1, [srcq+ssq*2]
    vpbroadcastd        xm2, [srcq+ssq*1]
    add                srcq, r8
    vpbroadcastd         m3, [srcq+ssq*0]
    vpblendd             m1, m0, 0x01     ; 0 2 2 _   2 _ _ _
    vpblendd             m3, m2, 0x03     ; 1 1 3 3   3 3 _ _
    vpbroadcastd         m0, [srcq+ssq*1]
    vpbroadcastd         m2, [srcq+ssq*2]
    vpblendd             m1, m0, 0x68     ; 0 2 2 4   2 4 4 _
    vpbroadcastd         m0, [srcq+r8   ]
    vbroadcasti128       m5, [deint_shuf4]
    vpblendd             m3, m2, 0xc0     ; 1 1 3 3   3 3 5 5
    vpblendd             m2, m3, m1, 0x55 ; 0 1 2 3   2 3 4 5
    vpblendd             m3, m1, 0xaa     ; 1 2 3 4   3 4 5 _
    punpcklbw            m1, m2, m3       ; 01  12    23  34
    vpblendd             m3, m0, 0x80     ; 1 2 3 4   3 4 5 6
    punpckhbw            m2, m3           ; 23  34    45  56
.v_w4_loop:
    lea                srcq, [srcq+ssq*4]
    pinsrd              xm0, [srcq+ssq*0], 1
    vpbroadcastd         m3, [srcq+ssq*1]
    vpbroadcastd         m4, [srcq+ssq*2]
    vpblendd             m3, m0, 0x03     ; 6 7 8 _   8 _ _ _
    vpbroadcastd         m0, [srcq+r8   ]
    vpblendd             m3, m4, 0x20     ; 6 7 8 _   8 9 _ _
    vpblendd             m3, m0, 0x40     ; 6 7 8 _   8 9 a _
    pshufb               m3, m5           ; 67  78    89  9a
    pmaddubsw            m4, m1, m8
    vperm2i128           m1, m2, m3, 0x21 ; 45  56    67  78
    pmaddubsw            m2, m9
    paddw                m4, m2
    mova                 m2, m3
    pmaddubsw            m3, m11
    paddw                m3, m4
    pmaddubsw            m4, m1, m10
    paddw                m3, m4
    pmulhrsw             m3, m7
    vextracti128        xm4, m3, 1
    movq       [dstq+dsq*0], xm3
    movhps     [dstq+dsq*2], xm3
    movq       [dstq+dsq*4], xm4
    movhps     [dstq+r6 *2], xm4
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .v_w4_loop
    RET
.v_w8:
    movq                xm1, [srcq+ssq*0]
    vpbroadcastq         m4, [srcq+ssq*1]
    vpbroadcastq         m2, [srcq+ssq*2]
    vpbroadcastq         m5, [srcq+r8   ]
    lea                srcq, [srcq+ssq*4]
    vpbroadcastq         m3, [srcq+ssq*0]
    vpbroadcastq         m6, [srcq+ssq*1]
    vpbroadcastq         m0, [srcq+ssq*2]
    vpblendd             m1, m4, 0x30
    vpblendd             m4, m2, 0x30
    punpcklbw            m1, m4 ; 01 12
    vpblendd             m2, m5, 0x30
    vpblendd             m5, m3, 0x30
    punpcklbw            m2, m5 ; 23 34
    vpblendd             m3, m6, 0x30
    vpblendd             m6, m0, 0x30
    punpcklbw            m3, m6 ; 45 56
.v_w8_loop:
    vpbroadcastq         m4, [srcq+r8   ]
    lea                srcq, [srcq+ssq*4]
    pmaddubsw            m5, m2, m9  ; a1
    pmaddubsw            m6, m2, m8  ; b0
    vpblendd             m2, m0, m4, 0x30
    vpbroadcastq         m0, [srcq+ssq*0]
    vpblendd             m4, m0, 0x30
    punpcklbw            m2, m4      ; 67 78
    pmaddubsw            m1, m8      ; a0
    pmaddubsw            m4, m3, m9  ; b1
    paddw                m5, m1
    mova                 m1, m3
    pmaddubsw            m3, m10     ; a2
    paddw                m6, m4
    paddw                m5, m3
    vpbroadcastq         m4, [srcq+ssq*1]
    vpblendd             m3, m0, m4, 0x30
    vpbroadcastq         m0, [srcq+ssq*2]
    vpblendd             m4, m0, 0x30
    punpcklbw            m3, m4      ; 89 9a
    pmaddubsw            m4, m2, m11 ; a3
    paddw                m5, m4
    pmaddubsw            m4, m2, m10 ; b2
    paddw                m6, m4
    pmaddubsw            m4, m3, m11 ; b3
    paddw                m6, m4
    pmulhrsw             m5, m7
    pmulhrsw             m6, m7
    mova         [dstq+dsq*0], xm5
    vextracti128 [dstq+dsq*2], m5, 1
    mova         [dstq+dsq*4], xm6
    vextracti128 [dstq+r6 *2], m6, 1
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .v_w8_loop
    RET
.v_w16:
    lea                 r6d, [wq*2-32]
    WIN64_PUSH_XMM       15
    lea                 r6d, [hq+r6*8]
.v_w16_loop0:
    vbroadcasti128       m4, [srcq+ssq*0]
    vbroadcasti128       m5, [srcq+ssq*1]
    lea                  r4, [srcq+ssq*2]
    vbroadcasti128       m0, [r4+ssq*1]
    vbroadcasti128       m6, [r4+ssq*0]
    lea                  r4, [r4+ssq*2]
    vbroadcasti128       m1, [r4+ssq*0]
    vbroadcasti128       m2, [r4+ssq*1]
    lea                  r4, [r4+ssq*2]
    vbroadcasti128       m3, [r4+ssq*0]
    mov                  r7, dstq
    shufpd               m4, m0, 0x0c
    shufpd               m5, m1, 0x0c
    punpcklbw            m1, m4, m5 ; 01
    punpckhbw            m4, m5     ; 34
    shufpd               m6, m2, 0x0c
    punpcklbw            m2, m5, m6 ; 12
    punpckhbw            m5, m6     ; 45
    shufpd               m0, m3, 0x0c
    punpcklbw            m3, m6, m0 ; 23
    punpckhbw            m6, m0     ; 56
.v_w16_loop:
    vbroadcasti128      m12, [r4+ssq*1]
    lea                  r4, [r4+ssq*2]
    pmaddubsw           m13, m1, m8  ; a0
    pmaddubsw           m14, m2, m8  ; b0
    mova                 m1, m3
    mova                 m2, m4
    pmaddubsw            m3, m9      ; a1
    pmaddubsw            m4, m9      ; b1
    paddw               m13, m3
    paddw               m14, m4
    mova                 m3, m5
    mova                 m4, m6
    pmaddubsw            m5, m10     ; a2
    pmaddubsw            m6, m10     ; b2
    paddw               m13, m5
    vbroadcasti128       m5, [r4+ssq*0]
    paddw               m14, m6
    shufpd               m6, m0, m12, 0x0d
    shufpd               m0, m12, m5, 0x0c
    punpcklbw            m5, m6, m0  ; 67
    punpckhbw            m6, m0      ; 78
    pmaddubsw           m12, m5, m11 ; a3
    paddw               m13, m12
    pmaddubsw           m12, m6, m11 ; b3
    paddw               m14, m12
    pmulhrsw            m13, m7
    pmulhrsw            m14, m7
    mova         [r7+dsq*0], m13
    mova         [r7+dsq*2], m14
    lea                  r7, [r7+dsq*4]
    sub                  hd, 2
    jg .v_w16_loop
    add                srcq, 16
    add                dstq, 32
    movzx                hd, r6b
    sub                 r6d, 1<<8
    jg .v_w16_loop0
    RET
.h:
.h_w4:
    test                myd, 0xf00
    jnz .hv
    vpbroadcastd         m4, [pw_8192]
    cmp                  wd, 4
    je mangle(private_prefix %+ _prep_6tap_8bpc_avx2).h_w4
    WIN64_SPILL_XMM      10
    vbroadcasti128       m5, [subpel_h_shufA]
    tzcnt                wd, wd
    vbroadcasti128       m6, [subpel_h_shufB]
    vbroadcasti128       m7, [subpel_h_shufC]
    shr                 mxd, 16
    sub                srcq, 3
    movzx                wd, word [r8+wq*2+table_offset(prep, _8tap_sharp_h)]
    vpbroadcastd         m8, [r8+mxq*8+subpel_filters-prep_avx2+0]
    vpbroadcastd         m9, [r8+mxq*8+subpel_filters-prep_avx2+4]
    add                  wq, r8
    jmp                  wq
.h_w8:
    movu                xm0, [srcq+ssq*0]
    vinserti128          m0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
%macro PREP_8TAP_H 0
    pshufb               m1, m0, m5
    pshufb               m2, m0, m6
    pshufb               m3, m0, m7
    pmaddubsw            m1, m8
    pmaddubsw            m0, m2, m8
    pmaddubsw            m2, m9
    pmaddubsw            m3, m9
    paddw                m1, m2
    paddw                m0, m3
    phaddw               m1, m0
    pmulhrsw             m1, m4
%endmacro
    PREP_8TAP_H
    mova         [dstq+dsq*0], xm1
    vextracti128 [dstq+dsq*2], m1, 1
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 2
    jg .h_w8
    RET
.h_w16:
    movu                xm0, [srcq+ssq*0+8*0]
    vinserti128          m0, [srcq+ssq*0+8*1], 1
    PREP_8TAP_H
    movu                xm0, [srcq+ssq*1+8*0]
    vinserti128          m0, [srcq+ssq*1+8*1], 1
    lea                srcq, [srcq+ssq*2]
    mova       [dstq+dsq*0], m1
    PREP_8TAP_H
    mova       [dstq+dsq*2], m1
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 2
    jg .h_w16
    RET
.h_w32:
    xor                 r6d, r6d
    jmp .h_start
.h_w64:
    mov                  r6, -32*1
    add                dstq, 32*2
    sub                srcq, r6
.h_start:
    mov                  r4, r6
.h_loop:
    movu                xm0, [srcq+r6+8*0]
    vinserti128          m0, [srcq+r6+8*1], 1
    PREP_8TAP_H
    movu                xm0, [srcq+r6+8*2]
    vinserti128          m0, [srcq+r6+8*3], 1
    mova   [dstq+r6*2+32*0], m1
    PREP_8TAP_H
    mova   [dstq+r6*2+32*1], m1
    add                  r6, 32
    jle .h_loop
    add                srcq, ssq
    lea                dstq, [dstq+dsq*2]
    mov                  r6, r4
    dec                  hd
    jg .h_loop
    RET
.hv:
    WIN64_SPILL_XMM      16
    cmp                  wd, 4
    je .hv_w4
    shr                 mxd, 16
    sub                srcq, 3
    vpbroadcastd        m10, [r8+mxq*8+subpel_filters-prep_avx2+0]
    vpbroadcastd        m11, [r8+mxq*8+subpel_filters-prep_avx2+4]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 4
    cmove               myd, mxd
    vpbroadcastq         m0, [r8+myq*8+subpel_filters-prep_avx2]
    lea                  r8, [ssq*3]
    sub                srcq, r8
    punpcklbw            m0, m0
    lea                 r6d, [wq*8-64]
    psraw                m0, 8 ; sign-extend
    lea                 r6d, [hq+r6*4]
    pshufd              m12, m0, q0000
    pshufd              m13, m0, q1111
    pshufd              m14, m0, q2222
    pshufd              m15, m0, q3333
.hv_w8_loop0:
    vbroadcasti128       m7, [subpel_h_shufA]
    movu                xm4, [srcq+ssq*0]
    lea                  r4, [srcq+ssq*2]
    vbroadcasti128       m8, [subpel_h_shufB]
    movu                xm5, [srcq+ssq*1]
    mov                  r7, dstq
    vbroadcasti128       m9, [subpel_h_shufC]
    movu                xm6, [r4+ssq*0]
    vbroadcasti128       m0, [r4+ssq*1]
    lea                  r4, [r4+ssq*2]
    vpblendd             m4, m0, 0xf0          ; 0 3
    vinserti128          m5, [r4+ssq*0], 1 ; 1 4
    vinserti128          m6, [r4+ssq*1], 1 ; 2 5
    lea                  r4, [r4+ssq*2]
    vinserti128          m0, [r4+ssq*0], 1 ; 3 6
    HV_H_8TAP_W8         m4, m1, m2, m3, m7, m8, m9
    HV_H_8TAP_W8         m5, m1, m2, m3, m7, m8, m9
    HV_H_8TAP_W8         m6, m1, m2, m3, m7, m8, m9
    HV_H_8TAP_W8         m0, m1, m2, m3, m7, m8, m9
    vpbroadcastd         m7, [pw_8192]
    vpermq               m4, m4, q3120
    vpermq               m5, m5, q3120
    vpermq               m6, m6, q3120
    pmulhrsw             m0, m7
    pmulhrsw             m4, m7
    pmulhrsw             m5, m7
    pmulhrsw             m6, m7
    vpermq               m7, m0, q3120
    punpcklwd            m1, m4, m5  ; 01
    punpckhwd            m4, m5      ; 34
    punpcklwd            m2, m5, m6  ; 12
    punpckhwd            m5, m6      ; 45
    punpcklwd            m3, m6, m7  ; 23
    punpckhwd            m6, m7      ; 56
.hv_w8_loop:
    vextracti128       [r7], m0, 1 ; not enough registers
    movu                xm0, [r4+ssq*1]
    lea                  r4, [r4+ssq*2]
    vinserti128          m0, [r4+ssq*0], 1 ; 7 8
    pmaddwd              m8, m1, m12 ; a0
    pmaddwd              m9, m2, m12 ; b0
    mova                 m1, m3
    mova                 m2, m4
    pmaddwd              m3, m13     ; a1
    pmaddwd              m4, m13     ; b1
    paddd                m8, m3
    paddd                m9, m4
    mova                 m3, m5
    mova                 m4, m6
    pmaddwd              m5, m14     ; a2
    pmaddwd              m6, m14     ; b2
    paddd                m8, m5
    paddd                m9, m6
    vbroadcasti128       m6, [subpel_h_shufB]
    vbroadcasti128       m7, [subpel_h_shufC]
    vbroadcasti128       m5, [subpel_h_shufA]
    HV_H_8TAP_W8         m0, m5, m6, m7, m5, m6, m7
    vpbroadcastd         m5, [pw_8192]
    vpbroadcastd         m7, [pd_32]
    vbroadcasti128       m6, [r7]
    pmulhrsw             m0, m5
    paddd                m8, m7
    paddd                m9, m7
    vpermq               m7, m0, q3120 ; 7 8
    shufpd               m6, m7, 0x04  ; 6 7
    punpcklwd            m5, m6, m7  ; 67
    punpckhwd            m6, m7      ; 78
    pmaddwd              m7, m5, m15 ; a3
    paddd                m8, m7
    pmaddwd              m7, m6, m15 ; b3
    paddd                m7, m9
    psrad                m8, 6
    psrad                m7, 6
    packssdw             m8, m7
    vpermq               m7, m8, q3120
    mova         [r7+dsq*0], xm7
    vextracti128 [r7+dsq*2], m7, 1
    lea                  r7, [r7+dsq*4]
    sub                  hd, 2
    jg .hv_w8_loop
    add                srcq, 8
    add                dstq, 16
    movzx                hd, r6b
    sub                 r6d, 1<<8
    jg .hv_w8_loop0
    RET
.hv_w4:
    movzx               mxd, mxb
    dec                srcq
    vpbroadcastd         m8, [r8+mxq*8+subpel_filters-prep_avx2+2]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 4
    cmove               myd, mxd
    vpbroadcastq         m0, [r8+myq*8+subpel_filters-prep_avx2]
    lea                  r8, [ssq*3]
    sub                srcq, r8
    mova                 m7, [subpel_h_shuf4]
    pmovzxbd             m9, [deint_shuf4]
    vpbroadcastd        m10, [pw_8192]
    punpcklbw            m0, m0
    psraw                m0, 8 ; sign-extend
    vpbroadcastd        m11, [pd_32]
    pshufd              m12, m0, q0000
    pshufd              m13, m0, q1111
    pshufd              m14, m0, q2222
    pshufd              m15, m0, q3333
    vpbroadcastq         m2, [srcq+ssq*0]
    vpbroadcastq         m4, [srcq+ssq*1]
    vpbroadcastq         m0, [srcq+ssq*2]
    vpbroadcastq         m5, [srcq+r8   ]
    lea                srcq, [srcq+ssq*4]
    vpbroadcastq         m3, [srcq+ssq*0]
    vpbroadcastq         m6, [srcq+ssq*1]
    vpbroadcastq         m1, [srcq+ssq*2]
    vpblendd             m2, m4, 0xcc ; 0 1
    vpblendd             m0, m5, 0xcc ; 2 3
    vpblendd             m3, m6, 0xcc ; 4 5
    pshufb               m2, m7 ; 00 01 10 11  02 03 12 13
    pshufb               m0, m7 ; 20 21 30 31  22 23 32 33
    pshufb               m3, m7 ; 40 41 50 51  42 43 52 53
    pshufb               m1, m7 ; 60 61 60 61  62 63 62 63
    pmaddubsw            m2, m8
    pmaddubsw            m0, m8
    pmaddubsw            m3, m8
    pmaddubsw            m1, m8
    phaddw               m2, m0 ; 0a 1a 2a 3a  0b 1b 2b 3b
    phaddw               m3, m1 ; 4a 5a 6a __  4b 5b 6b __
    pmulhrsw             m2, m10
    pmulhrsw             m3, m10
    lea                  r6, [dsq*3]
    palignr              m4, m3, m2, 4 ; 1a 2a 3a 4a  1b 2b 3b 4b
    punpcklwd            m1, m2, m4  ; 01 12
    punpckhwd            m2, m4      ; 23 34
    pshufd               m0, m3, q2121
    punpcklwd            m3, m0      ; 45 56
.hv_w4_loop:
    pmaddwd              m5, m1, m12 ; a0 b0
    pmaddwd              m6, m2, m12 ; c0 d0
    pmaddwd              m2, m13     ; a1 b1
    pmaddwd              m4, m3, m13 ; c1 d1
    mova                 m1, m3
    pmaddwd              m3, m14     ; a2 b2
    paddd                m5, m2
    vpbroadcastq         m2, [srcq+r8   ]
    lea                srcq, [srcq+ssq*4]
    paddd                m6, m4
    vpbroadcastq         m4, [srcq+ssq*0]
    paddd                m5, m3
    vpbroadcastq         m3, [srcq+ssq*1]
    vpblendd             m2, m4, 0xcc
    vpbroadcastq         m4, [srcq+ssq*2]
    vpblendd             m3, m4, 0xcc
    pshufb               m2, m7
    pshufb               m3, m7
    pmaddubsw            m2, m8
    pmaddubsw            m3, m8
    phaddw               m2, m3
    pmulhrsw             m2, m10
    palignr              m3, m2, m0, 12
    mova                 m0, m2
    punpcklwd            m2, m3, m0  ; 67 78
    punpckhwd            m3, m0      ; 89 9a
    pmaddwd              m4, m2, m14 ; c2 d2
    paddd                m6, m11
    paddd                m5, m11
    paddd                m6, m4
    pmaddwd              m4, m2, m15 ; a3 b3
    paddd                m5, m4
    pmaddwd              m4, m3, m15 ; c3 d3
    paddd                m6, m4
    psrad                m5, 6
    psrad                m6, 6
    packssdw             m5, m6
    vpermd               m5, m9, m5
    vextracti128        xm6, m5, 1
    movq       [dstq+dsq*0], xm5
    movhps     [dstq+dsq*2], xm5
    movq       [dstq+dsq*4], xm6
    movhps     [dstq+r6 *2], xm6
    lea                dstq, [dstq+dsq*8]
    sub                  hd, 4
    jg .hv_w4_loop
    RET

%macro movifprep 2
 %if isprep
    mov %1, %2
 %endif
%endmacro

%macro REMAP_REG 2
 %xdefine r%1  r%2
 %xdefine r%1q r%2q
 %xdefine r%1d r%2d
%endmacro

%macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0
 %if isprep
  %xdefine r14_save r14
  %assign %%i 14
  %rep 14
   %assign %%j %%i-1
   REMAP_REG %%i, %%j
   %assign %%i %%i-1
  %endrep
 %endif
%endmacro

%macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0
 %if isprep
  %assign %%i 1
  %rep 13
   %assign %%j %%i+1
   REMAP_REG %%i, %%j
   %assign %%i %%i+1
  %endrep
  %xdefine r14 r14_save
  %undef r14_save
 %endif
%endmacro

%macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged
    MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT
    RET
 %if %1
    MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
 %endif
%endmacro

%macro MC_8TAP_SCALED_H 8 ; dst, tmp[0-6]
    movq               xm%1, [srcq+ r4]
    movq               xm%2, [srcq+ r6]
    movhps             xm%1, [srcq+ r7]
    movhps             xm%2, [srcq+ r9]
    vinserti128         m%1, [srcq+r10], 1
    vinserti128         m%2, [srcq+r11], 1
    vpbroadcastq        m%5, [srcq+r13]
    vpbroadcastq        m%6, [srcq+ rX]
    add                srcq, ssq
    movq               xm%3, [srcq+ r4]
    movq               xm%4, [srcq+ r6]
    movhps             xm%3, [srcq+ r7]
    movhps             xm%4, [srcq+ r9]
    vinserti128         m%3, [srcq+r10], 1
    vinserti128         m%4, [srcq+r11], 1
    vpbroadcastq        m%7, [srcq+r13]
    vpbroadcastq        m%8, [srcq+ rX]
    add                srcq, ssq
    vpblendd            m%1, m%5, 0xc0
    vpblendd            m%2, m%6, 0xc0
    vpblendd            m%3, m%7, 0xc0
    vpblendd            m%4, m%8, 0xc0
    pmaddubsw           m%1, m15
    pmaddubsw           m%2, m10
    pmaddubsw           m%3, m15
    pmaddubsw           m%4, m10
    phaddw              m%1, m%2
    phaddw              m%3, m%4
    phaddw              m%1, m%3
    pmulhrsw            m%1, m12
%endmacro

%macro MC_8TAP_SCALED 1
%ifidn %1, put
 %assign isprep 0
cglobal put_8tap_scaled_8bpc, 4, 14, 16, 128, dst, ds, src, ss, w, h, mx, my, dx, dy
 %xdefine base_reg r12
 %define rndshift 10
%else
 %assign isprep 1
cglobal prep_8tap_scaled_8bpc, 4, 14, 16, 128, tmp, src, ss, w, h, mx, my, dx, dy
 %define tmp_stridem qword [rsp+120]
 %xdefine base_reg r11
 %define rndshift 6
%endif
    lea            base_reg, [%1_8tap_scaled_8bpc_avx2]
%define base base_reg-%1_8tap_scaled_8bpc_avx2
    tzcnt                wd, wm
    vpbroadcastd         m8, dxm
%if isprep && UNIX64
    movd               xm14, mxd
    vpbroadcastd        m14, xm14
    mov                 r5d, t0d
 DECLARE_REG_TMP 5, 7
%else
    vpbroadcastd        m14, mxm
%endif
    mov                 dyd, dym
%ifidn %1, put
 %if WIN64
    mov                 r8d, hm
  DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3
  %define hm r5m
  %define dxm r8m
 %else
  DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3
  %define hm r6m
 %endif
 %define dsm [rsp+112]
 %define rX r1
 %define rXd r1d
%else ; prep
 %if WIN64
    mov                 r7d, hm
  DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3
  %define hm r4m
  %define dxm r7m
 %else
  DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3
  %define hm [rsp+112]
 %endif
 MCT_8TAP_SCALED_REMAP_REGS_TO_PREV
 %define rX r14
 %define rXd r14d
%endif
    vpbroadcastd        m10, [base+pd_0x3ff]
    vpbroadcastd        m12, [base+pw_8192]
%ifidn %1, put
    vpbroadcastd        m13, [base+pd_512]
%else
    vpbroadcastd        m13, [base+pd_32]
%endif
    pxor                 m9, m9
    lea                ss3q, [ssq*3]
    movzx               r7d, t1b
    shr                 t1d, 16
    cmp                  hd, 6
    cmovs               t1d, r7d
    sub                srcq, ss3q
    cmp                 dyd, 1024
    je .dy1
    cmp                 dyd, 2048
    je .dy2
    movzx                wd, word [base+%1_8tap_scaled_avx2_table+wq*2]
    add                  wq, base_reg
    jmp                  wq
%ifidn %1, put
.w2:
    mov                 myd, mym
    movzx               t0d, t0b
    dec                srcq
    movd               xm15, t0d
    punpckldq            m8, m9, m8
    paddd               m14, m8 ; mx+dx*[0,1]
    vpbroadcastd        m11, [base+pd_0x4000]
    vpbroadcastd       xm15, xm15
    pand                 m8, m14, m10
    psrld                m8, 6
    paddd              xm15, xm8
    movd                r4d, xm15
    pextrd              r6d, xm15, 1
    vbroadcasti128       m5, [base+bdct_lb_dw]
    vbroadcasti128       m6, [base+subpel_s_shuf2]
    vpbroadcastd        m15, [base+subpel_filters+r4*8+2]
    vpbroadcastd         m7, [base+subpel_filters+r6*8+2]
    pcmpeqd              m8, m9
    psrld               m14, 10
    movq                xm0, [srcq+ssq*0]
    movq                xm1, [srcq+ssq*2]
    movhps              xm0, [srcq+ssq*1]
    movhps              xm1, [srcq+ss3q ]
    lea                srcq, [srcq+ssq*4]
    pshufb              m14, m5
    paddb               m14, m6
    vinserti128          m0, [srcq+ssq*0], 1
    vinserti128          m1, [srcq+ssq*2], 1
    vpbroadcastq         m2, [srcq+ssq*1]
    vpbroadcastq         m3, [srcq+ss3q ]
    lea                srcq, [srcq+ssq*4]
    vpblendd            m15, m7, 0xaa
    vpblendd             m0, m2, 0xc0       ; 0 1  4 5
    vpblendd             m1, m3, 0xc0       ; 2 3  6 7
    pblendvb            m15, m11, m8
    pshufb               m0, m14
    pshufb               m1, m14
    pmaddubsw            m0, m15
    pmaddubsw            m1, m15
    phaddw               m0, m1
    pmulhrsw             m0, m12            ; 0 1 2 3  4 5 6 7
    vextracti128        xm1, m0, 1          ; 4 5 6 7
    palignr             xm2, xm1, xm0, 4    ; 1 2 3 4
    punpcklwd           xm3, xm0, xm2       ; 01 12
    punpckhwd           xm0, xm2            ; 23 34
    pshufd              xm4, xm1, q0321     ; 5 6 7 _
    punpcklwd           xm2, xm1, xm4       ; 45 56
    punpckhwd           xm4, xm1, xm4       ; 67 __
.w2_loop:
    and                 myd, 0x3ff
    mov                 r6d, 64 << 24
    mov                 r4d, myd
    shr                 r4d, 6
    lea                 r4d, [t1+r4]
    cmovnz              r6q, [base+subpel_filters+r4*8]
    movq               xm11, r6q
    pmovsxbw           xm11, xm11
    pshufd              xm8, xm11, q0000
    pshufd              xm9, xm11, q1111
    pshufd             xm10, xm11, q2222
    pshufd             xm11, xm11, q3333
    pmaddwd             xm5, xm3, xm8
    pmaddwd             xm6, xm0, xm9
    pmaddwd             xm7, xm2, xm10
    pmaddwd             xm8, xm4, xm11
    paddd               xm5, xm6
    paddd               xm7, xm8
    paddd               xm5, xm13
    paddd               xm5, xm7
    psrad               xm5, 10
    packssdw            xm5, xm5
    packuswb            xm5, xm5
    pextrw           [dstq], xm5, 0
    add                dstq, dsq
    dec                  hd
    jz .ret
    add                 myd, dyd
    test                myd, ~0x3ff
    jz .w2_loop
    movq                xm5, [srcq]
    test                myd, 0x400
    jz .w2_skip_line
    add                srcq, ssq
    shufps              xm3, xm0, q1032     ; 01 12
    shufps              xm0, xm2, q1032     ; 23 34
    shufps              xm2, xm4, q1032     ; 45 56
    pshufb              xm5, xm14
    pmaddubsw           xm5, xm15
    phaddw              xm5, xm5
    pmulhrsw            xm5, xm12
    palignr             xm1, xm5, xm1, 12
    punpcklqdq          xm1, xm1            ; 6 7 6 7
    punpcklwd           xm4, xm1, xm5       ; 67 __
    jmp .w2_loop
.w2_skip_line:
    movhps              xm5, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mova                xm3, xm0            ; 01 12
    mova                xm0, xm2            ; 23 34
    pshufb              xm5, xm14
    pmaddubsw           xm5, xm15
    phaddw              xm5, xm5
    pmulhrsw            xm5, xm12           ; 6 7 6 7
    palignr             xm1, xm5, xm1, 8    ; 4 5 6 7
    pshufd              xm5, xm1, q0321     ; 5 6 7 _
    punpcklwd           xm2, xm1, xm5       ; 45 56
    punpckhwd           xm4, xm1, xm5       ; 67 __
    jmp .w2_loop
%endif
.w4:
    mov                 myd, mym
    vbroadcasti128       m7, [base+rescale_mul]
    movzx               t0d, t0b
    dec                srcq
    movd               xm15, t0d
    pmaddwd              m8, m7
    vpbroadcastd        m11, [base+pd_0x4000]
    vpbroadcastd       xm15, xm15
    paddd               m14, m8 ; mx+dx*[0-3]
    pand                 m0, m14, m10
    psrld                m0, 6
    paddd              xm15, xm0
    movd                r4d, xm15
    pextrd              r6d, xm15, 1
    pextrd             r11d, xm15, 2
    pextrd             r13d, xm15, 3
    movd               xm15, [base+subpel_filters+r4*8+2]
    vbroadcasti128       m5, [base+bdct_lb_dw]
    vpbroadcastq         m6, [base+subpel_s_shuf2]
    pinsrd             xm15, [base+subpel_filters+r6*8+2], 1
    pcmpeqd              m0, m9
    psrld               m14, 10
    movu                xm7, [srcq+ssq*0]
    movu                xm9, [srcq+ssq*1]
    pinsrd             xm15, [base+subpel_filters+r11*8+2], 2
    movu                xm8, [srcq+ssq*2]
    movu               xm10, [srcq+ss3q ]
    pinsrd             xm15, [base+subpel_filters+r13*8+2], 3
    lea                srcq, [srcq+ssq*4]
    pshufb              m14, m5
    paddb               m14, m6
    vinserti128          m7, [srcq+ssq*0], 1
    vinserti128          m9, [srcq+ssq*1], 1
    vinserti128         m15, xm15, 1
    vinserti128          m8, [srcq+ssq*2], 1
    vinserti128         m10, [srcq+ss3q ], 1
    lea                srcq, [srcq+ssq*4]
    pblendvb            m15, m11, m0
    pshufb               m7, m14
    pshufb               m9, m14
    pshufb               m8, m14
    pshufb              m10, m14
    pmaddubsw            m7, m15
    pmaddubsw            m9, m15
    pmaddubsw            m8, m15
    pmaddubsw           m10, m15
    phaddw               m7, m9
    phaddw               m8, m10
    pmulhrsw             m7, m12                ; 0 1  4 5
    pmulhrsw             m8, m12                ; 2 3  6 7
    vextracti128        xm9, m7, 1              ; 4 5
    vextracti128        xm3, m8, 1              ; 6 7
    shufps              xm4, xm7, xm8, q1032    ; 1 2
    shufps              xm5, xm8, xm9, q1032    ; 3 4
    shufps              xm6, xm9, xm3, q1032    ; 5 6
    psrldq             xm11, xm3, 8             ; 7 _
    punpcklwd           xm0, xm7, xm4   ; 01
    punpckhwd           xm7, xm4        ; 12
    punpcklwd           xm1, xm8, xm5   ; 23
    punpckhwd           xm8, xm5        ; 34
    punpcklwd           xm2, xm9, xm6   ; 45
    punpckhwd           xm9, xm6        ; 56
    punpcklwd           xm3, xm11       ; 67
    mova         [rsp+0x00], xm7
    mova         [rsp+0x10], xm8
    mova         [rsp+0x20], xm9
.w4_loop:
    and                 myd, 0x3ff
    mov                 r6d, 64 << 24
    mov                 r4d, myd
    shr                 r4d, 6
    lea                 r4d, [t1+r4]
    cmovnz              r6q, [base+subpel_filters+r4*8]
    movq               xm10, r6q
    pmovsxbw           xm10, xm10
    pshufd              xm7, xm10, q0000
    pshufd              xm8, xm10, q1111
    pshufd              xm9, xm10, q2222
    pshufd             xm10, xm10, q3333
    pmaddwd             xm4, xm0, xm7
    pmaddwd             xm5, xm1, xm8
    pmaddwd             xm6, xm2, xm9
    pmaddwd             xm7, xm3, xm10
    paddd               xm4, xm5
    paddd               xm6, xm7
    paddd               xm4, xm13
    paddd               xm4, xm6
    psrad               xm4, rndshift
    packssdw            xm4, xm4
%ifidn %1, put
    packuswb            xm4, xm4
    movd             [dstq], xm4
    add                dstq, dsq
%else
    movq             [tmpq], xm4
    add                tmpq, 8
%endif
    dec                  hd
    jz .ret
    add                 myd, dyd
    test                myd, ~0x3ff
    jz .w4_loop
    movu                xm4, [srcq]
    test                myd, 0x400
    jz .w4_skip_line
    mova                xm0, [rsp+0x00]
    mova         [rsp+0x00], xm1
    mova                xm1, [rsp+0x10]
    mova         [rsp+0x10], xm2
    mova                xm2, [rsp+0x20]
    mova         [rsp+0x20], xm3
    pshufb              xm4, xm14
    pmaddubsw           xm4, xm15
    phaddw              xm4, xm4
    pmulhrsw            xm4, xm12
    punpcklwd           xm3, xm11, xm4
    mova               xm11, xm4
    add                srcq, ssq
    jmp .w4_loop
.w4_skip_line:
    movu                xm5, [srcq+ssq*1]
    movu                 m6, [rsp+0x10]
    pshufb              xm4, xm14
    pshufb              xm5, xm14
    pmaddubsw           xm4, xm15
    pmaddubsw           xm5, xm15
    movu         [rsp+0x00], m6
    phaddw              xm4, xm5
    pmulhrsw            xm4, xm12
    punpcklwd           xm9, xm11, xm4
    mova         [rsp+0x20], xm9
    psrldq             xm11, xm4, 8
    mova                xm0, xm1
    mova                xm1, xm2
    mova                xm2, xm3
    punpcklwd           xm3, xm4, xm11
    lea                srcq, [srcq+ssq*2]
    jmp .w4_loop
.w8:
    mov      dword [rsp+48], 1
    movifprep   tmp_stridem, 16
    jmp .w_start
.w16:
    mov      dword [rsp+48], 2
    movifprep   tmp_stridem, 32
    jmp .w_start
.w32:
    mov      dword [rsp+48], 4
    movifprep   tmp_stridem, 64
    jmp .w_start
.w64:
    mov      dword [rsp+48], 8
    movifprep   tmp_stridem, 128
    jmp .w_start
.w128:
    mov      dword [rsp+48], 16
    movifprep   tmp_stridem, 256
.w_start:
%ifidn %1, put
    movifnidn           dsm, dsq
%endif
    shr                 t0d, 16
    sub                srcq, 3
    pmaddwd              m8, [base+rescale_mul]
    movd               xm15, t0d
    mov            [rsp+72], t0d
    mov            [rsp+56], srcq
    mov            [rsp+64], r0q ; dstq / tmpq
%if UNIX64
    mov                  hm, hd
%endif
    shl           dword dxm, 3 ; dx*8
    vpbroadcastd        m15, xm15
    paddd               m14, m8 ; mx+dx*[0-7]
    jmp .hloop
.hloop_prep:
    dec      dword [rsp+48]
    jz .ret
    add      qword [rsp+64], 8*(isprep+1)
    mov                  hd, hm
    vpbroadcastd         m8, dxm
    vpbroadcastd        m10, [base+pd_0x3ff]
    paddd               m14, m8, [rsp+16]
    vpbroadcastd        m15, [rsp+72]
    pxor                 m9, m9
    mov                srcq, [rsp+56]
    mov                 r0q, [rsp+64] ; dstq / tmpq
.hloop:
    vpbroadcastq        m11, [base+pq_0x40000000]
    pand                 m6, m14, m10
    psrld                m6, 6
    paddd               m15, m6
    pcmpeqd              m6, m9
    vextracti128        xm7, m15, 1
    movd                r4d, xm15
    pextrd              r6d, xm15, 2
    pextrd              r7d, xm15, 1
    pextrd              r9d, xm15, 3
    movd               r10d, xm7
    pextrd             r11d, xm7, 2
    pextrd             r13d, xm7, 1
    pextrd              rXd, xm7, 3
    movu           [rsp+16], m14
    movq               xm15, [base+subpel_filters+ r4*8]
    movq               xm10, [base+subpel_filters+ r6*8]
    movhps             xm15, [base+subpel_filters+ r7*8]
    movhps             xm10, [base+subpel_filters+ r9*8]
    vinserti128         m15, [base+subpel_filters+r10*8], 1
    vinserti128         m10, [base+subpel_filters+r11*8], 1
    vpbroadcastq         m9, [base+subpel_filters+r13*8]
    vpbroadcastq         m8, [base+subpel_filters+ rX*8]
    psrld               m14, 10
    vextracti128        xm7, m14, 1
    mova              [rsp], xm14
    movd                r4d, xm14
    pextrd              r6d, xm14, 2
    pextrd              r7d, xm14, 1
    pextrd              r9d, xm14, 3
    movd               r10d, xm7
    pextrd             r11d, xm7, 2
    pextrd             r13d, xm7, 1
    pextrd              rXd, xm7, 3
    pshufd               m5, m6, q1100
    pshufd               m6, m6, q3322
    vpblendd            m15, m9, 0xc0
    vpblendd            m10, m8, 0xc0
    pblendvb            m15, m11, m5
    pblendvb            m10, m11, m6
    vbroadcasti128      m14, [base+subpel_s_shuf8]
    MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7  ; 0a 1a 0b 1b
    MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8  ; 2a 3a 2b 3b
    MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9  ; 4a 5a 4b 5b
    MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b
    mov                 myd, mym
    mov                 dyd, dym
    pshufb               m0, m14    ; 01a 01b
    pshufb               m1, m14    ; 23a 23b
    pshufb               m2, m14    ; 45a 45b
    pshufb               m3, m14    ; 67a 67b
    vbroadcasti128      m14, [base+wswap]
.vloop:
    and                 myd, 0x3ff
    mov                 r6d, 64 << 24
    mov                 r4d, myd
    shr                 r4d, 6
    lea                 r4d, [t1+r4]
    cmovnz              r6q, [base+subpel_filters+r4*8]
    movq               xm11, r6q
    punpcklqdq         xm11, xm11
    pmovsxbw            m11, xm11
    pshufd               m8, m11, q0000
    pshufd               m9, m11, q1111
    pmaddwd              m4, m0, m8
    pmaddwd              m5, m1, m9
    pshufd               m8, m11, q2222
    pshufd              m11, m11, q3333
    pmaddwd              m6, m2, m8
    pmaddwd              m7, m3, m11
    paddd                m4, m5
    paddd                m6, m7
    paddd                m4, m13
    paddd                m4, m6
    psrad                m4, rndshift
    vextracti128        xm5, m4, 1
    packssdw            xm4, xm5
%ifidn %1, put
    packuswb            xm4, xm4
    movq             [dstq], xm4
    add                dstq, dsm
%else
    mova             [tmpq], xm4
    add                tmpq, tmp_stridem
%endif
    dec                  hd
    jz .hloop_prep
    add                 myd, dyd
    test                myd, ~0x3ff
    jz .vloop
    test                myd, 0x400
    mov            [rsp+52], myd
    mov                 r4d, [rsp+ 0]
    mov                 r6d, [rsp+ 8]
    mov                 r7d, [rsp+ 4]
    mov                 r9d, [rsp+12]
    jz .skip_line
    vpbroadcastq         m6, [srcq+r13]
    vpbroadcastq         m7, [srcq+ rX]
    movq                xm4, [srcq+ r4]
    movq                xm5, [srcq+ r6]
    movhps              xm4, [srcq+ r7]
    movhps              xm5, [srcq+ r9]
    vinserti128          m4, [srcq+r10], 1
    vinserti128          m5, [srcq+r11], 1
    add                srcq, ssq
    mov                 myd, [rsp+52]
    mov                 dyd, dym
    pshufb               m0, m14
    pshufb               m1, m14
    pshufb               m2, m14
    pshufb               m3, m14
    vpblendd             m4, m6, 0xc0
    vpblendd             m5, m7, 0xc0
    pmaddubsw            m4, m15
    pmaddubsw            m5, m10
    phaddw               m4, m5
    pslld                m5, m4, 16
    paddw                m4, m5
    pmulhrsw             m4, m12
    pblendw              m0, m1, 0xaa
    pblendw              m1, m2, 0xaa
    pblendw              m2, m3, 0xaa
    pblendw              m3, m4, 0xaa
    jmp .vloop
.skip_line:
    mova                 m0, m1
    mova                 m1, m2
    mova                 m2, m3
    vpbroadcastq         m7, [srcq+r13]
    vpbroadcastq         m8, [srcq+ rX]
    movq                xm3, [srcq+ r4]
    movq                xm4, [srcq+ r6]
    movhps              xm3, [srcq+ r7]
    movhps              xm4, [srcq+ r9]
    vinserti128          m3, [srcq+r10], 1
    vinserti128          m4, [srcq+r11], 1
    add                srcq, ssq
    movq                xm5, [srcq+ r4]
    movq                xm6, [srcq+ r6]
    movhps              xm5, [srcq+ r7]
    movhps              xm6, [srcq+ r9]
    vinserti128          m5, [srcq+r10], 1
    vinserti128          m6, [srcq+r11], 1
    vpbroadcastq         m9, [srcq+r13]
    vpbroadcastq        m11, [srcq+ rX]
    add                srcq, ssq
    mov                 myd, [rsp+52]
    mov                 dyd, dym
    vpblendd             m3, m7, 0xc0
    vpblendd             m4, m8, 0xc0
    vpblendd             m5, m9, 0xc0
    vpblendd             m6, m11, 0xc0
    pmaddubsw            m3, m15
    pmaddubsw            m4, m10
    pmaddubsw            m5, m15
    pmaddubsw            m6, m10
    phaddw               m3, m4
    phaddw               m5, m6
    psrld                m4, m3, 16
    pslld                m6, m5, 16
    paddw                m3, m4
    paddw                m5, m6
    pblendw              m3, m5, 0xaa
    pmulhrsw             m3, m12
    jmp .vloop
.dy1:
    movzx                wd, word [base+%1_8tap_scaled_avx2_dy1_table+wq*2]
    add                  wq, base_reg
    jmp                  wq
%ifidn %1, put
.dy1_w2:
    mov                 myd, mym
    movzx               t0d, t0b
    dec                srcq
    movd               xm15, t0d
    punpckldq            m8, m9, m8
    paddd               m14, m8 ; mx+dx*[0-1]
    vpbroadcastd        m11, [base+pd_0x4000]
    vpbroadcastd       xm15, xm15
    pand                 m8, m14, m10
    psrld                m8, 6
    paddd              xm15, xm8
    movd                r4d, xm15
    pextrd              r6d, xm15, 1
    vbroadcasti128       m5, [base+bdct_lb_dw]
    vbroadcasti128       m6, [base+subpel_s_shuf2]
    vpbroadcastd        m15, [base+subpel_filters+r4*8+2]
    vpbroadcastd         m7, [base+subpel_filters+r6*8+2]
    pcmpeqd              m8, m9
    psrld               m14, 10
    movq                xm0, [srcq+ssq*0]
    movq                xm1, [srcq+ssq*2]
    movhps              xm0, [srcq+ssq*1]
    movhps              xm1, [srcq+ss3q ]
    lea                srcq, [srcq+ssq*4]
    shr                 myd, 6
    mov                 r4d, 64 << 24
    lea                 myd, [t1+myq]
    cmovnz              r4q, [base+subpel_filters+myq*8]
    pshufb              m14, m5
    paddb               m14, m6
    vinserti128          m0, [srcq+ssq*0], 1
    vinserti128          m1, [srcq+ssq*2], 1
    vpbroadcastq         m2, [srcq+ssq*1]
    add                srcq, ss3q
    movq               xm10, r4q
    pmovsxbw           xm10, xm10
    vpblendd            m15, m7, 0xaa
    pblendvb            m15, m11, m8
    pshufd              xm8, xm10, q0000
    pshufd              xm9, xm10, q1111
    pshufd             xm11, xm10, q3333
    pshufd             xm10, xm10, q2222
    vpblendd             m0, m2, 0xc0
    pshufb               m1, m14
    pshufb               m0, m14
    pmaddubsw            m1, m15
    pmaddubsw            m0, m15
    phaddw               m0, m1
    pmulhrsw             m0, m12
    vextracti128        xm1, m0, 1
    palignr             xm2, xm1, xm0, 4
    pshufd              xm4, xm1, q2121
    punpcklwd           xm3, xm0, xm2       ; 01 12
    punpckhwd           xm0, xm2            ; 23 34
    punpcklwd           xm2, xm1, xm4       ; 45 56
.dy1_w2_loop:
    movq                xm1, [srcq+ssq*0]
    movhps              xm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pmaddwd             xm5, xm3, xm8
    pmaddwd             xm6, xm0, xm9
    pmaddwd             xm7, xm2, xm10
    mova                xm3, xm0
    mova                xm0, xm2
    paddd               xm5, xm13
    paddd               xm6, xm7
    pshufb              xm1, xm14
    pmaddubsw           xm1, xm15
    phaddw              xm1, xm1
    pmulhrsw            xm1, xm12
    palignr             xm7, xm1, xm4, 12
    punpcklwd           xm2, xm7, xm1     ; 67 78
    pmaddwd             xm7, xm2, xm11
    mova                xm4, xm1
    paddd               xm5, xm6
    paddd               xm5, xm7
    psrad               xm5, rndshift
    packssdw            xm5, xm5
    packuswb            xm5, xm5
    pextrw     [dstq+dsq*0], xm5, 0
    pextrw     [dstq+dsq*1], xm5, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .dy1_w2_loop
    RET
%endif
.dy1_w4:
    mov                 myd, mym
    vbroadcasti128       m7, [base+rescale_mul]
    movzx               t0d, t0b
    dec                srcq
    movd               xm15, t0d
    pmaddwd              m8, m7
    vpbroadcastd        m11, [base+pd_0x4000]
    vpbroadcastd       xm15, xm15
    paddd               m14, m8 ; mx+dx*[0-3]
    pand                 m8, m14, m10
    psrld                m8, 6
    paddd              xm15, xm8
    vpermq               m8, m8, q3120
    movd                r4d, xm15
    pextrd              r6d, xm15, 2
    pextrd             r11d, xm15, 1
    pextrd             r13d, xm15, 3
    movd               xm15, [base+subpel_filters+r4*8+2]
    vpbroadcastd         m7, [base+subpel_filters+r6*8+2]
    movu                xm2, [srcq+ssq*0]
    movu                xm3, [srcq+ssq*2]
    vbroadcasti128       m5, [base+bdct_lb_dw]
    vpbroadcastq         m6, [base+subpel_s_shuf2]
    pcmpeqd              m8, m9
    psrld               m14, 10
    pinsrd             xm15, [base+subpel_filters+r11*8+2], 1
    vpblendd             m7, [base+subpel_filters+r13*8+2-20], 0x20
    vinserti128          m2, [srcq+ssq*1], 1
    vinserti128          m3, [srcq+ss3q ], 1
    lea                srcq, [srcq+ssq*4]
    shr                 myd, 6
    mov                 r4d, 64 << 24
    lea                 myd, [t1+myq]
    cmovnz              r4q, [base+subpel_filters+myq*8]
    pshufb              m14, m5
    paddb               m14, m6
    movu                xm4, [srcq+ssq*0]
    movu                xm5, [srcq+ssq*2]
    vinserti128          m4, [srcq+ssq*1], 1
    add                srcq, ss3q
    vpblendd            m15, m7, 0x30
    punpcklqdq          m15, m15
    pblendvb            m15, m11, m8
    movq               xm10, r4q
    punpcklqdq         xm10, xm10
    pmovsxbw            m10, xm10
    pshufb               m2, m14
    pshufb               m3, m14
    pshufb               m4, m14
    pshufb              xm5, xm14
    vpermq               m2, m2, q3120
    vpermq               m3, m3, q3120
    vpermq               m4, m4, q3120
    vpermq               m5, m5, q3120
    pshufd               m7, m10, q0000
    pshufd               m8, m10, q1111
    pshufd               m9, m10, q2222
    pshufd              m10, m10, q3333
    pmaddubsw            m2, m15
    pmaddubsw            m3, m15
    pmaddubsw            m4, m15
    pmaddubsw            m5, m15
    phaddw               m2, m3
    phaddw               m4, m5
    pmulhrsw             m2, m12
    pmulhrsw             m4, m12
    palignr              m5, m4, m2, 4
    pshufd               m3, m4, q2121
    punpcklwd            m0, m2, m5     ; 01 12
    punpckhwd            m1, m2, m5     ; 23 34
    punpcklwd            m2, m4, m3     ; 45 56
.dy1_w4_loop:
    movu               xm11, [srcq+ssq*0]
    vinserti128         m11, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    pmaddwd              m4, m0, m7
    pmaddwd              m5, m1, m8
    pmaddwd              m6, m2, m9
    mova                 m0, m1
    mova                 m1, m2
    paddd                m4, m13
    paddd                m5, m6
    pshufb              m11, m14
    vpermq              m11, m11, q3120
    pmaddubsw           m11, m15
    phaddw              m11, m11
    pmulhrsw            m11, m12
    palignr              m6, m11, m3, 12
    punpcklwd            m2, m6, m11    ; 67 78
    mova                 m3, m11
    pmaddwd              m6, m2, m10
    paddd                m4, m5
    paddd                m4, m6
    psrad                m4, rndshift
    vextracti128        xm5, m4, 1
    packssdw            xm4, xm5
%ifidn %1, put
    packuswb            xm4, xm4
    pshuflw             xm4, xm4, q3120
    movd       [dstq+dsq*0], xm4
    pextrd     [dstq+dsq*1], xm4, 1
    lea                dstq, [dstq+dsq*2]
%else
    pshufd              xm4, xm4, q3120
    mova             [tmpq], xm4
    add                tmpq, 16
%endif
    sub                  hd, 2
    jg .dy1_w4_loop
    MC_8TAP_SCALED_RET
.dy1_w8:
    mov      dword [rsp+72], 1
    movifprep   tmp_stridem, 16
    jmp .dy1_w_start
.dy1_w16:
    mov      dword [rsp+72], 2
    movifprep   tmp_stridem, 32
    jmp .dy1_w_start
.dy1_w32:
    mov      dword [rsp+72], 4
    movifprep   tmp_stridem, 64
    jmp .dy1_w_start
.dy1_w64:
    mov      dword [rsp+72], 8
    movifprep   tmp_stridem, 128
    jmp .dy1_w_start
.dy1_w128:
    mov      dword [rsp+72], 16
    movifprep   tmp_stridem, 256
.dy1_w_start:
    mov                 myd, mym
%ifidn %1, put
    movifnidn           dsm, dsq
%endif
    shr                 t0d, 16
    sub                srcq, 3
    shr                 myd, 6
    mov                 r4d, 64 << 24
    lea                 myd, [t1+myq]
    cmovnz              r4q, [base+subpel_filters+myq*8]
    pmaddwd              m8, [base+rescale_mul]
    movd               xm15, t0d
    mov            [rsp+76], t0d
    mov            [rsp+80], srcq
    mov            [rsp+88], r0q ; dstq / tmpq
%if UNIX64
    mov                  hm, hd
%endif
    shl           dword dxm, 3 ; dx*8
    vpbroadcastd        m15, xm15
    paddd               m14, m8 ; mx+dx*[0-7]
    movq                xm0, r4q
    pmovsxbw            xm0, xm0
    mova           [rsp+96], xm0
    jmp .dy1_hloop
.dy1_hloop_prep:
    dec      dword [rsp+72]
    jz .ret
    add      qword [rsp+88], 8*(isprep+1)
    mov                  hd, hm
    vpbroadcastd         m8, dxm
    vpbroadcastd        m10, [base+pd_0x3ff]
    paddd               m14, m8, [rsp+32]
    vpbroadcastd        m15, [rsp+76]
    pxor                 m9, m9
    mov                srcq, [rsp+80]
    mov                 r0q, [rsp+88] ; dstq / tmpq
.dy1_hloop:
    vpbroadcastq        m11, [base+pq_0x40000000]
    pand                 m6, m14, m10
    psrld                m6, 6
    paddd               m15, m6
    pcmpeqd              m6, m9
    vextracti128        xm7, m15, 1
    movd                r4d, xm15
    pextrd              r6d, xm15, 2
    pextrd              r7d, xm15, 1
    pextrd              r9d, xm15, 3
    movd               r10d, xm7
    pextrd             r11d, xm7, 2
    pextrd             r13d, xm7, 1
    pextrd              rXd, xm7, 3
    movu           [rsp+32], m14
    movq               xm15, [base+subpel_filters+ r4*8]
    movq               xm10, [base+subpel_filters+ r6*8]
    movhps             xm15, [base+subpel_filters+ r7*8]
    movhps             xm10, [base+subpel_filters+ r9*8]
    vinserti128         m15, [base+subpel_filters+r10*8], 1
    vinserti128         m10, [base+subpel_filters+r11*8], 1
    vpbroadcastq         m9, [base+subpel_filters+r13*8]
    vpbroadcastq         m8, [base+subpel_filters+ rX*8]
    psrld               m14, 10
    vextracti128        xm7, m14, 1
    movq           [rsp+64], xm14
    movd                r4d, xm14
    pextrd              r6d, xm14, 2
    pextrd              r7d, xm14, 1
    pextrd              r9d, xm14, 3
    movd               r10d, xm7
    pextrd             r11d, xm7, 2
    pextrd             r13d, xm7, 1
    pextrd              rXd, xm7, 3
    pshufd               m5, m6, q1100
    pshufd               m6, m6, q3322
    vpblendd            m15, m9, 0xc0
    vpblendd            m10, m8, 0xc0
    pblendvb            m15, m11, m5
    pblendvb            m10, m11, m6
    vbroadcasti128      m14, [base+subpel_s_shuf8]
    MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7  ; 0a 1a 0b 1b
    MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8  ; 2a 3a 2b 3b
    MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9  ; 4a 5a 4b 5b
    MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b
    movu              [rsp], m10
    vpbroadcastd         m8, [rsp+0x60]
    vpbroadcastd         m9, [rsp+0x64]
    vpbroadcastd        m10, [rsp+0x68]
    vpbroadcastd        m11, [rsp+0x6c]
    pshufb               m0, m14    ; 01a 01b
    pshufb               m1, m14    ; 23a 23b
    pshufb               m2, m14    ; 45a 45b
    pshufb               m3, m14    ; 67a 67b
    vbroadcasti128      m14, [base+wswap]
.dy1_vloop:
    pmaddwd              m4, m0, m8
    pmaddwd              m5, m1, m9
    pmaddwd              m6, m2, m10
    pmaddwd              m7, m3, m11
    paddd                m4, m5
    paddd                m6, m7
    paddd                m4, m13
    paddd                m4, m6
    psrad                m4, rndshift
    vextracti128        xm5, m4, 1
    packssdw            xm4, xm5
%ifidn %1, put
    packuswb            xm4, xm4
    movq             [dstq], xm4
    add                dstq, dsm
%else
    mova             [tmpq], xm4
    add                tmpq, tmp_stridem
%endif
    dec                  hd
    jz .dy1_hloop_prep
    movq                xm4, [srcq+ r4]
    movq                xm5, [srcq+ r6]
    movhps              xm4, [srcq+ r7]
    movhps              xm5, [srcq+ r9]
    vinserti128          m4, [srcq+r10], 1
    vinserti128          m5, [srcq+r11], 1
    vpbroadcastq         m6, [srcq+r13]
    vpbroadcastq         m7, [srcq+ rX]
    add                srcq, ssq
    pshufb               m0, m14
    pshufb               m1, m14
    pshufb               m2, m14
    pshufb               m3, m14
    vpblendd             m4, m6, 0xc0
    vpblendd             m5, m7, 0xc0
    pmaddubsw            m4, m15
    pmaddubsw            m5, [rsp]
    phaddw               m4, m5
    pslld                m5, m4, 16
    paddw                m4, m5
    pmulhrsw             m4, m12
    pblendw              m0, m1, 0xaa
    pblendw              m1, m2, 0xaa
    pblendw              m2, m3, 0xaa
    pblendw              m3, m4, 0xaa
    jmp .dy1_vloop
.dy2:
    movzx                wd, word [base+%1_8tap_scaled_avx2_dy2_table+wq*2]
    add                  wq, base_reg
    jmp                  wq
%ifidn %1, put
.dy2_w2:
    mov                 myd, mym
    movzx               t0d, t0b
    dec                srcq
    movd               xm15, t0d
    punpckldq            m8, m9, m8
    paddd               m14, m8 ; mx+dx*[0-1]
    vpbroadcastd        m11, [base+pd_0x4000]
    vpbroadcastd       xm15, xm15
    pand                 m8, m14, m10
    psrld                m8, 6
    paddd              xm15, xm8
    movd                r4d, xm15
    pextrd              r6d, xm15, 1
    vbroadcasti128       m5, [base+bdct_lb_dw]
    vbroadcasti128       m6, [base+subpel_s_shuf2]
    vpbroadcastd        m15, [base+subpel_filters+r4*8+2]
    vpbroadcastd         m7, [base+subpel_filters+r6*8+2]
    pcmpeqd              m8, m9
    psrld               m14, 10
    movq                xm0, [srcq+ssq*0]
    vpbroadcastq         m2, [srcq+ssq*1]
    movhps              xm0, [srcq+ssq*2]
    vpbroadcastq         m3, [srcq+ss3q ]
    lea                srcq, [srcq+ssq*4]
    pshufb              m14, m5
    paddb               m14, m6
    vpblendd            m15, m7, 0xaa
    pblendvb            m15, m11, m8
    movhps              xm1, [srcq+ssq*0]
    vpbroadcastq         m4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    shr                 myd, 6
    mov                 r4d, 64 << 24
    lea                 myd, [t1+myq]
    cmovnz              r4q, [base+subpel_filters+myq*8]
    vpblendd             m0, m2, 0x30
    vpblendd             m1, m4, 0xc0
    vpblendd             m0, m3, 0xc0
    pshufb               m0, m14
    pshufb               m1, m14
    pmaddubsw            m0, m15
    pmaddubsw            m1, m15
    movq               xm11, r4q
    pmovsxbw           xm11, xm11
    phaddw               m0, m1
    pmulhrsw             m0, m12            ; 0 2 _ 4  1 3 _ 5
    pshufd              xm8, xm11, q0000
    pshufd              xm9, xm11, q1111
    pshufd             xm10, xm11, q2222
    pshufd             xm11, xm11, q3333
    pshufd               m2, m0, q3110      ; 0 2 2 4  1 3 3 5
    vextracti128        xm1, m2, 1
    punpcklwd           xm3, xm2, xm1       ; 01 23
    punpckhwd           xm2, xm1            ; 23 45
.dy2_w2_loop:
    movq                xm6, [srcq+ssq*0]
    vpbroadcastq         m7, [srcq+ssq*1]
    movhps              xm6, [srcq+ssq*2]
    vpbroadcastq         m1, [srcq+ss3q ]
    lea                srcq, [srcq+ssq*4]
    pmaddwd             xm4, xm3, xm8
    pmaddwd             xm5, xm2, xm9
    vpblendd             m6, m7, 0x30
    vpblendd             m6, m1, 0xc0
    pshufb               m6, m14
    pmaddubsw            m6, m15
    phaddw               m6, m6
    pmulhrsw             m6, m12
    palignr              m0, m6, m0, 8
    pshufd               m2, m0, q3221
    vextracti128        xm1, m2, 1
    punpcklwd           xm3, xm2, xm1       ; 45 67
    punpckhwd           xm2, xm1            ; 67 89
    pmaddwd             xm6, xm3, xm10
    pmaddwd             xm7, xm2, xm11
    paddd               xm4, xm5
    paddd               xm4, xm13
    paddd               xm6, xm7
    paddd               xm4, xm6
    psrad               xm4, rndshift
    packssdw            xm4, xm4
    packuswb            xm4, xm4
    pextrw     [dstq+dsq*0], xm4, 0
    pextrw     [dstq+dsq*1], xm4, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .dy2_w2_loop
    RET
%endif
.dy2_w4:
    mov                 myd, mym
    vbroadcasti128       m7, [base+rescale_mul]
    movzx               t0d, t0b
    dec                srcq
    movd               xm15, t0d
    pmaddwd              m8, m7
    vpbroadcastd        m11, [base+pd_0x4000]
    vpbroadcastd       xm15, xm15
    paddd               m14, m8 ; mx+dx*[0-3]
    pand                 m8, m14, m10
    psrld                m8, 6
    paddd              xm15, xm8
    movd                r4d, xm15
    pextrd              r6d, xm15, 1
    pextrd             r11d, xm15, 2
    pextrd             r13d, xm15, 3
    movd               xm15, [base+subpel_filters+r4*8+2]
    vbroadcasti128       m5, [base+bdct_lb_dw]
    vpbroadcastq         m6, [base+subpel_s_shuf2]
    pinsrd             xm15, [base+subpel_filters+r6*8+2], 1
    pcmpeqd              m8, m9
    psrld               m14, 10
    movu                xm0, [srcq+ssq*0]
    movu                xm2, [srcq+ssq*2]
    pinsrd             xm15, [base+subpel_filters+r11*8+2], 2
    movu                xm1, [srcq+ssq*1]
    movu                xm3, [srcq+ss3q ]
    pinsrd             xm15, [base+subpel_filters+r13*8+2], 3
    lea                srcq, [srcq+ssq*4]
    shr                 myd, 6
    mov                 r4d, 64 << 24
    lea                 myd, [t1+myq]
    cmovnz              r4q, [base+subpel_filters+myq*8]
    vinserti128         m15, xm15, 1
    pshufb              m14, m5
    paddb               m14, m6
    vinserti128          m2, [srcq+ssq*0], 1
    vinserti128          m3, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    pblendvb            m15, m11, m8
    pshufb              xm0, xm14
    pshufb               m2, m14
    pshufb              xm1, xm14
    pshufb               m3, m14
    pmaddubsw           xm0, xm15
    pmaddubsw            m2, m15
    pmaddubsw           xm1, xm15
    pmaddubsw            m3, m15
    movq               xm11, r4q
    punpcklqdq         xm11, xm11
    pmovsxbw            m11, xm11
    phaddw               m0, m2
    phaddw               m1, m3
    pmulhrsw             m0, m12    ; 0 2  _ 4
    pmulhrsw             m1, m12    ; 1 3  _ 5
    pshufd               m8, m11, q0000
    pshufd               m9, m11, q1111
    pshufd              m10, m11, q2222
    pshufd              m11, m11, q3333
    punpcklwd           xm2, xm0, xm1
    punpckhwd            m1, m0, m1     ; 23 45
    vinserti128          m0, m2, xm1, 1 ; 01 23
.dy2_w4_loop:
    movu                xm6, [srcq+ssq*0]
    movu                xm7, [srcq+ssq*1]
    vinserti128          m6, [srcq+ssq*2], 1
    vinserti128          m7, [srcq+ss3q ], 1
    lea                srcq, [srcq+ssq*4]
    pmaddwd              m4, m0, m8
    pmaddwd              m5, m1, m9
    pshufb               m6, m14
    pshufb               m7, m14
    pmaddubsw            m6, m15
    pmaddubsw            m7, m15
    psrld                m2, m6, 16
    pslld                m3, m7, 16
    paddw                m6, m2
    paddw                m7, m3
    pblendw              m6, m7, 0xaa   ; 67 89
    pmulhrsw             m6, m12
    paddd                m4, m5
    vperm2i128           m0, m1, m6, 0x21 ; 45 67
    mova                 m1, m6
    pmaddwd              m6, m0, m10
    pmaddwd              m7, m1, m11
    paddd                m4, m13
    paddd                m6, m7
    paddd                m4, m6
    psrad                m4, rndshift
    vextracti128        xm5, m4, 1
    packssdw            xm4, xm5
%ifidn %1, put
    packuswb            xm4, xm4
    movd       [dstq+dsq*0], xm4
    pextrd     [dstq+dsq*1], xm4, 1
    lea                dstq, [dstq+dsq*2]
%else
    mova             [tmpq], xm4
    add                tmpq, 16
%endif
    sub                  hd, 2
    jg .dy2_w4_loop
    MC_8TAP_SCALED_RET
.dy2_w8:
    mov      dword [rsp+40], 1
    movifprep   tmp_stridem, 16
    jmp .dy2_w_start
.dy2_w16:
    mov      dword [rsp+40], 2
    movifprep   tmp_stridem, 32
    jmp .dy2_w_start
.dy2_w32:
    mov      dword [rsp+40], 4
    movifprep   tmp_stridem, 64
    jmp .dy2_w_start
.dy2_w64:
    mov      dword [rsp+40], 8
    movifprep   tmp_stridem, 128
    jmp .dy2_w_start
.dy2_w128:
    mov      dword [rsp+40], 16
    movifprep   tmp_stridem, 256
.dy2_w_start:
    mov                 myd, mym
%ifidn %1, put
    movifnidn           dsm, dsq
%endif
    shr                 t0d, 16
    sub                srcq, 3
    shr                 myd, 6
    mov                 r4d, 64 << 24
    lea                 myd, [t1+myq]
    cmovnz              r4q, [base+subpel_filters+myq*8]
    pmaddwd              m8, [base+rescale_mul]
    movd               xm15, t0d
    mov            [rsp+64], t0d
    mov            [rsp+48], srcq
    mov            [rsp+56], r0q ; dstq / tmpq
%if UNIX64
    mov                  hm, hd
%endif
    shl           dword dxm, 3 ; dx*8
    vpbroadcastd        m15, xm15
    paddd               m14, m8 ; mx+dx*[0-7]
    movq                xm0, r4q
    pmovsxbw            xm0, xm0
    mova         [rsp+0x50], xm0
    jmp .dy2_hloop
.dy2_hloop_prep:
    dec      dword [rsp+40]
    jz .ret
    add      qword [rsp+56], 8*(isprep+1)
    mov                  hd, hm
    vpbroadcastd         m8, dxm
    vpbroadcastd        m10, [base+pd_0x3ff]
    paddd               m14, m8, [rsp]
    vpbroadcastd        m15, [rsp+64]
    pxor                 m9, m9
    mov                srcq, [rsp+48]
    mov                 r0q, [rsp+56] ; dstq / tmpq
.dy2_hloop:
    vpbroadcastq        m11, [base+pq_0x40000000]
    pand                 m6, m14, m10
    psrld                m6, 6
    paddd               m15, m6
    pcmpeqd              m6, m9
    vextracti128        xm7, m15, 1
    movd                r4d, xm15
    pextrd              r6d, xm15, 2
    pextrd              r7d, xm15, 1
    pextrd              r9d, xm15, 3
    movd               r10d, xm7
    pextrd             r11d, xm7, 2
    pextrd             r13d, xm7, 1
    pextrd              rXd, xm7, 3
    movu              [rsp], m14
    movq               xm15, [base+subpel_filters+ r4*8]
    movq               xm10, [base+subpel_filters+ r6*8]
    movhps             xm15, [base+subpel_filters+ r7*8]
    movhps             xm10, [base+subpel_filters+ r9*8]
    vinserti128         m15, [base+subpel_filters+r10*8], 1
    vinserti128         m10, [base+subpel_filters+r11*8], 1
    vpbroadcastq         m9, [base+subpel_filters+r13*8]
    vpbroadcastq         m8, [base+subpel_filters+ rX*8]
    psrld               m14, 10
    vextracti128        xm7, m14, 1
    movd                r4d, xm14
    pextrd              r6d, xm14, 2
    pextrd              r7d, xm14, 1
    pextrd              r9d, xm14, 3
    movd               r10d, xm7
    pextrd             r11d, xm7, 2
    pextrd             r13d, xm7, 1
    pextrd              rXd, xm7, 3
    pshufd               m5, m6, q1100
    pshufd               m6, m6, q3322
    vpblendd            m15, m9, 0xc0
    vpblendd            m10, m8, 0xc0
    pblendvb            m15, m11, m5
    pblendvb            m10, m11, m6
    vbroadcasti128      m14, [base+subpel_s_shuf8]
    MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7  ; 0a 1a 0b 1b
    MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8  ; 2a 3a 2b 3b
    MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9  ; 4a 5a 4b 5b
    MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b
    vpbroadcastd         m8, [rsp+0x50]
    vpbroadcastd         m9, [rsp+0x54]
    vpbroadcastd        m11, [rsp+0x58]
    vpbroadcastd         m4, [rsp+0x5c]
    pshufb               m0, m14    ; 01a 01b
    pshufb               m1, m14    ; 23a 23b
    pshufb               m2, m14    ; 45a 45b
    pshufb               m3, m14    ; 67a 67b
    SWAP                m14, m4
.dy2_vloop:
    pmaddwd              m4, m0, m8
    pmaddwd              m5, m1, m9
    pmaddwd              m6, m2, m11
    pmaddwd              m7, m3, m14
    paddd                m4, m5
    paddd                m6, m7
    paddd                m4, m13
    paddd                m4, m6
    psrad                m4, rndshift
    vextracti128        xm5, m4, 1
    packssdw            xm4, xm5
%ifidn %1, put
    packuswb            xm4, xm4
    movq             [dstq], xm4
    add                dstq, dsm
%else
    mova             [tmpq], xm4
    add                tmpq, tmp_stridem
%endif
    dec                  hd
    jz .dy2_hloop_prep
    mova                 m0, m1
    mova                 m1, m2
    mova                 m2, m3
    movq                xm3, [srcq+ r4]
    movq                xm4, [srcq+ r6]
    movhps              xm3, [srcq+ r7]
    movhps              xm4, [srcq+ r9]
    vinserti128          m3, [srcq+r10], 1
    vinserti128          m4, [srcq+r11], 1
    vpbroadcastq         m5, [srcq+r13]
    vpbroadcastq         m6, [srcq+ rX]
    add                srcq, ssq
    vpblendd             m3, m5, 0xc0
    vpblendd             m4, m6, 0xc0
    pmaddubsw            m3, m15
    pmaddubsw            m4, m10
    phaddw               m3, m4
    movq                xm4, [srcq+ r4]
    movq                xm5, [srcq+ r6]
    movhps              xm4, [srcq+ r7]
    movhps              xm5, [srcq+ r9]
    vinserti128          m4, [srcq+r10], 1
    vinserti128          m5, [srcq+r11], 1
    vpbroadcastq         m6, [srcq+r13]
    vpbroadcastq         m7, [srcq+ rX]
    add                srcq, ssq
    vpblendd             m4, m6, 0xc0
    vpblendd             m5, m7, 0xc0
    pmaddubsw            m4, m15
    pmaddubsw            m5, m10
    phaddw               m4, m5
    psrld                m5, m3, 16
    pslld                m6, m4, 16
    paddw                m3, m5
    paddw                m4, m6
    pblendw              m3, m4, 0xaa
    pmulhrsw             m3, m12
    jmp .dy2_vloop
.ret:
    MC_8TAP_SCALED_RET 0
%undef isprep
%endmacro

%macro BILIN_SCALED_FN 1
cglobal %1_bilin_scaled_8bpc
    mov                 t0d, (5*15 << 16) | 5*15
    mov                 t1d, t0d
    jmp mangle(private_prefix %+ _%1_8tap_scaled_8bpc %+ SUFFIX)
%endmacro

%if WIN64
DECLARE_REG_TMP 6, 5
%else
DECLARE_REG_TMP 6, 8
%endif

%define PUT_8TAP_SCALED_FN FN put_8tap_scaled,
%define PREP_8TAP_SCALED_FN FN prep_8tap_scaled,

BILIN_SCALED_FN put
PUT_8TAP_SCALED_FN sharp,   SHARP,   put_8tap_scaled_8bpc
PUT_8TAP_SCALED_FN smooth,  SMOOTH,  put_8tap_scaled_8bpc
PUT_8TAP_SCALED_FN regular, REGULAR
MC_8TAP_SCALED put

%if WIN64
DECLARE_REG_TMP 5, 4
%else
DECLARE_REG_TMP 6, 7
%endif

BILIN_SCALED_FN prep
PREP_8TAP_SCALED_FN sharp,   SHARP,   prep_8tap_scaled_8bpc
PREP_8TAP_SCALED_FN smooth,  SMOOTH,  prep_8tap_scaled_8bpc
PREP_8TAP_SCALED_FN regular, REGULAR
MC_8TAP_SCALED prep

%macro WARP_V 5 ; dst, 02, 46, 13, 57
    ; Can be done using gathers, but that's terribly slow on many CPU:s
    lea               tmp1d, [myq+gammaq*4]
    lea               tmp2d, [myq+gammaq*1]
    shr                 myd, 10
    shr               tmp1d, 10
    movq                xm8, [filterq+myq  *8]
    vinserti128          m8, [filterq+tmp1q*8], 1 ; a e
    lea               tmp1d, [tmp2q+gammaq*4]
    lea                 myd, [tmp2q+gammaq*1]
    shr               tmp2d, 10
    shr               tmp1d, 10
    movq                xm0, [filterq+tmp2q*8]
    vinserti128          m0, [filterq+tmp1q*8], 1 ; b f
    lea               tmp1d, [myq+gammaq*4]
    lea               tmp2d, [myq+gammaq*1]
    shr                 myd, 10
    shr               tmp1d, 10
    movq                xm9, [filterq+myq  *8]
    vinserti128          m9, [filterq+tmp1q*8], 1 ; c g
    lea               tmp1d, [tmp2q+gammaq*4]
    lea                 myd, [tmp2q+deltaq]       ; my += delta
    shr               tmp2d, 10
    shr               tmp1d, 10
    punpcklwd            m8, m0
    movq                xm0, [filterq+tmp2q*8]
    vinserti128          m0, [filterq+tmp1q*8], 1 ; d h
    punpcklwd            m0, m9, m0
    punpckldq            m9, m8, m0
    punpckhdq            m0, m8, m0
    punpcklbw            m8, m11, m9 ; a0 a2 b0 b2 c0 c2 d0 d2 << 8
    punpckhbw            m9, m11, m9 ; a4 a6 b4 b6 c4 c6 d4 d6 << 8
    pmaddwd             m%2, m8
    pmaddwd              m9, m%3
    punpcklbw            m8, m11, m0 ; a1 a3 b1 b3 c1 c3 d1 d3 << 8
    punpckhbw            m0, m11, m0 ; a5 a7 b5 b7 c5 c7 d5 d7 << 8
    pmaddwd              m8, m%4
    pmaddwd              m0, m%5
    paddd               m%2, m9
    paddd                m0, m8
    paddd               m%1, m0, m%2
%endmacro

cglobal warp_affine_8x8t_8bpc, 0, 14, 0, tmp, ts
%if WIN64
    sub                 rsp, 0xa0
%endif
    call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).main
.loop:
    psrad                m7, 13
    psrad                m0, 13
    packssdw             m7, m0
    pmulhrsw             m7, m14 ; (x + (1 << 6)) >> 7
    vpermq               m7, m7, q3120
    mova         [tmpq+tsq*0], xm7
    vextracti128 [tmpq+tsq*2], m7, 1
    dec                 r4d
    jz   mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).end
    call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).main2
    lea                tmpq, [tmpq+tsq*4]
    jmp .loop

cglobal warp_affine_8x8_8bpc, 0, 14, 0, dst, ds, src, ss, abcd, mx, tmp2, alpha, \
                                        beta, filter, tmp1, gamma, my, delta
%if WIN64
    %assign xmm_regs_used 16
    %assign stack_size_padded 0xa0
    SUB                 rsp, stack_size_padded
%endif
    call .main
    jmp .start
.loop:
    call .main2
    lea                dstq, [dstq+dsq*2]
.start:
    psrad                m7, 18
    psrad                m0, 18
    packusdw             m7, m0
    pavgw                m7, m11 ; (x + (1 << 10)) >> 11
    vextracti128        xm0, m7, 1
    packuswb            xm7, xm0
    pshufd              xm7, xm7, q3120
    movq       [dstq+dsq*0], xm7
    movhps     [dstq+dsq*1], xm7
    dec                 r4d
    jg .loop
.end:
    RET
ALIGN function_align
.main:
    ; Stack is offset due to call
    %assign stack_offset stack_offset + gprsize
    %assign stack_size stack_size + gprsize
    %assign stack_size_padded stack_size_padded + gprsize
    movifnidn         abcdq, abcdmp
    movifnidn           mxd, mxm
    WIN64_PUSH_XMM
    movsx            alphad, word [abcdq+2*0]
    movsx             betad, word [abcdq+2*1]
    mova                m12, [warp_8x8_shufA]
    mova                m13, [warp_8x8_shufB]
    vpbroadcastd        m14, [pw_8192]
    vpbroadcastd        m15, [pd_32768]
    pxor                m11, m11
    lea             filterq, [mc_warp_filter2]
    lea               tmp1q, [ssq*3+3]
    add                 mxd, 512+(3*64<<10)
    lea               tmp2d, [alphaq*3]
    sub                srcq, tmp1q    ; src -= src_stride*3 + 3
    sub               betad, tmp2d    ; beta -= alpha*3
    mov                 myd, r6m
    call .h
    psrld                m1, m0, 16
    call .h
    psrld                m4, m0, 16
    call .h
    pblendw              m1, m0, 0xaa ; 02
    call .h
    pblendw              m4, m0, 0xaa ; 13
    call .h
    psrld                m2, m1, 16
    pblendw              m2, m0, 0xaa ; 24
    call .h
    psrld                m5, m4, 16
    pblendw              m5, m0, 0xaa ; 35
    call .h
    psrld                m3, m2, 16
    pblendw              m3, m0, 0xaa ; 46
    movsx            gammad, word [abcdq+2*2]
    movsx            deltad, word [abcdq+2*3]
    add                 myd, 512+(3*64<<10)
    mov                 r4d, 4
    lea               tmp1d, [gammaq*3]
    sub              deltad, tmp1d    ; delta -= gamma*3
.main2:
    call .h
    psrld                m6, m5, 16
    pblendw              m6, m0, 0xaa ; 57
    WARP_V                7, 1, 3, 4, 6
    call .h
    mova                 m1, m2
    mova                 m2, m3
    psrld                m3, 16
    pblendw              m3, m0, 0xaa ; 68
    WARP_V                0, 4, 6, 1, 3
    mova                 m4, m5
    mova                 m5, m6
    ret
ALIGN function_align
.h:
    lea               tmp1d, [mxq+alphaq*4]
    lea               tmp2d, [mxq+alphaq*1]
    vbroadcasti128      m10, [srcq]
    shr                 mxd, 10
    shr               tmp1d, 10
    movq                xm8, [filterq+mxq  *8]
    vinserti128          m8, [filterq+tmp1q*8], 1
    lea               tmp1d, [tmp2q+alphaq*4]
    lea                 mxd, [tmp2q+alphaq*1]
    shr               tmp2d, 10
    shr               tmp1d, 10
    movq                xm0, [filterq+tmp2q*8]
    vinserti128          m0, [filterq+tmp1q*8], 1
    lea               tmp1d, [mxq+alphaq*4]
    lea               tmp2d, [mxq+alphaq*1]
    shr                 mxd, 10
    shr               tmp1d, 10
    movq                xm9, [filterq+mxq  *8]
    vinserti128          m9, [filterq+tmp1q*8], 1
    lea               tmp1d, [tmp2q+alphaq*4]
    lea                 mxd, [tmp2q+betaq] ; mx += beta
    shr               tmp2d, 10
    shr               tmp1d, 10
    punpcklqdq           m8, m0  ; 0 1   4 5
    movq                xm0, [filterq+tmp2q*8]
    vinserti128          m0, [filterq+tmp1q*8], 1
    punpcklqdq           m9, m0  ; 2 3   6 7
    pshufb               m0, m10, m12
    pmaddubsw            m0, m8
    pshufb              m10, m13
    pmaddubsw           m10, m9
    add                srcq, ssq
    phaddw               m0, m10
    pmaddwd              m0, m14 ; 17-bit intermediate, upshifted by 13
    paddd                m0, m15 ; rounded 14-bit result in upper 16 bits of dword
    ret

%macro BIDIR_FN 1 ; op
    %1                    0
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4_loop:
    %1_INC_PTR            2
    %1                    0
    lea                dstq, [dstq+strideq*4]
.w4:
    vextracti128        xm1, m0, 1
    movd   [dstq+strideq*0], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q ], xm1, 1
    sub                  hd, 8
    jl .ret
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq+strideq*0], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
    jg .w4_loop
.ret:
    RET
.w8_loop:
    %1_INC_PTR            2
    %1                    0
    lea                dstq, [dstq+strideq*4]
.w8:
    vextracti128        xm1, m0, 1
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm0
    movhps [dstq+stride3q ], xm1
    sub                  hd, 4
    jg .w8_loop
    RET
.w16_loop:
    %1_INC_PTR            4
    %1                    0
    lea                dstq, [dstq+strideq*4]
.w16:
    vpermq               m0, m0, q3120
    movu         [dstq+strideq*0], xm0
    vextracti128 [dstq+strideq*1], m0, 1
    %1                    2
    vpermq               m0, m0, q3120
    movu         [dstq+strideq*2], xm0
    vextracti128 [dstq+stride3q ], m0, 1
    sub                  hd, 4
    jg .w16_loop
    RET
.w32_loop:
    %1_INC_PTR            4
    %1                    0
    lea                dstq, [dstq+strideq*2]
.w32:
    vpermq               m0, m0, q3120
    movu   [dstq+strideq*0], m0
    %1                    2
    vpermq               m0, m0, q3120
    movu   [dstq+strideq*1], m0
    sub                  hd, 2
    jg .w32_loop
    RET
.w64_loop:
    %1_INC_PTR            4
    %1                    0
    add                dstq, strideq
.w64:
    vpermq               m0, m0, q3120
    mova        [dstq+32*0], m0
    %1                    2
    vpermq               m0, m0, q3120
    mova        [dstq+32*1], m0
    dec                  hd
    jg .w64_loop
    RET
%endmacro

%macro AVG 1 ; src_offset
    mova                 m0, [tmp1q+(%1+0)*32]
    paddw                m0, [tmp2q+(%1+0)*32]
    mova                 m1, [tmp1q+(%1+1)*32]
    paddw                m1, [tmp2q+(%1+1)*32]
    pmulhrsw             m0, m2
    pmulhrsw             m1, m2
    packuswb             m0, m1
%endmacro

%macro AVG_INC_PTR 1
    add               tmp1q, %1*32
    add               tmp2q, %1*32
%endmacro

cglobal avg_8bpc, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3
%define base r6-avg_avx2_table
    lea                  r6, [avg_avx2_table]
    tzcnt                wd, wm
    movifnidn            hd, hm
    movsxd               wq, dword [r6+wq*4]
    vpbroadcastd         m2, [base+pw_1024]
    add                  wq, r6
    BIDIR_FN            AVG

%macro W_AVG 1 ; src_offset
    ; (a * weight + b * (16 - weight) + 128) >> 8
    ; = ((a - b) * weight + (b << 4) + 128) >> 8
    ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4
    ; = ((((b - a) * (-weight     << 12)) >> 16) + b + 8) >> 4
    mova                 m0,     [tmp1q+(%1+0)*32]
    psubw                m2, m0, [tmp2q+(%1+0)*32]
    mova                 m1,     [tmp1q+(%1+1)*32]
    psubw                m3, m1, [tmp2q+(%1+1)*32]
    pmulhw               m2, m4
    pmulhw               m3, m4
    paddw                m0, m2
    paddw                m1, m3
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
%endmacro

%define W_AVG_INC_PTR AVG_INC_PTR

cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3
%define base r6-w_avg_avx2_table
    lea                  r6, [w_avg_avx2_table]
    tzcnt                wd, wm
    movifnidn            hd, hm
    vpbroadcastw         m4, r6m ; weight
    movsxd               wq, dword [r6+wq*4]
    vpbroadcastd         m5, [base+pw_2048]
    psllw                m4, 12 ; (weight-16) << 12 when interpreted as signed
    add                  wq, r6
    cmp           dword r6m, 7
    jg .weight_gt7
    mov                  r6, tmp1q
    pxor                 m0, m0
    mov               tmp1q, tmp2q
    psubw                m4, m0, m4 ; -weight
    mov               tmp2q, r6
.weight_gt7:
    BIDIR_FN          W_AVG

%macro MASK 1 ; src_offset
    ; (a * m + b * (64 - m) + 512) >> 10
    ; = ((a - b) * m + (b << 6) + 512) >> 10
    ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4
    vpermq               m3,     [maskq+%1*16], q3120
    mova                 m0,     [tmp2q+(%1+0)*32]
    psubw                m1, m0, [tmp1q+(%1+0)*32]
    psubb                m3, m4, m3
    paddw                m1, m1     ; (b - a) << 1
    paddb                m3, m3
    punpcklbw            m2, m4, m3 ; -m << 9
    pmulhw               m1, m2
    paddw                m0, m1
    mova                 m1,     [tmp2q+(%1+1)*32]
    psubw                m2, m1, [tmp1q+(%1+1)*32]
    paddw                m2, m2
    punpckhbw            m3, m4, m3
    pmulhw               m2, m3
    paddw                m1, m2
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
%endmacro

%macro MASK_INC_PTR 1
    add               maskq, %1*16
    add               tmp2q, %1*32
    add               tmp1q, %1*32
%endmacro

cglobal mask_8bpc, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-mask_avx2_table
    lea                  r7, [mask_avx2_table]
    tzcnt                wd, wm
    movifnidn            hd, hm
    mov               maskq, maskmp
    movsxd               wq, dword [r7+wq*4]
    vpbroadcastd         m5, [base+pw_2048]
    pxor                 m4, m4
    add                  wq, r7
    BIDIR_FN           MASK

cglobal blend_8bpc, 3, 7, 7, dst, ds, tmp, w, h, mask
%define base r6-blend_avx2_table
    lea                  r6, [blend_avx2_table]
    tzcnt                wd, wm
    movifnidn         maskq, maskmp
    movifnidn            hd, hm
    movsxd               wq, dword [r6+wq*4]
    vpbroadcastd         m4, [base+pb_64]
    vpbroadcastd         m5, [base+pw_512]
    sub                tmpq, maskq
    add                  wq, r6
    lea                  r6, [dsq*3]
    jmp                  wq
.w4:
    movd                xm0, [dstq+dsq*0]
    pinsrd              xm0, [dstq+dsq*1], 1
    vpbroadcastd        xm1, [dstq+dsq*2]
    pinsrd              xm1, [dstq+r6   ], 3
    mova                xm6, [maskq]
    psubb               xm3, xm4, xm6
    punpcklbw           xm2, xm3, xm6
    punpckhbw           xm3, xm6
    mova                xm6, [maskq+tmpq]
    add               maskq, 4*4
    punpcklbw           xm0, xm6
    punpckhbw           xm1, xm6
    pmaddubsw           xm0, xm2
    pmaddubsw           xm1, xm3
    pmulhrsw            xm0, xm5
    pmulhrsw            xm1, xm5
    packuswb            xm0, xm1
    movd       [dstq+dsq*0], xm0
    pextrd     [dstq+dsq*1], xm0, 1
    pextrd     [dstq+dsq*2], xm0, 2
    pextrd     [dstq+r6   ], xm0, 3
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 4
    jg .w4
    RET
.w8:
    movq                xm1, [dstq+dsq*0]
    movhps              xm1, [dstq+dsq*1]
    vpbroadcastq         m2, [dstq+dsq*2]
    vpbroadcastq         m3, [dstq+r6   ]
    mova                 m0, [maskq]
    mova                 m6, [maskq+tmpq]
    add               maskq, 8*4
    vpblendd             m1, m2, 0x30
    vpblendd             m1, m3, 0xc0
    psubb                m3, m4, m0
    punpcklbw            m2, m3, m0
    punpckhbw            m3, m0
    punpcklbw            m0, m1, m6
    punpckhbw            m1, m6
    pmaddubsw            m0, m2
    pmaddubsw            m1, m3
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
    vextracti128        xm1, m0, 1
    movq       [dstq+dsq*0], xm0
    movhps     [dstq+dsq*1], xm0
    movq       [dstq+dsq*2], xm1
    movhps     [dstq+r6   ], xm1
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 4
    jg .w8
    RET
.w16:
    mova                 m0, [maskq]
    movu                xm1, [dstq+dsq*0]
    vinserti128          m1, [dstq+dsq*1], 1
    psubb                m3, m4, m0
    punpcklbw            m2, m3, m0
    punpckhbw            m3, m0
    mova                 m6, [maskq+tmpq]
    add               maskq, 16*2
    punpcklbw            m0, m1, m6
    punpckhbw            m1, m6
    pmaddubsw            m0, m2
    pmaddubsw            m1, m3
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
    movu         [dstq+dsq*0], xm0
    vextracti128 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .w16
    RET
.w32:
    mova                 m0, [maskq]
    movu                 m1, [dstq]
    mova                 m6, [maskq+tmpq]
    add               maskq, 32
    psubb                m3, m4, m0
    punpcklbw            m2, m3, m0
    punpckhbw            m3, m0
    punpcklbw            m0, m1, m6
    punpckhbw            m1, m6
    pmaddubsw            m0, m2
    pmaddubsw            m1, m3
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
    movu             [dstq], m0
    add                dstq, dsq
    dec                  hd
    jg .w32
    RET
.w64:
    mova                 m0, [maskq+32*0]
    mova                 m1, [dstq+32*0]
    mova                 m2, [maskq+tmpq+32*0]
    psubb                m6, m4, m0
    punpcklbw            m3, m6, m0
    punpckhbw            m6, m0
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    mova                 m2, [maskq+32*1]
    pmaddubsw            m0, m3
    mova                 m3, [dstq+32*1]
    pmaddubsw            m1, m6
    mova                 m6, [maskq+tmpq+32*1]
    add               maskq, 64
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
    psubb                m1, m4, m2
    mova        [dstq+32*0], m0
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    punpcklbw            m2, m3, m6
    punpckhbw            m3, m6
    pmaddubsw            m2, m0
    pmaddubsw            m3, m1
    pmulhrsw             m2, m5
    pmulhrsw             m3, m5
    packuswb             m2, m3
    mova        [dstq+32*1], m2
    add                dstq, dsq
    dec                  hd
    jg .w64
    RET

cglobal emu_edge_8bpc, 10, 13, 1, bw, bh, iw, ih, x, y, dst, dstride, src, sstride, \
                             bottomext, rightext
    ; we assume that the buffer (stride) is larger than width, so we can
    ; safely overwrite by a few bytes

    ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride)
    xor                r12d, r12d
    lea                 r10, [ihq-1]
    cmp                  yq, ihq
    cmovs               r10, yq
    test                 yq, yq
    cmovs               r10, r12
    imul                r10, sstrideq
    add                srcq, r10

    ; ref += iclip(x, 0, iw - 1)
    lea                 r10, [iwq-1]
    cmp                  xq, iwq
    cmovs               r10, xq
    test                 xq, xq
    cmovs               r10, r12
    add                srcq, r10

    ; bottom_ext = iclip(y + bh - ih, 0, bh - 1)
    lea          bottomextq, [yq+bhq]
    sub          bottomextq, ihq
    lea                  r3, [bhq-1]
    cmovs        bottomextq, r12

    DEFINE_ARGS bw, bh, iw, ih, x, topext, dst, dstride, src, sstride, \
                bottomext, rightext

    ; top_ext = iclip(-y, 0, bh - 1)
    neg             topextq
    cmovs           topextq, r12
    cmp          bottomextq, bhq
    cmovns       bottomextq, r3
    cmp             topextq, bhq
    cmovg           topextq, r3

    ; right_ext = iclip(x + bw - iw, 0, bw - 1)
    lea           rightextq, [xq+bwq]
    sub           rightextq, iwq
    lea                  r2, [bwq-1]
    cmovs         rightextq, r12

    DEFINE_ARGS bw, bh, iw, ih, leftext, topext, dst, dstride, src, sstride, \
                bottomext, rightext

    ; left_ext = iclip(-x, 0, bw - 1)
    neg            leftextq
    cmovs          leftextq, r12
    cmp           rightextq, bwq
    cmovns        rightextq, r2
    cmp            leftextq, bwq
    cmovns         leftextq, r2

    DEFINE_ARGS bw, centerh, centerw, dummy, leftext, topext, \
                dst, dstride, src, sstride, bottomext, rightext

    ; center_h = bh - top_ext - bottom_ext
    lea                  r3, [bottomextq+topextq]
    sub            centerhq, r3

    ; blk += top_ext * PXSTRIDE(dst_stride)
    mov                  r2, topextq
    imul                 r2, dstrideq
    add                dstq, r2
    mov                 r9m, dstq

    ; center_w = bw - left_ext - right_ext
    mov            centerwq, bwq
    lea                  r3, [rightextq+leftextq]
    sub            centerwq, r3

%macro v_loop 3 ; need_left_ext, need_right_ext, suffix
.v_loop_%3:
%if %1
    ; left extension
    xor                  r3, r3
    vpbroadcastb         m0, [srcq]
.left_loop_%3:
    mova          [dstq+r3], m0
    add                  r3, 32
    cmp                  r3, leftextq
    jl .left_loop_%3

    ; body
    lea                 r12, [dstq+leftextq]
%endif
    xor                  r3, r3
.body_loop_%3:
    movu                 m0, [srcq+r3]
%if %1
    movu           [r12+r3], m0
%else
    movu          [dstq+r3], m0
%endif
    add                  r3, 32
    cmp                  r3, centerwq
    jl .body_loop_%3

%if %2
    ; right extension
%if %1
    add                 r12, centerwq
%else
    lea                 r12, [dstq+centerwq]
%endif
    xor                  r3, r3
    vpbroadcastb         m0, [srcq+centerwq-1]
.right_loop_%3:
    movu           [r12+r3], m0
    add                  r3, 32
    cmp                  r3, rightextq
    jl .right_loop_%3

%endif
    add                dstq, dstrideq
    add                srcq, sstrideq
    dec            centerhq
    jg .v_loop_%3
%endmacro

    test           leftextq, leftextq
    jnz .need_left_ext
    test          rightextq, rightextq
    jnz .need_right_ext
    v_loop                0, 0, 0
    jmp .body_done

.need_left_ext:
    test          rightextq, rightextq
    jnz .need_left_right_ext
    v_loop                1, 0, 1
    jmp .body_done

.need_left_right_ext:
    v_loop                1, 1, 2
    jmp .body_done

.need_right_ext:
    v_loop                0, 1, 3

.body_done:
    ; bottom edge extension
    test         bottomextq, bottomextq
    jz .top
    mov                srcq, dstq
    sub                srcq, dstrideq
    xor                  r1, r1
.bottom_x_loop:
    mova                 m0, [srcq+r1]
    lea                  r3, [dstq+r1]
    mov                  r4, bottomextq
.bottom_y_loop:
    mova               [r3], m0
    add                  r3, dstrideq
    dec                  r4
    jg .bottom_y_loop
    add                  r1, 32
    cmp                  r1, bwq
    jl .bottom_x_loop

.top:
    ; top edge extension
    test            topextq, topextq
    jz .end
    mov                srcq, r9m
    mov                dstq, dstm
    xor                  r1, r1
.top_x_loop:
    mova                 m0, [srcq+r1]
    lea                  r3, [dstq+r1]
    mov                  r4, topextq
.top_y_loop:
    mova               [r3], m0
    add                  r3, dstrideq
    dec                  r4
    jg .top_y_loop
    add                  r1, 32
    cmp                  r1, bwq
    jl .top_x_loop

.end:
    RET

%macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4
    mova                m%1, [tmp1q+32*%3]
    mova                 m1, [tmp2q+32*%3]
    psubw                m1, m%1
    pabsw               m%2, m1
    psubusw             m%2, m6, m%2
    psrlw               m%2, 8 ; 64 - m
    psllw                m2, m%2, 10
    pmulhw               m1, m2
    paddw               m%1, m1
    mova                 m1, [tmp1q+32*%4]
    mova                 m2, [tmp2q+32*%4]
    psubw                m2, m1
    pabsw                m3, m2
    psubusw              m3, m6, m3
    psrlw                m3, 8
%if %5
    packuswb            m%2, m3
    psubb               m%2, m5, m%2
    vpermq              m%2, m%2, q3120
%else
    phaddw              m%2, m3
%endif
    psllw                m3, 10
    pmulhw               m2, m3
    paddw                m1, m2
    pmulhrsw            m%1, m7
    pmulhrsw             m1, m7
    packuswb            m%1, m1
%endmacro

cglobal w_mask_420_8bpc, 4, 9, 14, dst, stride, tmp1, tmp2, w, h, mask, mstr, stride3
%define base r7-w_mask_420_avx2_table
    lea                  r7, [w_mask_420_avx2_table]
    tzcnt                wd, wm
    mov                 r6d, r8m ; sign
    movifnidn            hd, hm
    movsxd               wq, [r7+wq*4]
    vpbroadcastd         m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
    vpbroadcastd         m7, [base+pw_2048]
    pmovzxbd             m9, [base+deint_shuf4]
    vpbroadcastd         m8, [base+wm_420_sign+r6*4] ; 258 - sign
    add                  wq, r7
    W_MASK                0, 4, 0, 1
    mov               maskq, maskmp
    mov               mstrq, mstrmp
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4_loop:
    add               tmp1q, 4*32
    add               tmp2q, 4*32
    W_MASK                0, 4, 0, 1
    lea                dstq, [dstq+strideq*4]
    add               maskq, 16
.w4:
    vextracti128        xm1, m0, 1
    movd   [dstq+strideq*0], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q ], xm1, 1
    cmp                  hd, 8
    jl .w4_end
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq+strideq*0], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
    je .w4_end
    W_MASK                0, 5, 2, 3
    lea                dstq, [dstq+strideq*4]
    phaddd               m4, m5
    vextracti128        xm1, m0, 1
    psubw                m4, m8, m4
    psrlw                m4, 2
    vpermd               m4, m9, m4
    vextracti128        xm5, m4, 1
    packuswb            xm4, xm5
    movd   [dstq+strideq*0], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q], xm1, 1
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq+strideq*0], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
    mova            [maskq], xm4
    sub                  hd, 16
    jg .w4_loop
    RET
.w4_end:
    vextracti128        xm0, m4, 1
    vpblendd            xm1, xm4, xm0, 0x05
    vpblendd            xm4, xm0, 0x0a
    pshufd              xm1, xm1, q2301
    psubw               xm4, xm8, xm4
    psubw               xm4, xm1
    psrlw               xm4, 2
    packuswb            xm4, xm4
    movq            [maskq], xm4
    RET
.w8_loop:
    add               tmp1q, 2*32
    add               tmp2q, 2*32
    W_MASK                0, 4, 0, 1
    lea                dstq, [dstq+strideq*4]
    add               maskq, 8
.w8:
    vextracti128        xm2, m4, 1
    vextracti128        xm1, m0, 1
    psubw               xm4, xm8, xm4
    psubw               xm4, xm2
    psrlw               xm4, 2
    packuswb            xm4, xm4
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm0
    movhps [dstq+stride3q ], xm1
    movq            [maskq], xm4
    sub                  hd, 4
    jg .w8_loop
    RET
.w16_loop:
    add               tmp1q, 4*32
    add               tmp2q, 4*32
    W_MASK                0, 4, 0, 1
    lea                dstq, [dstq+strideq*4]
    add               maskq, 16
.w16:
    vpermq               m0, m0, q3120
    movu         [dstq+strideq*0], xm0
    vextracti128 [dstq+strideq*1], m0, 1
    W_MASK                0, 5, 2, 3
    punpckhqdq           m1, m4, m5
    punpcklqdq           m4, m5
    psubw                m1, m8, m1
    psubw                m1, m4
    psrlw                m1, 2
    vpermq               m0, m0, q3120
    packuswb             m1, m1
    vpermd               m1, m9, m1
    movu         [dstq+strideq*2], xm0
    vextracti128 [dstq+stride3q ], m0, 1
    mova            [maskq], xm1
    sub                  hd, 4
    jg .w16_loop
    RET
.w32_loop:
    add               tmp1q, 4*32
    add               tmp2q, 4*32
    W_MASK                0, 4, 0, 1
    lea                dstq, [dstq+strideq*2]
    add               maskq, 16
.w32:
    vpermq               m0, m0, q3120
    movu   [dstq+strideq*0], m0
    W_MASK                0, 5, 2, 3
    psubw                m4, m8, m4
    psubw                m4, m5
    psrlw                m4, 2
    vpermq               m0, m0, q3120
    packuswb             m4, m4
    vpermd               m4, m9, m4
    movu   [dstq+strideq*1], m0
    mova            [maskq], xm4
    sub                  hd, 2
    jg .w32_loop
    RET
.w64_loop_even:
    psubw               m10, m8, m4
    psubw               m11, m8, m5
    dec                  hd
.w64_loop:
    add               tmp1q, 4*32
    add               tmp2q, 4*32
    W_MASK                0, 4, 0, 1
    add                dstq, strideq
.w64:
    vpermq               m0, m0, q3120
    mova        [dstq+32*0], m0
    W_MASK                0, 5, 2, 3
    vpermq               m0, m0, q3120
    mova        [dstq+32*1], m0
    test                 hd, 1
    jz .w64_loop_even
    psubw                m4, m10, m4
    psubw                m5, m11, m5
    psrlw                m4, 2
    psrlw                m5, 2
    packuswb             m4, m5
    vpermd               m4, m9, m4
    mova            [maskq], m4
    add               maskq, mstrq
    dec                  hd
    jg .w64_loop
    RET

cglobal w_mask_422_8bpc, 4, 9, 11, dst, stride, tmp1, tmp2, w, h, mask, mstr, stride3
%define base r7-w_mask_422_avx2_table
    lea                  r7, [w_mask_422_avx2_table]
    tzcnt                wd, wm
    mov                 r6d, r8m ; sign
    movifnidn            hd, hm
    pxor                 m9, m9
    movsxd               wq, dword [r7+wq*4]
    vpbroadcastd         m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
    vpbroadcastd         m7, [base+pw_2048]
    pmovzxbd            m10, [base+deint_shuf4]
    vpbroadcastd         m8, [base+wm_422_sign+r6*4] ; 128 - sign
    add                  wq, r7
    mov               maskq, maskmp
    mov               mstrq, mstrmp
    W_MASK                0, 4, 0, 1
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4_loop:
    add               tmp1q, 32*4
    add               tmp2q, 32*4
    W_MASK                0, 4, 0, 1
    lea                dstq, [dstq+strideq*4]
    add               maskq, 32
.w4:
    vextracti128        xm1, m0, 1
    movd   [dstq+strideq*0], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q ], xm1, 1
    cmp                  hd, 8
    jl .w4_end
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq+strideq*0], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
    je .w4_end
    W_MASK                0, 5, 2, 3
    lea                dstq, [dstq+strideq*4]
    packuswb             m4, m5
    psubb                m5, m8, m4
    pavgb                m5, m9
    vpermd               m5, m10, m5
    vextracti128        xm1, m0, 1
    movd   [dstq+strideq*0], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q ], xm1, 1
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq+strideq*0], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
    mova            [maskq], m5
    sub                  hd, 16
    jg .w4_loop
    RET
.w4_end:
    vextracti128        xm5, m4, 1
    packuswb            xm4, xm5
    psubb               xm5, xm8, xm4
    pavgb               xm5, xm9
    pshufd              xm5, xm5, q3120
    mova            [maskq], xm5
    RET
.w8_loop:
    add               tmp1q, 32*2
    add               tmp2q, 32*2
    W_MASK                0, 4, 0, 1
    lea                dstq, [dstq+strideq*4]
    add               maskq, 16
.w8:
    vextracti128        xm5, m4, 1
    vextracti128        xm1, m0, 1
    packuswb            xm4, xm5
    psubb               xm5, xm8, xm4
    pavgb               xm5, xm9
    pshufd              xm5, xm5, q3120
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm0
    movhps [dstq+stride3q ], xm1
    mova            [maskq], xm5
    sub                  hd, 4
    jg .w8_loop
    RET
.w16_loop:
    add               tmp1q, 32*4
    add               tmp2q, 32*4
    W_MASK                0, 4, 0, 1
    lea                dstq, [dstq+strideq*4]
    add               maskq, 32
.w16:
    vpermq               m0, m0, q3120
    movu         [dstq+strideq*0], xm0
    vextracti128 [dstq+strideq*1], m0, 1
    W_MASK                0, 5, 2, 3
    packuswb             m4, m5
    psubb                m5, m8, m4
    pavgb                m5, m9
    vpermq               m0, m0, q3120
    vpermd               m5, m10, m5
    movu         [dstq+strideq*2], xm0
    vextracti128 [dstq+stride3q ], m0, 1
    mova            [maskq], m5
    sub                  hd, 4
    jg .w16_loop
    RET
.w32_loop:
    add               tmp1q, 32*4
    add               tmp2q, 32*4
    W_MASK                0, 4, 0, 1
    lea                dstq, [dstq+strideq*2]
    add               maskq, 32
.w32:
    vpermq               m0, m0, q3120
    movu   [dstq+strideq*0], m0
    W_MASK                0, 5, 2, 3
    packuswb             m4, m5
    psubb                m5, m8, m4
    pavgb                m5, m9
    vpermq               m0, m0, q3120
    vpermd               m5, m10, m5
    movu   [dstq+strideq*1], m0
    mova            [maskq], m5
    sub                  hd, 2
    jg .w32_loop
    RET
.w64_loop:
    add               tmp1q, 32*4
    add               tmp2q, 32*4
    W_MASK                0, 4, 0, 1
    add                dstq, strideq
    add               maskq, mstrq
.w64:
    vpermq               m0, m0, q3120
    mova        [dstq+32*0], m0
    W_MASK                0, 5, 2, 3
    packuswb             m4, m5
    psubb                m5, m8, m4
    pavgb                m5, m9
    vpermq               m0, m0, q3120
    vpermd               m5, m10, m5
    mova        [dstq+32*1], m0
    mova            [maskq], m5
    dec                  hd
    jg .w64_loop
    RET

cglobal w_mask_444_8bpc, 4, 8, 8, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_444_avx2_table
    lea                  r7, [w_mask_444_avx2_table]
    tzcnt                wd, wm
    movifnidn            hd, hm
    mov               maskq, maskmp
    movsxd               wq, dword [r7+wq*4]
    vpbroadcastd         m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
    vpbroadcastd         m5, [base+pb_64]
    vpbroadcastd         m7, [base+pw_2048]
    add                  wq, r7
    W_MASK                0, 4, 0, 1, 1
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4_loop:
    add               tmp1q, 32*2
    add               tmp2q, 32*2
    W_MASK                0, 4, 0, 1, 1
    lea                dstq, [dstq+strideq*4]
    add               maskq, 32
.w4:
    vextracti128        xm1, m0, 1
    movd   [dstq+strideq*0], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q ], xm1, 1
    mova            [maskq], m4
    sub                  hd, 8
    jl .w4_end
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq+strideq*0], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
    jg .w4_loop
.w4_end:
    RET
.w8_loop:
    add               tmp1q, 32*2
    add               tmp2q, 32*2
    W_MASK                0, 4, 0, 1, 1
    lea                dstq, [dstq+strideq*4]
    add               maskq, 32
.w8:
    vextracti128        xm1, m0, 1
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm0
    movhps [dstq+stride3q ], xm1
    mova            [maskq], m4
    sub                  hd, 4
    jg .w8_loop
    RET
.w16_loop:
    add               tmp1q, 32*2
    add               tmp2q, 32*2
    W_MASK                0, 4, 0, 1, 1
    lea                dstq, [dstq+strideq*2]
    add               maskq, 32
.w16:
    vpermq               m0, m0, q3120
    movu         [dstq+strideq*0], xm0
    vextracti128 [dstq+strideq*1], m0, 1
    mova            [maskq], m4
    sub                  hd, 2
    jg .w16_loop
    RET
.w32_loop:
    add               tmp1q, 32*2
    add               tmp2q, 32*2
    W_MASK                0, 4, 0, 1, 1
    add                dstq, strideq
    add               maskq, 32
.w32:
    vpermq               m0, m0, q3120
    movu             [dstq], m0
    mova            [maskq], m4
    dec                  hd
    jg .w32_loop
    RET
.w64_loop:
    add               tmp1q, 32*4
    add               tmp2q, 32*4
    W_MASK                0, 4, 0, 1, 1
    add                dstq, strideq
    add               maskq, 32*2
.w64:
    vpermq               m0, m0, q3120
    mova        [dstq+32*0], m0
    mova       [maskq+32*0], m4
    W_MASK                0, 4, 2, 3, 1
    vpermq               m0, m0, q3120
    mova        [dstq+32*1], m0
    mova       [maskq+32*1], m4
    dec                  hd
    jg .w64_loop
    RET

cglobal morph_8bpc, 4, 6, 9, dst, stride, a, b, w, h
    movd                xm0, ad
    movd                xm2, bd
    lea                  r2, [morph_avx2_table]
    vpbroadcastw         m0, xm0
    vpbroadcastd         m2, xm2
    vbroadcasti128       m3, [r2+morph_A-morph_avx2_table]
    DEFINE_ARGS dst, stride, _, stride3, w, h
    tzcnt                wd, wm
    movifnidn            hd, hm
    movsxd               wq, dword [r2+wq*4]
    add                  wq, r2
    lea            stride3q, [strideq*3]
    pslld                m1, m0, 16
    psrld                m0, 16         ; since a can be negative
    jmp                  wq
.w4:
    movd                xm4, [dstq+strideq*0]
    vpbroadcastd         m6, [dstq+strideq*2]
    movd                xm5, [dstq+strideq*1]
    vpbroadcastd         m7, [dstq+stride3q ]
    vpblendd             m4, m6, 0x10
    vpblendd             m5, m7, 0x10
    REPX    {pshufb  x, m3}, m4, m5
    REPX    {pmaddwd x, m0}, m4, m5
    REPX    {paddd   x, m2}, m4, m5
    packusdw             m4, m5
    psrlw                m4, 8
    packuswb             m4, m4
    vextracti128        xm5, m4, 1
    movd   [dstq+strideq*0], xm4
    pextrd [dstq+strideq*1], xm4, 1
    movd   [dstq+strideq*2], xm5
    pextrd [dstq+stride3q ], xm5, 1
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w4
    RET
.w8:
    movq                xm4, [dstq+strideq*0]
    movq                xm6, [dstq+strideq*1]
    vpbroadcastq         m5, [dstq+strideq*2]
    vpbroadcastq         m7, [dstq+stride3q ]
    vpblendd             m4, m5, 0xf0
    vpblendd             m6, m7, 0xf0
    REPX     {pshufb x, m3}, m4, m6
    pmaddwd              m5, m4, m1
    pmaddwd              m4, m0
    pmaddwd              m7, m6, m1
    pmaddwd              m6, m0
    REPX    {paddd   x, m2}, m5, m4, m7, m6
    packusdw             m4, m5
    packusdw             m6, m7
    REPX    {psrlw   x, 8 }, m4, m6
    packuswb             m4, m6
    vextracti128        xm5, m4, 1
    movq   [dstq+strideq*0], xm4
    movhps [dstq+strideq*1], xm4
    movq   [dstq+strideq*2], xm5
    movhps [dstq+stride3q ], xm5
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w8
    RET
.w16:
    vbroadcasti128       m4, [r2+morph_B-morph_avx2_table]
.w16_loop:
    movu                xm5, [dstq+strideq*0]
    vinserti128          m5, [dstq+strideq*1], 1
    call .body
    movu         [dstq+strideq*0], xm5
    vextracti128 [dstq+strideq*1], m5, 1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w16_loop
    RET
.w32:
    vbroadcasti128       m4, [r2+morph_B-morph_avx2_table]
.w32_loop:
    movu                 m5, [dstq]
    call .body
    movu             [dstq], m5
    add                dstq, strideq
    dec                  hd
    jg .w32_loop
    RET
.w64:
    vbroadcasti128       m4, [r2+morph_B-morph_avx2_table]
.w64_loop:
    mova                 m5, [dstq]
    call .body
    mova             [dstq], m5
    mova                 m5, [dstq+32]
    call .body
    mova          [dstq+32], m5
    add                dstq, strideq
    dec                  hd
    jg .w64_loop
    RET
.body:
    pshufb               m6, m5, m4
    pshufb               m5, m3
    pmaddwd              m7, m5, m1
    pmaddwd              m5, m0
    pmaddwd              m8, m6, m1
    pmaddwd              m6, m0
    REPX    {paddd   x, m2}, m7, m5, m8, m6
    packusdw             m5, m7
    packusdw             m6, m8
    REPX    {psrlw   x, 8 }, m5, m6
    packuswb             m5, m6
    ret

cglobal sad_refine_mv_8bpc, 4, 10, 16, 9 * 32, p0, p0s, p1, p1s, p0s3, h, p1s3, p0s5, p1s5
    movifnidn            hd, hm
    lea                  r9, [hq+4]
    lea               p1s3q, [p1sq*3]
    lea               p0s5q, [p0sq*5]
    lea               p1s5q, [p1sq*5]
    cmp           dword r4m, 8                  ; if w == 8
    lea               p0s3q, [p0sq*3]
    je .w8

    ; w=16
    imul                r9d, 40
    mov               [rsp], r9d
    movu                xm0, [p0q+2*p0sq]
    movu                xm1, [p1q+2*p1sq]
    movq                xm2, [p0q+2*p0sq+16]
    movq                xm3, [p1q+2*p1sq+16]
    movu                xm4, [p0q+2*p0s3q+8]
    movu                xm5, [p1q+2*p1s3q+8]
    movhps              xm2, [p0q+2*p0s3q]
    movhps              xm3, [p1q+2*p1s3q]
    vinserti128          m0, [p0q+4*p0sq], 1    ; p0[y=2|4,x=0-15]
    vinserti128          m1, [p1q+4*p1sq], 1    ; p1[y=2|4,x=0-15]
    vinserti128          m2, [p0q+4*p0sq+16], 1
    vinserti128          m3, [p1q+4*p1sq+16], 1
    vpbroadcastq         m6, [p0q+8*p0sq]
    vpbroadcastq         m7, [p1q+8*p1sq]
    vinserti128          m4, [p0q+8*p0sq+8], 1  ; p0[y=6|8,x=8-23]
    vinserti128          m5, [p1q+8*p1sq+8], 1  ; p1[y=6|8,x=8-23]
    vpblendd             m2, m6, 0xc0           ; p0[y=2|4,x=16-23 & y=6|8,x=0-7]
    vpblendd             m3, m7, 0xc0           ; p1[y=2|4,x=16-23 & y=6|8,x=0-7]
    movu                xm6, [p0q+2*p0s5q]
    movq                xm8, [p0q+2*p0s5q+16]
    movu                xm7, [p1q+2*p1s5q+8]
    movhps              xm8, [p1q+2*p1s5q]
    vinserti128          m6, [p0q+4*p0s3q], 1   ; p0[y=10|12,x=0-15]
    vinserti128          m7, [p1q+4*p1s3q+8], 1 ; p1[y=10|12,x=8-23]
    vinserti128          m8, [p0q+4*p0s3q+16], 1
    vpbroadcastq         m9, [p1q+4*p1s3q]
    vpblendd             m8, m9, 0xc0           ; p0[y=10|12,x=16-23] & p1[y=10|12,x=0-7]
    mov                 r9d, -1
    cmp            byte r6m, 0
    je .w16_main

    ; m0: p0[y=2|4,x=0-15]
    ; m1: p1[y=2|4,x=0-15]
    ; m2: p0[y=2|4,x=16-23 & y=6|8,x=0-7]
    ; m3: p1[y=2|4,x=16-23 & y=6|8,x=0-7]
    ; m4: p0[y=6|8,x=8-23]
    ; m5: p1[y=6|8,x=8-23]
    ; m6: p0[y=10|12,x=0-15]
    ; m7: p1[y=10|12,x=8-23]
    ; m8: p0[y=10|12,x=16-23] & p1[y=10|12,x=0-7]
    ; m9-15: free

    palignr              m9, m2, m0, 2          ; p0[y=2|4,x=2-17]
    palignr             m10, m3, m1, 2          ; p1[y=2|4,x=2-17]
    palignr             m11, m4, m2, 14         ; p0[y=6|8,x=6-21]
    palignr             m12, m5, m3, 14         ; p1[y=6|8,x=6-21]
    palignr             m13, m8, m6, 2          ; p0[y=10|12,x=2-17]
    palignr             m14, m7, m8, 10         ; p1[y=10|12,x=2-17]
    psadbw               m9, m10
    psadbw              m11, m12
    psadbw              m13, m14
    psrldq              m10, m2, 2              ; p0[y=2|4,x=18-23 & y=6|8,x=0-7], 2x0
    palignr             m12, m8, m7, 10         ; p1[y=10|12,x=18-23] &
                                                ; p0[y=10|12,x=16-23] & 2x?
    psrldq              m14, m3, 2              ; p1[y=2|4,x=18-23 & y=6|8,x=0-7], 2x0
    shufps              m10, m12, q2220         ; p0[y=2|4,x=18-21 & y=6|8,x=2-5 &
                                                ;    y=10|12,x=18-21 [2x]]
    shufps              m14, m12, q2020         ; p1[y=2|4,x=18-21 & y=6|8,x=2-5 &
                                                ;    y=10|12,x=18-21] &
                                                ; p0[y=10|12,x=18-21]
    paddw                m9, m11
    psadbw              m10, m14
    paddw                m9, m13
    paddw                m9, m10                ; sad(p0,p1) @ dy=0,dx=0

    cmp                  hd, 8
    je .skip_16x16_thr_acc

    lea                  r9, [p0q+p0s3q*4]      ; &p0[y=12]
    movu               xm10, [r9+p0sq*2]
    movq               xm12, [r9+p0sq*2+16]
    movu               xm14, [r9+p0s3q*2+8]
    movhps             xm12, [r9+p0s3q*2]
    vinserti128         m10, [r9+p0sq*4], 1     ; p0[y=14|16,x=0-15]
    vinserti128         m12, [r9+p0sq*4+16], 1
    vpbroadcastq        m15, [r9+p0sq*8]
    vinserti128         m14, [r9+p0sq*8+8], 1   ; p0[y=18|20,x=8-23]
    vpblendd            m12, m15, 0xc0          ; p0[y=14|16,x=16-23 &
                                                ;    y=18|20,x=0-7]
    lea                  r9, [p1q+p1s3q*4]
    movu               xm11, [r9+p1sq*2]
    movq               xm13, [r9+p1sq*2+16]
    movhps             xm13, [r9+p1s3q*2]
    vinserti128         m11, [r9+p1sq*4], 1    ; p1[y=14|16,x=0-15]
    vinserti128         m13, [r9+p1sq*4+16], 1
    vpbroadcastq        m15, [r9+p1sq*8]
    vpblendd            m13, m15, 0xc0          ; p1[y=14|16,x=16-23 &
                                                ;    y=18|20,x=0-7]
    movu               xm15, [r9+p1s3q*2+8]
    vinserti128         m15, [r9+p1sq*8+8], 1   ; p1[y=18|20,x=8-23]

    palignr             m10, m12, m10, 2        ; p0[y=14|16,x=2-17]
    palignr             m11, m13, m11, 2        ; p1[y=14|16,x=2-17]
    palignr             m14, m12, 14            ; p0[y=18|20,x=6-21]
    palignr             m15, m13, 14            ; p1[y=18|20,x=6-21]
    vpblendw            m13, m12, 10011001b
    psadbw              m10, m11
    psadbw              m14, m15
    psadbw              m12, m13
    paddw                m9, m10
    paddw               m12, m14
    paddw                m9, m12

.skip_16x16_thr_acc:
    vextracti128       xm10, m9, 1
    paddd               xm9, xm10
    punpckhqdq         xm10, xm9, xm9
    paddd               xm9, xm10
    movd                r9d, xm9
    inc                 r9d
    imul                r9d, 7
    shr                 r9d, 3                  ; (sad*7+7)>>3
    cmp                 r9d, [rsp]
    jl .ret_origin
.w16_main:
    mov            word r6m, r9w

    ; r6m: best_sad
    ; m0: p0[y=2|4,x=0-15]
    ; m1: p1[y=2|4,x=0-15]
    ; m2: p0[y=2|4,x=16-23 & y=6|8,x=0-7]
    ; m3: p1[y=2|4,x=16-23 & y=6|8,x=0-7]
    ; m4: p0[y=6|8,x=8-23]
    ; m5: p1[y=6|8,x=8-23]
    ; m6: p0[y=10|12,x=0-15]
    ; m7: p1[y=10|12,x=8-23]
    ; m8: p0[y=10|12,x=16-23] & p1[y=10|12,x=0-7]
    ; m9-15: free

    call .w16_line_top
    pslldq               m9, 2
    pslldq              m10, 4
    pslldq              m11, 6
    por                 m15, m9
    por                 m10, m11
    mova     [rsp+2*mmsize], m10
    mova     [rsp+3*mmsize], m15

    ; r6m: best_sad
    ; m0: p0[y=2|4,x=0-15]
    ; m1: p1[y=2|4,x=0-15]
    ; m2: p0[y=2|4,x=16-23 & y=6|8,x=0-7]
    ; m3: p1[y=2|4,x=16-23 & y=6|8,x=0-7]
    ; m4: p0[y=6|8,x=8-23]
    ; m5: p1[y=6|8,x=8-23]
    ; m6: p0[y=10|12,x=0-15]
    ; m7: p1[y=10|12,x=8-23]
    ; m8: p0[y=10|12,x=16-23] & p1[y=10|12,x=0-7]
    ; r2: sad(p0,p1) @ dy=0,dx=[-2..-1]
    ; r3: sad(p0,p1) @ dy=0,dx=[+1..+2]

    ; compare p0[y=0,2,4,6,8] with p1[y=4,6,8,10,12], i.e. dy=-2
    lea                  r9, [p1q+p1s3q*4]
    mova [rsp+4*mmsize+0*16], xm1
    mova [rsp+4*mmsize+1*16], xm3
    vextracti128 [rsp+4*mmsize+2*16], m6, 1
    vextracti128 [rsp+4*mmsize+3*16], m8, 1
    vpbroadcastq         m9, [p0q+16]
    vinserti128         m10, m6, [p0q], 1       ; p0[y=10|0,x=0-15]
    vpblendd             m9, m8, m9, 0x30       ; p0[y=10|0,x=16-23] &
                                                ; p1[y=10|12,x=0-7]
    movq               xm11, [r9+p1sq*2+16]
    vinserti128          m1, [r9+p1sq*2], 0     ; p1[y=14|4,x=0-15]
    vpblendd             m3, m11, 0x03          ; p1[y=14|4,x=16-23 & y=6|8,x=0-7]
    palignr              m6, m4, m2, 8
    punpckhqdq           m8, m4, m8
    palignr              m4, m2, m0, 8
    punpcklqdq           m2, m9, m0
    mova                 m0, m10

    ; r6m: best_sad
    ; m0: p0[y=10|0,x=0-15]
    ; m1: p1[y=14|4,x=0-15]
    ; m2: p0[y=10|0,x=16-23 & y=2|4,x=0-7]
    ; m3: p1[y=14|4,x=16-23 & y=6|8,x=0-7]
    ; m4: p0[y=2|4,x=8-23]
    ; m5: p1[y=6|8,x=8-23]
    ; m6: p0[y=6|8,x=0-15]
    ; m7: p1[y=10|12,x=8-23]
    ; m8: p0[y=6|8,x=16-23] & p1[y=10|12,x=0-7]
    ; r2: sad(p0,p1) @ dy=0,dx=[-2..-1]
    ; r3: sad(p0,p1) @ dy=0,dx=[+1..+2]
    ; r4-5: backup data for y=2[p1] and y=12[p0] for dy=+2

    call .w16_line_top_w_origin
    mova                m12, [rsp+8*mmsize]
    pslldq              m11, 2
    pslldq              m12, 4
    pslldq              m15, 6
    por                 m10, m11
    por                 m12, m15
    por                 m10, m12
    mova     [rsp+1*mmsize], m9
    mova     [rsp+0*mmsize], m10

    ; r6m: best_sad
    ; m0: p0[y=10|0,x=0-15]
    ; m1: p1[y=14|4,x=0-15]
    ; m2: p0[y=10|0,x=16-23 & y=2|4,x=0-7]
    ; m3: p1[y=14|4,x=16-23 & y=6|8,x=0-7]
    ; m4: p0[y=2|4,x=8-23]
    ; m5: p1[y=6|8,x=8-23]
    ; m6: p0[y=6|8,x=0-15]
    ; m7: p1[y=10|12,x=8-23]
    ; m8: p0[y=6|8,x=16-23] & p1[y=10|12,x=0-7]
    ; r0: sad(p0,p1) @ dy=-2,dx=[-2..+1]
    ; r1: sad(p0,p1) @ dy=-2,dx=+2
    ; r2: sad(p0,p1) @ dy=0,dx=[-2..-1]
    ; r3: sad(p0,p1) @ dy=0,dx=[+1..+2]
    ; r4-5: backup data for y=2[p1] and y=12[p0] for dy=+2

    ; compare p0[y=4,6,8,10,12] with p1[y=0,2,4,6,8], i.e. dy=+2
    vinserti128          m1, [rsp+4*mmsize+0*16], 0 ; p1[y=2|4,x=0-15]
    vinserti128          m3, [rsp+4*mmsize+1*16], 0 ; p1[y=2|4,x=0-15 &
                                                    ;    y=6|8,x=0-7]
    lea                  r9, [p0q+p0s3q*4]      ; &p0[y=12]
    vpbroadcastq         m9, [p1q]
    vinserti128          m7, [p1q+8], 1         ; p1[y=10|0,x=8-23]
    vpblendd             m8, m9, 0xc0           ; p0[y=6|8,x=16-23] &
                                                ; p1[y=10|0,x=0-7]

    palignr              m9, m4, m2, 8
    vinserti128         m10, m4, [r9+p0sq*2+8], 0
    vinserti128          m9, [r9+p0sq*2], 0
    palignr            xm11, xm2, xm0, 8
    vinserti128         m12, m0, [rsp+4*mmsize+4*8], 1
    vinserti128         m11, [rsp+4*mmsize+5*8], 1
    mova                 m0, m6
    mova                 m6, m9
    mova                 m4, m11
    punpcklqdq           m2, m8, m12
    punpckhqdq           m8, m10, m8
    call .w16_line_top_w_origin
    mova                m12, [rsp+8*mmsize]
    pslldq              m10, 6
    pslldq              m15, 4
    pslldq               m9, 6
    pslldq              m12, 2
    por                 m15, m9
    por                 m11, m12
    por                 m11, m15
    mova     [rsp+4*mmsize], m10
    mova     [rsp+5*mmsize], m11

    ; r6m: best_sad
    ; r0: sad(p0,p1) @ dy=-2,dx=[-2..+1]
    ; r1: sad(p0,p1) @ dy=-2,dx=+2
    ; r2: sad(p0,p1) @ dy=0,dx=[-2..-1]
    ; r3: sad(p0,p1) @ dy=0,dx=[+1..+2]
    ; r4: sad(p0,p1) @ dy=+2,dx=-2
    ; r5: sad(p0,p1) @ dy=+2,dx=[11..+2]

    ; compare p0[y=1,3,5,7,9,11] with p1[y=3,5,7,9,11,13], i.e. dy=-1
    lea                  r9, [p1q+p1sq]         ; &p1[y=1]
    movu                xm1, [r9+p1sq*2]
    movq                xm3, [r9+p1sq*2+16]
    movu                xm5, [r9+p1s3q*2+8]
    movhps              xm3, [r9+p1s3q*2]
    vinserti128          m1, [r9+p1sq*4], 1     ; p1[y=3|5,x=0-15]
    vinserti128          m3, [r9+p1sq*4+16], 1
    vinserti128          m5, [r9+p1sq*8+8], 1   ; p1[y=7|9,x=8-23]
    vpbroadcastq         m9, [r9+p1sq*8]
    movu                xm7, [r9+p1s5q*2+8]
    movhps              xm8, [r9+p1s5q*2]
    vinserti128          m7, [r9+p1s3q*4+8], 1  ; p1[y=11|13,x=8-23]
    vpbroadcastq        m10, [r9+p1s3q*4]
    lea                  r9, [p0q+p0sq]         ; &p0[y=1]
    movu                xm0, [r9]
    movq                xm2, [r9+16]
    movu                xm4, [r9+p0sq*4+8]
    movhps              xm2, [r9+p0sq*4]
    vinserti128          m0, [r9+p0sq*2], 1     ; p0[y=1|3,x=0-15]
    vinserti128          m2, [r9+p0sq*2+16], 1
    vinserti128          m4, [r9+p0s3q*2+8], 1  ; p0[y=5|7,x=8-23]
    vpbroadcastq        m11, [r9+p0s3q*2]
    movu                xm6, [r9+p0sq*8]
    movlps              xm8, [r9+p0sq*8+16]
    vinserti128          m6, [r9+p0s5q*2], 1    ; p0[y=9|11,x=0-15]
    vinserti128          m8, [r9+p0s5q*2+16], 1
    vpblendd             m3, m9, 0xc0           ; p1[y=3|5,x=16-23 & y=7|9,x=0-7]
    vpblendd             m2, m11, 0xc0          ; p0[y=1|3,x=16-23 & y=5|7,x=0-7]
    vpblendd             m8, m10, 0xc0          ; p0[y=9|11,x=16-23] &
                                                ; p1[y=11|13,x=0-7]
    call .w16_line_top_w_origin
    mova                m12, [rsp+8*mmsize]
    pslldq              m10, 2
    pslldq              m11, 4
    pslldq              m12, 6
    pslldq               m9, 2
    por                 m10, m11
    por                 m12, [rsp+1*mmsize]
    por                  m9, m15
    por                 m10, m12
    por                  m9, [rsp+2*mmsize]
    mova     [rsp+1*mmsize], m10
    mova     [rsp+2*mmsize], m9

    ; r6m: best_sad
    ; m0: p0[y=1|3,x=0-15]
    ; m1: p1[y=3|5,x=0-15]
    ; m2: p0[y=1|3,x=16-23 & y=5|7,x=0-7]
    ; m3: p1[y=3|5,x=16-23 & y=7|9,x=0-7]
    ; m4: p0[y=5|7,x=8-23]
    ; m5: p1[y=7|9,x=8-23]
    ; m6: p0[y=9|11,x=0-15]
    ; m7: p1[y=11|13,x=8-23]
    ; m8: p0[y=9|11,x=16-23] & p1[y=11|13,x=0-7]
    ; r0: sad(p0,p1) @ dy=-2,dx=[-2..+1]
    ; r1: sad(p0,p1) @ dy=-2,dx=+2 & dy=-1,dx=[-2..0]
    ; r2: sad(p0,p1) @ dy=-1,dx=[+1..+2] & dy=0,dx=[-2..-1]
    ; r3: sad(p0,p1) @ dy=0,dx=[+1..+2]
    ; r4: sad(p0,p1) @ dy=+2,dx=-2
    ; r5: sad(p0,p1) @ dy=+2,dx=[11..+2]

    ; compare p0[y=3,5,7,9,11,13] with p1[y=1,3,5,7,9,11], i.e. dy=+1
    vinserti128         m11, m0, [r9+p0s3q*4], 0 ; p0[y=13|3,x=0-15]
    movq                xm9, [r9+p0s3q*4+16]
    vinserti128          m7, [p1q+p1sq+8], 1    ; p1[y=11|1,x=8-23]
    vpbroadcastq        m10, [p1q+p1sq]
    vpblendd             m9, m2, m9, 0x3        ; p0[y=13|3,x=16-23] &
                                                ; p1[y=3|5,x=0-7]
    vpblendd             m8, m10, 0xc0          ; p0[y=9|11,x=16-23] &
                                                ; p1[y=11|1,x=0-7]
    palignr              m0, m4, m2, 8
    palignr              m2, m6, m4, 8
    palignr              m4, m8, m6, 8
    mova                 m6, m11
    vpblendd             m8, m9, 00110011b
    call .w16_line_top_w_origin
    mova                m12, [rsp+8*mmsize]
    pslldq              m10, 4
    pslldq              m11, 6
    pslldq              m15, 2
    pslldq               m9, 4
    por                 m12, m15
    por                  m9, [rsp+4*mmsize]
    por                 m10, m11
    por                 m14, m12, m9
    por                 m13, m10, [rsp+3*mmsize]

    ; r6m: best_sad
    ; r0: sad(p0,p1) @ dy=-2,dx=[-2..+1]
    ; r1: sad(p0,p1) @ dy=-2,dx=+2 & dy=-1,dx=[-2..0]
    ; r2: sad(p0,p1) @ dy=-1,dx=[+1..+2] & dy=0,dx=[-2..-1]
    ; m13: sad(p0,p1) @ dy=0,dx=[+1..+2] & dy=+1,dx=[-2..-1]
    ; m14: sad(p0,p1) @ dy=+1,dx=[0..+2] & dy=+2,dx=-2
    ; r5: sad(p0,p1) @ dy=+2,dx=[11..+2]

    cmp                  hd, 8
    jg .bottom_16x16

    mova                m15, [rsp+2*mmsize]
    jmp .skip_8x16_main

.bottom_16x16:
    lea                 p0q, [p0q+p0s3q*4]      ; &p0[y=12]
    lea                 p1q, [p1q+p1s3q*4]      ; &p1[y=12]

    ; compare p0[y=12,14,16,18] with p1[y=16,18,20,22], i.e. dy=-2
    movu                xm0, [p0q]
    movq                xm2, [p0q+16]
    movu                xm4, [p0q+p0sq*4+8]
    movhps              xm2, [p0q+p0sq*4]
    vinserti128          m0, [p0q+p0sq*2], 1    ; p0[y=12|14,x=0-15]
    vinserti128          m2, [p0q+p0sq*2+16], 1
    vpbroadcastq         m6, [p0q+p0s3q*2]
    vinserti128          m4, [p0q+p0s3q*2+8], 1 ; p0[y=16|18,x=8-23]
    movu                xm1, [p1q+p1sq*4]
    movq                xm3, [p1q+p1sq*4+16]
    movu                xm5, [p1q+p1sq*8+8]
    movhps              xm3, [p1q+p1sq*8]
    vinserti128          m1, [p1q+p1s3q*2], 1   ; p1[y=16|18,x=0-15]
    vinserti128          m3, [p1q+p1s3q*2+16], 1
    vpbroadcastq         m7, [p1q+p1s5q*2]
    vinserti128          m5, [p1q+p1s5q*2+8], 1 ; p1[y=20|22,x=8-23]
    vpblendd             m2, m6, 0xc0           ; p0[y=12|14,x=16-23 &
                                                ;    y=16|18,x=0-7]
    vpblendd             m3, m7, 0xc0           ; p1[y=16|18,x=16-23 &
                                                ;    y=20|22,x=0-7]
    call .w16_line_bottom_w_origin
    pslldq              m12, 2
    pslldq              m15, 4
    pslldq              m11, 6
    por                 m10, m12
    por                 m15, m11
    por                 m10, m15
    paddw                m9, [rsp+1*mmsize]
    paddw               m10, [rsp+0*mmsize]
    mova     [rsp+1*mmsize], m9
    mova     [rsp+0*mmsize], m10

    ; word r6m: best_sad
    ; m0: p0[y=12|14,x=0-15]
    ; m1: p1[y=16|18,x=0-15]
    ; m2: p0[y=12|14,x=16-23 & y=16|18,x=0-7]
    ; m3: p1[y=16|18,x=16-23 & y=20|22,x=0-7]
    ; m4: p0[y=16|18,x=8-23]
    ; m5: p1[y=20|22,x=8-23]
    ; r0: sad(p0,p1) @ dy=-2,dx=[-2..+1] (full)
    ; r1: sad(p0,p1) @ dy=-2,dx=+2 (full) & dy=-1,dx=[-2..0] (top)
    ; r2: sad(p0,p1) @ dy=-1,dx=[+1..+2] & dy=0,dx=[-2..-1] (top)
    ; m13: sad(p0,p1) @ dy=0,dx=[+1..+2] & dy=+1,dx=[-2..-1] (top)
    ; m14: sad(p0,p1) @ dy=+1,dx=[0..+2] & dy=+2,dx=-2 (top)
    ; r5: sad(p0,p1) @ dy=+2,dx=[-1..+2] (top)

    ; compare p0[y=14,16,18,20] with p1[y=14,16,18,20], i.e. dy=0
    vinserti128          m6, m0, [p0q+8*p0sq], 0 ; p0[y=20|14,x=0-15]
    vinserti128          m5, [p1q+2*p1sq+8], 1  ; p1[y=20|14,x=8-23]
    movq                xm7, [p0q+8*p0sq+16]
    vpbroadcastq         m8, [p1q+2*p1sq]
    vpblendd             m7, m2, m7, 0x3        ; p0[y=20|14,x=16-23 &
                                                ;    y=16|18,x=0-7]
    vpblendd             m3, m8, 0xc0           ; p1[y=16|18,x=16-23 &
                                                ;    y=20|14,x=0-7]
    palignr              m0, m4, m2, 8          ; p0[y=16|18,y=0-15]
    palignr              m2, m6, m4, 8          ; p0[y=16|18,x=16-23 &
                                                ;    y=20|14,x=0-7]
    palignr              m4, m7, m6, 8          ; p0[y=20|14,x=8-23]
    call .w16_line_bottom
    pslldq              m10, 4
    pslldq              m12, 6
    pslldq               m9, 2
    por                 m10, m12
    por                 m11, m9
    paddw               m10, [rsp+2*mmsize]
    paddw               m13, m11
    mova     [rsp+2*mmsize], m10

    ; compare p0[y=16,18,20,22] with p1[y=12,14,16,18], i.e. dy=+2
    vinserti128          m6, m4, [p0q+p0s5q*2+8], 1 ; p0[y=20|22,x=8-23]
    vinserti128          m5, [p1q+8], 0         ; p1[y=12|14,x=8-23]
    vpbroadcastq         m7, [p0q+p0s5q*2]
    movhps              xm8, [p1q]
    vpblendd             m7, m2, m7, 0xc0       ; p0[y=16|18,x=16-23 &
                                                ;    y=20|22,x=0-7]
    vpblendd             m3, m8, 0xc            ; p1[y=16|18,x=16-23 &
                                                ;    y=12|14,x=0-7]
    palignr              m4, m7, m0, 8          ; p0[y=16|18,x=8-23]
    palignr              m2, m0, m6, 8          ; p0[y=20|22,x=16-23 &
                                                ;    y=16|18,x=0-7]
    palignr              m0, m6, m7, 8          ; p0[y=20|22,x=0-15]
    call .w16_line_bottom_w_origin
    pslldq              m15, 2
    pslldq              m11, 4
    pslldq               m9, 6
    por                 m12, m15
    por                 m11, m9
    pslldq              m10, 6
    por                 m12, m11
    paddw               m14, m10
    paddw               m12, [rsp+5*mmsize]
    mova     [rsp+5*mmsize], m12

    ; word r6m: best_sad
    ; r0: sad(p0,p1) @ dy=-2,dx=[-2..+1] (full)
    ; r1: sad(p0,p1) @ dy=-2,dx=+2 (full) & dy=-1,dx=[-2..0] (top)
    ; r2: sad(p0,p1) @ dy=-1,dx=[+1..+2] (top) & dy=0,dx=[-2..-1] (full)
    ; m13: sad(p0,p1) @ dy=0,dx=[+1..+2] (full) & dy=+1,dx=[-2..-1] (top)
    ; m14: sad(p0,p1) @ dy=+1,dx=[0..+2] (top) & dy=+2,dx=-2 (full)
    ; r5: sad(p0,p1) @ dy=+2,dx=[-1..+2] (full)
    ; m0-12,15: free

    ; compare p0[y=13|15|17|19] with p1[y=15|17|19|21], i.e. dy=-1
    add                 p0q, p0sq               ; &p0[y=13]
    add                 p1q, p1sq               ; &p1[y=13]
    movu                xm0, [p0q]
    movq                xm2, [p0q+16]
    movu                xm4, [p0q+p0sq*4+8]
    movhps              xm2, [p0q+p0sq*4]
    vinserti128          m0, [p0q+p0sq*2], 1    ; p0[y=13|15,x=0-15]
    vinserti128          m2, [p0q+p0sq*2+16], 1
    vpbroadcastq         m6, [p0q+p0s3q*2]
    vinserti128          m4, [p0q+p0s3q*2+8], 1 ; p0[y=17|19,x=8-23]
    movu                xm1, [p1q+p1sq*2]
    movq                xm3, [p1q+p1sq*2+16]
    movu                xm5, [p1q+p1s3q*2+8]
    movhps              xm3, [p1q+p1s3q*2]
    vinserti128          m1, [p1q+p1sq*4], 1    ; p1[y=15|17,x=0-15]
    vinserti128          m3, [p1q+p1sq*4+16], 1
    vpbroadcastq         m7, [p1q+p1sq*8]
    vinserti128          m5, [p1q+p1sq*8+8], 1  ; p1[y=19|21,x=8-23]
    vpblendd             m2, m6, 0xc0           ; p0[y=13|15,x=16-23 &
                                                ;    y=17|19,x=0-7]
    vpblendd             m3, m7, 0xc0           ; p1[y=15|17,x=16-23 &
                                                ;    y=19|21,x=0-7]
    call .w16_line_bottom_w_origin
    pslldq              m10, 2
    pslldq              m12, 4
    pslldq              m15, 6
    pslldq               m9, 2
    por                 m10, m12
    por                 m11, m9
    por                 m10, m15
    paddw               m11, [rsp+2*mmsize]
    paddw               m10, [rsp+1*mmsize]
    mova     [rsp+2*mmsize], m11
    mova     [rsp+1*mmsize], m10

    ; compare p0[y=15|17|19|21] with p1[y=13|15|17|19], i.e. dy=+1
    vinserti128          m6, m0, [p0q+p0sq*8], 0 ; p0[y=21|15,x=0-15]
    vinserti128          m5, [p1q+8], 1         ; p1[y=19|13,x=8-23]
    movq                xm7, [p0q+p0sq*8+16]
    vpbroadcastq         m8, [p1q]
    vpblendd             m7, m2, m7, 0x3        ; p0[y=21|15,x=16-23 &
                                                ;    y=17|19,x=0-7]
    vpblendd             m3, m8, 0xc0           ; p1[y=15|17,x=16-23 &
                                                ;    y=19|13,x=0-7]
    palignr              m0, m4, m7, 8          ; p0[y=17|19,x=0-15]
    palignr              m2, m6, m4, 8          ; p0[y=17|19,x=16-23 &
                                                ;    y=21|15,x=0-7]
    palignr              m4, m7, m6, 8          ; p0[y=21|15,x=8-23]
    call .w16_line_bottom_w_origin
    pslldq              m11, 2
    pslldq               m9, 4
    pslldq              m10, 4
    pslldq              m12, 6
    por                 m15, m11
    por                 m10, m12
    por                 m15, m9
    paddw               m13, m10
    paddw               m14, m15
    mova                m15, [rsp+2*mmsize]

    jmp .skip_8x16_main

.w16_line_top_w_origin:
    ; dx=0
    palignr              m9, m2, m0, 2          ; p0[y=2|4,x=2-17]
    palignr             m10, m3, m1, 2          ; p1[y=2|4,x=2-17]
    palignr             m11, m4, m2, 14         ; p0[y=6|8,x=6-21]
    palignr             m12, m5, m3, 14         ; p1[y=6|8,x=6-21]
    palignr             m13, m8, m6, 2          ; p0[y=10|12,x=2-17]
    palignr             m14, m7, m8, 10         ; p1[y=10|12,x=2-17]
    psadbw               m9, m10
    psadbw              m11, m12
    psadbw              m13, m14
    psrldq              m10, m2, 2              ; p0[y=2|4,x=18-23 & y=6|8,x=0-7], 2x0
    palignr             m12, m8, m7, 10         ; p1[y=10|12,x=18-23] &
                                                ; p0[y=10|12,x=16-23] & 2x?
    psrldq              m14, m3, 2              ; p1[y=2|4,x=18-23 & y=6|8,x=0-7], 2x0
    shufps              m10, m12, q2220         ; p0[y=2|4,x=18-21 & y=6|8,x=2-5 &
                                                ;    y=10|12,x=18-21 [2x]]
    shufps              m14, m12, q2020         ; p1[y=2|4,x=18-21 & y=6|8,x=2-5 &
                                                ;    y=10|12,x=18-21] &
                                                ; p0[y=10|12,x=18-21]
    paddw                m9, m11
    psadbw              m10, m14
    paddw                m9, m13
    paddw                m9, m10                ; sad(p0,p1) @ dx=0
    mova [rsp+8*mmsize+gprsize], m9
.w16_line_top:
    ; dy=0,dx=-1
    palignr              m9, m2, m0, 1          ; p0[y=2|4,x=1-16]
    palignr             m10, m3, m1, 3          ; p1[y=2|4,x=3-18]
    palignr             m11, m4, m2, 13         ; p0[y=6|8,x=5-20]
    palignr             m12, m5, m3, 15         ; p1[y=6|8,x=7-22]
    palignr             m13, m8, m6, 1          ; p0[y=10|12,x=1-16]
    palignr             m14, m7, m8, 11         ; p1[y=10|12,x=3-18]
    psadbw               m9, m10
    psadbw              m11, m12
    psadbw              m13, m14
    psrldq              m10, m2, 1              ; p0[y=2|4,x=17-23 & y=6|8,x=0-7], 1x0
    pslldq              m12, m8, 11             ; 11x0,p0[y=10|12,x=16-20]
    psrldq              m14, m3, 3              ; p1[y=2|4,x=19-23 & y=6|8,x=0-7], 3x0
    psrldq              m15, m7, 11             ; p1[y=10|12,x=19-23],11x0
    shufps              m10, m12, q0320         ; p0[y=2|4,x=17-20 & y=6|8,x=1-4 &
                                                ;    y=10|12,x=17-20], 4x0
    shufps              m14, m15, q3020         ; p1[y=2|4,x=19-22 & y=6|8,x=3-6 &
                                                ;    y=10|12,x=19-22], 4x0
    paddw                m9, m11
    psadbw              m10, m14
    paddw                m9, m13
    paddw                m9, m10                ; sad(p0,p1) @ dx=-1
    mova [rsp+7*mmsize+gprsize], m9

    ; dx=-2
    ; m0 is good as-is                          ; p0[y=2|4,x=0-15]
    palignr             m10, m3, m1, 4          ; p1[y=2|4,x=4-19]
    palignr             m11, m4, m2, 12         ; p0[y=6|8,x=4-19]
    ; m5 is good as-is                          ; p1[y=6|8,x=8-23]
    ; m6 is good as-is                          ; p0[y=10|12,x=0-15]
    palignr             m13, m7, m8, 12         ; p1[y=10|12,x=4-19]
    punpckhqdq          m14, m7, m8             ; p1[y=10|12,x=16-23 & x=0-7]
    shufps              m12, m2, m8, q3020      ; p0[y=2|4,x=16-19 & y=6|8,x=0-3 &
                                                ;    y=10|12,x=16-19] & p1[y=10|12,x=4-7]
    shufps              m14, m3, m14, q3131     ; p1[y=2|4,x=20-23 & y=6|8,x=4-7 &
                                                ;    y=10|12,x=20-23 & y=10|12,x=4-7]
    psadbw               m9, m0, m10
    psadbw              m11, m5
    psadbw              m13, m6, m13
    psadbw              m14, m12
    paddw                m9, m11
    paddw               m13, m14
    paddw                m9, m13                ; sad(p0,p1) @ dx=-2
    mova [rsp+6*mmsize+gprsize], m9

    ; dx=+1
    palignr              m9, m2, m0, 3          ; p0[y=2|4,x=3-18]
    palignr             m10, m3, m1, 1          ; p1[y=2|4,x=1-16]
    palignr             m11, m4, m2, 15         ; p0[y=6|8,x=7-22]
    palignr             m12, m5, m3, 13         ; p1[y=6|8,x=5-20]
    palignr             m13, m8, m6, 3          ; p0[y=10|12,x=3-18]
    palignr             m14, m7, m8, 9          ; p1[y=10|12,x=1-16]
    psadbw               m9, m10
    psadbw              m11, m12
    psadbw              m13, m14
    psrldq              m10, m2, 3              ; p0[y=2|4,x=19-23 & y=6|8,x=0-7], 3x0
    pslldq              m12, m8, 9              ; 9x0,p0[y=10|12,x=16-22]
    psrldq              m14, m3, 1              ; p1[y=2|4,x=17-23 & y=6|8,x=0-7], 1x0
    psrldq              m15, m7, 9              ; p1[y=10|12,x=17-23],9x0
    shufps              m10, m12, q0320         ; p0[y=2|4,x=19-22 & y=6|8,x=3-6 &
                                                ;    y=10|12,x=19-22], 4x0
    shufps              m14, m15, q3020         ; p1[y=2|4,x=17-20 & y=6|8,x=1-4 &
                                                ;    y=10|12,x=17-20], 4x0
    paddw                m9, m11
    psadbw              m10, m14
    paddw                m9, m13
    paddw               m15, m9, m10            ; sad(p0,p1) @ dx=+1

    ; dx=+2
    palignr              m9, m2, m0, 4          ; p0[y=2|4,x=4-19]
    ; m1 is good as-is                          ; p1[y=2|4,x=0-15]
    ; m4 is good as-is                          ; p0[y=6|8,x=8-23]
    palignr             m11, m5, m3, 12         ; p1[y=6|8,x=4-19]
    palignr             m13, m8, m6, 4          ; p0[y=10|12,x=4-19]
    palignr             m14, m7, m8, 8          ; p1[y=10|12,x=0-15]
    punpckhqdq          m12, m7, m8             ; p1[y=10|12,x=16-23] &
                                                ; p1[y=10|12,x=0-7]
    shufps              m10, m2, m8, q3131      ; p0[y=2|4,x=20-23 & y=6|8,x=4-7 &
                                                ;    y=10|12,x=20-23] & p1[y=10|12,x=4-7]
    shufps              m12, m3, m12, q3020     ; p1[y=2|4,x=16-19 & y=6|8,x=0-3 &
                                                ;    y=10|12,x=16-19 & y=10|12,x=4-7]
    psadbw               m9, m1
    psadbw              m11, m4, m11
    psadbw              m13, m14
    psadbw              m10, m12
    paddw                m9, m11
    paddw               m13, m10
    paddw                m9, m13                ; sad(p0,p1) @ dx=+2

    mova                m10, [rsp+6*mmsize+gprsize]
    mova                m11, [rsp+7*mmsize+gprsize]
    ret
.w16_line_bottom_w_origin:
    ; dy=-2,dx=0
    palignr              m6, m2, m0, 2          ; p0[y=12|14,x=2-17]
    palignr              m7, m3, m1, 2          ; p1[y=16|18,x=2-17]
    palignr              m8, m4, m2, 14         ; p0[y=16|18,x=6-21]
    palignr              m9, m5, m3, 14         ; p1[y=20|22,x=6-21]
    vpblendw            m10, m2, m3, 01100110b
    psadbw               m6, m7
    psadbw               m8, m9
    psadbw              m10, m2
    paddw                m6, m8
    paddw               m15, m6, m10
    ; fall-through
.w16_line_bottom:
    ; dy=-2,dx=-1
    palignr              m6, m2, m0, 1          ; p0[y=12|14,x=1-16]
    palignr              m7, m3, m1, 3          ; p1[y=16|18,x=3-18]
    palignr              m8, m4, m2, 13         ; p0[y=16|18,x=5-20]
    palignr              m9, m5, m3, 15         ; p1[y=20|22,x=7-22]
    psrldq              m10, m2, 1
    psrldq              m11, m3, 3
    vpblendw            m10, m11, 11001100b
    psadbw               m6, m7
    psadbw               m8, m9
    psadbw              m10, m11
    paddw                m6, m8
    paddw               m12, m6, m10

    ; dy=-2,dx=+1
    palignr              m6, m2, m0, 3          ; p0[y=12|14,x=3-18]
    palignr              m7, m3, m1, 1          ; p1[y=16|18,x=1-16]
    palignr              m8, m4, m2, 15         ; p0[y=16|18,x=7-22]
    palignr              m9, m5, m3, 13         ; p1[y=20|22,x=5-20]
    psrldq              m10, m2, 3
    psrldq              m11, m3, 1
    vpblendw            m10, m11, 11001100b
    psadbw               m6, m7
    psadbw               m8, m9
    psadbw              m10, m11
    paddw                m6, m8
    paddw               m11, m6, m10

    ; dy=-2,dx=-2
    ; m0 is good as-is                          ; p0[y=12|14,x=0-15]
    palignr              m6, m3, m1, 4          ; p1[y=16|18,x=4-19]
    palignr              m8, m4, m2, 12         ; p0[y=16|18,x=4-19]
    ; m5 is good as-is                          ; p1[y=20|22,x=8-23]
    shufps              m10, m2, m3, q3120
    pshufd               m9, m3, q3131
    psadbw               m6, m0
    psadbw               m8, m5
    psadbw              m10, m9
    paddw                m6, m8
    paddw               m10, m6

    ; dy=-2,dx=+2
    palignr              m6, m2, m0, 4          ; p0[y=12|14,x=4-19]
    ; m1 is good as-is                          ; p1[y=16|18,x=0-15]
    ; m4 is good as-is                          ; p0[y=16|18,x=8-23]
    palignr              m8, m5, m3, 12         ; p1[y=20|22,x=4-19]
    shufps               m7, m3, m2, q3120
    pshufd               m9, m2, q3131
    psadbw               m6, m1
    psadbw               m8, m4
    psadbw               m9, m7
    paddw                m6, m8
    paddw                m9, m6
    ret

.w8:
    imul                r9d, 24
    mov               [rsp], r9d
    movu                xm0, [p0q+2*p0sq]
    movu                xm1, [p1q+2*p1sq]
    vinserti128          m0, [p0q+4*p0sq], 1    ; p0[y=2|4]
    vinserti128          m1, [p1q+4*p1sq], 1    ; p1[y=2|4]
    movu                xm2, [p0q+2*p0s3q]
    movu                xm3, [p1q+2*p1s3q]
    vinserti128          m2, [p0q+8*p0sq], 1    ; p0[y=6|8]
    vinserti128          m3, [p1q+8*p1sq], 1    ; p1[y=6|8]
    movu                xm4, [p0q+2*p0s5q]
    movu                xm5, [p1q+2*p1s5q]
    vinserti128          m4, [p0q+4*p0s3q], 1   ; p0[y=10|12]
    vinserti128          m5, [p1q+4*p1s3q], 1   ; p1[y=10|12]
    pcmpeqw              m6, m6
    psrldq               m6, 4                  ; 12x255,4x0
    pslldq               m7, m6, 2              ; 2x0,12x255,2x0
    pslldq               m6, 1                  ; 1x0,12x255,3x0
    mov                 r9d, -1
    cmp            byte r6m, 0
    je .w8_main
    vpblendvb            m9, m1, m0, m7
    vpblendvb           m10, m3, m2, m7
    vpblendvb           m11, m5, m4, m7         ; p1[2x],p0[12x],p1[2x]
    psadbw               m9, m1
    psadbw              m10, m3
    psadbw              m11, m5                 ; p0/1 sad on inner 12px
    paddw                m9, m10
    paddw                m9, m11
    cmp                  hd, 8
    je .skip_8x16_thr_acc
    lea                  r9, [p0q+4*p0s3q]
    movu               xm10, [r9+2*p0sq]
    movu               xm12, [r9+2*p0s3q]
    vinserti128         m10, [r9+4*p0sq], 1
    vinserti128         m12, [r9+8*p0sq], 1
    lea                  r9, [p1q+4*p1s3q]
    movu               xm11, [r9+2*p1sq]
    movu               xm13, [r9+2*p1s3q]
    vinserti128         m11, [r9+4*p1sq], 1
    vinserti128         m13, [r9+8*p1sq], 1
    vpblendvb           m10, m11, m10, m7
    vpblendvb           m12, m13, m12, m7
    psadbw              m10, m11
    psadbw              m12, m13                ; p0/1 sad on inner 12px
    paddw                m9, m10
    paddw                m9, m12
.skip_8x16_thr_acc:
    vextracti128       xm10, m9, 1
    paddd               xm9, xm10
    punpckhqdq         xm10, xm9, xm9
    paddd               xm9, xm10
    movd                r9d, xm9
    inc                 r9d
    imul                r9d, 7
    shr                 r9d, 3                  ; (sad*7+7)>>3
    cmp                 r9d, [rsp]
    jl .ret_origin

    ; r9d: best_sad
    ; m0/2/4: p0[y=2|4,6|8,10|12]
    ; m1/3/5: p1[y=2|4,6|8,10|12]
    ; m6: 1x0,12x255,3x0[both lanes]
    ; m7: 12x255,4x0[both lanes]
    ; m8: 2x0,12x255,2x0[both lanes]
    ; m9-15: free

.w8_main:
    mov                 r6m, r9w

    ; compare p0[y=2,4,6,8,10,12] with p1[y=2,4,6,8,10,12], i.e. dy=+0
    call .w8_line_top
    pslldq               m9, 4
    pslldq              m10, 6
    pslldq              m12, 2
    por                  m9, m10
    por                 m11, m12
    mova     [rsp+2*mmsize], m9
    mova     [rsp+3*mmsize], m11

    ; compare p0[y=0,2,4,6,8,10] with p1[y=4,6,8,10,12,14], i.e. dy=-2
    lea                  r9, [p1s5q+p1sq*2]     ; p1s7q
    vpblendd            m15, m1, m4, 11110000b
    vinserti128         m10, m4, [p0q], 1       ; p0[y=10|0]
    vinserti128          m1, [p1q+r9*2], 0      ; p1[y=14|4]
    mova                 m4, m2
    mova                 m2, m0
    mova                 m0, m10
    call .w8_line_top_w_origin
    pslldq              m10, 2
    pslldq               m8, 4
    pslldq              m11, 6
    por                  m9, m10
    por                  m8, m11
    mova     [rsp+1*mmsize], m12
    por                  m8, m9
    mova     [rsp+0*mmsize], m8

    ; compare p0[y=4,6,8,10,12,14] with p1[y=0,2,4,6,8,10], i.e. dy=+2
    lea                  r9, [p0s5q+p0sq*2]     ; p0s7q
    vinserti128          m5, [p1q], 1           ; p1[y=10|0]
    vpblendd             m1, m15, 00001111b     ; p1[y=2|4]
    vinserti128         m10, m2, [p0q+r9*2], 0  ; p0[y=14|4]
    vpblendd             m2, m0, m15, 11110000b ; p0[y=10|12]
    mova                 m0, m4
    mova                 m4, m10
    call .w8_line_top_w_origin
    pslldq               m8, 2
    pslldq              m11, 4
    pslldq              m12, 6
    pslldq               m9, 6
    por                 m10, m8
    por                 m11, m12
    mova     [rsp+4*mmsize], m9
    por                 m10, m11
    mova     [rsp+5*mmsize], m10

    ; r6m: best_sad
    ; m0-5: free
    ; m6: 1x0,12x255,3x0[both lanes]
    ; m7: 2x0,12x255,2x0[both lanes]
    ; m9-15: free
    ; r0: sad(p0,p1) @ dy=-2,dx=[-2..+1]
    ; r1: sad(p0,p1) @ dy=-2,dx=+2
    ; r2: sad(p0,p1) @ dy=+0,dx=[-2..-1]
    ; r3: sad(p0,p1) @ dy=+0,dx=[+1..+2]
    ; r4: sad(p0,p1) @ dy=+2,dx=-2
    ; r5: sad(p0,p1) @ dy=+2,dx=[-1..+2]

    ; compare p0[y=1,3,5,7,9,11] with p1[y=3,5,7,9,11,13], i.e. dy=-1
    lea                  r9, [p1q+p1s3q]
    movu                xm1, [r9]
    movu                xm3, [r9+p1sq*4]
    movu                xm5, [r9+p1sq*8]
    vinserti128          m1, [r9+p1sq*2], 1     ; p1[y=3|5]
    vinserti128          m3, [r9+p1s3q*2], 1    ; p1[y=7|9]
    vinserti128          m5, [r9+p1s5q*2], 1    ; p1[y=11|13]
    lea                  r9, [p0q+p0s3q]
    movu                xm0, [p0q+p0sq]
    movu                xm2, [r9+p0sq*2]
    movu                xm4, [r9+p0s3q*2]
    vinserti128          m0, [r9], 1            ; p0[y=1|3]
    vinserti128          m2, [r9+p0sq*4], 1     ; p0[y=5|7]
    vinserti128          m4, [r9+p0sq*8], 1     ; p0[y=9|11]
    call .w8_line_top_w_origin
    pslldq               m9, 2
    pslldq              m10, 4
    pslldq               m8, 6
    pslldq              m12, 2
    por                  m9, [rsp+1*mmsize]
    por                 m10, m8
    por                 m11, m12
    por                  m9, m10
    por                 m15, m11, [rsp+2*mmsize]
    mova     [rsp+1*mmsize], m9

    ; r6m: best_sad
    ; m0/2/4: p0[y=1|3,5|7,9|11]
    ; m1/3/5: p1[y=3|5,7|9,11|13]
    ; m6: 1x0,12x255,3x0[both lanes]
    ; m7: 2x0,12x255,2x0[both lanes]
    ; m9-14: free
    ; r0: sad(p0,p1) @ dy=-2,dx=[-2..+1]
    ; r1: sad(p0,p1) @ dy=-2,dx=+2 & dy=-1,dx=[-2..0]
    ; m15: sad(p0,p1) @ dy=-1,dx=[+1..+2] & dy=+0,dx=[-2..-1]
    ; r3: sad(p0,p1) @ dy=+0,dx=[+1..+2]
    ; r4: sad(p0,p1) @ dy=+2,dx=-2
    ; r5: sad(p0,p1) @ dy=+2,dx=[-1..+2]

    ; compare p0[y=3,5,7,9,11,13] with p1[y=1,3,5,7,9,11], i.e. dy=+1
    vinserti128         m10, m0, [r9+p0s5q*2], 0 ; p0[y=13|3]
    vinserti128          m5, [p1q+p1sq], 1      ; p1[y=11|1]
    mova                 m0, m2
    mova                 m2, m4
    mova                 m4, m10
    call .w8_line_top_w_origin
    pslldq               m9, 4
    pslldq              m10, 6
    pslldq              m11, 2
    pslldq              m12, 4
    por                  m9, m10
    por                  m8, m11
    por                 m12, [rsp+4*mmsize]
    por                 m13, m9, [rsp+3*mmsize]
    por                 m14, m8, m12

    ; r6m: best_sad
    ; m0-5: free
    ; m6: 1x0,12x255,3x0[both lanes]
    ; m7: 2x0,12x255,2x0[both lanes]
    ; m9-12: free
    ; r0: sad(p0,p1) @ dy=-2,dx=[-2..+1]
    ; r1: sad(p0,p1) @ dy=-2,dx=+2 & dy=-1,dx=[-2..0]
    ; m15: sad(p0,p1) @ dy=-1,dx=[+1..+2] & dy=+0,dx=[-2..-1]
    ; m13: sad(p0,p1) @ dy=+0,dx=[+1..+2] & dy=+1,dx=[-2..-1]
    ; m14: sad(p0,p1) @ dy=-1,dx=[0..+2] & dy=+2,dx=-2
    ; r5: sad(p0,p1) @ dy=+2,dx=[-1..+2]

    cmp                  hd, 8
    je .skip_8x16_main

    ; compare p0[y=14|16,18|12] with p1[y=18|20,22|16], i.e. dy=-2
    lea                 p0q, [p0q+p0s3q*4]      ; &p0[y=12]
    lea                 p1q, [p1q+p1s3q*4]      ; &p1[y=12]
    movu                xm0, [p0q+p0sq*2]
    movu                xm3, [p1q+p1s5q*2]
    vinserti128          m0, [p0q+p0sq*4], 1    ; p0[y=14|16]
    vinserti128          m3, [p1q+p1sq*4], 1    ; p1[y=22|16]
    movu                xm2, [p0q+p0s3q*2]
    movu                xm1, [p1q+p1s3q*2]
    vinserti128          m2, [p0q], 1           ; p0[y=18|12]
    vinserti128          m1, [p1q+p1sq*8], 1    ; p1[y=18|20]
    call .w8_line_bottom_w_origin
    pslldq               m8, 2
    pslldq               m4, 4
    pslldq               m9, 6
    por                  m5, m8
    por                  m4, m9
    por                  m5, m4
    paddw               m10, [rsp+1*mmsize]
    paddw                m5, [rsp+0*mmsize]
    mova     [rsp+1*mmsize], m10
    mova     [rsp+0*mmsize], m5

    ; compare p0[y=14|16,18|20] with p1[y=14|16,18|20], i.e. dy=0
    vinserti128         m10, m2, [p0q+p0sq*8], 1 ; p0[y=18|20]
    vinserti128          m3, [p1q+p1sq*2], 0    ; p1[y=14|16]
    mova                 m2, m0
    mova                 m0, m10
    call .w8_line_bottom
    pslldq               m5, 4
    pslldq               m8, 6
    pslldq              m10, 2
    por                  m5, m8
    por                  m9, m10
    paddw               m15, m5
    paddw               m13, m9

    ; compare p0[y=18|20,22|16] with p1[y=14|16,18|12], i.e. dy=+2
    vinserti128         m10, m2, [p0q+p0s5q*2], 0 ; p0[y=22|16]
    vinserti128          m1, [p1q], 1           ; p1[y=18|12]
    mova                 m2, m0
    mova                 m0, m10
    call .w8_line_bottom_w_origin
    pslldq               m5, 6
    pslldq               m4, 2
    pslldq               m9, 4
    pslldq              m10, 6
    por                  m8, m4
    por                  m9, m10
    por                  m8, m9
    paddw               m14, m5
    paddw               m12, m8, [rsp+5*mmsize]

    ; compare p0[y=13|15,17|19] with p1[y=15|17,19|21], i.e. dy=-1
    lea                  r9, [p1q+p1s3q*2]
    movu                xm1, [p1q+p1s3q]
    movu                xm3, [r9+p1sq]
    vinserti128          m1, [p1q+p1s5q], 1     ; p1[y=15|17]
    vinserti128          m3, [r9+p1s3q], 1      ; p1[y=19|21]
    lea                  r9, [p0q+p0s3q*2]
    movu                xm0, [p0q+p0sq]
    movu                xm2, [p0q+p0s5q]
    vinserti128          m0, [p0q+p0s3q], 1     ; p0[y=13|15]
    vinserti128          m2, [r9+p0sq], 1       ; p0[y=15|17]
    call .w8_line_bottom_w_origin
    pslldq               m5, 2
    pslldq               m8, 4
    pslldq               m4, 6
    pslldq              m10, 2
    por                  m5, m8
    por                  m9, m10
    por                  m5, m4
    paddw                m5, [rsp+1*mmsize]
    paddw               m15, m9
    mova     [rsp+1*mmsize], m5

    ; compare p0[y=21|15,17|19] with p1[y=19|13,15|17], i.e. dy=+1
    vinserti128         m10, m0, [r9+p0s3q], 0  ; p0[y=21|15]
    vinserti128          m3, [p1q+p1sq], 1      ; p1[y=19|13]
    mova                 m0, m2
    mova                 m2, m10
    call .w8_line_bottom_w_origin
    pslldq               m9, 2
    pslldq              m10, 4
    pslldq               m5, 4
    pslldq               m8, 6
    por                  m4, m9
    por                  m5, m8
    por                  m4, m10
    paddw               m13, m5
    paddw               m14, m4
    jmp .skip_8x16_main_noreload

.skip_8x16_main:
    mova                m12, [rsp+5*mmsize]
.skip_8x16_main_noreload:
    mova                m10, [rsp+0*mmsize]
    mova                m11, [rsp+1*mmsize]

    ; aggregate
    punpckhqdq           m0, m10, m11
    punpcklqdq          m10, m11
    paddw                m0, m10                ; sad(p0,p1) @ dy=-2,dx=[-2..+2] &
                                                ;              dy=-1,dx=[-2..0]
    punpckhqdq           m1, m15, m13
    punpcklqdq          m15, m13
    paddw                m1, m15                ; sad(p0,p1) @ dy=-1,dx=[+1..+2] &
                                                ;              dy=0,dx=[-2..+2] &
                                                ;              dy=+1,dx=[-2..-1]
    punpckhqdq           m2, m14, m12
    punpcklqdq          m14, m12
    paddw                m2, m14                ; sad(p0,p1) @ dy=+1,dx=[0..+2] &
                                                ;              dy=+2,dx=[-2..+2]

    ; r6m: best_sad
    ; m9: sad(p0,p1) @ dy=-1,dx=[+1..+2] & dy=0,dx=[-2..+2] & dy=+1,dx=[-2..-1]
    ; m11: sad(p0,p1) @ dy=-2,dx=[-2..+2] & dy=-1,dx=[-2..0]
    ; m12: sad(p0,p1) @ dy=+1,dx=[0..+2] & dy=+2,dx=[-2..+2]

    vextracti128        xm3, m0, 1
    vextracti128        xm4, m1, 1
    vextracti128        xm5, m2, 1
    paddw               xm0, xm3
    paddw               xm1, xm4
    paddw               xm2, xm5
    REPX  {phminposuw x, x}, xm0, xm1, xm2

    movd                r0d, xm0
    movd                r1d, xm1
    movd                r2d, xm2
    or                  r1d, 0x80000
    or                  r2d, 0x100000
    cmp                 r1w, r0w
    cmovb                r0, r1
    cmp                 r2w, r0w
    cmovb                r0, r2                 ; best_sad, excluding origin

    cmp                 r0w, word r6m
    jnb .ret_origin
    mov                  r1, r7m
    shr                  r0, 16
    lea                  r2, [sadrefinemv_idx2off]
    mov                 r0w, [r2+r0*2]
    mov           word [r1], r0w

    RET
.w8_line_top_w_origin:
    ; dx=0
    vpblendvb            m8, m0, m1, m7
    vpblendvb            m9, m2, m3, m7
    vpblendvb           m10, m4, m5, m7         ; mask out left 2px & right 2px
    psadbw               m8, m0
    psadbw               m9, m2
    psadbw              m10, m4
    paddw                m8, m9
    paddw                m8, m10                ; sad(p0,p1) @ dx=0
    ; fall-through
.w8_line_top:
    ; dx=-2
    psrldq               m9, m1, 4              ; p1[y=2|4,x=4-15]
    psrldq              m10, m3, 4              ; p1[y=6|8,x=4-15]
    psrldq              m11, m5, 4              ; p1[y=10|12,x=4-15]
    vpblendd             m9, m0, m9, 01110111b
    vpblendd            m10, m2, m10, 01110111b
    vpblendd            m11, m4, m11, 01110111b ; mask out right 4px
    psadbw               m9, m0
    psadbw              m10, m2
    psadbw              m11, m4
    paddw                m9, m10
    paddw                m9, m11                ; sad(p0,p1) @ dx=-2
    ; dx=-1
    psrldq              m10, m1, 2              ; p1[y=2|4,x=2-15]
    psrldq              m11, m3, 2              ; p1[y=6|8,x=2-15]
    psrldq              m12, m5, 2              ; p1[y=10|12,x=2-15]
    vpblendvb           m10, m0, m10, m6
    vpblendvb           m11, m2, m11, m6
    vpblendvb           m12, m4, m12, m6        ; mask out left 1px & right 3px
    psadbw              m10, m0
    psadbw              m11, m2
    psadbw              m12, m4
    paddw               m10, m11
    paddw               m10, m12                ; sad(p0,p1) @ dy=-2,dx=-1
    ; dx=+1
    psrldq              m11, m0, 2              ; p0[y=2|4,x=2-15]
    psrldq              m12, m2, 2              ; p0[y=6|8,x=2-15]
    psrldq              m13, m4, 2              ; p0[y=10|12,x=2-15]
    vpblendvb           m11, m1, m11, m6
    vpblendvb           m12, m3, m12, m6
    vpblendvb           m13, m5, m13, m6        ; mask out left 1px & right 3px
    psadbw              m11, m1
    psadbw              m12, m3
    psadbw              m13, m5
    paddw               m11, m12
    paddw               m11, m13                ; sad(p0,p1) @ dx=+1
    ; dx=+2
    psrldq              m12, m0, 4              ; p0[y=2|4,x=4-15]
    psrldq              m13, m2, 4              ; p0[y=6|8,x=4-15]
    psrldq              m14, m4, 4              ; p0[y=10|12,x=4-15]
    vpblendd            m12, m1, m12, 01110111b
    vpblendd            m13, m3, m13, 01110111b
    vpblendd            m14, m5, m14, 01110111b ; mask out right 4px
    psadbw              m12, m1
    psadbw              m13, m3
    psadbw              m14, m5
    paddw               m12, m13
    paddw               m12, m14                ; sad(p0,p1) @ dx=+2
    ret
.w8_line_bottom_w_origin:
    ; dx=0
    vpblendvb            m4, m0, m1, m7
    vpblendvb            m5, m2, m3, m7         ; mask out left 2px & right 2px
    psadbw               m4, m0
    psadbw               m5, m2
    paddw                m4, m5                 ; sad(p0,p1) @ dx=0
.w8_line_bottom:
    ; dy=-2,dx=-2
    psrldq               m5, m1, 4              ; p1[y=14|16,x=4-15]
    psrldq               m8, m3, 4              ; p1[y=18|20,x=4-15]
    vpblendd             m5, m0, m5, 01110111b
    vpblendd             m8, m2, m8, 01110111b  ; mask out right 4px
    psadbw               m5, m0
    psadbw               m8, m2
    paddw                m5, m8                 ; sad(p0,p1) @ dx=-2
    ; dy=-2,dx=-1
    psrldq               m8, m1, 2              ; p1[y=14|16,x=2-15]
    psrldq               m9, m3, 2              ; p1[y=18|20,x=2-15]
    vpblendvb            m8, m0, m8, m6
    vpblendvb            m9, m2, m9, m6         ; mask out left 1px & right 3px
    psadbw               m8, m0
    psadbw               m9, m2
    paddw                m8, m9                 ; sad(p0,p1) @ dx=-1
    ; dy=-2,dx=+1
    psrldq               m9, m0, 2              ; p0[y=14|16,x=2-15]
    psrldq              m10, m2, 2              ; p0[y=18|20,x=2-15]
    vpblendvb            m9, m1, m9, m6
    vpblendvb           m10, m3, m10, m6        ; mask out left 1px & right 3px
    psadbw               m9, m1
    psadbw              m10, m3
    paddw                m9, m10                ; sad(p0,p1) @ dx=+1
    ; dy=-2,dx=+2
    psrldq              m10, m0, 4              ; p0[y=14|16,x=4-15]
    psrldq              m11, m2, 4              ; p0[y=18|20,x=4-15]
    vpblendd            m10, m1, m10, 01110111b
    vpblendd            m11, m3, m11, 01110111b ; mask out right 4px
    psadbw              m10, m1
    psadbw              m11, m3
    paddw               m10, m11                ; sad(p0,p1) @ dx=+2
    ret
.ret_origin:
    mov                  r0, r7m
    mov           word [r0], 0                  ; o->y = o->x = 0
    RET

cglobal opfl_derive_mv_8bpc, 5, 7, 0, out, p0, p0s, p1, p1s, w, h, bs, reldist
    movifnidn            wd, wm
    cmp                  wd, 16
    jge .w16

%macro opfl_loop 1 ; bs
    mov                  hd, hm
    vpbroadcastd         m6, [pb_1_m1]
    vpbroadcastw        m14, reldistm
    mova                m15, [pw_4x84a10_4x42a5]

    movq                xm0, [p0q]
    movq                xm1, [p1q]
    vinserti128          m0, [p0q+p0sq*1], 1
    vinserti128          m1, [p1q+p1sq*1], 1
    punpcklbw            m0, m1
    pmaddubsw            m1, m0, m6         ; tmp1[y=0|1]
    pmaddubsw            m0, m14            ; tmp0[y=0|1]
    vpermq               m2, m0, q1010      ; tmp0[y=0|0]=q1
    mova                 m3, m2             ; q0
    sub                  hd, 2

    ; aggregators
%if %1 == 4
    vpbroadcastq        xm7, [pq_16]        ; bs * bs for u*u or v*v
%else
    movd                xm7, [pd_64]
%endif
    REPX        {pxor x, x}, xm8, xm10, xm11
    mova                xm9, xm7
    jmp %%loop_skipconst
%%loop:
    vpbroadcastd         m6, [pb_1_m1]
    vpbroadcastw        m14, reldistm
    vpbroadcastd        m15, [pw_4x84a10_4x42a5+16]
%%loop_skipconst:

    ; m0=tmp0[cur line]
    ; m1=tmp1[cur line]
    ; m2=q1
    ; m3=q0
    ; m6=pb_1_m1 [weights for tmp0]
    ; m7-11: res
    ; m4-5,12-13=free
    ; m14=reldistm [weights for tmp1]
    ; m15=gy0 weights (some permutation of [pw_4x84a10_4x42a5])

    lea                 p0q, [p0q+p0sq*2]
    lea                 p1q, [p1q+p1sq*2]
    movq                xm4, [p0q]
    movq                xm5, [p1q]
    vinserti128          m4, [p0q+p0sq*1], 1
    vinserti128          m5, [p1q+p1sq*1], 1
    punpcklbw            m4, m5
    pmaddubsw            m5, m4, m6         ; tmp1[y=2|3]
    pmaddubsw            m4, m14            ; tmp0[y=2|3]=q3
    vperm2i128           m6, m0, m4, 0x21   ; tmp0[y=1|2]=q2

    ; m0=tmp0[cur line]
    ; m1=tmp1[cur line]
    ; m2=q1
    ; m3=q0
    ; m4=tmp0[next line]=q3
    ; m5=tmp1[next line]
    ; m6=q2
    ; m7-11=res [u*u, u*v, v*v, u*w, v*w]
    ; m12-13=free
    ; m14=reldistm [weights for tmp1]
    ; m15=gy0 weights (some permutation of [pw_4x84a10_4x42a5])

%%loop_skipload:
    ; gy0
    vpbroadcastd        m14, [pd_64]
    psubw                m2, m6, m2         ; (q2 - q1)[y=0|1]
    psubw                m3, m4             ; (q0 - q3)[y=0|1]
    punpckhwd           m13, m2, m3
    punpcklwd            m2, m3
    REPX   {pmaddwd x, m15}, m13, m2
    psrad               m12, m13, 31
    psrad                m3, m2, 31
    REPX   {paddd   x, m14}, m13, m2
    paddd               m13, m12
    paddd                m2, m3
    REPX   {psrad   x, 7  }, m13, m2
    packssdw             m2, m13            ; gy0[y=0|1]=v

    ; m0=tmp0[cur line]
    ; m1=tmp1[cur line]
    ; m2=gy0
    ; m4=tmp0[next line]=q3
    ; m5=tmp1[next line]
    ; m6=q2
    ; m7-11=res [u*u, u*v, v*v, u*w, v*w]
    ; m3,12-13,15=free
    ; m14=pd_64

    ; gx0
    vbroadcasti128       m3, [shuf_left_1w]
    vbroadcasti128      m15, [shuf_right_1w]
    pshufb              m12, m0, m3         ; p1
    pshufb              m13, m0, m15        ; p2
    pshufb               m3, m12, m3        ; p0
    pshufb              m15, m13, m15       ; p3
    psubw               m13, m12            ; (p2 - p1)[y=0|1]
    vbroadcasti128      m12, [pw_3x42a5_1x84a10]
    psubw                m3, m15            ; (p0 - p3)[y=0|1]
    punpckhwd           m15, m13, m3
    punpcklwd           m13, m3
    pshufd               m3, m12, q0123     ; pw_1x84a10_3x42a5
    pmaddwd             m15, m12
    pmaddwd             m13, m3
    psrad               m12, m15, 31
    psrad                m3, m13, 31
    REPX   {paddd   x, m14}, m15, m13
    paddd               m15, m12
    paddd                m3, m13
    REPX   {psrad   x, 7  }, m15, m3
    packssdw             m3, m15            ; gx0[y=0|1]=u

    ; m0=tmp0[cur line]
    ; m1=tmp1[cur line] [w]
    ; m2=gy0 [v]
    ; m3=gx0 [u]
    ; m4=tmp0[next line]=q3
    ; m5=tmp1[next line]
    ; m6=q2
    ; m7-11=res [u*u, u*v, v*v, u*w, v*w]
    ; m12-15=free

    ; regression data
    pmaddwd             m12, m1, m2         ; v*w
    pmaddwd              m1, m3             ; u*w
    pmaddwd             m15, m2, m3         ; u*v
    REPX     {pmaddwd x, x}, m2, m3         ; v*v and u*u
    paddd               m11, m12
    paddd               m10, m1
    paddd                m8, m15
    paddd                m9, m2
    paddd                m7, m3

    test                 hd, %1-1
    jz %%end
    sub                  hd, 2
%%noret:
    mova                 m3, m0
    mova                 m0, m4
    mova                 m1, m5
    mova                 m2, m6
    jg %%loop
    ; last line
    vpermq               m4, m4, q3232      ; tmp0[y=3|3]=q2 & q3
    vpermq              m15, [pw_4x84a10_4x42a5], q1032
    mova                 m6, m4
    jmp %%loop_skipload

%%end:
    ; m7-11=res [u*u, u*v, v*v, u*w, v*w]
%if %1 == 8
    phaddd               m7, m8
    phaddd               m9, m10
    vextracti128       xm12, m11, 1
    phaddd               m7, m9
    paddd              xm11, xm12
    vextracti128        xm8, m7, 1
    punpckhqdq         xm12, xm11, xm11
    paddd               xm7, xm8
    paddd              xm11, xm12
    psrlq              xm12, xm11, 32
    paddd              xm11, xm12
    movu         [outq+0*4], xm7
    movd         [outq+4*4], xm11
%else
    phaddd               m7, m9
    phaddd               m8, m10
    vextracti128       xm12, m11, 1
    vextracti128        xm9, m7, 1
    vextracti128       xm10, m8, 1
    paddd              xm11, xm12
    paddd               xm7, xm9
    psrlq              xm12, xm11, 32
    paddd               xm8, xm10
    paddd              xm11, xm12
    punpckhdq           xm9, xm7, xm8
    punpckldq           xm7, xm8
    punpcklqdq          xm8, xm7, xm9
    punpckhqdq          xm7, xm9
    movu         [outq+0*4], xm8
    movd         [outq+4*4], xm11
    movu         [outq+5*4], xm7
    pextrd       [outq+9*4], xm11, 2
%endif
    sub                  hd, 2
    jl %%ret
    add                outq, 5*(32/%1)
%if %1 == 4
    vpbroadcastq        xm7, [pq_16]        ; bs * bs for u*u or v*v
%else
    movd                xm7, [pd_64]
%endif
    REPX        {pxor x, x}, xm8, xm10, xm11
    mova                xm9, xm7
    jmp %%noret
%%ret:
    RET
%endmacro

    PROLOGUE 5, 7, 16, out, p0, p0s, p1, p1s, w, h, bs, reldist
    cmp           dword bsm, 4
    je .bs4
    opfl_loop             8
.bs4:
    opfl_loop             4

.w16:
    PROLOGUE 5, 8, 16, 12 * mmsize * 4 * gprsize, out, p0, p0s, p1, p1s, w, h, bs, reldist
    lea                 r7d, [wd*5]
    shr                 r7d, 1
    mov [rsp+12*mmsize+0*gprsize], p0q
    mov [rsp+12*mmsize+1*gprsize], p1q
    mov [rsp+12*mmsize+2*gprsize], outq
    mov [rsp+12*mmsize+3*gprsize], r7
.w16_loop:
    mov                  hd, hm
    vpbroadcastd        m13, [pb_1_m1]
    vpbroadcastw        m14, reldistm
    mova                m15, [pw_4x84a10_4x42a5]

    movu                xm0, [p0q]
    movu                xm2, [p1q]
    vinserti128          m0, [p0q+p0sq*1], 1
    vinserti128          m2, [p1q+p1sq*1], 1
    punpckhbw            m1, m0, m2
    punpcklbw            m0, m2
    pmaddubsw            m7, m1, m13
    pmaddubsw            m6, m0, m13        ; tmp1[y=0|1]
    REPX {pmaddubsw x, m14}, m0, m1         ; tmp0[y=0|1]
    vpermq               m2, m0, q1010
    vpermq               m3, m1, q1010      ; tmp0[y=0|0]=q1
    mova                 m4, m2
    mova                 m5, m3             ; q0
    sub                  hd, 2

    ; aggregators
    movd               xm10, [pd_64]
    pxor               xm11, xm11
    mova    [rsp+ 2*mmsize], m10
    mova    [rsp+ 3*mmsize], m11
    mova    [rsp+ 4*mmsize], m10
    mova    [rsp+ 5*mmsize], m11
    mova    [rsp+ 6*mmsize], m11
    mova    [rsp+ 7*mmsize], m10
    mova    [rsp+ 8*mmsize], m11
    mova    [rsp+ 9*mmsize], m10
    mova    [rsp+10*mmsize], m11
    mova    [rsp+11*mmsize], m11
    jmp .loop_skipconst
.loop:
    vpbroadcastd        m13, [pb_1_m1]
    vpbroadcastw        m14, reldistm
    vpbroadcastd        m15, [pw_4x84a10_4x42a5+16]
.loop_skipconst:

    ; m0-1=tmp0[cur line]
    ; m2-3=q1
    ; m4-5=q0
    ; m6-7=tmp1[cur line]
    ; m8-12=free
    ; m13=[pb_1_m1]
    ; m14=reldistm [weights for tmp1]
    ; m15=gy0 weights (some permutation of [pw_4x84a10_4x42a5])

    lea                 p0q, [p0q+p0sq*2]
    lea                 p1q, [p1q+p1sq*2]
    movu                xm8, [p0q]
    movu               xm10, [p1q]
    vinserti128          m8, [p0q+p0sq*1], 1
    vinserti128         m10, [p1q+p1sq*1], 1
    punpckhbw            m9, m8, m10
    punpcklbw            m8, m10
    pmaddubsw           m11, m9, m13
    pmaddubsw           m10, m8, m13        ; tmp1[y=2|3]
    mova     [rsp+0*mmsize], m10
    mova     [rsp+1*mmsize], m11
    REPX {pmaddubsw x, m14}, m8, m9         ; tmp0[y=2|3]=q3
    vperm2i128          m12, m0, m8, 0x21
    vperm2i128          m13, m1, m9, 0x21   ; tmp0[y=1|2]=q2

    ; m0-1=tmp0[cur line]
    ; m2-3=q1
    ; m4-5=q0
    ; m6-7=tmp1[cur line]
    ; m8-9=tmp0[next line]=q3
    ; m10-11=free
    ; m12-13=q2
    ; m14=reldistm [weights for tmp1]
    ; m15=gy0 weights (some permutation of [pw_4x84a10_4x42a5])
    ; r0-1=tmp1[next line]
    ; r2-11=res

.loop_skipload:
    ; gy0
    psubw                m2, m12, m2
    psubw                m3, m13, m3        ; (q2 - q1)[y=0|1]
    psubw                m4, m8
    psubw                m5, m9             ; (q0 - q3)[y=0|1]
    punpckhwd           m14, m2, m4
    punpcklwd            m2, m4
    punpckhwd            m4, m3, m5
    punpcklwd            m3, m5
    vpbroadcastd         m5, [pd_64]
    REPX   {pmaddwd x, m15}, m14, m2, m4, m3
    psrad               m10, m14, 31
    psrad               m11, m2, 31
    REPX   {paddd   x, m5 }, m14, m2
    paddd               m14, m10
    paddd                m2, m11
    REPX   {psrad   x, 7  }, m14, m2
    psrad               m10, m4, 31
    psrad               m11, m3, 31
    REPX   {paddd   x, m5 }, m4, m3
    paddd                m4, m10
    paddd                m3, m11
    REPX   {psrad   x, 7  }, m4, m3
    packssdw             m2, m14
    packssdw             m3, m4             ; gy0[y=0|1]=v

    ; m0-1=tmp0[cur line]
    ; m2-3=gy0
    ; m5=pd_64
    ; m6-7=tmp1[cur line]
    ; m8-9=tmp0[next line]=q3
    ; m12-13=q2
    ; m4,10-11,14-15=free
    ; r0-1=tmp1[next line]
    ; r2-11=res

    ; gx0
    vbroadcasti128      m14, [shuf_left_1w]
    vbroadcasti128      m15, [shuf_right_1w]
    pshufb               m4, m0, m14        ; p1
    palignr             m11, m1, m0, 2      ; p2
    pshufb              m14, m4, m14        ; p0
    palignr             m10, m1, m0, 4      ; p3
    psubw               m11, m4             ; (p2 - p1)[y=0|1,x=0-7]
    psubw               m14, m10            ; (p0 - p3)[y=0|1,x=0-7]
    pshufb               m4, m1, m15        ; p2
    palignr             m10, m1, m0, 14     ; p1
    pshufb              m15, m4, m15        ; p3
    palignr              m5, m1, m0, 12     ; p0
    psubw                m4, m10            ; (p2 - p1)[y=0|1,x=8-15]
    psubw                m5, m15            ; (p0 - p3)[y=0|1,x=8-15]
    vbroadcasti128      m10, [pw_3x42a5_1x84a10]
    punpckhwd           m15, m11, m14
    punpcklwd           m11, m14
    punpckhwd           m14, m4, m5
    punpcklwd            m4, m5
    pshufd               m5, m10, q0123     ; pw_1x84a10_3x42a5
    pmaddwd             m14, m10
    pshufd              m10, m10, q1111     ; pw_4x42a5
    pmaddwd             m11, m5
    REPX   {pmaddwd x, m10}, m15, m4
    psrad                m5, m11, 31
    psrad               m10, m15, 31
    paddd               m11, m5
    paddd               m15, m10
    psrad                m5, m4, 31
    psrad               m10, m14, 31
    paddd                m5, m4
    vpbroadcastd         m4, [pd_64]
    paddd               m14, m10
    REPX  {paddd    x, m4 }, m11, m15, m5, m14
    REPX  {psrad    x, 7  }, m11, m15, m5, m14
    packssdw             m4, m11, m15
    packssdw             m5, m14            ; gx0[y=0|1]=u

    ; m0-1=tmp0[cur line]
    ; m2-3=gy0
    ; m4-5=gx0
    ; m6-7=tmp1[cur line]
    ; m8-9=tmp0[next line]=q3
    ; m12-13=q2
    ; m10-11,14-15=free
    ; r0-1=tmp1[next line]
    ; r2-11=res

    ; regression data
    pmaddwd             m14, m2, m6
    pmaddwd             m15, m3, m7         ; v*w
    pmaddwd              m6, m4
    pmaddwd              m7, m5             ; u*w
    pmaddwd             m10, m2, m4
    pmaddwd             m11, m3, m5         ; u*v
    REPX     {pmaddwd x, x}, m2, m3, m4, m5 ; v*v and u*u
    paddd                m4, [rsp+ 2*mmsize]
    paddd               m10, [rsp+ 3*mmsize]
    paddd                m2, [rsp+ 4*mmsize]
    paddd                m6, [rsp+ 5*mmsize]
    paddd               m14, [rsp+ 6*mmsize]
    paddd                m5, [rsp+ 7*mmsize]
    paddd               m11, [rsp+ 8*mmsize]
    paddd                m3, [rsp+ 9*mmsize]
    paddd                m7, [rsp+10*mmsize]
    paddd               m15, [rsp+11*mmsize]

    test                 hd, 7
    jz .end
    sub                  hd, 2
    mova    [rsp+ 2*mmsize], m4
    mova    [rsp+ 3*mmsize], m10
    mova    [rsp+ 4*mmsize], m2
    mova    [rsp+ 5*mmsize], m6
    mova    [rsp+ 6*mmsize], m14
    mova    [rsp+ 7*mmsize], m5
    mova    [rsp+ 8*mmsize], m11
    mova    [rsp+ 9*mmsize], m3
    mova    [rsp+10*mmsize], m7
    mova    [rsp+11*mmsize], m15
.noret:
    mova                 m4, m0
    mova                 m5, m1
    mova                 m0, m8
    mova                 m1, m9
    mova                 m2, m12
    mova                 m3, m13
    mova                 m6, [rsp+0*mmsize]
    mova                 m7, [rsp+1*mmsize]
    jg .loop
    ; last line
    REPX {vpermq x, x, q3232}, m8, m9       ; tmp0[y=3|3]=q2 & q3
    vpermq              m15, [pw_4x84a10_4x42a5], q1032
    mova                m12, m8
    mova                m13, m9
    jmp .loop_skipload

.end:
    ; m4-5,10-11,2-3,6-7,14-15=res [u*u, u*v, v*v, u*w, v*w]
    phaddd               m4, m10
    phaddd               m2, m6
    phaddd              m14, m5
    phaddd              m11, m3
    phaddd               m7, m15
    phaddd               m4, m2
    psrlq               m15, m7, 32
    phaddd              m14, m11
    paddd                m7, m15
    vextracti128        xm2, m4, 1
    vextracti128       xm11, m14, 1
    vextracti128       xm15, m7, 1
    paddd               xm4, xm2
    paddd              xm14, xm11
    paddd               xm7, xm15
    movu         [outq+0*4], xm4
    pshufd              xm7, xm7, q3120
    movu         [outq+4*4], xm14
    movq         [outq+8*4], xm7
    sub                  hd, 2
    jl .next_block
    add                outq, [rsp+12*mmsize+3*gprsize]
    movd               xm10, [pd_64]
    pxor               xm11, xm11
    mova    [rsp+ 2*mmsize], m10
    mova    [rsp+ 3*mmsize], m11
    mova    [rsp+ 4*mmsize], m10
    mova    [rsp+ 5*mmsize], m11
    mova    [rsp+ 6*mmsize], m11
    mova    [rsp+ 7*mmsize], m10
    mova    [rsp+ 8*mmsize], m11
    mova    [rsp+ 9*mmsize], m10
    mova    [rsp+10*mmsize], m11
    mova    [rsp+11*mmsize], m11
    jmp .noret
.next_block:
    sub                  wd, 16
    jz .ret
    mov                 p0q, [rsp+12*mmsize+0*gprsize]
    mov                 p1q, [rsp+12*mmsize+1*gprsize]
    mov                outq, [rsp+12*mmsize+2*gprsize]
    REPX        {add x, 16}, p0q, p1q
    add                outq, 4*5*2
    mov [rsp+12*mmsize+0*gprsize], p0q
    mov [rsp+12*mmsize+1*gprsize], p1q
    mov [rsp+12*mmsize+2*gprsize], outq
    jmp .w16_loop
.ret:
    RET

INIT_XMM avx2
cglobal sad8x8_8bpc, 4, 6, 6, p0, p0s, p1, p1s, p0s3, p1s3
    lea               p0s3q, [p0sq*3]
    lea               p1s3q, [p1sq*3]
    movq                 m0, [p0q]
    movq                 m1, [p1q]
    movhps               m0, [p0q+p0sq]
    movhps               m1, [p1q+p1sq]
    movq                 m2, [p0q+p0sq*2]
    movq                 m3, [p1q+p1sq*2]
    movhps               m2, [p0q+p0s3q]
    movhps               m3, [p1q+p1s3q]
    lea                 p0q, [p0q+p0sq*4]
    lea                 p1q, [p1q+p1sq*4]
    psadbw               m0, m1
    psadbw               m2, m3
    movq                 m1, [p0q]
    movq                 m3, [p1q]
    movhps               m1, [p0q+p0sq]
    movhps               m3, [p1q+p1sq]
    movq                 m4, [p0q+p0sq*2]
    movq                 m5, [p1q+p1sq*2]
    movhps               m4, [p0q+p0s3q]
    movhps               m5, [p1q+p1s3q]
    psadbw               m1, m3
    psadbw               m4, m5
    paddd                m0, m2
    paddd                m1, m4
    paddd                m0, m1
    punpckhqdq           m1, m0, m0
    paddd                m0, m1
    movd                eax, m0
    RET
