;***************************************************************************** ;* Copyright (C) 2013 x265 project ;* ;* Authors: Min Chen ;* Nabajit Deka ;* Praveen Kumar Tiwari ;* ;* This program is free software; you can redistribute it and/or modify ;* it under the terms of the GNU General Public License as published by ;* the Free Software Foundation; either version 2 of the License, or ;* (at your option) any later version. ;* ;* This program is distributed in the hope that it will be useful, ;* but WITHOUT ANY WARRANTY; without even the implied warranty of ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;* GNU General Public License for more details. ;* ;* You should have received a copy of the GNU General Public License ;* along with this program; if not, write to the Free Software ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. ;* ;* This program is also available under a commercial proprietary license. ;* For more information, contact us at license @ x265.com. ;*****************************************************************************/ %include "x86inc.asm" %include "x86util.asm" SECTION_RODATA 32 tab_Tm: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 db 8, 9,10,11, 9,10,11,12,10,11,12,13,11,12,13, 14 tab_Lm: db 0, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 8 db 2, 3, 4, 5, 6, 7, 8, 9, 3, 4, 5, 6, 7, 8, 9, 10 db 4, 5, 6, 7, 8, 9, 10, 11, 5, 6, 7, 8, 9, 10, 11, 12 db 6, 7, 8, 9, 10, 11, 12, 13, 7, 8, 9, 10, 11, 12, 13, 14 tab_Vm: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 db 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3 tab_Cm: db 0, 2, 1, 3, 0, 2, 1, 3, 0, 2, 1, 3, 0, 2, 1, 3 tab_c_512: times 8 dw 512 tab_c_526336: times 4 dd 8192*64+2048 tab_ChromaCoeff: db 0, 64, 0, 0 db -2, 58, 10, -2 db -4, 54, 16, -2 db -6, 46, 28, -4 db -4, 36, 36, -4 db -4, 28, 46, -6 db -2, 16, 54, -4 db -2, 10, 58, -2 tab_ChromaCoeffV: times 4 dw 0, 64 times 4 dw 0, 0 times 4 dw -2, 58 times 4 dw 10, -2 times 4 dw -4, 54 times 4 dw 16, -2 times 4 dw -6, 46 times 4 dw 28, -4 times 4 dw -4, 36 times 4 dw 36, -4 times 4 dw -4, 28 times 4 dw 46, -6 times 4 dw -2, 16 times 4 dw 54, -4 times 4 dw -2, 10 times 4 dw 58, -2 tab_LumaCoeff: db 0, 0, 0, 64, 0, 0, 0, 0 db -1, 4, -10, 58, 17, -5, 1, 0 db -1, 4, -11, 40, 40, -11, 4, -1 db 0, 1, -5, 17, 58, -10, 4, -1 tab_LumaCoeffV: times 4 dw 0, 0 times 4 dw 0, 64 times 4 dw 0, 0 times 4 dw 0, 0 times 4 dw -1, 4 times 4 dw -10, 58 times 4 dw 17, -5 times 4 dw 1, 0 times 4 dw -1, 4 times 4 dw -11, 40 times 4 dw 40, -11 times 4 dw 4, -1 times 4 dw 0, 1 times 4 dw -5, 17 times 4 dw 58, -10 times 4 dw 4, -1 tab_LumaCoeffVer: times 8 db 0, 0 times 8 db 0, 64 times 8 db 0, 0 times 8 db 0, 0 times 8 db -1, 4 times 8 db -10, 58 times 8 db 17, -5 times 8 db 1, 0 times 8 db -1, 4 times 8 db -11, 40 times 8 db 40, -11 times 8 db 4, -1 times 8 db 0, 1 times 8 db -5, 17 times 8 db 58, -10 times 8 db 4, -1 tab_c_128: times 16 db 0x80 tab_c_64_n64: times 8 db 64, -64 SECTION .text cextern pw_512 cextern pw_2000 %macro FILTER_H4_w2_2 3 movh %2, [srcq - 1] pshufb %2, %2, Tm0 movh %1, [srcq + srcstrideq - 1] pshufb %1, %1, Tm0 punpcklqdq %2, %1 pmaddubsw %2, coef2 phaddw %2, %2 pmulhrsw %2, %3 packuswb %2, %2 movd r4, %2 mov [dstq], r4w shr r4, 16 mov [dstq + dststrideq], r4w %endmacro ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_2x4(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_horiz_pp_2x4, 4, 6, 5, src, srcstride, dst, dststride %define coef2 m4 %define Tm0 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] %rep 2 FILTER_H4_w2_2 t0, t1, t2 lea srcq, [srcq + srcstrideq * 2] lea dstq, [dstq + dststrideq * 2] %endrep RET ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_2x8(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_horiz_pp_2x8, 4, 6, 5, src, srcstride, dst, dststride %define coef2 m4 %define Tm0 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] %rep 4 FILTER_H4_w2_2 t0, t1, t2 lea srcq, [srcq + srcstrideq * 2] lea dstq, [dstq + dststrideq * 2] %endrep RET ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_2x16(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_horiz_pp_2x16, 4, 6, 5, src, srcstride, dst, dststride %define coef2 m4 %define Tm0 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] mov r5d, 16/2 .loop: FILTER_H4_w2_2 t0, t1, t2 lea srcq, [srcq + srcstrideq * 2] lea dstq, [dstq + dststrideq * 2] dec r5d jnz .loop RET %macro FILTER_H4_w4_2 3 movh %2, [srcq - 1] pshufb %2, %2, Tm0 pmaddubsw %2, coef2 movh %1, [srcq + srcstrideq - 1] pshufb %1, %1, Tm0 pmaddubsw %1, coef2 phaddw %2, %1 pmulhrsw %2, %3 packuswb %2, %2 movd [dstq], %2 palignr %2, %2, 4 movd [dstq + dststrideq], %2 %endmacro ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_4x2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_horiz_pp_4x2, 4, 6, 5, src, srcstride, dst, dststride %define coef2 m4 %define Tm0 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] FILTER_H4_w4_2 t0, t1, t2 RET ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_4x4(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_horiz_pp_4x4, 4, 6, 5, src, srcstride, dst, dststride %define coef2 m4 %define Tm0 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] %rep 2 FILTER_H4_w4_2 t0, t1, t2 lea srcq, [srcq + srcstrideq * 2] lea dstq, [dstq + dststrideq * 2] %endrep RET ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_4x8(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_horiz_pp_4x8, 4, 6, 5, src, srcstride, dst, dststride %define coef2 m4 %define Tm0 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] %rep 4 FILTER_H4_w4_2 t0, t1, t2 lea srcq, [srcq + srcstrideq * 2] lea dstq, [dstq + dststrideq * 2] %endrep RET ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_4x16(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_horiz_pp_4x16, 4, 6, 5, src, srcstride, dst, dststride %define coef2 m4 %define Tm0 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] %rep 8 FILTER_H4_w4_2 t0, t1, t2 lea srcq, [srcq + srcstrideq * 2] lea dstq, [dstq + dststrideq * 2] %endrep RET ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_4x32(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_horiz_pp_4x32, 4, 6, 5, src, srcstride, dst, dststride %define coef2 m4 %define Tm0 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] mov r5d, 32/2 .loop: FILTER_H4_w4_2 t0, t1, t2 lea srcq, [srcq + srcstrideq * 2] lea dstq, [dstq + dststrideq * 2] dec r5d jnz .loop RET %macro FILTER_H4_w6 3 movu %1, [srcq - 1] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 pmulhrsw %2, %3 packuswb %2, %2 movd [dstq], %2 pextrw [dstq + 4], %2, 2 %endmacro %macro FILTER_H4_w8 3 movu %1, [srcq - 1] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 pmulhrsw %2, %3 packuswb %2, %2 movh [dstq], %2 %endmacro %macro FILTER_H4_w12 3 movu %1, [srcq - 1] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 pmulhrsw %2, %3 movu %1, [srcq - 1 + 8] pshufb %1, %1, Tm0 pmaddubsw %1, coef2 phaddw %1, %1 pmulhrsw %1, %3 packuswb %2, %1 movh [dstq], %2 pextrd [dstq + 8], %2, 2 %endmacro %macro FILTER_H4_w16 4 movu %1, [srcq - 1] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq - 1 + 8] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 pmulhrsw %2, %3 pmulhrsw %4, %3 packuswb %2, %4 movu [dstq], %2 %endmacro %macro FILTER_H4_w24 4 movu %1, [srcq - 1] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq - 1 + 8] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 pmulhrsw %2, %3 pmulhrsw %4, %3 packuswb %2, %4 movu [dstq], %2 movu %1, [srcq - 1 + 16] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 pmulhrsw %2, %3 packuswb %2, %2 movh [dstq + 16], %2 %endmacro %macro FILTER_H4_w32 4 movu %1, [srcq - 1] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq - 1 + 8] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 pmulhrsw %2, %3 pmulhrsw %4, %3 packuswb %2, %4 movu [dstq], %2 movu %1, [srcq - 1 + 16] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq - 1 + 24] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 pmulhrsw %2, %3 pmulhrsw %4, %3 packuswb %2, %4 movu [dstq + 16], %2 %endmacro %macro FILTER_H4_w16o 5 movu %1, [srcq + %5 - 1] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq + %5 - 1 + 8] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 pmulhrsw %2, %3 pmulhrsw %4, %3 packuswb %2, %4 movu [dstq + %5], %2 %endmacro %macro FILTER_H4_w48 4 FILTER_H4_w16o %1, %2, %3, %4, 0 FILTER_H4_w16o %1, %2, %3, %4, 16 FILTER_H4_w16o %1, %2, %3, %4, 32 %endmacro %macro FILTER_H4_w64 4 FILTER_H4_w16o %1, %2, %3, %4, 0 FILTER_H4_w16o %1, %2, %3, %4, 16 FILTER_H4_w16o %1, %2, %3, %4, 32 FILTER_H4_w16o %1, %2, %3, %4, 48 %endmacro ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro IPFILTER_CHROMA 2 INIT_XMM sse4 cglobal interp_4tap_horiz_pp_%1x%2, 4, 6, 6, src, srcstride, dst, dststride %define coef2 m5 %define Tm0 m4 %define Tm1 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif mov r5d, %2 pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] mova Tm1, [tab_Tm + 16] .loop: FILTER_H4_w%1 t0, t1, t2 add srcq, srcstrideq add dstq, dststrideq dec r5d jnz .loop RET %endmacro IPFILTER_CHROMA 6, 8 IPFILTER_CHROMA 8, 2 IPFILTER_CHROMA 8, 4 IPFILTER_CHROMA 8, 6 IPFILTER_CHROMA 8, 8 IPFILTER_CHROMA 8, 16 IPFILTER_CHROMA 8, 32 IPFILTER_CHROMA 12, 16 IPFILTER_CHROMA 6, 16 IPFILTER_CHROMA 8, 12 IPFILTER_CHROMA 8, 64 IPFILTER_CHROMA 12, 32 ;----------------------------------------------------------------------------- ; void interp_4tap_horiz_pp_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro IPFILTER_CHROMA_W 2 INIT_XMM sse4 cglobal interp_4tap_horiz_pp_%1x%2, 4, 6, 7, src, srcstride, dst, dststride %define coef2 m6 %define Tm0 m5 %define Tm1 m4 %define t3 m3 %define t2 m2 %define t1 m1 %define t0 m0 mov r4d, r4m %ifdef PIC lea r5, [tab_ChromaCoeff] movd coef2, [r5 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif mov r5d, %2 pshufd coef2, coef2, 0 mova t2, [tab_c_512] mova Tm0, [tab_Tm] mova Tm1, [tab_Tm + 16] .loop: FILTER_H4_w%1 t0, t1, t2, t3 add srcq, srcstrideq add dstq, dststrideq dec r5d jnz .loop RET %endmacro IPFILTER_CHROMA_W 16, 4 IPFILTER_CHROMA_W 16, 8 IPFILTER_CHROMA_W 16, 12 IPFILTER_CHROMA_W 16, 16 IPFILTER_CHROMA_W 16, 32 IPFILTER_CHROMA_W 32, 8 IPFILTER_CHROMA_W 32, 16 IPFILTER_CHROMA_W 32, 24 IPFILTER_CHROMA_W 24, 32 IPFILTER_CHROMA_W 32, 32 IPFILTER_CHROMA_W 16, 24 IPFILTER_CHROMA_W 16, 64 IPFILTER_CHROMA_W 32, 48 IPFILTER_CHROMA_W 24, 64 IPFILTER_CHROMA_W 32, 64 IPFILTER_CHROMA_W 64, 64 IPFILTER_CHROMA_W 64, 32 IPFILTER_CHROMA_W 64, 48 IPFILTER_CHROMA_W 48, 64 IPFILTER_CHROMA_W 64, 16 %macro FILTER_H8_W8 7-8 ; t0, t1, t2, t3, coef, c512, src, dst movu %1, %7 pshufb %2, %1, [tab_Lm + 0] pmaddubsw %2, %5 pshufb %3, %1, [tab_Lm + 16] pmaddubsw %3, %5 phaddw %2, %3 pshufb %4, %1, [tab_Lm + 32] pmaddubsw %4, %5 pshufb %1, %1, [tab_Lm + 48] pmaddubsw %1, %5 phaddw %4, %1 phaddw %2, %4 %if %0 == 8 pmulhrsw %2, %6 packuswb %2, %2 movh %8, %2 %endif %endmacro %macro FILTER_H8_W4 2 movu %1, [r0 - 3 + r5] pshufb %2, %1, [tab_Lm] pmaddubsw %2, m3 pshufb m7, %1, [tab_Lm + 16] pmaddubsw m7, m3 phaddw %2, m7 phaddw %2, %2 %endmacro ;---------------------------------------------------------------------------------------------------------------------------- ; void interp_8tap_horiz_%3_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx, int isRowExt) ;---------------------------------------------------------------------------------------------------------------------------- %macro IPFILTER_LUMA 3 INIT_XMM sse4 cglobal interp_8tap_horiz_%3_%1x%2, 4,7,8 mov r4d, r4m %ifdef PIC lea r6, [tab_LumaCoeff] movh m3, [r6 + r4 * 8] %else movh m3, [tab_LumaCoeff + r4 * 8] %endif punpcklqdq m3, m3 %ifidn %3, pp mova m2, [tab_c_512] %else mova m2, [pw_2000] %endif mov r4d, %2 %ifidn %3, ps add r3, r3 cmp r5m, byte 0 je .loopH lea r6, [r1 + 2 * r1] sub r0, r6 add r4d, 7 %endif .loopH: xor r5, r5 %rep %1 / 8 %ifidn %3, pp FILTER_H8_W8 m0, m1, m4, m5, m3, m2, [r0 - 3 + r5], [r2 + r5] %else FILTER_H8_W8 m0, m1, m4, m5, m3, UNUSED, [r0 - 3 + r5] psubw m1, m2 movu [r2 + 2 * r5], m1 %endif add r5, 8 %endrep %rep (%1 % 8) / 4 FILTER_H8_W4 m0, m1 %ifidn %3, pp pmulhrsw m1, m2 packuswb m1, m1 movd [r2 + r5], m1 %else psubw m1, m2 movh [r2 + 2 * r5], m1 %endif %endrep add r0, r1 add r2, r3 dec r4d jnz .loopH RET %endmacro ;-------------------------------------------------------------------------------------------------------------- ; void interp_8tap_horiz_pp_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- IPFILTER_LUMA 4, 4, pp IPFILTER_LUMA 4, 8, pp IPFILTER_LUMA 12, 16, pp IPFILTER_LUMA 4, 16, pp ;-------------------------------------------------------------------------------------------------------------- ; void interp_8tap_horiz_pp_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- %macro IPFILTER_LUMA_PP_W8 2 INIT_XMM sse4 cglobal interp_8tap_horiz_pp_%1x%2, 4,6,7 mov r4d, r4m %ifdef PIC lea r5, [tab_LumaCoeff] movh m3, [r5 + r4 * 8] %else movh m3, [tab_LumaCoeff + r4 * 8] %endif pshufd m0, m3, 0 ; m0 = coeff-L pshufd m1, m3, 0x55 ; m1 = coeff-H lea r5, [tab_Tm] ; r5 = shuffle mova m2, [pw_512] ; m2 = 512 mov r4d, %2 .loopH: %assign x 0 %rep %1 / 8 movu m3, [r0 - 3 + x] ; m3 = [F E D C B A 9 8 7 6 5 4 3 2 1 0] pshufb m4, m3, [r5 + 0*16] ; m4 = [6 5 4 3 5 4 3 2 4 3 2 1 3 2 1 0] pshufb m5, m3, [r5 + 1*16] ; m5 = [A 9 8 7 9 8 7 6 8 7 6 5 7 6 5 4] pshufb m3, [r5 + 2*16] ; m3 = [E D C B D C B A C B A 9 B A 9 8] pmaddubsw m4, m0 pmaddubsw m6, m5, m1 pmaddubsw m5, m0 pmaddubsw m3, m1 paddw m4, m6 paddw m5, m3 phaddw m4, m5 pmulhrsw m4, m2 packuswb m4, m4 movh [r2 + x], m4 %assign x x+8 %endrep add r0, r1 add r2, r3 dec r4d jnz .loopH RET %endmacro IPFILTER_LUMA_PP_W8 8, 4 IPFILTER_LUMA_PP_W8 8, 8 IPFILTER_LUMA_PP_W8 8, 16 IPFILTER_LUMA_PP_W8 8, 32 IPFILTER_LUMA_PP_W8 16, 4 IPFILTER_LUMA_PP_W8 16, 8 IPFILTER_LUMA_PP_W8 16, 12 IPFILTER_LUMA_PP_W8 16, 16 IPFILTER_LUMA_PP_W8 16, 32 IPFILTER_LUMA_PP_W8 16, 64 IPFILTER_LUMA_PP_W8 24, 32 IPFILTER_LUMA_PP_W8 32, 8 IPFILTER_LUMA_PP_W8 32, 16 IPFILTER_LUMA_PP_W8 32, 24 IPFILTER_LUMA_PP_W8 32, 32 IPFILTER_LUMA_PP_W8 32, 64 IPFILTER_LUMA_PP_W8 48, 64 IPFILTER_LUMA_PP_W8 64, 16 IPFILTER_LUMA_PP_W8 64, 32 IPFILTER_LUMA_PP_W8 64, 48 IPFILTER_LUMA_PP_W8 64, 64 ;---------------------------------------------------------------------------------------------------------------------------- ; void interp_8tap_horiz_ps_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx, int isRowExt) ;---------------------------------------------------------------------------------------------------------------------------- IPFILTER_LUMA 4, 4, ps IPFILTER_LUMA 8, 8, ps IPFILTER_LUMA 8, 4, ps IPFILTER_LUMA 4, 8, ps IPFILTER_LUMA 16, 16, ps IPFILTER_LUMA 16, 8, ps IPFILTER_LUMA 8, 16, ps IPFILTER_LUMA 16, 12, ps IPFILTER_LUMA 12, 16, ps IPFILTER_LUMA 16, 4, ps IPFILTER_LUMA 4, 16, ps IPFILTER_LUMA 32, 32, ps IPFILTER_LUMA 32, 16, ps IPFILTER_LUMA 16, 32, ps IPFILTER_LUMA 32, 24, ps IPFILTER_LUMA 24, 32, ps IPFILTER_LUMA 32, 8, ps IPFILTER_LUMA 8, 32, ps IPFILTER_LUMA 64, 64, ps IPFILTER_LUMA 64, 32, ps IPFILTER_LUMA 32, 64, ps IPFILTER_LUMA 64, 48, ps IPFILTER_LUMA 48, 64, ps IPFILTER_LUMA 64, 16, ps IPFILTER_LUMA 16, 64, ps ;----------------------------------------------------------------------------- ; Interpolate HV ;----------------------------------------------------------------------------- %macro FILTER_HV8_START 7 ; (t0, t1, t2, t3, t4, off_src, off_coeff) -> (t3, t5), (t4, t1), [2] mova %5, [r0 + (%6 + 0) * 16] mova %1, [r0 + (%6 + 1) * 16] mova %2, [r0 + (%6 + 2) * 16] punpcklwd %3, %5, %1 punpckhwd %5, %1 pmaddwd %3, [r5 + (%7) * 16] ; R3 = L[0+1] -- Row 0 pmaddwd %5, [r5 + (%7) * 16] ; R0 = H[0+1] punpcklwd %4, %1, %2 punpckhwd %1, %2 pmaddwd %4, [r5 + (%7) * 16] ; R4 = L[1+2] -- Row 1 pmaddwd %1, [r5 + (%7) * 16] ; R1 = H[1+2] %endmacro ; FILTER_HV8_START %macro FILTER_HV8_MID 10 ; (Row3, prevRow, sum0L, sum1L, sum0H, sum1H, t6, t7, off_src, off_coeff) -> [6] mova %8, [r0 + (%9 + 0) * 16] mova %1, [r0 + (%9 + 1) * 16] punpcklwd %7, %2, %8 punpckhwd %2, %8 pmaddwd %7, [r5 + %10 * 16] pmaddwd %2, [r5 + %10 * 16] paddd %3, %7 ; R3 = L[0+1+2+3] -- Row 0 paddd %5, %2 ; R0 = H[0+1+2+3] punpcklwd %7, %8, %1 punpckhwd %8, %1 pmaddwd %7, [r5 + %10 * 16] pmaddwd %8, [r5 + %10 * 16] paddd %4, %7 ; R4 = L[1+2+3+4] -- Row 1 paddd %6, %8 ; R1 = H[1+2+3+4] %endmacro ; FILTER_HV8_MID ; Round and Saturate %macro FILTER_HV8_END 4 ; output in [1, 3] paddd %1, [tab_c_526336] paddd %2, [tab_c_526336] paddd %3, [tab_c_526336] paddd %4, [tab_c_526336] psrad %1, 12 psrad %2, 12 psrad %3, 12 psrad %4, 12 packssdw %1, %2 packssdw %3, %4 ; TODO: is merge better? I think this way is short dependency link packuswb %1, %3 %endmacro ; FILTER_HV8_END ;----------------------------------------------------------------------------- ; void interp_8tap_hv_pp_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int idxX, int idxY) ;----------------------------------------------------------------------------- INIT_XMM ssse3 cglobal interp_8tap_hv_pp_8x8, 4, 7, 8, 0-15*16 %define coef m7 %define stk_buf rsp mov r4d, r4m mov r5d, r5m %ifdef PIC lea r6, [tab_LumaCoeff] movh coef, [r6 + r4 * 8] %else movh coef, [tab_LumaCoeff + r4 * 8] %endif punpcklqdq coef, coef ; move to row -3 lea r6, [r1 + r1 * 2] sub r0, r6 xor r6, r6 mov r4, rsp .loopH: FILTER_H8_W8 m0, m1, m2, m3, coef, [tab_c_512], [r0 - 3] psubw m1, [pw_2000] mova [r4], m1 add r0, r1 add r4, 16 inc r6 cmp r6, 8+7 jnz .loopH ; ready to phase V ; Here all of mN is free ; load coeff table shl r5, 6 lea r6, [tab_LumaCoeffV] lea r5, [r5 + r6] ; load intermedia buffer mov r0, stk_buf ; register mapping ; r0 - src ; r5 - coeff ; r6 - loop_i ; let's go xor r6, r6 ; TODO: this loop have more than 70 instructions, I think it is more than Intel loop decode cache .loopV: FILTER_HV8_START m1, m2, m3, m4, m0, 0, 0 FILTER_HV8_MID m6, m2, m3, m4, m0, m1, m7, m5, 3, 1 FILTER_HV8_MID m5, m6, m3, m4, m0, m1, m7, m2, 5, 2 FILTER_HV8_MID m6, m5, m3, m4, m0, m1, m7, m2, 7, 3 FILTER_HV8_END m3, m0, m4, m1 movh [r2], m3 movhps [r2 + r3], m3 lea r0, [r0 + 16 * 2] lea r2, [r2 + r3 * 2] inc r6 cmp r6, 8/2 jnz .loopV RET ;----------------------------------------------------------------------------- ;void interp_4tap_vert_pp_2x4(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_vert_pp_2x4, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif lea r4, [r1 * 3] lea r5, [r0 + 4 * r1] pshufb m0, [tab_Cm] mova m1, [tab_c_512] movd m2, [r0] movd m3, [r0 + r1] movd m4, [r0 + 2 * r1] movd m5, [r0 + r4] punpcklbw m2, m3 punpcklbw m6, m4, m5 punpcklbw m2, m6 pmaddubsw m2, m0 movd m6, [r5] punpcklbw m3, m4 punpcklbw m7, m5, m6 punpcklbw m3, m7 pmaddubsw m3, m0 phaddw m2, m3 pmulhrsw m2, m1 movd m7, [r5 + r1] punpcklbw m4, m5 punpcklbw m3, m6, m7 punpcklbw m4, m3 pmaddubsw m4, m0 movd m3, [r5 + 2 * r1] punpcklbw m5, m6 punpcklbw m7, m3 punpcklbw m5, m7 pmaddubsw m5, m0 phaddw m4, m5 pmulhrsw m4, m1 packuswb m2, m4 pextrw [r2], m2, 0 pextrw [r2 + r3], m2, 2 lea r2, [r2 + 2 * r3] pextrw [r2], m2, 4 pextrw [r2 + r3], m2, 6 RET ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_2x8(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W2_H4 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_2x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m0, [tab_Cm] mova m1, [tab_c_512] mov r4d, %2 lea r5, [3 * r1] .loop: movd m2, [r0] movd m3, [r0 + r1] movd m4, [r0 + 2 * r1] movd m5, [r0 + r5] punpcklbw m2, m3 punpcklbw m6, m4, m5 punpcklbw m2, m6 pmaddubsw m2, m0 lea r0, [r0 + 4 * r1] movd m6, [r0] punpcklbw m3, m4 punpcklbw m7, m5, m6 punpcklbw m3, m7 pmaddubsw m3, m0 phaddw m2, m3 pmulhrsw m2, m1 movd m7, [r0 + r1] punpcklbw m4, m5 punpcklbw m3, m6, m7 punpcklbw m4, m3 pmaddubsw m4, m0 movd m3, [r0 + 2 * r1] punpcklbw m5, m6 punpcklbw m7, m3 punpcklbw m5, m7 pmaddubsw m5, m0 phaddw m4, m5 pmulhrsw m4, m1 packuswb m2, m4 pextrw [r2], m2, 0 pextrw [r2 + r3], m2, 2 lea r2, [r2 + 2 * r3] pextrw [r2], m2, 4 pextrw [r2 + r3], m2, 6 lea r2, [r2 + 2 * r3] sub r4, 4 jnz .loop RET %endmacro FILTER_V4_W2_H4 2, 8 FILTER_V4_W2_H4 2, 16 ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_4x2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_vert_pp_4x2, 4, 6, 6 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m0, [tab_Cm] lea r5, [r0 + 2 * r1] movd m2, [r0] movd m3, [r0 + r1] movd m4, [r5] movd m5, [r5 + r1] punpcklbw m2, m3 punpcklbw m1, m4, m5 punpcklbw m2, m1 pmaddubsw m2, m0 movd m1, [r0 + 4 * r1] punpcklbw m3, m4 punpcklbw m5, m1 punpcklbw m3, m5 pmaddubsw m3, m0 phaddw m2, m3 pmulhrsw m2, [tab_c_512] packuswb m2, m2 movd [r2], m2 pextrd [r2 + r3], m2, 1 RET ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_4x4(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_vert_pp_4x4, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m0, [tab_Cm] mova m1, [tab_c_512] lea r5, [r0 + 4 * r1] lea r4, [r1 * 3] movd m2, [r0] movd m3, [r0 + r1] movd m4, [r0 + 2 * r1] movd m5, [r0 + r4] punpcklbw m2, m3 punpcklbw m6, m4, m5 punpcklbw m2, m6 pmaddubsw m2, m0 movd m6, [r5] punpcklbw m3, m4 punpcklbw m7, m5, m6 punpcklbw m3, m7 pmaddubsw m3, m0 phaddw m2, m3 pmulhrsw m2, m1 movd m7, [r5 + r1] punpcklbw m4, m5 punpcklbw m3, m6, m7 punpcklbw m4, m3 pmaddubsw m4, m0 movd m3, [r5 + 2 * r1] punpcklbw m5, m6 punpcklbw m7, m3 punpcklbw m5, m7 pmaddubsw m5, m0 phaddw m4, m5 pmulhrsw m4, m1 packuswb m2, m4 movd [r2], m2 pextrd [r2 + r3], m2, 1 lea r2, [r2 + 2 * r3] pextrd [r2], m2, 2 pextrd [r2 + r3], m2, 3 RET ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W4_H4 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_%1x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m0, [tab_Cm] mova m1, [tab_c_512] mov r4d, %2 lea r5, [3 * r1] .loop: movd m2, [r0] movd m3, [r0 + r1] movd m4, [r0 + 2 * r1] movd m5, [r0 + r5] punpcklbw m2, m3 punpcklbw m6, m4, m5 punpcklbw m2, m6 pmaddubsw m2, m0 lea r0, [r0 + 4 * r1] movd m6, [r0] punpcklbw m3, m4 punpcklbw m7, m5, m6 punpcklbw m3, m7 pmaddubsw m3, m0 phaddw m2, m3 pmulhrsw m2, m1 movd m7, [r0 + r1] punpcklbw m4, m5 punpcklbw m3, m6, m7 punpcklbw m4, m3 pmaddubsw m4, m0 movd m3, [r0 + 2 * r1] punpcklbw m5, m6 punpcklbw m7, m3 punpcklbw m5, m7 pmaddubsw m5, m0 phaddw m4, m5 pmulhrsw m4, m1 packuswb m2, m4 movd [r2], m2 pextrd [r2 + r3], m2, 1 lea r2, [r2 + 2 * r3] pextrd [r2], m2, 2 pextrd [r2 + r3], m2, 3 lea r2, [r2 + 2 * r3] sub r4, 4 jnz .loop RET %endmacro FILTER_V4_W4_H4 4, 8 FILTER_V4_W4_H4 4, 16 FILTER_V4_W4_H4 4, 32 %macro FILTER_V4_W8_H2 0 punpcklbw m1, m2 punpcklbw m7, m3, m0 pmaddubsw m1, m6 pmaddubsw m7, m5 paddw m1, m7 pmulhrsw m1, m4 packuswb m1, m1 %endmacro %macro FILTER_V4_W8_H3 0 punpcklbw m2, m3 punpcklbw m7, m0, m1 pmaddubsw m2, m6 pmaddubsw m7, m5 paddw m2, m7 pmulhrsw m2, m4 packuswb m2, m2 %endmacro %macro FILTER_V4_W8_H4 0 punpcklbw m3, m0 punpcklbw m7, m1, m2 pmaddubsw m3, m6 pmaddubsw m7, m5 paddw m3, m7 pmulhrsw m3, m4 packuswb m3, m3 %endmacro %macro FILTER_V4_W8_H5 0 punpcklbw m0, m1 punpcklbw m7, m2, m3 pmaddubsw m0, m6 pmaddubsw m7, m5 paddw m0, m7 pmulhrsw m0, m4 packuswb m0, m0 %endmacro %macro FILTER_V4_W8_8x2 2 FILTER_V4_W8 %1, %2 movq m0, [r0 + 4 * r1] FILTER_V4_W8_H2 movh [r2 + r3], m1 %endmacro %macro FILTER_V4_W8_8x4 2 FILTER_V4_W8_8x2 %1, %2 ;8x3 lea r6, [r0 + 4 * r1] movq m1, [r6 + r1] FILTER_V4_W8_H3 movh [r2 + 2 * r3], m2 ;8x4 movq m2, [r6 + 2 * r1] FILTER_V4_W8_H4 lea r5, [r2 + 2 * r3] movh [r5 + r3], m3 %endmacro %macro FILTER_V4_W8_8x6 2 FILTER_V4_W8_8x4 %1, %2 ;8x5 lea r6, [r6 + 2 * r1] movq m3, [r6 + r1] FILTER_V4_W8_H5 movh [r2 + 4 * r3], m0 ;8x6 movq m0, [r0 + 8 * r1] FILTER_V4_W8_H2 lea r5, [r2 + 4 * r3] movh [r5 + r3], m1 %endmacro ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W8 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_%1x%2, 4, 7, 8 mov r4d, r4m sub r0, r1 movq m0, [r0] movq m1, [r0 + r1] movq m2, [r0 + 2 * r1] lea r5, [r0 + 2 * r1] movq m3, [r5 + r1] punpcklbw m0, m1 punpcklbw m4, m2, m3 %ifdef PIC lea r6, [tab_ChromaCoeff] movd m5, [r6 + r4 * 4] %else movd m5, [tab_ChromaCoeff + r4 * 4] %endif pshufb m6, m5, [tab_Vm] pmaddubsw m0, m6 pshufb m5, [tab_Vm + 16] pmaddubsw m4, m5 paddw m0, m4 mova m4, [tab_c_512] pmulhrsw m0, m4 packuswb m0, m0 movh [r2], m0 %endmacro ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_8x2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- FILTER_V4_W8_8x2 8, 2 RET ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_8x4(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- FILTER_V4_W8_8x4 8, 4 RET ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_8x6(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- FILTER_V4_W8_8x6 8, 6 RET ;------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_4x2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_vert_ps_4x2, 4, 6, 6 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m0, [tab_Cm] movd m2, [r0] movd m3, [r0 + r1] lea r5, [r0 + 2 * r1] movd m4, [r5] movd m5, [r5 + r1] punpcklbw m2, m3 punpcklbw m1, m4, m5 punpcklbw m2, m1 pmaddubsw m2, m0 movd m1, [r0 + 4 * r1] punpcklbw m3, m4 punpcklbw m5, m1 punpcklbw m3, m5 pmaddubsw m3, m0 phaddw m2, m3 psubw m2, [pw_2000] movh [r2], m2 movhps [r2 + r3], m2 RET ;------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_4x4(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_vert_ps_4x4, 4, 6, 7 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m0, [tab_Cm] lea r4, [r1 * 3] lea r5, [r0 + 4 * r1] movd m2, [r0] movd m3, [r0 + r1] movd m4, [r0 + 2 * r1] movd m5, [r0 + r4] punpcklbw m2, m3 punpcklbw m6, m4, m5 punpcklbw m2, m6 pmaddubsw m2, m0 movd m6, [r5] punpcklbw m3, m4 punpcklbw m1, m5, m6 punpcklbw m3, m1 pmaddubsw m3, m0 phaddw m2, m3 mova m1, [pw_2000] psubw m2, m1 movh [r2], m2 movhps [r2 + r3], m2 movd m2, [r5 + r1] punpcklbw m4, m5 punpcklbw m3, m6, m2 punpcklbw m4, m3 pmaddubsw m4, m0 movd m3, [r5 + 2 * r1] punpcklbw m5, m6 punpcklbw m2, m3 punpcklbw m5, m2 pmaddubsw m5, m0 phaddw m4, m5 psubw m4, m1 lea r2, [r2 + 2 * r3] movh [r2], m4 movhps [r2 + r3], m4 RET ;--------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_%1x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;--------------------------------------------------------------------------------------------------------------- %macro FILTER_V_PS_W4_H4 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_%1x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m0, [tab_Cm] mova m1, [pw_2000] mov r4d, %2/4 lea r5, [3 * r1] .loop: movd m2, [r0] movd m3, [r0 + r1] movd m4, [r0 + 2 * r1] movd m5, [r0 + r5] punpcklbw m2, m3 punpcklbw m6, m4, m5 punpcklbw m2, m6 pmaddubsw m2, m0 lea r0, [r0 + 4 * r1] movd m6, [r0] punpcklbw m3, m4 punpcklbw m7, m5, m6 punpcklbw m3, m7 pmaddubsw m3, m0 phaddw m2, m3 psubw m2, m1 movh [r2], m2 movhps [r2 + r3], m2 movd m2, [r0 + r1] punpcklbw m4, m5 punpcklbw m3, m6, m2 punpcklbw m4, m3 pmaddubsw m4, m0 movd m3, [r0 + 2 * r1] punpcklbw m5, m6 punpcklbw m2, m3 punpcklbw m5, m2 pmaddubsw m5, m0 phaddw m4, m5 psubw m4, m1 lea r2, [r2 + 2 * r3] movh [r2], m4 movhps [r2 + r3], m4 lea r2, [r2 + 2 * r3] dec r4d jnz .loop RET %endmacro FILTER_V_PS_W4_H4 4, 8 FILTER_V_PS_W4_H4 4, 16 FILTER_V_PS_W4_H4 4, 32 ;-------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_8x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- %macro FILTER_V_PS_W8_H8_H16_H2 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_%1x%2, 4, 6, 7 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m5, [r5 + r4 * 4] %else movd m5, [tab_ChromaCoeff + r4 * 4] %endif pshufb m6, m5, [tab_Vm] pshufb m5, [tab_Vm + 16] mova m4, [pw_2000] mov r4d, %2/2 lea r5, [3 * r1] .loopH: movq m0, [r0] movq m1, [r0 + r1] movq m2, [r0 + 2 * r1] movq m3, [r0 + r5] punpcklbw m0, m1 punpcklbw m1, m2 punpcklbw m2, m3 pmaddubsw m0, m6 pmaddubsw m2, m5 paddw m0, m2 psubw m0, m4 movu [r2], m0 movq m0, [r0 + 4 * r1] punpcklbw m3, m0 pmaddubsw m1, m6 pmaddubsw m3, m5 paddw m1, m3 psubw m1, m4 movu [r2 + r3], m1 lea r0, [r0 + 2 * r1] lea r2, [r2 + 2 * r3] dec r4d jnz .loopH RET %endmacro FILTER_V_PS_W8_H8_H16_H2 8, 2 FILTER_V_PS_W8_H8_H16_H2 8, 4 FILTER_V_PS_W8_H8_H16_H2 8, 6 FILTER_V_PS_W8_H8_H16_H2 8, 12 FILTER_V_PS_W8_H8_H16_H2 8, 64 ;-------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_8x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- %macro FILTER_V_PS_W8_H8_H16_H32 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_%1x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m5, [r5 + r4 * 4] %else movd m5, [tab_ChromaCoeff + r4 * 4] %endif pshufb m6, m5, [tab_Vm] pshufb m5, [tab_Vm + 16] mova m4, [pw_2000] mov r4d, %2/4 lea r5, [3 * r1] .loop: movq m0, [r0] movq m1, [r0 + r1] movq m2, [r0 + 2 * r1] movq m3, [r0 + r5] punpcklbw m0, m1 punpcklbw m1, m2 punpcklbw m2, m3 pmaddubsw m0, m6 pmaddubsw m7, m2, m5 paddw m0, m7 psubw m0, m4 movu [r2], m0 lea r0, [r0 + 4 * r1] movq m0, [r0] punpcklbw m3, m0 pmaddubsw m1, m6 pmaddubsw m7, m3, m5 paddw m1, m7 psubw m1, m4 movu [r2 + r3], m1 movq m1, [r0 + r1] punpcklbw m0, m1 pmaddubsw m2, m6 pmaddubsw m0, m5 paddw m2, m0 psubw m2, m4 lea r2, [r2 + 2 * r3] movu [r2], m2 movq m2, [r0 + 2 * r1] punpcklbw m1, m2 pmaddubsw m3, m6 pmaddubsw m1, m5 paddw m3, m1 psubw m3, m4 movu [r2 + r3], m3 lea r2, [r2 + 2 * r3] dec r4d jnz .loop RET %endmacro FILTER_V_PS_W8_H8_H16_H32 8, 8 FILTER_V_PS_W8_H8_H16_H32 8, 16 FILTER_V_PS_W8_H8_H16_H32 8, 32 ;------------------------------------------------------------------------------------------------------------ ;void interp_4tap_vert_ps_6x8(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------ %macro FILTER_V_PS_W6 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_6x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m5, [r5 + r4 * 4] %else movd m5, [tab_ChromaCoeff + r4 * 4] %endif pshufb m6, m5, [tab_Vm] pshufb m5, [tab_Vm + 16] mova m4, [pw_2000] lea r5, [3 * r1] mov r4d, %2/4 .loop: movq m0, [r0] movq m1, [r0 + r1] movq m2, [r0 + 2 * r1] movq m3, [r0 + r5] punpcklbw m0, m1 punpcklbw m1, m2 punpcklbw m2, m3 pmaddubsw m0, m6 pmaddubsw m7, m2, m5 paddw m0, m7 psubw m0, m4 movh [r2], m0 pshufd m0, m0, 2 movd [r2 + 8], m0 lea r0, [r0 + 4 * r1] movq m0, [r0] punpcklbw m3, m0 pmaddubsw m1, m6 pmaddubsw m7, m3, m5 paddw m1, m7 psubw m1, m4 movh [r2 + r3], m1 pshufd m1, m1, 2 movd [r2 + r3 + 8], m1 movq m1, [r0 + r1] punpcklbw m0, m1 pmaddubsw m2, m6 pmaddubsw m0, m5 paddw m2, m0 psubw m2, m4 lea r2,[r2 + 2 * r3] movh [r2], m2 pshufd m2, m2, 2 movd [r2 + 8], m2 movq m2,[r0 + 2 * r1] punpcklbw m1, m2 pmaddubsw m3, m6 pmaddubsw m1, m5 paddw m3, m1 psubw m3, m4 movh [r2 + r3], m3 pshufd m3, m3, 2 movd [r2 + r3 + 8], m3 lea r2, [r2 + 2 * r3] dec r4d jnz .loop RET %endmacro FILTER_V_PS_W6 6, 8 FILTER_V_PS_W6 6, 16 ;--------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_12x16(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;--------------------------------------------------------------------------------------------------------------- %macro FILTER_V_PS_W12 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_12x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mov r4d, %2/2 .loop: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r0, [r0 + 2 * r1] movu m5, [r0] movu m7, [r0 + r1] punpcklbw m6, m5, m7 pmaddubsw m6, m0 paddw m4, m6 punpckhbw m6, m5, m7 pmaddubsw m6, m0 paddw m2, m6 mova m6, [pw_2000] psubw m4, m6 psubw m2, m6 movu [r2], m4 movh [r2 + 16], m2 punpcklbw m4, m3, m5 punpckhbw m3, m5 pmaddubsw m4, m1 pmaddubsw m3, m1 movu m2, [r0 + 2 * r1] punpcklbw m5, m7, m2 punpckhbw m7, m2 pmaddubsw m5, m0 pmaddubsw m7, m0 paddw m4, m5 paddw m3, m7 psubw m4, m6 psubw m3, m6 movu [r2 + r3], m4 movh [r2 + r3 + 16], m3 lea r2, [r2 + 2 * r3] dec r4d jnz .loop RET %endmacro FILTER_V_PS_W12 12, 16 FILTER_V_PS_W12 12, 32 ;--------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_16x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;--------------------------------------------------------------------------------------------------------------- %macro FILTER_V_PS_W16 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_%1x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mov r4d, %2/2 .loop: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r0, [r0 + 2 * r1] movu m5, [r0] movu m7, [r0 + r1] punpcklbw m6, m5, m7 pmaddubsw m6, m0 paddw m4, m6 punpckhbw m6, m5, m7 pmaddubsw m6, m0 paddw m2, m6 mova m6, [pw_2000] psubw m4, m6 psubw m2, m6 movu [r2], m4 movu [r2 + 16], m2 punpcklbw m4, m3, m5 punpckhbw m3, m5 pmaddubsw m4, m1 pmaddubsw m3, m1 movu m5, [r0 + 2 * r1] punpcklbw m2, m7, m5 punpckhbw m7, m5 pmaddubsw m2, m0 pmaddubsw m7, m0 paddw m4, m2 paddw m3, m7 psubw m4, m6 psubw m3, m6 movu [r2 + r3], m4 movu [r2 + r3 + 16], m3 lea r2, [r2 + 2 * r3] dec r4d jnz .loop RET %endmacro FILTER_V_PS_W16 16, 4 FILTER_V_PS_W16 16, 8 FILTER_V_PS_W16 16, 12 FILTER_V_PS_W16 16, 16 FILTER_V_PS_W16 16, 32 FILTER_V_PS_W16 16, 24 FILTER_V_PS_W16 16, 64 ;-------------------------------------------------------------------------------------------------------------- ;void interp_4tap_vert_ps_24x32(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- %macro FILTER_V4_PS_W24 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_24x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mov r4d, %2/2 .loop: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r5, [r0 + 2 * r1] movu m5, [r5] movu m7, [r5 + r1] punpcklbw m6, m5, m7 pmaddubsw m6, m0 paddw m4, m6 punpckhbw m6, m5, m7 pmaddubsw m6, m0 paddw m2, m6 mova m6, [pw_2000] psubw m4, m6 psubw m2, m6 movu [r2], m4 movu [r2 + 16], m2 punpcklbw m4, m3, m5 punpckhbw m3, m5 pmaddubsw m4, m1 pmaddubsw m3, m1 movu m2, [r5 + 2 * r1] punpcklbw m5, m7, m2 punpckhbw m7, m2 pmaddubsw m5, m0 pmaddubsw m7, m0 paddw m4, m5 paddw m3, m7 psubw m4, m6 psubw m3, m6 movu [r2 + r3], m4 movu [r2 + r3 + 16], m3 movq m2, [r0 + 16] movq m3, [r0 + r1 + 16] movq m4, [r5 + 16] movq m5, [r5 + r1 + 16] punpcklbw m2, m3 punpcklbw m7, m4, m5 pmaddubsw m2, m1 pmaddubsw m7, m0 paddw m2, m7 psubw m2, m6 movu [r2 + 32], m2 movq m2, [r5 + 2 * r1 + 16] punpcklbw m3, m4 punpcklbw m5, m2 pmaddubsw m3, m1 pmaddubsw m5, m0 paddw m3, m5 psubw m3, m6 movu [r2 + r3 + 32], m3 mov r0, r5 lea r2, [r2 + 2 * r3] dec r4d jnz .loop RET %endmacro FILTER_V4_PS_W24 24, 32 FILTER_V4_PS_W24 24, 64 ;--------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_32x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;--------------------------------------------------------------------------------------------------------------- %macro FILTER_V_PS_W32 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_%1x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mova m7, [pw_2000] mov r4d, %2 .loop: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r5, [r0 + 2 * r1] movu m3, [r5] movu m5, [r5 + r1] punpcklbw m6, m3, m5 punpckhbw m3, m5 pmaddubsw m6, m0 pmaddubsw m3, m0 paddw m4, m6 paddw m2, m3 psubw m4, m7 psubw m2, m7 movu [r2], m4 movu [r2 + 16], m2 movu m2, [r0 + 16] movu m3, [r0 + r1 + 16] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 movu m3, [r5 + 16] movu m5, [r5 + r1 + 16] punpcklbw m6, m3, m5 punpckhbw m3, m5 pmaddubsw m6, m0 pmaddubsw m3, m0 paddw m4, m6 paddw m2, m3 psubw m4, m7 psubw m2, m7 movu [r2 + 32], m4 movu [r2 + 48], m2 lea r0, [r0 + r1] lea r2, [r2 + r3] dec r4d jnz .loop RET %endmacro FILTER_V_PS_W32 32, 8 FILTER_V_PS_W32 32, 16 FILTER_V_PS_W32 32, 24 FILTER_V_PS_W32 32, 32 FILTER_V_PS_W32 32, 48 FILTER_V_PS_W32 32, 64 ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W8_H8_H16_H32 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_%1x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m5, [r5 + r4 * 4] %else movd m5, [tab_ChromaCoeff + r4 * 4] %endif pshufb m6, m5, [tab_Vm] pshufb m5, [tab_Vm + 16] mova m4, [tab_c_512] lea r5, [r1 * 3] mov r4d, %2 .loop: movq m0, [r0] movq m1, [r0 + r1] movq m2, [r0 + 2 * r1] movq m3, [r0 + r5] punpcklbw m0, m1 punpcklbw m1, m2 punpcklbw m2, m3 pmaddubsw m0, m6 pmaddubsw m7, m2, m5 paddw m0, m7 pmulhrsw m0, m4 packuswb m0, m0 movh [r2], m0 lea r0, [r0 + 4 * r1] movq m0, [r0] punpcklbw m3, m0 pmaddubsw m1, m6 pmaddubsw m7, m3, m5 paddw m1, m7 pmulhrsw m1, m4 packuswb m1, m1 movh [r2 + r3], m1 movq m1, [r0 + r1] punpcklbw m0, m1 pmaddubsw m2, m6 pmaddubsw m0, m5 paddw m2, m0 pmulhrsw m2, m4 movq m7, [r0 + 2 * r1] punpcklbw m1, m7 pmaddubsw m3, m6 pmaddubsw m1, m5 paddw m3, m1 pmulhrsw m3, m4 packuswb m2, m3 lea r2, [r2 + 2 * r3] movh [r2], m2 movhps [r2 + r3], m2 lea r2, [r2 + 2 * r3] sub r4, 4 jnz .loop RET %endmacro FILTER_V4_W8_H8_H16_H32 8, 8 FILTER_V4_W8_H8_H16_H32 8, 16 FILTER_V4_W8_H8_H16_H32 8, 32 FILTER_V4_W8_H8_H16_H32 8, 12 FILTER_V4_W8_H8_H16_H32 8, 64 ;----------------------------------------------------------------------------- ;void interp_4tap_vert_pp_6x8(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W6_H4 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_6x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m5, [r5 + r4 * 4] %else movd m5, [tab_ChromaCoeff + r4 * 4] %endif pshufb m6, m5, [tab_Vm] pshufb m5, [tab_Vm + 16] mova m4, [tab_c_512] mov r4d, %2 lea r5, [3 * r1] .loop: movq m0, [r0] movq m1, [r0 + r1] movq m2, [r0 + 2 * r1] movq m3, [r0 + r5] punpcklbw m0, m1 punpcklbw m1, m2 punpcklbw m2, m3 pmaddubsw m0, m6 pmaddubsw m7, m2, m5 paddw m0, m7 pmulhrsw m0, m4 packuswb m0, m0 movd [r2], m0 pextrw [r2 + 4], m0, 2 lea r0, [r0 + 4 * r1] movq m0, [r0] punpcklbw m3, m0 pmaddubsw m1, m6 pmaddubsw m7, m3, m5 paddw m1, m7 pmulhrsw m1, m4 packuswb m1, m1 movd [r2 + r3], m1 pextrw [r2 + r3 + 4], m1, 2 movq m1, [r0 + r1] punpcklbw m7, m0, m1 pmaddubsw m2, m6 pmaddubsw m7, m5 paddw m2, m7 pmulhrsw m2, m4 packuswb m2, m2 lea r2, [r2 + 2 * r3] movd [r2], m2 pextrw [r2 + 4], m2, 2 movq m2, [r0 + 2 * r1] punpcklbw m1, m2 pmaddubsw m3, m6 pmaddubsw m1, m5 paddw m3, m1 pmulhrsw m3, m4 packuswb m3, m3 movd [r2 + r3], m3 pextrw [r2 + r3 + 4], m3, 2 lea r2, [r2 + 2 * r3] sub r4, 4 jnz .loop RET %endmacro FILTER_V4_W6_H4 6, 8 FILTER_V4_W6_H4 6, 16 ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_12x16(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W12_H2 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_12x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mov r4d, %2 .loop: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r0, [r0 + 2 * r1] movu m5, [r0] movu m7, [r0 + r1] punpcklbw m6, m5, m7 pmaddubsw m6, m0 paddw m4, m6 punpckhbw m6, m5, m7 pmaddubsw m6, m0 paddw m2, m6 mova m6, [tab_c_512] pmulhrsw m4, m6 pmulhrsw m2, m6 packuswb m4, m2 movh [r2], m4 pextrd [r2 + 8], m4, 2 punpcklbw m4, m3, m5 punpckhbw m3, m5 pmaddubsw m4, m1 pmaddubsw m3, m1 movu m5, [r0 + 2 * r1] punpcklbw m2, m7, m5 punpckhbw m7, m5 pmaddubsw m2, m0 pmaddubsw m7, m0 paddw m4, m2 paddw m3, m7 pmulhrsw m4, m6 pmulhrsw m3, m6 packuswb m4, m3 movh [r2 + r3], m4 pextrd [r2 + r3 + 8], m4, 2 lea r2, [r2 + 2 * r3] sub r4, 2 jnz .loop RET %endmacro FILTER_V4_W12_H2 12, 16 FILTER_V4_W12_H2 12, 32 ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_16x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W16_H2 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_16x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mov r4d, %2/2 .loop: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r0, [r0 + 2 * r1] movu m5, [r0] movu m6, [r0 + r1] punpckhbw m7, m5, m6 pmaddubsw m7, m0 paddw m2, m7 punpcklbw m7, m5, m6 pmaddubsw m7, m0 paddw m4, m7 mova m7, [tab_c_512] pmulhrsw m4, m7 pmulhrsw m2, m7 packuswb m4, m2 movu [r2], m4 punpcklbw m4, m3, m5 punpckhbw m3, m5 pmaddubsw m4, m1 pmaddubsw m3, m1 movu m5, [r0 + 2 * r1] punpcklbw m2, m6, m5 punpckhbw m6, m5 pmaddubsw m2, m0 pmaddubsw m6, m0 paddw m4, m2 paddw m3, m6 pmulhrsw m4, m7 pmulhrsw m3, m7 packuswb m4, m3 movu [r2 + r3], m4 lea r2, [r2 + 2 * r3] dec r4d jnz .loop RET %endmacro FILTER_V4_W16_H2 16, 4 FILTER_V4_W16_H2 16, 8 FILTER_V4_W16_H2 16, 12 FILTER_V4_W16_H2 16, 16 FILTER_V4_W16_H2 16, 32 FILTER_V4_W16_H2 16, 24 FILTER_V4_W16_H2 16, 64 ;----------------------------------------------------------------------------- ;void interp_4tap_vert_pp_24x32(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W24 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_24x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mov r4d, %2 .loop: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r5, [r0 + 2 * r1] movu m5, [r5] movu m7, [r5 + r1] punpcklbw m6, m5, m7 pmaddubsw m6, m0 paddw m4, m6 punpckhbw m6, m5, m7 pmaddubsw m6, m0 paddw m2, m6 mova m6, [tab_c_512] pmulhrsw m4, m6 pmulhrsw m2, m6 packuswb m4, m2 movu [r2], m4 punpcklbw m4, m3, m5 punpckhbw m3, m5 pmaddubsw m4, m1 pmaddubsw m3, m1 movu m2, [r5 + 2 * r1] punpcklbw m5, m7, m2 punpckhbw m7, m2 pmaddubsw m5, m0 pmaddubsw m7, m0 paddw m4, m5 paddw m3, m7 pmulhrsw m4, m6 pmulhrsw m3, m6 packuswb m4, m3 movu [r2 + r3], m4 movq m2, [r0 + 16] movq m3, [r0 + r1 + 16] movq m4, [r5 + 16] movq m5, [r5 + r1 + 16] punpcklbw m2, m3 punpcklbw m4, m5 pmaddubsw m2, m1 pmaddubsw m4, m0 paddw m2, m4 pmulhrsw m2, m6 movq m3, [r0 + r1 + 16] movq m4, [r5 + 16] movq m5, [r5 + r1 + 16] movq m7, [r5 + 2 * r1 + 16] punpcklbw m3, m4 punpcklbw m5, m7 pmaddubsw m3, m1 pmaddubsw m5, m0 paddw m3, m5 pmulhrsw m3, m6 packuswb m2, m3 movh [r2 + 16], m2 movhps [r2 + r3 + 16], m2 mov r0, r5 lea r2, [r2 + 2 * r3] sub r4, 2 jnz .loop RET %endmacro FILTER_V4_W24 24, 32 FILTER_V4_W24 24, 64 ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_32x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W32 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_%1x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mova m7, [tab_c_512] mov r4d, %2 .loop: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r5, [r0 + 2 * r1] movu m3, [r5] movu m5, [r5 + r1] punpcklbw m6, m3, m5 punpckhbw m3, m5 pmaddubsw m6, m0 pmaddubsw m3, m0 paddw m4, m6 paddw m2, m3 pmulhrsw m4, m7 pmulhrsw m2, m7 packuswb m4, m2 movu [r2], m4 movu m2, [r0 + 16] movu m3, [r0 + r1 + 16] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 movu m3, [r5 + 16] movu m5, [r5 + r1 + 16] punpcklbw m6, m3, m5 punpckhbw m3, m5 pmaddubsw m6, m0 pmaddubsw m3, m0 paddw m4, m6 paddw m2, m3 pmulhrsw m4, m7 pmulhrsw m2, m7 packuswb m4, m2 movu [r2 + 16], m4 lea r0, [r0 + r1] lea r2, [r2 + r3] dec r4 jnz .loop RET %endmacro FILTER_V4_W32 32, 8 FILTER_V4_W32 32, 16 FILTER_V4_W32 32, 24 FILTER_V4_W32 32, 32 FILTER_V4_W32 32, 48 FILTER_V4_W32 32, 64 ;----------------------------------------------------------------------------- ; void interp_4tap_vert_pp_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------- %macro FILTER_V4_W16n_H2 2 INIT_XMM sse4 cglobal interp_4tap_vert_pp_%1x%2, 4, 7, 8 mov r4d, r4m sub r0, r1 %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mov r4d, %2/2 .loop: mov r6d, %1/16 .loopW: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r5, [r0 + 2 * r1] movu m5, [r5] movu m6, [r5 + r1] punpckhbw m7, m5, m6 pmaddubsw m7, m0 paddw m2, m7 punpcklbw m7, m5, m6 pmaddubsw m7, m0 paddw m4, m7 mova m7, [tab_c_512] pmulhrsw m4, m7 pmulhrsw m2, m7 packuswb m4, m2 movu [r2], m4 punpcklbw m4, m3, m5 punpckhbw m3, m5 pmaddubsw m4, m1 pmaddubsw m3, m1 movu m5, [r5 + 2 * r1] punpcklbw m2, m6, m5 punpckhbw m6, m5 pmaddubsw m2, m0 pmaddubsw m6, m0 paddw m4, m2 paddw m3, m6 pmulhrsw m4, m7 pmulhrsw m3, m7 packuswb m4, m3 movu [r2 + r3], m4 add r0, 16 add r2, 16 dec r6d jnz .loopW lea r0, [r0 + r1 * 2 - %1] lea r2, [r2 + r3 * 2 - %1] dec r4d jnz .loop RET %endmacro FILTER_V4_W16n_H2 64, 64 FILTER_V4_W16n_H2 64, 32 FILTER_V4_W16n_H2 64, 48 FILTER_V4_W16n_H2 48, 64 FILTER_V4_W16n_H2 64, 16 ;----------------------------------------------------------------------------- ; void filterConvertPelToShort(pixel *src, intptr_t srcStride, int16_t *dst, int width, int height) ;----------------------------------------------------------------------------- INIT_XMM ssse3 cglobal luma_p2s, 3, 7, 6 ; load width and height mov r3d, r3m mov r4d, r4m ; load constant mova m4, [tab_c_128] mova m5, [tab_c_64_n64] .loopH: xor r5d, r5d .loopW: lea r6, [r0 + r5] movh m0, [r6] punpcklbw m0, m4 pmaddubsw m0, m5 movh m1, [r6 + r1] punpcklbw m1, m4 pmaddubsw m1, m5 movh m2, [r6 + r1 * 2] punpcklbw m2, m4 pmaddubsw m2, m5 lea r6, [r6 + r1 * 2] movh m3, [r6 + r1] punpcklbw m3, m4 pmaddubsw m3, m5 add r5, 8 cmp r5, r3 jg .width4 movu [r2 + r5 * 2 + FENC_STRIDE * 0 - 16], m0 movu [r2 + r5 * 2 + FENC_STRIDE * 2 - 16], m1 movu [r2 + r5 * 2 + FENC_STRIDE * 4 - 16], m2 movu [r2 + r5 * 2 + FENC_STRIDE * 6 - 16], m3 je .nextH jmp .loopW .width4: movh [r2 + r5 * 2 + FENC_STRIDE * 0 - 16], m0 movh [r2 + r5 * 2 + FENC_STRIDE * 2 - 16], m1 movh [r2 + r5 * 2 + FENC_STRIDE * 4 - 16], m2 movh [r2 + r5 * 2 + FENC_STRIDE * 6 - 16], m3 .nextH: lea r0, [r0 + r1 * 4] add r2, FENC_STRIDE * 8 sub r4d, 4 jnz .loopH RET %macro PROCESS_LUMA_W4_4R 0 movd m0, [r0] movd m1, [r0 + r1] punpcklbw m2, m0, m1 ; m2=[0 1] lea r0, [r0 + 2 * r1] movd m0, [r0] punpcklbw m1, m0 ; m1=[1 2] punpcklqdq m2, m1 ; m2=[0 1 1 2] pmaddubsw m4, m2, [r6 + 0 * 16] ; m4=[0+1 1+2] movd m1, [r0 + r1] punpcklbw m5, m0, m1 ; m2=[2 3] lea r0, [r0 + 2 * r1] movd m0, [r0] punpcklbw m1, m0 ; m1=[3 4] punpcklqdq m5, m1 ; m5=[2 3 3 4] pmaddubsw m2, m5, [r6 + 1 * 16] ; m2=[2+3 3+4] paddw m4, m2 ; m4=[0+1+2+3 1+2+3+4] Row1-2 pmaddubsw m5, [r6 + 0 * 16] ; m5=[2+3 3+4] Row3-4 movd m1, [r0 + r1] punpcklbw m2, m0, m1 ; m2=[4 5] lea r0, [r0 + 2 * r1] movd m0, [r0] punpcklbw m1, m0 ; m1=[5 6] punpcklqdq m2, m1 ; m2=[4 5 5 6] pmaddubsw m1, m2, [r6 + 2 * 16] ; m1=[4+5 5+6] paddw m4, m1 ; m4=[0+1+2+3+4+5 1+2+3+4+5+6] Row1-2 pmaddubsw m2, [r6 + 1 * 16] ; m2=[4+5 5+6] paddw m5, m2 ; m5=[2+3+4+5 3+4+5+6] Row3-4 movd m1, [r0 + r1] punpcklbw m2, m0, m1 ; m2=[6 7] lea r0, [r0 + 2 * r1] movd m0, [r0] punpcklbw m1, m0 ; m1=[7 8] punpcklqdq m2, m1 ; m2=[6 7 7 8] pmaddubsw m1, m2, [r6 + 3 * 16] ; m1=[6+7 7+8] paddw m4, m1 ; m4=[0+1+2+3+4+5+6+7 1+2+3+4+5+6+7+8] Row1-2 end pmaddubsw m2, [r6 + 2 * 16] ; m2=[6+7 7+8] paddw m5, m2 ; m5=[2+3+4+5+6+7 3+4+5+6+7+8] Row3-4 movd m1, [r0 + r1] punpcklbw m2, m0, m1 ; m2=[8 9] movd m0, [r0 + 2 * r1] punpcklbw m1, m0 ; m1=[9 10] punpcklqdq m2, m1 ; m2=[8 9 9 10] pmaddubsw m2, [r6 + 3 * 16] ; m2=[8+9 9+10] paddw m5, m2 ; m5=[2+3+4+5+6+7+8+9 3+4+5+6+7+8+9+10] Row3-4 end %endmacro %macro PROCESS_LUMA_W8_4R 0 movq m0, [r0] movq m1, [r0 + r1] punpcklbw m0, m1 pmaddubsw m7, m0, [r6 + 0 *16] ;m7=[0+1] Row1 lea r0, [r0 + 2 * r1] movq m0, [r0] punpcklbw m1, m0 pmaddubsw m6, m1, [r6 + 0 *16] ;m6=[1+2] Row2 movq m1, [r0 + r1] punpcklbw m0, m1 pmaddubsw m5, m0, [r6 + 0 *16] ;m5=[2+3] Row3 pmaddubsw m0, [r6 + 1 * 16] paddw m7, m0 ;m7=[0+1+2+3] Row1 lea r0, [r0 + 2 * r1] movq m0, [r0] punpcklbw m1, m0 pmaddubsw m4, m1, [r6 + 0 *16] ;m4=[3+4] Row4 pmaddubsw m1, [r6 + 1 * 16] paddw m6, m1 ;m6 = [1+2+3+4] Row2 movq m1, [r0 + r1] punpcklbw m0, m1 pmaddubsw m2, m0, [r6 + 1 * 16] pmaddubsw m0, [r6 + 2 * 16] paddw m7, m0 ;m7=[0+1+2+3+4+5] Row1 paddw m5, m2 ;m5=[2+3+4+5] Row3 lea r0, [r0 + 2 * r1] movq m0, [r0] punpcklbw m1, m0 pmaddubsw m2, m1, [r6 + 1 * 16] pmaddubsw m1, [r6 + 2 * 16] paddw m6, m1 ;m6=[1+2+3+4+5+6] Row2 paddw m4, m2 ;m4=[3+4+5+6] Row4 movq m1, [r0 + r1] punpcklbw m0, m1 pmaddubsw m2, m0, [r6 + 2 * 16] pmaddubsw m0, [r6 + 3 * 16] paddw m7, m0 ;m7=[0+1+2+3+4+5+6+7] Row1 end paddw m5, m2 ;m5=[2+3+4+5+6+7] Row3 lea r0, [r0 + 2 * r1] movq m0, [r0] punpcklbw m1, m0 pmaddubsw m2, m1, [r6 + 2 * 16] pmaddubsw m1, [r6 + 3 * 16] paddw m6, m1 ;m6=[1+2+3+4+5+6+7+8] Row2 end paddw m4, m2 ;m4=[3+4+5+6+7+8] Row4 movq m1, [r0 + r1] punpcklbw m0, m1 pmaddubsw m0, [r6 + 3 * 16] paddw m5, m0 ;m5=[2+3+4+5+6+7+8+9] Row3 end movq m0, [r0 + 2 * r1] punpcklbw m1, m0 pmaddubsw m1, [r6 + 3 * 16] paddw m4, m1 ;m4=[3+4+5+6+7+8+9+10] Row4 end %endmacro ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_%3_4x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_LUMA_4xN 3 INIT_XMM sse4 cglobal interp_8tap_vert_%3_%1x%2, 5, 7, 6 lea r5, [3 * r1] sub r0, r5 shl r4d, 6 %ifidn %3,ps add r3d, r3d %endif %ifdef PIC lea r5, [tab_LumaCoeffVer] lea r6, [r5 + r4] %else lea r6, [tab_LumaCoeffVer + r4] %endif %ifidn %3,pp mova m3, [tab_c_512] %else mova m3, [pw_2000] %endif mov r4d, %2/4 lea r5, [4 * r1] .loopH: PROCESS_LUMA_W4_4R %ifidn %3,pp pmulhrsw m4, m3 pmulhrsw m5, m3 packuswb m4, m5 movd [r2], m4 pextrd [r2 + r3], m4, 1 lea r2, [r2 + 2 * r3] pextrd [r2], m4, 2 pextrd [r2 + r3], m4, 3 %else psubw m4, m3 psubw m5, m3 movlps [r2], m4 movhps [r2 + r3], m4 lea r2, [r2 + 2 * r3] movlps [r2], m5 movhps [r2 + r3], m5 %endif sub r0, r5 lea r2, [r2 + 2 * r3] dec r4d jnz .loopH RET %endmacro ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_pp_4x4(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_4xN 4, 4, pp ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_pp_4x8(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_4xN 4, 8, pp ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_pp_4x16(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_4xN 4, 16, pp ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_ps_4x4(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_4xN 4, 4, ps ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_ps_4x8(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_4xN 4, 8, ps ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_ps_4x16(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_4xN 4, 16, ps ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_%3_8x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_LUMA_8xN 3 INIT_XMM sse4 cglobal interp_8tap_vert_%3_%1x%2, 5, 7, 8 lea r5, [3 * r1] sub r0, r5 shl r4d, 6 %ifidn %3,ps add r3d, r3d %endif %ifdef PIC lea r5, [tab_LumaCoeffVer] lea r6, [r5 + r4] %else lea r6, [tab_LumaCoeffVer + r4] %endif %ifidn %3,pp mova m3, [tab_c_512] %else mova m3, [pw_2000] %endif mov r4d, %2/4 lea r5, [4 * r1] .loopH: PROCESS_LUMA_W8_4R %ifidn %3,pp pmulhrsw m7, m3 pmulhrsw m6, m3 pmulhrsw m5, m3 pmulhrsw m4, m3 packuswb m7, m6 packuswb m5, m4 movlps [r2], m7 movhps [r2 + r3], m7 lea r2, [r2 + 2 * r3] movlps [r2], m5 movhps [r2 + r3], m5 %else psubw m7, m3 psubw m6, m3 psubw m5, m3 psubw m4, m3 movu [r2], m7 movu [r2 + r3], m6 lea r2, [r2 + 2 * r3] movu [r2], m5 movu [r2 + r3], m4 %endif sub r0, r5 lea r2, [r2 + 2 * r3] dec r4d jnz .loopH RET %endmacro ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_pp_8x4(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_8xN 8, 4, pp ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_pp_8x8(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_8xN 8, 8, pp ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_pp_8x16(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_8xN 8, 16, pp ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_pp_8x32(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_8xN 8, 32, pp ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_ps_8x4(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_8xN 8, 4, ps ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_ps_8x8(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_8xN 8, 8, ps ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_ps_8x16(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_8xN 8, 16, ps ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_ps_8x32(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_8xN 8, 32, ps ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_%3_12x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_LUMA_12xN 3 INIT_XMM sse4 cglobal interp_8tap_vert_%3_%1x%2, 5, 7, 8 lea r5, [3 * r1] sub r0, r5 shl r4d, 6 %ifidn %3,ps add r3d, r3d %endif %ifdef PIC lea r5, [tab_LumaCoeffVer] lea r6, [r5 + r4] %else lea r6, [tab_LumaCoeffVer + r4] %endif %ifidn %3,pp mova m3, [tab_c_512] %else mova m3, [pw_2000] %endif mov r4d, %2/4 .loopH: PROCESS_LUMA_W8_4R %ifidn %3,pp pmulhrsw m7, m3 pmulhrsw m6, m3 pmulhrsw m5, m3 pmulhrsw m4, m3 packuswb m7, m6 packuswb m5, m4 movlps [r2], m7 movhps [r2 + r3], m7 lea r5, [r2 + 2 * r3] movlps [r5], m5 movhps [r5 + r3], m5 %else psubw m7, m3 psubw m6, m3 psubw m5, m3 psubw m4, m3 movu [r2], m7 movu [r2 + r3], m6 lea r5, [r2 + 2 * r3] movu [r5], m5 movu [r5 + r3], m4 %endif lea r5, [8 * r1 - 8] sub r0, r5 %ifidn %3,pp add r2, 8 %else add r2, 16 %endif PROCESS_LUMA_W4_4R %ifidn %3,pp pmulhrsw m4, m3 pmulhrsw m5, m3 packuswb m4, m5 movd [r2], m4 pextrd [r2 + r3], m4, 1 lea r5, [r2 + 2 * r3] pextrd [r5], m4, 2 pextrd [r5 + r3], m4, 3 %else psubw m4, m3 psubw m5, m3 movlps [r2], m4 movhps [r2 + r3], m4 lea r5, [r2 + 2 * r3] movlps [r5], m5 movhps [r5 + r3], m5 %endif lea r5, [4 * r1 + 8] sub r0, r5 %ifidn %3,pp lea r2, [r2 + 4 * r3 - 8] %else lea r2, [r2 + 4 * r3 - 16] %endif dec r4d jnz .loopH RET %endmacro ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_pp_12x16(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_12xN 12, 16, pp ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_ps_12x16(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_12xN 12, 16, ps ;------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_%3_%1x%2(pixel *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_LUMA 3 INIT_XMM sse4 cglobal interp_8tap_vert_%3_%1x%2, 5, 7, 8 ,0-gprsize lea r5, [3 * r1] sub r0, r5 shl r4d, 6 %ifidn %3,ps add r3d, r3d %endif %ifdef PIC lea r5, [tab_LumaCoeffVer] lea r6, [r5 + r4] %else lea r6, [tab_LumaCoeffVer + r4] %endif %ifidn %3,pp mova m3, [tab_c_512] %else mova m3, [pw_2000] %endif mov dword [rsp], %2/4 .loopH: mov r4d, (%1/8) .loopW: PROCESS_LUMA_W8_4R %ifidn %3,pp pmulhrsw m7, m3 pmulhrsw m6, m3 pmulhrsw m5, m3 pmulhrsw m4, m3 packuswb m7, m6 packuswb m5, m4 movlps [r2], m7 movhps [r2 + r3], m7 lea r5, [r2 + 2 * r3] movlps [r5], m5 movhps [r5 + r3], m5 %else psubw m7, m3 psubw m6, m3 psubw m5, m3 psubw m4, m3 movu [r2], m7 movu [r2 + r3], m6 lea r5, [r2 + 2 * r3] movu [r5], m5 movu [r5 + r3], m4 %endif lea r5, [8 * r1 - 8] sub r0, r5 %ifidn %3,pp add r2, 8 %else add r2, 16 %endif dec r4d jnz .loopW lea r0, [r0 + 4 * r1 - %1] %ifidn %3,pp lea r2, [r2 + 4 * r3 - %1] %else lea r2, [r2 + 4 * r3 - 2 * %1] %endif dec dword [rsp] jnz .loopH RET %endmacro FILTER_VER_LUMA 16, 4, pp FILTER_VER_LUMA 16, 8, pp FILTER_VER_LUMA 16, 12, pp FILTER_VER_LUMA 16, 16, pp FILTER_VER_LUMA 16, 32, pp FILTER_VER_LUMA 16, 64, pp FILTER_VER_LUMA 24, 32, pp FILTER_VER_LUMA 32, 8, pp FILTER_VER_LUMA 32, 16, pp FILTER_VER_LUMA 32, 24, pp FILTER_VER_LUMA 32, 32, pp FILTER_VER_LUMA 32, 64, pp FILTER_VER_LUMA 48, 64, pp FILTER_VER_LUMA 64, 16, pp FILTER_VER_LUMA 64, 32, pp FILTER_VER_LUMA 64, 48, pp FILTER_VER_LUMA 64, 64, pp FILTER_VER_LUMA 16, 4, ps FILTER_VER_LUMA 16, 8, ps FILTER_VER_LUMA 16, 12, ps FILTER_VER_LUMA 16, 16, ps FILTER_VER_LUMA 16, 32, ps FILTER_VER_LUMA 16, 64, ps FILTER_VER_LUMA 24, 32, ps FILTER_VER_LUMA 32, 8, ps FILTER_VER_LUMA 32, 16, ps FILTER_VER_LUMA 32, 24, ps FILTER_VER_LUMA 32, 32, ps FILTER_VER_LUMA 32, 64, ps FILTER_VER_LUMA 48, 64, ps FILTER_VER_LUMA 64, 16, ps FILTER_VER_LUMA 64, 32, ps FILTER_VER_LUMA 64, 48, ps FILTER_VER_LUMA 64, 64, ps %macro PROCESS_LUMA_SP_W4_4R 0 movq m0, [r0] movq m1, [r0 + r1] punpcklwd m0, m1 ;m0=[0 1] pmaddwd m0, [r6 + 0 *16] ;m0=[0+1] Row1 lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m1, m4 ;m1=[1 2] pmaddwd m1, [r6 + 0 *16] ;m1=[1+2] Row2 movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[2 3] pmaddwd m2, m4, [r6 + 0 *16] ;m2=[2+3] Row3 pmaddwd m4, [r6 + 1 * 16] paddd m0, m4 ;m0=[0+1+2+3] Row1 lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m5, m4 ;m5=[3 4] pmaddwd m3, m5, [r6 + 0 *16] ;m3=[3+4] Row4 pmaddwd m5, [r6 + 1 * 16] paddd m1, m5 ;m1 = [1+2+3+4] Row2 movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[4 5] pmaddwd m6, m4, [r6 + 1 * 16] paddd m2, m6 ;m2=[2+3+4+5] Row3 pmaddwd m4, [r6 + 2 * 16] paddd m0, m4 ;m0=[0+1+2+3+4+5] Row1 lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m5, m4 ;m5=[5 6] pmaddwd m6, m5, [r6 + 1 * 16] paddd m3, m6 ;m3=[3+4+5+6] Row4 pmaddwd m5, [r6 + 2 * 16] paddd m1, m5 ;m1=[1+2+3+4+5+6] Row2 movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[6 7] pmaddwd m6, m4, [r6 + 2 * 16] paddd m2, m6 ;m2=[2+3+4+5+6+7] Row3 pmaddwd m4, [r6 + 3 * 16] paddd m0, m4 ;m0=[0+1+2+3+4+5+6+7] Row1 end lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m5, m4 ;m5=[7 8] pmaddwd m6, m5, [r6 + 2 * 16] paddd m3, m6 ;m3=[3+4+5+6+7+8] Row4 pmaddwd m5, [r6 + 3 * 16] paddd m1, m5 ;m1=[1+2+3+4+5+6+7+8] Row2 end movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[8 9] pmaddwd m4, [r6 + 3 * 16] paddd m2, m4 ;m2=[2+3+4+5+6+7+8+9] Row3 end movq m4, [r0 + 2 * r1] punpcklwd m5, m4 ;m5=[9 10] pmaddwd m5, [r6 + 3 * 16] paddd m3, m5 ;m3=[3+4+5+6+7+8+9+10] Row4 end %endmacro ;-------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_sp_%1x%2(int16_t *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_LUMA_SP 2 INIT_XMM sse4 cglobal interp_8tap_vert_sp_%1x%2, 5, 7, 8 ,0-gprsize add r1d, r1d lea r5, [r1 + 2 * r1] sub r0, r5 shl r4d, 6 %ifdef PIC lea r5, [tab_LumaCoeffV] lea r6, [r5 + r4] %else lea r6, [tab_LumaCoeffV + r4] %endif mova m7, [tab_c_526336] mov dword [rsp], %2/4 .loopH: mov r4d, (%1/4) .loopW: PROCESS_LUMA_SP_W4_4R paddd m0, m7 paddd m1, m7 paddd m2, m7 paddd m3, m7 psrad m0, 12 psrad m1, 12 psrad m2, 12 psrad m3, 12 packssdw m0, m1 packssdw m2, m3 packuswb m0, m2 movd [r2], m0 pextrd [r2 + r3], m0, 1 lea r5, [r2 + 2 * r3] pextrd [r5], m0, 2 pextrd [r5 + r3], m0, 3 lea r5, [8 * r1 - 2 * 4] sub r0, r5 add r2, 4 dec r4d jnz .loopW lea r0, [r0 + 4 * r1 - 2 * %1] lea r2, [r2 + 4 * r3 - %1] dec dword [rsp] jnz .loopH RET %endmacro ;-------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_sp_%1x%2(int16_t *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- FILTER_VER_LUMA_SP 4, 4 FILTER_VER_LUMA_SP 8, 8 FILTER_VER_LUMA_SP 8, 4 FILTER_VER_LUMA_SP 4, 8 FILTER_VER_LUMA_SP 16, 16 FILTER_VER_LUMA_SP 16, 8 FILTER_VER_LUMA_SP 8, 16 FILTER_VER_LUMA_SP 16, 12 FILTER_VER_LUMA_SP 12, 16 FILTER_VER_LUMA_SP 16, 4 FILTER_VER_LUMA_SP 4, 16 FILTER_VER_LUMA_SP 32, 32 FILTER_VER_LUMA_SP 32, 16 FILTER_VER_LUMA_SP 16, 32 FILTER_VER_LUMA_SP 32, 24 FILTER_VER_LUMA_SP 24, 32 FILTER_VER_LUMA_SP 32, 8 FILTER_VER_LUMA_SP 8, 32 FILTER_VER_LUMA_SP 64, 64 FILTER_VER_LUMA_SP 64, 32 FILTER_VER_LUMA_SP 32, 64 FILTER_VER_LUMA_SP 64, 48 FILTER_VER_LUMA_SP 48, 64 FILTER_VER_LUMA_SP 64, 16 FILTER_VER_LUMA_SP 16, 64 ; TODO: combin of U and V is more performance, but need more register ; TODO: use two path for height alignment to 4 and otherwise may improvement 10% performance, but code is more complex, so I disable it INIT_XMM ssse3 cglobal chroma_p2s, 3, 7, 4 ; load width and height mov r3d, r3m mov r4d, r4m ; load constant mova m2, [tab_c_128] mova m3, [tab_c_64_n64] .loopH: xor r5d, r5d .loopW: lea r6, [r0 + r5] movh m0, [r6] punpcklbw m0, m2 pmaddubsw m0, m3 movh m1, [r6 + r1] punpcklbw m1, m2 pmaddubsw m1, m3 add r5d, 8 cmp r5d, r3d lea r6, [r2 + r5 * 2] jg .width4 movu [r6 + FENC_STRIDE / 2 * 0 - 16], m0 movu [r6 + FENC_STRIDE / 2 * 2 - 16], m1 je .nextH jmp .loopW .width4: test r3d, 4 jz .width2 test r3d, 2 movh [r6 + FENC_STRIDE / 2 * 0 - 16], m0 movh [r6 + FENC_STRIDE / 2 * 2 - 16], m1 lea r6, [r6 + 8] pshufd m0, m0, 2 pshufd m1, m1, 2 jz .nextH .width2: movd [r6 + FENC_STRIDE / 2 * 0 - 16], m0 movd [r6 + FENC_STRIDE / 2 * 2 - 16], m1 .nextH: lea r0, [r0 + r1 * 2] add r2, FENC_STRIDE / 2 * 4 sub r4d, 2 jnz .loopH RET %macro PROCESS_CHROMA_SP_W4_4R 0 movq m0, [r0] movq m1, [r0 + r1] punpcklwd m0, m1 ;m0=[0 1] pmaddwd m0, [r6 + 0 *16] ;m0=[0+1] Row1 lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m1, m4 ;m1=[1 2] pmaddwd m1, [r6 + 0 *16] ;m1=[1+2] Row2 movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[2 3] pmaddwd m2, m4, [r6 + 0 *16] ;m2=[2+3] Row3 pmaddwd m4, [r6 + 1 * 16] paddd m0, m4 ;m0=[0+1+2+3] Row1 done lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m5, m4 ;m5=[3 4] pmaddwd m3, m5, [r6 + 0 *16] ;m3=[3+4] Row4 pmaddwd m5, [r6 + 1 * 16] paddd m1, m5 ;m1 = [1+2+3+4] Row2 movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[4 5] pmaddwd m4, [r6 + 1 * 16] paddd m2, m4 ;m2=[2+3+4+5] Row3 movq m4, [r0 + 2 * r1] punpcklwd m5, m4 ;m5=[5 6] pmaddwd m5, [r6 + 1 * 16] paddd m3, m5 ;m3=[3+4+5+6] Row4 %endmacro ;-------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_sp_%1x%2(int16_t *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_CHROMA_SP 2 INIT_XMM sse4 cglobal interp_4tap_vert_sp_%1x%2, 5, 7, 7 ,0-gprsize add r1d, r1d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r6, [r5 + r4] %else lea r6, [tab_ChromaCoeffV + r4] %endif mova m6, [tab_c_526336] mov dword [rsp], %2/4 .loopH: mov r4d, (%1/4) .loopW: PROCESS_CHROMA_SP_W4_4R paddd m0, m6 paddd m1, m6 paddd m2, m6 paddd m3, m6 psrad m0, 12 psrad m1, 12 psrad m2, 12 psrad m3, 12 packssdw m0, m1 packssdw m2, m3 packuswb m0, m2 movd [r2], m0 pextrd [r2 + r3], m0, 1 lea r5, [r2 + 2 * r3] pextrd [r5], m0, 2 pextrd [r5 + r3], m0, 3 lea r5, [4 * r1 - 2 * 4] sub r0, r5 add r2, 4 dec r4d jnz .loopW lea r0, [r0 + 4 * r1 - 2 * %1] lea r2, [r2 + 4 * r3 - %1] dec dword [rsp] jnz .loopH RET %endmacro FILTER_VER_CHROMA_SP 4, 4 FILTER_VER_CHROMA_SP 4, 8 FILTER_VER_CHROMA_SP 16, 16 FILTER_VER_CHROMA_SP 16, 8 FILTER_VER_CHROMA_SP 16, 12 FILTER_VER_CHROMA_SP 12, 16 FILTER_VER_CHROMA_SP 16, 4 FILTER_VER_CHROMA_SP 4, 16 FILTER_VER_CHROMA_SP 32, 32 FILTER_VER_CHROMA_SP 32, 16 FILTER_VER_CHROMA_SP 16, 32 FILTER_VER_CHROMA_SP 32, 24 FILTER_VER_CHROMA_SP 24, 32 FILTER_VER_CHROMA_SP 32, 8 FILTER_VER_CHROMA_SP 16, 24 FILTER_VER_CHROMA_SP 16, 64 FILTER_VER_CHROMA_SP 12, 32 FILTER_VER_CHROMA_SP 4, 32 FILTER_VER_CHROMA_SP 32, 64 FILTER_VER_CHROMA_SP 32, 48 FILTER_VER_CHROMA_SP 24, 64 FILTER_VER_CHROMA_SP 64, 64 FILTER_VER_CHROMA_SP 64, 32 FILTER_VER_CHROMA_SP 64, 48 FILTER_VER_CHROMA_SP 48, 64 FILTER_VER_CHROMA_SP 64, 16 %macro PROCESS_CHROMA_SP_W2_4R 1 movd m0, [r0] movd m1, [r0 + r1] punpcklwd m0, m1 ;m0=[0 1] lea r0, [r0 + 2 * r1] movd m2, [r0] punpcklwd m1, m2 ;m1=[1 2] punpcklqdq m0, m1 ;m0=[0 1 1 2] pmaddwd m0, [%1 + 0 *16] ;m0=[0+1 1+2] Row 1-2 movd m1, [r0 + r1] punpcklwd m2, m1 ;m2=[2 3] lea r0, [r0 + 2 * r1] movd m3, [r0] punpcklwd m1, m3 ;m2=[3 4] punpcklqdq m2, m1 ;m2=[2 3 3 4] pmaddwd m4, m2, [%1 + 1 * 16] ;m4=[2+3 3+4] Row 1-2 pmaddwd m2, [%1 + 0 * 16] ;m2=[2+3 3+4] Row 3-4 paddd m0, m4 ;m0=[0+1+2+3 1+2+3+4] Row 1-2 movd m1, [r0 + r1] punpcklwd m3, m1 ;m3=[4 5] movd m4, [r0 + 2 * r1] punpcklwd m1, m4 ;m1=[5 6] punpcklqdq m3, m1 ;m2=[4 5 5 6] pmaddwd m3, [%1 + 1 * 16] ;m3=[4+5 5+6] Row 3-4 paddd m2, m3 ;m2=[2+3+4+5 3+4+5+6] Row 3-4 %endmacro ;------------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vertical_sp_%1x%2(int16_t *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_CHROMA_SP_W2_4R 2 INIT_XMM sse4 cglobal interp_4tap_vert_sp_%1x%2, 5, 6, 6 add r1d, r1d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r5, [r5 + r4] %else lea r5, [tab_ChromaCoeffV + r4] %endif mova m5, [tab_c_526336] mov r4d, (%2/4) .loopH: PROCESS_CHROMA_SP_W2_4R r5 paddd m0, m5 paddd m2, m5 psrad m0, 12 psrad m2, 12 packssdw m0, m2 packuswb m0, m0 pextrw [r2], m0, 0 pextrw [r2 + r3], m0, 1 lea r2, [r2 + 2 * r3] pextrw [r2], m0, 2 pextrw [r2 + r3], m0, 3 lea r2, [r2 + 2 * r3] dec r4d jnz .loopH RET %endmacro FILTER_VER_CHROMA_SP_W2_4R 2, 4 FILTER_VER_CHROMA_SP_W2_4R 2, 8 FILTER_VER_CHROMA_SP_W2_4R 2, 16 ;-------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_sp_4x2(int16_t *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- INIT_XMM sse4 cglobal interp_4tap_vert_sp_4x2, 5, 6, 5 add r1d, r1d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r5, [r5 + r4] %else lea r5, [tab_ChromaCoeffV + r4] %endif mova m4, [tab_c_526336] movq m0, [r0] movq m1, [r0 + r1] punpcklwd m0, m1 ;m0=[0 1] pmaddwd m0, [r5 + 0 *16] ;m0=[0+1] Row1 lea r0, [r0 + 2 * r1] movq m2, [r0] punpcklwd m1, m2 ;m1=[1 2] pmaddwd m1, [r5 + 0 *16] ;m1=[1+2] Row2 movq m3, [r0 + r1] punpcklwd m2, m3 ;m4=[2 3] pmaddwd m2, [r5 + 1 * 16] paddd m0, m2 ;m0=[0+1+2+3] Row1 done paddd m0, m4 psrad m0, 12 movq m2, [r0 + 2 * r1] punpcklwd m3, m2 ;m5=[3 4] pmaddwd m3, [r5 + 1 * 16] paddd m1, m3 ;m1 = [1+2+3+4] Row2 done paddd m1, m4 psrad m1, 12 packssdw m0, m1 packuswb m0, m0 movd [r2], m0 pextrd [r2 + r3], m0, 1 RET ;------------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vertical_sp_6x%2(int16_t *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_CHROMA_SP_W6_H4 2 INIT_XMM sse4 cglobal interp_4tap_vert_sp_6x%2, 5, 7, 7 add r1d, r1d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r6, [r5 + r4] %else lea r6, [tab_ChromaCoeffV + r4] %endif mova m6, [tab_c_526336] mov r4d, %2/4 .loopH: PROCESS_CHROMA_SP_W4_4R paddd m0, m6 paddd m1, m6 paddd m2, m6 paddd m3, m6 psrad m0, 12 psrad m1, 12 psrad m2, 12 psrad m3, 12 packssdw m0, m1 packssdw m2, m3 packuswb m0, m2 movd [r2], m0 pextrd [r2 + r3], m0, 1 lea r5, [r2 + 2 * r3] pextrd [r5], m0, 2 pextrd [r5 + r3], m0, 3 lea r5, [4 * r1 - 2 * 4] sub r0, r5 add r2, 4 PROCESS_CHROMA_SP_W2_4R r6 paddd m0, m6 paddd m2, m6 psrad m0, 12 psrad m2, 12 packssdw m0, m2 packuswb m0, m0 pextrw [r2], m0, 0 pextrw [r2 + r3], m0, 1 lea r2, [r2 + 2 * r3] pextrw [r2], m0, 2 pextrw [r2 + r3], m0, 3 sub r0, 2 * 4 lea r2, [r2 + 2 * r3 - 4] dec r4d jnz .loopH RET %endmacro FILTER_VER_CHROMA_SP_W6_H4 6, 8 FILTER_VER_CHROMA_SP_W6_H4 6, 16 %macro PROCESS_CHROMA_SP_W8_2R 0 movu m1, [r0] movu m3, [r0 + r1] punpcklwd m0, m1, m3 pmaddwd m0, [r5 + 0 * 16] ;m0 = [0l+1l] Row1l punpckhwd m1, m3 pmaddwd m1, [r5 + 0 * 16] ;m1 = [0h+1h] Row1h movu m4, [r0 + 2 * r1] punpcklwd m2, m3, m4 pmaddwd m2, [r5 + 0 * 16] ;m2 = [1l+2l] Row2l punpckhwd m3, m4 pmaddwd m3, [r5 + 0 * 16] ;m3 = [1h+2h] Row2h lea r0, [r0 + 2 * r1] movu m5, [r0 + r1] punpcklwd m6, m4, m5 pmaddwd m6, [r5 + 1 * 16] ;m6 = [2l+3l] Row1l paddd m0, m6 ;m0 = [0l+1l+2l+3l] Row1l sum punpckhwd m4, m5 pmaddwd m4, [r5 + 1 * 16] ;m6 = [2h+3h] Row1h paddd m1, m4 ;m1 = [0h+1h+2h+3h] Row1h sum movu m4, [r0 + 2 * r1] punpcklwd m6, m5, m4 pmaddwd m6, [r5 + 1 * 16] ;m6 = [3l+4l] Row2l paddd m2, m6 ;m2 = [1l+2l+3l+4l] Row2l sum punpckhwd m5, m4 pmaddwd m5, [r5 + 1 * 16] ;m1 = [3h+4h] Row2h paddd m3, m5 ;m3 = [1h+2h+3h+4h] Row2h sum %endmacro ;-------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_sp_8x%2(int16_t *src, intptr_t srcStride, pixel *dst, intptr_t dstStride, int coeffIdx) ;-------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_CHROMA_SP_W8_H2 2 INIT_XMM sse2 cglobal interp_4tap_vert_sp_%1x%2, 5, 6, 8 add r1d, r1d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r5, [r5 + r4] %else lea r5, [tab_ChromaCoeffV + r4] %endif mova m7, [tab_c_526336] mov r4d, %2/2 .loopH: PROCESS_CHROMA_SP_W8_2R paddd m0, m7 paddd m1, m7 paddd m2, m7 paddd m3, m7 psrad m0, 12 psrad m1, 12 psrad m2, 12 psrad m3, 12 packssdw m0, m1 packssdw m2, m3 packuswb m0, m2 movlps [r2], m0 movhps [r2 + r3], m0 lea r2, [r2 + 2 * r3] dec r4d jnz .loopH RET %endmacro FILTER_VER_CHROMA_SP_W8_H2 8, 2 FILTER_VER_CHROMA_SP_W8_H2 8, 4 FILTER_VER_CHROMA_SP_W8_H2 8, 6 FILTER_VER_CHROMA_SP_W8_H2 8, 8 FILTER_VER_CHROMA_SP_W8_H2 8, 16 FILTER_VER_CHROMA_SP_W8_H2 8, 32 FILTER_VER_CHROMA_SP_W8_H2 8, 12 FILTER_VER_CHROMA_SP_W8_H2 8, 64 ;----------------------------------------------------------------------------------------------------------------------------- ; void interp_4tap_horiz_ps_2x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx, int isRowExt) ;----------------------------------------------------------------------------------------------------------------------------- %macro FILTER_HORIZ_CHROMA_2xN 2 INIT_XMM sse4 cglobal interp_4tap_horiz_ps_%1x%2, 4, 7, 4, src, srcstride, dst, dststride %define coef2 m3 %define Tm0 m2 %define t1 m1 %define t0 m0 dec srcq mov r4d, r4m add dststrided, dststrided %ifdef PIC lea r6, [tab_ChromaCoeff] movd coef2, [r6 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t1, [pw_2000] mova Tm0, [tab_Tm] mov r4d, %2 cmp r5m, byte 0 je .loopH sub srcq, srcstrideq add r4d, 3 .loopH: movh t0, [srcq] pshufb t0, t0, Tm0 pmaddubsw t0, coef2 phaddw t0, t0 psubw t0, t1 movd [dstq], t0 lea srcq, [srcq + srcstrideq] lea dstq, [dstq + dststrideq] dec r4d jnz .loopH RET %endmacro FILTER_HORIZ_CHROMA_2xN 2, 4 FILTER_HORIZ_CHROMA_2xN 2, 8 FILTER_HORIZ_CHROMA_2xN 2, 16 ;----------------------------------------------------------------------------------------------------------------------------- ; void interp_4tap_horiz_ps_4x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx, int isRowExt) ;----------------------------------------------------------------------------------------------------------------------------- %macro FILTER_HORIZ_CHROMA_4xN 2 INIT_XMM sse4 cglobal interp_4tap_horiz_ps_%1x%2, 4, 7, 4, src, srcstride, dst, dststride %define coef2 m3 %define Tm0 m2 %define t1 m1 %define t0 m0 dec srcq mov r4d, r4m add dststrided, dststrided %ifdef PIC lea r6, [tab_ChromaCoeff] movd coef2, [r6 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t1, [pw_2000] mova Tm0, [tab_Tm] mov r4d, %2 cmp r5m, byte 0 je .loopH sub srcq, srcstrideq add r4d, 3 .loopH: movh t0, [srcq] pshufb t0, t0, Tm0 pmaddubsw t0, coef2 phaddw t0, t0 psubw t0, t1 movlps [dstq], t0 lea srcq, [srcq + srcstrideq] lea dstq, [dstq + dststrideq] dec r4d jnz .loopH RET %endmacro FILTER_HORIZ_CHROMA_4xN 4, 2 FILTER_HORIZ_CHROMA_4xN 4, 4 FILTER_HORIZ_CHROMA_4xN 4, 8 FILTER_HORIZ_CHROMA_4xN 4, 16 FILTER_HORIZ_CHROMA_4xN 4, 32 %macro PROCESS_CHROMA_W6 3 movu %1, [srcq] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 psubw %2, %3 movh [dstq], %2 pshufd %2, %2, 2 movd [dstq + 8], %2 %endmacro %macro PROCESS_CHROMA_W12 3 movu %1, [srcq] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 psubw %2, %3 movu [dstq], %2 movu %1, [srcq + 8] pshufb %1, %1, Tm0 pmaddubsw %1, coef2 phaddw %1, %1 psubw %1, %3 movh [dstq + 16], %1 %endmacro ;----------------------------------------------------------------------------------------------------------------------------- ; void interp_4tap_horiz_ps_6x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx, int isRowExt) ;----------------------------------------------------------------------------------------------------------------------------- %macro FILTER_HORIZ_CHROMA 2 INIT_XMM sse4 cglobal interp_4tap_horiz_ps_%1x%2, 4, 7, 6, src, srcstride, dst, dststride %define coef2 m5 %define Tm0 m4 %define Tm1 m3 %define t2 m2 %define t1 m1 %define t0 m0 dec srcq mov r4d, r4m add dststrided, dststrided %ifdef PIC lea r6, [tab_ChromaCoeff] movd coef2, [r6 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [pw_2000] mova Tm0, [tab_Tm] mova Tm1, [tab_Tm + 16] mov r4d, %2 cmp r5m, byte 0 je .loopH sub srcq, srcstrideq add r4d, 3 .loopH: PROCESS_CHROMA_W%1 t0, t1, t2 add srcq, srcstrideq add dstq, dststrideq dec r4d jnz .loopH RET %endmacro FILTER_HORIZ_CHROMA 6, 8 FILTER_HORIZ_CHROMA 12, 16 FILTER_HORIZ_CHROMA 6, 16 FILTER_HORIZ_CHROMA 12, 32 %macro PROCESS_CHROMA_W8 3 movu %1, [srcq] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 psubw %2, %3 movu [dstq], %2 %endmacro ;----------------------------------------------------------------------------------------------------------------------------- ; void interp_4tap_horiz_ps_8x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx, int isRowExt) ;----------------------------------------------------------------------------------------------------------------------------- %macro FILTER_HORIZ_CHROMA_8xN 2 INIT_XMM sse4 cglobal interp_4tap_horiz_ps_%1x%2, 4, 7, 6, src, srcstride, dst, dststride %define coef2 m5 %define Tm0 m4 %define Tm1 m3 %define t2 m2 %define t1 m1 %define t0 m0 dec srcq mov r4d, r4m add dststrided, dststrided %ifdef PIC lea r6, [tab_ChromaCoeff] movd coef2, [r6 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [pw_2000] mova Tm0, [tab_Tm] mova Tm1, [tab_Tm + 16] mov r4d, %2 cmp r5m, byte 0 je .loopH sub srcq, srcstrideq add r4d, 3 .loopH: PROCESS_CHROMA_W8 t0, t1, t2 add srcq, srcstrideq add dstq, dststrideq dec r4d jnz .loopH RET %endmacro FILTER_HORIZ_CHROMA_8xN 8, 2 FILTER_HORIZ_CHROMA_8xN 8, 4 FILTER_HORIZ_CHROMA_8xN 8, 6 FILTER_HORIZ_CHROMA_8xN 8, 8 FILTER_HORIZ_CHROMA_8xN 8, 16 FILTER_HORIZ_CHROMA_8xN 8, 32 FILTER_HORIZ_CHROMA_8xN 8, 12 FILTER_HORIZ_CHROMA_8xN 8, 64 %macro PROCESS_CHROMA_W16 4 movu %1, [srcq] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq + 8] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 psubw %2, %3 psubw %4, %3 movu [dstq], %2 movu [dstq + 16], %4 %endmacro %macro PROCESS_CHROMA_W24 4 movu %1, [srcq] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq + 8] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 psubw %2, %3 psubw %4, %3 movu [dstq], %2 movu [dstq + 16], %4 movu %1, [srcq + 16] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 psubw %2, %3 movu [dstq + 32], %2 %endmacro %macro PROCESS_CHROMA_W32 4 movu %1, [srcq] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq + 8] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 psubw %2, %3 psubw %4, %3 movu [dstq], %2 movu [dstq + 16], %4 movu %1, [srcq + 16] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq + 24] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 psubw %2, %3 psubw %4, %3 movu [dstq + 32], %2 movu [dstq + 48], %4 %endmacro %macro PROCESS_CHROMA_W16o 5 movu %1, [srcq + %5] pshufb %2, %1, Tm0 pmaddubsw %2, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %2, %1 movu %1, [srcq + %5 + 8] pshufb %4, %1, Tm0 pmaddubsw %4, coef2 pshufb %1, %1, Tm1 pmaddubsw %1, coef2 phaddw %4, %1 psubw %2, %3 psubw %4, %3 movu [dstq + %5 * 2], %2 movu [dstq + %5 * 2 + 16], %4 %endmacro %macro PROCESS_CHROMA_W48 4 PROCESS_CHROMA_W16o %1, %2, %3, %4, 0 PROCESS_CHROMA_W16o %1, %2, %3, %4, 16 PROCESS_CHROMA_W16o %1, %2, %3, %4, 32 %endmacro %macro PROCESS_CHROMA_W64 4 PROCESS_CHROMA_W16o %1, %2, %3, %4, 0 PROCESS_CHROMA_W16o %1, %2, %3, %4, 16 PROCESS_CHROMA_W16o %1, %2, %3, %4, 32 PROCESS_CHROMA_W16o %1, %2, %3, %4, 48 %endmacro ;------------------------------------------------------------------------------------------------------------------------------ ; void interp_4tap_horiz_ps_%1x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx, int isRowExt) ;------------------------------------------------------------------------------------------------------------------------------ %macro FILTER_HORIZ_CHROMA_WxN 2 INIT_XMM sse4 cglobal interp_4tap_horiz_ps_%1x%2, 4, 7, 7, src, srcstride, dst, dststride %define coef2 m6 %define Tm0 m5 %define Tm1 m4 %define t3 m3 %define t2 m2 %define t1 m1 %define t0 m0 dec srcq mov r4d, r4m add dststrided, dststrided %ifdef PIC lea r6, [tab_ChromaCoeff] movd coef2, [r6 + r4 * 4] %else movd coef2, [tab_ChromaCoeff + r4 * 4] %endif pshufd coef2, coef2, 0 mova t2, [pw_2000] mova Tm0, [tab_Tm] mova Tm1, [tab_Tm + 16] mov r4d, %2 cmp r5m, byte 0 je .loopH sub srcq, srcstrideq add r4d, 3 .loopH: PROCESS_CHROMA_W%1 t0, t1, t2, t3 add srcq, srcstrideq add dstq, dststrideq dec r4d jnz .loopH RET %endmacro FILTER_HORIZ_CHROMA_WxN 16, 4 FILTER_HORIZ_CHROMA_WxN 16, 8 FILTER_HORIZ_CHROMA_WxN 16, 12 FILTER_HORIZ_CHROMA_WxN 16, 16 FILTER_HORIZ_CHROMA_WxN 16, 32 FILTER_HORIZ_CHROMA_WxN 24, 32 FILTER_HORIZ_CHROMA_WxN 32, 8 FILTER_HORIZ_CHROMA_WxN 32, 16 FILTER_HORIZ_CHROMA_WxN 32, 24 FILTER_HORIZ_CHROMA_WxN 32, 32 FILTER_HORIZ_CHROMA_WxN 16, 24 FILTER_HORIZ_CHROMA_WxN 16, 64 FILTER_HORIZ_CHROMA_WxN 24, 64 FILTER_HORIZ_CHROMA_WxN 32, 48 FILTER_HORIZ_CHROMA_WxN 32, 64 FILTER_HORIZ_CHROMA_WxN 64, 64 FILTER_HORIZ_CHROMA_WxN 64, 32 FILTER_HORIZ_CHROMA_WxN 64, 48 FILTER_HORIZ_CHROMA_WxN 48, 64 FILTER_HORIZ_CHROMA_WxN 64, 16 ;--------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_%1x%2(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;--------------------------------------------------------------------------------------------------------------- %macro FILTER_V_PS_W16n 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_%1x%2, 4, 7, 8 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m1, m0, [tab_Vm] pshufb m0, [tab_Vm + 16] mov r4d, %2/2 .loop: mov r6d, %1/16 .loopW: movu m2, [r0] movu m3, [r0 + r1] punpcklbw m4, m2, m3 punpckhbw m2, m3 pmaddubsw m4, m1 pmaddubsw m2, m1 lea r5, [r0 + 2 * r1] movu m5, [r5] movu m7, [r5 + r1] punpcklbw m6, m5, m7 pmaddubsw m6, m0 paddw m4, m6 punpckhbw m6, m5, m7 pmaddubsw m6, m0 paddw m2, m6 mova m6, [pw_2000] psubw m4, m6 psubw m2, m6 movu [r2], m4 movu [r2 + 16], m2 punpcklbw m4, m3, m5 punpckhbw m3, m5 pmaddubsw m4, m1 pmaddubsw m3, m1 movu m5, [r5 + 2 * r1] punpcklbw m2, m7, m5 punpckhbw m7, m5 pmaddubsw m2, m0 pmaddubsw m7, m0 paddw m4, m2 paddw m3, m7 psubw m4, m6 psubw m3, m6 movu [r2 + r3], m4 movu [r2 + r3 + 16], m3 add r0, 16 add r2, 32 dec r6d jnz .loopW lea r0, [r0 + r1 * 2 - %1] lea r2, [r2 + r3 * 2 - %1 * 2] dec r4d jnz .loop RET %endmacro FILTER_V_PS_W16n 64, 64 FILTER_V_PS_W16n 64, 32 FILTER_V_PS_W16n 64, 48 FILTER_V_PS_W16n 48, 64 FILTER_V_PS_W16n 64, 16 ;------------------------------------------------------------------------------------------------------------ ;void interp_4tap_vert_ps_2x4(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------ INIT_XMM sse4 cglobal interp_4tap_vert_ps_2x4, 4, 6, 7 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m0, [tab_Cm] lea r5, [3 * r1] movd m2, [r0] movd m3, [r0 + r1] movd m4, [r0 + 2 * r1] movd m5, [r0 + r5] punpcklbw m2, m3 punpcklbw m6, m4, m5 punpcklbw m2, m6 pmaddubsw m2, m0 lea r0, [r0 + 4 * r1] movd m6, [r0] punpcklbw m3, m4 punpcklbw m1, m5, m6 punpcklbw m3, m1 pmaddubsw m3, m0 phaddw m2, m3 mova m1, [pw_2000] psubw m2, m1 movd [r2], m2 pextrd [r2 + r3], m2, 2 movd m2, [r0 + r1] punpcklbw m4, m5 punpcklbw m3, m6, m2 punpcklbw m4, m3 pmaddubsw m4, m0 movd m3, [r0 + 2 * r1] punpcklbw m5, m6 punpcklbw m2, m3 punpcklbw m5, m2 pmaddubsw m5, m0 phaddw m4, m5 psubw m4, m1 lea r2, [r2 + 2 * r3] movd [r2], m4 pextrd [r2 + r3], m4, 2 RET ;------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ps_2x8(pixel *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------- %macro FILTER_V_PS_W2 2 INIT_XMM sse4 cglobal interp_4tap_vert_ps_2x%2, 4, 6, 8 mov r4d, r4m sub r0, r1 add r3d, r3d %ifdef PIC lea r5, [tab_ChromaCoeff] movd m0, [r5 + r4 * 4] %else movd m0, [tab_ChromaCoeff + r4 * 4] %endif pshufb m0, [tab_Cm] mova m1, [pw_2000] lea r5, [3 * r1] mov r4d, %2/4 .loop: movd m2, [r0] movd m3, [r0 + r1] movd m4, [r0 + 2 * r1] movd m5, [r0 + r5] punpcklbw m2, m3 punpcklbw m6, m4, m5 punpcklbw m2, m6 pmaddubsw m2, m0 lea r0, [r0 + 4 * r1] movd m6, [r0] punpcklbw m3, m4 punpcklbw m7, m5, m6 punpcklbw m3, m7 pmaddubsw m3, m0 phaddw m2, m3 psubw m2, m1 movd [r2], m2 pshufd m2, m2, 2 movd [r2 + r3], m2 movd m2, [r0 + r1] punpcklbw m4, m5 punpcklbw m3, m6, m2 punpcklbw m4, m3 pmaddubsw m4, m0 movd m3, [r0 + 2 * r1] punpcklbw m5, m6 punpcklbw m2, m3 punpcklbw m5, m2 pmaddubsw m5, m0 phaddw m4, m5 psubw m4, m1 lea r2, [r2 + 2 * r3] movd [r2], m4 pshufd m4 , m4 ,2 movd [r2 + r3], m4 lea r2, [r2 + 2 * r3] dec r4d jnz .loop RET %endmacro FILTER_V_PS_W2 2, 8 FILTER_V_PS_W2 2, 16 ;----------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ss_%1x%2(int16_t *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_CHROMA_SS 2 INIT_XMM sse2 cglobal interp_4tap_vert_ss_%1x%2, 5, 7, 6 ,0-gprsize add r1d, r1d add r3d, r3d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r6, [r5 + r4] %else lea r6, [tab_ChromaCoeffV + r4] %endif mov dword [rsp], %2/4 .loopH: mov r4d, (%1/4) .loopW: PROCESS_CHROMA_SP_W4_4R psrad m0, 6 psrad m1, 6 psrad m2, 6 psrad m3, 6 packssdw m0, m1 packssdw m2, m3 movlps [r2], m0 movhps [r2 + r3], m0 lea r5, [r2 + 2 * r3] movlps [r5], m2 movhps [r5 + r3], m2 lea r5, [4 * r1 - 2 * 4] sub r0, r5 add r2, 2 * 4 dec r4d jnz .loopW lea r0, [r0 + 4 * r1 - 2 * %1] lea r2, [r2 + 4 * r3 - 2 * %1] dec dword [rsp] jnz .loopH RET %endmacro FILTER_VER_CHROMA_SS 4, 4 FILTER_VER_CHROMA_SS 4, 8 FILTER_VER_CHROMA_SS 16, 16 FILTER_VER_CHROMA_SS 16, 8 FILTER_VER_CHROMA_SS 16, 12 FILTER_VER_CHROMA_SS 12, 16 FILTER_VER_CHROMA_SS 16, 4 FILTER_VER_CHROMA_SS 4, 16 FILTER_VER_CHROMA_SS 32, 32 FILTER_VER_CHROMA_SS 32, 16 FILTER_VER_CHROMA_SS 16, 32 FILTER_VER_CHROMA_SS 32, 24 FILTER_VER_CHROMA_SS 24, 32 FILTER_VER_CHROMA_SS 32, 8 FILTER_VER_CHROMA_SS 16, 24 FILTER_VER_CHROMA_SS 12, 32 FILTER_VER_CHROMA_SS 4, 32 FILTER_VER_CHROMA_SS 32, 64 FILTER_VER_CHROMA_SS 16, 64 FILTER_VER_CHROMA_SS 32, 48 FILTER_VER_CHROMA_SS 24, 64 FILTER_VER_CHROMA_SS 64, 64 FILTER_VER_CHROMA_SS 64, 32 FILTER_VER_CHROMA_SS 64, 48 FILTER_VER_CHROMA_SS 48, 64 FILTER_VER_CHROMA_SS 64, 16 ;--------------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vertical_ss_%1x%2(int16_t *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;--------------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_CHROMA_SS_W2_4R 2 INIT_XMM sse4 cglobal interp_4tap_vert_ss_%1x%2, 5, 6, 5 add r1d, r1d add r3d, r3d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r5, [r5 + r4] %else lea r5, [tab_ChromaCoeffV + r4] %endif mov r4d, (%2/4) .loopH: PROCESS_CHROMA_SP_W2_4R r5 psrad m0, 6 psrad m2, 6 packssdw m0, m2 movd [r2], m0 pextrd [r2 + r3], m0, 1 lea r2, [r2 + 2 * r3] pextrd [r2], m0, 2 pextrd [r2 + r3], m0, 3 lea r2, [r2 + 2 * r3] dec r4d jnz .loopH RET %endmacro FILTER_VER_CHROMA_SS_W2_4R 2, 4 FILTER_VER_CHROMA_SS_W2_4R 2, 8 FILTER_VER_CHROMA_SS_W2_4R 2, 16 ;--------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ss_4x2(int16_t *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;--------------------------------------------------------------------------------------------------------------- INIT_XMM sse2 cglobal interp_4tap_vert_ss_4x2, 5, 6, 4 add r1d, r1d add r3d, r3d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r5, [r5 + r4] %else lea r5, [tab_ChromaCoeffV + r4] %endif movq m0, [r0] movq m1, [r0 + r1] punpcklwd m0, m1 ;m0=[0 1] pmaddwd m0, [r5 + 0 *16] ;m0=[0+1] Row1 lea r0, [r0 + 2 * r1] movq m2, [r0] punpcklwd m1, m2 ;m1=[1 2] pmaddwd m1, [r5 + 0 *16] ;m1=[1+2] Row2 movq m3, [r0 + r1] punpcklwd m2, m3 ;m4=[2 3] pmaddwd m2, [r5 + 1 * 16] paddd m0, m2 ;m0=[0+1+2+3] Row1 done psrad m0, 6 movq m2, [r0 + 2 * r1] punpcklwd m3, m2 ;m5=[3 4] pmaddwd m3, [r5 + 1 * 16] paddd m1, m3 ;m1=[1+2+3+4] Row2 done psrad m1, 6 packssdw m0, m1 movlps [r2], m0 movhps [r2 + r3], m0 RET ;------------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vertical_ss_6x8(int16_t *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;------------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_CHROMA_SS_W6_H4 2 INIT_XMM sse4 cglobal interp_4tap_vert_ss_6x%2, 5, 7, 6 add r1d, r1d add r3d, r3d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r6, [r5 + r4] %else lea r6, [tab_ChromaCoeffV + r4] %endif mov r4d, %2/4 .loopH: PROCESS_CHROMA_SP_W4_4R psrad m0, 6 psrad m1, 6 psrad m2, 6 psrad m3, 6 packssdw m0, m1 packssdw m2, m3 movlps [r2], m0 movhps [r2 + r3], m0 lea r5, [r2 + 2 * r3] movlps [r5], m2 movhps [r5 + r3], m2 lea r5, [4 * r1 - 2 * 4] sub r0, r5 add r2, 2 * 4 PROCESS_CHROMA_SP_W2_4R r6 psrad m0, 6 psrad m2, 6 packssdw m0, m2 movd [r2], m0 pextrd [r2 + r3], m0, 1 lea r2, [r2 + 2 * r3] pextrd [r2], m0, 2 pextrd [r2 + r3], m0, 3 sub r0, 2 * 4 lea r2, [r2 + 2 * r3 - 2 * 4] dec r4d jnz .loopH RET %endmacro FILTER_VER_CHROMA_SS_W6_H4 6, 8 FILTER_VER_CHROMA_SS_W6_H4 6, 16 ;---------------------------------------------------------------------------------------------------------------- ; void interp_4tap_vert_ss_8x%2(int16_t *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;---------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_CHROMA_SS_W8_H2 2 INIT_XMM sse2 cglobal interp_4tap_vert_ss_%1x%2, 5, 6, 7 add r1d, r1d add r3d, r3d sub r0, r1 shl r4d, 5 %ifdef PIC lea r5, [tab_ChromaCoeffV] lea r5, [r5 + r4] %else lea r5, [tab_ChromaCoeffV + r4] %endif mov r4d, %2/2 .loopH: PROCESS_CHROMA_SP_W8_2R psrad m0, 6 psrad m1, 6 psrad m2, 6 psrad m3, 6 packssdw m0, m1 packssdw m2, m3 movu [r2], m0 movu [r2 + r3], m2 lea r2, [r2 + 2 * r3] dec r4d jnz .loopH RET %endmacro FILTER_VER_CHROMA_SS_W8_H2 8, 2 FILTER_VER_CHROMA_SS_W8_H2 8, 4 FILTER_VER_CHROMA_SS_W8_H2 8, 6 FILTER_VER_CHROMA_SS_W8_H2 8, 8 FILTER_VER_CHROMA_SS_W8_H2 8, 16 FILTER_VER_CHROMA_SS_W8_H2 8, 32 FILTER_VER_CHROMA_SS_W8_H2 8, 12 FILTER_VER_CHROMA_SS_W8_H2 8, 64 ;----------------------------------------------------------------------------------------------------------------- ; void interp_8tap_vert_ss_%1x%2(int16_t *src, intptr_t srcStride, int16_t *dst, intptr_t dstStride, int coeffIdx) ;----------------------------------------------------------------------------------------------------------------- %macro FILTER_VER_LUMA_SS 2 INIT_XMM sse2 cglobal interp_8tap_vert_ss_%1x%2, 5, 7, 7 ,0-gprsize add r1d, r1d add r3d, r3d lea r5, [3 * r1] sub r0, r5 shl r4d, 6 %ifdef PIC lea r5, [tab_LumaCoeffV] lea r6, [r5 + r4] %else lea r6, [tab_LumaCoeffV + r4] %endif mov dword [rsp], %2/4 .loopH: mov r4d, (%1/4) .loopW: movq m0, [r0] movq m1, [r0 + r1] punpcklwd m0, m1 ;m0=[0 1] pmaddwd m0, [r6 + 0 *16] ;m0=[0+1] Row1 lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m1, m4 ;m1=[1 2] pmaddwd m1, [r6 + 0 *16] ;m1=[1+2] Row2 movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[2 3] pmaddwd m2, m4, [r6 + 0 *16] ;m2=[2+3] Row3 pmaddwd m4, [r6 + 1 * 16] paddd m0, m4 ;m0=[0+1+2+3] Row1 lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m5, m4 ;m5=[3 4] pmaddwd m3, m5, [r6 + 0 *16] ;m3=[3+4] Row4 pmaddwd m5, [r6 + 1 * 16] paddd m1, m5 ;m1 = [1+2+3+4] Row2 movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[4 5] pmaddwd m6, m4, [r6 + 1 * 16] paddd m2, m6 ;m2=[2+3+4+5] Row3 pmaddwd m4, [r6 + 2 * 16] paddd m0, m4 ;m0=[0+1+2+3+4+5] Row1 lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m5, m4 ;m5=[5 6] pmaddwd m6, m5, [r6 + 1 * 16] paddd m3, m6 ;m3=[3+4+5+6] Row4 pmaddwd m5, [r6 + 2 * 16] paddd m1, m5 ;m1=[1+2+3+4+5+6] Row2 movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[6 7] pmaddwd m6, m4, [r6 + 2 * 16] paddd m2, m6 ;m2=[2+3+4+5+6+7] Row3 pmaddwd m4, [r6 + 3 * 16] paddd m0, m4 ;m0=[0+1+2+3+4+5+6+7] Row1 end psrad m0, 6 lea r0, [r0 + 2 * r1] movq m4, [r0] punpcklwd m5, m4 ;m5=[7 8] pmaddwd m6, m5, [r6 + 2 * 16] paddd m3, m6 ;m3=[3+4+5+6+7+8] Row4 pmaddwd m5, [r6 + 3 * 16] paddd m1, m5 ;m1=[1+2+3+4+5+6+7+8] Row2 end psrad m1, 6 packssdw m0, m1 movlps [r2], m0 movhps [r2 + r3], m0 movq m5, [r0 + r1] punpcklwd m4, m5 ;m4=[8 9] pmaddwd m4, [r6 + 3 * 16] paddd m2, m4 ;m2=[2+3+4+5+6+7+8+9] Row3 end psrad m2, 6 movq m4, [r0 + 2 * r1] punpcklwd m5, m4 ;m5=[9 10] pmaddwd m5, [r6 + 3 * 16] paddd m3, m5 ;m3=[3+4+5+6+7+8+9+10] Row4 end psrad m3, 6 packssdw m2, m3 movlps [r2 + 2 * r3], m2 lea r5, [3 * r3] movhps [r2 + r5], m2 lea r5, [8 * r1 - 2 * 4] sub r0, r5 add r2, 2 * 4 dec r4d jnz .loopW lea r0, [r0 + 4 * r1 - 2 * %1] lea r2, [r2 + 4 * r3 - 2 * %1] dec dword [rsp] jnz .loopH RET %endmacro FILTER_VER_LUMA_SS 4, 4 FILTER_VER_LUMA_SS 8, 8 FILTER_VER_LUMA_SS 8, 4 FILTER_VER_LUMA_SS 4, 8 FILTER_VER_LUMA_SS 16, 16 FILTER_VER_LUMA_SS 16, 8 FILTER_VER_LUMA_SS 8, 16 FILTER_VER_LUMA_SS 16, 12 FILTER_VER_LUMA_SS 12, 16 FILTER_VER_LUMA_SS 16, 4 FILTER_VER_LUMA_SS 4, 16 FILTER_VER_LUMA_SS 32, 32 FILTER_VER_LUMA_SS 32, 16 FILTER_VER_LUMA_SS 16, 32 FILTER_VER_LUMA_SS 32, 24 FILTER_VER_LUMA_SS 24, 32 FILTER_VER_LUMA_SS 32, 8 FILTER_VER_LUMA_SS 8, 32 FILTER_VER_LUMA_SS 64, 64 FILTER_VER_LUMA_SS 64, 32 FILTER_VER_LUMA_SS 32, 64 FILTER_VER_LUMA_SS 64, 48 FILTER_VER_LUMA_SS 48, 64 FILTER_VER_LUMA_SS 64, 16 FILTER_VER_LUMA_SS 16, 64