/***************************************************************************** * Copyright (C) 2020-2021 MulticoreWare, Inc * * Authors: Hongbin Liu * Sebastian Pop * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. * * This program is also available under a commercial proprietary license. * For more information, contact us at license @ x265.com. *****************************************************************************/ #include "asm.S" #ifdef __APPLE__ .section __RODATA,__rodata #else .section .rodata #endif .align 4 .text .macro SAD_START_4 f ldr s0, [x0] add x0, x0, x1 ldr s1, [x2] add x2, x2, x3 ld1 {v1.s}[1], [x2], x3 ld1 {v0.s}[1], [x0], x1 \f v16.8h, v0.8b, v1.8b .endm .macro SAD_4 h .rept \h / 2 - 1 SAD_START_4 uabal .endr .endm .macro SAD_START_8 f ld1 {v0.8b}, [x0], x1 ld1 {v1.8b}, [x2], x3 ld1 {v2.8b}, [x0], x1 ld1 {v3.8b}, [x2], x3 \f v16.8h, v0.8b, v1.8b \f v17.8h, v2.8b, v3.8b .endm .macro SAD_8 h .rept \h / 2 - 1 SAD_START_8 uabal .endr .endm .macro SAD_START_16 f ld1 {v0.16b}, [x0], x1 ld1 {v1.16b}, [x2], x3 ld1 {v2.16b}, [x0], x1 ld1 {v3.16b}, [x2], x3 \f v16.8h, v0.8b, v1.8b \f\()2 v17.8h, v0.16b, v1.16b uabal v16.8h, v2.8b, v3.8b uabal2 v17.8h, v2.16b, v3.16b .endm .macro SAD_16 h .rept \h / 2 - 1 SAD_START_16 uabal .endr .endm .macro SAD_START_32 movi v16.16b, #0 movi v17.16b, #0 movi v18.16b, #0 movi v19.16b, #0 .endm .macro SAD_32 ld1 {v0.16b-v1.16b}, [x0], x1 ld1 {v2.16b-v3.16b}, [x2], x3 ld1 {v4.16b-v5.16b}, [x0], x1 ld1 {v6.16b-v7.16b}, [x2], x3 uabal v16.8h, v0.8b, v2.8b uabal2 v17.8h, v0.16b, v2.16b uabal v18.8h, v1.8b, v3.8b uabal2 v19.8h, v1.16b, v3.16b uabal v16.8h, v4.8b, v6.8b uabal2 v17.8h, v4.16b, v6.16b uabal v18.8h, v5.8b, v7.8b uabal2 v19.8h, v5.16b, v7.16b .endm .macro SAD_END_32 add v16.8h, v16.8h, v17.8h add v17.8h, v18.8h, v19.8h add v16.8h, v16.8h, v17.8h uaddlv s0, v16.8h fmov w0, s0 ret .endm .macro SAD_START_64 movi v16.16b, #0 movi v17.16b, #0 movi v18.16b, #0 movi v19.16b, #0 movi v20.16b, #0 movi v21.16b, #0 movi v22.16b, #0 movi v23.16b, #0 .endm .macro SAD_64 ld1 {v0.16b-v3.16b}, [x0], x1 ld1 {v4.16b-v7.16b}, [x2], x3 ld1 {v24.16b-v27.16b}, [x0], x1 ld1 {v28.16b-v31.16b}, [x2], x3 uabal v16.8h, v0.8b, v4.8b uabal2 v17.8h, v0.16b, v4.16b uabal v18.8h, v1.8b, v5.8b uabal2 v19.8h, v1.16b, v5.16b uabal v20.8h, v2.8b, v6.8b uabal2 v21.8h, v2.16b, v6.16b uabal v22.8h, v3.8b, v7.8b uabal2 v23.8h, v3.16b, v7.16b uabal v16.8h, v24.8b, v28.8b uabal2 v17.8h, v24.16b, v28.16b uabal v18.8h, v25.8b, v29.8b uabal2 v19.8h, v25.16b, v29.16b uabal v20.8h, v26.8b, v30.8b uabal2 v21.8h, v26.16b, v30.16b uabal v22.8h, v27.8b, v31.8b uabal2 v23.8h, v27.16b, v31.16b .endm .macro SAD_END_64 add v16.8h, v16.8h, v17.8h add v17.8h, v18.8h, v19.8h add v16.8h, v16.8h, v17.8h uaddlp v16.4s, v16.8h add v18.8h, v20.8h, v21.8h add v19.8h, v22.8h, v23.8h add v17.8h, v18.8h, v19.8h uaddlp v17.4s, v17.8h add v16.4s, v16.4s, v17.4s uaddlv d0, v16.4s fmov x0, d0 ret .endm .macro SAD_START_12 movrel x12, sad12_mask ld1 {v31.16b}, [x12] movi v16.16b, #0 movi v17.16b, #0 .endm .macro SAD_12 ld1 {v0.16b}, [x0], x1 and v0.16b, v0.16b, v31.16b ld1 {v1.16b}, [x2], x3 and v1.16b, v1.16b, v31.16b ld1 {v2.16b}, [x0], x1 and v2.16b, v2.16b, v31.16b ld1 {v3.16b}, [x2], x3 and v3.16b, v3.16b, v31.16b uabal v16.8h, v0.8b, v1.8b uabal2 v17.8h, v0.16b, v1.16b uabal v16.8h, v2.8b, v3.8b uabal2 v17.8h, v2.16b, v3.16b .endm .macro SAD_END_12 add v16.8h, v16.8h, v17.8h uaddlv s0, v16.8h fmov w0, s0 ret .endm .macro SAD_START_24 movi v16.16b, #0 movi v17.16b, #0 movi v18.16b, #0 sub x1, x1, #16 sub x3, x3, #16 .endm .macro SAD_24 ld1 {v0.16b}, [x0], #16 ld1 {v1.8b}, [x0], x1 ld1 {v2.16b}, [x2], #16 ld1 {v3.8b}, [x2], x3 ld1 {v4.16b}, [x0], #16 ld1 {v5.8b}, [x0], x1 ld1 {v6.16b}, [x2], #16 ld1 {v7.8b}, [x2], x3 uabal v16.8h, v0.8b, v2.8b uabal2 v17.8h, v0.16b, v2.16b uabal v18.8h, v1.8b, v3.8b uabal v16.8h, v4.8b, v6.8b uabal2 v17.8h, v4.16b, v6.16b uabal v18.8h, v5.8b, v7.8b .endm .macro SAD_END_24 add v16.8h, v16.8h, v17.8h add v16.8h, v16.8h, v18.8h uaddlv s0, v16.8h fmov w0, s0 ret .endm .macro SAD_START_48 movi v16.16b, #0 movi v17.16b, #0 movi v18.16b, #0 movi v19.16b, #0 movi v20.16b, #0 movi v21.16b, #0 .endm .macro SAD_48 ld1 {v0.16b-v2.16b}, [x0], x1 ld1 {v4.16b-v6.16b}, [x2], x3 ld1 {v24.16b-v26.16b}, [x0], x1 ld1 {v28.16b-v30.16b}, [x2], x3 uabal v16.8h, v0.8b, v4.8b uabal2 v17.8h, v0.16b, v4.16b uabal v18.8h, v1.8b, v5.8b uabal2 v19.8h, v1.16b, v5.16b uabal v20.8h, v2.8b, v6.8b uabal2 v21.8h, v2.16b, v6.16b uabal v16.8h, v24.8b, v28.8b uabal2 v17.8h, v24.16b, v28.16b uabal v18.8h, v25.8b, v29.8b uabal2 v19.8h, v25.16b, v29.16b uabal v20.8h, v26.8b, v30.8b uabal2 v21.8h, v26.16b, v30.16b .endm .macro SAD_END_48 add v16.8h, v16.8h, v17.8h add v17.8h, v18.8h, v19.8h add v16.8h, v16.8h, v17.8h uaddlv s0, v16.8h fmov w0, s0 add v18.8h, v20.8h, v21.8h uaddlv s1, v18.8h fmov w1, s1 add w0, w0, w1 ret .endm // Fully unrolled. .macro SAD_FUNC w, h function PFX(pixel_sad_\w\()x\h\()_neon) SAD_START_\w uabdl SAD_\w \h .if \w > 4 add v16.8h, v16.8h, v17.8h .endif uaddlv s0, v16.8h fmov w0, s0 ret endfunc .endm // Loop unrolled 4. .macro SAD_FUNC_LOOP w, h function PFX(pixel_sad_\w\()x\h\()_neon) SAD_START_\w mov w9, #\h/8 .loop_\w\()x\h: sub w9, w9, #1 .rept 4 SAD_\w .endr cbnz w9, .loop_\w\()x\h SAD_END_\w endfunc .endm SAD_FUNC 4, 4 SAD_FUNC 4, 8 SAD_FUNC 4, 16 SAD_FUNC 8, 4 SAD_FUNC 8, 8 SAD_FUNC 8, 16 SAD_FUNC 8, 32 SAD_FUNC 16, 4 SAD_FUNC 16, 8 SAD_FUNC 16, 12 SAD_FUNC 16, 16 SAD_FUNC 16, 32 SAD_FUNC 16, 64 SAD_FUNC_LOOP 32, 8 SAD_FUNC_LOOP 32, 16 SAD_FUNC_LOOP 32, 24 SAD_FUNC_LOOP 32, 32 SAD_FUNC_LOOP 32, 64 SAD_FUNC_LOOP 64, 16 SAD_FUNC_LOOP 64, 32 SAD_FUNC_LOOP 64, 48 SAD_FUNC_LOOP 64, 64 SAD_FUNC_LOOP 12, 16 SAD_FUNC_LOOP 24, 32 SAD_FUNC_LOOP 48, 64 // SAD_X3 and SAD_X4 code start .macro SAD_X_START_4 h, x, f ldr s0, [x0] add x0, x0, x9 ld1 {v0.s}[1], [x0], x9 ldr s1, [x1] add x1, x1, x5 ld1 {v1.s}[1], [x1], x5 ldr s2, [x2] add x2, x2, x5 ld1 {v2.s}[1], [x2], x5 ldr s3, [x3] add x3, x3, x5 ld1 {v3.s}[1], [x3], x5 \f v16.8h, v0.8b, v1.8b \f v17.8h, v0.8b, v2.8b \f v18.8h, v0.8b, v3.8b .if \x == 4 ld1 {v4.s}[0], [x4], x5 ld1 {v4.s}[1], [x4], x5 \f v19.8h, v0.8b, v4.8b .endif .endm .macro SAD_X_4 h, x .rept \h/2 - 1 SAD_X_START_4 \h, \x, uabal .endr .endm .macro SAD_X_END_4 x uaddlv s0, v16.8h uaddlv s1, v17.8h uaddlv s2, v18.8h stp s0, s1, [x6] .if \x == 3 str s2, [x6, #8] .elseif \x == 4 uaddlv s3, v19.8h stp s2, s3, [x6, #8] .endif ret .endm .macro SAD_X_START_8 h, x, f ld1 {v0.8b}, [x0], x9 ld1 {v1.8b}, [x1], x5 ld1 {v2.8b}, [x2], x5 ld1 {v3.8b}, [x3], x5 \f v16.8h, v0.8b, v1.8b \f v17.8h, v0.8b, v2.8b \f v18.8h, v0.8b, v3.8b .if \x == 4 ld1 {v4.8b}, [x4], x5 \f v19.8h, v0.8b, v4.8b .endif .endm .macro SAD_X_8 h x .rept \h - 1 SAD_X_START_8 \h, \x, uabal .endr .endm .macro SAD_X_END_8 x SAD_X_END_4 \x .endm .macro SAD_X_START_12 h, x, f ld1 {v0.16b}, [x0], x9 and v0.16b, v0.16b, v31.16b ld1 {v1.16b}, [x1], x5 and v1.16b, v1.16b, v31.16b ld1 {v2.16b}, [x2], x5 and v2.16b, v2.16b, v31.16b ld1 {v3.16b}, [x3], x5 and v3.16b, v3.16b, v31.16b \f v16.8h, v1.8b, v0.8b \f\()2 v20.8h, v1.16b, v0.16b \f v17.8h, v2.8b, v0.8b \f\()2 v21.8h, v2.16b, v0.16b \f v18.8h, v3.8b, v0.8b \f\()2 v22.8h, v3.16b, v0.16b .if \x == 4 ld1 {v4.16b}, [x4], x5 and v4.16b, v4.16b, v31.16b \f v19.8h, v4.8b, v0.8b \f\()2 v23.8h, v4.16b, v0.16b .endif .endm .macro SAD_X_12 h x .rept \h - 1 SAD_X_START_12 \h, \x, uabal .endr .endm .macro SAD_X_END_12 x SAD_X_END_16 \x .endm .macro SAD_X_START_16 h, x, f ld1 {v0.16b}, [x0], x9 ld1 {v1.16b}, [x1], x5 ld1 {v2.16b}, [x2], x5 ld1 {v3.16b}, [x3], x5 \f v16.8h, v1.8b, v0.8b \f\()2 v20.8h, v1.16b, v0.16b \f v17.8h, v2.8b, v0.8b \f\()2 v21.8h, v2.16b, v0.16b \f v18.8h, v3.8b, v0.8b \f\()2 v22.8h, v3.16b, v0.16b .if \x == 4 ld1 {v4.16b}, [x4], x5 \f v19.8h, v4.8b, v0.8b \f\()2 v23.8h, v4.16b, v0.16b .endif .endm .macro SAD_X_16 h x .rept \h - 1 SAD_X_START_16 \h, \x, uabal .endr .endm .macro SAD_X_END_16 x add v16.8h, v16.8h, v20.8h add v17.8h, v17.8h, v21.8h add v18.8h, v18.8h, v22.8h .if \x == 4 add v19.8h, v19.8h, v23.8h .endif SAD_X_END_4 \x .endm .macro SAD_X_START_24 x SAD_X_START_32 \x sub x5, x5, #16 sub x9, x9, #16 .endm .macro SAD_X_24 base v1 v2 ld1 {v0.16b}, [ \base ], #16 ld1 {v1.8b}, [ \base ], x5 uabal \v1\().8h, v0.8b, v6.8b uabal \v1\().8h, v1.8b, v7.8b uabal2 \v2\().8h, v0.16b, v6.16b .endm .macro SAD_X_END_24 x SAD_X_END_16 \x .endm .macro SAD_X_START_32 x movi v16.16b, #0 movi v17.16b, #0 movi v18.16b, #0 movi v20.16b, #0 movi v21.16b, #0 movi v22.16b, #0 .if \x == 4 movi v19.16b, #0 movi v23.16b, #0 .endif .endm .macro SAD_X_32 base v1 v2 ld1 {v0.16b-v1.16b}, [ \base ], x5 uabal \v1\().8h, v0.8b, v6.8b uabal \v1\().8h, v1.8b, v7.8b uabal2 \v2\().8h, v0.16b, v6.16b uabal2 \v2\().8h, v1.16b, v7.16b .endm .macro SAD_X_END_32 x SAD_X_END_16 \x .endm .macro SAD_X_START_48 x SAD_X_START_32 \x .endm .macro SAD_X_48 x1 v1 v2 ld1 {v0.16b-v2.16b}, [ \x1 ], x5 uabal \v1\().8h, v0.8b, v4.8b uabal \v1\().8h, v1.8b, v5.8b uabal \v1\().8h, v2.8b, v6.8b uabal2 \v2\().8h, v0.16b, v4.16b uabal2 \v2\().8h, v1.16b, v5.16b uabal2 \v2\().8h, v2.16b, v6.16b .endm .macro SAD_X_END_48 x SAD_X_END_64 \x .endm .macro SAD_X_START_64 x SAD_X_START_32 \x .endm .macro SAD_X_64 x1 v1 v2 ld1 {v0.16b-v3.16b}, [ \x1 ], x5 uabal \v1\().8h, v0.8b, v4.8b uabal \v1\().8h, v1.8b, v5.8b uabal \v1\().8h, v2.8b, v6.8b uabal \v1\().8h, v3.8b, v7.8b uabal2 \v2\().8h, v0.16b, v4.16b uabal2 \v2\().8h, v1.16b, v5.16b uabal2 \v2\().8h, v2.16b, v6.16b uabal2 \v2\().8h, v3.16b, v7.16b .endm .macro SAD_X_END_64 x uaddlp v16.4s, v16.8h uaddlp v17.4s, v17.8h uaddlp v18.4s, v18.8h uaddlp v20.4s, v20.8h uaddlp v21.4s, v21.8h uaddlp v22.4s, v22.8h add v16.4s, v16.4s, v20.4s add v17.4s, v17.4s, v21.4s add v18.4s, v18.4s, v22.4s trn2 v20.2d, v16.2d, v16.2d trn2 v21.2d, v17.2d, v17.2d trn2 v22.2d, v18.2d, v18.2d add v16.2s, v16.2s, v20.2s add v17.2s, v17.2s, v21.2s add v18.2s, v18.2s, v22.2s uaddlp v16.1d, v16.2s uaddlp v17.1d, v17.2s uaddlp v18.1d, v18.2s stp s16, s17, [x6], #8 .if \x == 3 str s18, [x6] .elseif \x == 4 uaddlp v19.4s, v19.8h uaddlp v23.4s, v23.8h add v19.4s, v19.4s, v23.4s trn2 v23.2d, v19.2d, v19.2d add v19.2s, v19.2s, v23.2s uaddlp v19.1d, v19.2s stp s18, s19, [x6] .endif ret .endm // static void x264_pixel_sad_x3_##size(pixel *fenc, pixel *pix0, pixel *pix1, pixel *pix2, intptr_t i_stride, int scores[3]) // static void x264_pixel_sad_x4_##size(pixel *fenc, pixel *pix0, pixel *pix1,pixel *pix2, pixel *pix3, intptr_t i_stride, int scores[4]) .macro SAD_X_FUNC x, w, h function PFX(sad_x\x\()_\w\()x\h\()_neon) mov x9, #FENC_STRIDE // Make function arguments for x == 3 look like x == 4. .if \x == 3 mov x6, x5 mov x5, x4 .endif .if \w == 12 movrel x12, sad12_mask ld1 {v31.16b}, [x12] .endif SAD_X_START_\w \h, \x, uabdl SAD_X_\w \h, \x SAD_X_END_\w \x endfunc .endm .macro SAD_X_LOOP x, w, h function PFX(sad_x\x\()_\w\()x\h\()_neon) mov x9, #FENC_STRIDE // Make function arguments for x == 3 look like x == 4. .if \x == 3 mov x6, x5 mov x5, x4 .endif SAD_X_START_\w \x mov w12, #\h/4 .loop_sad_x\x\()_\w\()x\h: sub w12, w12, #1 .rept 4 .if \w == 24 ld1 {v6.16b}, [x0], #16 ld1 {v7.8b}, [x0], x9 .elseif \w == 32 ld1 {v6.16b-v7.16b}, [x0], x9 .elseif \w == 48 ld1 {v4.16b-v6.16b}, [x0], x9 .elseif \w == 64 ld1 {v4.16b-v7.16b}, [x0], x9 .endif SAD_X_\w x1, v16, v20 SAD_X_\w x2, v17, v21 SAD_X_\w x3, v18, v22 .if \x == 4 SAD_X_\w x4, v19, v23 .endif .endr cbnz w12, .loop_sad_x\x\()_\w\()x\h SAD_X_END_\w \x endfunc .endm SAD_X_FUNC 3, 4, 4 SAD_X_FUNC 3, 4, 8 SAD_X_FUNC 3, 4, 16 SAD_X_FUNC 3, 8, 4 SAD_X_FUNC 3, 8, 8 SAD_X_FUNC 3, 8, 16 SAD_X_FUNC 3, 8, 32 SAD_X_FUNC 3, 12, 16 SAD_X_FUNC 3, 16, 4 SAD_X_FUNC 3, 16, 8 SAD_X_FUNC 3, 16, 12 SAD_X_FUNC 3, 16, 16 SAD_X_FUNC 3, 16, 32 SAD_X_FUNC 3, 16, 64 SAD_X_LOOP 3, 24, 32 SAD_X_LOOP 3, 32, 8 SAD_X_LOOP 3, 32, 16 SAD_X_LOOP 3, 32, 24 SAD_X_LOOP 3, 32, 32 SAD_X_LOOP 3, 32, 64 SAD_X_LOOP 3, 48, 64 SAD_X_LOOP 3, 64, 16 SAD_X_LOOP 3, 64, 32 SAD_X_LOOP 3, 64, 48 SAD_X_LOOP 3, 64, 64 SAD_X_FUNC 4, 4, 4 SAD_X_FUNC 4, 4, 8 SAD_X_FUNC 4, 4, 16 SAD_X_FUNC 4, 8, 4 SAD_X_FUNC 4, 8, 8 SAD_X_FUNC 4, 8, 16 SAD_X_FUNC 4, 8, 32 SAD_X_FUNC 4, 12, 16 SAD_X_FUNC 4, 16, 4 SAD_X_FUNC 4, 16, 8 SAD_X_FUNC 4, 16, 12 SAD_X_FUNC 4, 16, 16 SAD_X_FUNC 4, 16, 32 SAD_X_FUNC 4, 16, 64 SAD_X_LOOP 4, 24, 32 SAD_X_LOOP 4, 32, 8 SAD_X_LOOP 4, 32, 16 SAD_X_LOOP 4, 32, 24 SAD_X_LOOP 4, 32, 32 SAD_X_LOOP 4, 32, 64 SAD_X_LOOP 4, 48, 64 SAD_X_LOOP 4, 64, 16 SAD_X_LOOP 4, 64, 32 SAD_X_LOOP 4, 64, 48 SAD_X_LOOP 4, 64, 64 const sad12_mask, align=8 .byte 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0 endconst