/*****************************************************************************
 * Copyright (C) 2020-2021 MulticoreWare, Inc
 *
 * Authors: Yimeng Su <yimeng.su@huawei.com>
 *          Hongbin Liu <liuhongbin1@huawei.com>
 *          Sebastian Pop <spop@amazon.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
 *
 * This program is also available under a commercial proprietary license.
 * For more information, contact us at license @ x265.com.
 *****************************************************************************/

#include "asm.S"

#ifdef __APPLE__
.section __RODATA,__rodata
#else
.section .rodata
#endif

.align 4

.text

// uint64_t pixel_var(const pixel* pix, intptr_t i_stride)
function PFX(pixel_var_8x8_neon)
    ldr             d4, [x0]                 // pix[x]
    add             x0, x0, x1
    uxtl            v0.8h, v4.8b             // sum = pix[x]
    umull           v1.8h, v4.8b, v4.8b
    uaddlp          v1.4s, v1.8h             // sqr = pix[x] * pix[x]

.rept 7
    ldr             d4, [x0]                 // pix[x]
    add             x0, x0, x1
    umull           v31.8h, v4.8b, v4.8b
    uaddw           v0.8h, v0.8h, v4.8b      // sum += pix[x]
    uadalp          v1.4s, v31.8h            // sqr += pix[x] * pix[x]
.endr
    uaddlv          s0, v0.8h
    uaddlv          d1, v1.4s
    fmov            w0, s0
    fmov            x1, d1
    orr             x0, x0, x1, lsl #32      // return sum + ((uint64_t)sqr << 32);
    ret
endfunc

.macro pixel_var_start
    movi            v0.16b, #0
    movi            v1.16b, #0
    movi            v2.16b, #0
    movi            v3.16b, #0
.endm

.macro pixel_var_1 v
    uaddw           v0.8h, v0.8h, \v\().8b
    umull           v30.8h, \v\().8b, \v\().8b
    uaddw2          v1.8h, v1.8h, \v\().16b
    umull2          v31.8h, \v\().16b, \v\().16b
    uadalp          v2.4s, v30.8h
    uadalp          v3.4s, v31.8h
.endm

.macro pixel_var_end
    uaddlv          s0, v0.8h
    uaddlv          s1, v1.8h
    add             v2.4s, v2.4s, v3.4s
    fadd            s0, s0, s1
    uaddlv          d2, v2.4s
    fmov            w0, s0
    fmov            x2, d2
    orr             x0, x0, x2, lsl #32
.endm

function PFX(pixel_var_16x16_neon)
    pixel_var_start
    mov             w12, #16
.loop_var_16:
    sub             w12, w12, #1
    ldr             q4, [x0]
    add             x0, x0, x1
    pixel_var_1 v4
    cbnz            w12, .loop_var_16
    pixel_var_end
    ret
endfunc

function PFX(pixel_var_32x32_neon)
    pixel_var_start
    mov             w12, #32
.loop_var_32:
    sub             w12, w12, #1
    ld1             {v4.16b-v5.16b}, [x0], x1
    pixel_var_1 v4
    pixel_var_1 v5
    cbnz            w12, .loop_var_32
    pixel_var_end
    ret
endfunc

function PFX(pixel_var_64x64_neon)
    pixel_var_start
    mov             w12, #64
.loop_var_64:
    sub             w12, w12, #1
    ld1             {v4.16b-v7.16b}, [x0], x1
    pixel_var_1 v4
    pixel_var_1 v5
    pixel_var_1 v6
    pixel_var_1 v7
    cbnz            w12, .loop_var_64
    pixel_var_end
    ret
endfunc

// void getResidual4_neon(const pixel* fenc, const pixel* pred, int16_t* residual, intptr_t stride)
function PFX(getResidual4_neon)
    lsl             x4, x3, #1
.rept 2
    ldr             d0, [x0]
    ldr             d1, [x1]
    add             x0, x0, x3
    add             x1, x1, x3
    ldr             d2, [x0]
    ldr             d3, [x1]
    add             x0, x0, x3
    add             x1, x1, x3
    usubl           v4.8h, v0.8b, v1.8b
    usubl           v5.8h, v2.8b, v3.8b
    str             d4, [x2]
    add             x2, x2, x4
    str             d5, [x2]
    add             x2, x2, x4
.endr
    ret
endfunc

function PFX(getResidual8_neon)
    lsl             x4, x3, #1
.rept 4
    ldr             d0, [x0]
    ldr             d1, [x1]
    add             x0, x0, x3
    add             x1, x1, x3
    ldr             d2, [x0]
    ldr             d3, [x1]
    add             x0, x0, x3
    add             x1, x1, x3
    usubl           v4.8h, v0.8b, v1.8b
    usubl           v5.8h, v2.8b, v3.8b
    str             q4, [x2]
    add             x2, x2, x4
    str             q5, [x2]
    add             x2, x2, x4
.endr
    ret
endfunc

function PFX(getResidual16_neon)
    lsl             x4, x3, #1
.rept 8
    ld1             {v0.16b}, [x0], x3
    ld1             {v1.16b}, [x1], x3
    ld1             {v2.16b}, [x0], x3
    ld1             {v3.16b}, [x1], x3
    usubl           v4.8h, v0.8b, v1.8b
    usubl2          v5.8h, v0.16b, v1.16b
    usubl           v6.8h, v2.8b, v3.8b
    usubl2          v7.8h, v2.16b, v3.16b
    st1             {v4.8h-v5.8h}, [x2], x4
    st1             {v6.8h-v7.8h}, [x2], x4
.endr
    ret
endfunc

function PFX(getResidual32_neon)
    lsl             x4, x3, #1
    mov             w12, #4
.loop_residual_32:
    sub             w12, w12, #1
.rept 4
    ld1             {v0.16b-v1.16b}, [x0], x3
    ld1             {v2.16b-v3.16b}, [x1], x3
    ld1             {v4.16b-v5.16b}, [x0], x3
    ld1             {v6.16b-v7.16b}, [x1], x3
    usubl           v16.8h, v0.8b, v2.8b
    usubl2          v17.8h, v0.16b, v2.16b
    usubl           v18.8h, v1.8b, v3.8b
    usubl2          v19.8h, v1.16b, v3.16b
    usubl           v20.8h, v4.8b, v6.8b
    usubl2          v21.8h, v4.16b, v6.16b
    usubl           v22.8h, v5.8b, v7.8b
    usubl2          v23.8h, v5.16b, v7.16b
    st1             {v16.8h-v19.8h}, [x2], x4
    st1             {v20.8h-v23.8h}, [x2], x4
.endr
    cbnz            w12, .loop_residual_32
    ret
endfunc

// void pixel_sub_ps_neon(int16_t* a, intptr_t dstride, const pixel* b0, const pixel* b1, intptr_t sstride0, intptr_t sstride1)
function PFX(pixel_sub_ps_4x4_neon)
    lsl             x1, x1, #1
.rept 2
    ld1             {v0.8b}, [x2], x4
    ld1             {v1.8b}, [x3], x5
    ld1             {v2.8b}, [x2], x4
    ld1             {v3.8b}, [x3], x5
    usubl           v4.8h, v0.8b, v1.8b
    usubl           v5.8h, v2.8b, v3.8b
    st1             {v4.4h}, [x0], x1
    st1             {v5.4h}, [x0], x1
.endr
    ret
endfunc

function PFX(pixel_sub_ps_8x8_neon)
    lsl             x1, x1, #1
.rept 4
    ld1             {v0.8b}, [x2], x4
    ld1             {v1.8b}, [x3], x5
    ld1             {v2.8b}, [x2], x4
    ld1             {v3.8b}, [x3], x5
    usubl           v4.8h, v0.8b, v1.8b
    usubl           v5.8h, v2.8b, v3.8b
    st1             {v4.8h}, [x0], x1
    st1             {v5.8h}, [x0], x1
.endr
    ret
endfunc

function PFX(pixel_sub_ps_16x16_neon)
    lsl             x1, x1, #1
.rept 8
    ld1             {v0.16b}, [x2], x4
    ld1             {v1.16b}, [x3], x5
    ld1             {v2.16b}, [x2], x4
    ld1             {v3.16b}, [x3], x5
    usubl           v4.8h, v0.8b, v1.8b
    usubl2          v5.8h, v0.16b, v1.16b
    usubl           v6.8h, v2.8b, v3.8b
    usubl2          v7.8h, v2.16b, v3.16b
    st1             {v4.8h-v5.8h}, [x0], x1
    st1             {v6.8h-v7.8h}, [x0], x1
.endr
    ret
endfunc

function PFX(pixel_sub_ps_32x32_neon)
    lsl             x1, x1, #1
    mov             w12, #4
.loop_sub_ps_32:
    sub             w12, w12, #1
.rept 4
    ld1             {v0.16b-v1.16b}, [x2], x4
    ld1             {v2.16b-v3.16b}, [x3], x5
    ld1             {v4.16b-v5.16b}, [x2], x4
    ld1             {v6.16b-v7.16b}, [x3], x5
    usubl           v16.8h, v0.8b, v2.8b
    usubl2          v17.8h, v0.16b, v2.16b
    usubl           v18.8h, v1.8b, v3.8b
    usubl2          v19.8h, v1.16b, v3.16b
    usubl           v20.8h, v4.8b, v6.8b
    usubl2          v21.8h, v4.16b, v6.16b
    usubl           v22.8h, v5.8b, v7.8b
    usubl2          v23.8h, v5.16b, v7.16b
    st1             {v16.8h-v19.8h}, [x0], x1
    st1             {v20.8h-v23.8h}, [x0], x1
.endr
    cbnz            w12, .loop_sub_ps_32
    ret
endfunc

function PFX(pixel_sub_ps_64x64_neon)
    lsl             x1, x1, #1
    sub             x1, x1, #64
    mov             w12, #16
.loop_sub_ps_64:
    sub             w12, w12, #1
.rept 4
    ld1             {v0.16b-v3.16b}, [x2], x4
    ld1             {v4.16b-v7.16b}, [x3], x5
    usubl           v16.8h, v0.8b, v4.8b
    usubl2          v17.8h, v0.16b, v4.16b
    usubl           v18.8h, v1.8b, v5.8b
    usubl2          v19.8h, v1.16b, v5.16b
    usubl           v20.8h, v2.8b, v6.8b
    usubl2          v21.8h, v2.16b, v6.16b
    usubl           v22.8h, v3.8b, v7.8b
    usubl2          v23.8h, v3.16b, v7.16b
    st1             {v16.8h-v19.8h}, [x0], #64
    st1             {v20.8h-v23.8h}, [x0], x1
.endr
    cbnz            w12, .loop_sub_ps_64
    ret
endfunc

// chroma sub_ps
function PFX(pixel_sub_ps_4x8_neon)
    lsl             x1, x1, #1
.rept 4
    ld1             {v0.8b}, [x2], x4
    ld1             {v1.8b}, [x3], x5
    ld1             {v2.8b}, [x2], x4
    ld1             {v3.8b}, [x3], x5
    usubl           v4.8h, v0.8b, v1.8b
    usubl           v5.8h, v2.8b, v3.8b
    st1             {v4.4h}, [x0], x1
    st1             {v5.4h}, [x0], x1
.endr
    ret
endfunc

function PFX(pixel_sub_ps_8x16_neon)
    lsl             x1, x1, #1
.rept 8
    ld1             {v0.8b}, [x2], x4
    ld1             {v1.8b}, [x3], x5
    ld1             {v2.8b}, [x2], x4
    ld1             {v3.8b}, [x3], x5
    usubl           v4.8h, v0.8b, v1.8b
    usubl           v5.8h, v2.8b, v3.8b
    st1             {v4.8h}, [x0], x1
    st1             {v5.8h}, [x0], x1
.endr
    ret
endfunc

function PFX(pixel_sub_ps_16x32_neon)
    lsl             x1, x1, #1
.rept 16
    ld1             {v0.16b}, [x2], x4
    ld1             {v1.16b}, [x3], x5
    ld1             {v2.16b}, [x2], x4
    ld1             {v3.16b}, [x3], x5
    usubl           v4.8h, v0.8b, v1.8b
    usubl2          v5.8h, v0.16b, v1.16b
    usubl           v6.8h, v2.8b, v3.8b
    usubl2          v7.8h, v2.16b, v3.16b
    st1             {v4.8h-v5.8h}, [x0], x1
    st1             {v6.8h-v7.8h}, [x0], x1
.endr
    ret
endfunc

function PFX(pixel_sub_ps_32x64_neon)
    lsl             x1, x1, #1
    mov             w12, #8
.loop_sub_ps_32x64:
    sub             w12, w12, #1
.rept 4
    ld1             {v0.16b-v1.16b}, [x2], x4
    ld1             {v2.16b-v3.16b}, [x3], x5
    ld1             {v4.16b-v5.16b}, [x2], x4
    ld1             {v6.16b-v7.16b}, [x3], x5
    usubl           v16.8h, v0.8b, v2.8b
    usubl2          v17.8h, v0.16b, v2.16b
    usubl           v18.8h, v1.8b, v3.8b
    usubl2          v19.8h, v1.16b, v3.16b
    usubl           v20.8h, v4.8b, v6.8b
    usubl2          v21.8h, v4.16b, v6.16b
    usubl           v22.8h, v5.8b, v7.8b
    usubl2          v23.8h, v5.16b, v7.16b
    st1             {v16.8h-v19.8h}, [x0], x1
    st1             {v20.8h-v23.8h}, [x0], x1
.endr
    cbnz            w12, .loop_sub_ps_32x64
    ret
endfunc

// void x265_pixel_add_ps_neon(pixel* a, intptr_t dstride, const pixel* b0, const int16_t* b1, intptr_t sstride0, intptr_t sstride1);
function PFX(pixel_add_ps_4x4_neon)
    lsl             x5, x5, #1
.rept 2
    ld1             {v0.8b}, [x2], x4
    ld1             {v1.8b}, [x2], x4
    ld1             {v2.4h}, [x3], x5
    ld1             {v3.4h}, [x3], x5
    uxtl            v0.8h, v0.8b
    uxtl            v1.8h, v1.8b
    add             v4.8h, v0.8h, v2.8h
    add             v5.8h, v1.8h, v3.8h
    sqxtun          v4.8b, v4.8h
    sqxtun          v5.8b, v5.8h
    st1             {v4.s}[0], [x0], x1
    st1             {v5.s}[0], [x0], x1
.endr
    ret
endfunc

function PFX(pixel_add_ps_8x8_neon)
    lsl             x5, x5, #1
.rept 4
    ld1             {v0.8b}, [x2], x4
    ld1             {v1.8b}, [x2], x4
    ld1             {v2.8h}, [x3], x5
    ld1             {v3.8h}, [x3], x5
    uxtl            v0.8h, v0.8b
    uxtl            v1.8h, v1.8b
    add             v4.8h, v0.8h, v2.8h
    add             v5.8h, v1.8h, v3.8h
    sqxtun          v4.8b, v4.8h
    sqxtun          v5.8b, v5.8h
    st1             {v4.8b}, [x0], x1
    st1             {v5.8b}, [x0], x1
.endr
    ret
endfunc

.macro pixel_add_ps_16xN_neon h
function PFX(pixel_add_ps_16x\h\()_neon)
    lsl             x5, x5, #1
    mov             w12, #\h / 8
.loop_add_ps_16x\h\():
    sub             w12, w12, #1
.rept 4
    ld1             {v0.16b}, [x2], x4
    ld1             {v1.16b}, [x2], x4
    ld1             {v16.8h-v17.8h}, [x3], x5
    ld1             {v18.8h-v19.8h}, [x3], x5
    uxtl            v4.8h, v0.8b
    uxtl2           v5.8h, v0.16b
    uxtl            v6.8h, v1.8b
    uxtl2           v7.8h, v1.16b
    add             v24.8h, v4.8h, v16.8h
    add             v25.8h, v5.8h, v17.8h
    add             v26.8h, v6.8h, v18.8h
    add             v27.8h, v7.8h, v19.8h
    sqxtun          v4.8b, v24.8h
    sqxtun2         v4.16b, v25.8h
    sqxtun          v5.8b, v26.8h
    sqxtun2         v5.16b, v27.8h
    st1             {v4.16b}, [x0], x1
    st1             {v5.16b}, [x0], x1
.endr
    cbnz            w12, .loop_add_ps_16x\h
    ret
endfunc
.endm

pixel_add_ps_16xN_neon 16
pixel_add_ps_16xN_neon 32

.macro pixel_add_ps_32xN_neon h
 function PFX(pixel_add_ps_32x\h\()_neon)
    lsl             x5, x5, #1
    mov             w12, #\h / 4
.loop_add_ps_32x\h\():
    sub             w12, w12, #1
.rept 4
    ld1             {v0.16b-v1.16b}, [x2], x4
    ld1             {v16.8h-v19.8h}, [x3], x5
    uxtl            v4.8h, v0.8b
    uxtl2           v5.8h, v0.16b
    uxtl            v6.8h, v1.8b
    uxtl2           v7.8h, v1.16b
    add             v24.8h, v4.8h, v16.8h
    add             v25.8h, v5.8h, v17.8h
    add             v26.8h, v6.8h, v18.8h
    add             v27.8h, v7.8h, v19.8h
    sqxtun          v4.8b, v24.8h
    sqxtun2         v4.16b, v25.8h
    sqxtun          v5.8b, v26.8h
    sqxtun2         v5.16b, v27.8h
    st1             {v4.16b-v5.16b}, [x0], x1
.endr
    cbnz            w12, .loop_add_ps_32x\h
    ret
endfunc
.endm

pixel_add_ps_32xN_neon 32
pixel_add_ps_32xN_neon 64

function PFX(pixel_add_ps_64x64_neon)
    lsl             x5, x5, #1
    sub             x5, x5, #64
    mov             w12, #32
.loop_add_ps_64x64:
    sub             w12, w12, #1
.rept 2
    ld1             {v0.16b-v3.16b}, [x2], x4
    ld1             {v16.8h-v19.8h}, [x3], #64
    ld1             {v20.8h-v23.8h}, [x3], x5
    uxtl            v4.8h, v0.8b
    uxtl2           v5.8h, v0.16b
    uxtl            v6.8h, v1.8b
    uxtl2           v7.8h, v1.16b
    uxtl            v24.8h, v2.8b
    uxtl2           v25.8h, v2.16b
    uxtl            v26.8h, v3.8b
    uxtl2           v27.8h, v3.16b
    add             v0.8h, v4.8h, v16.8h
    add             v1.8h, v5.8h, v17.8h
    add             v2.8h, v6.8h, v18.8h
    add             v3.8h, v7.8h, v19.8h
    add             v4.8h, v24.8h, v20.8h
    add             v5.8h, v25.8h, v21.8h
    add             v6.8h, v26.8h, v22.8h
    add             v7.8h, v27.8h, v23.8h
    sqxtun          v0.8b, v0.8h
    sqxtun2         v0.16b, v1.8h
    sqxtun          v1.8b, v2.8h
    sqxtun2         v1.16b, v3.8h
    sqxtun          v2.8b, v4.8h
    sqxtun2         v2.16b, v5.8h
    sqxtun          v3.8b, v6.8h
    sqxtun2         v3.16b, v7.8h
    st1             {v0.16b-v3.16b}, [x0], x1
.endr
    cbnz            w12, .loop_add_ps_64x64
    ret
endfunc

// Chroma add_ps
function PFX(pixel_add_ps_4x8_neon)
    lsl             x5, x5, #1
.rept 4
    ld1             {v0.8b}, [x2], x4
    ld1             {v1.8b}, [x2], x4
    ld1             {v2.4h}, [x3], x5
    ld1             {v3.4h}, [x3], x5
    uxtl            v0.8h, v0.8b
    uxtl            v1.8h, v1.8b
    add             v4.8h, v0.8h, v2.8h
    add             v5.8h, v1.8h, v3.8h
    sqxtun          v4.8b, v4.8h
    sqxtun          v5.8b, v5.8h
    st1             {v4.s}[0], [x0], x1
    st1             {v5.s}[0], [x0], x1
.endr
    ret
endfunc

function PFX(pixel_add_ps_8x16_neon)
    lsl             x5, x5, #1
.rept 8
    ld1             {v0.8b}, [x2], x4
    ld1             {v1.8b}, [x2], x4
    ld1             {v2.8h}, [x3], x5
    ld1             {v3.8h}, [x3], x5
    uxtl            v0.8h, v0.8b
    uxtl            v1.8h, v1.8b
    add             v4.8h, v0.8h, v2.8h
    add             v5.8h, v1.8h, v3.8h
    sqxtun          v4.8b, v4.8h
    sqxtun          v5.8b, v5.8h
    st1             {v4.8b}, [x0], x1
    st1             {v5.8b}, [x0], x1
.endr
    ret
endfunc

// void scale1D_128to64(pixel *dst, const pixel *src)
function PFX(scale1D_128to64_neon)
.rept 2
    ld2             {v0.16b, v1.16b}, [x1], #32
    ld2             {v2.16b, v3.16b}, [x1], #32
    ld2             {v4.16b, v5.16b}, [x1], #32
    ld2             {v6.16b, v7.16b}, [x1], #32
    urhadd          v0.16b, v0.16b, v1.16b
    urhadd          v1.16b, v2.16b, v3.16b
    urhadd          v2.16b, v4.16b, v5.16b
    urhadd          v3.16b, v6.16b, v7.16b
    st1             {v0.16b-v3.16b}, [x0], #64
.endr
    ret
endfunc

.macro scale2D_1  v0, v1
    uaddlp          \v0\().8h, \v0\().16b
    uaddlp          \v1\().8h, \v1\().16b
    add             \v0\().8h, \v0\().8h, \v1\().8h
.endm

// void scale2D_64to32(pixel* dst, const pixel* src, intptr_t stride)
function PFX(scale2D_64to32_neon)
    mov             w12, #32
.loop_scale2D:
    ld1             {v0.16b-v3.16b}, [x1], x2
    sub             w12, w12, #1
    ld1             {v4.16b-v7.16b}, [x1], x2
    scale2D_1       v0, v4
    scale2D_1       v1, v5
    scale2D_1       v2, v6
    scale2D_1       v3, v7
    uqrshrn         v0.8b, v0.8h, #2
    uqrshrn2        v0.16b, v1.8h, #2
    uqrshrn         v1.8b, v2.8h, #2
    uqrshrn2        v1.16b, v3.8h, #2
    st1             {v0.16b-v1.16b}, [x0], #32
    cbnz            w12, .loop_scale2D
    ret
endfunc

// void planecopy_cp_c(const uint8_t* src, intptr_t srcStride, pixel* dst, intptr_t dstStride, int width, int height, int shift)
function PFX(pixel_planecopy_cp_neon)
    dup             v2.16b, w6
    sub             x5, x5, #1
.loop_h:
    mov             x6, x0
    mov             x12, x2
    mov             x7, #0
.loop_w:
    ldr             q0, [x6], #16
    ushl            v0.16b, v0.16b, v2.16b
    str             q0, [x12], #16
    add             x7, x7, #16
    cmp             x7, x4
    blt             .loop_w

    add             x0, x0, x1
    add             x2, x2, x3
    sub             x5, x5, #1
    cbnz            x5, .loop_h

// handle last row
    mov             x5, x4
    lsr             x5, x5, #3
.loopW8:
    ldr             d0, [x0], #8
    ushl            v0.8b, v0.8b, v2.8b
    str             d0, [x2], #8
    sub             x4, x4, #8
    sub             x5, x5, #1
    cbnz            x5, .loopW8

    mov             x5, #8
    sub             x5, x5, x4
    sub             x0, x0, x5
    sub             x2, x2, x5
    ldr             d0, [x0]
    ushl            v0.8b, v0.8b, v2.8b
    str             d0, [x2]
    ret
endfunc

//******* satd *******
.macro satd_4x4_neon
    ldr             s1, [x2]
    ldr             s0, [x0]
    add             x2, x2, x3
    add             x0, x0, x1
    ldr             s3, [x2]
    ldr             s2, [x0]
    add             x2, x2, x3
    add             x0, x0, x1

    ld1             {v1.s}[1], [x2], x3
    ld1             {v0.s}[1], [x0], x1
    ld1             {v3.s}[1], [x2], x3
    ld1             {v2.s}[1], [x0], x1

    usubl           v4.8h, v0.8b, v1.8b
    usubl           v5.8h, v2.8b, v3.8b

    add             v6.8h, v4.8h, v5.8h
    sub             v7.8h, v4.8h, v5.8h

    mov             v4.d[0], v6.d[1]
    add             v0.4h, v6.4h, v4.4h
    sub             v2.4h, v6.4h, v4.4h

    mov             v5.d[0], v7.d[1]
    add             v1.4h, v7.4h, v5.4h
    sub             v3.4h, v7.4h, v5.4h

    trn1            v4.4h, v0.4h, v1.4h
    trn2            v5.4h, v0.4h, v1.4h

    trn1            v6.4h, v2.4h, v3.4h
    trn2            v7.4h, v2.4h, v3.4h

    add             v0.4h, v4.4h, v5.4h
    sub             v1.4h, v4.4h, v5.4h

    add             v2.4h, v6.4h, v7.4h
    sub             v3.4h, v6.4h, v7.4h

    trn1            v4.2s, v0.2s, v1.2s
    trn2            v5.2s, v0.2s, v1.2s

    trn1            v6.2s, v2.2s, v3.2s
    trn2            v7.2s, v2.2s, v3.2s

    abs             v4.4h, v4.4h
    abs             v5.4h, v5.4h
    abs             v6.4h, v6.4h
    abs             v7.4h, v7.4h

    smax            v1.4h, v4.4h, v5.4h
    smax            v2.4h, v6.4h, v7.4h

    add             v0.4h, v1.4h, v2.4h
    uaddlp          v0.2s, v0.4h
    uaddlp          v0.1d, v0.2s
.endm

// int satd_4x4(const pixel* pix1, intptr_t stride_pix1, const pixel* pix2, intptr_t stride_pix2)
function PFX(pixel_satd_4x4_neon)
    satd_4x4_neon
    fmov            x0, d0
    ret
endfunc

.macro x265_satd_4x8_8x4_end_neon
    add             v0.8h, v4.8h, v6.8h
    add             v1.8h, v5.8h, v7.8h
    sub             v2.8h, v4.8h, v6.8h
    sub             v3.8h, v5.8h, v7.8h

    trn1            v16.8h, v0.8h, v1.8h
    trn2            v17.8h, v0.8h, v1.8h
    add             v4.8h, v16.8h, v17.8h
    trn1            v18.8h, v2.8h, v3.8h
    trn2            v19.8h, v2.8h, v3.8h
    sub             v5.8h, v16.8h, v17.8h
    add             v6.8h, v18.8h, v19.8h
    sub             v7.8h, v18.8h, v19.8h
    trn1            v0.4s, v4.4s, v6.4s
    trn2            v2.4s, v4.4s, v6.4s
    abs             v0.8h, v0.8h
    trn1            v1.4s, v5.4s, v7.4s
    trn2            v3.4s, v5.4s, v7.4s
    abs             v2.8h, v2.8h
    abs             v1.8h, v1.8h
    abs             v3.8h, v3.8h
    umax            v0.8h, v0.8h, v2.8h
    umax            v1.8h, v1.8h, v3.8h
    add             v0.8h, v0.8h, v1.8h
    uaddlv          s0, v0.8h
.endm

.macro pixel_satd_4x8_neon
    ld1r            {v1.2s}, [x2], x3
    ld1r            {v0.2s}, [x0], x1
    ld1r            {v3.2s}, [x2], x3
    ld1r            {v2.2s}, [x0], x1
    ld1r            {v5.2s}, [x2], x3
    ld1r            {v4.2s}, [x0], x1
    ld1r            {v7.2s}, [x2], x3
    ld1r            {v6.2s}, [x0], x1

    ld1             {v1.s}[1], [x2], x3
    ld1             {v0.s}[1], [x0], x1
    usubl           v0.8h, v0.8b, v1.8b
    ld1             {v3.s}[1], [x2], x3
    ld1             {v2.s}[1], [x0], x1
    usubl           v1.8h, v2.8b, v3.8b
    ld1             {v5.s}[1], [x2], x3
    ld1             {v4.s}[1], [x0], x1
    usubl           v2.8h, v4.8b, v5.8b
    ld1             {v7.s}[1], [x2], x3
    add             v4.8h, v0.8h, v1.8h
    sub             v5.8h, v0.8h, v1.8h
    ld1             {v6.s}[1], [x0], x1
    usubl           v3.8h, v6.8b, v7.8b
    add             v6.8h, v2.8h, v3.8h
    sub             v7.8h, v2.8h, v3.8h
    x265_satd_4x8_8x4_end_neon
.endm

// template<int w, int h>
// int satd4(const pixel* pix1, intptr_t stride_pix1, const pixel* pix2, intptr_t stride_pix2)
function PFX(pixel_satd_4x8_neon)
    pixel_satd_4x8_neon
    mov             w0, v0.s[0]
    ret
endfunc

function PFX(pixel_satd_4x16_neon)
    mov             w4, #0
    pixel_satd_4x8_neon
    mov             w5, v0.s[0]
    add             w4, w4, w5
    pixel_satd_4x8_neon
    mov             w5, v0.s[0]
    add             w0, w5, w4
    ret
endfunc

function PFX(pixel_satd_4x32_neon)
    mov             w4, #0
.rept 4
    pixel_satd_4x8_neon
    mov             w5, v0.s[0]
    add             w4, w4, w5
.endr
    mov             w0, w4
    ret
endfunc

function PFX(pixel_satd_12x16_neon)
    mov             x4, x0
    mov             x5, x2
    mov             w7, #0
    pixel_satd_4x8_neon
    mov             w6, v0.s[0]
    add             w7, w7, w6
    pixel_satd_4x8_neon
    mov             w6, v0.s[0]
    add             w7, w7, w6

    add             x0, x4, #4
    add             x2, x5, #4
    pixel_satd_4x8_neon
    mov             w6, v0.s[0]
    add             w7, w7, w6
    pixel_satd_4x8_neon
    mov             w6, v0.s[0]
    add             w7, w7, w6

    add             x0, x4, #8
    add             x2, x5, #8
    pixel_satd_4x8_neon
    mov             w6, v0.s[0]
    add             w7, w7, w6
    pixel_satd_4x8_neon
    mov             w6, v0.s[0]
    add             w0, w7, w6
    ret
endfunc

function PFX(pixel_satd_12x32_neon)
    mov             x4, x0
    mov             x5, x2
    mov             w7, #0
.rept 4
    pixel_satd_4x8_neon
    mov             w6, v0.s[0]
    add             w7, w7, w6
.endr

    add             x0, x4, #4
    add             x2, x5, #4
.rept 4
    pixel_satd_4x8_neon
    mov             w6, v0.s[0]
    add             w7, w7, w6
.endr

    add             x0, x4, #8
    add             x2, x5, #8
.rept 4
    pixel_satd_4x8_neon
    mov             w6, v0.s[0]
    add             w7, w7, w6
.endr

    mov             w0, w7
    ret
endfunc

function PFX(pixel_satd_8x4_neon)
    mov             x4, x0
    mov             x5, x2
    satd_4x4_neon
    add             x0, x4, #4
    add             x2, x5, #4
    umov            x6, v0.d[0]
    satd_4x4_neon
    umov            x0, v0.d[0]
    add             x0, x0, x6
    ret
endfunc

.macro LOAD_DIFF_8x4 v0 v1 v2 v3
    ld1             {v0.8b}, [x0], x1
    ld1             {v1.8b}, [x2], x3
    ld1             {v2.8b}, [x0], x1
    ld1             {v3.8b}, [x2], x3
    ld1             {v4.8b}, [x0], x1
    ld1             {v5.8b}, [x2], x3
    ld1             {v6.8b}, [x0], x1
    ld1             {v7.8b}, [x2], x3
    usubl           \v0, v0.8b, v1.8b
    usubl           \v1, v2.8b, v3.8b
    usubl           \v2, v4.8b, v5.8b
    usubl           \v3, v6.8b, v7.8b
.endm

.macro LOAD_DIFF_16x4 v0 v1 v2 v3 v4 v5 v6 v7
    ld1             {v0.16b}, [x0], x1
    ld1             {v1.16b}, [x2], x3
    ld1             {v2.16b}, [x0], x1
    ld1             {v3.16b}, [x2], x3
    ld1             {v4.16b}, [x0], x1
    ld1             {v5.16b}, [x2], x3
    ld1             {v6.16b}, [x0], x1
    ld1             {v7.16b}, [x2], x3
    usubl           \v0, v0.8b, v1.8b
    usubl           \v1, v2.8b, v3.8b
    usubl           \v2, v4.8b, v5.8b
    usubl           \v3, v6.8b, v7.8b
    usubl2          \v4, v0.16b, v1.16b
    usubl2          \v5, v2.16b, v3.16b
    usubl2          \v6, v4.16b, v5.16b
    usubl2          \v7, v6.16b, v7.16b
.endm

function PFX(satd_16x4_neon), export=0
    LOAD_DIFF_16x4  v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
    b               PFX(satd_8x4v_8x8h_neon)
endfunc

function PFX(satd_8x8_neon), export=0
    LOAD_DIFF_8x4   v16.8h, v17.8h, v18.8h, v19.8h
    LOAD_DIFF_8x4   v20.8h, v21.8h, v22.8h, v23.8h
    b               PFX(satd_8x4v_8x8h_neon)
endfunc

// one vertical hadamard pass and two horizontal
function PFX(satd_8x4v_8x8h_neon), export=0
    HADAMARD4_V     v16.8h, v18.8h, v17.8h, v19.8h, v0.8h, v2.8h, v1.8h, v3.8h
    HADAMARD4_V     v20.8h, v21.8h, v22.8h, v23.8h, v0.8h, v1.8h, v2.8h, v3.8h
    trn4            v0.8h, v1.8h, v2.8h, v3.8h, v16.8h, v17.8h, v18.8h, v19.8h
    trn4            v4.8h, v5.8h, v6.8h, v7.8h, v20.8h, v21.8h, v22.8h, v23.8h
    SUMSUB_ABCD     v16.8h, v17.8h, v18.8h, v19.8h, v0.8h, v1.8h, v2.8h, v3.8h
    SUMSUB_ABCD     v20.8h, v21.8h, v22.8h, v23.8h, v4.8h, v5.8h, v6.8h, v7.8h
    trn4            v0.4s, v2.4s, v1.4s, v3.4s, v16.4s, v18.4s, v17.4s, v19.4s
    trn4            v4.4s, v6.4s, v5.4s, v7.4s, v20.4s, v22.4s, v21.4s, v23.4s
    ABS8            v0.8h, v1.8h, v2.8h, v3.8h, v4.8h, v5.8h, v6.8h, v7.8h
    smax            v0.8h, v0.8h, v2.8h
    smax            v1.8h, v1.8h, v3.8h
    smax            v2.8h, v4.8h, v6.8h
    smax            v3.8h, v5.8h, v7.8h
    ret
endfunc

function PFX(pixel_satd_8x8_neon)
    mov             x10, x30
    bl              PFX(satd_8x8_neon)
    add             v0.8h, v0.8h, v1.8h
    add             v1.8h, v2.8h, v3.8h
    add             v0.8h, v0.8h, v1.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_8x12_neon)
    mov             x4, x0
    mov             x5, x2
    mov             x7, #0
    satd_4x4_neon
    umov            x6, v0.d[0]
    add             x7, x7, x6
    add             x0, x4, #4
    add             x2, x5, #4
    satd_4x4_neon
    umov            x6, v0.d[0]
    add             x7, x7, x6
.rept 2
    sub             x0, x0, #4
    sub             x2, x2, #4
    mov             x4, x0
    mov             x5, x2
    satd_4x4_neon
    umov            x6, v0.d[0]
    add             x7, x7, x6
    add             x0, x4, #4
    add             x2, x5, #4
    satd_4x4_neon
    umov            x6, v0.d[0]
    add             x7, x7, x6
.endr
    mov             x0, x7
    ret
endfunc

function PFX(pixel_satd_8x16_neon)
    mov             x10, x30
    bl              PFX(satd_8x8_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
    bl              PFX(satd_8x8_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_8x32_neon)
    mov             x10, x30
    bl              PFX(satd_8x8_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
.rept 3
    bl              PFX(satd_8x8_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_8x64_neon)
    mov             x10, x30
    bl              PFX(satd_8x8_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
.rept 7
    bl              PFX(satd_8x8_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_16x4_neon)
    mov             x10, x30
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_16x8_neon)
    mov             x10, x30
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_16x12_neon)
    mov             x10, x30
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
.rept 2
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_16x16_neon)
    mov             x10, x30
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
.rept 3
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_16x24_neon)
    mov             x10, x30
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
.rept 5
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

.macro pixel_satd_16x32_neon
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
.rept 7
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
.endm

function PFX(pixel_satd_16x32_neon)
    mov             x10, x30
    pixel_satd_16x32_neon
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_16x64_neon)
    mov             x10, x30
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v0.8h, v1.8h
    add             v31.8h, v2.8h, v3.8h
.rept 15
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_24x32_neon)
    mov             x10, x30
    mov             x7, #0
    mov             x4, x0
    mov             x5, x2
.rept 3
    movi            v30.8h, #0
    movi            v31.8h, #0
.rept 4
    bl              PFX(satd_8x8_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w6, v0.s[0]
    add             x7, x7, x6
    add             x4, x4, #8
    add             x5, x5, #8
    mov             x0, x4
    mov             x2, x5
.endr
    mov             x0, x7
    ret             x10
endfunc

function PFX(pixel_satd_24x64_neon)
    mov             x10, x30
    mov             x7, #0
    mov             x4, x0
    mov             x5, x2
.rept 3
    movi            v30.8h, #0
    movi            v31.8h, #0
.rept 4
    bl              PFX(satd_8x8_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w6, v0.s[0]
    add             x7, x7, x6
    add             x4, x4, #8
    add             x5, x5, #8
    mov             x0, x4
    mov             x2, x5
.endr
    sub             x4, x4, #24
    sub             x5, x5, #24
    add             x0, x4, x1, lsl #5
    add             x2, x5, x3, lsl #5
    mov             x4, x0
    mov             x5, x2
.rept 3
    movi            v30.8h, #0
    movi            v31.8h, #0
.rept 4
    bl              PFX(satd_8x8_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w6, v0.s[0]
    add             x7, x7, x6
    add             x4, x4, #8
    add             x5, x5, #8
    mov             x0, x4
    mov             x2, x5
.endr
    mov             x0, x7
    ret             x10
endfunc

.macro pixel_satd_32x8
    mov             x4, x0
    mov             x5, x2
.rept 2
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
    add             x0, x4, #16
    add             x2, x5, #16
.rept 2
    bl              PFX(satd_16x4_neon)
    add             v30.8h, v30.8h, v0.8h
    add             v31.8h, v31.8h, v1.8h
    add             v30.8h, v30.8h, v2.8h
    add             v31.8h, v31.8h, v3.8h
.endr
.endm

.macro satd_32x16_neon
    movi            v30.8h, #0
    movi            v31.8h, #0
    pixel_satd_32x8
    sub             x0, x0, #16
    sub             x2, x2, #16
    pixel_satd_32x8
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w6, v0.s[0]
.endm

.macro satd_64x16_neon
    mov             x8, x0
    mov             x9, x2
    satd_32x16_neon
    add             x7, x7, x6
    add             x0, x8, #32
    add             x2, x9, #32
    satd_32x16_neon
    add             x7, x7, x6
.endm

function PFX(pixel_satd_32x8_neon)
    mov             x10, x30
    mov             x7, #0
    mov             x4, x0
    mov             x5, x2
    movi            v30.8h, #0
    movi            v31.8h, #0
    pixel_satd_32x8
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    ret             x10
endfunc

function PFX(pixel_satd_32x16_neon)
    mov             x10, x30
    satd_32x16_neon
    mov             x0, x6
    ret             x10
endfunc

function PFX(pixel_satd_32x24_neon)
    mov             x10, x30
    satd_32x16_neon
    movi            v30.8h, #0
    movi            v31.8h, #0
    sub             x0, x0, #16
    sub             x2, x2, #16
    pixel_satd_32x8
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    add             x0, x0, x6
    ret             x10
endfunc

function PFX(pixel_satd_32x32_neon)
    mov             x10, x30
    mov             x7, #0
    satd_32x16_neon
    sub             x0, x0, #16
    sub             x2, x2, #16
    add             x7, x7, x6
    satd_32x16_neon
    add             x0, x7, x6
    ret             x10
endfunc

function PFX(pixel_satd_32x48_neon)
    mov             x10, x30
    mov             x7, #0
.rept 2
    satd_32x16_neon
    sub             x0, x0, #16
    sub             x2, x2, #16
    add             x7, x7, x6
.endr
    satd_32x16_neon
    add             x0, x7, x6
    ret             x10
endfunc

function PFX(pixel_satd_32x64_neon)
    mov             x10, x30
    mov             x7, #0
.rept 3
    satd_32x16_neon
    sub             x0, x0, #16
    sub             x2, x2, #16
    add             x7, x7, x6
.endr
    satd_32x16_neon
    add             x0, x7, x6
    ret             x10
endfunc

function PFX(pixel_satd_64x16_neon)
    mov             x10, x30
    mov             x7, #0
    satd_64x16_neon
    mov             x0, x7
    ret             x10
endfunc

function PFX(pixel_satd_64x32_neon)
    mov             x10, x30
    mov             x7, #0
    satd_64x16_neon
    sub             x0, x0, #48
    sub             x2, x2, #48
    satd_64x16_neon
    mov             x0, x7
    ret             x10
endfunc

function PFX(pixel_satd_64x48_neon)
    mov             x10, x30
    mov             x7, #0
.rept 2
    satd_64x16_neon
    sub             x0, x0, #48
    sub             x2, x2, #48
.endr
    satd_64x16_neon
    mov             x0, x7
    ret             x10
endfunc

function PFX(pixel_satd_64x64_neon)
    mov             x10, x30
    mov             x7, #0
.rept 3
    satd_64x16_neon
    sub             x0, x0, #48
    sub             x2, x2, #48
.endr
    satd_64x16_neon
    mov             x0, x7
    ret             x10
endfunc

function PFX(pixel_satd_48x64_neon)
    mov             x10, x30
    mov             x7, #0
    mov             x8, x0
    mov             x9, x2
.rept 3
    satd_32x16_neon
    sub             x0, x0, #16
    sub             x2, x2, #16
    add             x7, x7, x6
.endr
    satd_32x16_neon
    add             x7, x7, x6

    add             x0, x8, #32
    add             x2, x9, #32
    pixel_satd_16x32_neon
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w6, v0.s[0]
    add             x7, x7, x6

    movi            v30.8h, #0
    movi            v31.8h, #0
    pixel_satd_16x32_neon
    add             v0.8h, v30.8h, v31.8h
    uaddlv          s0, v0.8h
    mov             w6, v0.s[0]
    add             x0, x7, x6
    ret             x10
endfunc

function PFX(sa8d_8x8_neon), export=0
    LOAD_DIFF_8x4   v16.8h, v17.8h, v18.8h, v19.8h
    LOAD_DIFF_8x4   v20.8h, v21.8h, v22.8h, v23.8h
    HADAMARD4_V     v16.8h, v18.8h, v17.8h, v19.8h, v0.8h, v2.8h, v1.8h, v3.8h
    HADAMARD4_V     v20.8h, v21.8h, v22.8h, v23.8h, v0.8h, v1.8h, v2.8h, v3.8h
    SUMSUB_ABCD     v0.8h, v16.8h, v1.8h, v17.8h, v16.8h, v20.8h, v17.8h, v21.8h
    SUMSUB_ABCD     v2.8h, v18.8h, v3.8h, v19.8h, v18.8h, v22.8h, v19.8h, v23.8h
    trn4            v4.8h, v5.8h, v6.8h, v7.8h, v0.8h, v1.8h, v2.8h, v3.8h
    trn4            v20.8h, v21.8h, v22.8h, v23.8h, v16.8h, v17.8h, v18.8h, v19.8h
    SUMSUB_ABCD     v2.8h, v3.8h, v24.8h, v25.8h, v20.8h, v21.8h, v4.8h, v5.8h
    SUMSUB_ABCD     v0.8h, v1.8h, v4.8h, v5.8h, v22.8h, v23.8h, v6.8h, v7.8h
    trn4            v20.4s, v22.4s, v21.4s, v23.4s, v2.4s, v0.4s, v3.4s, v1.4s
    trn4            v16.4s, v18.4s, v17.4s, v19.4s, v24.4s, v4.4s, v25.4s, v5.4s
    SUMSUB_ABCD     v0.8h, v2.8h, v1.8h, v3.8h, v20.8h, v22.8h, v21.8h, v23.8h
    SUMSUB_ABCD     v4.8h, v6.8h, v5.8h, v7.8h, v16.8h, v18.8h, v17.8h, v19.8h
    trn4            v16.2d, v20.2d, v17.2d, v21.2d, v0.2d, v4.2d, v1.2d, v5.2d
    trn4            v18.2d, v22.2d, v19.2d, v23.2d, v2.2d, v6.2d, v3.2d, v7.2d
    ABS8            v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
    smax            v16.8h, v16.8h, v20.8h
    smax            v17.8h, v17.8h, v21.8h
    smax            v18.8h, v18.8h, v22.8h
    smax            v19.8h, v19.8h, v23.8h
    add             v0.8h, v16.8h, v17.8h
    add             v1.8h, v18.8h, v19.8h
    ret
endfunc

function PFX(pixel_sa8d_8x8_neon)
    mov             x10, x30
    bl              PFX(sa8d_8x8_neon)
    add             v0.8h, v0.8h, v1.8h
    uaddlv          s0, v0.8h
    mov             w0, v0.s[0]
    add             w0, w0, #1
    lsr             w0, w0, #1
    ret             x10
endfunc

function PFX(pixel_sa8d_8x16_neon)
    mov             x10, x30
    bl              PFX(sa8d_8x8_neon)
    add             v0.8h, v0.8h, v1.8h
    uaddlv          s0, v0.8h
    mov             w5, v0.s[0]
    add             w5, w5, #1
    lsr             w5, w5, #1
    bl              PFX(sa8d_8x8_neon)
    add             v0.8h, v0.8h, v1.8h
    uaddlv          s0, v0.8h
    mov             w4, v0.s[0]
    add             w4, w4, #1
    lsr             w4, w4, #1
    add             w0, w4, w5
    ret             x10
endfunc

.macro sa8d_16x16 reg
    bl              PFX(sa8d_8x8_neon)
    uaddlp          v30.4s, v0.8h
    uaddlp          v31.4s, v1.8h
    bl              PFX(sa8d_8x8_neon)
    uadalp          v30.4s, v0.8h
    uadalp          v31.4s, v1.8h
    sub             x0, x0, x1, lsl #4
    sub             x2, x2, x3, lsl #4
    add             x0, x0, #8
    add             x2, x2, #8
    bl              PFX(sa8d_8x8_neon)
    uadalp          v30.4s, v0.8h
    uadalp          v31.4s, v1.8h
    bl              PFX(sa8d_8x8_neon)
    uadalp          v30.4s, v0.8h
    uadalp          v31.4s, v1.8h
    add             v0.4s, v30.4s, v31.4s
    addv            s0, v0.4s
    mov             \reg, v0.s[0]
    add             \reg, \reg, #1
    lsr             \reg, \reg, #1
.endm

function PFX(pixel_sa8d_16x16_neon)
    mov             x10, x30
    sa8d_16x16      w0
    ret             x10
endfunc

function PFX(pixel_sa8d_16x32_neon)
    mov             x10, x30
    sa8d_16x16      w4
    sub             x0, x0, #8
    sub             x2, x2, #8
    sa8d_16x16      w5
    add             w0, w4, w5
    ret             x10
endfunc

function PFX(pixel_sa8d_32x32_neon)
    mov             x10, x30
    sa8d_16x16      w4
    sub             x0, x0, x1, lsl #4
    sub             x2, x2, x3, lsl #4
    add             x0, x0, #8
    add             x2, x2, #8
    sa8d_16x16      w5
    sub             x0, x0, #24
    sub             x2, x2, #24
    sa8d_16x16      w6
    sub             x0, x0, x1, lsl #4
    sub             x2, x2, x3, lsl #4
    add             x0, x0, #8
    add             x2, x2, #8
    sa8d_16x16      w7
    add             w4, w4, w5
    add             w6, w6, w7
    add             w0, w4, w6
    ret             x10
endfunc

function PFX(pixel_sa8d_32x64_neon)
    mov             x10, x30
    mov             w11, #4
    mov             w9, #0
.loop_sa8d_32:
    sub             w11, w11, #1
    sa8d_16x16      w4
    sub             x0, x0, x1, lsl #4
    sub             x2, x2, x3, lsl #4
    add             x0, x0, #8
    add             x2, x2, #8
    sa8d_16x16      w5
    add             w4, w4, w5
    add             w9, w9, w4
    sub             x0, x0, #24
    sub             x2, x2, #24
    cbnz            w11, .loop_sa8d_32
    mov             w0, w9
    ret             x10
endfunc

function PFX(pixel_sa8d_64x64_neon)
    mov             x10, x30
    mov             w11, #4
    mov             w9, #0
.loop_sa8d_64:
    sub             w11, w11, #1
    sa8d_16x16      w4
    sub             x0, x0, x1, lsl #4
    sub             x2, x2, x3, lsl #4
    add             x0, x0, #8
    add             x2, x2, #8
    sa8d_16x16      w5
    sub             x0, x0, x1, lsl #4
    sub             x2, x2, x3, lsl #4
    add             x0, x0, #8
    add             x2, x2, #8
    sa8d_16x16      w6
    sub             x0, x0, x1, lsl #4
    sub             x2, x2, x3, lsl #4
    add             x0, x0, #8
    add             x2, x2, #8
    sa8d_16x16      w7
    add             w4, w4, w5
    add             w6, w6, w7
    add             w8, w4, w6
    add             w9, w9, w8

    sub             x0, x0, #56
    sub             x2, x2, #56
    cbnz            w11, .loop_sa8d_64
    mov             w0, w9
    ret             x10
endfunc

/***** dequant_scaling*****/
// void dequant_scaling_c(const int16_t* quantCoef, const int32_t* deQuantCoef, int16_t* coef, int num, int per, int shift)
function PFX(dequant_scaling_neon)
    add             x5, x5, #4              // shift + 4
    lsr             x3, x3, #3              // num / 8
    cmp             x5, x4
    blt             .dequant_skip

    mov             x12, #1
    sub             x6, x5, x4          // shift - per
    sub             x6, x6, #1          // shift - per - 1
    lsl             x6, x12, x6         // 1 << shift - per - 1 (add)
    dup             v0.4s, w6
    sub             x7, x4, x5          // per - shift
    dup             v3.4s, w7

.dequant_loop1:
    ld1             {v19.8h}, [x0], #16 // quantCoef
    ld1             {v2.4s}, [x1], #16  // deQuantCoef
    ld1             {v20.4s}, [x1], #16
    sub             x3, x3, #1
    sxtl            v1.4s, v19.4h
    sxtl2           v19.4s, v19.8h

    mul             v1.4s, v1.4s, v2.4s // quantCoef * deQuantCoef
    mul             v19.4s, v19.4s, v20.4s
    add             v1.4s, v1.4s, v0.4s // quantCoef * deQuantCoef + add
    add             v19.4s, v19.4s, v0.4s

    sshl            v1.4s, v1.4s, v3.4s
    sshl            v19.4s, v19.4s, v3.4s
    sqxtn           v16.4h, v1.4s       // x265_clip3
    sqxtn2          v16.8h, v19.4s
    st1             {v16.8h}, [x2], #16
    cbnz            x3, .dequant_loop1
    ret

.dequant_skip:
    sub             x6, x4, x5          // per - shift
    dup             v0.8h, w6

.dequant_loop2:
    ld1             {v19.8h}, [x0], #16 // quantCoef
    ld1             {v2.4s}, [x1], #16  // deQuantCoef
    ld1             {v20.4s}, [x1], #16
    sub             x3, x3, #1
    sxtl            v1.4s, v19.4h
    sxtl2           v19.4s, v19.8h

    mul             v1.4s, v1.4s, v2.4s // quantCoef * deQuantCoef
    mul             v19.4s, v19.4s, v20.4s
    sqxtn           v16.4h, v1.4s       // x265_clip3
    sqxtn2          v16.8h, v19.4s

    sqshl           v16.8h, v16.8h, v0.8h // coefQ << per - shift
    st1             {v16.8h}, [x2], #16
    cbnz            x3, .dequant_loop2
    ret
endfunc

// void dequant_normal_c(const int16_t* quantCoef, int16_t* coef, int num, int scale, int shift)
function PFX(dequant_normal_neon)
    lsr             w2, w2, #4              // num / 16
    neg             w4, w4
    dup             v0.8h, w3
    dup             v1.4s, w4

.dqn_loop1:
    ld1             {v2.8h, v3.8h}, [x0], #32
    smull           v16.4s, v2.4h, v0.4h
    smull2          v17.4s, v2.8h, v0.8h
    smull           v18.4s, v3.4h, v0.4h
    smull2          v19.4s, v3.8h, v0.8h

    srshl           v16.4s, v16.4s, v1.4s
    srshl           v17.4s, v17.4s, v1.4s
    srshl           v18.4s, v18.4s, v1.4s
    srshl           v19.4s, v19.4s, v1.4s

    sqxtn           v2.4h, v16.4s
    sqxtn2          v2.8h, v17.4s
    sqxtn           v3.4h, v18.4s
    sqxtn2          v3.8h, v19.4s

    sub             w2, w2, #1
    st1             {v2.8h, v3.8h}, [x1], #32
    cbnz            w2, .dqn_loop1
    ret
endfunc

/********* ssim ***********/
// void ssim_4x4x2_core(const pixel* pix1, intptr_t stride1, const pixel* pix2, intptr_t stride2, int sums[2][4])
function PFX(ssim_4x4x2_core_neon)
    ld1             {v0.8b}, [x0], x1
    ld1             {v1.8b}, [x0], x1
    ld1             {v2.8b}, [x0], x1
    ld1             {v3.8b}, [x0], x1

    ld1             {v4.8b}, [x2], x3
    ld1             {v5.8b}, [x2], x3
    ld1             {v6.8b}, [x2], x3
    ld1             {v7.8b}, [x2], x3

    umull           v16.8h, v0.8b, v0.8b
    umull           v17.8h, v1.8b, v1.8b
    umull           v18.8h, v2.8b, v2.8b
    uaddlp          v30.4s, v16.8h
    umull           v19.8h, v3.8b, v3.8b
    umull           v20.8h, v4.8b, v4.8b
    umull           v21.8h, v5.8b, v5.8b
    uadalp          v30.4s, v17.8h
    umull           v22.8h, v6.8b, v6.8b
    umull           v23.8h, v7.8b, v7.8b

    umull           v24.8h, v0.8b, v4.8b
    uadalp          v30.4s, v18.8h
    umull           v25.8h, v1.8b, v5.8b
    umull           v26.8h, v2.8b, v6.8b
    umull           v27.8h, v3.8b, v7.8b
    uadalp          v30.4s, v19.8h

    uaddl           v28.8h, v0.8b, v1.8b
    uaddl           v29.8h, v4.8b, v5.8b
    uadalp          v30.4s, v20.8h
    uaddlp          v31.4s, v24.8h

    uaddw           v28.8h, v28.8h, v2.8b
    uaddw           v29.8h, v29.8h, v6.8b
    uadalp          v30.4s, v21.8h
    uadalp          v31.4s, v25.8h

    uaddw           v28.8h, v28.8h, v3.8b
    uaddw           v29.8h, v29.8h, v7.8b
    uadalp          v30.4s, v22.8h
    uadalp          v31.4s, v26.8h

    uaddlp          v28.4s, v28.8h
    uaddlp          v29.4s, v29.8h
    uadalp          v30.4s, v23.8h
    uadalp          v31.4s, v27.8h

    addp            v28.4s, v28.4s, v28.4s
    addp            v29.4s, v29.4s, v29.4s
    addp            v30.4s, v30.4s, v30.4s
    addp            v31.4s, v31.4s, v31.4s

    st4             {v28.2s, v29.2s, v30.2s, v31.2s}, [x4]
    ret
endfunc

// int psyCost_pp(const pixel* source, intptr_t sstride, const pixel* recon, intptr_t rstride)
function PFX(psyCost_4x4_neon)
    ld1r            {v4.2s}, [x0], x1
    ld1r            {v5.2s}, [x0], x1
    ld1             {v4.s}[1], [x0], x1
    ld1             {v5.s}[1], [x0], x1

    ld1r            {v6.2s}, [x2], x3
    ld1r            {v7.2s}, [x2], x3
    ld1             {v6.s}[1], [x2], x3
    ld1             {v7.s}[1], [x2], x3

    uaddl           v2.8h, v4.8b, v5.8b
    usubl           v3.8h, v4.8b, v5.8b
    uaddl           v18.8h, v6.8b, v7.8b
    usubl           v19.8h, v6.8b, v7.8b

    mov             v20.d[0], v2.d[1]
    add             v0.4h, v2.4h, v20.4h
    sub             v1.4h, v2.4h, v20.4h
    mov             v21.d[0], v3.d[1]
    add             v22.4h, v3.4h, v21.4h
    sub             v23.4h, v3.4h, v21.4h

    mov             v24.d[0], v18.d[1]
    add             v16.4h, v18.4h, v24.4h
    sub             v17.4h, v18.4h, v24.4h
    mov             v25.d[0], v19.d[1]
    add             v26.4h, v19.4h, v25.4h
    sub             v27.4h, v19.4h, v25.4h

    mov             v0.d[1], v22.d[0]
    mov             v1.d[1], v23.d[0]
    trn1            v22.8h, v0.8h, v1.8h
    trn2            v23.8h, v0.8h, v1.8h
    mov             v16.d[1], v26.d[0]
    mov             v17.d[1], v27.d[0]
    trn1            v26.8h, v16.8h, v17.8h
    trn2            v27.8h, v16.8h, v17.8h

    add             v2.8h, v22.8h, v23.8h
    sub             v3.8h, v22.8h, v23.8h
    add             v18.8h, v26.8h, v27.8h
    sub             v19.8h, v26.8h, v27.8h

    uaddl           v20.8h, v4.8b, v5.8b
    uaddl           v21.8h, v6.8b, v7.8b

    trn1            v0.4s, v2.4s, v3.4s
    trn2            v1.4s, v2.4s, v3.4s
    trn1            v16.4s, v18.4s, v19.4s
    trn2            v17.4s, v18.4s, v19.4s
    abs             v0.8h, v0.8h
    abs             v16.8h, v16.8h
    abs             v1.8h, v1.8h
    abs             v17.8h, v17.8h

    uaddlv          s20, v20.8h
    uaddlv          s21, v21.8h
    mov             v20.s[1], v21.s[0]

    smax            v0.8h, v0.8h, v1.8h
    smax            v16.8h, v16.8h, v17.8h

    trn1            v4.2d, v0.2d, v16.2d
    trn2            v5.2d, v0.2d, v16.2d
    add             v0.8h, v4.8h, v5.8h
    mov             v4.d[0], v0.d[1]
    uaddlv          s0, v0.4h
    uaddlv          s4, v4.4h

    ushr            v20.2s, v20.2s, #2
    mov             v0.s[1], v4.s[0]
    sub             v0.2s, v0.2s, v20.2s
    mov             w0, v0.s[0]
    mov             w1, v0.s[1]
    subs            w0, w0, w1
    cneg            w0, w0, mi

    ret
endfunc

// uint32_t quant_c(const int16_t* coef, const int32_t* quantCoeff, int32_t* deltaU, int16_t* qCoef, int qBits, int add, int numCoeff)
function PFX(quant_neon)
    mov             w9, #1
    lsl             w9, w9, w4
    dup             v0.2s, w9
    neg             w9, w4
    dup             v1.4s, w9
    add             w9, w9, #8
    dup             v2.4s, w9
    dup             v3.4s, w5

    lsr             w6, w6, #2
    eor             v4.16b, v4.16b, v4.16b
    eor             w10, w10, w10
    eor             v17.16b, v17.16b, v17.16b

.loop_quant:

    ld1             {v18.4h}, [x0], #8
    ld1             {v7.4s}, [x1], #16
    sxtl            v6.4s, v18.4h

    cmlt            v5.4s, v6.4s, #0

    abs             v6.4s, v6.4s


    mul             v6.4s, v6.4s, v7.4s

    add             v7.4s, v6.4s, v3.4s
    sshl            v7.4s, v7.4s, v1.4s

    mls             v6.4s, v7.4s, v0.s[0]
    sshl            v16.4s, v6.4s, v2.4s
    st1             {v16.4s}, [x2], #16

    // numsig
    cmeq            v16.4s, v7.4s, v17.4s
    add             v4.4s, v4.4s, v16.4s
    add             w10, w10, #4

    // level *= sign
    eor             v16.16b, v7.16b, v5.16b
    sub             v16.4s, v16.4s, v5.4s
    sqxtn           v5.4h, v16.4s
    st1             {v5.4h}, [x3], #8

    subs            w6, w6, #1
    b.ne             .loop_quant

    addv            s4, v4.4s
    mov             w9, v4.s[0]
    add             w0, w10, w9
    ret
endfunc

// uint32_t nquant_c(const int16_t* coef, const int32_t* quantCoeff, int16_t* qCoef, int qBits, int add, int numCoeff)
function PFX(nquant_neon)
    neg             x12, x3
    dup             v0.4s, w12             // q0= -qbits
    dup             v1.4s, w4              // add

    lsr             w5, w5, #2
    movi            v4.4s, #0              // v4= accumulate numsig
    mov             x4, #0
    movi            v22.4s, #0

.loop_nquant:
    ld1             {v16.4h}, [x0], #8
    sub             w5, w5, #1
    sxtl            v19.4s, v16.4h         // v19 = coef[blockpos]

    cmlt            v18.4s, v19.4s, #0     // v18 = sign

    abs             v19.4s, v19.4s         // v19 = level=abs(coef[blockpos])
    ld1             {v20.4s}, [x1], #16    // v20 = quantCoeff[blockpos]
    mul             v19.4s, v19.4s, v20.4s // v19 = tmplevel = abs(level) * quantCoeff[blockpos];

    add             v20.4s, v19.4s, v1.4s  // v20 = tmplevel+add
    sshl            v20.4s, v20.4s, v0.4s  // v20 = level =(tmplevel+add) >> qbits

    // numsig
    cmeq            v21.4s, v20.4s, v22.4s
    add             v4.4s, v4.4s, v21.4s
    add             x4, x4, #4

    eor             v21.16b, v20.16b, v18.16b
    sub             v21.4s, v21.4s, v18.4s
    sqxtn           v16.4h, v21.4s
    abs             v17.4h, v16.4h
    st1             {v17.4h}, [x2], #8

    cbnz            w5, .loop_nquant

    uaddlv          d4, v4.4s
    fmov            x12, d4
    add             x0, x4, x12
    ret
endfunc

// void ssimDist_c(const pixel* fenc, uint32_t fStride, const pixel* recon, intptr_t rstride, uint64_t *ssBlock, int shift, uint64_t *ac_k)
.macro ssimDist_start
    movi            v0.16b, #0
    movi            v1.16b, #0
.endm

.macro ssimDist_1  v4 v5
    sub             v20.8h, \v4\().8h, \v5\().8h
    smull           v16.4s, \v4\().4h, \v4\().4h
    smull2          v17.4s, \v4\().8h, \v4\().8h
    smull           v18.4s, v20.4h, v20.4h
    smull2          v19.4s, v20.8h, v20.8h
    add             v0.4s, v0.4s, v16.4s
    add             v0.4s, v0.4s, v17.4s
    add             v1.4s, v1.4s, v18.4s
    add             v1.4s, v1.4s, v19.4s
.endm

.macro ssimDist_end
    uaddlv          d0, v0.4s
    uaddlv          d1, v1.4s
    str             d0, [x6]
    str             d1, [x4]
.endm

function PFX(ssimDist4_neon)
    ssimDist_start
.rept 4
    ldr             s4, [x0]
    ldr             s5, [x2]
    add             x0, x0, x1
    add             x2, x2, x3
    uxtl            v4.8h, v4.8b
    uxtl            v5.8h, v5.8b
    sub             v2.4h, v4.4h, v5.4h
    smull           v3.4s, v4.4h, v4.4h
    smull           v2.4s, v2.4h, v2.4h
    add             v0.4s, v0.4s, v3.4s
    add             v1.4s, v1.4s, v2.4s
.endr
    ssimDist_end
    ret
endfunc

function PFX(ssimDist8_neon)
    ssimDist_start
.rept 8
    ldr             d4, [x0]
    ldr             d5, [x2]
    add             x0, x0, x1
    add             x2, x2, x3
    uxtl            v4.8h, v4.8b
    uxtl            v5.8h, v5.8b
    ssimDist_1      v4, v5
.endr
    ssimDist_end
    ret
endfunc

function PFX(ssimDist16_neon)
    mov w12, #16
    ssimDist_start
.loop_ssimDist16:
    sub             w12, w12, #1
    ldr             q4, [x0]
    ldr             q5, [x2]
    add             x0, x0, x1
    add             x2, x2, x3
    uxtl            v6.8h, v4.8b
    uxtl            v7.8h, v5.8b
    uxtl2           v4.8h, v4.16b
    uxtl2           v5.8h, v5.16b
    ssimDist_1      v6, v7
    ssimDist_1      v4, v5
    cbnz            w12, .loop_ssimDist16
    ssimDist_end
    ret
endfunc

function PFX(ssimDist32_neon)
    mov w12, #32
    ssimDist_start
.loop_ssimDist32:
    sub             w12, w12, #1
    ldp             q4, q5, [x0]
    ldp             q6, q7, [x2]
    add             x0, x0, x1
    add             x2, x2, x3
    uxtl            v21.8h, v4.8b
    uxtl            v22.8h, v6.8b
    uxtl            v23.8h, v5.8b
    uxtl            v24.8h, v7.8b
    uxtl2           v25.8h, v4.16b
    uxtl2           v26.8h, v6.16b
    uxtl2           v27.8h, v5.16b
    uxtl2           v28.8h, v7.16b
    ssimDist_1      v21, v22
    ssimDist_1      v23, v24
    ssimDist_1      v25, v26
    ssimDist_1      v27, v28
    cbnz            w12, .loop_ssimDist32
    ssimDist_end
    ret
endfunc

function PFX(ssimDist64_neon)
    mov w12, #64
    ssimDist_start
.loop_ssimDist64:
    sub             w12, w12, #1
    ldp             q4, q5, [x0]
    ldp             q6, q7, [x2]
    uxtl            v21.8h, v4.8b
    uxtl            v22.8h, v6.8b
    uxtl            v23.8h, v5.8b
    uxtl            v24.8h, v7.8b
    uxtl2           v25.8h, v4.16b
    uxtl2           v26.8h, v6.16b
    uxtl2           v27.8h, v5.16b
    uxtl2           v28.8h, v7.16b
    ssimDist_1      v21, v22
    ssimDist_1      v23, v24
    ssimDist_1      v25, v26
    ssimDist_1      v27, v28
    ldp             q4, q5, [x0, #32]
    ldp             q6, q7, [x2, #32]
    add             x0, x0, x1
    add             x2, x2, x3
    uxtl            v21.8h, v4.8b
    uxtl            v22.8h, v6.8b
    uxtl            v23.8h, v5.8b
    uxtl            v24.8h, v7.8b
    uxtl2           v25.8h, v4.16b
    uxtl2           v26.8h, v6.16b
    uxtl2           v27.8h, v5.16b
    uxtl2           v28.8h, v7.16b
    ssimDist_1      v21, v22
    ssimDist_1      v23, v24
    ssimDist_1      v25, v26
    ssimDist_1      v27, v28
    cbnz            w12, .loop_ssimDist64
    ssimDist_end
    ret
endfunc

// void normFact_c(const pixel* src, uint32_t blockSize, int shift, uint64_t *z_k)

.macro normFact_start
    movi            v0.16b, #0
.endm

.macro normFact_1  v4
    smull           v16.4s, \v4\().4h, \v4\().4h
    smull2          v17.4s, \v4\().8h, \v4\().8h
    add             v0.4s, v0.4s, v16.4s
    add             v0.4s, v0.4s, v17.4s
.endm

.macro normFact_end
    uaddlv          d0, v0.4s
    str             d0, [x3]
.endm


function PFX(normFact8_neon)
    normFact_start
.rept 8
    ldr             d4, [x0]
    add             x0, x0, x1
    uxtl            v4.8h, v4.8b
    normFact_1      v4
.endr
    normFact_end
    ret
endfunc

function PFX(normFact16_neon)
    mov w12, #16
    normFact_start
.loop_normFact16:
    sub             w12, w12, #1
    ldr             q4, [x0]
    add             x0, x0, x1
    uxtl            v5.8h, v4.8b
    uxtl2           v4.8h, v4.16b
    normFact_1      v5
    normFact_1      v4
    cbnz            w12, .loop_normFact16
    normFact_end
    ret
endfunc

function PFX(normFact32_neon)
    mov w12, #32
    normFact_start
.loop_normFact32:
    sub             w12, w12, #1
    ldp             q4, q5, [x0]
    add             x0, x0, x1
    uxtl            v6.8h, v4.8b
    uxtl2           v4.8h, v4.16b
    uxtl            v7.8h, v5.8b
    uxtl2           v5.8h, v5.16b
    normFact_1      v4
    normFact_1      v5
    normFact_1      v6
    normFact_1      v7
    cbnz            w12, .loop_normFact32
    normFact_end
    ret
endfunc

function PFX(normFact64_neon)
    mov w12, #64
    normFact_start
.loop_normFact64:
    sub             w12, w12, #1
    ldp             q4, q5, [x0]
    uxtl            v6.8h, v4.8b
    uxtl2           v4.8h, v4.16b
    uxtl            v7.8h, v5.8b
    uxtl2           v5.8h, v5.16b
    normFact_1      v4
    normFact_1      v5
    normFact_1      v6
    normFact_1      v7
    ldp             q4, q5, [x0, #32]
    add             x0, x0, x1
    uxtl            v6.8h, v4.8b
    uxtl2           v4.8h, v4.16b
    uxtl            v7.8h, v5.8b
    uxtl2           v5.8h, v5.16b
    normFact_1      v4
    normFact_1      v5
    normFact_1      v6
    normFact_1      v7
    cbnz            w12, .loop_normFact64
    normFact_end
    ret
endfunc

// void weight_pp_c(const pixel* src, pixel* dst, intptr_t stride, int width, int height, int w0, int round, int shift, int offset)
function PFX(weight_pp_neon)
    sub             x2, x2, x3
    ldr             w9, [sp]              // offset
    lsl             w5, w5, #6            // w0 << correction

    // count trailing zeros in w5 and compare against shift right amount.
    rbit            w10, w5
    clz             w10, w10
    cmp             w10, w7
    b.lt            .unfoldedShift

    // shift right only removes trailing zeros: hoist LSR out of the loop.
    lsr             w10, w5, w7           // w0 << correction >> shift
    dup             v25.16b, w10
    lsr             w6, w6, w7            // round >> shift
    add             w6, w6, w9            // round >> shift + offset
    dup             v26.8h, w6

    // Check arithmetic range.
    mov             w11, #255
    madd            w11, w11, w10, w6
    add             w11, w11, w9
    lsr             w11, w11, #16
    cbnz            w11, .widenTo32Bit

    // 16-bit arithmetic is enough.
.loopHpp:
    mov             x12, x3
.loopWpp:
    ldr             q0, [x0], #16
    sub             x12, x12, #16
    umull           v1.8h, v0.8b, v25.8b  // val *= w0 << correction >> shift
    umull2          v2.8h, v0.16b, v25.16b
    add             v1.8h, v1.8h, v26.8h  // val += round >> shift + offset
    add             v2.8h, v2.8h, v26.8h
    sqxtun          v0.8b, v1.8h          // val = x265_clip(val)
    sqxtun2         v0.16b, v2.8h
    str             q0, [x1], #16
    cbnz            x12, .loopWpp
    add             x1, x1, x2
    add             x0, x0, x2
    sub             x4, x4, #1
    cbnz            x4, .loopHpp
    ret

    // 32-bit arithmetic is needed.
.widenTo32Bit:
.loopHpp32:
    mov             x12, x3
.loopWpp32:
    ldr             d0, [x0], #8
    sub             x12, x12, #8
    uxtl            v0.8h, v0.8b
    umull           v1.4s, v0.4h, v25.4h  // val *= w0 << correction >> shift
    umull2          v2.4s, v0.8h, v25.8h
    add             v1.4s, v1.4s, v26.4s  // val += round >> shift + offset
    add             v2.4s, v2.4s, v26.4s
    sqxtn           v0.4h, v1.4s          // val = x265_clip(val)
    sqxtn2          v0.8h, v2.4s
    sqxtun          v0.8b, v0.8h
    str             d0, [x1], #8
    cbnz            x12, .loopWpp32
    add             x1, x1, x2
    add             x0, x0, x2
    sub             x4, x4, #1
    cbnz            x4, .loopHpp32
    ret

    // The shift right cannot be moved out of the loop.
.unfoldedShift:
    dup             v25.8h, w5            // w0 << correction
    dup             v26.4s, w6            // round
    neg             w7, w7                // -shift
    dup             v27.4s, w7
    dup             v29.4s, w9            // offset
.loopHppUS:
    mov             x12, x3
.loopWppUS:
    ldr             d0, [x0], #8
    sub             x12, x12, #8
    uxtl            v0.8h, v0.8b
    umull           v1.4s, v0.4h, v25.4h  // val *= w0
    umull2          v2.4s, v0.8h, v25.8h
    add             v1.4s, v1.4s, v26.4s  // val += round
    add             v2.4s, v2.4s, v26.4s
    sshl            v1.4s, v1.4s, v27.4s  // val >>= shift
    sshl            v2.4s, v2.4s, v27.4s
    add             v1.4s, v1.4s, v29.4s  // val += offset
    add             v2.4s, v2.4s, v29.4s
    sqxtn           v0.4h, v1.4s          // val = x265_clip(val)
    sqxtn2          v0.8h, v2.4s
    sqxtun          v0.8b, v0.8h
    str             d0, [x1], #8
    cbnz            x12, .loopWppUS
    add             x1, x1, x2
    add             x0, x0, x2
    sub             x4, x4, #1
    cbnz            x4, .loopHppUS
    ret
endfunc

// int scanPosLast(
//     const uint16_t *scan,      // x0
//     const coeff_t *coeff,      // x1
//     uint16_t *coeffSign,       // x2
//     uint16_t *coeffFlag,       // x3
//     uint8_t *coeffNum,         // x4
//     int numSig,                // x5
//     const uint16_t* scanCG4x4, // x6
//     const int trSize)          // x7
function PFX(scanPosLast_neon)
    // convert unit of Stride(trSize) to int16_t
    add             x7, x7, x7

    // load scan table and convert to Byte
    ldp             q0, q1, [x6]
    xtn             v0.8b, v0.8h
    xtn2            v0.16b, v1.8h   // v0 - Zigzag scan table

    movrel          x10, g_SPL_and_mask
    ldr             q28, [x10]      // v28 = mask for pmovmskb
    movi            v31.16b, #0     // v31 = {0, ..., 0}
    add             x10, x7, x7     // 2*x7
    add             x11, x10, x7    // 3*x7
    add             x9, x4, #1      // CG count

.loop_spl:
    // position of current CG
    ldrh            w6, [x0], #32
    add             x6, x1, x6, lsl #1

    // loading current CG
    ldr             d2, [x6]
    ldr             d3, [x6, x7]
    ldr             d4, [x6, x10]
    ldr             d5, [x6, x11]
    mov             v2.d[1], v3.d[0]
    mov             v4.d[1], v5.d[0]
    sqxtn           v2.8b, v2.8h
    sqxtn2          v2.16b, v4.8h

    // Zigzag
    tbl             v3.16b, {v2.16b}, v0.16b

    // get sign
    cmhi            v5.16b, v3.16b, v31.16b   // v5 = non-zero
    cmlt            v3.16b, v3.16b, #0        // v3 = negative

    // val - w13 = pmovmskb(v3)
    and             v3.16b, v3.16b, v28.16b
    mov             d4, v3.d[1]
    addv            b13, v3.8b
    addv            b14, v4.8b
    fmov            w13, s13
    fmov            w14, s14
    orr             w13, w13, w14, lsl #8

    // mask - w15 = pmovmskb(v5)
    and             v5.16b, v5.16b, v28.16b
    mov             d6, v5.d[1]
    addv            b15, v5.8b
    addv            b16, v6.8b
    fmov            w15, s15
    fmov            w14, s16
    orr             w15, w15, w14, lsl #8

    // coeffFlag = reverse_bit(w15) in 16-bit
    rbit            w12, w15
    lsr             w12, w12, #16
    fmov            s30, w12
    strh            w12, [x3], #2

    // accelerate by preparing w13 = w13 & w15
    and             w13, w13, w15
    mov             x14, xzr
.loop_spl_1:
    cbz             w15, .pext_end
    clz             w6, w15
    lsl             w13, w13, w6
    lsl             w15, w15, w6
    extr            w14, w14, w13, #31
    bfm             w15, wzr, #1, #0
    b               .loop_spl_1
.pext_end:
    strh            w14, [x2], #2

    // compute coeffNum = popcount(coeffFlag)
    cnt             v30.8b, v30.8b
    addp            v30.8b, v30.8b, v30.8b
    fmov            w6, s30
    sub             x5, x5, x6
    strb            w6, [x4], #1

    cbnz            x5, .loop_spl

    // count trailing zeros
    rbit            w13, w12
    clz             w13, w13
    lsr             w12, w12, w13
    strh            w12, [x3, #-2]

    // get last pos
    sub             x9, x4, x9
    lsl             x0, x9, #4
    eor             w13, w13, #15
    add             x0, x0, x13
    ret
endfunc


const g_SPL_and_mask, align=8
.byte 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
endconst
