mirror of
https://github.com/FFmpeg/FFmpeg.git
synced 2024-11-26 19:01:44 +02:00
38cd829dce
Currently it is done in several different ways, which might cause needless dependencies or in case of tx_float_neon.S is incorrect. Reviewed-by: Martin Storsjö <martin@martin.st> Signed-off-by: Reimar Döffinger <Reimar.Doeffinger@gmx.de>
1295 lines
49 KiB
ArmAsm
1295 lines
49 KiB
ArmAsm
/*
|
|
* Copyright (c) Lynne
|
|
*
|
|
* This file is part of FFmpeg.
|
|
*
|
|
* FFmpeg is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
*
|
|
* FFmpeg is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
* License along with FFmpeg; if not, write to the Free Software
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
*/
|
|
|
|
#include "libavutil/aarch64/asm.S"
|
|
|
|
/* Open `doc/transforms.md` to see the code upon which the transforms here were
|
|
* based upon.
|
|
*
|
|
* File conventions:
|
|
* GPRs: x0-x3 - arguments, untouched
|
|
* x4 - Lookup table base pointer
|
|
* x5-x6 - macro ld1 temps/function scratch
|
|
* x7-x9 - FFT table state
|
|
* x10-x17 - lookup table/macro scratch
|
|
* w19-w20 - current/target length when needed
|
|
* x21-x22 - len*2, len*6
|
|
*
|
|
* Vectors: v0-v7 - coefficients
|
|
* v8-v15 - coefficients when needed, otherwise untouched
|
|
* v16-v30 - used as needed
|
|
* v31 - -1.0, +1.0, -1.0, +1.0. Never touched after loading.
|
|
*
|
|
* Stack: backup for v8-v15 and x19-x22 when needed, and transform lengths
|
|
*/
|
|
|
|
#define M_SQRT1_2 0.707106781186547524401
|
|
#define COS16_1 0.92387950420379638671875
|
|
#define COS16_3 0.3826834261417388916015625
|
|
|
|
/* We only ever load this once at the start, and then live with losing an
|
|
* entire register as we need to lug this all the time everywhere.
|
|
* Clearly should be integrated into an fsadd and fmlsa, but "muh RISC!". */
|
|
const subadd, align=4
|
|
.float -1.0, 1.0, -1.0, 1.0
|
|
endconst
|
|
|
|
.macro LOAD_SUBADD
|
|
movrel x5, subadd
|
|
ld1 { v31.4s }, [x5]
|
|
.endm
|
|
|
|
.macro SETUP_LUT no_lut=0
|
|
.if \no_lut == 0
|
|
ldr x4, [x0, #8]
|
|
.endif
|
|
.endm
|
|
|
|
.macro LOAD_INPUT dst1, dst2, dst3, dst4, src, no_lut=0, discont=0
|
|
.if \no_lut == 1
|
|
.if \discont == 1
|
|
ldp q\dst1\(), q\dst2\(), [\src\()]
|
|
ldp q\dst3\(), q\dst4\(), [\src\(), #32]
|
|
add \src\(), \src\(), #64
|
|
.else
|
|
ld1 { v\dst1\().4s, v\dst2\().4s, v\dst3\().4s, v\dst4\().4s }, [\src], #64
|
|
.endif
|
|
.else
|
|
ldp w10, w11, [x4, #0 ]
|
|
ldp w12, w13, [x4, #8 ]
|
|
ldp w14, w15, [x4, #16]
|
|
ldp w16, w17, [x4, #24]
|
|
|
|
add x4, x4, #32
|
|
|
|
ldr d\dst1, [\src, x10, lsl #3]
|
|
add x11, \src, x11, lsl #3
|
|
ldr d\dst2, [\src, x12, lsl #3]
|
|
add x13, \src, x13, lsl #3
|
|
ldr d\dst3, [\src, x14, lsl #3]
|
|
add x15, \src, x15, lsl #3
|
|
ldr d\dst4, [\src, x16, lsl #3]
|
|
add x17, \src, x17, lsl #3
|
|
|
|
ld1 { v\dst1\().d }[1], [x11]
|
|
ld1 { v\dst2\().d }[1], [x13]
|
|
ld1 { v\dst3\().d }[1], [x15]
|
|
ld1 { v\dst4\().d }[1], [x17]
|
|
.endif
|
|
.endm
|
|
|
|
.macro FFT4 e0, o0, standalone
|
|
fadd v16.4s, \e0\().4s, \o0\().4s // r1..4
|
|
fsub \e0\().4s, \e0\().4s, \o0\().4s // t1..4
|
|
|
|
rev64 v18.4s, \e0\().4s
|
|
|
|
zip2 \o0\().2d, v16.2d, \e0\().2d
|
|
zip1 v17.2d, v16.2d, \e0\().2d
|
|
|
|
mov \o0\().d[1], v18.d[1]
|
|
|
|
fadd \e0\().4s, v17.4s, \o0\().4s // a1,2 b1,4
|
|
fsub v16.4s, v17.4s, \o0\().4s // a3,4 b3,2
|
|
|
|
mov \o0\().16b, v16.16b // Swap once again...
|
|
mov \o0\().s[3], \e0\().s[3]
|
|
mov \e0\().s[3], v16.s[3]
|
|
|
|
.if \standalone == 0
|
|
uzp2 \o0\().2d, \e0\().2d, \o0\().2d
|
|
uzp1 \e0\().2d, \e0\().2d, v16.2d
|
|
.endif
|
|
.endm
|
|
|
|
const shuf_4pt_x2, align=4
|
|
.byte 24, 25, 26, 27 // reg2, 3
|
|
.byte 12, 13, 14, 15 // reg1, 4
|
|
.byte 8, 9, 10, 11 // reg1, 3
|
|
.byte 28, 29, 30, 31 // reg2, 4
|
|
endconst
|
|
|
|
// Identical to FFT4, but does 2 transforms in parallel, with no deinterleaving
|
|
.macro FFT4_X2 e0, o0, e1, o1, \
|
|
t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22
|
|
|
|
fadd \t0\().4s, \e0\().4s, \o0\().4s // r1234
|
|
fadd \t2\().4s, \e1\().4s, \o1\().4s // r1234
|
|
fsub \e0\().4s, \e0\().4s, \o0\().4s // t1234
|
|
fsub \e1\().4s, \e1\().4s, \o1\().4s // t1234
|
|
|
|
movrel x5, shuf_4pt_x2
|
|
|
|
rev64 \t4\().4s, \e0\().4s
|
|
rev64 \t5\().4s, \e1\().4s
|
|
|
|
zip2 \o0\().2d, \t0\().2d, \e0\().2d // t3,4 r3,4
|
|
zip2 \o1\().2d, \t2\().2d, \e1\().2d // t3,4 r3,4
|
|
|
|
ld1 { \t6\().16b }, [x5]
|
|
|
|
mov \o0\().d[1], \t4\().d[1]
|
|
mov \o1\().d[1], \t5\().d[1]
|
|
|
|
zip1 \t1\().2d, \t0\().2d, \e0\().2d // t1,2 r1,2
|
|
zip1 \t3\().2d, \t2\().2d, \e1\().2d // t1,2 r1,2
|
|
|
|
fsub \t4\().4s, \t1\().4s, \o0\().4s // a34 b32
|
|
fadd \t5\().4s, \t1\().4s, \o0\().4s // a12 b14
|
|
fsub \t2\().4s, \t3\().4s, \o1\().4s // a34 b32
|
|
fadd \t3\().4s, \t3\().4s, \o1\().4s // a12 b14
|
|
|
|
// TODO: experiment with movs instead of tables here
|
|
tbl \o0\().16b, { \t4\().16b, \t5\().16b }, \t6\().16b // b1234
|
|
tbl \o1\().16b, { \t2\().16b, \t3\().16b }, \t6\().16b // b1234
|
|
|
|
zip1 \e0\().2d, \t5\().2d, \t4\().2d // a1234
|
|
// zip2 \o0\().2d, \t5\().2d, \t4\().2d // b1432
|
|
zip1 \e1\().2d, \t3\().2d, \t2\().2d // a1234
|
|
// zip2 \o1\().2d, \t3\().2d, \t2\().2d // b1432
|
|
// rev64 \o0\().4s, \o0\().4s // b4123
|
|
// rev64 \o1\().4s, \o1\().4s // b4123
|
|
// ext \o0\().16b, \o0\().16b, \o0\().16b, #4 // b1234
|
|
// ext \o1\().16b, \o1\().16b, \o1\().16b, #4 // b1234
|
|
.endm
|
|
|
|
const tab_8pt, align=4
|
|
.float M_SQRT1_2, -M_SQRT1_2, -M_SQRT1_2, M_SQRT1_2
|
|
endconst
|
|
|
|
.macro FFT8 e0, e1, o0, o1, \
|
|
t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22
|
|
|
|
movrel x5, tab_8pt
|
|
|
|
fsub \t1\().4s, \e1\().4s, \o1\().4s // j1234
|
|
fadd \o1\().4s, \e1\().4s, \o1\().4s // k1234
|
|
fsub \t0\().4s, \e0\().4s, \o0\().4s // r1234
|
|
fadd \o0\().4s, \e0\().4s, \o0\().4s // q1234
|
|
|
|
ld1 { \t5\().4s }, [x5]
|
|
|
|
ext \t4\().16b, \o1\().16b, \o1\().16b, #12
|
|
rev64 \t4\().4s, \t4\().4s
|
|
|
|
ext \t2\().16b, \o0\().16b, \t4\().16b, #8 // o0[0,1], o1[3,2]
|
|
mov \o0\().d[1], \t4\().d[1] // o0[3, 4]; o1[1, 4]
|
|
|
|
fsub \e1\().4s, \o0\().4s, \t2\().4s // s34, g43
|
|
fadd \t2\().4s, \o0\().4s, \t2\().4s // s12, g12
|
|
|
|
rev64 \t6\().4s, v31.4s // 1, -1, 1, -1
|
|
dup \o0\().2d, \t0\().d[0] // r1212
|
|
dup \o1\().2d, \t0\().d[1] // r3434
|
|
|
|
rev64 \t4\().4s, \e1\().4s // xxg34
|
|
rev64 \o1\().4s, \o1\().4s // r4343
|
|
|
|
ext \t6\().16b, v31.16b, \t6\().16b, #8 // -1, 1, 1, -1
|
|
zip1 \t3\().2d, \t2\().2d, \e1\().2d // s1234
|
|
zip2 \t2\().2d, \t2\().2d, \t4\().2d // g1234
|
|
|
|
fadd \e0\().4s, \t3\().4s, \t2\().4s // out_e1
|
|
fsub \e1\().4s, \t3\().4s, \t2\().4s // out_e2
|
|
|
|
fmul \t1\().4s, \t1\().4s, \t5\().4s // j * +--+M_SQRT1_2
|
|
fmls \o0\().4s, \o1\().4s, \t6\().4s // z1234
|
|
|
|
rev64 \t4\().4s, \t1\().4s // j2143
|
|
fmla \t1\().4s, \t4\().4s, v31.4s // l2143
|
|
|
|
rev64 \t4\().4s, \t1\().4s // l1234
|
|
ext \t4\().16b, \t4\().16b, \t4\().16b, #8 // l3412
|
|
|
|
fmla \t4\().4s, \t1\().4s, v31.4s // t1234
|
|
|
|
fadd \o1\().4s, \o0\().4s, \t4\().4s // out_o2
|
|
fsub \o0\().4s, \o0\().4s, \t4\().4s // out_o1
|
|
.endm
|
|
|
|
// Identical as FFT8, but does 2 transforms in parallel
|
|
.macro FFT8_X2 e0, e1, o0, o1, e2, e3, o2, o3
|
|
|
|
movrel x5, tab_8pt
|
|
|
|
fadd v19.4s, \e3\().4s, \o3\().4s // k1234
|
|
fadd v17.4s, \e1\().4s, \o1\().4s // k1234
|
|
fadd v18.4s, \e2\().4s, \o2\().4s // q1234
|
|
fadd v16.4s, \e0\().4s, \o0\().4s // q1234
|
|
|
|
ld1 { v23.4s }, [x5]
|
|
|
|
ext v22.16b, v19.16b, v19.16b, #12
|
|
ext v21.16b, v17.16b, v17.16b, #12
|
|
|
|
rev64 v22.4s, v22.4s
|
|
rev64 v21.4s, v21.4s
|
|
|
|
ext v19.16b, v18.16b, v22.16b, #8
|
|
ext v17.16b, v16.16b, v21.16b, #8
|
|
|
|
mov v18.d[1], v22.d[1]
|
|
mov v21.d[0], v16.d[0]
|
|
|
|
fadd v22.4s, v18.4s, v19.4s // s12, g12
|
|
fsub v19.4s, v18.4s, v19.4s // s34, g43
|
|
fsub v18.4s, v21.4s, v17.4s // s34, g43
|
|
fadd v16.4s, v21.4s, v17.4s // s12, g12
|
|
|
|
fsub \e0\().4s, \e0\().4s, \o0\().4s // r1234
|
|
fsub v20.4s, \e1\().4s, \o1\().4s // j1234
|
|
fsub \e2\().4s, \e2\().4s, \o2\().4s // r1234
|
|
fsub v21.4s, \e3\().4s, \o3\().4s // j1234
|
|
|
|
rev64 v24.4s, v31.4s // 1, -1, 1, -1
|
|
zip1 v17.2d, v16.2d, v18.2d // s1234
|
|
zip1 \e1\().2d, v22.2d, v19.2d // s1234
|
|
|
|
rev64 v18.4s, v18.4s // xxg34
|
|
rev64 v19.4s, v19.4s // xxg34
|
|
|
|
zip2 v16.2d, v16.2d, v18.2d // g1234
|
|
zip2 \e3\().2d, v22.2d, v19.2d // g1234
|
|
|
|
dup \o0\().2d, \e0\().d[0] // r1212
|
|
dup \o1\().2d, \e0\().d[1] // r3434
|
|
dup \o2\().2d, \e2\().d[0] // r1212
|
|
dup \o3\().2d, \e2\().d[1] // r3434
|
|
|
|
fadd \e2\().4s, \e1\().4s, \e3\().4s // out_e1
|
|
fsub \e3\().4s, \e1\().4s, \e3\().4s // out_e2
|
|
fadd \e0\().4s, v17.4s, v16.4s // out_e1
|
|
fsub \e1\().4s, v17.4s, v16.4s // out_e2
|
|
|
|
ext v24.16b, v31.16b, v24.16b, #8 // -1, 1, 1, -1
|
|
rev64 \o1\().4s, \o1\().4s // r4343
|
|
rev64 \o3\().4s, \o3\().4s // r4343
|
|
|
|
fmul v19.4s, v20.4s, v23.4s // j * +--+M_SQRT1_2
|
|
fmul v21.4s, v21.4s, v23.4s // j * +--+M_SQRT1_2
|
|
|
|
rev64 v20.4s, v19.4s // j2143
|
|
rev64 v18.4s, v21.4s // j2143
|
|
|
|
fmls \o0\().4s, \o1\().4s, v24.4s // z1234
|
|
fmls \o2\().4s, \o3\().4s, v24.4s // z1234
|
|
|
|
fmla v19.4s, v20.4s, v31.4s // l2143
|
|
fmla v21.4s, v18.4s, v31.4s // l2143
|
|
|
|
rev64 v20.4s, v19.4s // l1234
|
|
rev64 v18.4s, v21.4s // l1234
|
|
ext v20.16b, v20.16b, v20.16b, #8 // l3412
|
|
ext v18.16b, v18.16b, v18.16b, #8 // l3412
|
|
|
|
fmla v20.4s, v19.4s, v31.4s // t1234
|
|
fmla v18.4s, v21.4s, v31.4s // t1234
|
|
|
|
fadd \o1\().4s, \o0\().4s, v20.4s // out_o2
|
|
fadd \o3\().4s, \o2\().4s, v18.4s // out_o2
|
|
fsub \o0\().4s, \o0\().4s, v20.4s // out_o1
|
|
fsub \o2\().4s, \o2\().4s, v18.4s // out_o1
|
|
.endm
|
|
|
|
const tab_16pt, align=4
|
|
.float -COS16_1, COS16_1, -COS16_3, COS16_3 // Could be +-+- too
|
|
.float COS16_3, COS16_3, COS16_1, COS16_1
|
|
.float 1.0, 1.0, M_SQRT1_2, M_SQRT1_2
|
|
endconst
|
|
|
|
// 16-point FFT
|
|
// t3, t4, t5, t6 must be sequential
|
|
.macro FFT16 e0, e1, e2, e3, o0, o1, o2, o3, \
|
|
t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22
|
|
|
|
FFT8 \e0, \e1, \e2, \e3, \t0, \t1, \t2, \t3, \t4, \t5, \t6
|
|
FFT4_X2 \o0, \o1, \o2, \o3, \t0, \t1, \t2, \t3, \t4, \t5, \t6
|
|
|
|
movrel x5, tab_16pt
|
|
|
|
rev64 \t0\().4s, \o0\().4s // z[ 8, 9].imre
|
|
rev64 \t1\().4s, \o2\().4s // z[10,11].imre
|
|
|
|
ins \t0\().d[0], xzr
|
|
ins \t1\().d[0], xzr
|
|
|
|
ld1 { \t4\().4s, \t5\().4s, \t6\().4s }, [x5]
|
|
// TODO: We could derive \t4\() or \t5\() from either, but it seems cheaper to load
|
|
|
|
fmla \o2\().4s, \t1\().4s, v31.4s // s[4567]
|
|
fmls \o0\().4s, \t0\().4s, v31.4s // s[0123]
|
|
|
|
fmul \t2\().4s, \o1\().4s, \t4\().4s
|
|
fmul \t3\().4s, \o3\().4s, \t4\().4s
|
|
|
|
rev64 \o3\().4s, \o3\().4s
|
|
rev64 \o1\().4s, \o1\().4s
|
|
|
|
fmla \t3\().4s, \o3\().4s, \t5\().4s // s[12, 13, 14, 15]
|
|
fmls \t2\().4s, \o1\().4s, \t5\().4s // s[ 8, 9, 10, 11]
|
|
|
|
fmul \t1\().4s, \o2\().4s, \t6\().4s // s[4567] * mult
|
|
fmul \t0\().4s, \o0\().4s, \t6\().4s // s[0123] * mult
|
|
|
|
mov \o1\().16b, \t3\().16b
|
|
mov \o2\().16b, \t1\().16b
|
|
|
|
fsub \t3\().4s, \t3\().4s, \t2\().4s // y34, u34
|
|
fsub \t1\().4s, \t1\().4s, \t0\().4s // w34, x34
|
|
|
|
fadd \t2\().4s, \t2\().4s, \o1\().4s // y56, u56
|
|
rev64 \t3\().4s, \t3\().4s
|
|
fadd \t0\().4s, \t0\().4s, \o2\().4s // w56, x56
|
|
rev64 \t1\().4s, \t1\().4s
|
|
|
|
fmul \t2\().4s, \t2\().4s, v31.4s
|
|
fmul \t1\().4s, \t1\().4s, v31.4s
|
|
|
|
fadd \o3\().4s, \e3\().4s, \t3\().4s
|
|
fsub \o2\().4s, \e3\().4s, \t3\().4s
|
|
fsub \o1\().4s, \e2\().4s, \t2\().4s
|
|
fadd \o0\().4s, \e2\().4s, \t2\().4s
|
|
|
|
fsub \e2\().4s, \e0\().4s, \t0\().4s
|
|
fadd \e0\().4s, \e0\().4s, \t0\().4s
|
|
fsub \e3\().4s, \e1\().4s, \t1\().4s
|
|
fadd \e1\().4s, \e1\().4s, \t1\().4s
|
|
.endm
|
|
|
|
function ff_tx_fft2_float_neon, export=1
|
|
ld2r { v0.2d, v1.2d }, [x2]
|
|
|
|
fneg v2.2s, v1.2s
|
|
mov v2.d[1], v1.d[0]
|
|
|
|
fsub v2.4s, v0.4s, v2.4s
|
|
|
|
st1 { v2.4s }, [x1]
|
|
ret
|
|
endfunc
|
|
|
|
.macro FFT4_FN name, inv
|
|
function ff_tx_fft4_\name\()_float_neon, export=1
|
|
ld1 {v0.4s, v1.4s}, [x2]
|
|
|
|
.if \inv == 1
|
|
mov v2.d[0], v0.d[1]
|
|
mov v0.d[1], v1.d[1]
|
|
mov v1.d[1], v2.d[0]
|
|
.endif
|
|
|
|
FFT4 v0, v1, 1
|
|
|
|
st1 { v0.4s, v1.4s }, [x1]
|
|
ret
|
|
endfunc
|
|
.endm
|
|
|
|
FFT4_FN fwd, 0
|
|
FFT4_FN inv, 1
|
|
|
|
.macro FFT8_FN name, no_perm
|
|
function ff_tx_fft8_\name\()_neon, export=1
|
|
SETUP_LUT \no_perm
|
|
LOAD_INPUT 0, 1, 2, 3, x2, \no_perm
|
|
|
|
LOAD_SUBADD
|
|
FFT8 v0, v1, v2, v3
|
|
|
|
zip1 v16.2d, v0.2d, v2.2d
|
|
zip2 v17.2d, v0.2d, v2.2d
|
|
zip1 v18.2d, v1.2d, v3.2d
|
|
zip2 v19.2d, v1.2d, v3.2d
|
|
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1]
|
|
|
|
ret
|
|
endfunc
|
|
.endm
|
|
|
|
FFT8_FN float, 0
|
|
FFT8_FN ns_float, 1
|
|
|
|
.macro FFT16_FN name, no_perm
|
|
function ff_tx_fft16_\name\()_neon, export=1
|
|
SETUP_LUT \no_perm
|
|
LOAD_INPUT 0, 1, 2, 3, x2, \no_perm
|
|
LOAD_INPUT 4, 5, 6, 7, x2, \no_perm
|
|
|
|
LOAD_SUBADD
|
|
FFT16 v0, v1, v2, v3, v4, v5, v6, v7
|
|
|
|
zip1 v20.2d, v0.2d, v4.2d
|
|
zip2 v21.2d, v0.2d, v4.2d
|
|
zip1 v22.2d, v1.2d, v6.2d
|
|
zip2 v23.2d, v1.2d, v6.2d
|
|
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64
|
|
|
|
zip1 v24.2d, v2.2d, v5.2d
|
|
zip2 v25.2d, v2.2d, v5.2d
|
|
zip1 v26.2d, v3.2d, v7.2d
|
|
zip2 v27.2d, v3.2d, v7.2d
|
|
st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1]
|
|
|
|
ret
|
|
endfunc
|
|
.endm
|
|
|
|
FFT16_FN float, 0
|
|
FFT16_FN ns_float, 1
|
|
|
|
.macro SETUP_SR_RECOMB len, re, im, dec
|
|
ldr w5, =(\len - 4*7)
|
|
movrel \re, X(ff_tx_tab_\len\()_float)
|
|
add \im, \re, x5
|
|
mov \dec, #-32
|
|
|
|
.if \len > 32
|
|
mov x21, #2*\len
|
|
add x22, x21, x21, lsl #1
|
|
.endif
|
|
.endm
|
|
|
|
.macro SR_COMBINE e0, e1, e2, e3, e4, e5, e6, e7, \
|
|
o0, o1, o2, o3, o4, o5, o6, o7, \
|
|
re, im, dec, swap_im, \
|
|
t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, \
|
|
t6=v22, t7=v23, t8=v24, t9=v25, ta=v26, tb=v27
|
|
|
|
ld1 { \t8\().4s, \t9\().4s }, [\im], \dec
|
|
ld1 { \t0\().4s, \t1\().4s }, [\re], #32
|
|
|
|
.if \swap_im == 1
|
|
ext \t2\().16b, \t9\().16b, \t9\().16b, #8
|
|
ext \t3\().16b, \t8\().16b, \t8\().16b, #8
|
|
.else
|
|
ext \t2\().16b, \t8\().16b, \t8\().16b, #8
|
|
ext \t3\().16b, \t9\().16b, \t9\().16b, #8
|
|
.endif
|
|
|
|
trn1 \t4\().4s, \t0\().4s, \t0\().4s // cos0022
|
|
trn2 \t0\().4s, \t0\().4s, \t0\().4s // cos4466
|
|
trn1 \t5\().4s, \t1\().4s, \t1\().4s // cos1133
|
|
trn2 \t1\().4s, \t1\().4s, \t1\().4s // cos5577
|
|
|
|
rev64 \t6\().4s, \o0\().4s // E m2[0,1].imre
|
|
rev64 \t7\().4s, \o2\().4s // O m2[0,1].imre
|
|
rev64 \t8\().4s, \o4\().4s // E m2[2,3].imre
|
|
rev64 \t9\().4s, \o6\().4s // O m2[2,3].imre
|
|
|
|
fmul \t6\().4s, \t6\().4s, \t4\().4s // E m2[0,1].imre*t1[0,2]
|
|
fmul \t7\().4s, \t7\().4s, \t0\().4s // O m2[0,1].imre*t1[0,2]
|
|
fmul \t8\().4s, \t8\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2]
|
|
fmul \t9\().4s, \t9\().4s, \t0\().4s // O m2[2,3].imre*t1[0,2]
|
|
|
|
rev64 \ta\().4s, \o1\().4s // E m3[0,1].imre
|
|
rev64 \tb\().4s, \o3\().4s // O m3[0,1].imre
|
|
rev64 \t4\().4s, \o5\().4s // E m3[2,3].imre
|
|
rev64 \t0\().4s, \o7\().4s // O m3[2,3].imre
|
|
|
|
fmul \ta\().4s, \ta\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6]
|
|
fmul \tb\().4s, \tb\().4s, \t1\().4s // O m3[0,1].imre*t1[4,6]
|
|
fmul \t4\().4s, \t4\().4s, \t5\().4s // E m3[2,3].imre*t1[4,6]
|
|
fmul \t0\().4s, \t0\().4s, \t1\().4s // O m3[2,3].imre*t1[4,6]
|
|
|
|
trn1 \t5\().4s, \t3\().4s, \t3\().4s // wim2200
|
|
trn2 \t3\().4s, \t3\().4s, \t3\().4s // wim3311
|
|
trn1 \t1\().4s, \t2\().4s, \t2\().4s // wim6644
|
|
trn2 \t2\().4s, \t2\().4s, \t2\().4s // wim7755
|
|
|
|
fmul \t5\().4s, \t5\().4s, v31.4s
|
|
fmul \t3\().4s, \t3\().4s, v31.4s
|
|
fmul \t1\().4s, \t1\().4s, v31.4s
|
|
fmul \t2\().4s, \t2\().4s, v31.4s
|
|
|
|
fmla \t7\().4s, \o2\().4s, \t5\().4s // O w0123
|
|
fmls \t9\().4s, \o6\().4s, \t5\().4s // O j0123
|
|
fmla \t6\().4s, \o0\().4s, \t3\().4s // E w0123
|
|
fmls \t8\().4s, \o4\().4s, \t3\().4s // E j0123
|
|
|
|
fmla \ta\().4s, \o1\().4s, \t2\().4s // E w4567
|
|
fmla \tb\().4s, \o3\().4s, \t1\().4s // O w4567
|
|
fmls \t4\().4s, \o5\().4s, \t2\().4s // E j4567
|
|
fmls \t0\().4s, \o7\().4s, \t1\().4s // O j4567
|
|
|
|
fsub \t2\().4s, \t7\().4s, \t9\().4s
|
|
fsub \t1\().4s, \t8\().4s, \t6\().4s
|
|
fsub \t3\().4s, \t4\().4s, \ta\().4s
|
|
fsub \t5\().4s, \t0\().4s, \tb\().4s
|
|
|
|
fadd \t6\().4s, \t8\().4s, \t6\().4s
|
|
fadd \t7\().4s, \t9\().4s, \t7\().4s
|
|
fadd \t8\().4s, \t4\().4s, \ta\().4s
|
|
fadd \t9\().4s, \t0\().4s, \tb\().4s
|
|
|
|
fmul \t1\().4s, \t1\().4s, v31.4s
|
|
fmul \t2\().4s, \t2\().4s, v31.4s
|
|
fmul \t3\().4s, \t3\().4s, v31.4s
|
|
fmul \t5\().4s, \t5\().4s, v31.4s
|
|
|
|
rev64 \t6\().4s, \t6\().4s
|
|
rev64 \t8\().4s, \t8\().4s
|
|
rev64 \t7\().4s, \t7\().4s
|
|
rev64 \t9\().4s, \t9\().4s
|
|
|
|
fsub \o0\().4s, \e0\().4s, \t6\().4s
|
|
fsub \o1\().4s, \e1\().4s, \t8\().4s
|
|
fsub \o2\().4s, \e2\().4s, \t1\().4s
|
|
fsub \o3\().4s, \e3\().4s, \t3\().4s
|
|
|
|
fsub \o4\().4s, \e4\().4s, \t7\().4s
|
|
fsub \o5\().4s, \e6\().4s, \t9\().4s
|
|
fadd \o6\().4s, \e5\().4s, \t2\().4s
|
|
fsub \o7\().4s, \e7\().4s, \t5\().4s
|
|
|
|
fadd \e0\().4s, \e0\().4s, \t6\().4s
|
|
fadd \e1\().4s, \e1\().4s, \t8\().4s
|
|
fadd \e2\().4s, \e2\().4s, \t1\().4s
|
|
fadd \e3\().4s, \e3\().4s, \t3\().4s
|
|
|
|
fadd \e4\().4s, \e4\().4s, \t7\().4s
|
|
fsub \e5\().4s, \e5\().4s, \t2\().4s // swapped
|
|
fadd \e6\().4s, \e6\().4s, \t9\().4s // swapped
|
|
fadd \e7\().4s, \e7\().4s, \t5\().4s
|
|
.endm
|
|
|
|
.macro SR_COMBINE_HALF e0, e1, e2, e3, \
|
|
o0, o1, o2, o3, \
|
|
c0, c1, c2, c3, \
|
|
t0, t1, t2, t3, t4, t5, part
|
|
|
|
.if \part == 0
|
|
trn1 \t4\().4s, \c0\().4s, \c0\().4s // cos0022
|
|
trn1 \c1\().4s, \c1\().4s, \c1\().4s // cos1133
|
|
.else
|
|
trn2 \t4\().4s, \c0\().4s, \c0\().4s // cos0022
|
|
trn2 \c1\().4s, \c1\().4s, \c1\().4s // cos1133
|
|
.endif
|
|
.if \part == 0
|
|
trn2 \t5\().4s, \c2\().4s, \c2\().4s // wim7755
|
|
trn2 \c3\().4s, \c3\().4s, \c3\().4s // wim3311
|
|
.else
|
|
trn1 \t5\().4s, \c2\().4s, \c2\().4s // wim7755
|
|
trn1 \c3\().4s, \c3\().4s, \c3\().4s // wim3311
|
|
.endif
|
|
|
|
fmul \t5\().4s, \t5\().4s, v31.4s
|
|
fmul \c3\().4s, \c3\().4s, v31.4s
|
|
|
|
rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre
|
|
rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre
|
|
rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre
|
|
rev64 \t3\().4s, \o3\().4s // E m3[2,3].imre
|
|
|
|
fmul \o0\().4s, \o0\().4s, \c3\().4s // E m2[0,1].imre*t1[0,2]
|
|
fmul \o1\().4s, \o1\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6]
|
|
fmla \o0\().4s, \t0\().4s, \t4\().4s // E w0123
|
|
fmla \o1\().4s, \t2\().4s, \c1\().4s // E w4567
|
|
|
|
fmul \t1\().4s, \t1\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2]
|
|
fmul \t3\().4s, \t3\().4s, \c1\().4s // E m3[2,3].imre*t1[4,6]
|
|
fmls \t1\().4s, \o2\().4s, \c3\().4s // E j0123
|
|
fmls \t3\().4s, \o3\().4s, \t5\().4s // E j4567
|
|
|
|
fsub \t0\().4s, \t1\().4s, \o0\().4s
|
|
fadd \t1\().4s, \t1\().4s, \o0\().4s
|
|
fadd \t2\().4s, \t3\().4s, \o1\().4s
|
|
fsub \t3\().4s, \t3\().4s, \o1\().4s
|
|
|
|
fmul \t0\().4s, \t0\().4s, v31.4s
|
|
fmul \t3\().4s, \t3\().4s, v31.4s
|
|
|
|
rev64 \t1\().4s, \t1\().4s
|
|
rev64 \t2\().4s, \t2\().4s
|
|
|
|
.if \part == 0
|
|
fsub \o0\().4s, \e0\().4s, \t1\().4s
|
|
fsub \o1\().4s, \e1\().4s, \t2\().4s
|
|
fsub \o2\().4s, \e2\().4s, \t0\().4s
|
|
fsub \o3\().4s, \e3\().4s, \t3\().4s
|
|
.else
|
|
fsub \o0\().4s, \e0\().4s, \t1\().4s
|
|
fadd \o2\().4s, \e1\().4s, \t2\().4s
|
|
fsub \o1\().4s, \e2\().4s, \t0\().4s
|
|
fadd \o3\().4s, \e3\().4s, \t3\().4s
|
|
.endif
|
|
|
|
.if \part == 0
|
|
fadd \e0\().4s, \e0\().4s, \t1\().4s
|
|
fadd \e1\().4s, \e1\().4s, \t2\().4s
|
|
fadd \e2\().4s, \e2\().4s, \t0\().4s
|
|
fadd \e3\().4s, \e3\().4s, \t3\().4s
|
|
.else
|
|
fadd \e0\().4s, \e0\().4s, \t1\().4s
|
|
fsub \e1\().4s, \e1\().4s, \t2\().4s // swapped
|
|
fadd \e2\().4s, \e2\().4s, \t0\().4s // swapped
|
|
fsub \e3\().4s, \e3\().4s, \t3\().4s
|
|
.endif
|
|
.endm
|
|
|
|
/* Same as SR_COMBINE_HALF, but heroically tries to use 3 temporary registers
|
|
* without touching the tables. */
|
|
.macro SR_COMBINE_LITE e0, e1, e2, e3, \
|
|
o0, o1, o2, o3, \
|
|
c0, c1, c2, c3, \
|
|
t0, t1, t2, part
|
|
|
|
rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre
|
|
rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre
|
|
.if \part == 0
|
|
trn2 \t2\().4s, \c3\().4s, \c3\().4s // wim3311
|
|
.else
|
|
trn1 \t2\().4s, \c3\().4s, \c3\().4s // wim3311
|
|
.endif
|
|
fmul \t2\().4s, \t2\().4s, v31.4s
|
|
fmul \o2\().4s, \o2\().4s, \t2\().4s
|
|
fmul \o0\().4s, \o0\().4s, \t2\().4s // E m2[0,1].imre*t1[0,2]
|
|
.if \part == 0
|
|
trn1 \t2\().4s, \c0\().4s, \c0\().4s // cos0022
|
|
.else
|
|
trn2 \t2\().4s, \c0\().4s, \c0\().4s // cos0022
|
|
.endif
|
|
fmul \t1\().4s, \t1\().4s, \t2\().4s // E m2[2,3].imre*t1[0,2]
|
|
fmla \o0\().4s, \t0\().4s, \t2\().4s // E w0123
|
|
fsub \t1\().4s, \t1\().4s, \o2\().4s // E j0123
|
|
|
|
rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre
|
|
rev64 \o2\().4s, \o3\().4s // E m3[2,3].imre
|
|
|
|
.if \part == 0
|
|
trn2 \t0\().4s, \c2\().4s, \c2\().4s // wim7755
|
|
.else
|
|
trn1 \t0\().4s, \c2\().4s, \c2\().4s // wim7755
|
|
.endif
|
|
fmul \t0\().4s, \t0\().4s, v31.4s
|
|
|
|
fmul \o1\().4s, \o1\().4s, \t0\().4s // E m3[0,1].imre*t1[4,6]
|
|
fmul \o3\().4s, \o3\().4s, \t0\().4s
|
|
|
|
.if \part == 0
|
|
trn1 \t0\().4s, \c1\().4s, \c1\().4s // cos1133
|
|
.else
|
|
trn2 \t0\().4s, \c1\().4s, \c1\().4s // cos1133
|
|
.endif
|
|
fmul \o2\().4s, \o2\().4s, \t0\().4s // E m3[2,3].imre*t1[4,6]
|
|
fmla \o1\().4s, \t2\().4s, \t0\().4s // E w4567
|
|
fsub \o2\().4s, \o2\().4s, \o3\().4s // E j4567
|
|
|
|
fsub \t0\().4s, \t1\().4s, \o0\().4s
|
|
fadd \o0\().4s, \t1\().4s, \o0\().4s
|
|
fadd \t2\().4s, \o2\().4s, \o1\().4s
|
|
fsub \t1\().4s, \o2\().4s, \o1\().4s
|
|
|
|
fmul \t0\().4s, \t0\().4s, v31.4s
|
|
fmul \t1\().4s, \t1\().4s, v31.4s
|
|
|
|
rev64 \t2\().4s, \t2\().4s
|
|
rev64 \o0\().4s, \o0\().4s
|
|
|
|
.if \part == 0
|
|
fsub \o1\().4s, \e1\().4s, \t2\().4s
|
|
fsub \o2\().4s, \e2\().4s, \t0\().4s
|
|
fsub \o3\().4s, \e3\().4s, \t1\().4s
|
|
.else
|
|
fadd \o2\().4s, \e1\().4s, \t0\().4s
|
|
fsub \o1\().4s, \e2\().4s, \t2\().4s
|
|
fadd \o3\().4s, \e3\().4s, \t1\().4s
|
|
.endif
|
|
|
|
.if \part == 0
|
|
fadd \e1\().4s, \e1\().4s, \t2\().4s
|
|
fadd \e2\().4s, \e2\().4s, \t0\().4s
|
|
fadd \e3\().4s, \e3\().4s, \t1\().4s
|
|
.else
|
|
fsub \e1\().4s, \e1\().4s, \t0\().4s // swapped
|
|
fadd \e2\().4s, \e2\().4s, \t2\().4s // swapped
|
|
fsub \e3\().4s, \e3\().4s, \t1\().4s
|
|
.endif
|
|
|
|
mov \t1\().16b, \o0\().16b
|
|
|
|
fsub \o0\().4s, \e0\().4s, \t1\().4s
|
|
fadd \e0\().4s, \e0\().4s, \t1\().4s
|
|
.endm
|
|
|
|
.macro SR_COMBINE_4 len, part, off
|
|
add x10, x1, x21
|
|
add x11, x1, x21, lsl #1
|
|
add x12, x1, x22
|
|
|
|
ldp q0, q1, [x1, #((0 + \part)*32 + \off)]
|
|
ldp q4, q5, [x1, #((2 + \part)*32 + \off)]
|
|
ldp q2, q3, [x10, #((0 + \part)*32 + \off)]
|
|
ldp q6, q7, [x10, #((2 + \part)*32 + \off)]
|
|
|
|
ldp q8, q9, [x11, #((0 + \part)*32 + \off)]
|
|
ldp q10, q11, [x11, #((2 + \part)*32 + \off)]
|
|
ldp q12, q13, [x12, #((0 + \part)*32 + \off)]
|
|
ldp q14, q15, [x12, #((2 + \part)*32 + \off)]
|
|
|
|
SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \
|
|
v8, v9, v10, v11, v12, v13, v14, v15, \
|
|
x7, x8, x9, 0
|
|
|
|
stp q0, q1, [x1, #((0 + \part)*32 + \off)]
|
|
stp q4, q5, [x1, #((2 + \part)*32 + \off)]
|
|
stp q2, q3, [x10, #((0 + \part)*32 + \off)]
|
|
stp q6, q7, [x10, #((2 + \part)*32 + \off)]
|
|
|
|
stp q8, q9, [x11, #((0 + \part)*32 + \off)]
|
|
stp q12, q13, [x11, #((2 + \part)*32 + \off)]
|
|
stp q10, q11, [x12, #((0 + \part)*32 + \off)]
|
|
stp q14, q15, [x12, #((2 + \part)*32 + \off)]
|
|
.endm
|
|
|
|
.macro SR_COMBINE_FULL len, off=0
|
|
add x10, x1, x21
|
|
add x11, x1, x21, lsl #1
|
|
add x12, x1, x22
|
|
|
|
SR_COMBINE_4 \len, 0, \off
|
|
SR_COMBINE_4 \len, 1, \off
|
|
SR_COMBINE_4 \len, 4, \off
|
|
SR_COMBINE_4 \len, 5, \off
|
|
.endm
|
|
|
|
.macro SR_COMBINE_D2 part, off
|
|
add x10, x1, #((\part)*32 + \off)
|
|
add x11, x14, #((\part)*32 + \off)
|
|
add x12, x15, #((\part)*32 + \off)
|
|
add x13, x16, #((\part)*32 + \off)
|
|
|
|
ldp q0, q1, [x10]
|
|
ldp q4, q5, [x10, #(2*32)]
|
|
ldp q2, q3, [x11]
|
|
ldp q6, q7, [x11, #(2*32)]
|
|
|
|
ldp q8, q9, [x12]
|
|
ldp q10, q11, [x12, #(2*32)]
|
|
ldp q12, q13, [x13]
|
|
ldp q14, q15, [x13, #(2*32)]
|
|
|
|
SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \
|
|
v8, v9, v10, v11, v12, v13, v14, v15, \
|
|
x7, x8, x9, 0, \
|
|
v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27
|
|
|
|
zip1 v16.2d, v0.2d, v4.2d
|
|
zip2 v17.2d, v0.2d, v4.2d
|
|
zip1 v18.2d, v1.2d, v5.2d
|
|
zip2 v19.2d, v1.2d, v5.2d
|
|
|
|
zip1 v20.2d, v2.2d, v6.2d
|
|
zip2 v21.2d, v2.2d, v6.2d
|
|
zip1 v22.2d, v3.2d, v7.2d
|
|
zip2 v23.2d, v3.2d, v7.2d
|
|
|
|
ldp q0, q1, [x10, #(1*32)]
|
|
ldp q4, q5, [x10, #(3*32)]
|
|
ldp q2, q3, [x11, #(1*32)]
|
|
ldp q6, q7, [x11, #(3*32)]
|
|
|
|
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10], #64
|
|
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x11], #64
|
|
|
|
zip1 v20.2d, v8.2d, v12.2d
|
|
zip2 v21.2d, v8.2d, v12.2d
|
|
zip1 v22.2d, v9.2d, v13.2d
|
|
zip2 v23.2d, v9.2d, v13.2d
|
|
zip1 v24.2d, v10.2d, v14.2d
|
|
zip2 v25.2d, v10.2d, v14.2d
|
|
zip1 v26.2d, v11.2d, v15.2d
|
|
zip2 v27.2d, v11.2d, v15.2d
|
|
|
|
ldp q8, q9, [x12, #(1*32)]
|
|
ldp q10, q11, [x12, #(3*32)]
|
|
ldp q12, q13, [x13, #(1*32)]
|
|
ldp q14, q15, [x13, #(3*32)]
|
|
|
|
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12], #64
|
|
st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13], #64
|
|
|
|
SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \
|
|
v8, v9, v10, v11, v12, v13, v14, v15, \
|
|
x7, x8, x9, 0, \
|
|
v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27
|
|
|
|
zip1 v16.2d, v0.2d, v4.2d
|
|
zip2 v17.2d, v0.2d, v4.2d
|
|
zip1 v18.2d, v1.2d, v5.2d
|
|
zip2 v19.2d, v1.2d, v5.2d
|
|
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10]
|
|
|
|
zip1 v16.2d, v2.2d, v6.2d
|
|
zip2 v17.2d, v2.2d, v6.2d
|
|
zip1 v18.2d, v3.2d, v7.2d
|
|
zip2 v19.2d, v3.2d, v7.2d
|
|
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x11]
|
|
|
|
zip1 v20.2d, v8.2d, v12.2d
|
|
zip2 v21.2d, v8.2d, v12.2d
|
|
zip1 v22.2d, v9.2d, v13.2d
|
|
zip2 v23.2d, v9.2d, v13.2d
|
|
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12]
|
|
|
|
zip1 v24.2d, v10.2d, v14.2d
|
|
zip2 v25.2d, v10.2d, v14.2d
|
|
zip1 v26.2d, v11.2d, v15.2d
|
|
zip2 v27.2d, v11.2d, v15.2d
|
|
st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13]
|
|
.endm
|
|
|
|
.macro SR_COMBINE_DINT off=0
|
|
add x14, x1, x21
|
|
add x15, x1, x21, lsl #1
|
|
add x16, x1, x22
|
|
|
|
SR_COMBINE_D2 0, \off
|
|
SR_COMBINE_D2 4, \off
|
|
.endm
|
|
|
|
.macro FFT32_FN name, no_perm
|
|
function ff_tx_fft32_\name\()_neon, export=1
|
|
stp d14, d15, [sp, #-16*4]!
|
|
stp d8, d9, [sp, #16*3]
|
|
stp d10, d11, [sp, #16*2]
|
|
stp d12, d13, [sp, #16]
|
|
|
|
LOAD_SUBADD
|
|
SETUP_SR_RECOMB 32, x7, x8, x9
|
|
|
|
SETUP_LUT \no_perm
|
|
LOAD_INPUT 0, 1, 2, 3, x2, \no_perm
|
|
LOAD_INPUT 4, 5, 6, 7, x2, \no_perm
|
|
LOAD_INPUT 8, 9, 10, 11, x2, \no_perm
|
|
LOAD_INPUT 12, 13, 14, 15, x2, \no_perm
|
|
|
|
FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15
|
|
FFT16 v0, v1, v2, v3, v4, v5, v6, v7
|
|
|
|
SR_COMBINE v0, v1, v2, v3, v4, v5, v6, v7, \
|
|
v8, v9, v10, v11, v12, v13, v14, v15, \
|
|
x7, x8, x9, 0
|
|
|
|
zip1 v16.2d, v0.2d, v4.2d
|
|
zip2 v17.2d, v0.2d, v4.2d
|
|
zip1 v18.2d, v1.2d, v6.2d
|
|
zip2 v19.2d, v1.2d, v6.2d
|
|
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1], #64
|
|
|
|
zip1 v20.2d, v2.2d, v5.2d
|
|
zip2 v21.2d, v2.2d, v5.2d
|
|
zip1 v22.2d, v3.2d, v7.2d
|
|
zip2 v23.2d, v3.2d, v7.2d
|
|
st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64
|
|
|
|
zip1 v24.2d, v8.2d, v12.2d
|
|
zip2 v25.2d, v8.2d, v12.2d
|
|
zip1 v26.2d, v9.2d, v13.2d
|
|
zip2 v27.2d, v9.2d, v13.2d
|
|
st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1], #64
|
|
|
|
zip1 v28.2d, v10.2d, v14.2d
|
|
zip2 v29.2d, v10.2d, v14.2d
|
|
zip1 v30.2d, v11.2d, v15.2d
|
|
zip2 v31.2d, v11.2d, v15.2d
|
|
st1 { v28.4s, v29.4s, v30.4s, v31.4s }, [x1]
|
|
|
|
ldp d12, d13, [sp, #16]
|
|
ldp d10, d11, [sp, #16*2]
|
|
ldp d8, d9, [sp, #16*3]
|
|
ldp d14, d15, [sp], #16*4
|
|
|
|
ret
|
|
endfunc
|
|
.endm
|
|
|
|
FFT32_FN float, 0
|
|
FFT32_FN ns_float, 1
|
|
|
|
.macro cmp_imm reg, imm
|
|
.if \imm >= 4096
|
|
cmp \reg, #((\imm)/4096), lsl #12
|
|
.else
|
|
cmp \reg, #(\imm)
|
|
.endif
|
|
.endm
|
|
|
|
.macro SR_TRANSFORM_DEF len, next=0
|
|
\len:
|
|
stp x20, x30, [sp, #-16]!
|
|
mov w20, #(\len/4)
|
|
mov x5, #((\len*4) - (\len/1))
|
|
add x1, x1, x5
|
|
bl 32b
|
|
mov x5, #((\len*2) - (\len/2))
|
|
add x1, x1, x5
|
|
bl 32b
|
|
ldp x20, x30, [sp], #16
|
|
ldr w5, =(\len*6 + \len/2)
|
|
sub x1, x1, x5
|
|
|
|
SETUP_SR_RECOMB \len, x7, x8, x9
|
|
|
|
.if \next\() != 0
|
|
cmp_imm w19, \len
|
|
b.eq 0f
|
|
|
|
mov w5, #(\len/128)
|
|
\len\()5:
|
|
SR_COMBINE_FULL \len
|
|
add x1, x1, 8*32
|
|
subs w5, w5, 1
|
|
b.gt \len\()5b
|
|
|
|
cmp_imm w20, \len
|
|
b.gt \next\()f
|
|
ret
|
|
.endif
|
|
.endm
|
|
|
|
.macro FFT_SPLIT_RADIX_FN name, no_perm
|
|
function ff_tx_fft_sr_\name\()_neon, export=1
|
|
stp x21, x22, [sp, #-16*6]!
|
|
stp d8, d9, [sp, #16*5]
|
|
stp d10, d11, [sp, #16*4]
|
|
stp d12, d13, [sp, #16*3]
|
|
stp d14, d15, [sp, #16*2]
|
|
stp x19, x20, [sp, #16]
|
|
|
|
ldr w19, [x0, #0] // global target
|
|
mov w20, w19 // local length
|
|
|
|
LOAD_SUBADD
|
|
SETUP_LUT \no_perm
|
|
|
|
32:
|
|
SETUP_SR_RECOMB 32, x7, x8, x9
|
|
|
|
LOAD_INPUT 0, 1, 2, 3, x2, \no_perm
|
|
LOAD_INPUT 4, 6, 5, 7, x2, \no_perm, 1
|
|
LOAD_INPUT 8, 9, 10, 11, x2, \no_perm
|
|
LOAD_INPUT 12, 13, 14, 15, x2, \no_perm
|
|
|
|
FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15
|
|
FFT16 v0, v1, v2, v3, v4, v6, v5, v7
|
|
|
|
SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \
|
|
v8, v9, v10, v11, v12, v13, v14, v15, \
|
|
x7, x8, x9, 0
|
|
|
|
stp q2, q3, [x1, #32*1]
|
|
stp q6, q7, [x1, #32*3]
|
|
stp q10, q11, [x1, #32*5]
|
|
stp q14, q15, [x1, #32*7]
|
|
|
|
cmp w20, #32
|
|
b.gt 64f
|
|
|
|
stp q0, q1, [x1, #32*0]
|
|
stp q4, q5, [x1, #32*2]
|
|
stp q8, q9, [x1, #32*4]
|
|
stp q12, q13, [x1, #32*6]
|
|
|
|
ret
|
|
64:
|
|
SETUP_SR_RECOMB 64, x7, x8, x9
|
|
|
|
LOAD_INPUT 2, 3, 10, 11, x2, \no_perm, 1
|
|
LOAD_INPUT 6, 14, 7, 15, x2, \no_perm, 1
|
|
|
|
FFT16 v2, v3, v10, v11, v6, v14, v7, v15
|
|
|
|
LOAD_INPUT 16, 17, 18, 19, x2, \no_perm
|
|
LOAD_INPUT 20, 22, 21, 23, x2, \no_perm, 1
|
|
|
|
FFT16 v16, v17, v18, v19, v20, v22, v21, v23, \
|
|
v24, v25, v26, v27, v28, v29, v30
|
|
|
|
ld1 { v26.4s, v27.4s }, [x8], x9
|
|
ldp q24, q25, [x7], #32
|
|
|
|
ext v26.16b, v26.16b, v26.16b, #8
|
|
ext v27.16b, v27.16b, v27.16b, #8
|
|
|
|
cmp w19, #64
|
|
b.eq 2f // custom deinterleave
|
|
|
|
// TODO: investigate doing the 2 combines like in deinterleave
|
|
// TODO: experiment with spilling to gprs and converting to HALF or full
|
|
SR_COMBINE_LITE v0, v1, v8, v9, \
|
|
v2, v3, v16, v17, \
|
|
v24, v25, v26, v27, \
|
|
v28, v29, v30, 0
|
|
|
|
stp q0, q1, [x1, #32* 0]
|
|
stp q8, q9, [x1, #32* 4]
|
|
stp q2, q3, [x1, #32* 8]
|
|
stp q16, q17, [x1, #32*12]
|
|
|
|
SR_COMBINE_HALF v4, v5, v12, v13, \
|
|
v6, v7, v20, v21, \
|
|
v24, v25, v26, v27, \
|
|
v28, v29, v30, v0, v1, v8, 1
|
|
|
|
stp q4, q20, [x1, #32* 2]
|
|
stp q12, q21, [x1, #32* 6]
|
|
stp q6, q5, [x1, #32*10]
|
|
stp q7, q13, [x1, #32*14]
|
|
|
|
ldp q2, q3, [x1, #32*1]
|
|
ldp q6, q7, [x1, #32*3]
|
|
ldp q12, q13, [x1, #32*5]
|
|
ldp q16, q17, [x1, #32*7]
|
|
|
|
SR_COMBINE v2, v3, v12, v13, v6, v16, v7, v17, \
|
|
v10, v11, v14, v15, v18, v19, v22, v23, \
|
|
x7, x8, x9, 0, \
|
|
v24, v25, v26, v27, v28, v29, v30, v8, v0, v1, v4, v5
|
|
|
|
stp q2, q3, [x1, #32* 1]
|
|
stp q6, q7, [x1, #32* 3]
|
|
stp q12, q13, [x1, #32* 5]
|
|
stp q16, q17, [x1, #32* 7]
|
|
|
|
stp q10, q11, [x1, #32* 9]
|
|
stp q18, q19, [x1, #32*11]
|
|
stp q14, q15, [x1, #32*13]
|
|
stp q22, q23, [x1, #32*15]
|
|
|
|
cmp w20, #64
|
|
b.gt 128f
|
|
ret
|
|
128:
|
|
stp x20, x30, [sp, #-16]!
|
|
mov w20, #32
|
|
add x1, x1, #16*32
|
|
bl 32b
|
|
add x1, x1, #8*32
|
|
bl 32b
|
|
ldp x20, x30, [sp], #16
|
|
sub x1, x1, #24*32
|
|
|
|
SETUP_SR_RECOMB 128, x7, x8, x9
|
|
|
|
cmp w19, #128
|
|
b.eq 0f
|
|
|
|
SR_COMBINE_FULL 128
|
|
|
|
cmp w20, #128
|
|
b.gt 256f
|
|
ret
|
|
256:
|
|
stp x20, x30, [sp, #-16]!
|
|
mov w20, #64
|
|
add x1, x1, #32*32
|
|
bl 32b
|
|
add x1, x1, #16*32
|
|
bl 32b
|
|
ldp x20, x30, [sp], #16
|
|
sub x1, x1, #48*32
|
|
|
|
SETUP_SR_RECOMB 256, x7, x8, x9
|
|
|
|
cmp w19, #256
|
|
b.eq 0f
|
|
|
|
SR_COMBINE_FULL 256
|
|
SR_COMBINE_FULL 256, 8*32
|
|
|
|
cmp w20, #256
|
|
b.gt 512f
|
|
ret
|
|
512:
|
|
stp x20, x30, [sp, #-16]!
|
|
mov w20, #128
|
|
add x1, x1, #64*32
|
|
bl 32b
|
|
add x1, x1, #32*32
|
|
bl 32b
|
|
ldp x20, x30, [sp], #16
|
|
sub x1, x1, #96*32
|
|
|
|
SETUP_SR_RECOMB 512, x7, x8, x9
|
|
|
|
cmp w19, #512
|
|
b.eq 0f
|
|
|
|
mov x5, 4
|
|
5125:
|
|
SR_COMBINE_FULL 512
|
|
add x1, x1, 8*32
|
|
subs w5, w5, 1
|
|
b.gt 5125b
|
|
|
|
cmp w20, #512
|
|
b.gt 1024f
|
|
|
|
ret
|
|
1024:
|
|
stp x20, x30, [sp, #-16]!
|
|
mov w20, #256
|
|
add x1, x1, #96*32
|
|
bl 32b
|
|
add x1, x1, #64*32
|
|
bl 32b
|
|
ldp x20, x30, [sp], #16
|
|
mov x5, #192*32
|
|
sub x1, x1, x5
|
|
|
|
SETUP_SR_RECOMB 1024, x7, x8, x9
|
|
|
|
cmp w19, #1024
|
|
b.eq 0f
|
|
|
|
mov w5, 8
|
|
10245:
|
|
SR_COMBINE_FULL 1024
|
|
add x1, x1, 8*32
|
|
subs w5, w5, 1
|
|
b.gt 10245b
|
|
|
|
cmp w20, #1024
|
|
b.gt 2048f
|
|
|
|
ret
|
|
|
|
SR_TRANSFORM_DEF 2048, 4096
|
|
SR_TRANSFORM_DEF 4096, 8192
|
|
SR_TRANSFORM_DEF 8192, 16384
|
|
SR_TRANSFORM_DEF 16384, 32768
|
|
SR_TRANSFORM_DEF 32768, 65536
|
|
SR_TRANSFORM_DEF 65536, 131072
|
|
SR_TRANSFORM_DEF 131072
|
|
|
|
0: // general deinterleave loop
|
|
SR_COMBINE_DINT
|
|
add x1, x1, #32*8
|
|
subs w19, w19, #32*4
|
|
b.gt 0b
|
|
|
|
ldp x19, x20, [sp, #16]
|
|
ldp d14, d15, [sp, #16*2]
|
|
ldp d12, d13, [sp, #16*3]
|
|
ldp d10, d11, [sp, #16*4]
|
|
ldp d8, d9, [sp, #16*5]
|
|
ldp x21, x22, [sp], #16*6
|
|
|
|
ret
|
|
|
|
2: // special case for 64 point deinterleave
|
|
mov x10, v23.d[0]
|
|
mov x11, v23.d[1]
|
|
|
|
SR_COMBINE_LITE v0, v1, v8, v9, \
|
|
v2, v3, v16, v17, \
|
|
v24, v25, v26, v27, \
|
|
v28, v29, v30, 0
|
|
|
|
SR_COMBINE_HALF v4, v5, v12, v13, \
|
|
v6, v7, v20, v21, \
|
|
v24, v25, v26, v27, \
|
|
v28, v29, v30, v23, v24, v26, 1
|
|
|
|
zip1 v23.2d, v0.2d, v4.2d
|
|
zip2 v24.2d, v0.2d, v4.2d
|
|
zip1 v25.2d, v1.2d, v20.2d
|
|
zip2 v26.2d, v1.2d, v20.2d
|
|
|
|
zip1 v27.2d, v8.2d, v12.2d
|
|
zip2 v28.2d, v8.2d, v12.2d
|
|
zip1 v29.2d, v9.2d, v21.2d
|
|
zip2 v30.2d, v9.2d, v21.2d
|
|
|
|
mov v20.16b, v5.16b
|
|
mov v21.16b, v7.16b
|
|
mov x12, x1
|
|
add x13, x1, #32* 4
|
|
add x14, x1, #32* 8
|
|
add x15, x1, #32*12
|
|
|
|
zip1 v4.2d, v2.2d, v6.2d
|
|
zip2 v5.2d, v2.2d, v6.2d
|
|
zip1 v6.2d, v3.2d, v20.2d
|
|
zip2 v7.2d, v3.2d, v20.2d
|
|
|
|
zip1 v0.2d, v16.2d, v21.2d
|
|
zip2 v1.2d, v16.2d, v21.2d
|
|
zip1 v2.2d, v17.2d, v13.2d
|
|
zip2 v3.2d, v17.2d, v13.2d
|
|
|
|
// stp is faster by a little on A53, but this is faster on M1s (theory)
|
|
ldp q8, q9, [x1, #32*1]
|
|
ldp q12, q13, [x1, #32*5]
|
|
|
|
st1 { v23.4s, v24.4s, v25.4s, v26.4s }, [x12], #64 // 32* 0...1
|
|
st1 { v27.4s, v28.4s, v29.4s, v30.4s }, [x13], #64 // 32* 4...5
|
|
st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x14], #64 // 32* 8...9
|
|
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x15], #64 // 32*12..13
|
|
|
|
mov v23.d[0], x10
|
|
mov v23.d[1], x11
|
|
|
|
ldp q6, q7, [x1, #32*3]
|
|
ldp q16, q17, [x1, #32*7]
|
|
|
|
SR_COMBINE v8, v9, v12, v13, v6, v16, v7, v17, \
|
|
v10, v11, v14, v15, v18, v19, v22, v23, \
|
|
x7, x8, x9, 0, \
|
|
v24, v25, v26, v27, v28, v29, v30, v4, v0, v1, v5, v20
|
|
|
|
zip1 v0.2d, v8.2d, v6.2d
|
|
zip2 v1.2d, v8.2d, v6.2d
|
|
zip1 v2.2d, v9.2d, v7.2d
|
|
zip2 v3.2d, v9.2d, v7.2d
|
|
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x12]
|
|
|
|
zip1 v4.2d, v12.2d, v16.2d
|
|
zip2 v5.2d, v12.2d, v16.2d
|
|
zip1 v6.2d, v13.2d, v17.2d
|
|
zip2 v7.2d, v13.2d, v17.2d
|
|
st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x13]
|
|
|
|
zip1 v0.2d, v10.2d, v18.2d
|
|
zip2 v1.2d, v10.2d, v18.2d
|
|
zip1 v2.2d, v11.2d, v19.2d
|
|
zip2 v3.2d, v11.2d, v19.2d
|
|
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x14]
|
|
|
|
zip1 v4.2d, v14.2d, v22.2d
|
|
zip2 v5.2d, v14.2d, v22.2d
|
|
zip1 v6.2d, v15.2d, v23.2d
|
|
zip2 v7.2d, v15.2d, v23.2d
|
|
st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x15]
|
|
|
|
ldp x19, x20, [sp, #16]
|
|
ldp d14, d15, [sp, #16*2]
|
|
ldp d12, d13, [sp, #16*3]
|
|
ldp d10, d11, [sp, #16*4]
|
|
ldp d8, d9, [sp, #16*5]
|
|
ldp x21, x22, [sp], #16*6
|
|
|
|
ret
|
|
endfunc
|
|
.endm
|
|
|
|
FFT_SPLIT_RADIX_FN float, 0
|
|
FFT_SPLIT_RADIX_FN ns_float, 1
|