mirror of
https://github.com/FFmpeg/FFmpeg.git
synced 2024-12-23 12:43:46 +02:00
87552d54d3
The previous implementation targeted DTS Coherent Acoustics, which only
requires nbits == 4 (fft16()). This case was (and still is) linked directly
rather than being indirected through ff_fft_calc_vfp(), but now the full
range from radix-4 up to radix-65536 is available. This benefits other codecs
such as AAC and AC3.
The implementaion is based upon the C version, with each routine larger than
radix-16 calling a hierarchy of smaller FFT functions, then performing a
post-processing pass. This pass benefits a lot from loop unrolling to
counter the long pipelines in the VFP. A relaxed calling standard also
reduces the overhead of the call hierarchy, and avoiding the excessive
inlining performed by GCC probably helps with I-cache utilisation too.
I benchmarked the result by measuring the number of gperftools samples that
hit anywhere in the AAC decoder (starting from aac_decode_frame()) or
specifically in the FFT routines (fft4() to fft512() and pass()) for the
same sample AAC stream:
Before After
Mean StdDev Mean StdDev Confidence Change
Audio decode 2245.5 53.1 1599.6 43.8 100.0% +40.4%
FFT routines 940.6 22.0 348.1 20.8 100.0% +170.2%
Signed-off-by: Martin Storsjö <martin@martin.st>
535 lines
19 KiB
ArmAsm
535 lines
19 KiB
ArmAsm
/*
|
|
* Copyright (c) 2013 RISC OS Open Ltd
|
|
* Author: Ben Avison <bavison@riscosopen.org>
|
|
*
|
|
* This file is part of Libav.
|
|
*
|
|
* Libav is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
*
|
|
* Libav is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
* License along with Libav; if not, write to the Free Software
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
*/
|
|
|
|
#include "libavutil/arm/asm.S"
|
|
|
|
@ The fftx_internal_vfp versions of the functions obey a modified AAPCS:
|
|
@ VFP is in RunFast mode, vector length 4, stride 1 thoroughout, and
|
|
@ all single-precision VFP registers may be corrupted on exit. The a2
|
|
@ register may not be clobbered in these functions, as it holds the
|
|
@ stored original FPSCR.
|
|
|
|
function ff_fft_calc_vfp, export=1
|
|
ldr ip, [a1, #0] @ nbits
|
|
mov a1, a2
|
|
A ldr pc, [pc, ip, lsl #2]
|
|
A .word 0
|
|
A .word 0
|
|
A .word 0
|
|
T movrel a2, (fft_tab_vfp - 8)
|
|
T ldr pc, [a2, ip, lsl #2]
|
|
T endfunc
|
|
T const fft_tab_vfp
|
|
.word fft4_vfp
|
|
.word fft8_vfp
|
|
.word X(ff_fft16_vfp) @ this one alone is exported
|
|
.word fft32_vfp
|
|
.word fft64_vfp
|
|
.word fft128_vfp
|
|
.word fft256_vfp
|
|
.word fft512_vfp
|
|
.word fft1024_vfp
|
|
.word fft2048_vfp
|
|
.word fft4096_vfp
|
|
.word fft8192_vfp
|
|
.word fft16384_vfp
|
|
.word fft32768_vfp
|
|
.word fft65536_vfp
|
|
A endfunc
|
|
|
|
function fft4_vfp
|
|
vldr d0, [a1, #0*2*4] @ s0,s1 = z[0]
|
|
vldr d4, [a1, #1*2*4] @ s8,s9 = z[1]
|
|
vldr d1, [a1, #2*2*4] @ s2,s3 = z[2]
|
|
vldr d5, [a1, #3*2*4] @ s10,s11 = z[3]
|
|
@ stall
|
|
vadd.f s12, s0, s8 @ i0
|
|
vadd.f s13, s1, s9 @ i1
|
|
vadd.f s14, s2, s10 @ i2
|
|
vadd.f s15, s3, s11 @ i3
|
|
vsub.f s8, s0, s8 @ i4
|
|
vsub.f s9, s1, s9 @ i5
|
|
vsub.f s10, s2, s10 @ i6
|
|
vsub.f s11, s3, s11 @ i7
|
|
@ stall
|
|
@ stall
|
|
vadd.f s0, s12, s14 @ z[0].re
|
|
vsub.f s4, s12, s14 @ z[2].re
|
|
vadd.f s1, s13, s15 @ z[0].im
|
|
vsub.f s5, s13, s15 @ z[2].im
|
|
vadd.f s7, s9, s10 @ z[3].im
|
|
vsub.f s3, s9, s10 @ z[1].im
|
|
vadd.f s2, s8, s11 @ z[1].re
|
|
vsub.f s6, s8, s11 @ z[3].re
|
|
@ stall
|
|
@ stall
|
|
vstr d0, [a1, #0*2*4]
|
|
vstr d2, [a1, #2*2*4]
|
|
@ stall
|
|
@ stall
|
|
vstr d1, [a1, #1*2*4]
|
|
vstr d3, [a1, #3*2*4]
|
|
|
|
bx lr
|
|
endfunc
|
|
|
|
.macro macro_fft8_head
|
|
@ FFT4
|
|
vldr d4, [a1, #0 * 2*4]
|
|
vldr d6, [a1, #1 * 2*4]
|
|
vldr d5, [a1, #2 * 2*4]
|
|
vldr d7, [a1, #3 * 2*4]
|
|
@ BF
|
|
vldr d12, [a1, #4 * 2*4]
|
|
vadd.f s16, s8, s12 @ vector op
|
|
vldr d14, [a1, #5 * 2*4]
|
|
vldr d13, [a1, #6 * 2*4]
|
|
vldr d15, [a1, #7 * 2*4]
|
|
vsub.f s20, s8, s12 @ vector op
|
|
vadd.f s0, s16, s18
|
|
vsub.f s2, s16, s18
|
|
vadd.f s1, s17, s19
|
|
vsub.f s3, s17, s19
|
|
vadd.f s7, s21, s22
|
|
vsub.f s5, s21, s22
|
|
vadd.f s4, s20, s23
|
|
vsub.f s6, s20, s23
|
|
vsub.f s20, s24, s28 @ vector op
|
|
vstr d0, [a1, #0 * 2*4] @ transfer s0-s7 to s24-s31 via memory
|
|
vstr d1, [a1, #1 * 2*4]
|
|
vldr s0, cos1pi4
|
|
vadd.f s16, s24, s28 @ vector op
|
|
vstr d2, [a1, #2 * 2*4]
|
|
vstr d3, [a1, #3 * 2*4]
|
|
vldr d12, [a1, #0 * 2*4]
|
|
@ TRANSFORM
|
|
vmul.f s20, s20, s0 @ vector x scalar op
|
|
vldr d13, [a1, #1 * 2*4]
|
|
vldr d14, [a1, #2 * 2*4]
|
|
vldr d15, [a1, #3 * 2*4]
|
|
@ BUTTERFLIES
|
|
vadd.f s0, s18, s16
|
|
vadd.f s1, s17, s19
|
|
vsub.f s2, s17, s19
|
|
vsub.f s3, s18, s16
|
|
vadd.f s4, s21, s20
|
|
vsub.f s5, s21, s20
|
|
vadd.f s6, s22, s23
|
|
vsub.f s7, s22, s23
|
|
vadd.f s8, s0, s24 @ vector op
|
|
vstr d0, [a1, #0 * 2*4] @ transfer s0-s3 to s12-s15 via memory
|
|
vstr d1, [a1, #1 * 2*4]
|
|
vldr d6, [a1, #0 * 2*4]
|
|
vldr d7, [a1, #1 * 2*4]
|
|
vadd.f s1, s5, s6
|
|
vadd.f s0, s7, s4
|
|
vsub.f s2, s5, s6
|
|
vsub.f s3, s7, s4
|
|
vsub.f s12, s24, s12 @ vector op
|
|
vsub.f s5, s29, s1
|
|
vsub.f s4, s28, s0
|
|
vsub.f s6, s30, s2
|
|
vsub.f s7, s31, s3
|
|
vadd.f s16, s0, s28 @ vector op
|
|
vstr d6, [a1, #4 * 2*4]
|
|
vstr d7, [a1, #6 * 2*4]
|
|
vstr d4, [a1, #0 * 2*4]
|
|
vstr d5, [a1, #2 * 2*4]
|
|
vstr d2, [a1, #5 * 2*4]
|
|
vstr d3, [a1, #7 * 2*4]
|
|
.endm
|
|
|
|
.macro macro_fft8_tail
|
|
vstr d8, [a1, #1 * 2*4]
|
|
vstr d9, [a1, #3 * 2*4]
|
|
.endm
|
|
|
|
function .Lfft8_internal_vfp
|
|
macro_fft8_head
|
|
macro_fft8_tail
|
|
bx lr
|
|
endfunc
|
|
|
|
function fft8_vfp
|
|
ldr a3, =0x03030000 @ RunFast mode, vector length 4, stride 1
|
|
fmrx a2, FPSCR
|
|
fmxr FPSCR, a3
|
|
vpush {s16-s31}
|
|
mov ip, lr
|
|
bl .Lfft8_internal_vfp
|
|
vpop {s16-s31}
|
|
fmxr FPSCR, a2
|
|
bx ip
|
|
endfunc
|
|
|
|
.align 3
|
|
cos1pi4: @ cos(1*pi/4) = sqrt(2)
|
|
.float 0.707106769084930419921875
|
|
cos1pi8: @ cos(1*pi/8) = sqrt(2+sqrt(2))/2
|
|
.float 0.92387950420379638671875
|
|
cos3pi8: @ cos(2*pi/8) = sqrt(2-sqrt(2))/2
|
|
.float 0.3826834261417388916015625
|
|
|
|
function .Lfft16_internal_vfp
|
|
macro_fft8_head
|
|
@ FFT4(z+8)
|
|
vldr d10, [a1, #8 * 2*4]
|
|
vldr d12, [a1, #9 * 2*4]
|
|
vldr d11, [a1, #10 * 2*4]
|
|
vldr d13, [a1, #11 * 2*4]
|
|
macro_fft8_tail
|
|
vadd.f s16, s20, s24 @ vector op
|
|
@ FFT4(z+12)
|
|
vldr d4, [a1, #12 * 2*4]
|
|
vldr d6, [a1, #13 * 2*4]
|
|
vldr d5, [a1, #14 * 2*4]
|
|
vsub.f s20, s20, s24 @ vector op
|
|
vldr d7, [a1, #15 * 2*4]
|
|
vadd.f s0, s16, s18
|
|
vsub.f s4, s16, s18
|
|
vadd.f s1, s17, s19
|
|
vsub.f s5, s17, s19
|
|
vadd.f s7, s21, s22
|
|
vsub.f s3, s21, s22
|
|
vadd.f s2, s20, s23
|
|
vsub.f s6, s20, s23
|
|
vadd.f s16, s8, s12 @ vector op
|
|
vstr d0, [a1, #8 * 2*4]
|
|
vstr d2, [a1, #10 * 2*4]
|
|
vstr d1, [a1, #9 * 2*4]
|
|
vsub.f s20, s8, s12
|
|
vstr d3, [a1, #11 * 2*4]
|
|
@ TRANSFORM(z[2],z[6],z[10],z[14],cos1pi4,cos1pi4)
|
|
vldr d12, [a1, #10 * 2*4]
|
|
vadd.f s0, s16, s18
|
|
vadd.f s1, s17, s19
|
|
vsub.f s6, s16, s18
|
|
vsub.f s7, s17, s19
|
|
vsub.f s3, s21, s22
|
|
vadd.f s2, s20, s23
|
|
vadd.f s5, s21, s22
|
|
vsub.f s4, s20, s23
|
|
vstr d0, [a1, #12 * 2*4]
|
|
vmov s0, s6
|
|
@ TRANSFORM(z[1],z[5],z[9],z[13],cos1pi8,cos3pi8)
|
|
vldr d6, [a1, #9 * 2*4]
|
|
vstr d1, [a1, #13 * 2*4]
|
|
vldr d1, cos1pi4 @ s2 = cos1pi4, s3 = cos1pi8
|
|
vstr d2, [a1, #15 * 2*4]
|
|
vldr d7, [a1, #13 * 2*4]
|
|
vadd.f s4, s25, s24
|
|
vsub.f s5, s25, s24
|
|
vsub.f s6, s0, s7
|
|
vadd.f s7, s0, s7
|
|
vmul.f s20, s12, s3 @ vector op
|
|
@ TRANSFORM(z[3],z[7],z[11],z[15],cos3pi8,cos1pi8)
|
|
vldr d4, [a1, #11 * 2*4]
|
|
vldr d5, [a1, #15 * 2*4]
|
|
vldr s1, cos3pi8
|
|
vmul.f s24, s4, s2 @ vector * scalar op
|
|
vmul.f s28, s12, s1 @ vector * scalar op
|
|
vmul.f s12, s8, s1 @ vector * scalar op
|
|
vadd.f s4, s20, s29
|
|
vsub.f s5, s21, s28
|
|
vsub.f s6, s22, s31
|
|
vadd.f s7, s23, s30
|
|
vmul.f s8, s8, s3 @ vector * scalar op
|
|
vldr d8, [a1, #1 * 2*4]
|
|
vldr d9, [a1, #5 * 2*4]
|
|
vldr d10, [a1, #3 * 2*4]
|
|
vldr d11, [a1, #7 * 2*4]
|
|
vldr d14, [a1, #2 * 2*4]
|
|
vadd.f s0, s6, s4
|
|
vadd.f s1, s5, s7
|
|
vsub.f s2, s5, s7
|
|
vsub.f s3, s6, s4
|
|
vadd.f s4, s12, s9
|
|
vsub.f s5, s13, s8
|
|
vsub.f s6, s14, s11
|
|
vadd.f s7, s15, s10
|
|
vadd.f s12, s0, s16 @ vector op
|
|
vstr d0, [a1, #1 * 2*4]
|
|
vstr d1, [a1, #5 * 2*4]
|
|
vldr d4, [a1, #1 * 2*4]
|
|
vldr d5, [a1, #5 * 2*4]
|
|
vadd.f s0, s6, s4
|
|
vadd.f s1, s5, s7
|
|
vsub.f s2, s5, s7
|
|
vsub.f s3, s6, s4
|
|
vsub.f s8, s16, s8 @ vector op
|
|
vstr d6, [a1, #1 * 2*4]
|
|
vstr d7, [a1, #5 * 2*4]
|
|
vldr d15, [a1, #6 * 2*4]
|
|
vsub.f s4, s20, s0
|
|
vsub.f s5, s21, s1
|
|
vsub.f s6, s22, s2
|
|
vsub.f s7, s23, s3
|
|
vadd.f s20, s0, s20 @ vector op
|
|
vstr d4, [a1, #9 * 2*4]
|
|
@ TRANSFORM_ZERO(z[0],z[4],z[8],z[12])
|
|
vldr d6, [a1, #8 * 2*4]
|
|
vstr d5, [a1, #13 * 2*4]
|
|
vldr d7, [a1, #12 * 2*4]
|
|
vstr d2, [a1, #11 * 2*4]
|
|
vldr d8, [a1, #0 * 2*4]
|
|
vstr d3, [a1, #15 * 2*4]
|
|
vldr d9, [a1, #4 * 2*4]
|
|
vadd.f s0, s26, s24
|
|
vadd.f s1, s25, s27
|
|
vsub.f s2, s25, s27
|
|
vsub.f s3, s26, s24
|
|
vadd.f s4, s14, s12
|
|
vadd.f s5, s13, s15
|
|
vsub.f s6, s13, s15
|
|
vsub.f s7, s14, s12
|
|
vadd.f s8, s0, s28 @ vector op
|
|
vstr d0, [a1, #3 * 2*4]
|
|
vstr d1, [a1, #7 * 2*4]
|
|
vldr d6, [a1, #3 * 2*4]
|
|
vldr d7, [a1, #7 * 2*4]
|
|
vsub.f s0, s16, s4
|
|
vsub.f s1, s17, s5
|
|
vsub.f s2, s18, s6
|
|
vsub.f s3, s19, s7
|
|
vsub.f s12, s28, s12 @ vector op
|
|
vadd.f s16, s4, s16 @ vector op
|
|
vstr d10, [a1, #3 * 2*4]
|
|
vstr d11, [a1, #7 * 2*4]
|
|
vstr d4, [a1, #2 * 2*4]
|
|
vstr d5, [a1, #6 * 2*4]
|
|
vstr d0, [a1, #8 * 2*4]
|
|
vstr d1, [a1, #12 * 2*4]
|
|
vstr d6, [a1, #10 * 2*4]
|
|
vstr d7, [a1, #14 * 2*4]
|
|
vstr d8, [a1, #0 * 2*4]
|
|
vstr d9, [a1, #4 * 2*4]
|
|
|
|
bx lr
|
|
endfunc
|
|
|
|
function ff_fft16_vfp, export=1
|
|
ldr a3, =0x03030000 @ RunFast mode, vector length 4, stride 1
|
|
fmrx a2, FPSCR
|
|
fmxr FPSCR, a3
|
|
vpush {s16-s31}
|
|
mov ip, lr
|
|
bl .Lfft16_internal_vfp
|
|
vpop {s16-s31}
|
|
fmxr FPSCR, a2
|
|
bx ip
|
|
endfunc
|
|
|
|
.macro pass n, z0, z1, z2, z3
|
|
add v6, v5, #4*2*\n
|
|
@ TRANSFORM_ZERO(z[0],z[o1],z[o2],z[o3])
|
|
@ TRANSFORM(z[1],z[o1+1],z[o2+1],z[o3+1],wre[1],wim[-1])
|
|
@ TRANSFORM(z[0],z[o1],z[o2],z[o3],wre[0],wim[0])
|
|
@ TRANSFORM(z[1],z[o1+1],z[o2+1],z[o3+1],wre[1],wim[-1])
|
|
vldr d8, [\z2, #8*(o2+1)] @ s16,s17
|
|
vldmdb v6!, {s2}
|
|
vldr d9, [\z3, #8*(o3+1)] @ s18,s19
|
|
vldmia v5!, {s0,s1} @ s0 is unused
|
|
vldr s7, [\z2, #8*o2] @ t1
|
|
vmul.f s20, s16, s2 @ vector * scalar
|
|
vldr s0, [\z3, #8*o3] @ t5
|
|
vldr s6, [\z2, #8*o2+4] @ t2
|
|
vldr s3, [\z3, #8*o3+4] @ t6
|
|
vmul.f s16, s16, s1 @ vector * scalar
|
|
ldr a4, =\n-1
|
|
1: add \z0, \z0, #8*2
|
|
.if \n*4*2 >= 512
|
|
add \z1, \z1, #8*2
|
|
.endif
|
|
.if \n*4*2 >= 256
|
|
add \z2, \z2, #8*2
|
|
.endif
|
|
.if \n*4*2 >= 512
|
|
add \z3, \z3, #8*2
|
|
.endif
|
|
@ up to 2 stalls (VFP vector issuing / waiting for s0)
|
|
@ depending upon whether this is the first iteration and
|
|
@ how many add instructions are inserted above
|
|
vadd.f s4, s0, s7 @ t5
|
|
vadd.f s5, s6, s3 @ t6
|
|
vsub.f s6, s6, s3 @ t4
|
|
vsub.f s7, s0, s7 @ t3
|
|
vldr d6, [\z0, #8*0-8*2] @ s12,s13
|
|
vadd.f s0, s16, s21 @ t1
|
|
vldr d7, [\z1, #8*o1-8*2] @ s14,s15
|
|
vsub.f s1, s18, s23 @ t5
|
|
vadd.f s8, s4, s12 @ vector + vector
|
|
@ stall (VFP vector issuing)
|
|
@ stall (VFP vector issuing)
|
|
@ stall (VFP vector issuing)
|
|
vsub.f s4, s12, s4
|
|
vsub.f s5, s13, s5
|
|
vsub.f s6, s14, s6
|
|
vsub.f s7, s15, s7
|
|
vsub.f s2, s17, s20 @ t2
|
|
vadd.f s3, s19, s22 @ t6
|
|
vstr d4, [\z0, #8*0-8*2] @ s8,s9
|
|
vstr d5, [\z1, #8*o1-8*2] @ s10,s11
|
|
@ stall (waiting for s5)
|
|
vstr d2, [\z2, #8*o2-8*2] @ s4,s5
|
|
vadd.f s4, s1, s0 @ t5
|
|
vstr d3, [\z3, #8*o3-8*2] @ s6,s7
|
|
vsub.f s7, s1, s0 @ t3
|
|
vadd.f s5, s2, s3 @ t6
|
|
vsub.f s6, s2, s3 @ t4
|
|
vldr d6, [\z0, #8*1-8*2] @ s12,s13
|
|
vldr d7, [\z1, #8*(o1+1)-8*2] @ s14,s15
|
|
vldr d4, [\z2, #8*o2] @ s8,s9
|
|
vldmdb v6!, {s2,s3}
|
|
vldr d5, [\z3, #8*o3] @ s10,s11
|
|
vadd.f s20, s4, s12 @ vector + vector
|
|
vldmia v5!, {s0,s1}
|
|
vldr d8, [\z2, #8*(o2+1)] @ s16,s17
|
|
@ stall (VFP vector issuing)
|
|
vsub.f s4, s12, s4
|
|
vsub.f s5, s13, s5
|
|
vsub.f s6, s14, s6
|
|
vsub.f s7, s15, s7
|
|
vmul.f s12, s8, s3 @ vector * scalar
|
|
vstr d10, [\z0, #8*1-8*2] @ s20,s21
|
|
vldr d9, [\z3, #8*(o3+1)] @ s18,s19
|
|
vstr d11, [\z1, #8*(o1+1)-8*2] @ s22,s23
|
|
vmul.f s8, s8, s0 @ vector * scalar
|
|
vstr d2, [\z2, #8*(o2+1)-8*2] @ s4,s5
|
|
@ stall (waiting for s7)
|
|
vstr d3, [\z3, #8*(o3+1)-8*2] @ s6,s7
|
|
vmul.f s20, s16, s2 @ vector * scalar
|
|
@ stall (VFP vector issuing)
|
|
@ stall (VFP vector issuing)
|
|
@ stall (VFP vector issuing)
|
|
vadd.f s7, s8, s13 @ t1
|
|
vsub.f s6, s9, s12 @ t2
|
|
vsub.f s0, s10, s15 @ t5
|
|
vadd.f s3, s11, s14 @ t6
|
|
vmul.f s16, s16, s1 @ vector * scalar
|
|
subs a4, a4, #1
|
|
bne 1b
|
|
@ What remains is identical to the first two indentations of
|
|
@ the above, but without the increment of z
|
|
vadd.f s4, s0, s7 @ t5
|
|
vadd.f s5, s6, s3 @ t6
|
|
vsub.f s6, s6, s3 @ t4
|
|
vsub.f s7, s0, s7 @ t3
|
|
vldr d6, [\z0, #8*0] @ s12,s13
|
|
vadd.f s0, s16, s21 @ t1
|
|
vldr d7, [\z1, #8*o1] @ s14,s15
|
|
vsub.f s1, s18, s23 @ t5
|
|
vadd.f s8, s4, s12 @ vector + vector
|
|
vsub.f s4, s12, s4
|
|
vsub.f s5, s13, s5
|
|
vsub.f s6, s14, s6
|
|
vsub.f s7, s15, s7
|
|
vsub.f s2, s17, s20 @ t2
|
|
vadd.f s3, s19, s22 @ t6
|
|
vstr d4, [\z0, #8*0] @ s8,s9
|
|
vstr d5, [\z1, #8*o1] @ s10,s11
|
|
vstr d2, [\z2, #8*o2] @ s4,s5
|
|
vadd.f s4, s1, s0 @ t5
|
|
vstr d3, [\z3, #8*o3] @ s6,s7
|
|
vsub.f s7, s1, s0 @ t3
|
|
vadd.f s5, s2, s3 @ t6
|
|
vsub.f s6, s2, s3 @ t4
|
|
vldr d6, [\z0, #8*1] @ s12,s13
|
|
vldr d7, [\z1, #8*(o1+1)] @ s14,s15
|
|
vadd.f s20, s4, s12 @ vector + vector
|
|
vsub.f s4, s12, s4
|
|
vsub.f s5, s13, s5
|
|
vsub.f s6, s14, s6
|
|
vsub.f s7, s15, s7
|
|
vstr d10, [\z0, #8*1] @ s20,s21
|
|
vstr d11, [\z1, #8*(o1+1)] @ s22,s23
|
|
vstr d2, [\z2, #8*(o2+1)] @ s4,s5
|
|
vstr d3, [\z3, #8*(o3+1)] @ s6,s7
|
|
.endm
|
|
|
|
.macro def_fft n, n2, n4
|
|
function .Lfft\n\()_internal_vfp
|
|
.if \n >= 512
|
|
push {v1-v6,lr}
|
|
.elseif \n >= 256
|
|
push {v1-v2,v5-v6,lr}
|
|
.else
|
|
push {v1,v5-v6,lr}
|
|
.endif
|
|
mov v1, a1
|
|
bl .Lfft\n2\()_internal_vfp
|
|
add a1, v1, #8*(\n/4)*2
|
|
bl .Lfft\n4\()_internal_vfp
|
|
movrelx v5, X(ff_cos_\n), a1
|
|
add a1, v1, #8*(\n/4)*3
|
|
bl .Lfft\n4\()_internal_vfp
|
|
.if \n >= 512
|
|
.set o1, 0*(\n/4/2)
|
|
.set o2, 0*(\n/4/2)
|
|
.set o3, 0*(\n/4/2)
|
|
add v2, v1, #8*2*(\n/4/2)
|
|
add v3, v1, #8*4*(\n/4/2)
|
|
add v4, v1, #8*6*(\n/4/2)
|
|
pass (\n/4/2), v1, v2, v3, v4
|
|
pop {v1-v6,pc}
|
|
.elseif \n >= 256
|
|
.set o1, 2*(\n/4/2)
|
|
.set o2, 0*(\n/4/2)
|
|
.set o3, 2*(\n/4/2)
|
|
add v2, v1, #8*4*(\n/4/2)
|
|
pass (\n/4/2), v1, v1, v2, v2
|
|
pop {v1-v2,v5-v6,pc}
|
|
.else
|
|
.set o1, 2*(\n/4/2)
|
|
.set o2, 4*(\n/4/2)
|
|
.set o3, 6*(\n/4/2)
|
|
pass (\n/4/2), v1, v1, v1, v1
|
|
pop {v1,v5-v6,pc}
|
|
.endif
|
|
endfunc
|
|
|
|
function fft\n\()_vfp
|
|
ldr a3, =0x03030000 /* RunFast mode, vector length 4, stride 1 */
|
|
fmrx a2, FPSCR
|
|
fmxr FPSCR, a3
|
|
vpush {s16-s31}
|
|
mov ip, lr
|
|
bl .Lfft\n\()_internal_vfp
|
|
vpop {s16-s31}
|
|
fmxr FPSCR, a2
|
|
bx ip
|
|
endfunc
|
|
|
|
.ltorg
|
|
.endm
|
|
|
|
def_fft 32, 16, 8
|
|
def_fft 64, 32, 16
|
|
def_fft 128, 64, 32
|
|
def_fft 256, 128, 64
|
|
def_fft 512, 256, 128
|
|
def_fft 1024, 512, 256
|
|
def_fft 2048, 1024, 512
|
|
def_fft 4096, 2048, 1024
|
|
def_fft 8192, 4096, 2048
|
|
def_fft 16384, 8192, 4096
|
|
def_fft 32768, 16384, 8192
|
|
def_fft 65536, 32768, 16384
|