mirror of
https://github.com/FFmpeg/FFmpeg.git
synced 2024-11-21 10:55:51 +02:00
mdct15: add inverse transform postrotation SIMD
2.5ms frames: Before (c): 2638 decicycles in postrotate, 2097040 runs, 112 skips After (sse3): 1467 decicycles in postrotate, 2097083 runs, 69 skips After (avx2): 1244 decicycles in postrotate, 2097085 runs, 67 skips 5ms frames: Before (c): 4987 decicycles in postrotate, 1048371 runs, 205 skips After (sse3): 2644 decicycles in postrotate, 1048509 runs, 67 skips After (avx2): 2031 decicycles in postrotate, 1048523 runs, 53 skips 10ms frames: Before (c): 9153 decicycles in postrotate, 523575 runs, 713 skips After (sse3): 5110 decicycles in postrotate, 523726 runs, 562 skips After (avx2): 3738 decicycles in postrotate, 524223 runs, 65 skips 20ms frames: Before (c): 17857 decicycles in postrotate, 261866 runs, 278 skips After (sse3): 10041 decicycles in postrotate, 261746 runs, 398 skips After (avx2): 7050 decicycles in postrotate, 262116 runs, 28 skips Improves total decoding performance for real world content by 9% with avx2. Signed-off-by: Rostislav Pehlivanov <atomnuker@gmail.com>
This commit is contained in:
Rostislav Pehlivanov
parent
b0c61209cd
commit
70eb77b34e
@ -65,11 +65,11 @@ static inline int init_pfa_reindex_tabs(MDCT15Context *s)
|
||||
const int inv_1 = l_ptwo << ((4 - b_ptwo) & 3); /* (2^b_ptwo)^-1 mod 15 */
|
||||
const int inv_2 = 0xeeeeeeef & ((1U << b_ptwo) - 1); /* 15^-1 mod 2^b_ptwo */
|
||||
|
||||
s->pfa_prereindex = av_malloc(15 * l_ptwo * sizeof(*s->pfa_prereindex));
|
||||
s->pfa_prereindex = av_malloc_array(15 * l_ptwo, sizeof(*s->pfa_prereindex));
|
||||
if (!s->pfa_prereindex)
|
||||
return 1;
|
||||
|
||||
s->pfa_postreindex = av_malloc(15 * l_ptwo * sizeof(*s->pfa_postreindex));
|
||||
s->pfa_postreindex = av_malloc_array(15 * l_ptwo, sizeof(*s->pfa_postreindex));
|
||||
if (!s->pfa_postreindex)
|
||||
return 1;
|
||||
|
||||
@ -225,15 +225,22 @@ static void imdct15_half(MDCT15Context *s, float *dst, const float *src,
|
||||
for (i = 0; i < 15; i++)
|
||||
s->ptwo_fft.fft_calc(&s->ptwo_fft, s->tmp + l_ptwo*i);
|
||||
|
||||
/* Reindex again, apply twiddles and output */
|
||||
s->postreindex(z, s->tmp, s->twiddle_exptab, s->pfa_postreindex, len8);
|
||||
}
|
||||
|
||||
static void postrotate_c(FFTComplex *out, FFTComplex *in, FFTComplex *exp,
|
||||
int *lut, ptrdiff_t len8)
|
||||
{
|
||||
int i;
|
||||
|
||||
/* Reindex again, apply twiddles and output */
|
||||
for (i = 0; i < len8; i++) {
|
||||
const int i0 = len8 + i, i1 = len8 - i - 1;
|
||||
const int s0 = s->pfa_postreindex[i0], s1 = s->pfa_postreindex[i1];
|
||||
const int s0 = lut[i0], s1 = lut[i1];
|
||||
|
||||
CMUL(z[i1].re, z[i0].im, s->tmp[s1].im, s->tmp[s1].re,
|
||||
s->twiddle_exptab[i1].im, s->twiddle_exptab[i1].re);
|
||||
CMUL(z[i0].re, z[i1].im, s->tmp[s0].im, s->tmp[s0].re,
|
||||
s->twiddle_exptab[i0].im, s->twiddle_exptab[i0].re);
|
||||
CMUL(out[i1].re, out[i0].im, in[s1].im, in[s1].re, exp[i1].im, exp[i1].re);
|
||||
CMUL(out[i0].re, out[i1].im, in[s0].im, in[s0].re, exp[i0].im, exp[i0].re);
|
||||
}
|
||||
}
|
||||
|
||||
@ -253,13 +260,14 @@ av_cold int ff_mdct15_init(MDCT15Context **ps, int inverse, int N, double scale)
|
||||
if (!s)
|
||||
return AVERROR(ENOMEM);
|
||||
|
||||
s->fft_n = N - 1;
|
||||
s->len4 = len2 / 2;
|
||||
s->len2 = len2;
|
||||
s->inverse = inverse;
|
||||
s->fft15 = fft15_c;
|
||||
s->mdct = mdct15;
|
||||
s->imdct_half = imdct15_half;
|
||||
s->fft_n = N - 1;
|
||||
s->len4 = len2 / 2;
|
||||
s->len2 = len2;
|
||||
s->inverse = inverse;
|
||||
s->fft15 = fft15_c;
|
||||
s->mdct = mdct15;
|
||||
s->imdct_half = imdct15_half;
|
||||
s->postreindex = postrotate_c;
|
||||
|
||||
if (ff_fft_init(&s->ptwo_fft, N - 1, s->inverse) < 0)
|
||||
goto fail;
|
||||
@ -271,7 +279,7 @@ av_cold int ff_mdct15_init(MDCT15Context **ps, int inverse, int N, double scale)
|
||||
if (!s->tmp)
|
||||
goto fail;
|
||||
|
||||
s->twiddle_exptab = av_malloc_array(s->len4, sizeof(*s->twiddle_exptab));
|
||||
s->twiddle_exptab = av_malloc_array(s->len4, sizeof(*s->twiddle_exptab));
|
||||
if (!s->twiddle_exptab)
|
||||
goto fail;
|
||||
|
||||
|
||||
@ -42,6 +42,9 @@ typedef struct MDCT15Context {
|
||||
/* 15-point FFT */
|
||||
void (*fft15)(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
|
||||
|
||||
/* PFA postrotate and exptab */
|
||||
void (*postreindex)(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
|
||||
|
||||
/* Calculate a full 2N -> N MDCT */
|
||||
void (*mdct)(struct MDCT15Context *s, float *dst, const float *src, ptrdiff_t stride);
|
||||
|
||||
|
||||
@ -22,14 +22,21 @@
|
||||
|
||||
%include "libavutil/x86/x86util.asm"
|
||||
|
||||
%if ARCH_X86_64
|
||||
SECTION_RODATA 32
|
||||
|
||||
SECTION_RODATA
|
||||
perm_neg: dd 2, 5, 3, 4, 6, 1, 7, 0
|
||||
perm_pos: dd 0, 7, 1, 6, 4, 3, 5, 2
|
||||
sign_adjust_r: times 4 dd 0x80000000, 0x00000000
|
||||
|
||||
sign_adjust_5: dd 0x00000000, 0x80000000, 0x80000000, 0x00000000
|
||||
|
||||
SECTION .text
|
||||
|
||||
%if ARCH_X86_64
|
||||
|
||||
;*****************************************************************************************
|
||||
;void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
|
||||
;*****************************************************************************************
|
||||
%macro FFT5 3 ; %1 - in_offset, %2 - dst1 (64bit used), %3 - dst2
|
||||
VBROADCASTSD m0, [inq + %1] ; in[ 0].re, in[ 0].im, in[ 0].re, in[ 0].im
|
||||
movsd xm1, [inq + 1*16 + 8 + %1] ; in[ 3].re, in[ 3].im, 0, 0
|
||||
@ -103,9 +110,6 @@ SECTION .text
|
||||
movhps [%2q + strideq*4], xm1
|
||||
%endmacro
|
||||
|
||||
;*****************************************************************************************
|
||||
;void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
|
||||
;*****************************************************************************************
|
||||
INIT_YMM avx
|
||||
cglobal fft15, 4, 6, 14, out, in, exptab, stride, stride3, stride5
|
||||
%define out0q inq
|
||||
@ -138,4 +142,83 @@ cglobal fft15, 4, 6, 14, out, in, exptab, stride, stride3, stride5
|
||||
|
||||
RET
|
||||
|
||||
%endif ; ARCH_X86_64
|
||||
|
||||
;*******************************************************************************************************
|
||||
;void ff_mdct15_postreindex(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
|
||||
;*******************************************************************************************************
|
||||
%macro LUT_LOAD_4D 3
|
||||
mov r4d, [lutq + %3q*4 + 0]
|
||||
movsd xmm%1, [inq + r4q*8]
|
||||
mov r4d, [lutq + %3q*4 + 4]
|
||||
movhps xmm%1, [inq + r4q*8]
|
||||
%if cpuflag(avx2)
|
||||
mov r4d, [lutq + %3q*4 + 8]
|
||||
movsd %2, [inq + r4q*8]
|
||||
mov r4d, [lutq + %3q*4 + 12]
|
||||
movhps %2, [inq + r4q*8]
|
||||
vinsertf128 %1, %1, %2, 1
|
||||
%endif
|
||||
%endmacro
|
||||
|
||||
%macro POSTROTATE_FN 1
|
||||
cglobal mdct15_postreindex, 5, 7, 8 + cpuflag(avx2)*2, out, in, exp, lut, len8, offset_p, offset_n
|
||||
|
||||
xor offset_nq, offset_nq
|
||||
lea offset_pq, [len8q*2 - %1]
|
||||
|
||||
movaps m7, [sign_adjust_r]
|
||||
|
||||
%if cpuflag(avx2)
|
||||
movaps m8, [perm_pos]
|
||||
movaps m9, [perm_neg]
|
||||
%endif
|
||||
|
||||
.loop:
|
||||
movups m0, [expq + offset_pq*8] ; exp[p0].re, exp[p0].im, exp[p1].re, exp[p1].im, exp[p2].re, exp[p2].im, exp[p3].re, exp[p3].im
|
||||
movups m1, [expq + offset_nq*8] ; exp[n3].re, exp[n3].im, exp[n2].re, exp[n2].im, exp[n1].re, exp[n1].im, exp[n0].re, exp[n0].im
|
||||
|
||||
LUT_LOAD_4D m3, xm4, offset_p ; in[p0].re, in[p0].im, in[p1].re, in[p1].im, in[p2].re, in[p2].im, in[p3].re, in[p3].im
|
||||
LUT_LOAD_4D m4, xm5, offset_n ; in[n3].re, in[n3].im, in[n2].re, in[n2].im, in[n1].re, in[n1].im, in[n0].re, in[n0].im
|
||||
|
||||
mulps m5, m3, m0 ; in[p].reim * exp[p].reim
|
||||
mulps m6, m4, m1 ; in[n].reim * exp[n].reim
|
||||
|
||||
xorps m5, m7 ; in[p].re *= -1, in[p].im *= 1
|
||||
xorps m6, m7 ; in[n].re *= -1, in[n].im *= 1
|
||||
|
||||
shufps m3, m3, m3, q2301 ; in[p].imre
|
||||
shufps m4, m4, m4, q2301 ; in[n].imre
|
||||
|
||||
mulps m3, m0 ; in[p].imre * exp[p].reim
|
||||
mulps m4, m1 ; in[n].imre * exp[n].reim
|
||||
|
||||
haddps m3, m6 ; out[n0].im, out[n1].im, out[n3].re, out[n2].re, out[n2].im, out[n3].im, out[n1].re, out[n0].re
|
||||
haddps m5, m4 ; out[p0].re, out[p1].re, out[p3].im, out[p2].im, out[p2].re, out[p3].re, out[p1].im, out[p0].im
|
||||
|
||||
%if cpuflag(avx2)
|
||||
vpermps m3, m9, m3 ; out[n3].im, out[n3].re, out[n2].im, out[n2].re, out[n1].im, out[n1].re, out[n0].im, out[n0].re
|
||||
vpermps m5, m8, m5 ; out[p0].re, out[p0].im, out[p1].re, out[p1].im, out[p2].re, out[p2].im, out[p3].re, out[p3].im
|
||||
%else
|
||||
shufps m3, m3, m3, q0312
|
||||
shufps m5, m5, m5, q2130
|
||||
%endif
|
||||
|
||||
movups [outq + offset_nq*8], m3
|
||||
movups [outq + offset_pq*8], m5
|
||||
|
||||
sub offset_pq, %1
|
||||
add offset_nq, %1
|
||||
cmp offset_nq, offset_pq
|
||||
jle .loop
|
||||
|
||||
REP_RET
|
||||
%endmacro
|
||||
|
||||
INIT_XMM sse3
|
||||
POSTROTATE_FN 2
|
||||
|
||||
%if ARCH_X86_64 && HAVE_AVX2_EXTERNAL
|
||||
INIT_YMM avx2
|
||||
POSTROTATE_FN 4
|
||||
%endif
|
||||
|
||||
@ -25,6 +25,9 @@
|
||||
#include "libavutil/x86/cpu.h"
|
||||
#include "libavcodec/mdct15.h"
|
||||
|
||||
void ff_mdct15_postreindex_sse3(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
|
||||
void ff_mdct15_postreindex_avx2(FFTComplex *out, FFTComplex *in, FFTComplex *exp, int *lut, ptrdiff_t len8);
|
||||
|
||||
void ff_fft15_avx(FFTComplex *out, FFTComplex *in, FFTComplex *exptab, ptrdiff_t stride);
|
||||
|
||||
static void perm_twiddles(MDCT15Context *s)
|
||||
@ -85,11 +88,17 @@ av_cold void ff_mdct15_init_x86(MDCT15Context *s)
|
||||
int adjust_twiddles = 0;
|
||||
int cpu_flags = av_get_cpu_flags();
|
||||
|
||||
if (EXTERNAL_SSE3(cpu_flags))
|
||||
s->postreindex = ff_mdct15_postreindex_sse3;
|
||||
|
||||
if (ARCH_X86_64 && EXTERNAL_AVX(cpu_flags)) {
|
||||
s->fft15 = ff_fft15_avx;
|
||||
adjust_twiddles = 1;
|
||||
}
|
||||
|
||||
if (ARCH_X86_64 && EXTERNAL_AVX2_FAST(cpu_flags))
|
||||
s->postreindex = ff_mdct15_postreindex_avx2;
|
||||
|
||||
if (adjust_twiddles)
|
||||
perm_twiddles(s);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user