mirror of
https://github.com/FFmpeg/FFmpeg.git
synced 2024-12-28 20:53:54 +02:00
2718a3be1f
x64 always has MMX, MMXEXT, SSE and SSE2 and this means that some functions for MMX, MMXEXT, SSE and 3dnow are always overridden by other functions (unless one e.g. explicitly disables SSE2). So given that the only systems which benefit from ff_vector_fmul_window_3dnowext are truely ancient 32bit AMD x86s it is removed. Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
462 lines
13 KiB
NASM
462 lines
13 KiB
NASM
;*****************************************************************************
|
|
;* x86-optimized Float DSP functions
|
|
;*
|
|
;* Copyright 2006 Loren Merritt
|
|
;*
|
|
;* This file is part of FFmpeg.
|
|
;*
|
|
;* FFmpeg is free software; you can redistribute it and/or
|
|
;* modify it under the terms of the GNU Lesser General Public
|
|
;* License as published by the Free Software Foundation; either
|
|
;* version 2.1 of the License, or (at your option) any later version.
|
|
;*
|
|
;* FFmpeg is distributed in the hope that it will be useful,
|
|
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
;* Lesser General Public License for more details.
|
|
;*
|
|
;* You should have received a copy of the GNU Lesser General Public
|
|
;* License along with FFmpeg; if not, write to the Free Software
|
|
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
;******************************************************************************
|
|
|
|
%include "libavutil/x86/x86util.asm"
|
|
|
|
SECTION_RODATA 32
|
|
pd_reverse: dd 7, 6, 5, 4, 3, 2, 1, 0
|
|
|
|
SECTION .text
|
|
|
|
;-----------------------------------------------------------------------------
|
|
; void vector_fmul(float *dst, const float *src0, const float *src1, int len)
|
|
;-----------------------------------------------------------------------------
|
|
%macro VECTOR_FMUL 0
|
|
cglobal vector_fmul, 4,4,2, dst, src0, src1, len
|
|
lea lenq, [lend*4 - 64]
|
|
ALIGN 16
|
|
.loop:
|
|
%assign a 0
|
|
%rep 32/mmsize
|
|
mova m0, [src0q + lenq + (a+0)*mmsize]
|
|
mova m1, [src0q + lenq + (a+1)*mmsize]
|
|
mulps m0, m0, [src1q + lenq + (a+0)*mmsize]
|
|
mulps m1, m1, [src1q + lenq + (a+1)*mmsize]
|
|
mova [dstq + lenq + (a+0)*mmsize], m0
|
|
mova [dstq + lenq + (a+1)*mmsize], m1
|
|
%assign a a+2
|
|
%endrep
|
|
|
|
sub lenq, 64
|
|
jge .loop
|
|
REP_RET
|
|
%endmacro
|
|
|
|
INIT_XMM sse
|
|
VECTOR_FMUL
|
|
%if HAVE_AVX_EXTERNAL
|
|
INIT_YMM avx
|
|
VECTOR_FMUL
|
|
%endif
|
|
|
|
;-----------------------------------------------------------------------------
|
|
; void vector_dmul(double *dst, const double *src0, const double *src1, int len)
|
|
;-----------------------------------------------------------------------------
|
|
%macro VECTOR_DMUL 0
|
|
cglobal vector_dmul, 4,4,4, dst, src0, src1, len
|
|
lea lend, [lenq*8 - mmsize*4]
|
|
ALIGN 16
|
|
.loop:
|
|
movaps m0, [src0q + lenq + 0*mmsize]
|
|
movaps m1, [src0q + lenq + 1*mmsize]
|
|
movaps m2, [src0q + lenq + 2*mmsize]
|
|
movaps m3, [src0q + lenq + 3*mmsize]
|
|
mulpd m0, m0, [src1q + lenq + 0*mmsize]
|
|
mulpd m1, m1, [src1q + lenq + 1*mmsize]
|
|
mulpd m2, m2, [src1q + lenq + 2*mmsize]
|
|
mulpd m3, m3, [src1q + lenq + 3*mmsize]
|
|
movaps [dstq + lenq + 0*mmsize], m0
|
|
movaps [dstq + lenq + 1*mmsize], m1
|
|
movaps [dstq + lenq + 2*mmsize], m2
|
|
movaps [dstq + lenq + 3*mmsize], m3
|
|
|
|
sub lenq, mmsize*4
|
|
jge .loop
|
|
RET
|
|
%endmacro
|
|
|
|
INIT_XMM sse2
|
|
VECTOR_DMUL
|
|
%if HAVE_AVX_EXTERNAL
|
|
INIT_YMM avx
|
|
VECTOR_DMUL
|
|
%endif
|
|
|
|
;------------------------------------------------------------------------------
|
|
; void ff_vector_fmac_scalar(float *dst, const float *src, float mul, int len)
|
|
;------------------------------------------------------------------------------
|
|
|
|
%macro VECTOR_FMAC_SCALAR 0
|
|
%if UNIX64
|
|
cglobal vector_fmac_scalar, 3,3,5, dst, src, len
|
|
%else
|
|
cglobal vector_fmac_scalar, 4,4,5, dst, src, mul, len
|
|
%endif
|
|
%if ARCH_X86_32
|
|
VBROADCASTSS m0, mulm
|
|
%else
|
|
%if WIN64
|
|
SWAP 0, 2
|
|
%endif
|
|
shufps xm0, xm0, 0
|
|
%if cpuflag(avx)
|
|
vinsertf128 m0, m0, xm0, 1
|
|
%endif
|
|
%endif
|
|
lea lenq, [lend*4-64]
|
|
.loop:
|
|
%if cpuflag(fma3)
|
|
mova m1, [dstq+lenq]
|
|
mova m2, [dstq+lenq+1*mmsize]
|
|
fmaddps m1, m0, [srcq+lenq], m1
|
|
fmaddps m2, m0, [srcq+lenq+1*mmsize], m2
|
|
%else ; cpuflag
|
|
mulps m1, m0, [srcq+lenq]
|
|
mulps m2, m0, [srcq+lenq+1*mmsize]
|
|
%if mmsize < 32
|
|
mulps m3, m0, [srcq+lenq+2*mmsize]
|
|
mulps m4, m0, [srcq+lenq+3*mmsize]
|
|
%endif ; mmsize
|
|
addps m1, m1, [dstq+lenq]
|
|
addps m2, m2, [dstq+lenq+1*mmsize]
|
|
%if mmsize < 32
|
|
addps m3, m3, [dstq+lenq+2*mmsize]
|
|
addps m4, m4, [dstq+lenq+3*mmsize]
|
|
%endif ; mmsize
|
|
%endif ; cpuflag
|
|
mova [dstq+lenq], m1
|
|
mova [dstq+lenq+1*mmsize], m2
|
|
%if mmsize < 32
|
|
mova [dstq+lenq+2*mmsize], m3
|
|
mova [dstq+lenq+3*mmsize], m4
|
|
%endif ; mmsize
|
|
sub lenq, 64
|
|
jge .loop
|
|
REP_RET
|
|
%endmacro
|
|
|
|
INIT_XMM sse
|
|
VECTOR_FMAC_SCALAR
|
|
%if HAVE_AVX_EXTERNAL
|
|
INIT_YMM avx
|
|
VECTOR_FMAC_SCALAR
|
|
%endif
|
|
%if HAVE_FMA3_EXTERNAL
|
|
INIT_YMM fma3
|
|
VECTOR_FMAC_SCALAR
|
|
%endif
|
|
|
|
;------------------------------------------------------------------------------
|
|
; void ff_vector_fmul_scalar(float *dst, const float *src, float mul, int len)
|
|
;------------------------------------------------------------------------------
|
|
|
|
%macro VECTOR_FMUL_SCALAR 0
|
|
%if UNIX64
|
|
cglobal vector_fmul_scalar, 3,3,2, dst, src, len
|
|
%else
|
|
cglobal vector_fmul_scalar, 4,4,3, dst, src, mul, len
|
|
%endif
|
|
%if ARCH_X86_32
|
|
movss m0, mulm
|
|
%elif WIN64
|
|
SWAP 0, 2
|
|
%endif
|
|
shufps m0, m0, 0
|
|
lea lenq, [lend*4-mmsize]
|
|
.loop:
|
|
mova m1, [srcq+lenq]
|
|
mulps m1, m0
|
|
mova [dstq+lenq], m1
|
|
sub lenq, mmsize
|
|
jge .loop
|
|
REP_RET
|
|
%endmacro
|
|
|
|
INIT_XMM sse
|
|
VECTOR_FMUL_SCALAR
|
|
|
|
;------------------------------------------------------------------------------
|
|
; void ff_vector_dmac_scalar(double *dst, const double *src, double mul,
|
|
; int len)
|
|
;------------------------------------------------------------------------------
|
|
|
|
%macro VECTOR_DMAC_SCALAR 0
|
|
%if ARCH_X86_32
|
|
cglobal vector_dmac_scalar, 2,4,5, dst, src, mul, len, lenaddr
|
|
mov lenq, lenaddrm
|
|
VBROADCASTSD m0, mulm
|
|
%else
|
|
%if UNIX64
|
|
cglobal vector_dmac_scalar, 3,3,5, dst, src, len
|
|
%else
|
|
cglobal vector_dmac_scalar, 4,4,5, dst, src, mul, len
|
|
SWAP 0, 2
|
|
%endif
|
|
movlhps xm0, xm0
|
|
%if cpuflag(avx)
|
|
vinsertf128 m0, m0, xm0, 1
|
|
%endif
|
|
%endif
|
|
lea lenq, [lend*8-mmsize*4]
|
|
.loop:
|
|
%if cpuflag(fma3)
|
|
movaps m1, [dstq+lenq]
|
|
movaps m2, [dstq+lenq+1*mmsize]
|
|
movaps m3, [dstq+lenq+2*mmsize]
|
|
movaps m4, [dstq+lenq+3*mmsize]
|
|
fmaddpd m1, m0, [srcq+lenq], m1
|
|
fmaddpd m2, m0, [srcq+lenq+1*mmsize], m2
|
|
fmaddpd m3, m0, [srcq+lenq+2*mmsize], m3
|
|
fmaddpd m4, m0, [srcq+lenq+3*mmsize], m4
|
|
%else ; cpuflag
|
|
mulpd m1, m0, [srcq+lenq]
|
|
mulpd m2, m0, [srcq+lenq+1*mmsize]
|
|
mulpd m3, m0, [srcq+lenq+2*mmsize]
|
|
mulpd m4, m0, [srcq+lenq+3*mmsize]
|
|
addpd m1, m1, [dstq+lenq]
|
|
addpd m2, m2, [dstq+lenq+1*mmsize]
|
|
addpd m3, m3, [dstq+lenq+2*mmsize]
|
|
addpd m4, m4, [dstq+lenq+3*mmsize]
|
|
%endif ; cpuflag
|
|
movaps [dstq+lenq], m1
|
|
movaps [dstq+lenq+1*mmsize], m2
|
|
movaps [dstq+lenq+2*mmsize], m3
|
|
movaps [dstq+lenq+3*mmsize], m4
|
|
sub lenq, mmsize*4
|
|
jge .loop
|
|
REP_RET
|
|
%endmacro
|
|
|
|
INIT_XMM sse2
|
|
VECTOR_DMAC_SCALAR
|
|
%if HAVE_AVX_EXTERNAL
|
|
INIT_YMM avx
|
|
VECTOR_DMAC_SCALAR
|
|
%endif
|
|
%if HAVE_FMA3_EXTERNAL
|
|
INIT_YMM fma3
|
|
VECTOR_DMAC_SCALAR
|
|
%endif
|
|
|
|
;------------------------------------------------------------------------------
|
|
; void ff_vector_dmul_scalar(double *dst, const double *src, double mul,
|
|
; int len)
|
|
;------------------------------------------------------------------------------
|
|
|
|
%macro VECTOR_DMUL_SCALAR 0
|
|
%if ARCH_X86_32
|
|
cglobal vector_dmul_scalar, 3,4,3, dst, src, mul, len, lenaddr
|
|
mov lenq, lenaddrm
|
|
%elif UNIX64
|
|
cglobal vector_dmul_scalar, 3,3,3, dst, src, len
|
|
%else
|
|
cglobal vector_dmul_scalar, 4,4,3, dst, src, mul, len
|
|
%endif
|
|
%if ARCH_X86_32
|
|
VBROADCASTSD m0, mulm
|
|
%else
|
|
%if WIN64
|
|
SWAP 0, 2
|
|
%endif
|
|
movlhps xm0, xm0
|
|
%if cpuflag(avx)
|
|
vinsertf128 ym0, ym0, xm0, 1
|
|
%endif
|
|
%endif
|
|
lea lenq, [lend*8-2*mmsize]
|
|
.loop:
|
|
mulpd m1, m0, [srcq+lenq ]
|
|
mulpd m2, m0, [srcq+lenq+mmsize]
|
|
movaps [dstq+lenq ], m1
|
|
movaps [dstq+lenq+mmsize], m2
|
|
sub lenq, 2*mmsize
|
|
jge .loop
|
|
REP_RET
|
|
%endmacro
|
|
|
|
INIT_XMM sse2
|
|
VECTOR_DMUL_SCALAR
|
|
%if HAVE_AVX_EXTERNAL
|
|
INIT_YMM avx
|
|
VECTOR_DMUL_SCALAR
|
|
%endif
|
|
|
|
;-----------------------------------------------------------------------------
|
|
; vector_fmul_window(float *dst, const float *src0,
|
|
; const float *src1, const float *win, int len);
|
|
;-----------------------------------------------------------------------------
|
|
INIT_XMM sse
|
|
cglobal vector_fmul_window, 5, 6, 6, dst, src0, src1, win, len, len1
|
|
shl lend, 2
|
|
lea len1q, [lenq - mmsize]
|
|
add src0q, lenq
|
|
add dstq, lenq
|
|
add winq, lenq
|
|
neg lenq
|
|
.loop:
|
|
mova m0, [winq + lenq]
|
|
mova m4, [src0q + lenq]
|
|
mova m1, [winq + len1q]
|
|
mova m5, [src1q + len1q]
|
|
shufps m1, m1, 0x1b
|
|
shufps m5, m5, 0x1b
|
|
mova m2, m0
|
|
mova m3, m1
|
|
mulps m2, m4
|
|
mulps m3, m5
|
|
mulps m1, m4
|
|
mulps m0, m5
|
|
addps m2, m3
|
|
subps m1, m0
|
|
shufps m2, m2, 0x1b
|
|
mova [dstq + lenq], m1
|
|
mova [dstq + len1q], m2
|
|
sub len1q, mmsize
|
|
add lenq, mmsize
|
|
jl .loop
|
|
REP_RET
|
|
|
|
;-----------------------------------------------------------------------------
|
|
; vector_fmul_add(float *dst, const float *src0, const float *src1,
|
|
; const float *src2, int len)
|
|
;-----------------------------------------------------------------------------
|
|
%macro VECTOR_FMUL_ADD 0
|
|
cglobal vector_fmul_add, 5,5,4, dst, src0, src1, src2, len
|
|
lea lenq, [lend*4 - 2*mmsize]
|
|
ALIGN 16
|
|
.loop:
|
|
mova m0, [src0q + lenq]
|
|
mova m1, [src0q + lenq + mmsize]
|
|
%if cpuflag(fma3)
|
|
mova m2, [src2q + lenq]
|
|
mova m3, [src2q + lenq + mmsize]
|
|
fmaddps m0, m0, [src1q + lenq], m2
|
|
fmaddps m1, m1, [src1q + lenq + mmsize], m3
|
|
%else
|
|
mulps m0, m0, [src1q + lenq]
|
|
mulps m1, m1, [src1q + lenq + mmsize]
|
|
addps m0, m0, [src2q + lenq]
|
|
addps m1, m1, [src2q + lenq + mmsize]
|
|
%endif
|
|
mova [dstq + lenq], m0
|
|
mova [dstq + lenq + mmsize], m1
|
|
|
|
sub lenq, 2*mmsize
|
|
jge .loop
|
|
REP_RET
|
|
%endmacro
|
|
|
|
INIT_XMM sse
|
|
VECTOR_FMUL_ADD
|
|
%if HAVE_AVX_EXTERNAL
|
|
INIT_YMM avx
|
|
VECTOR_FMUL_ADD
|
|
%endif
|
|
%if HAVE_FMA3_EXTERNAL
|
|
INIT_YMM fma3
|
|
VECTOR_FMUL_ADD
|
|
%endif
|
|
|
|
;-----------------------------------------------------------------------------
|
|
; void vector_fmul_reverse(float *dst, const float *src0, const float *src1,
|
|
; int len)
|
|
;-----------------------------------------------------------------------------
|
|
%macro VECTOR_FMUL_REVERSE 0
|
|
cglobal vector_fmul_reverse, 4,4,2, dst, src0, src1, len
|
|
%if cpuflag(avx2)
|
|
movaps m2, [pd_reverse]
|
|
%endif
|
|
lea lenq, [lend*4 - 2*mmsize]
|
|
ALIGN 16
|
|
.loop:
|
|
%if cpuflag(avx2)
|
|
vpermps m0, m2, [src1q]
|
|
vpermps m1, m2, [src1q+mmsize]
|
|
%elif cpuflag(avx)
|
|
vmovaps xmm0, [src1q + 16]
|
|
vinsertf128 m0, m0, [src1q], 1
|
|
vshufps m0, m0, m0, q0123
|
|
vmovaps xmm1, [src1q + mmsize + 16]
|
|
vinsertf128 m1, m1, [src1q + mmsize], 1
|
|
vshufps m1, m1, m1, q0123
|
|
%else
|
|
mova m0, [src1q]
|
|
mova m1, [src1q + mmsize]
|
|
shufps m0, m0, q0123
|
|
shufps m1, m1, q0123
|
|
%endif
|
|
mulps m0, m0, [src0q + lenq + mmsize]
|
|
mulps m1, m1, [src0q + lenq]
|
|
movaps [dstq + lenq + mmsize], m0
|
|
movaps [dstq + lenq], m1
|
|
add src1q, 2*mmsize
|
|
sub lenq, 2*mmsize
|
|
jge .loop
|
|
REP_RET
|
|
%endmacro
|
|
|
|
INIT_XMM sse
|
|
VECTOR_FMUL_REVERSE
|
|
%if HAVE_AVX_EXTERNAL
|
|
INIT_YMM avx
|
|
VECTOR_FMUL_REVERSE
|
|
%endif
|
|
%if HAVE_AVX2_EXTERNAL
|
|
INIT_YMM avx2
|
|
VECTOR_FMUL_REVERSE
|
|
%endif
|
|
|
|
; float scalarproduct_float_sse(const float *v1, const float *v2, int len)
|
|
INIT_XMM sse
|
|
cglobal scalarproduct_float, 3,3,2, v1, v2, offset
|
|
shl offsetd, 2
|
|
add v1q, offsetq
|
|
add v2q, offsetq
|
|
neg offsetq
|
|
xorps xmm0, xmm0
|
|
.loop:
|
|
movaps xmm1, [v1q+offsetq]
|
|
mulps xmm1, [v2q+offsetq]
|
|
addps xmm0, xmm1
|
|
add offsetq, 16
|
|
js .loop
|
|
movhlps xmm1, xmm0
|
|
addps xmm0, xmm1
|
|
movss xmm1, xmm0
|
|
shufps xmm0, xmm0, 1
|
|
addss xmm0, xmm1
|
|
%if ARCH_X86_64 == 0
|
|
movss r0m, xmm0
|
|
fld dword r0m
|
|
%endif
|
|
RET
|
|
|
|
;-----------------------------------------------------------------------------
|
|
; void ff_butterflies_float(float *src0, float *src1, int len);
|
|
;-----------------------------------------------------------------------------
|
|
INIT_XMM sse
|
|
cglobal butterflies_float, 3,3,3, src0, src1, len
|
|
shl lend, 2
|
|
add src0q, lenq
|
|
add src1q, lenq
|
|
neg lenq
|
|
.loop:
|
|
mova m0, [src0q + lenq]
|
|
mova m1, [src1q + lenq]
|
|
subps m2, m0, m1
|
|
addps m0, m0, m1
|
|
mova [src1q + lenq], m2
|
|
mova [src0q + lenq], m0
|
|
add lenq, mmsize
|
|
jl .loop
|
|
REP_RET
|