mirror of
https://github.com/FFmpeg/FFmpeg.git
synced 2024-12-28 20:53:54 +02:00
x86/vvcdec: inter, add optical flow avx2 code
BDoF used about 10%–25% of the CPU for some clips. Here are the FPS for one run; please ignore the negative values, as they may be due to round-to-round variation clips | before | after | delta --------------------------------------------|--------|-------|------ RitualDance_1920x1080_60_10_420_37_RA.266 | 310.0 | 363.0 | 14.60% NovosobornayaSquare_1920x1080.bin | 322.3 | 339.7 | 5.12% Tango2_3840x2160_60_10_420_27_LD.266 | 71.0 | 68.7 | -3.35% RitualDance_1920x1080_60_10_420_32_LD.266 | 250.0 | 245.3 | -1.92% Chimera_8bit_1080P_1000_frames.vvc | 359.3 | 422.7 | 15.00% BQTerrace_1920x1080_60_10_420_22_RA.vvc | 142.3 | 147.7 | 3.66% Reviewed-by: James Almer <jamrial@gmail.com>
This commit is contained in:
Nuo Mi
parent
f851abb4b3
commit
15eb10c6de
@ -6,5 +6,6 @@ OBJS-$(CONFIG_VVC_DECODER) += x86/vvc/vvcdsp_init.o \
|
||||
X86ASM-OBJS-$(CONFIG_VVC_DECODER) += x86/vvc/vvc_alf.o \
|
||||
x86/vvc/vvc_dmvr.o \
|
||||
x86/vvc/vvc_mc.o \
|
||||
x86/vvc/vvc_of.o \
|
||||
x86/vvc/vvc_sad.o \
|
||||
x86/h26x/h2656_inter.o
|
||||
|
||||
385
libavcodec/x86/vvc/vvc_of.asm
Normal file
385
libavcodec/x86/vvc/vvc_of.asm
Normal file
@ -0,0 +1,385 @@
|
||||
; /*
|
||||
; * Provide AVX2 luma optical flow functions for VVC decoding
|
||||
; * Copyright (c) 2024 Nuo Mi
|
||||
; *
|
||||
; * This file is part of FFmpeg.
|
||||
; *
|
||||
; * FFmpeg is free software; you can redistribute it and/or
|
||||
; * modify it under the terms of the GNU Lesser General Public
|
||||
; * License as published by the Free Software Foundation; either
|
||||
; * version 2.1 of the License, or (at your option) any later version.
|
||||
; *
|
||||
; * FFmpeg is distributed in the hope that it will be useful,
|
||||
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
; * Lesser General Public License for more details.
|
||||
; *
|
||||
; * You should have received a copy of the GNU Lesser General Public
|
||||
; * License along with FFmpeg; if not, write to the Free Software
|
||||
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
; */
|
||||
%include "libavutil/x86/x86util.asm"
|
||||
|
||||
%define MAX_PB_SIZE 128
|
||||
%define SRC_STRIDE (MAX_PB_SIZE * 2)
|
||||
%define SRC_PS 2 ; source pixel size, sizeof(int16_t)
|
||||
%define BDOF_STACK_SIZE 10 ; (4 + 1) * 2, 4 lines + the first line, *2 for h and v
|
||||
%define bdof_stack_offset(line) ((line) * 2 % BDOF_STACK_SIZE * mmsize)
|
||||
%define SHIFT 6
|
||||
%define SHIFT2 4
|
||||
|
||||
SECTION_RODATA 32
|
||||
pd_15 times 8 dd 15
|
||||
pd_m15 times 8 dd -15
|
||||
|
||||
pb_shuffle_w8 times 2 db 0, 1, 0xff, 0xff, 8, 9, 0xff, 0xff, 6, 7, 0xff, 0xff, 14, 15, 0xff, 0xff
|
||||
pb_shuffle_w16 times 2 db 0, 1, 0xff, 0xff, 6, 7, 0xff, 0xff, 8, 9, 0xff, 0xff, 14, 15, 0xff, 0xff
|
||||
pd_perm_w16 dd 0, 2, 1, 4, 3, 6, 5, 7
|
||||
%if ARCH_X86_64
|
||||
|
||||
%if HAVE_AVX2_EXTERNAL
|
||||
|
||||
SECTION .text
|
||||
|
||||
INIT_YMM avx2
|
||||
|
||||
; dst = (src0 >> shift) - (src1 >> shift)
|
||||
%macro DIFF 5 ; dst, src0, src1, shift, tmp
|
||||
psraw %1, %2, %4
|
||||
psraw %5, %3, %4
|
||||
psubw %1, %5
|
||||
%endmacro
|
||||
|
||||
%macro LOAD_GRAD_H 4 ; dst, src, off, tmp
|
||||
movu %1, [%2 + %3 + 2 * SRC_PS]
|
||||
movu %4, [%2 + %3]
|
||||
|
||||
DIFF %1, %1, %4, SHIFT, %4
|
||||
%endmacro
|
||||
|
||||
%macro SUM_GRAD 2 ;(dst/grad0, grad1)
|
||||
paddw %1, %2
|
||||
psraw %1, 1 ; shift3
|
||||
%endmacro
|
||||
|
||||
%macro APPLY_BDOF_MIN_BLOCK_LINE 5 ; dst, vx, vy, tmp, line_num
|
||||
%define off bdof_stack_offset(%5)
|
||||
pmullw %1, %2, [rsp + off + 0 * mmsize] ; vx * (gradient_h[0] - gradient_h[1])
|
||||
pmullw %4, %3, [rsp + off + 1 * mmsize] ; vy * (gradient_v[0] - gradient_v[1])
|
||||
paddw %1, [src0q + (%5 + 1) * SRC_STRIDE + SRC_PS]
|
||||
paddw %4, [src1q + (%5 + 1) * SRC_STRIDE + SRC_PS]
|
||||
paddsw %1, %4 ; src0[x] + src1[x] + bdof_offset
|
||||
pmulhrsw %1, m11
|
||||
CLIPW %1, m9, m10
|
||||
%endmacro
|
||||
|
||||
%macro SAVE_8BPC 2 ; dst, src
|
||||
packuswb m%2, m%2
|
||||
vpermq m%2, m%2, q0020
|
||||
|
||||
cmp wd, 16
|
||||
je %%w16
|
||||
movq %1, xm%2
|
||||
jmp %%wend
|
||||
%%w16:
|
||||
movu %1, xm%2
|
||||
%%wend:
|
||||
%endmacro
|
||||
|
||||
%macro SAVE_16BPC 2 ; dst, src
|
||||
cmp wd, 16
|
||||
je %%w16
|
||||
movu %1, xm%2
|
||||
jmp %%wend
|
||||
%%w16:
|
||||
movu %1, m%2
|
||||
%%wend:
|
||||
%endmacro
|
||||
|
||||
%macro SAVE 2 ; dst, src
|
||||
cmp pixel_maxd, (1 << 8) - 1
|
||||
jne %%save_16bpc
|
||||
SAVE_8BPC %1, %2
|
||||
jmp %%end
|
||||
%%save_16bpc:
|
||||
SAVE_16BPC %1, %2
|
||||
%%end:
|
||||
%endmacro
|
||||
|
||||
; [rsp + even * mmsize] are gradient_h[0] - gradient_h[1]
|
||||
; [rsp + odd * mmsize] are gradient_v[0] - gradient_v[1]
|
||||
%macro APPLY_BDOF_MIN_BLOCK 4 ; block_num, vx, vy, bd
|
||||
pxor m9, m9
|
||||
|
||||
movd xm10, pixel_maxd
|
||||
vpbroadcastw m10, xm10
|
||||
|
||||
lea tmp0d, [pixel_maxd + 1]
|
||||
movd xm11, tmp0d
|
||||
VPBROADCASTW m11, xm11 ;shift_4 for pmulhrsw
|
||||
|
||||
APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 0
|
||||
SAVE [dstq + 0 * dsq], 6
|
||||
|
||||
APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 1
|
||||
SAVE [dstq + 1 * dsq], 6
|
||||
|
||||
APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 2
|
||||
SAVE [dstq + 2 * dsq], 6
|
||||
|
||||
APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 3
|
||||
SAVE [dstq + ds3q], 6
|
||||
%endmacro
|
||||
|
||||
%macro SUM_MIN_BLOCK_W16 4 ; src/dst, shuffle, perm, tmp
|
||||
pshufb %4, %1, %2
|
||||
vpermd %4, %3, %4
|
||||
paddw %1, %4
|
||||
%endmacro
|
||||
|
||||
%macro SUM_MIN_BLOCK_W8 3 ; src/dst, shuffle, tmp
|
||||
pshufb %3, %1, %2
|
||||
paddw %1, %3
|
||||
%endmacro
|
||||
|
||||
%macro BDOF_PROF_GRAD 2 ; line_no, last_line
|
||||
%assign i0 (%1 + 0) % 3
|
||||
%assign j0 (%1 + 1) % 3
|
||||
%assign k0 (%1 + 2) % 3
|
||||
%assign i1 3 + (%1 + 0) % 3
|
||||
%assign j1 3 + (%1 + 1) % 3
|
||||
%assign k1 3 + (%1 + 2) % 3
|
||||
|
||||
; we cached src0 in m0 to m2
|
||||
%define t0 m %+ i0
|
||||
%define c0 m %+ j0
|
||||
%define b0 m %+ k0
|
||||
|
||||
; we cached src1 in m3 to m5
|
||||
%define t1 m %+ i1
|
||||
%define c1 m %+ j1
|
||||
%define b1 m %+ k1
|
||||
%define ndiff t1
|
||||
%define off bdof_stack_offset(%1)
|
||||
|
||||
movu b0, [src0q + (%1 + 2) * SRC_STRIDE + SRC_PS]
|
||||
movu b1, [src1q + (%1 + 2) * SRC_STRIDE + SRC_PS]
|
||||
|
||||
; gradient_v[0], gradient_v[1]
|
||||
DIFF m6, b0, t0, SHIFT, t0
|
||||
DIFF m7, b1, t1, SHIFT, t1
|
||||
|
||||
; save gradient_v[0] - gradient_v[1]
|
||||
psubw m10, m6, m7
|
||||
mova [rsp + off + mmsize], m10
|
||||
|
||||
; gradient_h[0], gradient_h[1]
|
||||
LOAD_GRAD_H m8, src0q, (%1 + 1) * SRC_STRIDE, t0
|
||||
LOAD_GRAD_H m9, src1q, (%1 + 1) * SRC_STRIDE, t1
|
||||
|
||||
; save gradient_h[0] - gradient_h[1]
|
||||
psubw m11, m8, m9
|
||||
mova [rsp + off], m11
|
||||
|
||||
SUM_GRAD m8, m9 ; temph
|
||||
SUM_GRAD m6, m7 ; tempv
|
||||
|
||||
DIFF ndiff, c1, c0, SHIFT2, t0 ; -diff
|
||||
|
||||
psignw m7, ndiff, m8 ; sgxdi
|
||||
psignw m9, ndiff, m6 ; sgydi
|
||||
psignw m10, m8, m6 ; sgxgy
|
||||
|
||||
pabsw m6, m6 ; sgy2
|
||||
pabsw m8, m8 ; sgx2
|
||||
|
||||
; use t0, t1 as temporary buffers
|
||||
cmp wd, 16
|
||||
|
||||
je %%w16
|
||||
mova t0, [pb_shuffle_w8]
|
||||
SUM_MIN_BLOCK_W8 m6, t0, m11
|
||||
SUM_MIN_BLOCK_W8 m7, t0, m11
|
||||
SUM_MIN_BLOCK_W8 m8, t0, m11
|
||||
SUM_MIN_BLOCK_W8 m9, t0, m11
|
||||
SUM_MIN_BLOCK_W8 m10, t0, m11
|
||||
jmp %%wend
|
||||
|
||||
%%w16:
|
||||
mova t0, [pb_shuffle_w16]
|
||||
mova t1, [pd_perm_w16]
|
||||
SUM_MIN_BLOCK_W16 m6, t0, t1, m11
|
||||
SUM_MIN_BLOCK_W16 m7, t0, t1, m11
|
||||
SUM_MIN_BLOCK_W16 m8, t0, t1, m11
|
||||
SUM_MIN_BLOCK_W16 m9, t0, t1, m11
|
||||
SUM_MIN_BLOCK_W16 m10, t0, t1, m11
|
||||
|
||||
%%wend:
|
||||
vpblendd m11, m8, m7, 10101010b
|
||||
vpblendd m7, m8, m7, 01010101b
|
||||
pshufd m7, m7, q2301
|
||||
paddw m8, m7, m11 ;4 x (2sgx2, 2sgxdi)
|
||||
|
||||
vpblendd m11, m6, m9, 10101010b
|
||||
vpblendd m9, m6, m9, 01010101b
|
||||
pshufd m9, m9, q2301
|
||||
paddw m6, m9, m11 ;4 x (2sgy2, 2sgydi)
|
||||
|
||||
vpblendw m11, m8, m6, 10101010b
|
||||
vpblendw m6, m8, m6, 01010101b
|
||||
pshuflw m6, m6, q2301
|
||||
pshufhw m6, m6, q2301
|
||||
paddw m8, m6, m11 ; 4 x (4sgx2, 4sgy2, 4sgxdi, 4sgydi)
|
||||
|
||||
%if (%1) == 0 || (%2)
|
||||
; pad for top and bottom
|
||||
paddw m8, m8
|
||||
paddw m10, m10
|
||||
%endif
|
||||
|
||||
paddw m12, m8
|
||||
paddw m13, m10
|
||||
%endmacro
|
||||
|
||||
|
||||
%macro LOG2 5 ; log_sum, src, cmp, shift, tmp
|
||||
pcmpgtw %5, %2, %3
|
||||
pandd %5, %4
|
||||
paddw %1, %5
|
||||
|
||||
psrlw %2, %5
|
||||
psrlw %4, 1
|
||||
psrlw %3, %4
|
||||
%endmacro
|
||||
|
||||
%macro LOG2 2 ; dst/src, offset
|
||||
pextrw tmp0d, xm%1, %2
|
||||
bsr tmp0d, tmp0d
|
||||
pinsrw xm%1, tmp0d, %2
|
||||
%endmacro
|
||||
|
||||
%macro LOG2 1 ; dst/src
|
||||
LOG2 %1, 0
|
||||
LOG2 %1, 1
|
||||
LOG2 %1, 2
|
||||
LOG2 %1, 3
|
||||
LOG2 %1, 4
|
||||
LOG2 %1, 5
|
||||
LOG2 %1, 6
|
||||
LOG2 %1, 7
|
||||
%endmacro
|
||||
|
||||
; %1: 4 (sgx2, sgy2, sgxdi, gydi)
|
||||
; %2: 4 (4sgxgy)
|
||||
%macro BDOF_VX_VY 2 ;
|
||||
pshufd m6, m%1, q0032
|
||||
punpckldq m%1, m6
|
||||
vextracti128 xm7, m%1, 1
|
||||
|
||||
punpcklqdq m8, m%1, m7 ; 4 (sgx2, sgy2)
|
||||
punpckhqdq m9, m%1, m7 ; 4 (sgxdi, sgydi)
|
||||
mova m10, m8
|
||||
LOG2 10 ; 4 (log2(sgx2), log2(sgy2))
|
||||
|
||||
; Promote to dword since vpsrlvw is AVX-512 only
|
||||
pmovsxwd m8, xm8
|
||||
pmovsxwd m9, xm9
|
||||
pmovsxwd m10, xm10
|
||||
|
||||
pslld m9, 2 ; 4 (log2(sgx2) << 2, log2(sgy2) << 2)
|
||||
|
||||
psignd m11, m9, m8
|
||||
vpsravd m11, m11, m10
|
||||
CLIPD m11, [pd_m15], [pd_15] ; 4 (vx, junk)
|
||||
|
||||
pshuflw m%1, m11, q0000
|
||||
pshufhw m%1, m%1, q0000 ; 4 (2junk, 2vx)
|
||||
|
||||
psllq m6, m%2, 32
|
||||
paddw m%2, m6
|
||||
|
||||
pmaddwd m%2, m%1 ; 4 (junk, vx * sgxgy)
|
||||
psrad m%2, 1
|
||||
psubd m9, m%2 ; 4 (junk, (sgydi << 2) - (vx * sgxgy >> 1))
|
||||
|
||||
psignd m9, m8
|
||||
vpsravd m%2, m9, m10
|
||||
CLIPD m%2, [pd_m15], [pd_15] ; 4 (junk, vy)
|
||||
|
||||
pshuflw m%2, m%2, q2222
|
||||
pshufhw m%2, m%2, q2222 ; 4 (4vy)
|
||||
%endmacro
|
||||
|
||||
|
||||
%macro BDOF_MINI_BLOCKS 2 ; (block_num, last_block)
|
||||
|
||||
%if (%1) == 0
|
||||
movu m0, [src0q + 0 * SRC_STRIDE + SRC_PS]
|
||||
movu m1, [src0q + 1 * SRC_STRIDE + SRC_PS]
|
||||
movu m3, [src1q + 0 * SRC_STRIDE + SRC_PS]
|
||||
movu m4, [src1q + 1 * SRC_STRIDE + SRC_PS]
|
||||
|
||||
pxor m12, m12
|
||||
pxor m13, m13
|
||||
|
||||
BDOF_PROF_GRAD 0, 0
|
||||
%endif
|
||||
|
||||
mova m14, m12
|
||||
mova m15, m13
|
||||
|
||||
pxor m12, m12
|
||||
pxor m13, m13
|
||||
BDOF_PROF_GRAD %1 * 4 + 1, 0
|
||||
BDOF_PROF_GRAD %1 * 4 + 2, 0
|
||||
paddw m14, m12
|
||||
paddw m15, m13
|
||||
|
||||
pxor m12, m12
|
||||
pxor m13, m13
|
||||
BDOF_PROF_GRAD %1 * 4 + 3, %2
|
||||
%if (%2) == 0
|
||||
BDOF_PROF_GRAD %1 * 4 + 4, 0
|
||||
%endif
|
||||
paddw m14, m12
|
||||
paddw m15, m13
|
||||
|
||||
BDOF_VX_VY 14, 15
|
||||
APPLY_BDOF_MIN_BLOCK %1, m14, m15, bd
|
||||
lea dstq, [dstq + 4 * dsq]
|
||||
%endmacro
|
||||
|
||||
;void ff_vvc_apply_bdof_%1(uint8_t *dst, const ptrdiff_t dst_stride, int16_t *src0, int16_t *src1,
|
||||
; const int w, const int h, const int int pixel_max)
|
||||
%macro BDOF_AVX2 0
|
||||
cglobal vvc_apply_bdof, 7, 10, 16, BDOF_STACK_SIZE*32, dst, ds, src0, src1, w, h, pixel_max, ds3, tmp0, tmp1
|
||||
|
||||
lea ds3q, [dsq * 3]
|
||||
sub src0q, SRC_STRIDE + SRC_PS
|
||||
sub src1q, SRC_STRIDE + SRC_PS
|
||||
|
||||
BDOF_MINI_BLOCKS 0, 0
|
||||
|
||||
cmp hd, 16
|
||||
je .h16
|
||||
BDOF_MINI_BLOCKS 1, 1
|
||||
jmp .end
|
||||
|
||||
.h16:
|
||||
BDOF_MINI_BLOCKS 1, 0
|
||||
BDOF_MINI_BLOCKS 2, 0
|
||||
BDOF_MINI_BLOCKS 3, 1
|
||||
|
||||
.end:
|
||||
RET
|
||||
%endmacro
|
||||
|
||||
%macro VVC_OF_AVX2 0
|
||||
BDOF_AVX2
|
||||
%endmacro
|
||||
|
||||
VVC_OF_AVX2
|
||||
|
||||
%endif ; HAVE_AVX2_EXTERNAL
|
||||
|
||||
%endif ; ARCH_X86_64
|
||||
@ -102,6 +102,20 @@ DMVR_PROTOTYPES( 8, avx2)
|
||||
DMVR_PROTOTYPES(10, avx2)
|
||||
DMVR_PROTOTYPES(12, avx2)
|
||||
|
||||
void ff_vvc_apply_bdof_avx2(uint8_t *dst, ptrdiff_t dst_stride, \
|
||||
const int16_t *src0, const int16_t *src1, int w, int h, int pixel_max); \
|
||||
|
||||
#define OF_PROTOTYPES(bd, opt) \
|
||||
static void ff_vvc_apply_bdof_##bd##_##opt(uint8_t *dst, ptrdiff_t dst_stride, \
|
||||
const int16_t *src0, const int16_t *src1, int w, int h) \
|
||||
{ \
|
||||
ff_vvc_apply_bdof##_##opt(dst, dst_stride, src0, src1, w, h, (1 << bd) - 1); \
|
||||
} \
|
||||
|
||||
OF_PROTOTYPES( 8, avx2)
|
||||
OF_PROTOTYPES(10, avx2)
|
||||
OF_PROTOTYPES(12, avx2)
|
||||
|
||||
#define ALF_BPC_PROTOTYPES(bpc, opt) \
|
||||
void BF(ff_vvc_alf_filter_luma, bpc, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
|
||||
const uint8_t *src, ptrdiff_t src_stride, ptrdiff_t width, ptrdiff_t height, \
|
||||
@ -328,6 +342,10 @@ ALF_FUNCS(16, 12, avx2)
|
||||
c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_##bd##_avx2; \
|
||||
} while (0)
|
||||
|
||||
#define OF_INIT(bd) do { \
|
||||
c->inter.apply_bdof = ff_vvc_apply_bdof_##bd##_avx2; \
|
||||
} while (0)
|
||||
|
||||
#define ALF_INIT(bd) do { \
|
||||
c->alf.filter[LUMA] = ff_vvc_alf_filter_luma_##bd##_avx2; \
|
||||
c->alf.filter[CHROMA] = ff_vvc_alf_filter_chroma_##bd##_avx2; \
|
||||
@ -352,6 +370,7 @@ void ff_vvc_dsp_init_x86(VVCDSPContext *const c, const int bd)
|
||||
ALF_INIT(8);
|
||||
AVG_INIT(8, avx2);
|
||||
MC_LINKS_AVX2(8);
|
||||
OF_INIT(8);
|
||||
DMVR_INIT(8);
|
||||
SAD_INIT();
|
||||
}
|
||||
@ -365,6 +384,7 @@ void ff_vvc_dsp_init_x86(VVCDSPContext *const c, const int bd)
|
||||
AVG_INIT(10, avx2);
|
||||
MC_LINKS_AVX2(10);
|
||||
MC_LINKS_16BPC_AVX2(10);
|
||||
OF_INIT(10);
|
||||
DMVR_INIT(10);
|
||||
SAD_INIT();
|
||||
}
|
||||
@ -378,6 +398,7 @@ void ff_vvc_dsp_init_x86(VVCDSPContext *const c, const int bd)
|
||||
AVG_INIT(12, avx2);
|
||||
MC_LINKS_AVX2(12);
|
||||
MC_LINKS_16BPC_AVX2(12);
|
||||
OF_INIT(12);
|
||||
DMVR_INIT(12);
|
||||
SAD_INIT();
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user