diff --git a/libavfilter/x86/scene_sad.asm b/libavfilter/x86/scene_sad.asm index 2cd9dddb5c..181c290893 100644 --- a/libavfilter/x86/scene_sad.asm +++ b/libavfilter/x86/scene_sad.asm @@ -24,25 +24,47 @@ %include "libavutil/x86/x86util.asm" +SECTION_RODATA + +pw_1: times 32 dw 1 + SECTION .text -%macro SAD_INIT 0 -cglobal scene_sad, 6, 7, 2, src1, stride1, src2, stride2, width, end, x +%macro SAD_INIT 1 ; depth +cglobal scene_sad%1, 6, 7, 3, src1, stride1, src2, stride2, width, end, x add src1q, widthq add src2q, widthq neg widthq pxor m1, m1 %endmacro +%macro PSADQ 4 ; depth, dst, [src2], tmp +%if %1 == 8 + psadbw %2, %3 +%else + psubw %2, %3 + pabsw %2, %2 + pmaddwd %2, [pw_1] + %if mmsize == 64 + vextracti32x8 ymm%4, %2, 1 + paddd ymm%2, ymm%4 + pmovzxdq %2, ymm%2 + %else + vextracti128 xmm%4, %2, 1 + paddd xmm%2, xmm%4 + pmovzxdq %2, xmm%2 + %endif +%endif +%endmacro -%macro SAD_LOOP 0 +%macro SAD_LOOP 1 ; depth .nextrow: mov xq, widthq .loop: movu m0, [src1q + xq] - psadbw m0, [src2q + xq] + PSADQ %1, m0, [src2q + xq], m2 paddq m1, m0 add xq, mmsize jl .loop @@ -57,25 +79,27 @@ RET %endmacro -%macro SAD_FRAMES 0 - SAD_INIT - SAD_LOOP +%macro SAD_FRAMES 1 ; depth + SAD_INIT %1 + SAD_LOOP %1 %endmacro INIT_XMM sse2 -SAD_FRAMES +SAD_FRAMES 8 %if HAVE_AVX2_EXTERNAL INIT_YMM avx2 -SAD_FRAMES +SAD_FRAMES 8 +SAD_FRAMES 16 %endif %if HAVE_AVX512_EXTERNAL INIT_ZMM avx512 -SAD_FRAMES +SAD_FRAMES 8 +SAD_FRAMES 16 %endif diff --git a/libavfilter/x86/scene_sad_init.c b/libavfilter/x86/scene_sad_init.c index 2d631b376a..9863839b4e 100644 --- a/libavfilter/x86/scene_sad_init.c +++ b/libavfilter/x86/scene_sad_init.c @@ -36,13 +36,31 @@ static void FUNC_NAME(SCENE_SAD_PARAMS) { \ *sum += sad[0]; \ } +#define SCENE_SAD16_FUNC(FUNC_NAME, ASM_FUNC_NAME, MMSIZE) \ +void ASM_FUNC_NAME(SCENE_SAD_PARAMS); \ + \ +static void FUNC_NAME(SCENE_SAD_PARAMS) { \ + uint64_t sad[MMSIZE / 8] = {0}; \ + ptrdiff_t bytes = (width << 1) & ~(MMSIZE - 1); \ + *sum = 0; \ + ASM_FUNC_NAME(src1, stride1, src2, stride2, bytes, height, sad); \ + for (int i = 0; i < MMSIZE / 8; i++) \ + *sum += sad[i]; \ + ff_scene_sad16_c(src1 + bytes, stride1, \ + src2 + bytes, stride2, \ + width - (bytes >> 1), height, sad); \ + *sum += sad[0]; \ +} + #if HAVE_X86ASM -SCENE_SAD_FUNC(scene_sad_sse2, ff_scene_sad_sse2, 16) +SCENE_SAD_FUNC(scene_sad_sse2, ff_scene_sad8_sse2, 16) #if HAVE_AVX2_EXTERNAL -SCENE_SAD_FUNC(scene_sad_avx2, ff_scene_sad_avx2, 32) +SCENE_SAD_FUNC(scene_sad_avx2, ff_scene_sad8_avx2, 32) +SCENE_SAD16_FUNC(scene_sad16_avx2, ff_scene_sad16_avx2, 32) #endif #if HAVE_AVX512_EXTERNAL -SCENE_SAD_FUNC(scene_sad_avx512, ff_scene_sad_avx512, 64) +SCENE_SAD_FUNC(scene_sad_avx512, ff_scene_sad8_avx512, 64) +SCENE_SAD16_FUNC(scene_sad16_avx512, ff_scene_sad16_avx512, 64) #endif #endif @@ -61,6 +79,15 @@ ff_scene_sad_fn ff_scene_sad_get_fn_x86(int depth) #endif if (EXTERNAL_SSE2(cpu_flags)) return scene_sad_sse2; + } else if (depth < 16) { /* this routine is only safe up to 15 bits */ +#if HAVE_AVX512_EXTERNAL + if (EXTERNAL_AVX512(cpu_flags)) + return scene_sad16_avx512; +#endif +#if HAVE_AVX2_EXTERNAL + if (EXTERNAL_AVX2_FAST(cpu_flags)) + return scene_sad16_avx2; +#endif } #endif return NULL;