avfilter/x86/scene_sad: add high bit depth AVX2/AVX512 version

Since psadbw only exists for 8-bits, we have to emulate it for 16-bit
inputs. The simplest sequence is to use a normal subtraction, which is safe
as long as the inputs do not exceed 32767 - so limit this implementation
to 15-bit inputs and below.

For 16-bit inputs, we could in theory instead use a pminw / pmaxw to ensure
the resulting difference does not overflow, but this is slower, and also
breaks the subsequent use of pmaddwd, so I opted to skip 16-bit SIMD for
now.

scene_sad10_c:                                      114175.6 ( 1.00x)
scene_sad10_avx2:                                     9617.7 (11.87x)
scene_sad10_avx512:                                   5208.8 (21.92x)
scene_sad12_c:                                      114537.8 ( 1.00x)
scene_sad12_avx2:                                     9614.0 (11.91x)
scene_sad12_avx512:                                   5186.3 (22.08x)
scene_sad14_c:                                      114113.9 ( 1.00x)
scene_sad14_avx2:                                     9612.9 (11.87x)
scene_sad14_avx512:                                   5186.0 (22.00x)
scene_sad15_c:                                      114108.9 ( 1.00x)
scene_sad15_avx2:                                     9612.3 (11.87x)
scene_sad15_avx512:                                   5186.4 (22.00x)
scene_sad16_c:                                      114136.0 ( 1.00x)

libavfilter/x86/scene_sad.asm

@@ -24,25 +24,47 @@
 
 %include "libavutil/x86/x86util.asm"
 
+SECTION_RODATA
+
+pw_1: times 32 dw 1
+
 SECTION .text
 
 
-%macro SAD_INIT 0
-cglobal scene_sad, 6, 7, 2, src1, stride1, src2, stride2, width, end, x
+%macro SAD_INIT 1 ; depth
+cglobal scene_sad%1, 6, 7, 3, src1, stride1, src2, stride2, width, end, x
     add     src1q, widthq
     add     src2q, widthq
     neg    widthq
     pxor       m1, m1
 %endmacro
 
+%macro PSADQ 4 ; depth, dst, [src2], tmp
+%if %1 == 8
+        psadbw %2, %3
+%else
+        psubw     %2, %3
+        pabsw     %2, %2
+        pmaddwd   %2, [pw_1]
+    %if mmsize == 64
+        vextracti32x8 ymm%4, %2, 1
+        paddd     ymm%2, ymm%4
+        pmovzxdq  %2, ymm%2
+    %else
+        vextracti128 xmm%4, %2, 1
+        paddd     xmm%2, xmm%4
+        pmovzxdq  %2, xmm%2
+    %endif
+%endif
+%endmacro
 
-%macro SAD_LOOP 0
+%macro SAD_LOOP 1 ; depth
 .nextrow:
     mov        xq, widthq
 
     .loop:
         movu            m0, [src1q + xq]
-        psadbw          m0, [src2q + xq]
+        PSADQ %1,       m0, [src2q + xq], m2
         paddq           m1, m0
         add             xq, mmsize
     jl .loop
@@ -57,25 +79,27 @@ RET
 %endmacro
 
 
-%macro SAD_FRAMES 0
-    SAD_INIT
-    SAD_LOOP
+%macro SAD_FRAMES 1 ; depth
+    SAD_INIT %1
+    SAD_LOOP %1
 %endmacro
 
 
 INIT_XMM sse2
-SAD_FRAMES
+SAD_FRAMES 8
 
 %if HAVE_AVX2_EXTERNAL
 
 INIT_YMM avx2
-SAD_FRAMES
+SAD_FRAMES 8
+SAD_FRAMES 16
 
 %endif
 
 %if HAVE_AVX512_EXTERNAL
 
 INIT_ZMM avx512
-SAD_FRAMES
+SAD_FRAMES 8
+SAD_FRAMES 16
 
 %endif

libavfilter/x86/scene_sad_init.c

@@ -36,13 +36,31 @@ static void FUNC_NAME(SCENE_SAD_PARAMS) {                                     \
     *sum += sad[0];                                                           \
 }
 
+#define SCENE_SAD16_FUNC(FUNC_NAME, ASM_FUNC_NAME, MMSIZE)                    \
+void ASM_FUNC_NAME(SCENE_SAD_PARAMS);                                         \
+                                                                              \
+static void FUNC_NAME(SCENE_SAD_PARAMS) {                                     \
+    uint64_t sad[MMSIZE / 8] = {0};                                           \
+    ptrdiff_t bytes = (width << 1) & ~(MMSIZE - 1);                           \
+    *sum = 0;                                                                 \
+    ASM_FUNC_NAME(src1, stride1, src2, stride2, bytes, height, sad);          \
+    for (int i = 0; i < MMSIZE / 8; i++)                                      \
+        *sum += sad[i];                                                       \
+    ff_scene_sad16_c(src1 + bytes, stride1,                                   \
+                     src2 + bytes, stride2,                                   \
+                     width - (bytes >> 1), height, sad);                      \
+    *sum += sad[0];                                                           \
+}
+
 #if HAVE_X86ASM
-SCENE_SAD_FUNC(scene_sad_sse2, ff_scene_sad_sse2, 16)
+SCENE_SAD_FUNC(scene_sad_sse2, ff_scene_sad8_sse2, 16)
 #if HAVE_AVX2_EXTERNAL
-SCENE_SAD_FUNC(scene_sad_avx2, ff_scene_sad_avx2, 32)
+SCENE_SAD_FUNC(scene_sad_avx2,     ff_scene_sad8_avx2,  32)
+SCENE_SAD16_FUNC(scene_sad16_avx2, ff_scene_sad16_avx2, 32)
 #endif
 #if HAVE_AVX512_EXTERNAL
-SCENE_SAD_FUNC(scene_sad_avx512, ff_scene_sad_avx512, 64)
+SCENE_SAD_FUNC(scene_sad_avx512,     ff_scene_sad8_avx512,  64)
+SCENE_SAD16_FUNC(scene_sad16_avx512, ff_scene_sad16_avx512, 64)
 #endif
 #endif
 
@@ -61,6 +79,15 @@ ff_scene_sad_fn ff_scene_sad_get_fn_x86(int depth)
 #endif
         if (EXTERNAL_SSE2(cpu_flags))
             return scene_sad_sse2;
+    } else if (depth < 16) { /* this routine is only safe up to 15 bits */
+#if HAVE_AVX512_EXTERNAL
+        if (EXTERNAL_AVX512(cpu_flags))
+            return scene_sad16_avx512;
+#endif
+#if HAVE_AVX2_EXTERNAL
+        if (EXTERNAL_AVX2_FAST(cpu_flags))
+            return scene_sad16_avx2;
+#endif
     }
 #endif
     return NULL;