avfilter/vf_colordetect: detect fully opaque alpha planes

It can be useful to know if the alpha plane consists of fully opaque
pixels or not, in which case it can e.g. safely be stripped.

This only requires a very minor modification to the AVX2 routines, adding
an extra AND on the read alpha value with the reference alpha value, and a
single extra cheap test per line.

detect_alpha_8_full_c:                                2849.1 ( 1.00x)
detect_alpha_8_full_avx2:                              260.3 (10.95x)
detect_alpha_8_full_avx512icl:                         130.2 (21.87x)
detect_alpha_8_limited_c:                             8349.2 ( 1.00x)
detect_alpha_8_limited_avx2:                           756.6 (11.04x)
detect_alpha_8_limited_avx512icl:                      364.2 (22.93x)
detect_alpha_16_full_c:                               1652.8 ( 1.00x)
detect_alpha_16_full_avx2:                             236.5 ( 6.99x)
detect_alpha_16_full_avx512icl:                        134.6 (12.28x)
detect_alpha_16_limited_c:                            5263.1 ( 1.00x)
detect_alpha_16_limited_avx2:                          797.4 ( 6.60x)
detect_alpha_16_limited_avx512icl:                     400.3 (13.15x)

doc/filters.texi

@@ -9880,7 +9880,7 @@ which indicates that this is a full range YUV source.
 @item alpha_mode
 Detect if the source contains color values above the alpha channel, which
 indicates that the alpha channel is independent (straight), rather than
-premultiplied.
+premultiplied. Also detects if the alpha plane is fully opaque or not.
 @item all
 Enable detection of all of the above properties. This is the default.
 @end table

libavfilter/vf_colordetect.c

@@ -180,10 +180,9 @@ static int detect_alpha(AVFilterContext *ctx, void *arg,
         ret = s->dsp.detect_alpha(in->data[i] + y_start * stride, stride,
                                   alpha, alpha_stride, w, h_slice, alpha_max,
                                   mpeg_range, offset);
-        if (ret) {
-            atomic_store(&s->detected_alpha, ret);
+        ret |= atomic_fetch_or_explicit(&s->detected_alpha, ret, memory_order_relaxed);
+        if (ret == FF_ALPHA_STRAIGHT)
             break;
-        }
     }
 
     return 0;
@@ -197,7 +196,9 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
 
     if (s->mode & COLOR_DETECT_COLOR_RANGE && s->detected_range == AVCOL_RANGE_UNSPECIFIED)
         ff_filter_execute(ctx, detect_range, in, NULL, nb_threads);
-    if (s->mode & COLOR_DETECT_ALPHA_MODE && s->detected_alpha == FF_ALPHA_UNDETERMINED)
+
+    if (s->mode & COLOR_DETECT_ALPHA_MODE && s->detected_alpha != FF_ALPHA_NONE &&
+        s->detected_alpha != FF_ALPHA_STRAIGHT)
         ff_filter_execute(ctx, detect_alpha, in, NULL, nb_threads);
 
     return ff_filter_frame(inlink->dst->outputs[0], in);
@@ -218,9 +219,10 @@ static av_cold void uninit(AVFilterContext *ctx)
 
     if (s->mode & COLOR_DETECT_ALPHA_MODE) {
         av_log(ctx, AV_LOG_INFO, "  Alpha mode: %s\n",
-               s->detected_alpha == FF_ALPHA_NONE     ? "none" :
-               s->detected_alpha == FF_ALPHA_STRAIGHT ? "straight / independent"
-                                                      : "undertermined");
+               s->detected_alpha == FF_ALPHA_NONE        ? "none" :
+               s->detected_alpha == FF_ALPHA_STRAIGHT    ? "straight" :
+               s->detected_alpha == FF_ALPHA_TRANSPARENT ? "undetermined"
+                                                         : "opaque");
     }
 }
 

libavfilter/vf_colordetect.h

@@ -27,9 +27,10 @@
 #include <libavutil/pixfmt.h>
 
 enum FFAlphaDetect {
-    FF_ALPHA_NONE = -1,
+    FF_ALPHA_NONE         = -1,
     FF_ALPHA_UNDETERMINED = 0,
-    FF_ALPHA_STRAIGHT,
+    FF_ALPHA_TRANSPARENT  = 1 << 0, ///< alpha < alpha_max
+    FF_ALPHA_STRAIGHT     = (1 << 1) | FF_ALPHA_TRANSPARENT, ///< alpha < pixel
     /* No way to positively identify premultiplied alpha */
 };
 
@@ -113,16 +114,19 @@ ff_detect_alpha_full_c(const uint8_t *color, ptrdiff_t color_stride,
                        ptrdiff_t width, ptrdiff_t height,
                        int alpha_max, int mpeg_range, int offset)
 {
+    uint8_t transparent = 0;
     while (height--) {
-        uint8_t cond = 0;
-        for (int x = 0; x < width; x++)
-            cond |= color[x] > alpha[x];
-        if (cond)
+        uint8_t straight = 0;
+        for (int x = 0; x < width; x++) {
+            straight  |= color[x] > alpha[x];
+            transparent |= alpha[x] != alpha_max;
+        }
+        if (straight)
             return FF_ALPHA_STRAIGHT;
         color += color_stride;
         alpha += alpha_stride;
     }
-    return 0;
+    return transparent ? FF_ALPHA_TRANSPARENT : 0;
 }
 
 static inline int
@@ -131,16 +135,19 @@ ff_detect_alpha_limited_c(const uint8_t *color, ptrdiff_t color_stride,
                           ptrdiff_t width, ptrdiff_t height,
                           int alpha_max, int mpeg_range, int offset)
 {
+    uint8_t transparent = 0;
     while (height--) {
-        uint8_t cond = 0;
-        for (int x = 0; x < width; x++)
-            cond |= alpha_max * color[x] - offset > mpeg_range * alpha[x];
-        if (cond)
+        uint8_t straight = 0;
+        for (int x = 0; x < width; x++) {
+            straight  |= alpha_max * color[x] - offset > mpeg_range * alpha[x];
+            transparent |= alpha[x] != alpha_max;
+        }
+        if (straight)
             return FF_ALPHA_STRAIGHT;
         color += color_stride;
         alpha += alpha_stride;
     }
-    return 0;
+    return transparent ? FF_ALPHA_TRANSPARENT : 0;
 }
 
 static inline int
@@ -149,18 +156,21 @@ ff_detect_alpha16_full_c(const uint8_t *color, ptrdiff_t color_stride,
                          ptrdiff_t width, ptrdiff_t height,
                          int alpha_max, int mpeg_range, int offset)
 {
+    uint8_t transparent = 0;
     while (height--) {
         const uint16_t *color16 = (const uint16_t *) color;
         const uint16_t *alpha16 = (const uint16_t *) alpha;
-        uint8_t cond = 0;
-        for (int x = 0; x < width; x++)
-            cond |= color16[x] > alpha16[x];
-        if (cond)
+        uint8_t straight = 0;
+        for (int x = 0; x < width; x++) {
+            straight  |= color16[x] > alpha16[x];
+            transparent |= alpha16[x] != alpha_max;
+        }
+        if (straight)
             return FF_ALPHA_STRAIGHT;
         color += color_stride;
         alpha += alpha_stride;
     }
-    return 0;
+    return transparent ? FF_ALPHA_TRANSPARENT : 0;
 }
 
 static inline int
@@ -169,17 +179,19 @@ ff_detect_alpha16_limited_c(const uint8_t *color, ptrdiff_t color_stride,
                             ptrdiff_t width, ptrdiff_t height,
                             int alpha_max, int mpeg_range, int offset)
 {
+    uint8_t transparent = 0;
     while (height--) {
         const uint16_t *color16 = (const uint16_t *) color;
         const uint16_t *alpha16 = (const uint16_t *) alpha;
         for (int x = 0; x < width; x++) {
             if ((int64_t) alpha_max * color16[x] - offset > (int64_t) mpeg_range * alpha16[x])
                 return FF_ALPHA_STRAIGHT;
+            transparent |= alpha16[x] != alpha_max;
         }
         color += color_stride;
         alpha += alpha_stride;
     }
-    return 0;
+    return transparent ? FF_ALPHA_TRANSPARENT : 0;
 }
 
 #endif /* AVFILTER_COLORDETECT_H */

libavfilter/x86/vf_colordetect.asm

@@ -69,8 +69,19 @@ cglobal detect_range%1, 4, 7, 5, data, stride, width, height, mpeg_min, mpeg_max
     RET
 %endmacro
 
+%define FF_ALPHA_STRAIGHT 0x3
+
 %macro detect_alpha_fn 3 ; suffix, hsuffix, range
-cglobal detect_alpha%1_%3, 6, 7, 6, color, color_stride, alpha, alpha_stride, width, height, x
+%if ARCH_X86_64
+cglobal detect_alpha%1_%3, 6, 8, 7, color, color_stride, alpha, alpha_stride, width, height, ret, x
+%else
+cglobal detect_alpha%1_%3, 1, 6, 7, color, ret, alpha, x, width, height
+%define color_strideq r1m
+%define alpha_strideq r3m
+    mov alphad,  r2m
+    mov widthd,  r4m
+    mov heightd, r5m
+%endif
     pxor m0, m0
     add colorq, widthq
     add alphaq, widthq
@@ -79,17 +90,23 @@ cglobal detect_alpha%1_%3, 6, 7, 6, color, color_stride, alpha, alpha_stride, wi
     vpbroadcast%2 m3, r6m ; alpha_max
     vpbroadcast%2 m4, r7m ; mpeg_range
     vpbroadcast%2 m5, r8m ; offset
+%else
+    vpbroadcast%1 m3, r6m ; alpha_max
 %endif
+    mova m6, m3
+    xor retd, retd
 .lineloop:
     mov xq, widthq
     .loop:
     %ifidn %3, full
         movu m1, [colorq + xq]
         movu m2, [alphaq + xq]
+        pand m6, m2
         pmaxu%1 m1, m2
     %else
         pmovzx%1%2 m1, [colorq + xq]
         pmovzx%1%2 m2, [alphaq + xq]
+        pand m6, m2
         pmull%2 m1, m3
         pmull%2 m2, m4
     %ifidn %1, b
@@ -121,15 +138,28 @@ cglobal detect_alpha%1_%3, 6, 7, 6, color, color_stride, alpha, alpha_stride, wi
 %endif
     jnz .found
 
+%if cpuflag(avx512)
+    vpandnq m1, m6, m3 ; m1 = ~m6 & m3
+    vptestmq k1, m1, m1
+    kortestb k1, k1
+    setnz xb
+%else
+    ptest m6, m3
+    setnc xb ; CF = !(~m6 & m3)
+%endif
+    or retb, xb
+
     add colorq, color_strideq
     add alphaq, alpha_strideq
     dec heightq
     jg .lineloop
-    xor eax, eax
+%ifnidn retd, eax
+    mov eax, retd
+%endif
     RET
 
 .found:
-    mov eax, 1
+    mov eax, FF_ALPHA_STRAIGHT
     RET
 %endmacro
 

tests/checkasm/vf_colordetect.c

@@ -87,7 +87,7 @@ static void check_alpha_detect(int depth, enum AVColorRange range)
     LOCAL_ALIGNED_32(uint8_t, luma,  [HEIGHT * STRIDE]);
     LOCAL_ALIGNED_32(uint8_t, alpha, [HEIGHT * STRIDE]);
     memset(luma,  0x80, HEIGHT * STRIDE);
-    memset(alpha, 0xFF, HEIGHT * STRIDE);
+    memset(alpha, 0xF0, HEIGHT * STRIDE);
 
     /* Try and force overflow */
     if (depth > 8 && range == AVCOL_RANGE_MPEG) {