avfilter/vf_overlay: configure alpha mode on the link

And use the link-tagged value instead of the hard-coded parameter.

doc/filters.texi

@@ -18875,7 +18875,8 @@ See @ref{framesync}.
 
 @item alpha
 Set format of alpha of the overlaid video, it can be @var{straight} or
-@var{premultiplied}. Default is @var{straight}.
+@var{premultiplied}, or @var{auto} to choose the alpha mode automatically.
+Default is @var{auto}.
 @end table
 
 The @option{x}, and @option{y} expressions can contain the following

libavfilter/vf_overlay.c

@@ -233,6 +233,13 @@ static int query_formats(const AVFilterContext *ctx,
     AVFilterFormats *formats;
     int ret;
 
+    if (s->alpha_mode != AVALPHA_MODE_UNSPECIFIED) {
+        formats = ff_make_formats_list_singleton(s->alpha_mode);
+        ret = ff_formats_ref(formats, &cfg_in[OVERLAY]->alpha_modes);
+        if (ret < 0)
+            return ret;
+    }
+
     switch (s->format) {
     case OVERLAY_FORMAT_YUV420:
         main_formats    = main_pix_fmts_yuv420;
@@ -750,117 +757,128 @@ static int config_input_main(AVFilterLink *inlink)
     s->main_is_packed_rgb =
         ff_fill_rgba_map(s->main_rgba_map, inlink->format) >= 0;
     s->main_has_alpha = ff_fmt_is_in(inlink->format, alpha_pix_fmts);
-    switch (s->format) {
+    return 0;
-    case OVERLAY_FORMAT_YUV420:
+}
-        s->blend_slice = s->main_has_alpha ? blend_slice_yuva420 : blend_slice_yuv420;
+
-        break;
+static int init_slice_fn(AVFilterContext *ctx)
-    case OVERLAY_FORMAT_YUV420P10:
+{
-        s->blend_slice = s->main_has_alpha ? blend_slice_yuva420p10 : blend_slice_yuv420p10;
+    OverlayContext *s = ctx->priv;
-        break;
+    const AVFilterLink *main = ctx->inputs[MAIN];
-    case OVERLAY_FORMAT_YUV422:
+    const AVFilterLink *overlay = ctx->inputs[OVERLAY];
-        s->blend_slice = s->main_has_alpha ? blend_slice_yuva422 : blend_slice_yuv422;
+
-        break;
+    switch (overlay->alpha_mode) {
-    case OVERLAY_FORMAT_YUV422P10:
+    case AVALPHA_MODE_UNSPECIFIED:
-        s->blend_slice = s->main_has_alpha ? blend_slice_yuva422p10 : blend_slice_yuv422p10;
+    case AVALPHA_MODE_STRAIGHT:
-        break;
+        switch (s->format) {
-    case OVERLAY_FORMAT_YUV444:
+        case OVERLAY_FORMAT_YUV420:
-        s->blend_slice = s->main_has_alpha ? blend_slice_yuva444 : blend_slice_yuv444;
+            s->blend_slice = s->main_has_alpha ? blend_slice_yuva420 : blend_slice_yuv420;
-        break;
-    case OVERLAY_FORMAT_YUV444P10:
-        s->blend_slice = s->main_has_alpha ? blend_slice_yuva444p10 : blend_slice_yuv444p10;
-        break;
-    case OVERLAY_FORMAT_RGB:
-        s->blend_slice = s->main_has_alpha ? blend_slice_rgba : blend_slice_rgb;
-        break;
-    case OVERLAY_FORMAT_GBRP:
-        s->blend_slice = s->main_has_alpha ? blend_slice_gbrap : blend_slice_gbrp;
-        break;
-    case OVERLAY_FORMAT_AUTO:
-        switch (inlink->format) {
-        case AV_PIX_FMT_YUVA420P:
-            s->blend_slice = blend_slice_yuva420;
             break;
-        case AV_PIX_FMT_YUVA420P10:
+        case OVERLAY_FORMAT_YUV420P10:
-            s->blend_slice = blend_slice_yuva420p10;
+            s->blend_slice = s->main_has_alpha ? blend_slice_yuva420p10 : blend_slice_yuv420p10;
             break;
-        case AV_PIX_FMT_YUVA422P:
+        case OVERLAY_FORMAT_YUV422:
-            s->blend_slice = blend_slice_yuva422;
+            s->blend_slice = s->main_has_alpha ? blend_slice_yuva422 : blend_slice_yuv422;
             break;
-        case AV_PIX_FMT_YUVA422P10:
+        case OVERLAY_FORMAT_YUV422P10:
-            s->blend_slice = blend_slice_yuva422p10;
+            s->blend_slice = s->main_has_alpha ? blend_slice_yuva422p10 : blend_slice_yuv422p10;
             break;
-        case AV_PIX_FMT_YUVA444P:
+        case OVERLAY_FORMAT_YUV444:
-            s->blend_slice = blend_slice_yuva444;
+            s->blend_slice = s->main_has_alpha ? blend_slice_yuva444 : blend_slice_yuv444;
             break;
-        case AV_PIX_FMT_YUVA444P10:
+        case OVERLAY_FORMAT_YUV444P10:
-            s->blend_slice = blend_slice_yuva444p10;
+            s->blend_slice = s->main_has_alpha ? blend_slice_yuva444p10 : blend_slice_yuv444p10;
             break;
-        case AV_PIX_FMT_ARGB:
+        case OVERLAY_FORMAT_RGB:
-        case AV_PIX_FMT_RGBA:
+            s->blend_slice = s->main_has_alpha ? blend_slice_rgba : blend_slice_rgb;
-        case AV_PIX_FMT_BGRA:
-        case AV_PIX_FMT_ABGR:
-            s->blend_slice = blend_slice_rgba;
             break;
-        case AV_PIX_FMT_GBRAP:
+        case OVERLAY_FORMAT_GBRP:
-            s->blend_slice = blend_slice_gbrap;
+            s->blend_slice = s->main_has_alpha ? blend_slice_gbrap : blend_slice_gbrp;
             break;
-        default:
+        case OVERLAY_FORMAT_AUTO:
-            av_assert0(0);
+            switch (main->format) {
+            case AV_PIX_FMT_YUVA420P:
+                s->blend_slice = blend_slice_yuva420;
+                break;
+            case AV_PIX_FMT_YUVA420P10:
+                s->blend_slice = blend_slice_yuva420p10;
+                break;
+            case AV_PIX_FMT_YUVA422P:
+                s->blend_slice = blend_slice_yuva422;
+                break;
+            case AV_PIX_FMT_YUVA422P10:
+                s->blend_slice = blend_slice_yuva422p10;
+                break;
+            case AV_PIX_FMT_YUVA444P:
+                s->blend_slice = blend_slice_yuva444;
+                break;
+            case AV_PIX_FMT_YUVA444P10:
+                s->blend_slice = blend_slice_yuva444p10;
+                break;
+            case AV_PIX_FMT_ARGB:
+            case AV_PIX_FMT_RGBA:
+            case AV_PIX_FMT_BGRA:
+            case AV_PIX_FMT_ABGR:
+                s->blend_slice = blend_slice_rgba;
+                break;
+            case AV_PIX_FMT_GBRAP:
+                s->blend_slice = blend_slice_gbrap;
+                break;
+            default:
+                av_unreachable("Invalid pixel format for overlay");
+                break;
+            }
+            break;
+        }
+        break;
+
+    case AVALPHA_MODE_PREMULTIPLIED:
+        switch (s->format) {
+        case OVERLAY_FORMAT_YUV420:
+            s->blend_slice = s->main_has_alpha ? blend_slice_yuva420_pm : blend_slice_yuv420_pm;
+            break;
+        case OVERLAY_FORMAT_YUV422:
+            s->blend_slice = s->main_has_alpha ? blend_slice_yuva422_pm : blend_slice_yuv422_pm;
+            break;
+        case OVERLAY_FORMAT_YUV444:
+            s->blend_slice = s->main_has_alpha ? blend_slice_yuva444_pm : blend_slice_yuv444_pm;
+            break;
+        case OVERLAY_FORMAT_RGB:
+            s->blend_slice = s->main_has_alpha ? blend_slice_rgba_pm : blend_slice_rgb_pm;
+            break;
+        case OVERLAY_FORMAT_GBRP:
+            s->blend_slice = s->main_has_alpha ? blend_slice_gbrap_pm : blend_slice_gbrp_pm;
+            break;
+        case OVERLAY_FORMAT_AUTO:
+            switch (main->format) {
+            case AV_PIX_FMT_YUVA420P:
+                s->blend_slice = blend_slice_yuva420_pm;
+                break;
+            case AV_PIX_FMT_YUVA422P:
+                s->blend_slice = blend_slice_yuva422_pm;
+                break;
+            case AV_PIX_FMT_YUVA444P:
+                s->blend_slice = blend_slice_yuva444_pm;
+                break;
+            case AV_PIX_FMT_ARGB:
+            case AV_PIX_FMT_RGBA:
+            case AV_PIX_FMT_BGRA:
+            case AV_PIX_FMT_ABGR:
+                s->blend_slice = blend_slice_rgba_pm;
+                break;
+            case AV_PIX_FMT_GBRAP:
+                s->blend_slice = blend_slice_gbrap_pm;
+                break;
+            default:
+                av_unreachable("Invalid pixel format for overlay");
+                break;
+            }
             break;
         }
         break;
     }
 
-    if (!s->alpha_format)
-        goto end;
-
-    switch (s->format) {
-    case OVERLAY_FORMAT_YUV420:
-        s->blend_slice = s->main_has_alpha ? blend_slice_yuva420_pm : blend_slice_yuv420_pm;
-        break;
-    case OVERLAY_FORMAT_YUV422:
-        s->blend_slice = s->main_has_alpha ? blend_slice_yuva422_pm : blend_slice_yuv422_pm;
-        break;
-    case OVERLAY_FORMAT_YUV444:
-        s->blend_slice = s->main_has_alpha ? blend_slice_yuva444_pm : blend_slice_yuv444_pm;
-        break;
-    case OVERLAY_FORMAT_RGB:
-        s->blend_slice = s->main_has_alpha ? blend_slice_rgba_pm : blend_slice_rgb_pm;
-        break;
-    case OVERLAY_FORMAT_GBRP:
-        s->blend_slice = s->main_has_alpha ? blend_slice_gbrap_pm : blend_slice_gbrp_pm;
-        break;
-    case OVERLAY_FORMAT_AUTO:
-        switch (inlink->format) {
-        case AV_PIX_FMT_YUVA420P:
-            s->blend_slice = blend_slice_yuva420_pm;
-            break;
-        case AV_PIX_FMT_YUVA422P:
-            s->blend_slice = blend_slice_yuva422_pm;
-            break;
-        case AV_PIX_FMT_YUVA444P:
-            s->blend_slice = blend_slice_yuva444_pm;
-            break;
-        case AV_PIX_FMT_ARGB:
-        case AV_PIX_FMT_RGBA:
-        case AV_PIX_FMT_BGRA:
-        case AV_PIX_FMT_ABGR:
-            s->blend_slice = blend_slice_rgba_pm;
-            break;
-        case AV_PIX_FMT_GBRAP:
-            s->blend_slice = blend_slice_gbrap_pm;
-            break;
-        default:
-            av_assert0(0);
-            break;
-        }
-        break;
-    }
-
-end:
 #if ARCH_X86
-    ff_overlay_init_x86(s, s->format, inlink->format,
+    ff_overlay_init_x86(s, s->format, main->format, overlay->alpha_mode, s->main_has_alpha);
-                        s->alpha_format, s->main_has_alpha);
 #endif
 
     return 0;
@@ -903,6 +921,8 @@ static int do_blend(FFFrameSync *fs)
         s->y < mainpic->height && s->y + second->height >= 0) {
         ThreadData td;
 
+        init_slice_fn(ctx);
+
         td.dst = mainpic;
         td.src = second;
         ff_filter_execute(ctx, s->blend_slice, &td, NULL, FFMIN(FFMAX(1, FFMIN3(s->y + second->height, FFMIN(second->height, mainpic->height), mainpic->height - s->y)),
@@ -953,9 +973,11 @@ static const AVOption overlay_options[] = {
         { "gbrp",   "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_GBRP},   .flags = FLAGS, .unit = "format" },
         { "auto",   "", 0, AV_OPT_TYPE_CONST, {.i64=OVERLAY_FORMAT_AUTO},   .flags = FLAGS, .unit = "format" },
     { "repeatlast", "repeat overlay of the last overlay frame", OFFSET(fs.opt_repeatlast), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS },
-    { "alpha", "alpha format", OFFSET(alpha_format), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, FLAGS, .unit = "alpha_format" },
+    { "alpha", "alpha format", OFFSET(alpha_mode), AV_OPT_TYPE_INT, {.i64=AVALPHA_MODE_UNSPECIFIED}, 0, AVALPHA_MODE_NB-1, FLAGS, .unit = "alpha_mode" },
-        { "straight",      "", 0, AV_OPT_TYPE_CONST, {.i64=0}, .flags = FLAGS, .unit = "alpha_format" },
+        { "auto",          "", 0, AV_OPT_TYPE_CONST, {.i64=AVALPHA_MODE_UNSPECIFIED},   .flags = FLAGS, .unit = "alpha_mode" },
-        { "premultiplied", "", 0, AV_OPT_TYPE_CONST, {.i64=1}, .flags = FLAGS, .unit = "alpha_format" },
+        { "unknown",       "", 0, AV_OPT_TYPE_CONST, {.i64=AVALPHA_MODE_UNSPECIFIED},   .flags = FLAGS, .unit = "alpha_mode" },
+        { "straight",      "", 0, AV_OPT_TYPE_CONST, {.i64=AVALPHA_MODE_STRAIGHT},      .flags = FLAGS, .unit = "alpha_mode" },
+        { "premultiplied", "", 0, AV_OPT_TYPE_CONST, {.i64=AVALPHA_MODE_PREMULTIPLIED}, .flags = FLAGS, .unit = "alpha_mode" },
     { NULL }
 };
 

libavfilter/vf_overlay.h

@@ -62,7 +62,7 @@ typedef struct OverlayContext {
     uint8_t overlay_rgba_map[4];
     uint8_t overlay_has_alpha;
     int format;                 ///< OverlayFormat
-    int alpha_format;
+    int alpha_mode;
     int eval_mode;              ///< EvalMode
 
     FFFrameSync fs;
@@ -83,6 +83,6 @@ typedef struct OverlayContext {
 } OverlayContext;
 
 void ff_overlay_init_x86(OverlayContext *s, int format, int pix_format,
-                         int alpha_format, int main_has_alpha);
+                         enum AVAlphaMode, int main_has_alpha);
 
 #endif /* AVFILTER_OVERLAY_H */

libavfilter/x86/vf_overlay_init.c

@@ -33,14 +33,14 @@ int ff_overlay_row_22_sse4(uint8_t *d, uint8_t *da, uint8_t *s, uint8_t *a,
                            int w, ptrdiff_t alinesize);
 
 av_cold void ff_overlay_init_x86(OverlayContext *s, int format, int pix_format,
-                                 int alpha_format, int main_has_alpha)
+                                 enum AVAlphaMode alpha_mode, int main_has_alpha)
 {
     int cpu_flags = av_get_cpu_flags();
 
     if (EXTERNAL_SSE4(cpu_flags) &&
         (format == OVERLAY_FORMAT_YUV444 ||
          format == OVERLAY_FORMAT_GBRP) &&
-        alpha_format == 0 && main_has_alpha == 0) {
+        alpha_mode != AVALPHA_MODE_PREMULTIPLIED && main_has_alpha == 0) {
         s->blend_row[0] = ff_overlay_row_44_sse4;
         s->blend_row[1] = ff_overlay_row_44_sse4;
         s->blend_row[2] = ff_overlay_row_44_sse4;
@@ -49,7 +49,7 @@ av_cold void ff_overlay_init_x86(OverlayContext *s, int format, int pix_format,
     if (EXTERNAL_SSE4(cpu_flags) &&
         (pix_format == AV_PIX_FMT_YUV420P) &&
         (format == OVERLAY_FORMAT_YUV420) &&
-        alpha_format == 0 && main_has_alpha == 0) {
+        alpha_mode != AVALPHA_MODE_PREMULTIPLIED && main_has_alpha == 0) {
         s->blend_row[0] = ff_overlay_row_44_sse4;
         s->blend_row[1] = ff_overlay_row_20_sse4;
         s->blend_row[2] = ff_overlay_row_20_sse4;
@@ -57,7 +57,7 @@ av_cold void ff_overlay_init_x86(OverlayContext *s, int format, int pix_format,
 
     if (EXTERNAL_SSE4(cpu_flags) &&
         (format == OVERLAY_FORMAT_YUV422) &&
-        alpha_format == 0 && main_has_alpha == 0) {
+        alpha_mode != AVALPHA_MODE_PREMULTIPLIED && main_has_alpha == 0) {
         s->blend_row[0] = ff_overlay_row_44_sse4;
         s->blend_row[1] = ff_overlay_row_22_sse4;
         s->blend_row[2] = ff_overlay_row_22_sse4;