avfilter/avfiltergraph: allow different conversion filters per merger

Instead of hard-coding the assumption that all video properties are handled
by vf_scale, generalize the struct slightly to allow different conversion
filters for each property being merged. The avfiltergraph code creates a list
of conversion filters needed and inserts all of them at once.

Because a conversion filter might itself e.g. not support all formats,
it's possible that we need to go through the whole process of negotiating
formats multiple times, and keep adding conversion filters until all of them
are settled. Do this by simply jumping back to the beginning of the loop
to ensure that the `convertor_count` field remains contiguous.

libavfilter/avfiltergraph.c

@@ -508,7 +508,7 @@ static void print_filter_formats(void *log_ctx, int level, const AVFilterContext
  */
 static int query_formats(AVFilterGraph *graph, void *log_ctx)
 {
-    int i, j, ret;
+    int i, j, k, ret;
     int converter_count = 0;
     int count_queried = 0;        /* successful calls to query_formats() */
     int count_merged = 0;         /* successful merge of formats lists */
@@ -530,14 +530,16 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx)
     }
 
     /* go through and merge as many format lists as possible */
+retry:
     for (i = 0; i < graph->nb_filters; i++) {
         AVFilterContext *filter = graph->filters[i];
 
         for (j = 0; j < filter->nb_inputs; j++) {
             AVFilterLink *link = filter->inputs[j];
             const AVFilterNegotiation *neg;
-            unsigned neg_step;
-            int convert_needed = 0;
+            AVFilterContext *conv[4];
+            const char *conv_filters[4], *conv_opts[4] = {0};
+            unsigned neg_step, num_conv = 0;
 
             if (!link)
                 continue;
@@ -549,8 +551,17 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx)
                 void *a = FF_FIELD_AT(void *, m->offset, link->incfg);
                 void *b = FF_FIELD_AT(void *, m->offset, link->outcfg);
                 if (a && b && a != b && !m->can_merge(a, b)) {
-                    convert_needed = 1;
-                    break;
+                    for (k = 0; k < num_conv; k++) {
+                        if (conv_filters[k] == m->conversion_filter)
+                            break;
+                    }
+                    if (k == num_conv) {
+                        av_assert1(num_conv < FF_ARRAY_ELEMS(conv_filters));
+                        conv_filters[num_conv] = m->conversion_filter;
+                        if (m->conversion_opts_offset)
+                            conv_opts[num_conv] = FF_FIELD_AT(char *, m->conversion_opts_offset, *graph);
+                        num_conv++;
+                    }
                 }
             }
             for (neg_step = 0; neg_step < neg->nb_mergers; neg_step++) {
@@ -561,22 +572,28 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx)
                     count_delayed++;
                 } else if (a == b) {
                     count_already_merged++;
-                } else if (!convert_needed) {
+                } else if (!num_conv) {
                     count_merged++;
                     ret = m->merge(a, b);
                     if (ret < 0)
                         return ret;
-                    if (!ret)
-                        convert_needed = 1;
+                    if (!ret) {
+                        conv_filters[num_conv] = m->conversion_filter;
+                        if (m->conversion_opts_offset)
+                            conv_opts[num_conv] = FF_FIELD_AT(char *, m->conversion_opts_offset, *graph);
+                        num_conv++;
+                    }
                 }
             }
 
-            if (convert_needed) {
-                AVFilterContext *convert;
+            /**
+             * Couldn't merge format lists; auto-insert conversion filters
+             * in reverse order to keep the order consistent with the list
+             * of mergers, since they are prepended onto the existing link
+             */
+            for (k = num_conv - 1; k >= 0; k--) {
                 const AVFilter *filter;
-                AVFilterLink *inlink, *outlink;
                 char inst_name[30];
-                const char *opts;
 
                 if (fffiltergraph(graph)->disable_auto_convert) {
                     av_log(log_ctx, AV_LOG_ERROR,
@@ -587,28 +604,30 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx)
                     return AVERROR(EINVAL);
                 }
 
-                /* couldn't merge format lists. auto-insert conversion filter */
-                if (!(filter = avfilter_get_by_name(neg->conversion_filter))) {
+                if (!(filter = avfilter_get_by_name(conv_filters[k]))) {
                     av_log(log_ctx, AV_LOG_ERROR,
                            "'%s' filter not present, cannot convert formats.\n",
-                           neg->conversion_filter);
+                           conv_filters[k]);
                     print_link_formats(log_ctx, AV_LOG_ERROR, link);
                     return AVERROR(EINVAL);
                 }
                 snprintf(inst_name, sizeof(inst_name), "auto_%s_%d",
-                         neg->conversion_filter, converter_count++);
-                opts = FF_FIELD_AT(char *, neg->conversion_opts_offset, *graph);
-                ret = avfilter_graph_create_filter(&convert, filter, inst_name, opts, NULL, graph);
+                         conv_filters[k], converter_count++);
+                ret = avfilter_graph_create_filter(&conv[k], filter, inst_name,
+                                                   conv_opts[k], NULL, graph);
                 if (ret < 0)
                     return ret;
-                if ((ret = avfilter_insert_filter(link, convert, 0, 0)) < 0)
+                if ((ret = avfilter_insert_filter(link, conv[k], 0, 0)) < 0)
                     return ret;
 
-                if ((ret = filter_query_formats(convert)) < 0)
+                if ((ret = filter_query_formats(conv[k])) < 0)
                     return ret;
+            }
 
-                inlink  = convert->inputs[0];
-                outlink = convert->outputs[0];
+            /* preemptively settle formats of auto filters */
+            for (k = 0; k < num_conv; k++) {
+                AVFilterLink *inlink  = conv[k]->inputs[0];
+                AVFilterLink *outlink = conv[k]->outputs[0];
                 av_assert0( inlink->incfg.formats->refcount > 0);
                 av_assert0( inlink->outcfg.formats->refcount > 0);
                 av_assert0(outlink->incfg.formats->refcount > 0);
@@ -632,11 +651,15 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx)
                     av_assert0(outlink-> incfg.channel_layouts->refcount > 0);
                     av_assert0(outlink->outcfg.channel_layouts->refcount > 0);
                 }
+
 #define MERGE(merger, link)                                                  \
     ((merger)->merge(FF_FIELD_AT(void *, (merger)->offset, (link)->incfg),   \
                      FF_FIELD_AT(void *, (merger)->offset, (link)->outcfg)))
+
                 for (neg_step = 0; neg_step < neg->nb_mergers; neg_step++) {
                     const AVFilterFormatsMerger *m = &neg->mergers[neg_step];
+                    if (m->conversion_filter != conv_filters[k])
+                        continue;
                     if ((ret = MERGE(m,  inlink)) <= 0 ||
                         (ret = MERGE(m, outlink)) <= 0) {
                         if (ret < 0)
@@ -646,9 +669,17 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx)
                                "'%s' and the filter '%s'\n", link->src->name, link->dst->name);
                         print_link_formats(log_ctx, AV_LOG_ERROR, link);
                         return AVERROR(ENOSYS);
+                    } else {
+                        count_merged += 2;
                     }
                 }
             }
+
+            /* if there is more than one auto filter, we may need another round
+             * to fully settle formats due to possible cross-incompatibilities
+             * between the auto filters themselves */
+            if (num_conv > 1)
+                goto retry;
         }
     }
 

libavfilter/formats.c

@@ -345,21 +345,32 @@ static int merge_generic(void *a, void *b)
     return merge_generic_internal(a, b, 0);
 }
 
+#define CONVERSION_FILTER_SWSCALE \
+    .conversion_filter = "scale", \
+    .conversion_opts_offset = offsetof(AVFilterGraph, scale_sws_opts),
+
+#define CONVERSION_FILTER_ARESAMPLE \
+    .conversion_filter = "aresample", \
+    .conversion_opts_offset = offsetof(AVFilterGraph, aresample_swr_opts),
+
 static const AVFilterFormatsMerger mergers_video[] = {
     {
         .offset     = offsetof(AVFilterFormatsConfig, formats),
         .merge      = merge_pix_fmts,
         .can_merge  = can_merge_pix_fmts,
+        CONVERSION_FILTER_SWSCALE
     },
     {
         .offset     = offsetof(AVFilterFormatsConfig, color_spaces),
         .merge      = merge_generic,
         .can_merge  = can_merge_generic,
+        CONVERSION_FILTER_SWSCALE
     },
     {
         .offset     = offsetof(AVFilterFormatsConfig, color_ranges),
         .merge      = merge_generic,
         .can_merge  = can_merge_generic,
+        CONVERSION_FILTER_SWSCALE
     },
 };
 
@@ -368,31 +379,30 @@ static const AVFilterFormatsMerger mergers_audio[] = {
         .offset     = offsetof(AVFilterFormatsConfig, channel_layouts),
         .merge      = merge_channel_layouts,
         .can_merge  = can_merge_channel_layouts,
+        CONVERSION_FILTER_ARESAMPLE
     },
     {
         .offset     = offsetof(AVFilterFormatsConfig, samplerates),
         .merge      = merge_samplerates,
         .can_merge  = can_merge_samplerates,
+        CONVERSION_FILTER_ARESAMPLE
     },
     {
         .offset     = offsetof(AVFilterFormatsConfig, formats),
         .merge      = merge_sample_fmts,
         .can_merge  = can_merge_sample_fmts,
+        CONVERSION_FILTER_ARESAMPLE
     },
 };
 
 static const AVFilterNegotiation negotiate_video = {
     .nb_mergers = FF_ARRAY_ELEMS(mergers_video),
     .mergers = mergers_video,
-    .conversion_filter = "scale",
-    .conversion_opts_offset = offsetof(AVFilterGraph, scale_sws_opts),
 };
 
 static const AVFilterNegotiation negotiate_audio = {
     .nb_mergers = FF_ARRAY_ELEMS(mergers_audio),
     .mergers = mergers_audio,
-    .conversion_filter = "aresample",
-    .conversion_opts_offset = offsetof(AVFilterGraph, aresample_swr_opts),
 };
 
 const AVFilterNegotiation *ff_filter_get_negotiation(AVFilterLink *link)

libavfilter/formats.h

@@ -470,6 +470,8 @@ typedef struct AVFilterFormatMerger {
     unsigned offset;
     int (*merge)(void *a, void *b);
     int (*can_merge)(const void *a, const void *b);
+    const char *conversion_filter;
+    unsigned conversion_opts_offset;
 } AVFilterFormatsMerger;
 
 /**
@@ -560,8 +562,6 @@ typedef struct AVFilterFormatMerger {
 typedef struct AVFilterNegotiation {
     unsigned nb_mergers;
     const AVFilterFormatsMerger *mergers;
-    const char *conversion_filter;
-    unsigned conversion_opts_offset;
 } AVFilterNegotiation;
 
 const AVFilterNegotiation *ff_filter_get_negotiation(AVFilterLink *link);