From 563e1df5d6ae4fc6ba8257268f5493aee9168e9e Mon Sep 17 00:00:00 2001
From: Nicolas George <george@nsup.org>
Date: Thu, 16 Apr 2020 16:40:40 +0200
Subject: [PATCH] lavfi/formats: add ff_formats_pixdesc_filter().

---
 libavfilter/formats.c | 41 +++++++++++++++++++++++++++++++++++++++++
 libavfilter/formats.h | 10 ++++++++++
 2 files changed, 51 insertions(+)

diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index 33c64668a0..784f604483 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -373,6 +373,47 @@ const int64_t avfilter_all_channel_layouts[] = {
     -1
 };
 
+int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned rej)
+{
+    unsigned nb_formats, fmt, flags;
+    AVFilterFormats *formats = NULL;
+
+    while (1) {
+        nb_formats = 0;
+        for (fmt = 0;; fmt++) {
+            const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
+            if (!desc)
+                break;
+            flags = desc->flags;
+            if (!(desc->flags & AV_PIX_FMT_FLAG_HWACCEL) &&
+                !(desc->flags & AV_PIX_FMT_FLAG_PLANAR) &&
+                (desc->log2_chroma_w || desc->log2_chroma_h))
+                flags |= FF_PIX_FMT_FLAG_SW_FLAT_SUB;
+            if ((flags & (want | rej)) != want)
+                continue;
+            if (formats)
+                formats->formats[nb_formats] = fmt;
+            nb_formats++;
+        }
+        if (formats) {
+            av_assert0(formats->nb_formats == nb_formats);
+            *rfmts = formats;
+            return 0;
+        }
+        formats = av_mallocz(sizeof(*formats));
+        if (!formats)
+            return AVERROR(ENOMEM);
+        formats->nb_formats = nb_formats;
+        if (nb_formats) {
+            formats->formats = av_malloc_array(nb_formats, sizeof(*formats->formats));
+            if (!formats->formats) {
+                av_freep(&formats);
+                return AVERROR(ENOMEM);
+            }
+        }
+    }
+}
+
 // AVFilterFormats *avfilter_make_all_channel_layouts(void)
 // {
 //     return avfilter_make_format64_list(avfilter_all_channel_layouts);
diff --git a/libavfilter/formats.h b/libavfilter/formats.h
index 870809b5a0..cc588f30d8 100644
--- a/libavfilter/formats.h
+++ b/libavfilter/formats.h
@@ -221,6 +221,16 @@ int ff_add_format(AVFilterFormats **avff, int64_t fmt);
 av_warn_unused_result
 AVFilterFormats *ff_all_formats(enum AVMediaType type);
 
+/**
+ * Construct a formats list containing all pixel formats with certain
+ * properties
+ */
+av_warn_unused_result
+int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned rej);
+
+//* format is software, non-planar with sub-sampling
+#define FF_PIX_FMT_FLAG_SW_FLAT_SUB (1 << 24)
+
 /**
  * Construct a formats list containing all planar sample formats.
  */