avfilter: add an LCEVC decoding filter

Signed-off-by: James Almer <jamrial@gmail.com>

Changelog

@@ -25,6 +25,7 @@ version <next>:
 - Vulkan H.265 encoder
 - stream specifiers in fftools can now match by stream disposition
 - LCEVC enhancement data exporting in H.26x and MP4/ISOBMFF
+- LCEVC filter
 
 
 version 7.0:

configure

@@ -246,6 +246,7 @@ External library support:
   --enable-libklvanc       enable Kernel Labs VANC processing [no]
   --enable-libkvazaar      enable HEVC encoding via libkvazaar [no]
   --enable-liblc3          enable LC3 de/encoding via liblc3 [no]
+  --enable-liblcevc-dec    enable LCEVC decoding via liblcevc-dec [no]
   --enable-liblensfun      enable lensfun lens correction [no]
   --enable-libmodplug      enable ModPlug via libmodplug [no]
   --enable-libmp3lame      enable MP3 encoding via libmp3lame [no]
@@ -1930,6 +1931,7 @@ EXTERNAL_LIBRARY_LIST="
     libklvanc
     libkvazaar
     liblc3
+    liblcevc_dec
     libmodplug
     libmp3lame
     libmysofa
@@ -3893,6 +3895,7 @@ identity_filter_select="scene_sad"
 interlace_filter_deps="gpl"
 kerndeint_filter_deps="gpl"
 ladspa_filter_deps="ladspa libdl"
+lcevc_filter_deps="liblcevc_dec"
 lensfun_filter_deps="liblensfun version3"
 libplacebo_filter_deps="libplacebo vulkan"
 lv2_filter_deps="lv2"
@@ -6921,6 +6924,7 @@ enabled libklvanc         && require libklvanc libklvanc/vanc.h klvanc_context_c
 enabled libkvazaar        && require_pkg_config libkvazaar "kvazaar >= 2.0.0" kvazaar.h kvz_api_get
 enabled liblc3            && require_pkg_config liblc3 "lc3 >= 1.1.0" lc3.h lc3_hr_setup_encoder
 enabled liblensfun        && require_pkg_config liblensfun lensfun lensfun.h lf_db_create
+enabled liblcevc_dec      && require_pkg_config liblcevc_dec "lcevc_dec >= 2.0.0" "LCEVC/lcevc_dec.h" LCEVC_CreateDecoder
 
 if enabled libmfx && enabled libvpl; then
    die "ERROR: can not use libmfx and libvpl together"

doc/general_contents.texi

@@ -160,6 +160,19 @@ Go to @url{http://lame.sourceforge.net/} and follow the
 instructions for installing the library.
 Then pass @code{--enable-libmp3lame} to configure to enable it.
 
+@section LCEVCdec
+
+FFmpeg can make use of the liblcevc_dec library for LCEVC enhacement layer
+decoding on supported bitstreams.
+
+Go to @url{https://github.com/v-novaltd/LCEVCdec} and follow the instructions
+for installing the library. Then pass @code{--enable-liblcevc-dec} to configure to
+enable it.
+
+@float NOTE
+LCEVCdec is under the BSD-3-Clause-Clear License.
+@end float
+
 @section libilbc
 
 iLBC is a narrowband speech codec that has been made freely available

libavfilter/Makefile

@@ -360,6 +360,7 @@ OBJS-$(CONFIG_INTERLEAVE_FILTER)             += f_interleave.o
 OBJS-$(CONFIG_KERNDEINT_FILTER)              += vf_kerndeint.o
 OBJS-$(CONFIG_KIRSCH_FILTER)                 += vf_convolution.o
 OBJS-$(CONFIG_LAGFUN_FILTER)                 += vf_lagfun.o
+OBJS-$(CONFIG_LCEVC_FILTER)                  += vf_lcevc.o
 OBJS-$(CONFIG_LATENCY_FILTER)                += f_latency.o
 OBJS-$(CONFIG_LENSCORRECTION_FILTER)         += vf_lenscorrection.o
 OBJS-$(CONFIG_LENSFUN_FILTER)                += vf_lensfun.o

libavfilter/allfilters.c

@@ -337,6 +337,7 @@ extern const AVFilter ff_vf_kerndeint;
 extern const AVFilter ff_vf_kirsch;
 extern const AVFilter ff_vf_lagfun;
 extern const AVFilter ff_vf_latency;
+extern const AVFilter ff_vf_lcevc;
 extern const AVFilter ff_vf_lenscorrection;
 extern const AVFilter ff_vf_lensfun;
 extern const AVFilter ff_vf_libplacebo;

libavfilter/version.h

@@ -31,8 +31,8 @@
 
 #include "version_major.h"
 
-#define LIBAVFILTER_VERSION_MINOR   2
-#define LIBAVFILTER_VERSION_MICRO 102
+#define LIBAVFILTER_VERSION_MINOR   3
+#define LIBAVFILTER_VERSION_MICRO 100
 
 
 #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \

libavfilter/vf_lcevc.c

@@ -0,0 +1,430 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * Copyright (c) 2024 James Almer <jamrial@gmail.com>
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdint.h>
+
+#include <LCEVC/lcevc_dec.h>
+
+#include "libavutil/internal.h"
+#include "libavutil/opt.h"
+#include "filters.h"
+#include "video.h"
+
+typedef struct LCEVCContext {
+    LCEVC_DecoderHandle decoder;
+    int w, h;
+} LCEVCContext;
+
+static LCEVC_ColorFormat map_format(int format)
+{
+    switch (format) {
+    case AV_PIX_FMT_YUV420P:
+        return LCEVC_I420_8;
+    case AV_PIX_FMT_YUV420P10:
+        return LCEVC_I420_10_LE;
+    case AV_PIX_FMT_NV12:
+        return LCEVC_NV12_8;
+    case AV_PIX_FMT_NV21:
+        return LCEVC_NV21_8;
+    case AV_PIX_FMT_GRAY8:
+        return LCEVC_GRAY_8;
+    case AV_PIX_FMT_GRAY10LE:
+        return LCEVC_GRAY_10_LE;
+    }
+
+    return LCEVC_ColorFormat_Unknown;
+}
+
+static inline LCEVC_ColorRange map_range(int range)
+{
+    switch (range) {
+    case AVCOL_RANGE_MPEG:
+        return LCEVC_ColorRange_Limited;
+    case AVCOL_RANGE_JPEG:
+        return LCEVC_ColorRange_Full;
+    }
+
+    return LCEVC_ColorRange_Unknown;
+}
+
+static inline enum AVColorRange map_av_range(int range)
+{
+    switch (range) {
+    case LCEVC_ColorRange_Limited:
+        return AVCOL_RANGE_MPEG;
+    case LCEVC_ColorRange_Full:
+        return AVCOL_RANGE_JPEG;
+    }
+
+    return AVCOL_RANGE_UNSPECIFIED;
+}
+
+static int alloc_base_frame(AVFilterLink *inlink, const AVFrame *in,
+                            LCEVC_PictureHandle *picture)
+{
+    AVFilterContext *ctx = inlink->dst;
+    LCEVCContext *lcevc = ctx->priv;
+    LCEVC_PictureDesc desc;
+    LCEVC_PicturePlaneDesc planes[AV_VIDEO_MAX_PLANES] = { 0 };
+    LCEVC_ColorFormat fmt = map_format(in->format);
+    int width = in->width - in->crop_left - in->crop_right;
+    int height = in->height - in->crop_top - in->crop_bottom;
+    LCEVC_ReturnCode res;
+
+    res = LCEVC_DefaultPictureDesc(&desc, fmt, width, height);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_DefaultPictureDesc failed\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    for (int i = 0; i < AV_VIDEO_MAX_PLANES; i++) {
+        planes[i].firstSample = in->data[i];
+        planes[i].rowByteStride = in->linesize[i];
+    }
+
+    desc.cropTop    = in->crop_top;
+    desc.cropBottom = in->crop_bottom;
+    desc.cropLeft   = in->crop_left;
+    desc.cropRight  = in->crop_right;
+    desc.sampleAspectRatioNum = in->sample_aspect_ratio.num;
+    desc.sampleAspectRatioDen = in->sample_aspect_ratio.den;
+    desc.colorRange = map_range(in->color_range);
+    desc.colorPrimaries = (LCEVC_ColorPrimaries)in->color_primaries;
+    desc.matrixCoefficients = (LCEVC_MatrixCoefficients)in->colorspace;
+    desc.transferCharacteristics = (LCEVC_TransferCharacteristics)in->color_trc;
+    av_log(ctx, AV_LOG_DEBUG, "in  PTS %"PRId64", %dx%d, "
+                              "%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER", "
+                              "SAR %d:%d\n",
+           in->pts, in->width, in->height,
+           in->crop_top, in->crop_bottom, in->crop_left, in->crop_right,
+           in->sample_aspect_ratio.num, in->sample_aspect_ratio.den);
+
+    res = LCEVC_AllocPictureExternal(lcevc->decoder, &desc, NULL, planes, picture);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_AllocPictureExternal to allocate a buffer for a base frame\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    return 0;
+}
+
+static int send_frame(AVFilterLink *inlink, AVFrame *in)
+{
+    AVFilterContext *ctx = inlink->dst;
+    LCEVCContext *lcevc = ctx->priv;
+    LCEVC_PictureHandle picture;
+    const AVFrameSideData *sd = av_frame_get_side_data(in, AV_FRAME_DATA_LCEVC);
+    LCEVC_ReturnCode res;
+    int ret;
+
+    ret = alloc_base_frame(inlink, in, &picture);
+    if (ret < 0)
+        return ret;
+
+    if (sd) {
+        res = LCEVC_SendDecoderEnhancementData(lcevc->decoder, in->pts, 0, sd->data, sd->size);
+        if (res == LCEVC_Again)
+            return AVERROR(EAGAIN);
+        else if (res != LCEVC_Success) {
+            av_log(ctx, AV_LOG_ERROR, "LCEVC_SendDecoderEnhancementData failed\n");
+            return AVERROR_EXTERNAL;
+        }
+    }
+
+    res = LCEVC_SendDecoderBase(lcevc->decoder, in->pts, 0, picture, -1, in);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_SendDecoderBase failed\n");
+        LCEVC_FreePicture(lcevc->decoder, picture);
+        return AVERROR_EXTERNAL;
+    }
+
+    return 0;
+}
+
+static int alloc_enhanced_frame(AVFilterLink *inlink, const AVFrame *out,
+                                LCEVC_PictureHandle *picture)
+{
+    AVFilterContext *ctx = inlink->dst;
+    LCEVCContext *lcevc = ctx->priv;
+    LCEVC_PictureDesc desc;
+    LCEVC_PicturePlaneDesc planes[AV_VIDEO_MAX_PLANES] = { 0 };
+    LCEVC_ColorFormat fmt = map_format(out->format);
+    LCEVC_ReturnCode res;
+
+    res = LCEVC_DefaultPictureDesc(&desc, fmt, out->width, out->height);
+    if (res != LCEVC_Success)
+        return AVERROR_EXTERNAL;
+
+    for (int i = 0; i < AV_VIDEO_MAX_PLANES; i++) {
+        planes[i].firstSample = out->data[i];
+        planes[i].rowByteStride = out->linesize[i];
+    }
+
+    res = LCEVC_AllocPictureExternal(lcevc->decoder, &desc, NULL, planes, picture);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_AllocPictureExternal to allocate a buffer for an enhanced frame\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    return 0;
+}
+
+static int generate_output(AVFilterLink *inlink, AVFrame *out)
+{
+    AVFilterContext *ctx = inlink->dst;
+    AVFilterLink *outlink = ctx->outputs[0];
+    LCEVCContext *lcevc = ctx->priv;
+    LCEVC_PictureDesc desc;
+    LCEVC_DecodeInformation info;
+    LCEVC_PictureHandle picture;
+    LCEVC_ReturnCode res;
+
+    res = LCEVC_ReceiveDecoderPicture(lcevc->decoder, &picture, &info);
+    if (res == LCEVC_Again) {
+        int64_t pts;
+        int status;
+        if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
+            av_frame_free(&out);
+            ff_outlink_set_status(outlink, status, pts);
+            return 0;
+        }
+        // this shouldn't be reachable, but instead of asserting, just error out
+        return AVERROR_BUG;
+    } else if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_ReceiveDecoderPicture failed\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    av_frame_copy_props(out, (AVFrame *)info.baseUserData);
+    av_frame_remove_side_data(out, AV_FRAME_DATA_LCEVC);
+
+    av_frame_free((AVFrame **)&info.baseUserData);
+
+    res = LCEVC_GetPictureDesc(lcevc->decoder, picture, &desc);
+    LCEVC_FreePicture(lcevc->decoder, picture);
+
+    out->crop_top = desc.cropTop;
+    out->crop_bottom = desc.cropBottom;
+    out->crop_left = desc.cropLeft;
+    out->crop_right = desc.cropRight;
+    out->sample_aspect_ratio.num = outlink->sample_aspect_ratio.num = desc.sampleAspectRatioNum;
+    out->sample_aspect_ratio.den = outlink->sample_aspect_ratio.den = desc.sampleAspectRatioDen;
+    out->color_range = map_range(desc.colorRange);
+    out->color_primaries = (enum AVColorPrimaries)desc.colorPrimaries;
+    out->colorspace = (enum AVColorSpace)desc.matrixCoefficients;
+    out->color_trc = (enum AVColorTransferCharacteristic)desc.transferCharacteristics;
+    out->width = outlink->w = desc.width + out->crop_left + out->crop_right;
+    out->height = outlink->h = desc.height + out->crop_top + out->crop_bottom;
+
+    av_log(ctx, AV_LOG_DEBUG, "out PTS %"PRId64", %dx%d, "
+                              "%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER", "
+                              "SAR %d:%d, "
+                              "hasEnhancement %d, enhanced %d\n",
+           out->pts, out->width, out->height,
+           out->crop_top, out->crop_bottom, out->crop_left, out->crop_right,
+           out->sample_aspect_ratio.num, out->sample_aspect_ratio.den,
+           info.hasEnhancement, info.enhanced);
+
+    return ff_filter_frame(outlink, out);
+}
+
+static int receive_frame(AVFilterLink *inlink, AVFrame *out)
+{
+    AVFilterContext *ctx = inlink->dst;
+    LCEVCContext *lcevc = ctx->priv;
+    LCEVC_PictureHandle picture;
+    LCEVC_ReturnCode res;
+    int ret;
+
+    ret = alloc_enhanced_frame(inlink, out, &picture);
+    if (ret < 0)
+        return ret;
+
+    res = LCEVC_SendDecoderPicture(lcevc->decoder, picture);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_SendDecoderPicture failed\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    return generate_output(inlink, out);
+}
+
+static int config_props(AVFilterLink *outlink)
+{
+    AVFilterContext *ctx = outlink->src;
+    AVFilterLink *inlink = ctx->inputs[0];
+    LCEVCContext *lcevc = ctx->priv;
+
+    outlink->w = lcevc->w = inlink->w * 2 / FFMAX(inlink->sample_aspect_ratio.den, 1);
+    outlink->h = lcevc->h = inlink->h * 2 / FFMAX(inlink->sample_aspect_ratio.den, 1);
+    outlink->sample_aspect_ratio = (AVRational) { 0, 1 };
+
+    return 0;
+}
+
+static void flush_bases(AVFilterContext *ctx)
+{
+    LCEVCContext *lcevc = ctx->priv;
+    LCEVC_PictureHandle picture;
+
+    while (LCEVC_ReceiveDecoderBase(lcevc->decoder, &picture) == LCEVC_Success)
+        LCEVC_FreePicture(lcevc->decoder, picture);
+}
+
+static int activate(AVFilterContext *ctx)
+{
+    LCEVCContext *lcevc   = ctx->priv;
+    AVFilterLink *inlink  = ctx->inputs[0];
+    AVFilterLink *outlink = ctx->outputs[0];
+    AVFrame *in, *out;
+    int status, ret;
+
+    FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
+
+    ret = ff_inlink_consume_frame(inlink, &in);
+    if (ret < 0)
+        return ret;
+    if (!ret) {
+        int64_t pts;
+        if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
+            if (!status)
+                ff_outlink_set_status(outlink, status, pts);
+        }
+        if (!status)
+            FF_FILTER_FORWARD_WANTED(outlink, inlink);
+    }
+
+    if (in) {
+       if (in->width  != inlink->w ||
+            in->height != inlink->h ||
+            in->sample_aspect_ratio.den != inlink->sample_aspect_ratio.den ||
+            in->sample_aspect_ratio.num != inlink->sample_aspect_ratio.num) {
+            inlink->dst->inputs[0]->w                       = in->width;
+            inlink->dst->inputs[0]->h                       = in->height;
+            inlink->dst->inputs[0]->sample_aspect_ratio.den = in->sample_aspect_ratio.den;
+            inlink->dst->inputs[0]->sample_aspect_ratio.num = in->sample_aspect_ratio.num;
+
+            config_props(outlink);
+        }
+
+        ret = send_frame(inlink, in);
+        if (ret < 0)
+            return ret;
+    }
+
+    out = ff_get_video_buffer(outlink, lcevc->w, lcevc->h);
+    if (!out)
+        return AVERROR(ENOMEM);
+
+    ret = receive_frame(inlink, out);
+    if (ret < 0) {
+        av_frame_free(&out);
+        return ret;
+    }
+
+    flush_bases(ctx);
+
+    return ret;
+}
+
+static void log_callback(LCEVC_DecoderHandle dec, LCEVC_Event event,
+                         LCEVC_PictureHandle pic, const LCEVC_DecodeInformation *info,
+                         const uint8_t *data, uint32_t size, void *logctx)
+{
+    if (event != LCEVC_Log) // shouldn't happen
+        return;
+
+    if (strlen(data) != size) // sanitize input
+        return;
+
+    av_log(logctx, AV_LOG_INFO, "LCEVC Log: %s\n", data);
+}
+
+static av_cold int init(AVFilterContext *ctx)
+{
+    LCEVCContext *lcevc = ctx->priv;
+    LCEVC_AccelContextHandle dummy = { 0 };
+    const int32_t event = LCEVC_Log;
+    LCEVC_ReturnCode res;
+
+    res = LCEVC_CreateDecoder(&lcevc->decoder, dummy);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_CreateDecoder failed\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    res = LCEVC_ConfigureDecoderInt(lcevc->decoder, "log_level", 4);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_ConfigureDecoderInt failed to set \"log_level\"\n");
+        return AVERROR_EXTERNAL;
+    }
+    res = LCEVC_ConfigureDecoderIntArray(lcevc->decoder, "events", 1, &event);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_ConfigureDecoderIntArray failed to set \"events\"\n");
+        return AVERROR_EXTERNAL;
+    }
+    res = LCEVC_SetDecoderEventCallback(lcevc->decoder, log_callback, ctx);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_SetDecoderEventCallback failed\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    res = LCEVC_InitializeDecoder(lcevc->decoder);
+    if (res != LCEVC_Success) {
+        av_log(ctx, AV_LOG_ERROR, "LCEVC_InitializeDecoder failed\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    return 0;
+}
+
+static av_cold void uninit(AVFilterContext *ctx)
+{
+    LCEVCContext *lcevc = ctx->priv;
+
+    LCEVC_DestroyDecoder(lcevc->decoder);
+}
+
+static const AVFilterPad lcevc_outputs[] = {
+    {
+        .name         = "default",
+        .type         = AVMEDIA_TYPE_VIDEO,
+        .config_props = config_props,
+    },
+};
+
+static const enum AVPixelFormat pix_fmts[] = {
+    AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10LE,
+    AV_PIX_FMT_NV12, AV_PIX_FMT_NV21,
+    AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY10LE,
+    AV_PIX_FMT_NONE
+};
+
+const AVFilter ff_vf_lcevc = {
+    .name          = "lcevc",
+    .description   = NULL_IF_CONFIG_SMALL("LCEVC"),
+    .activate      = activate,
+    FILTER_INPUTS(ff_video_default_filterpad),
+    FILTER_OUTPUTS(lcevc_outputs),
+    FILTER_PIXFMTS_ARRAY(pix_fmts),
+    .priv_size     = sizeof(LCEVCContext),
+    .init          = init,
+    .uninit        = uninit,
+};