avcodec/vvc/dec: support applying film grain by the decoder

Signed-off-by: Wu Jianhua <toqsxw@outlook.com>

libavcodec/Makefile

@@ -811,7 +811,7 @@ OBJS-$(CONFIG_VP9_V4L2M2M_DECODER)     += v4l2_m2m_dec.o
 OBJS-$(CONFIG_VQA_DECODER)             += vqavideo.o
 OBJS-$(CONFIG_VQC_DECODER)             += vqcdec.o
 OBJS-$(CONFIG_VVC_DECODER)             += executor.o h2645data.o
-OBJS-$(CONFIG_VVC_SEI)                 += h2645_sei.o aom_film_grain.o
+OBJS-$(CONFIG_VVC_SEI)                 += h2645_sei.o aom_film_grain.o h274.o
 OBJS-$(CONFIG_WADY_DPCM_DECODER)       += dpcm.o
 OBJS-$(CONFIG_WAVARC_DECODER)          += wavarc.o
 OBJS-$(CONFIG_WAVPACK_DECODER)         += wavpack.o wavpackdata.o dsd.o

libavcodec/vvc/dec.c

@@ -26,9 +26,12 @@
 #include "libavcodec/hwconfig.h"
 #include "libavcodec/profiles.h"
 #include "libavutil/refstruct.h"
+#include "libavcodec/aom_film_grain.h"
+#include "libavcodec/thread.h"
 #include "libavutil/cpu.h"
 #include "libavutil/mem.h"
 #include "libavutil/thread.h"
+#include "libavutil/film_grain_params.h"
 
 #include "dec.h"
 #include "ctu.h"
@@ -618,6 +621,14 @@ static int ref_frame(VVCFrame *dst, const VVCFrame *src)
     av_refstruct_replace(&dst->sps, src->sps);
     av_refstruct_replace(&dst->pps, src->pps);
 
+    if (src->needs_fg) {
+        ret = av_frame_ref(dst->frame_grain, src->frame_grain);
+        if (ret < 0)
+            return ret;
+
+        dst->needs_fg = src->needs_fg;
+    }
+
     av_refstruct_replace(&dst->progress, src->progress);
 
     av_refstruct_replace(&dst->tab_dmvr_mvf, src->tab_dmvr_mvf);
@@ -651,6 +662,7 @@ static av_cold void frame_context_free(VVCFrameContext *fc)
     for (int i = 0; i < FF_ARRAY_ELEMS(fc->DPB); i++) {
         ff_vvc_unref_frame(fc, &fc->DPB[i], ~0);
         av_frame_free(&fc->DPB[i].frame);
+        av_frame_free(&fc->DPB[i].frame_grain);
     }
 
     ff_vvc_frame_thread_free(fc);
@@ -673,6 +685,10 @@ static av_cold int frame_context_init(VVCFrameContext *fc, AVCodecContext *avctx
         fc->DPB[j].frame = av_frame_alloc();
         if (!fc->DPB[j].frame)
             return AVERROR(ENOMEM);
+
+        fc->DPB[j].frame_grain = av_frame_alloc();
+        if (!fc->DPB[j].frame_grain)
+            return AVERROR(ENOMEM);
     }
     fc->cu_pool = av_refstruct_pool_alloc(sizeof(CodingUnit), 0);
     if (!fc->cu_pool)
@@ -743,6 +759,41 @@ static int set_side_data(VVCContext *s, VVCFrameContext *fc)
         NULL, fc->ps.sps->bit_depth, fc->ps.sps->bit_depth, fc->ref->poc);
 }
 
+static int check_film_grain(VVCContext *s, VVCFrameContext *fc)
+{
+    int ret;
+
+    fc->ref->needs_fg = (fc->sei.common.film_grain_characteristics &&
+        fc->sei.common.film_grain_characteristics->present ||
+        fc->sei.common.aom_film_grain.enable) &&
+        !(s->avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
+        !s->avctx->hwaccel;
+
+    if (fc->ref->needs_fg &&
+        (fc->sei.common.film_grain_characteristics->present &&
+            !ff_h274_film_grain_params_supported(fc->sei.common.film_grain_characteristics->model_id,
+                fc->ref->frame->format) ||
+            !av_film_grain_params_select(fc->ref->frame))) {
+        av_log_once(s->avctx, AV_LOG_WARNING, AV_LOG_DEBUG, &s->film_grain_warning_shown,
+            "Unsupported film grain parameters. Ignoring film grain.\n");
+        fc->ref->needs_fg = 0;
+    }
+
+    if (fc->ref->needs_fg) {
+        fc->ref->frame_grain->format = fc->ref->frame->format;
+        fc->ref->frame_grain->width  = fc->ref->frame->width;
+        fc->ref->frame_grain->height = fc->ref->frame->height;
+
+        ret = ff_thread_get_buffer(s->avctx, fc->ref->frame_grain, 0);
+        if (ret < 0)
+            return ret;
+
+        return av_frame_copy_props(fc->ref->frame_grain, fc->ref->frame);
+    }
+
+    return 0;
+}
+
 static int frame_start(VVCContext *s, VVCFrameContext *fc, SliceContext *sc)
 {
     const VVCPH *ph                 = &fc->ps.ph;
@@ -762,6 +813,10 @@ static int frame_start(VVCContext *s, VVCFrameContext *fc, SliceContext *sc)
     if (ret < 0)
         goto fail;
 
+    ret = check_film_grain(s, fc);
+    if (ret < 0)
+        goto fail;
+
     if (!IS_IDR(s))
         ff_vvc_bump_frame(s, fc);
 
@@ -1019,15 +1074,43 @@ fail:
     return ret;
 }
 
+static int frame_end(VVCContext *s, VVCFrameContext *fc)
+{
+    const AVFilmGrainParams *fgp;
+    int ret = 0;
+
+    if (fc->ref->needs_fg) {
+        av_assert0(fc->ref->frame_grain->buf[0]);
+        fgp = av_film_grain_params_select(fc->ref->frame);
+        switch (fgp->type) {
+        case AV_FILM_GRAIN_PARAMS_NONE:
+            av_assert0(0);
+            return AVERROR_BUG;
+        case AV_FILM_GRAIN_PARAMS_H274:
+            ret = ff_h274_apply_film_grain(fc->ref->frame_grain, fc->ref->frame,
+                &s->h274db, fgp);
+            break;
+        case AV_FILM_GRAIN_PARAMS_AV1:
+            ret = ff_aom_apply_film_grain(fc->ref->frame_grain, fc->ref->frame, fgp);
+            break;
+        }
+    }
+
+    return ret;
+}
+
 static int wait_delayed_frame(VVCContext *s, AVFrame *output, int *got_output)
 {
     VVCFrameContext *delayed = get_frame_context(s, s->fcs, s->nb_frames - s->nb_delayed);
     int ret                  = ff_vvc_frame_wait(s, delayed);
 
-    if (!ret && delayed->output_frame->buf[0] && output) {
+    if (!ret) {
+        ret = frame_end(s, delayed);
+        if (ret >= 0 && delayed->output_frame->buf[0] && output) {
             av_frame_move_ref(output, delayed->output_frame);
             *got_output = 1;
         }
+    }
     s->nb_delayed--;
 
     return ret;

libavcodec/vvc/dec.h

@@ -26,6 +26,7 @@
 
 #include "libavcodec/videodsp.h"
 #include "libavcodec/vvc.h"
+#include "libavcodec/h274.h"
 
 #include "ps.h"
 #include "dsp.h"
@@ -71,12 +72,15 @@ typedef struct VVCWindow {
 
 typedef struct VVCFrame {
     struct AVFrame *frame;
-
+    struct AVFrame *frame_grain;
     const VVCSPS *sps;                          ///< RefStruct reference
     const VVCPPS *pps;                          ///< RefStruct reference
     struct MvField *tab_dmvr_mvf;               ///< RefStruct reference
     RefPicListTab **rpl_tab;                    ///< RefStruct reference
     RefPicListTab  *rpl;                        ///< RefStruct reference
+
+    int needs_fg;                               ///< 1 if grain needs to be applied by the decoder
+
     int nb_rpl_elems;
 
     int ctb_count;
@@ -218,6 +222,7 @@ typedef struct VVCContext {
     CodedBitstreamFragment current_frame;
 
     VVCParamSets ps;
+    H274FilmGrainDatabase h274db;
 
     int temporal_id;        ///< temporal_id_plus1 - 1
     int poc_tid0;
@@ -228,6 +233,7 @@ typedef struct VVCContext {
     enum VVCNALUnitType vcl_unit_type;
     int no_output_before_recovery_flag; ///< NoOutputBeforeRecoveryFlag
     int gdr_recovery_point_poc;         ///< recoveryPointPocVal
+    int film_grain_warning_shown;
 
     /**
      * Sequence counters for decoded and output frames, so that old

libavcodec/vvc/refs.c

@@ -52,6 +52,12 @@ void ff_vvc_unref_frame(VVCFrameContext *fc, VVCFrame *frame, int flags)
         frame->flags = 0;
     if (!frame->flags) {
         av_frame_unref(frame->frame);
+
+        if (frame->needs_fg) {
+            av_frame_unref(frame->frame_grain);
+            frame->needs_fg = 0;
+        }
+
         av_refstruct_unref(&frame->sps);
         av_refstruct_unref(&frame->pps);
         av_refstruct_unref(&frame->progress);
@@ -285,7 +291,10 @@ int ff_vvc_output_frame(VVCContext *s, VVCFrameContext *fc, AVFrame *out, const
             if (frame->flags & VVC_FRAME_FLAG_CORRUPT)
                 frame->frame->flags |= AV_FRAME_FLAG_CORRUPT;
 
-            ret = av_frame_ref(out, frame->frame);
+            ret = av_frame_ref(out, frame->needs_fg ? frame->frame_grain : frame->frame);
+            if (ret < 0)
+                return ret;
+
             if (frame->flags & VVC_FRAME_FLAG_BUMPING)
                 ff_vvc_unref_frame(fc, frame, VVC_FRAME_FLAG_OUTPUT | VVC_FRAME_FLAG_BUMPING);
             else