avcodec/vvc: support decoding prefix and suffix nal units

Signed-off-by: Wu Jianhua <toqsxw@outlook.com>

configure

@@ -2661,6 +2661,7 @@ CONFIG_EXTRA="
     vp56dsp
     vp8dsp
     vulkan_encode
+    vvc_sei
     wma_freqs
     wmv2dsp
 "
@@ -2910,6 +2911,7 @@ mpegvideoenc_select="aandcttables fdctdsp me_cmp mpegvideo pixblockdsp"
 msmpeg4dec_select="h263_decoder"
 msmpeg4enc_select="h263_encoder"
 vc1dsp_select="h264chroma qpeldsp startcode"
+vvc_sei_select="atsc_a53 golomb"
 wmv2dsp_select="qpeldsp"
 
 # decoders / encoders
@@ -3147,7 +3149,7 @@ vp6f_decoder_select="vp6_decoder"
 vp7_decoder_select="h264pred videodsp vp8dsp"
 vp8_decoder_select="h264pred videodsp vp8dsp"
 vp9_decoder_select="videodsp vp9_parser vp9_superframe_split_bsf"
-vvc_decoder_select="cabac cbs_h266 golomb videodsp"
+vvc_decoder_select="cabac cbs_h266 golomb videodsp vvc_sei"
 wcmv_decoder_select="inflate_wrapper"
 webp_decoder_select="vp8_decoder exif"
 wmalossless_decoder_select="llauddsp"

libavcodec/Makefile

@@ -811,6 +811,7 @@ OBJS-$(CONFIG_VP9_V4L2M2M_DECODER)     += v4l2_m2m_dec.o
 OBJS-$(CONFIG_VQA_DECODER)             += vqavideo.o
 OBJS-$(CONFIG_VQC_DECODER)             += vqcdec.o
 OBJS-$(CONFIG_VVC_DECODER)             += executor.o h2645data.o
+OBJS-$(CONFIG_VVC_SEI)                 += h2645_sei.o aom_film_grain.o
 OBJS-$(CONFIG_WADY_DPCM_DECODER)       += dpcm.o
 OBJS-$(CONFIG_WAVARC_DECODER)          += wavarc.o
 OBJS-$(CONFIG_WAVPACK_DECODER)         += wavpack.o wavpackdata.o dsd.o

libavcodec/vvc/Makefile

@@ -14,4 +14,5 @@ OBJS-$(CONFIG_VVC_DECODER)          +=  vvc/dec.o           \
                                         vvc/mvs.o           \
                                         vvc/ps.o            \
                                         vvc/refs.o          \
+                                        vvc/sei.o           \
                                         vvc/thread.o        \

libavcodec/vvc/dec.c

@@ -657,6 +657,7 @@ static av_cold void frame_context_free(VVCFrameContext *fc)
     pic_arrays_free(fc);
     av_frame_free(&fc->output_frame);
     ff_vvc_frame_ps_free(&fc->ps);
+    ff_vvc_sei_reset(&fc->sei);
 }
 
 static av_cold int frame_context_init(VVCFrameContext *fc, AVCodecContext *avctx)
@@ -699,6 +700,10 @@ static int frame_context_setup(VVCFrameContext *fc, VVCContext *s)
                     return ret;
             }
         }
+
+        ret = ff_vvc_sei_replace(&fc->sei, &prev->sei);
+        if (ret < 0)
+            return ret;
     }
 
     if (IS_IDR(s)) {
@@ -714,6 +719,22 @@ static int frame_context_setup(VVCFrameContext *fc, VVCContext *s)
     return 0;
 }
 
+/* SEI does not affect decoding, so we ignore the return value */
+static void decode_prefix_sei(VVCFrameContext *fc, VVCContext *s)
+{
+    CodedBitstreamFragment *frame = &s->current_frame;
+
+    for (int i = 0; i < frame->nb_units; i++) {
+        const CodedBitstreamUnit *unit = frame->units + i;
+
+        if (unit->type == VVC_PREFIX_SEI_NUT) {
+            int ret = ff_vvc_sei_decode(&fc->sei, unit->content_ref, fc);
+            if (ret < 0)
+                return;
+        }
+    }
+}
+
 static int frame_start(VVCContext *s, VVCFrameContext *fc, SliceContext *sc)
 {
     const VVCPH *ph                 = &fc->ps.ph;
@@ -727,6 +748,8 @@ static int frame_start(VVCContext *s, VVCFrameContext *fc, SliceContext *sc)
     if ((ret = ff_vvc_set_new_ref(s, fc, &fc->frame)) < 0)
         goto fail;
 
+    decode_prefix_sei(fc, s);
+
     if (!IS_IDR(s))
         ff_vvc_bump_frame(s, fc);
 
@@ -931,6 +954,15 @@ static int decode_nal_unit(VVCContext *s, VVCFrameContext *fc, AVBufferRef *buf_
         if (ret < 0)
             return ret;
         break;
+    case VVC_PREFIX_SEI_NUT:
+        /* handle by decode_prefix_sei() */
+        break;
+
+    case VVC_SUFFIX_SEI_NUT:
+        /* SEI does not affect decoding, so we ignore the return value*/
+        if (fc)
+            ff_vvc_sei_decode(&fc->sei, unit->content_ref, fc);
+        break;
     }
 
     return 0;

libavcodec/vvc/dec.h

@@ -29,6 +29,7 @@
 
 #include "ps.h"
 #include "dsp.h"
+#include "sei.h"
 
 #define LUMA                    0
 #define CHROMA                  1
@@ -124,6 +125,7 @@ typedef struct VVCFrameContext {
     struct AVFrame *output_frame;
 
     VVCFrameParamSets ps;
+    VVCSEI sei;
 
     SliceContext  **slices;
     int nb_slices;

libavcodec/vvc/sei.c

@@ -0,0 +1,55 @@
+/*
+ * VVC Supplementary Enhancement Information messages
+ *
+ * copyright (c) 2024 Wu Jianhua <toqsxw@outlook.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "sei.h"
+#include "dec.h"
+#include "libavutil/refstruct.h"
+
+int ff_vvc_sei_decode(VVCSEI *s, const H266RawSEI *sei, const struct VVCFrameContext *fc)
+{
+    if (!sei)
+        return AVERROR_INVALIDDATA;
+
+    for (int i = 0; i < sei->message_list.nb_messages; i++) {
+        SEIRawMessage *message = &sei->message_list.messages[i];
+
+        switch (message->payload_type) {
+        default:
+            av_log(fc->log_ctx, AV_LOG_DEBUG, "Skipped %s SEI %d\n",
+                sei->nal_unit_header.nal_unit_type == VVC_PREFIX_SEI_NUT ?
+                    "PREFIX" : "SUFFIX", message->payload_type);
+            return FF_H2645_SEI_MESSAGE_UNHANDLED;
+        }
+    }
+
+    return 0;
+}
+
+int ff_vvc_sei_replace(VVCSEI *dst, const VVCSEI *src)
+{
+    return ff_h2645_sei_ctx_replace(&dst->common, &src->common);
+}
+
+void ff_vvc_sei_reset(VVCSEI *s)
+{
+    ff_h2645_sei_reset(&s->common);
+}

libavcodec/vvc/sei.h

@@ -0,0 +1,45 @@
+/*
+ * VVC Supplementary Enhancement Information messages
+ *
+ * copyright (c) 2024 Wu Jianhua <toqsxw@outlook.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_VVC_SEI_H
+#define AVCODEC_VVC_SEI_H
+
+#include <stdint.h>
+
+#include "libavcodec/get_bits.h"
+#include "libavcodec/cbs.h"
+#include "libavcodec/cbs_h266.h"
+#include "libavcodec/h2645_sei.h"
+#include "libavcodec/sei.h"
+#include "libavcodec/vvc.h"
+
+typedef struct VVCSEI {
+    H2645SEI common;
+} VVCSEI;
+
+struct VVCFrameContext;
+
+int ff_vvc_sei_decode(VVCSEI *s, const H266RawSEI *sei, const struct VVCFrameContext *fc);
+int ff_vvc_sei_replace(VVCSEI *dst, const VVCSEI *src);
+void ff_vvc_sei_reset(VVCSEI *s);
+
+#endif /* AVCODEC_VVC_SEI_H */