libavutil: introduce AVFilmGrainParams side data

This patch introduces a new frame side data type AVFilmGrainParams for use
with video codecs which support it.

It can save a lot of memory used for duplicate processed reference frames and
reduce copies when applying film grain during presentation.

doc/APIchanges

@@ -15,6 +15,10 @@ libavutil:     2017-10-21
 
 API changes, most recent first:
 
+2020-11-25 - xxxxxxxxxx - lavu 56.61.100 - film_grain_params.h
+  Adds a new API for extracting codec film grain parameters as side data.
+  Adds a new AVFrameSideDataType entry AV_FRAME_DATA_FILM_GRAIN_PARAMS for it.
+
 2020-xx-xx - xxxxxxxxxx - lavf 58.64.100 - avformat.h
   Add AVSTREAM_EVENT_FLAG_NEW_PACKETS.
 

libavutil/Makefile

@@ -84,6 +84,7 @@ HEADERS = adler32.h                                                     \
           xtea.h                                                        \
           tea.h                                                         \
           tx.h                                                          \
+          film_grain_params.h                                           \
 
 HEADERS-$(CONFIG_LZO)                   += lzo.h
 
@@ -170,6 +171,7 @@ OBJS = adler32.o                                                        \
        tx_double.o                                                      \
        tx_int32.o                                                       \
        video_enc_params.o                                               \
+       film_grain_params.o                                              \
 
 
 OBJS-$(CONFIG_CUDA)                     += hwcontext_cuda.o

libavutil/film_grain_params.c

@@ -0,0 +1,42 @@
+/**
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "film_grain_params.h"
+
+AVFilmGrainParams *av_film_grain_params_alloc(size_t *size)
+{
+    AVFilmGrainParams *params = av_mallocz(sizeof(AVFilmGrainParams));
+
+    if (size)
+        *size = sizeof(*params);
+
+    return params;
+}
+
+AVFilmGrainParams *av_film_grain_params_create_side_data(AVFrame *frame)
+{
+    AVFrameSideData *side_data = av_frame_new_side_data(frame,
+                                                        AV_FRAME_DATA_FILM_GRAIN_PARAMS,
+                                                        sizeof(AVFilmGrainParams));
+    if (!side_data)
+        return NULL;
+
+    memset(side_data->data, 0, sizeof(AVFilmGrainParams));
+
+    return (AVFilmGrainParams *)side_data->data;
+}

libavutil/film_grain_params.h

@@ -0,0 +1,167 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVUTIL_FILM_GRAIN_PARAMS_H
+#define AVUTIL_FILM_GRAIN_PARAMS_H
+
+#include "frame.h"
+
+enum AVFilmGrainParamsType {
+    AV_FILM_GRAM_PARAMS_NONE = 0,
+
+    /**
+     * The union is valid when interpreted as AVFilmGrainAOMParams (codec.aom)
+     */
+    AV_FILM_GRAM_PARAMS_AV1,
+};
+
+/**
+ * This structure describes how to handle film grain synthesis for AOM codecs.
+ *
+ * @note The struct must be allocated as part of AVFilmGrainParams using
+ *       av_film_grain_params_alloc(). Its size is not a part of the public ABI.
+ */
+typedef struct AVFilmGrainAOMParams {
+    /**
+     * Number of points, and the scale and value for each point of the
+     * piecewise linear scaling function for the uma plane.
+     */
+    int num_y_points;
+    uint8_t y_points[14][2 /* value, scaling */];
+
+    /**
+     * Signals whether to derive the chroma scaling function from the luma.
+     * Not equivalent to copying the luma values and scales.
+     */
+    int chroma_scaling_from_luma;
+
+    /**
+     * If chroma_scaling_from_luma is set to 0, signals the chroma scaling
+     * function parameters.
+     */
+    int num_uv_points[2];
+    uint8_t uv_points[2][10][2 /* value, scaling */];
+
+    /**
+     * Specifies the shift applied to the chroma components. For AV1, its within
+     * [8; 11] and determines the range and quantization of the film grain.
+     */
+    int scaling_shift;
+
+    /**
+     * Specifies the auto-regression lag. The number of coefficients is given by
+     * 2*ar_coeff_lag(ar_coeff_lag - 1), with an extra one for the chroma.
+     */
+    int ar_coeff_lag;
+
+    /**
+     * Luma auto-regression coefficients.
+     */
+    int8_t ar_coeffs_y[24];
+
+    /**
+     * Chroma auto-regression coefficients.
+     */
+    int8_t ar_coeffs_uv[2][25];
+
+    /**
+     * Specifies the range of the auto-regressive coefficients. Values of 6,
+     * 7, 8 and so on represent a range of [-2, 2), [-1, 1), [-0.5, 0.5) and
+     * so on. For AV1 must be between 6 and 9.
+     */
+    int ar_coeff_shift;
+
+    /**
+     * Signals the down shift applied to the generated gaussian numbers during
+     * synthesis.
+     */
+    int grain_scale_shift;
+
+    /**
+     * Specifies the luma/chroma multipliers for the index to the component
+     * scaling function.
+     */
+    int uv_mult[2];
+    int uv_mult_luma[2];
+
+    /**
+     * Offset used for component scaling function. For AV1 its a 9-bit value
+     * with a range [-256, 255]
+     */
+    int uv_offset[2];
+
+    /**
+     * Signals whether to overlap film grain blocks.
+     */
+    int overlap_flag;
+
+    /**
+     * Signals to clip to limited color levels after film grain application.
+     */
+    int limit_output_range;
+} AVFilmGrainAOMParams;
+
+/**
+ * This structure describes how to handle film grain synthesis in video
+ * for specific codecs. Must be present on every frame where film grain is
+ * meant to be synthesised for correct presentation.
+ *
+ * @note The struct must be allocated with av_film_grain_params_alloc() and
+ *       its size is not a part of the public ABI.
+ */
+typedef struct AVFilmGrainParams {
+    /**
+     * Specifies the codec for which this structure is valid.
+     */
+    enum AVFilmGrainParamsType type;
+
+    /**
+     * Seed to use for the synthesis process, if the codec allows for it.
+     */
+    uint64_t seed;
+
+    /**
+     * Additional fields may be added both here and in any structure included.
+     * If a codec's film grain structure differs slightly over another
+     * codec's, fields within may change meaning depending on the type.
+     */
+    union {
+        AVFilmGrainAOMParams aom;
+    } codec;
+} AVFilmGrainParams;
+
+/**
+ * Allocate an AVFilmGrainParams structure and set its fields to
+ * default values. The resulting struct can be freed using av_freep().
+ * If size is not NULL it will be set to the number of bytes allocated.
+ *
+ * @return An AVFilmGrainParams filled with default values or NULL
+ *         on failure.
+ */
+AVFilmGrainParams *av_film_grain_params_alloc(size_t *size);
+
+/**
+ * Allocate a complete AVFilmGrainParams and add it to the frame.
+ *
+ * @param frame The frame which side data is added to.
+ *
+ * @return The AVFilmGrainParams structure to be filled by caller.
+ */
+AVFilmGrainParams *av_film_grain_params_create_side_data(AVFrame *frame);
+
+#endif /* AVUTIL_FILM_GRAIN_PARAMS_H */

libavutil/frame.c

@@ -851,6 +851,7 @@ const char *av_frame_side_data_name(enum AVFrameSideDataType type)
     case AV_FRAME_DATA_REGIONS_OF_INTEREST: return "Regions Of Interest";
     case AV_FRAME_DATA_VIDEO_ENC_PARAMS:            return "Video encoding parameters";
     case AV_FRAME_DATA_SEI_UNREGISTERED:            return "H.26[45] User Data Unregistered SEI message";
+    case AV_FRAME_DATA_FILM_GRAIN_PARAMS:           return "Film grain parameters";
     }
     return NULL;
 }

libavutil/frame.h

@@ -192,6 +192,12 @@ enum AVFrameSideDataType {
      * uuid_iso_iec_11578 followed by AVFrameSideData.size - 16 bytes of user_data_payload_byte.
      */
     AV_FRAME_DATA_SEI_UNREGISTERED,
+
+    /**
+     * Film grain parameters for a frame, described by AVFilmGrainParameters.
+     * Must be present for every frame which should have film grain applied.
+     */
+    AV_FRAME_DATA_FILM_GRAIN_PARAMS,
 };
 
 enum AVActiveFormatDescription {

libavutil/version.h

@@ -79,7 +79,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  56
-#define LIBAVUTIL_VERSION_MINOR  60
+#define LIBAVUTIL_VERSION_MINOR  61
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \