From 5c0348f3d61ce850fa33fe79b5d2bc49f7b52683 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Tue, 12 Jul 2022 11:25:09 +0200 Subject: [PATCH] lavc: add a codec flag for propagating opaque from frames to packets This is intended to be a more convenient replacement for reordered_opaque. Add support for it in the two encoders that offer AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE: libx264 and libx265. Other encoders will be supported in future commits. --- doc/APIchanges | 3 +++ libavcodec/avcodec.h | 26 +++++++++++++++++++++ libavcodec/encode.c | 7 ++++++ libavcodec/libx264.c | 29 +++++++++++++++++++++++ libavcodec/libx265.c | 48 +++++++++++++++++++++++++++----------- libavcodec/options_table.h | 1 + libavcodec/version.h | 2 +- 7 files changed, 101 insertions(+), 15 deletions(-) diff --git a/doc/APIchanges b/doc/APIchanges index a11acadecd..148962cb40 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -14,6 +14,9 @@ libavutil: 2021-04-27 API changes, most recent first: +2023-01-xx - xxxxxxxxxx - lavc 59.59.100 - avcodec.h + Add AV_CODEC_FLAG_COPY_OPAQUE. + 2023-01-13 - xxxxxxxxxx - lavu 57.44.100 - ambient_viewing_environment.h frame.h Adds a new structure for holding H.274 Ambient Viewing Environment metadata, AVAmbientViewingEnvironment. diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h index 4874b443f9..832b9d9575 100644 --- a/libavcodec/avcodec.h +++ b/libavcodec/avcodec.h @@ -241,6 +241,32 @@ typedef struct RcOverride{ * @ref AV_CODEC_CAP_ENCODER_RECON_FRAME capability. */ #define AV_CODEC_FLAG_RECON_FRAME (1 << 6) +/** + * Request the encoder to propagate each frame's AVFrame.opaque and + * AVFrame.opaque_ref values to its corresponding output AVPacket. + * + * May only be set on encoders that have the + * @ref AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE capability flag. + * + * @note + * While in typical cases one input frame produces exactly one output packet + * (perhaps after a delay), in general the mapping of frames to packets is + * M-to-N, so + * - Any number of input frames may be associated with any given output packet. + * This includes zero - e.g. some encoders may output packets that carry only + * metadata about the whole stream. + * - A given input frame may be associated with any number of output packets. + * Again this includes zero - e.g. some encoders may drop frames under certain + * conditions. + * . + * This implies that when using this flag, the caller must NOT assume that + * - a given input frame's opaques will necessarily appear on some output packet; + * - every output packet will have some non-NULL opaque value. + * . + * When an output packet contains multiple frames, the opaque values will be + * taken from the first of those. + */ +#define AV_CODEC_FLAG_COPY_OPAQUE (1 << 7) /** * Use internal 2pass ratecontrol in first pass mode. */ diff --git a/libavcodec/encode.c b/libavcodec/encode.c index e0b3e43840..d6c73dc044 100644 --- a/libavcodec/encode.c +++ b/libavcodec/encode.c @@ -636,6 +636,13 @@ int ff_encode_preinit(AVCodecContext *avctx) return AVERROR(EINVAL); } + if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE && + !(avctx->codec->capabilities & AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE)) { + av_log(avctx, AV_LOG_ERROR, "The copy_opaque flag is set, but the " + "encoder does not support it.\n"); + return AVERROR(EINVAL); + } + switch (avctx->codec_type) { case AVMEDIA_TYPE_VIDEO: ret = encode_preinit_video(avctx); break; case AVMEDIA_TYPE_AUDIO: ret = encode_preinit_audio(avctx); break; diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c index 2bbd9044b6..6d22a1726e 100644 --- a/libavcodec/libx264.c +++ b/libavcodec/libx264.c @@ -21,6 +21,7 @@ #include "config_components.h" +#include "libavutil/buffer.h" #include "libavutil/eval.h" #include "libavutil/internal.h" #include "libavutil/opt.h" @@ -51,6 +52,9 @@ typedef struct X264Opaque { int64_t reordered_opaque; int64_t wallclock; + + void *frame_opaque; + AVBufferRef *frame_opaque_ref; } X264Opaque; typedef struct X264Context { @@ -133,6 +137,11 @@ static void X264_log(void *p, int level, const char *fmt, va_list args) av_vlog(p, level_map[level], fmt, args); } +static void opaque_uninit(X264Opaque *o) +{ + av_buffer_unref(&o->frame_opaque_ref); + memset(o, 0, sizeof(*o)); +} static int encode_nals(AVCodecContext *ctx, AVPacket *pkt, const x264_nal_t *nals, int nnal) @@ -440,6 +449,15 @@ static int setup_frame(AVCodecContext *ctx, const AVFrame *frame, pic->i_pts = frame->pts; + opaque_uninit(opaque); + + if (ctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) { + opaque->frame_opaque = frame->opaque; + ret = av_buffer_replace(&opaque->frame_opaque_ref, frame->opaque_ref); + if (ret < 0) + goto fail; + } + opaque->reordered_opaque = frame->reordered_opaque; opaque->wallclock = wallclock; if (ctx->export_side_data & AV_CODEC_EXPORT_DATA_PRFT) @@ -594,6 +612,14 @@ static int X264_frame(AVCodecContext *ctx, AVPacket *pkt, const AVFrame *frame, out_opaque < &x4->reordered_opaque[x4->nb_reordered_opaque]) { ctx->reordered_opaque = out_opaque->reordered_opaque; wallclock = out_opaque->wallclock; + + if (ctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) { + pkt->opaque = out_opaque->frame_opaque; + pkt->opaque_ref = out_opaque->frame_opaque_ref; + out_opaque->frame_opaque_ref = NULL; + } + + opaque_uninit(out_opaque); } else { // Unexpected opaque pointer on picture output av_log(ctx, AV_LOG_ERROR, "Unexpected opaque pointer; " @@ -634,6 +660,9 @@ static av_cold int X264_close(AVCodecContext *avctx) X264Context *x4 = avctx->priv_data; av_freep(&x4->sei); + + for (int i = 0; i < x4->nb_reordered_opaque; i++) + opaque_uninit(&x4->reordered_opaque[i]); av_freep(&x4->reordered_opaque); #if X264_BUILD >= 161 diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c index 25de3c669b..a71f1f76d3 100644 --- a/libavcodec/libx265.c +++ b/libavcodec/libx265.c @@ -28,6 +28,7 @@ #include #include "libavutil/avassert.h" +#include "libavutil/buffer.h" #include "libavutil/internal.h" #include "libavutil/common.h" #include "libavutil/opt.h" @@ -43,6 +44,9 @@ typedef struct ReorderedData { int64_t reordered_opaque; + void *frame_opaque; + AVBufferRef *frame_opaque_ref; + int in_use; } ReorderedData; @@ -121,7 +125,7 @@ static int rd_get(libx265Context *ctx) static void rd_release(libx265Context *ctx, int idx) { av_assert0(idx >= 0 && idx < ctx->nb_rd); - + av_buffer_unref(&ctx->rd[idx].frame_opaque_ref); memset(&ctx->rd[idx], 0, sizeof(ctx->rd[idx])); } @@ -132,6 +136,8 @@ static av_cold int libx265_encode_close(AVCodecContext *avctx) ctx->api->param_free(ctx->params); av_freep(&ctx->sei_data); + for (int i = 0; i < ctx->nb_rd; i++) + rd_release(ctx, i); av_freep(&ctx->rd); if (ctx->encoder) @@ -582,6 +588,9 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt, sei->numPayloads = 0; if (pic) { + ReorderedData *rd; + int rd_idx; + for (i = 0; i < 3; i++) { x265pic.planes[i] = pic->data[i]; x265pic.stride[i] = pic->linesize[i]; @@ -600,20 +609,25 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt, if (ret < 0) return ret; - if (pic->reordered_opaque) { - ReorderedData *rd; - int rd_idx = rd_get(ctx); - - if (rd_idx < 0) { - free_picture(ctx, &x265pic); - return rd_idx; - } - - x265pic.userData = (void*)(intptr_t)(rd_idx + 1); - - rd = &ctx->rd[rd_idx]; - rd->reordered_opaque = pic->reordered_opaque; + rd_idx = rd_get(ctx); + if (rd_idx < 0) { + free_picture(ctx, &x265pic); + return rd_idx; } + rd = &ctx->rd[rd_idx]; + + rd->reordered_opaque = pic->reordered_opaque; + if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) { + rd->frame_opaque = pic->opaque; + ret = av_buffer_replace(&rd->frame_opaque_ref, pic->opaque_ref); + if (ret < 0) { + rd_release(ctx, rd_idx); + free_picture(ctx, &x265pic); + return ret; + } + } + + x265pic.userData = (void*)(intptr_t)(rd_idx + 1); if (ctx->a53_cc) { void *sei_data; @@ -742,6 +756,12 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt, avctx->reordered_opaque = rd->reordered_opaque; + if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) { + pkt->opaque = rd->frame_opaque; + pkt->opaque_ref = rd->frame_opaque_ref; + rd->frame_opaque_ref = NULL; + } + rd_release(ctx, idx); } else avctx->reordered_opaque = 0; diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h index 7924ca6144..4b38c42043 100644 --- a/libavcodec/options_table.h +++ b/libavcodec/options_table.h @@ -58,6 +58,7 @@ static const AVOption avcodec_options[] = { {"loop", "use loop filter", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_LOOP_FILTER }, INT_MIN, INT_MAX, V|E, "flags"}, {"qscale", "use fixed qscale", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_QSCALE }, INT_MIN, INT_MAX, 0, "flags"}, {"recon_frame", "export reconstructed frames", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_RECON_FRAME}, .unit = "flags"}, +{"copy_opaque", "propagate opaque values", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_COPY_OPAQUE}, .unit = "flags"}, {"pass1", "use internal 2-pass ratecontrol in first pass mode", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_PASS1 }, INT_MIN, INT_MAX, 0, "flags"}, {"pass2", "use internal 2-pass ratecontrol in second pass mode", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_PASS2 }, INT_MIN, INT_MAX, 0, "flags"}, {"gray", "only decode/encode grayscale", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_GRAY }, INT_MIN, INT_MAX, V|E|D, "flags"}, diff --git a/libavcodec/version.h b/libavcodec/version.h index 7ac8d2b2fe..752adc81f8 100644 --- a/libavcodec/version.h +++ b/libavcodec/version.h @@ -29,7 +29,7 @@ #include "version_major.h" -#define LIBAVCODEC_VERSION_MINOR 58 +#define LIBAVCODEC_VERSION_MINOR 59 #define LIBAVCODEC_VERSION_MICRO 100 #define LIBAVCODEC_VERSION_INT AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \