lavc: rework handling of refcounted_frames=0

Use only proper AVFrame API (no assigning of whole frames, since that
hardcodes sizeof(AVFrame) into lavc).

Make a copy of the side data, so the caller can use av_frame_unref/free
on non-refcounted frames, eliminating the need for
avcodec_get_frame_defaults()/avcodec_free_frame().

libavcodec/internal.h

@@ -83,7 +83,7 @@ typedef struct AVCodecInternal {
      */
     int last_audio_frame;
 
-    AVFrame to_free;
+    AVFrame *to_free;
 
     FramePool *pool;
 

libavcodec/utils.c

@@ -895,6 +895,12 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, const AVCodec *code
         goto free_and_end;
     }
 
+    avctx->internal->to_free = av_frame_alloc();
+    if (!avctx->internal->to_free) {
+        ret = AVERROR(ENOMEM);
+        goto free_and_end;
+    }
+
     if (codec->priv_data_size > 0) {
         if (!avctx->priv_data) {
             avctx->priv_data = av_mallocz(codec->priv_data_size);
@@ -1373,6 +1379,51 @@ fail:
     return AVERROR_INVALIDDATA;
 }
 
+static int unrefcount_frame(AVCodecInternal *avci, AVFrame *frame)
+{
+    int ret;
+
+    /* move the original frame to our backup */
+    av_frame_unref(avci->to_free);
+    av_frame_move_ref(avci->to_free, frame);
+
+    /* now copy everything except the AVBufferRefs back
+     * note that we make a COPY of the side data, so calling av_frame_free() on
+     * the caller's frame will work properly */
+    ret = av_frame_copy_props(frame, avci->to_free);
+    if (ret < 0)
+        return ret;
+
+    memcpy(frame->data,     avci->to_free->data,     sizeof(frame->data));
+    memcpy(frame->linesize, avci->to_free->linesize, sizeof(frame->linesize));
+    if (avci->to_free->extended_data != avci->to_free->data) {
+        int planes = av_get_channel_layout_nb_channels(avci->to_free->channel_layout);
+        int size   = planes * sizeof(*frame->extended_data);
+
+        if (!size) {
+            av_frame_unref(frame);
+            return AVERROR_BUG;
+        }
+
+        frame->extended_data = av_malloc(size);
+        if (!frame->extended_data) {
+            av_frame_unref(frame);
+            return AVERROR(ENOMEM);
+        }
+        memcpy(frame->extended_data, avci->to_free->extended_data,
+               size);
+    } else
+        frame->extended_data = frame->data;
+
+    frame->format         = avci->to_free->format;
+    frame->width          = avci->to_free->width;
+    frame->height         = avci->to_free->height;
+    frame->channel_layout = avci->to_free->channel_layout;
+    frame->nb_samples     = avci->to_free->nb_samples;
+
+    return 0;
+}
+
 int attribute_align_arg avcodec_decode_video2(AVCodecContext *avctx, AVFrame *picture,
                                               int *got_picture_ptr,
                                               AVPacket *avpkt)
@@ -1394,9 +1445,6 @@ int attribute_align_arg avcodec_decode_video2(AVCodecContext *avctx, AVFrame *pi
 
     avcodec_get_frame_defaults(picture);
 
-    if (!avctx->refcounted_frames)
-        av_frame_unref(&avci->to_free);
-
     if ((avctx->codec->capabilities & CODEC_CAP_DELAY) || avpkt->size || (avctx->active_thread_type & FF_THREAD_FRAME)) {
         if (HAVE_THREADS && avctx->active_thread_type & FF_THREAD_FRAME)
             ret = ff_thread_decode_frame(avctx, picture, got_picture_ptr,
@@ -1418,9 +1466,9 @@ int attribute_align_arg avcodec_decode_video2(AVCodecContext *avctx, AVFrame *pi
 
         if (*got_picture_ptr) {
             if (!avctx->refcounted_frames) {
-                avci->to_free = *picture;
+                int err = unrefcount_frame(avci, picture);
-                avci->to_free.extended_data = avci->to_free.data;
+                if (err < 0)
-                memset(picture->buf, 0, sizeof(picture->buf));
+                    return err;
             }
 
             avctx->frame_number++;
@@ -1458,9 +1506,6 @@ int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
 
     avcodec_get_frame_defaults(frame);
 
-    if (!avctx->refcounted_frames)
-        av_frame_unref(&avci->to_free);
-
     if ((avctx->codec->capabilities & CODEC_CAP_DELAY) || avpkt->size) {
         ret = avctx->codec->decode(avctx, frame, got_frame_ptr, avpkt);
         if (ret >= 0 && *got_frame_ptr) {
@@ -1470,11 +1515,9 @@ int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
                 frame->format = avctx->sample_fmt;
 
             if (!avctx->refcounted_frames) {
-                avci->to_free = *frame;
+                int err = unrefcount_frame(avci, frame);
-                avci->to_free.extended_data = avci->to_free.data;
+                if (err < 0)
-                memset(frame->buf, 0, sizeof(frame->buf));
+                    return err;
-                frame->extended_buf    = NULL;
-                frame->nb_extended_buf = 0;
             }
         } else
             av_frame_unref(frame);
@@ -1540,8 +1583,7 @@ av_cold int avcodec_close(AVCodecContext *avctx)
         if (avctx->codec && avctx->codec->close)
             avctx->codec->close(avctx);
         avctx->coded_frame = NULL;
-        if (!avctx->refcounted_frames)
+        av_frame_free(&avctx->internal->to_free);
-            av_frame_unref(&avctx->internal->to_free);
         for (i = 0; i < FF_ARRAY_ELEMS(pool->pools); i++)
             av_buffer_pool_uninit(&pool->pools[i]);
         av_freep(&avctx->internal->pool);
@@ -1818,7 +1860,7 @@ void avcodec_flush_buffers(AVCodecContext *avctx)
         avctx->codec->flush(avctx);
 
     if (!avctx->refcounted_frames)
-        av_frame_unref(&avctx->internal->to_free);
+        av_frame_unref(avctx->internal->to_free);
 }
 
 int av_get_exact_bits_per_sample(enum AVCodecID codec_id)