diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index d2fa73fb75..97d066f8f9 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -2666,6 +2666,9 @@ void avcodec_flush_buffers(AVCodecContext *avctx)
 
     avctx->pts_correction_last_pts =
     avctx->pts_correction_last_dts = INT64_MIN;
+
+    if (!avctx->refcounted_frames)
+        av_frame_unref(&avctx->internal->to_free);
 }
 
 int av_get_exact_bits_per_sample(enum AVCodecID codec_id)