diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 4136d9fda2..f4aeb19aef 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -2104,6 +2104,8 @@ int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
             if(frame->nb_samples <= avctx->internal->skip_samples){
                 *got_frame_ptr = 0;
                 avctx->internal->skip_samples -= frame->nb_samples;
+                if (avctx->refcounted_frames)
+                    av_frame_unref(frame);
                 av_log(avctx, AV_LOG_DEBUG, "skip whole frame, skip left: %d\n",
                        avctx->internal->skip_samples);
             } else {