diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index d16be90c8f..dd40c7fafb 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -657,6 +657,7 @@ typedef struct AVStream {
         int duration_count;
         double duration_error[MAX_STD_TIMEBASES];
         int64_t codec_info_duration;
+        int nb_decoded_frames;
     } *info;
 #if !FF_API_REORDER_PRIVATE
     const uint8_t *cur_ptr;
diff --git a/libavformat/utils.c b/libavformat/utils.c
index 81eee97a84..1158079992 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -2119,7 +2119,7 @@ static int has_codec_parameters(AVCodecContext *avctx)
 static int has_decode_delay_been_guessed(AVStream *st)
 {
     return st->codec->codec_id != CODEC_ID_H264 ||
-        st->codec_info_nb_frames >= 6 + st->codec->has_b_frames;
+        st->info->nb_decoded_frames >= 6;
 }
 
 static int try_decode_frame(AVStream *st, AVPacket *avpkt, AVDictionary **options)
@@ -2145,6 +2145,8 @@ static int try_decode_frame(AVStream *st, AVPacket *avpkt, AVDictionary **option
             avcodec_get_frame_defaults(&picture);
             ret = avcodec_decode_video2(st->codec, &picture,
                                         &got_picture, avpkt);
+            if (got_picture)
+                st->info->nb_decoded_frames++;
             break;
         case AVMEDIA_TYPE_AUDIO:
             data_size = FFMAX(avpkt->size, AVCODEC_MAX_AUDIO_FRAME_SIZE);