diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 976044ce2c..26711c0d85 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -815,6 +815,7 @@ int ff_h264_execute_ref_pic_marking(H264Context *h)
     if (   err >= 0
         && h->long_ref_count==0
         && (   h->short_ref_count<=2
+            || pps_ref_count[0] <= 2 && pps_ref_count[1] <= 1 && h->avctx->has_b_frames
             || pps_ref_count[0] <= 1 + (h->picture_structure != PICT_FRAME) && pps_ref_count[1] <= 1)
         && pps_ref_count[0]<=2 + (h->picture_structure != PICT_FRAME) + (2*!h->has_recovery_point)
         && h->cur_pic_ptr->f->pict_type == AV_PICTURE_TYPE_I){