diff --git a/libavcodec/dxva2_hevc.c b/libavcodec/dxva2_hevc.c
index c91bcf3eeb..6b239d9917 100644
--- a/libavcodec/dxva2_hevc.c
+++ b/libavcodec/dxva2_hevc.c
@@ -184,7 +184,7 @@ static void fill_picture_parameters(const AVCodecContext *avctx, AVDXVAContext *
             const HEVCFrame *frame = NULL; \
             while (!frame && j < rpl->nb_refs) \
                 frame = rpl->ref[j++]; \
-            if (frame) \
+            if (frame && frame->flags & (HEVC_FRAME_FLAG_LONG_REF | HEVC_FRAME_FLAG_SHORT_REF)) \
                 pp->ref_list[i] = get_refpic_index(pp, ff_dxva2_get_surface_index(avctx, ctx, frame->frame)); \
             else \
                 pp->ref_list[i] = 0xff; \