diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 050ca92292..74840e5909 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -337,7 +337,7 @@ int ff_h264_build_ref_list(H264Context *h, H264SliceContext *sl)
                         break;
                 }
                 if (i >= 0)
-                    ref->pic_id = pred;
+                    pic_id = pred;
                 break;
             }
             case 2: {
@@ -354,7 +354,6 @@ int ff_h264_build_ref_list(H264Context *h, H264SliceContext *sl)
                 ref = h->long_ref[long_idx];
                 assert(!(ref && !ref->reference));
                 if (ref && (ref->reference & pic_structure)) {
-                    ref->pic_id = pic_id;
                     assert(ref->long_ref);
                     i = 0;
                 } else {
@@ -376,7 +375,7 @@ int ff_h264_build_ref_list(H264Context *h, H264SliceContext *sl)
                 for (i = index; i + 1 < sl->ref_count[list]; i++) {
                     if (sl->ref_list[list][i].parent &&
                         ref->long_ref == sl->ref_list[list][i].parent->long_ref &&
-                        ref->pic_id   == sl->ref_list[list][i].pic_id)
+                        pic_id        == sl->ref_list[list][i].pic_id)
                         break;
                 }
                 for (; i > index; i--) {