diff --git a/libavcodec/cbs_av1.c b/libavcodec/cbs_av1.c
index b2a1d2ded1..75e9cb78df 100644
--- a/libavcodec/cbs_av1.c
+++ b/libavcodec/cbs_av1.c
@@ -939,8 +939,6 @@ static int cbs_av1_read_unit(CodedBitstreamContext *ctx,
         priv->spatial_id  = 0;
     }
 
-    priv->ref = (AV1ReferenceFrameState *)&priv->read_ref;
-
     switch (obu->header.obu_type) {
     case AV1_OBU_SEQUENCE_HEADER:
         {
@@ -1084,8 +1082,6 @@ static int cbs_av1_write_obu(CodedBitstreamContext *ctx,
     td = NULL;
     start_pos = put_bits_count(pbc);
 
-    priv->ref = (AV1ReferenceFrameState *)&priv->write_ref;
-
     switch (obu->header.obu_type) {
     case AV1_OBU_SEQUENCE_HEADER:
         {
diff --git a/libavcodec/cbs_av1.h b/libavcodec/cbs_av1.h
index fdc629b00c..f5fed220a5 100644
--- a/libavcodec/cbs_av1.h
+++ b/libavcodec/cbs_av1.h
@@ -441,9 +441,7 @@ typedef struct CodedBitstreamAV1Context {
     int tile_cols;
     int tile_rows;
 
-    AV1ReferenceFrameState *ref;
-    AV1ReferenceFrameState read_ref[AV1_NUM_REF_FRAMES];
-    AV1ReferenceFrameState write_ref[AV1_NUM_REF_FRAMES];
+    AV1ReferenceFrameState ref[AV1_NUM_REF_FRAMES];
 } CodedBitstreamAV1Context;