diff --git a/libavcodec/dovi_rpu.c b/libavcodec/dovi_rpu.c
index b67978403f..5130a9598d 100644
--- a/libavcodec/dovi_rpu.c
+++ b/libavcodec/dovi_rpu.c
@@ -66,7 +66,6 @@ void ff_dovi_ctx_replace(DOVIContext *s, const DOVIContext *s0)
     for (int i = 0; i <= DOVI_MAX_DM_ID; i++)
         ff_refstruct_replace(&s->vdr[i], s0->vdr[i]);
     ff_refstruct_replace(&s->ext_blocks, s0->ext_blocks);
-    s->num_ext_blocks = s0->num_ext_blocks;
 }
 
 int ff_dovi_guess_profile_hevc(const AVDOVIRpuDataHeader *hdr)
diff --git a/libavcodec/dovi_rpu.h b/libavcodec/dovi_rpu.h
index 24a8353bdc..ed5bfa7b26 100644
--- a/libavcodec/dovi_rpu.h
+++ b/libavcodec/dovi_rpu.h
@@ -31,6 +31,12 @@
 #include "codec_par.h"
 
 #define DOVI_MAX_DM_ID 15
+
+typedef struct DOVIExt {
+    AVDOVIDmData dm[AV_DOVI_MAX_EXT_BLOCKS];
+    int num_dm;
+} DOVIExt;
+
 typedef struct DOVIContext {
     void *logctx;
 
@@ -70,8 +76,7 @@ typedef struct DOVIContext {
      * Currently active extension blocks, updates on every ff_dovi_rpu_parse()
      * or ff_dovi_rpu_generate().
      */
-    AVDOVIDmData *ext_blocks;
-    int num_ext_blocks;
+    DOVIExt *ext_blocks; ///< RefStruct, or NULL if no extension blocks
 
     /**
      * Private fields internal to dovi_rpu.c
diff --git a/libavcodec/dovi_rpudec.c b/libavcodec/dovi_rpudec.c
index 0ddc923539..1650547c80 100644
--- a/libavcodec/dovi_rpudec.c
+++ b/libavcodec/dovi_rpudec.c
@@ -33,7 +33,7 @@
 int ff_dovi_get_metadata(DOVIContext *s, AVDOVIMetadata **out_metadata)
 {
     AVDOVIMetadata *dovi;
-    size_t dovi_size, ext_sz;
+    size_t dovi_size;
 
     if (!s->mapping || !s->color)
         return 0; /* incomplete dovi metadata */
@@ -47,10 +47,14 @@ int ff_dovi_get_metadata(DOVIContext *s, AVDOVIMetadata **out_metadata)
     COPY(AVDOVIRpuDataHeader, av_dovi_get_header(dovi), &s->header, ext_mapping_idc_5_7);
     COPY(AVDOVIDataMapping, av_dovi_get_mapping(dovi), s->mapping, nlq_pivots);
     COPY(AVDOVIColorMetadata, av_dovi_get_color(dovi), s->color, source_diagonal);
-    ext_sz = FFMIN(sizeof(AVDOVIDmData), dovi->ext_block_size);
-    for (int i = 0; i < s->num_ext_blocks; i++)
-        memcpy(av_dovi_get_ext(dovi, i), &s->ext_blocks[i], ext_sz);
-    dovi->num_ext_blocks = s->num_ext_blocks;
+
+    if (s->ext_blocks) {
+        const DOVIExt *ext = s->ext_blocks;
+        size_t ext_sz = FFMIN(sizeof(AVDOVIDmData), dovi->ext_block_size);
+        for (int i = 0; i < ext->num_dm; i++)
+            memcpy(av_dovi_get_ext(dovi, i), &ext->dm[i], ext_sz);
+        dovi->num_ext_blocks = ext->num_dm;
+    }
 
     *out_metadata = dovi;
     return dovi_size;
@@ -279,20 +283,24 @@ static int parse_ext_v2(DOVIContext *s, GetBitContext *gb, AVDOVIDmData *dm,
 static int parse_ext_blocks(DOVIContext *s, GetBitContext *gb, int ver)
 {
     int num_ext_blocks, ext_block_length, start_pos, parsed_bits, ret;
+    DOVIExt *ext = s->ext_blocks;
 
     num_ext_blocks = get_ue_golomb_31(gb);
     align_get_bits(gb);
-    if (s->num_ext_blocks + num_ext_blocks > AV_DOVI_MAX_EXT_BLOCKS)
-        return AVERROR_INVALIDDATA;
 
-    if (!s->ext_blocks) {
-        s->ext_blocks = ff_refstruct_allocz(sizeof(AVDOVIDmData) * AV_DOVI_MAX_EXT_BLOCKS);
-        if (!s->ext_blocks)
+    if (!ext) {
+        ext = s->ext_blocks = ff_refstruct_allocz(sizeof(*s->ext_blocks));
+        if (!ext)
             return AVERROR(ENOMEM);
     }
 
     while (num_ext_blocks--) {
-        AVDOVIDmData *dm = &s->ext_blocks[s->num_ext_blocks++];
+        AVDOVIDmData *dm;
+
+        if (ext->num_dm >= FF_ARRAY_ELEMS(ext->dm))
+            return AVERROR_INVALIDDATA;
+        dm = &ext->dm[ext->num_dm++];
+
         ext_block_length = get_ue_golomb_31(gb);
         dm->level = get_bits(gb, 8);
         start_pos = get_bits_count(gb);
@@ -666,7 +674,8 @@ int ff_dovi_rpu_parse(DOVIContext *s, const uint8_t *rpu, size_t rpu_size,
         color->source_diagonal = get_bits(gb, 10);
 
         /* Parse extension blocks */
-        s->num_ext_blocks = 0;
+        if (s->ext_blocks)
+            s->ext_blocks->num_dm = 0;
         if ((ret = parse_ext_blocks(s, gb, 1)) < 0) {
             ff_dovi_ctx_unref(s);
             return ret;
@@ -680,7 +689,7 @@ int ff_dovi_rpu_parse(DOVIContext *s, const uint8_t *rpu, size_t rpu_size,
         }
     } else {
         s->color = &ff_dovi_color_default;
-        s->num_ext_blocks = 0;
+        ff_refstruct_unref(&s->ext_blocks);
     }
 
     return 0;
diff --git a/libavcodec/dovi_rpuenc.c b/libavcodec/dovi_rpuenc.c
index 25e520dd92..667d681c25 100644
--- a/libavcodec/dovi_rpuenc.c
+++ b/libavcodec/dovi_rpuenc.c
@@ -583,7 +583,7 @@ int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
     }
 
     if (metadata->num_ext_blocks && !s->ext_blocks) {
-        s->ext_blocks = ff_refstruct_allocz(sizeof(AVDOVIDmData) * AV_DOVI_MAX_EXT_BLOCKS);
+        s->ext_blocks = ff_refstruct_allocz(sizeof(*s->ext_blocks));
         if (!s->ext_blocks)
             return AVERROR(ENOMEM);
     }
@@ -717,7 +717,7 @@ int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
     }
 
     if (vdr_dm_metadata_present) {
-        size_t ext_sz;
+        DOVIExt *ext = s->ext_blocks;
         const int denom = profile == 4 ? (1 << 30) : (1 << 28);
         set_ue_golomb(pb, color->dm_metadata_id); /* affected_dm_id */
         set_ue_golomb(pb, color->dm_metadata_id); /* current_dm_id */
@@ -756,13 +756,15 @@ int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
                 generate_ext_v2(pb, av_dovi_get_ext(metadata, i));
         }
 
-        ext_sz = FFMIN(sizeof(AVDOVIDmData), metadata->ext_block_size);
-        for (int i = 0; i < metadata->num_ext_blocks; i++)
-            memcpy(&s->ext_blocks[i], av_dovi_get_ext(metadata, i), ext_sz);
-        s->num_ext_blocks = metadata->num_ext_blocks;
+        if (ext) {
+            size_t ext_sz = FFMIN(sizeof(AVDOVIDmData), metadata->ext_block_size);
+            for (int i = 0; i < metadata->num_ext_blocks; i++)
+                memcpy(&ext->dm[i], av_dovi_get_ext(metadata, i), ext_sz);
+            ext->num_dm = metadata->num_ext_blocks;
+        }
     } else {
         s->color = &ff_dovi_color_default;
-        s->num_ext_blocks = 0;
+        ff_refstruct_unref(&s->ext_blocks);
     }
 
     flush_put_bits(pb);