diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c
index 7aaae9b792..7ee375d4d0 100644
--- a/libavcodec/vp9.c
+++ b/libavcodec/vp9.c
@@ -112,10 +112,20 @@ static int vp9_frame_alloc(AVCodecContext *avctx, VP9Frame *f)
         return ret;
 
     sz = 64 * s->sb_cols * s->sb_rows;
-    f->extradata = av_buffer_allocz(sz * (1 + sizeof(VP9mvrefPair)));
+    if (sz != s->frame_extradata_pool_size) {
+        av_buffer_pool_uninit(&s->frame_extradata_pool);
+        s->frame_extradata_pool = av_buffer_pool_init(sz * (1 + sizeof(VP9mvrefPair)), NULL);
+        if (!s->frame_extradata_pool) {
+            s->frame_extradata_pool_size = 0;
+            goto fail;
+        }
+        s->frame_extradata_pool_size = sz;
+    }
+    f->extradata = av_buffer_pool_get(s->frame_extradata_pool);
     if (!f->extradata) {
         goto fail;
     }
+    memset(f->extradata->data, 0, f->extradata->size);
 
     f->segmentation_map = f->extradata->data;
     f->mv = (VP9mvrefPair *) (f->extradata->data + sz);
@@ -1210,6 +1220,7 @@ static av_cold int vp9_decode_free(AVCodecContext *avctx)
             vp9_frame_unref(avctx, &s->s.frames[i]);
         av_frame_free(&s->s.frames[i].tf.f);
     }
+    av_buffer_pool_uninit(&s->frame_extradata_pool);
     for (i = 0; i < 8; i++) {
         if (s->s.refs[i].f->buf[0])
             ff_thread_release_buffer(avctx, &s->s.refs[i]);
diff --git a/libavcodec/vp9dec.h b/libavcodec/vp9dec.h
index 66573edc79..de02b146f0 100644
--- a/libavcodec/vp9dec.h
+++ b/libavcodec/vp9dec.h
@@ -152,6 +152,10 @@ typedef struct VP9Context {
     int block_alloc_using_2pass;
     uint16_t mvscale[3][2];
     uint8_t mvstep[3][2];
+
+    // frame specific buffer pools
+    AVBufferPool *frame_extradata_pool;
+    int frame_extradata_pool_size;
 } VP9Context;
 
 struct VP9TileData {