diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c
index 76fd37248b..7b7deb3aa6 100644
--- a/libavcodec/vp9.c
+++ b/libavcodec/vp9.c
@@ -3826,6 +3826,8 @@ static int vp9_decode_frame(AVCodecContext *ctx, void *frame,
         }
         s->prob_ctx[s->framectxid].p = s->prob.p;
         ff_thread_finish_setup(ctx);
+    } else if (!s->refreshctx) {
+        ff_thread_finish_setup(ctx);
     }
 
     do {