diff --git a/libavcodec/h261dec.c b/libavcodec/h261dec.c
index eb544e6046..0d8cd8c20d 100644
--- a/libavcodec/h261dec.c
+++ b/libavcodec/h261dec.c
@@ -79,6 +79,7 @@ static av_cold int h261_decode_init(AVCodecContext *avctx)
     avctx->pix_fmt = AV_PIX_FMT_YUV420P;
 
     h->gob_start_code_skipped = 0;
+    ff_mpv_idct_init(s);
 
     ff_thread_once(&init_static_once, h261_decode_init_static);
 
@@ -595,10 +596,6 @@ static int h261_decode_frame(AVCodecContext *avctx, void *data,
 retry:
     init_get_bits(&s->gb, buf, buf_size * 8);
 
-    if (!s->context_initialized)
-        // we need the IDCT permutation for reading a custom matrix
-        ff_mpv_idct_init(s);
-
     ret = h261_decode_picture_header(h);
 
     /* skip if the header was thrashed */