diff --git a/libavcodec/h263dec.c b/libavcodec/h263dec.c
index ddbada9841..fd1e36b68a 100644
--- a/libavcodec/h263dec.c
+++ b/libavcodec/h263dec.c
@@ -449,7 +449,6 @@ int ff_h263_decode_frame(AVCodecContext *avctx, AVFrame *pict,
         return 0;
     }
 
-retry:
     // s->gb might be overridden in ff_mpeg4_decode_picture_header() below.
     ret = init_get_bits8(&s->gb, buf, buf_size);
     if (ret < 0)
@@ -505,8 +504,7 @@ retry:
     if (CONFIG_MPEG4_DECODER && avctx->codec_id == AV_CODEC_ID_MPEG4) {
         if (s->pict_type != AV_PICTURE_TYPE_B && s->mb_num/2 > get_bits_left(&s->gb))
             return AVERROR_INVALIDDATA;
-        if (ff_mpeg4_workaround_bugs(avctx) == 1)
-            goto retry;
+        ff_mpeg4_workaround_bugs(avctx);
         if (s->studio_profile != (s->idsp.idct == NULL))
             ff_mpv_idct_init(s);
     }
diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c
index 5d675d2a78..2eb663603c 100644
--- a/libavcodec/mpeg4videodec.c
+++ b/libavcodec/mpeg4videodec.c
@@ -3021,7 +3021,36 @@ static int decode_user_data(Mpeg4DecContext *ctx, GetBitContext *gb)
     return 0;
 }
 
-int ff_mpeg4_workaround_bugs(AVCodecContext *avctx)
+static av_cold void permute_quant_matrix(uint16_t matrix[64],
+                                         const uint8_t new_perm[64],
+                                         const uint8_t old_perm[64])
+{
+    uint16_t tmp[64];
+
+    memcpy(tmp, matrix, sizeof(tmp));
+    for (int i = 0; i < 64; ++i)
+        matrix[new_perm[i]] = tmp[old_perm[i]];
+}
+
+static av_cold void switch_to_xvid_idct(AVCodecContext *const avctx,
+                                        MpegEncContext *const s)
+{
+    uint8_t old_permutation[64];
+
+    memcpy(old_permutation, s->idsp.idct_permutation, sizeof(old_permutation));
+
+    avctx->idct_algo = FF_IDCT_XVID;
+    ff_mpv_idct_init(s);
+    ff_permute_scantable(s->permutated_intra_h_scantable,
+                         s->alternate_scan ? ff_alternate_vertical_scan : ff_alternate_horizontal_scan,
+                         s->idsp.idct_permutation);
+
+    // Normal (i.e. non-studio) MPEG-4 does not use the chroma matrices.
+    permute_quant_matrix(s->inter_matrix, s->idsp.idct_permutation, old_permutation);
+    permute_quant_matrix(s->intra_matrix, s->idsp.idct_permutation, old_permutation);
+}
+
+void ff_mpeg4_workaround_bugs(AVCodecContext *avctx)
 {
     Mpeg4DecContext *ctx = avctx->priv_data;
     MpegEncContext *s = &ctx->m;
@@ -3129,13 +3158,9 @@ int ff_mpeg4_workaround_bugs(AVCodecContext *avctx)
                ctx->divx_version, ctx->divx_build, s->divx_packed ? "p" : "");
 
     if (CONFIG_MPEG4_DECODER && ctx->xvid_build >= 0 &&
-        avctx->idct_algo == FF_IDCT_AUTO) {
-        avctx->idct_algo = FF_IDCT_XVID;
-        ff_mpv_idct_init(s);
-        return 1;
+        avctx->idct_algo == FF_IDCT_AUTO && !s->studio_profile) {
+        switch_to_xvid_idct(avctx, s);
     }
-
-    return 0;
 }
 
 static int decode_vop_header(Mpeg4DecContext *ctx, GetBitContext *gb,
diff --git a/libavcodec/mpeg4videodec.h b/libavcodec/mpeg4videodec.h
index d3de81c5a9..59a7251ce2 100644
--- a/libavcodec/mpeg4videodec.h
+++ b/libavcodec/mpeg4videodec.h
@@ -111,7 +111,7 @@ void ff_mpeg4_mcsel_motion(MpegEncContext *s,
 int ff_mpeg4_decode_partitions(Mpeg4DecContext *ctx);
 int ff_mpeg4_decode_video_packet_header(Mpeg4DecContext *ctx);
 int ff_mpeg4_decode_studio_slice_header(Mpeg4DecContext *ctx);
-int ff_mpeg4_workaround_bugs(AVCodecContext *avctx);
+void ff_mpeg4_workaround_bugs(AVCodecContext *avctx);
 void ff_mpeg4_pred_ac(MpegEncContext *s, int16_t *block, int n,
                       int dir);
 int ff_mpeg4_frame_end(AVCodecContext *avctx, const AVPacket *pkt);