diff --git a/libavcodec/nvdec_mpeg12.c b/libavcodec/nvdec_mpeg12.c
index 9a9030d8d3..746d720888 100644
--- a/libavcodec/nvdec_mpeg12.c
+++ b/libavcodec/nvdec_mpeg12.c
@@ -80,8 +80,9 @@ static int nvdec_mpeg12_start_frame(AVCodecContext *avctx, const uint8_t *buffer
     };
 
     for (i = 0; i < 64; ++i) {
-        ppc->QuantMatrixIntra[i] = s->intra_matrix[i];
-        ppc->QuantMatrixInter[i] = s->inter_matrix[i];
+        int n = s->idsp.idct_permutation[i];
+        ppc->QuantMatrixIntra[i] = s->intra_matrix[n];
+        ppc->QuantMatrixInter[i] = s->inter_matrix[n];
     }
 
     return 0;