diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c
index 69bc003490..5fd77073da 100644
--- a/libavcodec/mjpegdec.c
+++ b/libavcodec/mjpegdec.c
@@ -854,9 +854,9 @@ static int decode_block(MJpegDecodeContext *s, int16_t *block, int component,
         i += ((unsigned)code) >> 4;
             code &= 0xf;
         if (code) {
-            if (code > MIN_CACHE_BITS - 16)
-                UPDATE_CACHE(re, &s->gb);
-
+            // GET_VLC updates the cache if parsing reaches the second stage.
+            // So we have at least MIN_CACHE_BITS - 9 > 15 bits left here
+            // and don't need to refill the cache.
             {
                 int cache = GET_CACHE(re, &s->gb);
                 int sign  = (~cache) >> 31;
@@ -918,8 +918,6 @@ static int decode_block_progressive(MJpegDecodeContext *s, int16_t *block,
             code &= 0xF;
             if (code) {
                 i += run;
-                if (code > MIN_CACHE_BITS - 16)
-                    UPDATE_CACHE(re, &s->gb);
 
                 {
                     int cache = GET_CACHE(re, &s->gb);
@@ -950,7 +948,8 @@ static int decode_block_progressive(MJpegDecodeContext *s, int16_t *block,
                 } else {
                     val = (1 << run);
                     if (run) {
-                        UPDATE_CACHE(re, &s->gb);
+                        // Given that GET_VLC reloads internally, we always
+                        // have at least 16 bits in the cache here.
                         val += NEG_USR32(GET_CACHE(re, &s->gb), run);
                         LAST_SKIP_BITS(re, &s->gb, run);
                     }
@@ -1012,7 +1011,6 @@ static int decode_block_refinement(MJpegDecodeContext *s, int16_t *block,
 
             if (code & 0xF) {
                 run = ((unsigned) code) >> 4;
-                UPDATE_CACHE(re, &s->gb);
                 val = SHOW_UBITS(re, &s->gb, 1);
                 LAST_SKIP_BITS(re, &s->gb, 1);
                 ZERO_RUN;
@@ -1033,7 +1031,8 @@ static int decode_block_refinement(MJpegDecodeContext *s, int16_t *block,
                     val = run;
                     run = (1 << run);
                     if (val) {
-                        UPDATE_CACHE(re, &s->gb);
+                        // Given that GET_VLC reloads internally, we always
+                        // have at least 16 bits in the cache here.
                         run += SHOW_UBITS(re, &s->gb, val);
                         LAST_SKIP_BITS(re, &s->gb, val);
                     }