diff --git a/libavcodec/h261enc.c b/libavcodec/h261enc.c
index dabab9d80a..5253e82129 100644
--- a/libavcodec/h261enc.c
+++ b/libavcodec/h261enc.c
@@ -46,6 +46,10 @@ static struct VLCLUT {
     uint16_t code;
 } vlc_lut[H261_MAX_RUN + 1][32 /* 0..2 * H261_MAX_LEN are used */];
 
+// Not const despite never being initialized because doing so would
+// put it into .rodata instead of .bss and bloat the binary.
+// mv_penalty exists so that the motion estimation code can avoid branches.
+static uint8_t mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
 static uint8_t uni_h261_rl_len     [64 * 128];
 static uint8_t uni_h261_rl_len_last[64 * 128];
 static uint8_t h261_mv_codes[64][2];
@@ -370,6 +374,8 @@ av_cold int ff_h261_encode_init(MpegEncContext *s)
     s->max_qcoeff       = 127;
     s->ac_esc_length    = H261_ESC_LEN;
 
+    s->me.mv_penalty = mv_penalty;
+
     s->intra_ac_vlc_length      = s->inter_ac_vlc_length      = uni_h261_rl_len;
     s->intra_ac_vlc_last_length = s->inter_ac_vlc_last_length = uni_h261_rl_len_last;
     ff_thread_once(&init_static_once, h261_encode_init_static);
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 8c22dbb5f5..1fe69f89a6 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -93,7 +93,6 @@ static int dct_quantize_c(MpegEncContext *s,
                           int qscale, int *overflow);
 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
 
-static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
 
 static const AVOption mpv_generic_options[] = {
@@ -287,7 +286,6 @@ static void mpv_encode_defaults(MpegEncContext *s)
 
     ff_thread_once(&init_static_once, mpv_encode_init_static);
 
-    s->me.mv_penalty = default_mv_penalty;
     s->fcode_tab     = default_fcode_tab;
 
     s->input_picture_number  = 0;