diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index a9ed4f78db..1a96520ff9 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -386,17 +386,13 @@ static int alloc_scratch_buffers(H264Context *h, int linesize)
     // edge emu needs blocksize + filter length - 1
     // (= 21x21 for  h264)
     h->edge_emu_buffer = av_mallocz(alloc_size * 2 * 21);
-    h->me.scratchpad   = av_mallocz(alloc_size * 2 * 16 * 2);
 
-    if (!h->bipred_scratchpad || !h->edge_emu_buffer || !h->me.scratchpad) {
+    if (!h->bipred_scratchpad || !h->edge_emu_buffer) {
         av_freep(&h->bipred_scratchpad);
         av_freep(&h->edge_emu_buffer);
-        av_freep(&h->me.scratchpad);
         return AVERROR(ENOMEM);
     }
 
-    h->me.temp = h->me.scratchpad;
-
     return 0;
 }
 
@@ -1252,7 +1248,6 @@ static void free_tables(H264Context *h, int free_rbsp)
         av_freep(&hx->bipred_scratchpad);
         av_freep(&hx->edge_emu_buffer);
         av_freep(&hx->dc_val_base);
-        av_freep(&hx->me.scratchpad);
         av_freep(&hx->er.mb_index2xy);
         av_freep(&hx->er.error_status_table);
         av_freep(&hx->er.er_temp_buffer);
@@ -1424,7 +1419,6 @@ static void clone_tables(H264Context *dst, H264Context *src, int i)
     dst->cur_pic                = src->cur_pic;
     dst->bipred_scratchpad      = NULL;
     dst->edge_emu_buffer        = NULL;
-    dst->me.scratchpad          = NULL;
     ff_h264_pred_init(&dst->hpc, src->avctx->codec_id, src->sps.bit_depth_luma,
                       src->sps.chroma_format_idc);
 }
@@ -1809,7 +1803,6 @@ static int decode_update_thread_context(AVCodecContext *dst,
         memset(h->pps_buffers, 0, sizeof(h->pps_buffers));
 
         memset(&h->er, 0, sizeof(h->er));
-        memset(&h->me, 0, sizeof(h->me));
         memset(&h->mb, 0, sizeof(h->mb));
         memset(&h->mb_luma_dc, 0, sizeof(h->mb_luma_dc));
         memset(&h->mb_padding, 0, sizeof(h->mb_padding));
@@ -3499,8 +3492,8 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     int needs_reinit = 0;
     int field_pic_flag, bottom_field_flag;
 
-    h->me.qpel_put = h->h264qpel.put_h264_qpel_pixels_tab;
-    h->me.qpel_avg = h->h264qpel.avg_h264_qpel_pixels_tab;
+    h->qpel_put = h->h264qpel.put_h264_qpel_pixels_tab;
+    h->qpel_avg = h->h264qpel.avg_h264_qpel_pixels_tab;
 
     first_mb_in_slice = get_ue_golomb_long(&h->gb);
 
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index a18c96ded0..0b199462e9 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -30,6 +30,7 @@
 
 #include "libavutil/intreadwrite.h"
 #include "cabac.h"
+#include "dsputil.h"
 #include "error_resilience.h"
 #include "get_bits.h"
 #include "mpegvideo.h"
@@ -335,7 +336,6 @@ typedef struct H264Context {
     H264DSPContext h264dsp;
     H264ChromaContext h264chroma;
     H264QpelContext h264qpel;
-    MotionEstContext me;
     ParseContext parse_context;
     GetBitContext gb;
     DSPContext       dsp;
@@ -744,6 +744,10 @@ typedef struct H264Context {
     AVBufferPool *mb_type_pool;
     AVBufferPool *motion_val_pool;
     AVBufferPool *ref_index_pool;
+
+    /* Motion Estimation */
+    qpel_mc_func (*qpel_put)[16];
+    qpel_mc_func (*qpel_avg)[16];
 } H264Context;
 
 extern const uint8_t ff_h264_chroma_qp[7][QP_MAX_NUM + 1]; ///< One chroma qp table for each possible bit depth (8-14).
diff --git a/libavcodec/h264_mb_template.c b/libavcodec/h264_mb_template.c
index da3a9263de..1c13825a85 100644
--- a/libavcodec/h264_mb_template.c
+++ b/libavcodec/h264_mb_template.c
@@ -172,14 +172,14 @@ static av_noinline void FUNC(hl_decode_mb)(H264Context *h)
         } else if (is_h264) {
             if (chroma422) {
                 FUNC(hl_motion_422)(h, dest_y, dest_cb, dest_cr,
-                              h->me.qpel_put, h->h264chroma.put_h264_chroma_pixels_tab,
-                              h->me.qpel_avg, h->h264chroma.avg_h264_chroma_pixels_tab,
+                              h->qpel_put, h->h264chroma.put_h264_chroma_pixels_tab,
+                              h->qpel_avg, h->h264chroma.avg_h264_chroma_pixels_tab,
                               h->h264dsp.weight_h264_pixels_tab,
                               h->h264dsp.biweight_h264_pixels_tab);
             } else {
                 FUNC(hl_motion_420)(h, dest_y, dest_cb, dest_cr,
-                              h->me.qpel_put, h->h264chroma.put_h264_chroma_pixels_tab,
-                              h->me.qpel_avg, h->h264chroma.avg_h264_chroma_pixels_tab,
+                              h->qpel_put, h->h264chroma.put_h264_chroma_pixels_tab,
+                              h->qpel_avg, h->h264chroma.avg_h264_chroma_pixels_tab,
                               h->h264dsp.weight_h264_pixels_tab,
                               h->h264dsp.biweight_h264_pixels_tab);
             }
@@ -351,8 +351,8 @@ static av_noinline void FUNC(hl_decode_mb_444)(H264Context *h)
                                linesize, 0, 1, SIMPLE, PIXEL_SHIFT);
         } else {
             FUNC(hl_motion_444)(h, dest[0], dest[1], dest[2],
-                      h->me.qpel_put, h->h264chroma.put_h264_chroma_pixels_tab,
-                      h->me.qpel_avg, h->h264chroma.avg_h264_chroma_pixels_tab,
+                      h->qpel_put, h->h264chroma.put_h264_chroma_pixels_tab,
+                      h->qpel_avg, h->h264chroma.avg_h264_chroma_pixels_tab,
                       h->h264dsp.weight_h264_pixels_tab,
                       h->h264dsp.biweight_h264_pixels_tab);
         }