diff --git a/libavcodec/bsf/h264_mp4toannexb.c b/libavcodec/bsf/h264_mp4toannexb.c
index dda064287e..36b09b4dd6 100644
--- a/libavcodec/bsf/h264_mp4toannexb.c
+++ b/libavcodec/bsf/h264_mp4toannexb.c
@@ -93,6 +93,11 @@ static int h264_extradata_to_annexb(AVBSFContext *ctx,
     const int padding                   = AV_INPUT_BUFFER_PADDING_SIZE;
     int length_size, pps_offset = 0;
 
+    if (extradata_size < 7) {
+        av_log(ctx, AV_LOG_ERROR, "Invalid extradata size: %d\n", extradata_size);
+        return AVERROR_INVALIDDATA;
+    }
+
     bytestream2_init(gb, extradata, extradata_size);
 
     bytestream2_skipu(gb, 4);
@@ -262,16 +267,11 @@ static int h264_mp4toannexb_init(AVBSFContext *ctx)
         (extra_size >= 4 && AV_RB32(ctx->par_in->extradata) == 1)) {
         av_log(ctx, AV_LOG_VERBOSE,
                "The input looks like it is Annex B already\n");
-    } else if (extra_size >= 7) {
-        return h264_extradata_to_annexb(ctx,
-                                        ctx->par_in->extradata,
-                                        ctx->par_in->extradata_size);
-    } else {
-        av_log(ctx, AV_LOG_ERROR, "Invalid extradata size: %d\n", extra_size);
-        return AVERROR_INVALIDDATA;
+        return 0;
     }
-
-    return 0;
+    return h264_extradata_to_annexb(ctx,
+                                    ctx->par_in->extradata,
+                                    ctx->par_in->extradata_size);
 }
 
 static int h264_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *opkt)
@@ -293,10 +293,12 @@ static int h264_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *opkt)
 
     extradata = av_packet_get_side_data(in, AV_PKT_DATA_NEW_EXTRADATA,
                                         &extradata_size);
-    if (extradata) {
+    if (extradata && extradata[0] == 1) {
         ret = h264_extradata_to_annexb(ctx, extradata, extradata_size);
         if (ret < 0)
             goto fail;
+        av_packet_side_data_remove(in->side_data, &in->side_data_elems,
+                                   AV_PKT_DATA_NEW_EXTRADATA);
     }
 
     /* nothing to filter */
diff --git a/tests/fate/h264.mak b/tests/fate/h264.mak
index 00b153cc93..8aa780ed4b 100644
--- a/tests/fate/h264.mak
+++ b/tests/fate/h264.mak
@@ -225,7 +225,7 @@ FATE_H264-$(call FRAMECRC, MOV, H264) += fate-h264-unescaped-extradata
 # this sample contains field-coded frames, with both fields in a single packet
 FATE_H264-$(call FRAMECRC, MOV, H264) += fate-h264-twofields-packet
 
-FATE_H264-$(call DEMMUX, MOV, H264, H264_MP4TOANNEXB_BSF SCALE_FILTER) += fate-h264-bsf-mp4toannexb-new-extradata
+FATE_H264-$(call DEMMUX, MOV, H264, H264_MP4TOANNEXB_BSF H264_METADATA_BSF SCALE_FILTER) += fate-h264-bsf-mp4toannexb-new-extradata
 
 FATE_H264-$(call DEMMUX, MOV, H264, H264_MP4TOANNEXB_BSF) += fate-h264-bsf-mp4toannexb \
                                                              fate-h264-bsf-mp4toannexb-2 \
@@ -440,7 +440,7 @@ fate-h264-bsf-mp4toannexb:                        CMD = md5 -i $(TARGET_SAMPLES)
 fate-h264-bsf-mp4toannexb-2:                      CMD = md5 -i $(TARGET_SAMPLES)/h264/ps_prefix_first_idr.mp4 -c:v copy -f h264
 fate-h264-bsf-mp4toannexb-2:                      CMP = oneline
 fate-h264-bsf-mp4toannexb-2:                      REF = cffcfa6a2d0b58c9de1f5785f099f41d
-fate-h264-bsf-mp4toannexb-new-extradata:          CMD = stream_remux mov $(TARGET_SAMPLES)/h264/extradata-reload-multi-stsd.mov "" h264 "-map 0:v"
+fate-h264-bsf-mp4toannexb-new-extradata:          CMD = stream_remux mov $(TARGET_SAMPLES)/h264/extradata-reload-multi-stsd.mov "" h264 "-bsf h264_mp4toannexb,h264_metadata -map 0:v"
 fate-h264-bsf-dts2pts:                            CMD = transcode "h264" $(TARGET_SAMPLES)/h264-conformance/CAPAMA3_Sand_F.264 \
                                                         mov "-c:v copy -bsf:v dts2pts -frames:v 50" "-c:v copy"
 fate-h264_mp4toannexb_ticket5927:                 CMD = transcode "mp4" $(TARGET_SAMPLES)/h264/thezerotheorem-cut.mp4 \