diff --git a/libavfilter/f_select.c b/libavfilter/f_select.c
index 5c7372c976..755e10a399 100644
--- a/libavfilter/f_select.c
+++ b/libavfilter/f_select.c
@@ -209,9 +209,13 @@ static int config_input(AVFilterLink *inlink)
 {
     SelectContext *select = inlink->dst->priv;
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
+    int is_yuv = !(desc->flags & AV_PIX_FMT_FLAG_RGB) &&
+                 (desc->flags & AV_PIX_FMT_FLAG_PLANAR) &&
+                 desc->nb_components >= 3;
 
     select->bitdepth = desc->comp[0].depth;
-    select->nb_planes = av_pix_fmt_count_planes(inlink->format);
+    select->nb_planes = is_yuv ? 1 : av_pix_fmt_count_planes(inlink->format);
+
     for (int plane = 0; plane < select->nb_planes; plane++) {
         ptrdiff_t line_size = av_image_get_linesize(inlink->format, inlink->w, plane);
         int vsub = desc->log2_chroma_h;
diff --git a/tests/ref/fate/filter-metadata-scenedetect b/tests/ref/fate/filter-metadata-scenedetect
index 7ce2d6794e..36c033bc0e 100644
--- a/tests/ref/fate/filter-metadata-scenedetect
+++ b/tests/ref/fate/filter-metadata-scenedetect
@@ -1,11 +1,11 @@
 pkt_pts=1620|tag:lavfi.scene_score=1.000000
-pkt_pts=4140|tag:lavfi.scene_score=0.668643
-pkt_pts=5800|tag:lavfi.scene_score=0.996721
-pkt_pts=6720|tag:lavfi.scene_score=0.357390
-pkt_pts=8160|tag:lavfi.scene_score=0.886268
-pkt_pts=9760|tag:lavfi.scene_score=0.926219
-pkt_pts=14080|tag:lavfi.scene_score=0.650033
+pkt_pts=4140|tag:lavfi.scene_score=0.923403
+pkt_pts=5800|tag:lavfi.scene_score=1.000000
+pkt_pts=6720|tag:lavfi.scene_score=0.475643
+pkt_pts=8160|tag:lavfi.scene_score=1.000000
+pkt_pts=9760|tag:lavfi.scene_score=1.000000
+pkt_pts=14080|tag:lavfi.scene_score=0.874623
 pkt_pts=15700|tag:lavfi.scene_score=1.000000
-pkt_pts=18500|tag:lavfi.scene_score=0.316402
-pkt_pts=20040|tag:lavfi.scene_score=0.269509
+pkt_pts=18500|tag:lavfi.scene_score=0.422509
+pkt_pts=20040|tag:lavfi.scene_score=0.352360
 pkt_pts=21760|tag:lavfi.scene_score=1.000000