diff --git a/libavformat/concatdec.c b/libavformat/concatdec.c
index 8fff9cc2cb..bbe13136fa 100644
--- a/libavformat/concatdec.c
+++ b/libavformat/concatdec.c
@@ -603,7 +603,6 @@ static int concat_read_packet(AVFormatContext *avf, AVPacket *pkt)
             av_packet_unref(pkt);
             continue;
         }
-        pkt->stream_index = cs->out_stream_index;
         break;
     }
     if ((ret = filter_packet(avf, cs, pkt)))
@@ -646,6 +645,7 @@ static int concat_read_packet(AVFormatContext *avf, AVPacket *pkt)
         }
     }
 
+    pkt->stream_index = cs->out_stream_index;
     return ret;
 }