diff --git a/libavformat/mov.c b/libavformat/mov.c
index 3491d680db..60309b4869 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -5050,7 +5050,7 @@ static int mov_read_sidx(MOVContext *c, AVIOContext *pb, MOVAtom atom)
                 }
             }
         }
-        for (i = 0; i < c->fc->nb_streams; i++) {
+        if (ref_st) for (i = 0; i < c->fc->nb_streams; i++) {
             st = c->fc->streams[i];
             sc = st->priv_data;
             if (!sc->has_sidx) {