diff --git a/libavformat/oggdec.c b/libavformat/oggdec.c
index fc26314316..a36dfae234 100644
--- a/libavformat/oggdec.c
+++ b/libavformat/oggdec.c
@@ -223,6 +223,7 @@ static int ogg_new_stream(AVFormatContext *s, uint32_t serial)
     os->bufsize = DECODER_BUFFER_SIZE;
     os->buf = av_malloc(os->bufsize + FF_INPUT_BUFFER_PADDING_SIZE);
     os->header = -1;
+    os->start_granule = OGG_NOGRANULE_VALUE;
     if (!os->buf)
         return AVERROR(ENOMEM);
 
@@ -609,6 +610,9 @@ static int ogg_read_header(AVFormatContext *s)
         } else if (os->codec && os->nb_header < os->codec->nb_header) {
             av_log(s, AV_LOG_WARNING, "Number of headers (%d) mismatch for stream %d\n", os->nb_header, i);
         }
+        if (os->start_granule != OGG_NOGRANULE_VALUE)
+            os->lastpts = s->streams[i]->start_time =
+                ogg_gptopts(s, i, os->start_granule, NULL);
     }
 
     //linear granulepos seek from end
diff --git a/libavformat/oggdec.h b/libavformat/oggdec.h
index 79d8179d9d..7b93175383 100644
--- a/libavformat/oggdec.h
+++ b/libavformat/oggdec.h
@@ -67,6 +67,7 @@ struct ogg_stream {
     unsigned int pduration;
     uint32_t serial;
     uint64_t granule;
+    uint64_t start_granule;
     int64_t lastpts;
     int64_t lastdts;
     int64_t sync_pos;   ///< file offset of the first page needed to reconstruct the current packet
@@ -105,6 +106,8 @@ struct ogg {
 #define OGG_FLAG_BOS  2
 #define OGG_FLAG_EOS  4
 
+#define OGG_NOGRANULE_VALUE -1ull
+
 extern const struct ogg_codec ff_celt_codec;
 extern const struct ogg_codec ff_dirac_codec;
 extern const struct ogg_codec ff_flac_codec;
diff --git a/libavformat/oggparseskeleton.c b/libavformat/oggparseskeleton.c
index a9e4fc9ed0..4b02c838f2 100644
--- a/libavformat/oggparseskeleton.c
+++ b/libavformat/oggparseskeleton.c
@@ -30,7 +30,8 @@ static int skeleton_header(AVFormatContext *s, int idx)
     AVStream *st = s->streams[idx];
     uint8_t *buf = os->buf + os->pstart;
     int version_major, version_minor;
-    int64_t start_num, start_den, start_granule;
+    int64_t start_num, start_den;
+    uint64_t start_granule;
     int target_idx, start_time;
 
     strcpy(st->codec->codec_name, "skeleton");
@@ -73,12 +74,13 @@ static int skeleton_header(AVFormatContext *s, int idx)
 
         target_idx = ogg_find_stream(ogg, AV_RL32(buf+12));
         start_granule = AV_RL64(buf+36);
-        if (target_idx >= 0 && start_granule != -1) {
-            int64_t pts = ogg_gptopts(s, target_idx, start_granule, NULL);
-            if (pts == AV_NOPTS_VALUE)
-                return -1;
-            ogg->streams[target_idx].lastpts =
-            s->streams[target_idx]->start_time = pts;
+        if (os->start_granule != OGG_NOGRANULE_VALUE) {
+            av_log_missing_feature(s, "multiple fisbone for the "
+                                      "same stream\n", 0);
+            return 1;
+        }
+        if (target_idx >= 0 && start_granule != OGG_NOGRANULE_VALUE) {
+            os->start_granule = start_granule;
         }
     }