diff --git a/libavformat/oggenc.c b/libavformat/oggenc.c
index 315b8da049..c92ee0cfa4 100644
--- a/libavformat/oggenc.c
+++ b/libavformat/oggenc.c
@@ -195,7 +195,8 @@ static int ogg_buffer_page(AVFormatContext *s, OGGStreamContext *oggstream)
 }
 
 static int ogg_buffer_data(AVFormatContext *s, AVStream *st,
-                           uint8_t *data, unsigned size, int64_t granule)
+                           uint8_t *data, unsigned size, int64_t granule,
+                           int header)
 {
     OGGStreamContext *oggstream = st->priv_data;
     OGGContext *ogg = s->priv_data;
@@ -207,9 +208,9 @@ static int ogg_buffer_data(AVFormatContext *s, AVStream *st,
     // For theora, keyframes also need to have a timestamp to correctly mark
     // them as such, otherwise seeking will not work correctly at the very
     // least with old libogg versions.
-    // Do not try to flush empty packets though, that will create broken files.
+    // Do not try to flush header packets though, that will create broken files.
     if (st->codec->codec_id == CODEC_ID_THEORA &&
-        oggstream->page.size &&
+        !header &&
         (ogg_granule_to_timestamp(oggstream, granule) >
          ogg_granule_to_timestamp(oggstream, oggstream->last_granule) + 1 ||
          ogg_key_granule(oggstream, granule))) {
@@ -440,7 +441,7 @@ static int ogg_write_header(AVFormatContext *s)
     for (j = 0; j < s->nb_streams; j++) {
         OGGStreamContext *oggstream = s->streams[j]->priv_data;
         ogg_buffer_data(s, s->streams[j], oggstream->header[0],
-                        oggstream->header_len[0], 0);
+                        oggstream->header_len[0], 0, 1);
         oggstream->page.flags |= 2; // bos
         ogg_buffer_page(s, oggstream);
     }
@@ -450,7 +451,7 @@ static int ogg_write_header(AVFormatContext *s)
         for (i = 1; i < 3; i++) {
             if (oggstream && oggstream->header_len[i])
                 ogg_buffer_data(s, st, oggstream->header[i],
-                                oggstream->header_len[i], 0);
+                                oggstream->header_len[i], 0, 1);
         }
         ogg_buffer_page(s, oggstream);
     }
@@ -501,7 +502,7 @@ static int ogg_write_packet(AVFormatContext *s, AVPacket *pkt)
     } else
         granule = pkt->pts + pkt->duration;
 
-    ret = ogg_buffer_data(s, st, pkt->data, pkt->size, granule);
+    ret = ogg_buffer_data(s, st, pkt->data, pkt->size, granule, 0);
     if (ret < 0)
         return ret;
 
diff --git a/tests/ref/lavf-fate/ogg_vp3 b/tests/ref/lavf-fate/ogg_vp3
index 9f36eeebc2..635dd3e5a9 100644
--- a/tests/ref/lavf-fate/ogg_vp3
+++ b/tests/ref/lavf-fate/ogg_vp3
@@ -1,3 +1,3 @@
-cf4184d736685fb698bb3be4656353bd *./tests/data/lavf-fate/lavf.ogg
-417698 ./tests/data/lavf-fate/lavf.ogg
+adbe6c30bdfe934dc5ae397f4db2960d *./tests/data/lavf-fate/lavf.ogg
+417644 ./tests/data/lavf-fate/lavf.ogg
 ./tests/data/lavf-fate/lavf.ogg CRC=0x45a9df79