diff --git a/libavformat/segment.c b/libavformat/segment.c
index 0e6421ea5d..80e4bf851c 100644
--- a/libavformat/segment.c
+++ b/libavformat/segment.c
@@ -116,6 +116,7 @@ typedef struct SegmentContext {
     int64_t initial_offset;    ///< initial timestamps offset, expressed in microseconds
     char *reference_stream_specifier; ///< reference stream specifier
     int   reference_stream_index;
+    int64_t reference_stream_first_pts;    ///< initial timestamp, expressed in microseconds
     int   break_non_keyframes;
     int   write_empty;
 
@@ -746,6 +747,8 @@ static int seg_init(AVFormatContext *s)
            seg->reference_stream_index,
            av_get_media_type_string(s->streams[seg->reference_stream_index]->codecpar->codec_type));
 
+    seg->reference_stream_first_pts = AV_NOPTS_VALUE;
+
     seg->oformat = av_guess_format(seg->format, s->url, NULL);
 
     if (!seg->oformat)
@@ -898,6 +901,18 @@ calc_times:
             pkt->flags & AV_PKT_FLAG_KEY,
             pkt->stream_index == seg->reference_stream_index ? seg->frame_count : -1);
 
+    if (seg->reference_stream_first_pts == AV_NOPTS_VALUE &&
+        pkt->stream_index == seg->reference_stream_index &&
+        pkt->pts != AV_NOPTS_VALUE) {
+        seg->reference_stream_first_pts = av_rescale_q(pkt->pts, st->time_base, AV_TIME_BASE_Q);
+    }
+
+    if (seg->reference_stream_first_pts != AV_NOPTS_VALUE) {
+        end_pts += (INT64_MAX - end_pts >= seg->reference_stream_first_pts) ?
+                    seg->reference_stream_first_pts :
+                    INT64_MAX - end_pts;
+    }
+
     if (pkt->pts != AV_NOPTS_VALUE)
         pkt_pts_avtb = av_rescale_q(pkt->pts, st->time_base, AV_TIME_BASE_Q);
 
diff --git a/tests/ref/fate/segment-mp4-to-ts b/tests/ref/fate/segment-mp4-to-ts
index 54fcadb846..2994416270 100644
--- a/tests/ref/fate/segment-mp4-to-ts
+++ b/tests/ref/fate/segment-mp4-to-ts
@@ -23,7 +23,7 @@
 0,      50400,      54000,     3600,      607, 0xc53c2339, F=0x0, S=1,        1
 0,      54000,      72000,     3600,     4755, 0x2f642b58, F=0x0, S=1,        1
 0,      57600,      64800,     3600,     1182, 0xbe1a4847, F=0x0, S=1,        1
-0,      61200,      61200,     3600,      809, 0x8d948a4e, F=0x2, S=1,        1
+0,      61200,      61200,     3600,      809, 0x8d948a4e, F=0x0, S=1,        1
 0,      64800,      68400,     3600,      656, 0x4fa03c2b, F=0x0, S=1,        1
 0,      68400,      86400,     3600,    26555, 0x5629b584, S=1,        1
 0,      72000,      79200,     3600,     1141, 0x761b31e8, F=0x0, S=1,        1
@@ -71,7 +71,7 @@
 0,     223200,     226800,     3600,      809, 0x84e37fee, F=0x0, S=1,        1
 0,     226800,     244800,     3600,     4541, 0xd4e5c0de, F=0x0, S=1,        1
 0,     230400,     237600,     3600,     1545, 0x0099fc98, F=0x0, S=1,        1
-0,     234000,     234000,     3600,      929, 0xfd72d049, F=0x0, S=1,        1
+0,     234000,     234000,     3600,      929, 0xfd72d049, F=0x2, S=1,        1
 0,     237600,     241200,     3600,      829, 0xcfda9e96, F=0x0, S=1,        1
 0,     241200,     259200,     3600,    24220, 0x5ca21d71, S=1,        1
 0,     244800,     252000,     3600,     1422, 0xcde6cc34, F=0x0, S=1,        1