diff --git a/libavformat/rtpdec.c b/libavformat/rtpdec.c
index d2041878cf..bd3566e9f8 100644
--- a/libavformat/rtpdec.c
+++ b/libavformat/rtpdec.c
@@ -80,6 +80,8 @@ static int rtcp_parse_packet(RTPDemuxContext *s, const unsigned char *buf, int l
     if (buf[1] != 200)
         return -1;
     s->last_rtcp_ntp_time = AV_RB64(buf + 8);
+    if (s->first_rtcp_ntp_time == AV_NOPTS_VALUE)
+        s->first_rtcp_ntp_time = s->last_rtcp_ntp_time;
     s->last_rtcp_timestamp = AV_RB32(buf + 16);
     return 0;
 }
@@ -326,6 +328,7 @@ RTPDemuxContext *rtp_parse_open(AVFormatContext *s1, AVStream *st, URLContext *r
         return NULL;
     s->payload_type = payload_type;
     s->last_rtcp_ntp_time = AV_NOPTS_VALUE;
+    s->first_rtcp_ntp_time = AV_NOPTS_VALUE;
     s->ic = s1;
     s->st = st;
     s->rtp_payload_data = rtp_payload_data;
@@ -433,7 +436,7 @@ static void finalize_packet(RTPDemuxContext *s, AVPacket *pkt, uint32_t timestam
         /* compute pts from timestamp with received ntp_time */
         delta_timestamp = timestamp - s->last_rtcp_timestamp;
         /* convert to the PTS timebase */
-        addend = av_rescale(s->last_rtcp_ntp_time, s->st->time_base.den, (uint64_t)s->st->time_base.num << 32);
+        addend = av_rescale(s->last_rtcp_ntp_time - s->first_rtcp_ntp_time, s->st->time_base.den, (uint64_t)s->st->time_base.num << 32);
         pkt->pts = addend + delta_timestamp;
     }
 }
diff --git a/libavformat/rtpdec.h b/libavformat/rtpdec.h
index 1754d2bce1..599e13d2a2 100644
--- a/libavformat/rtpdec.h
+++ b/libavformat/rtpdec.h
@@ -174,6 +174,7 @@ struct RTPDemuxContext {
 
     /* rtcp sender statistics receive */
     int64_t last_rtcp_ntp_time;    // TODO: move into statistics
+    int64_t first_rtcp_ntp_time;   // TODO: move into statistics
     uint32_t last_rtcp_timestamp;  // TODO: move into statistics
 
     /* rtcp sender statistics */
diff --git a/libavformat/rtsp.c b/libavformat/rtsp.c
index b0fa2eae25..0931201f81 100644
--- a/libavformat/rtsp.c
+++ b/libavformat/rtsp.c
@@ -1796,8 +1796,27 @@ static int rtsp_fetch_packet(AVFormatContext *s, AVPacket *pkt)
         return AVERROR_EOF;
     if (rt->transport == RTSP_TRANSPORT_RDT) {
         ret = ff_rdt_parse_packet(rtsp_st->transport_priv, pkt, buf, len);
-    } else
+    } else {
         ret = rtp_parse_packet(rtsp_st->transport_priv, pkt, buf, len);
+        if (ret < 0) {
+            /* Either bad packet, or a RTCP packet. Check if the
+             * first_rtcp_ntp_time field was initialized. */
+            RTPDemuxContext *rtpctx = rtsp_st->transport_priv;
+            if (rtpctx->first_rtcp_ntp_time != AV_NOPTS_VALUE) {
+                /* first_rtcp_ntp_time has been initialized for this stream,
+                 * copy the same value to all other uninitialized streams,
+                 * in order to map their timestamp origin to the same ntp time
+                 * as this one. */
+                int i;
+                for (i = 0; i < rt->nb_rtsp_streams; i++) {
+                    RTPDemuxContext *rtpctx2 = rtsp_st->transport_priv;
+                    if (rtpctx2 &&
+                        rtpctx2->first_rtcp_ntp_time == AV_NOPTS_VALUE)
+                        rtpctx2->first_rtcp_ntp_time = rtpctx->first_rtcp_ntp_time;
+                }
+            }
+        }
+    }
     if (ret < 0)
         goto redo;
     if (ret == 1)