diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index d53181e427..ef5bb13908 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -523,8 +523,6 @@ typedef struct OutputStream {
     /* dts of the last packet sent to the muxing queue, in AV_TIME_BASE_Q */
     int64_t last_mux_dts;
 
-    // the timebase of the packets sent to the muxer
-    AVRational mux_timebase;
     AVRational enc_timebase;
 
     Encoder *enc;
diff --git a/fftools/ffmpeg_enc.c b/fftools/ffmpeg_enc.c
index 96424272bf..3c9fdd3237 100644
--- a/fftools/ffmpeg_enc.c
+++ b/fftools/ffmpeg_enc.c
@@ -463,8 +463,6 @@ int enc_open(OutputStream *ost, AVFrame *frame)
     if (ost->st->time_base.num <= 0 || ost->st->time_base.den <= 0)
         ost->st->time_base = av_add_q(ost->enc_ctx->time_base, (AVRational){0, 1});
 
-    ost->mux_timebase = enc_ctx->time_base;
-
     ret = of_stream_init(of, ost);
     if (ret < 0)
         return ret;
diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c
index ab9bb398c1..54410aac5c 100644
--- a/fftools/ffmpeg_mux.c
+++ b/fftools/ffmpeg_mux.c
@@ -76,7 +76,23 @@ static int write_packet(Muxer *mux, OutputStream *ost, AVPacket *pkt)
     if (ost->type == AVMEDIA_TYPE_VIDEO && ost->vsync_method == VSYNC_DROP)
         pkt->pts = pkt->dts = AV_NOPTS_VALUE;
 
-    av_packet_rescale_ts(pkt, pkt->time_base, ost->st->time_base);
+    // rescale timestamps to the stream timebase
+    if (ost->type == AVMEDIA_TYPE_AUDIO && !ost->enc) {
+        // use av_rescale_delta() for streamcopying audio, to preserve
+        // accuracy with coarse input timebases
+        int duration = av_get_audio_frame_duration2(ost->st->codecpar, pkt->size);
+
+        if (!duration)
+            duration = ost->st->codecpar->frame_size;
+
+        pkt->dts = av_rescale_delta(pkt->time_base, pkt->dts,
+                                    (AVRational){1, ost->st->codecpar->sample_rate}, duration,
+                                    &ms->ts_rescale_delta_last, ost->st->time_base);
+        pkt->pts = pkt->dts;
+
+        pkt->duration = av_rescale_q(pkt->duration, pkt->time_base, ost->st->time_base);
+    } else
+        av_packet_rescale_ts(pkt, pkt->time_base, ost->st->time_base);
     pkt->time_base = ost->st->time_base;
 
     if (!(s->oformat->flags & AVFMT_NOTIMESTAMPS)) {
@@ -392,7 +408,7 @@ int of_streamcopy(OutputStream *ost, const AVPacket *pkt, int64_t dts)
     OutputFile *of = output_files[ost->file_index];
     MuxStream  *ms = ms_from_ost(ost);
     int64_t start_time = (of->start_time == AV_NOPTS_VALUE) ? 0 : of->start_time;
-    int64_t ost_tb_start_time = av_rescale_q(start_time, AV_TIME_BASE_Q, ost->mux_timebase);
+    int64_t ts_offset;
     AVPacket *opkt = ms->pkt;
     int ret;
 
@@ -425,27 +441,17 @@ int of_streamcopy(OutputStream *ost, const AVPacket *pkt, int64_t dts)
     if (ret < 0)
         return ret;
 
-    opkt->time_base = ost->mux_timebase;
+    ts_offset = av_rescale_q(start_time, AV_TIME_BASE_Q, opkt->time_base);
 
     if (pkt->pts != AV_NOPTS_VALUE)
-        opkt->pts = av_rescale_q(pkt->pts, pkt->time_base, opkt->time_base) - ost_tb_start_time;
+        opkt->pts -= ts_offset;
 
     if (pkt->dts == AV_NOPTS_VALUE) {
         opkt->dts = av_rescale_q(dts, AV_TIME_BASE_Q, opkt->time_base);
     } else if (ost->st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
-        int duration = av_get_audio_frame_duration2(ost->par_in, pkt->size);
-        if(!duration)
-            duration = ost->par_in->frame_size;
-        opkt->dts = av_rescale_delta(pkt->time_base, pkt->dts,
-                                    (AVRational){1, ost->par_in->sample_rate}, duration,
-                                    &ms->ts_rescale_delta_last, opkt->time_base);
-        /* dts will be set immediately afterwards to what pts is now */
-        opkt->pts = opkt->dts - ost_tb_start_time;
-    } else
-        opkt->dts = av_rescale_q(pkt->dts, pkt->time_base, opkt->time_base);
-    opkt->dts -= ost_tb_start_time;
-
-    opkt->duration = av_rescale_q(pkt->duration, pkt->time_base, opkt->time_base);
+        opkt->pts = opkt->dts - ts_offset;
+    }
+    opkt->dts -= ts_offset;
 
     {
         int ret = trigger_fix_sub_duration_heartbeat(ost, pkt);
@@ -511,10 +517,6 @@ static int thread_start(Muxer *mux)
         MuxStream     *ms = ms_from_ost(ost);
         AVPacket *pkt;
 
-        /* try to improve muxing time_base (only possible if nothing has been written yet) */
-        if (!av_fifo_can_read(ms->muxing_queue))
-            ost->mux_timebase = ost->st->time_base;
-
         while (av_fifo_read(ms->muxing_queue, &pkt, 1) >= 0) {
             ret = thread_submit_packet(mux, ost, pkt);
             if (pkt) {
diff --git a/fftools/ffmpeg_mux.h b/fftools/ffmpeg_mux.h
index 7f34b86548..a2bb4dfc7d 100644
--- a/fftools/ffmpeg_mux.h
+++ b/fftools/ffmpeg_mux.h
@@ -75,7 +75,7 @@ typedef struct MuxStream {
     int64_t    stream_duration;
     AVRational stream_duration_tb;
 
-    // audio streamcopy - state for av_rescale_delta()
+    // state for av_rescale_delta() call for audio in write_packet()
     int64_t ts_rescale_delta_last;
 
     // combined size of all the packets sent to the muxer
diff --git a/fftools/ffmpeg_mux_init.c b/fftools/ffmpeg_mux_init.c
index 0289cdabad..cf4cd2d5b7 100644
--- a/fftools/ffmpeg_mux_init.c
+++ b/fftools/ffmpeg_mux_init.c
@@ -1090,8 +1090,6 @@ static int streamcopy_init(const Muxer *mux, OutputStream *ost)
         }
     }
 
-    ost->mux_timebase = ist->st->time_base;
-
 fail:
     avcodec_free_context(&codec_ctx);
     av_dict_free(&codec_opts);