diff --git a/libavfilter/af_atempo.c b/libavfilter/af_atempo.c
index 4d1c68aa7e..ad12786584 100644
--- a/libavfilter/af_atempo.c
+++ b/libavfilter/af_atempo.c
@@ -123,8 +123,9 @@ typedef struct {
     // tempo scaling factor:
     double tempo;
 
-    // cumulative alignment drift:
-    int drift;
+    // a snapshot of previous fragment input and output position values
+    // captured when the tempo scale factor was set most recently:
+    int64_t origin[2];
 
     // current/previous fragment ring-buffer:
     AudioFragment frag[2];
@@ -159,6 +160,16 @@ static const AVOption atempo_options[] = {
 
 AVFILTER_DEFINE_CLASS(atempo);
 
+inline static AudioFragment *yae_curr_frag(ATempoContext *atempo)
+{
+    return &atempo->frag[atempo->nfrag % 2];
+}
+
+inline static AudioFragment *yae_prev_frag(ATempoContext *atempo)
+{
+    return &atempo->frag[(atempo->nfrag + 1) % 2];
+}
+
 /**
  * Reset filter to initial state, do not deallocate existing local buffers.
  */
@@ -168,13 +179,15 @@ static void yae_clear(ATempoContext *atempo)
     atempo->head = 0;
     atempo->tail = 0;
 
-    atempo->drift = 0;
     atempo->nfrag = 0;
     atempo->state = YAE_LOAD_FRAGMENT;
 
     atempo->position[0] = 0;
     atempo->position[1] = 0;
 
+    atempo->origin[0] = 0;
+    atempo->origin[1] = 0;
+
     atempo->frag[0].position[0] = 0;
     atempo->frag[0].position[1] = 0;
     atempo->frag[0].nsamples    = 0;
@@ -308,6 +321,7 @@ static int yae_reset(ATempoContext *atempo,
 
 static int yae_set_tempo(AVFilterContext *ctx, const char *arg_tempo)
 {
+    const AudioFragment *prev;
     ATempoContext *atempo = ctx->priv;
     char   *tail = NULL;
     double tempo = av_strtod(arg_tempo, &tail);
@@ -323,20 +337,13 @@ static int yae_set_tempo(AVFilterContext *ctx, const char *arg_tempo)
         return AVERROR(EINVAL);
     }
 
+    prev = yae_prev_frag(atempo);
+    atempo->origin[0] = prev->position[0] + atempo->window / 2;
+    atempo->origin[1] = prev->position[1] + atempo->window / 2;
     atempo->tempo = tempo;
     return 0;
 }
 
-inline static AudioFragment *yae_curr_frag(ATempoContext *atempo)
-{
-    return &atempo->frag[atempo->nfrag % 2];
-}
-
-inline static AudioFragment *yae_prev_frag(ATempoContext *atempo)
-{
-    return &atempo->frag[(atempo->nfrag + 1) % 2];
-}
-
 /**
  * A helper macro for initializing complex data buffer with scalar data
  * of a given type.
@@ -689,12 +696,21 @@ static int yae_adjust_position(ATempoContext *atempo)
     const AudioFragment *prev = yae_prev_frag(atempo);
     AudioFragment       *frag = yae_curr_frag(atempo);
 
+    const double prev_output_position =
+        (double)(prev->position[1] - atempo->origin[1] + atempo->window / 2);
+
+    const double ideal_output_position =
+        (double)(prev->position[0] - atempo->origin[0] + atempo->window / 2) /
+        atempo->tempo;
+
+    const int drift = (int)(prev_output_position - ideal_output_position);
+
     const int delta_max  = atempo->window / 2;
     const int correction = yae_align(frag,
                                      prev,
                                      atempo->window,
                                      delta_max,
-                                     atempo->drift,
+                                     drift,
                                      atempo->correlation,
                                      atempo->complex_to_real);
 
@@ -704,9 +720,6 @@ static int yae_adjust_position(ATempoContext *atempo)
 
         // clear so that the fragment can be reloaded:
         frag->nsamples = 0;
-
-        // update cumulative correction drift counter:
-        atempo->drift += correction;
     }
 
     return correction;