diff --git a/libavformat/aviobuf.c b/libavformat/aviobuf.c
index 5841966c9b..941691adb6 100644
--- a/libavformat/aviobuf.c
+++ b/libavformat/aviobuf.c
@@ -667,7 +667,7 @@ int ff_rewind_with_probe_data(ByteIOContext *s, unsigned char *buf, int buf_size
 {
     int64_t buffer_start;
     int buffer_size;
-    int overlap, new_size;
+    int overlap, new_size, alloc_size;
 
     if (s->write_flag)
         return AVERROR(EINVAL);
@@ -681,17 +681,20 @@ int ff_rewind_with_probe_data(ByteIOContext *s, unsigned char *buf, int buf_size
     overlap = buf_size - buffer_start;
     new_size = buf_size + buffer_size - overlap;
 
-    if (new_size > buf_size) {
-        if (!(buf = av_realloc(buf, new_size)))
+    alloc_size = FFMAX(s->buffer_size, new_size);
+    if (alloc_size > buf_size)
+        if (!(buf = av_realloc(buf, alloc_size)))
             return AVERROR(ENOMEM);
 
+    if (new_size > buf_size) {
         memcpy(buf + buf_size, s->buffer + overlap, buffer_size - overlap);
         buf_size = new_size;
     }
 
     av_free(s->buffer);
     s->buf_ptr = s->buffer = buf;
-    s->pos = s->buffer_size = buf_size;
+    s->buffer_size = alloc_size;
+    s->pos = buf_size;
     s->buf_end = s->buf_ptr + buf_size;
     s->eof_reached = 0;
     s->must_flush = 0;