diff --git a/libavcodec/pthread.c b/libavcodec/pthread.c
index 1ca72b44f2..c582bfc64c 100644
--- a/libavcodec/pthread.c
+++ b/libavcodec/pthread.c
@@ -107,7 +107,8 @@ typedef struct PerThreadContext {
     AVCodecContext *avctx;          ///< Context used to decode packets passed to this thread.
 
     AVPacket       avpkt;           ///< Input packet (for decoding) or output (for encoding).
-    int            allocated_buf_size; ///< Size allocated for avpkt.data
+    uint8_t       *buf;             ///< backup storage for packet data when the input packet is not refcounted
+    int            allocated_buf_size; ///< Size allocated for buf
 
     AVFrame frame;                  ///< Output frame (for decoding) or input (for encoding).
     int     got_frame;              ///< The output of got_picture_ptr from the last avcodec_decode_video() call.
@@ -541,7 +542,6 @@ static int submit_packet(PerThreadContext *p, AVPacket *avpkt)
     FrameThreadContext *fctx = p->parent;
     PerThreadContext *prev_thread = fctx->prev_thread;
     const AVCodec *codec = p->avctx->codec;
-    uint8_t *buf = p->avpkt.data;
 
     if (!avpkt->size && !(codec->capabilities & CODEC_CAP_DELAY)) return 0;
 
@@ -565,11 +565,16 @@ static int submit_packet(PerThreadContext *p, AVPacket *avpkt)
         }
     }
 
-    av_fast_malloc(&buf, &p->allocated_buf_size, avpkt->size + FF_INPUT_BUFFER_PADDING_SIZE);
+    av_buffer_unref(&p->avpkt.buf);
     p->avpkt = *avpkt;
-    p->avpkt.data = buf;
-    memcpy(buf, avpkt->data, avpkt->size);
-    memset(buf + avpkt->size, 0, FF_INPUT_BUFFER_PADDING_SIZE);
+    if (avpkt->buf)
+        p->avpkt.buf = av_buffer_ref(avpkt->buf);
+    else {
+        av_fast_malloc(&p->buf, &p->allocated_buf_size, avpkt->size + FF_INPUT_BUFFER_PADDING_SIZE);
+        p->avpkt.data = p->buf;
+        memcpy(p->buf, avpkt->data, avpkt->size);
+        memset(p->buf + avpkt->size, 0, FF_INPUT_BUFFER_PADDING_SIZE);
+    }
 
     p->state = STATE_SETTING_UP;
     pthread_cond_signal(&p->input_cond);
@@ -791,7 +796,8 @@ static void frame_thread_free(AVCodecContext *avctx, int thread_count)
         pthread_cond_destroy(&p->input_cond);
         pthread_cond_destroy(&p->progress_cond);
         pthread_cond_destroy(&p->output_cond);
-        av_freep(&p->avpkt.data);
+        av_buffer_unref(&p->avpkt.buf);
+        av_freep(&p->buf);
 
         if (i) {
             av_freep(&p->avctx->priv_data);