diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index 9d63e32c25..47fda6905e 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -2532,13 +2532,13 @@ typedef struct AVCodecContext {
      */
     int (*execute2)(struct AVCodecContext *c, int (*func)(struct AVCodecContext *c2, void *arg, int jobnr, int threadnr), void *arg2, int *ret, int count);
 
+#if FF_API_THREAD_OPAQUE
     /**
-     * thread opaque
-     * Can be used by execute() to store some per AVCodecContext stuff.
-     * - encoding: set by execute()
-     * - decoding: set by execute()
+     * @deprecated this field should not be used from outside of lavc
      */
+    attribute_deprecated
     void *thread_opaque;
+#endif
 
     /**
      * noise vs. sse weight for the nsse comparsion function
diff --git a/libavcodec/internal.h b/libavcodec/internal.h
index 21331377c3..9a57209c69 100644
--- a/libavcodec/internal.h
+++ b/libavcodec/internal.h
@@ -86,6 +86,8 @@ typedef struct AVCodecInternal {
     AVFrame to_free;
 
     FramePool *pool;
+
+    void *thread_ctx;
 } AVCodecInternal;
 
 struct AVCodecDefault {
diff --git a/libavcodec/options.c b/libavcodec/options.c
index 8d36c01183..2e41ce4704 100644
--- a/libavcodec/options.c
+++ b/libavcodec/options.c
@@ -150,7 +150,6 @@ int avcodec_copy_context(AVCodecContext *dest, const AVCodecContext *src)
     dest->codec           = NULL;
     dest->slice_offset    = NULL;
     dest->hwaccel         = NULL;
-    dest->thread_opaque   = NULL;
     dest->internal        = NULL;
 
     /* reallocate values that should be allocated separately */
diff --git a/libavcodec/pthread_frame.c b/libavcodec/pthread_frame.c
index d16078f454..c2d12c854f 100644
--- a/libavcodec/pthread_frame.c
+++ b/libavcodec/pthread_frame.c
@@ -46,7 +46,7 @@
 #include "libavutil/mem.h"
 
 /**
- * Context used by codec threads and stored in their AVCodecContext thread_opaque.
+ * Context used by codec threads and stored in their AVCodecInternal thread_ctx.
  */
 typedef struct PerThreadContext {
     struct FrameThreadContext *parent;
@@ -93,7 +93,7 @@ typedef struct PerThreadContext {
 } PerThreadContext;
 
 /**
- * Context stored in the client AVCodecContext thread_opaque.
+ * Context stored in the client AVCodecInternal thread_ctx.
  */
 typedef struct FrameThreadContext {
     PerThreadContext *threads;     ///< The contexts for each thread.
@@ -365,7 +365,7 @@ int ff_thread_decode_frame(AVCodecContext *avctx,
                            AVFrame *picture, int *got_picture_ptr,
                            AVPacket *avpkt)
 {
-    FrameThreadContext *fctx = avctx->thread_opaque;
+    FrameThreadContext *fctx = avctx->internal->thread_ctx;
     int finished = fctx->next_finished;
     PerThreadContext *p;
     int err;
@@ -441,7 +441,7 @@ void ff_thread_report_progress(ThreadFrame *f, int n, int field)
 
     if (!progress || progress[field] >= n) return;
 
-    p = f->owner->thread_opaque;
+    p = f->owner->internal->thread_ctx;
 
     if (f->owner->debug&FF_DEBUG_THREADS)
         av_log(f->owner, AV_LOG_DEBUG, "%p finished %d field %d\n", progress, n, field);
@@ -459,7 +459,7 @@ void ff_thread_await_progress(ThreadFrame *f, int n, int field)
 
     if (!progress || progress[field] >= n) return;
 
-    p = f->owner->thread_opaque;
+    p = f->owner->internal->thread_ctx;
 
     if (f->owner->debug&FF_DEBUG_THREADS)
         av_log(f->owner, AV_LOG_DEBUG, "thread awaiting %d field %d from %p\n", n, field, progress);
@@ -471,7 +471,7 @@ void ff_thread_await_progress(ThreadFrame *f, int n, int field)
 }
 
 void ff_thread_finish_setup(AVCodecContext *avctx) {
-    PerThreadContext *p = avctx->thread_opaque;
+    PerThreadContext *p = avctx->internal->thread_ctx;
 
     if (!(avctx->active_thread_type&FF_THREAD_FRAME)) return;
 
@@ -500,7 +500,7 @@ static void park_frame_worker_threads(FrameThreadContext *fctx, int thread_count
 
 void ff_frame_thread_free(AVCodecContext *avctx, int thread_count)
 {
-    FrameThreadContext *fctx = avctx->thread_opaque;
+    FrameThreadContext *fctx = avctx->internal->thread_ctx;
     const AVCodec *codec = avctx->codec;
     int i;
 
@@ -544,16 +544,16 @@ void ff_frame_thread_free(AVCodecContext *avctx, int thread_count)
 
         if (i) {
             av_freep(&p->avctx->priv_data);
-            av_freep(&p->avctx->internal);
             av_freep(&p->avctx->slice_offset);
         }
 
+        av_freep(&p->avctx->internal);
         av_freep(&p->avctx);
     }
 
     av_freep(&fctx->threads);
     pthread_mutex_destroy(&fctx->buffer_mutex);
-    av_freep(&avctx->thread_opaque);
+    av_freep(&avctx->internal->thread_ctx);
 }
 
 int ff_frame_thread_init(AVCodecContext *avctx)
@@ -583,7 +583,7 @@ int ff_frame_thread_init(AVCodecContext *avctx)
         return 0;
     }
 
-    avctx->thread_opaque = fctx = av_mallocz(sizeof(FrameThreadContext));
+    avctx->internal->thread_ctx = fctx = av_mallocz(sizeof(FrameThreadContext));
 
     fctx->threads = av_mallocz(sizeof(PerThreadContext) * thread_count);
     pthread_mutex_init(&fctx->buffer_mutex, NULL);
@@ -608,9 +608,16 @@ int ff_frame_thread_init(AVCodecContext *avctx)
         }
 
         *copy = *src;
-        copy->thread_opaque = p;
         copy->pkt = &p->avpkt;
 
+        copy->internal = av_malloc(sizeof(AVCodecInternal));
+        if (!copy->internal) {
+            err = AVERROR(ENOMEM);
+            goto error;
+        }
+        *copy->internal = *src->internal;
+        copy->internal->thread_ctx = p;
+
         if (!i) {
             src = copy;
 
@@ -625,12 +632,6 @@ int ff_frame_thread_init(AVCodecContext *avctx)
                 goto error;
             }
             memcpy(copy->priv_data, src->priv_data, codec->priv_data_size);
-            copy->internal = av_malloc(sizeof(AVCodecInternal));
-            if (!copy->internal) {
-                err = AVERROR(ENOMEM);
-                goto error;
-            }
-            *copy->internal = *src->internal;
             copy->internal->is_copy = 1;
 
             if (codec->init_thread_copy)
@@ -654,9 +655,9 @@ error:
 void ff_thread_flush(AVCodecContext *avctx)
 {
     int i;
-    FrameThreadContext *fctx = avctx->thread_opaque;
+    FrameThreadContext *fctx = avctx->internal->thread_ctx;
 
-    if (!avctx->thread_opaque) return;
+    if (!fctx) return;
 
     park_frame_worker_threads(fctx, avctx->thread_count);
     if (fctx->prev_thread) {
@@ -681,7 +682,7 @@ void ff_thread_flush(AVCodecContext *avctx)
 
 int ff_thread_get_buffer(AVCodecContext *avctx, ThreadFrame *f, int flags)
 {
-    PerThreadContext *p = avctx->thread_opaque;
+    PerThreadContext *p = avctx->internal->thread_ctx;
     int err;
 
     f->owner = avctx;
@@ -743,7 +744,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
 
 void ff_thread_release_buffer(AVCodecContext *avctx, ThreadFrame *f)
 {
-    PerThreadContext *p = avctx->thread_opaque;
+    PerThreadContext *p = avctx->internal->thread_ctx;
     FrameThreadContext *fctx;
     AVFrame *dst, *tmp;
 FF_DISABLE_DEPRECATION_WARNINGS
diff --git a/libavcodec/pthread_slice.c b/libavcodec/pthread_slice.c
index cf9805ffd0..d7c73f0884 100644
--- a/libavcodec/pthread_slice.c
+++ b/libavcodec/pthread_slice.c
@@ -63,7 +63,7 @@ typedef struct SliceThreadContext {
 static void* attribute_align_arg worker(void *v)
 {
     AVCodecContext *avctx = v;
-    SliceThreadContext *c = avctx->thread_opaque;
+    SliceThreadContext *c = avctx->internal->thread_ctx;
     unsigned last_execute = 0;
     int our_job = c->job_count;
     int thread_count = avctx->thread_count;
@@ -98,7 +98,7 @@ static void* attribute_align_arg worker(void *v)
 
 void ff_slice_thread_free(AVCodecContext *avctx)
 {
-    SliceThreadContext *c = avctx->thread_opaque;
+    SliceThreadContext *c = avctx->internal->thread_ctx;
     int i;
 
     pthread_mutex_lock(&c->current_job_lock);
@@ -113,7 +113,7 @@ void ff_slice_thread_free(AVCodecContext *avctx)
     pthread_cond_destroy(&c->current_job_cond);
     pthread_cond_destroy(&c->last_job_cond);
     av_free(c->workers);
-    av_freep(&avctx->thread_opaque);
+    av_freep(&avctx->internal->thread_ctx);
 }
 
 static av_always_inline void thread_park_workers(SliceThreadContext *c, int thread_count)
@@ -125,7 +125,7 @@ static av_always_inline void thread_park_workers(SliceThreadContext *c, int thre
 
 static int thread_execute(AVCodecContext *avctx, action_func* func, void *arg, int *ret, int job_count, int job_size)
 {
-    SliceThreadContext *c = avctx->thread_opaque;
+    SliceThreadContext *c = avctx->internal->thread_ctx;
     int dummy_ret;
 
     if (!(avctx->active_thread_type&FF_THREAD_SLICE) || avctx->thread_count <= 1)
@@ -158,7 +158,7 @@ static int thread_execute(AVCodecContext *avctx, action_func* func, void *arg, i
 
 static int thread_execute2(AVCodecContext *avctx, action_func2* func2, void *arg, int *ret, int job_count)
 {
-    SliceThreadContext *c = avctx->thread_opaque;
+    SliceThreadContext *c = avctx->internal->thread_ctx;
     c->func2 = func2;
     return thread_execute(avctx, NULL, arg, ret, job_count, 0);
 }
@@ -198,7 +198,7 @@ int ff_slice_thread_init(AVCodecContext *avctx)
         return -1;
     }
 
-    avctx->thread_opaque = c;
+    avctx->internal->thread_ctx = c;
     c->current_job = 0;
     c->job_count = 0;
     c->job_size = 0;
diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 262f081d18..7b5c796b2a 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -1567,7 +1567,7 @@ av_cold int avcodec_close(AVCodecContext *avctx)
     if (avcodec_is_open(avctx)) {
         FramePool *pool = avctx->internal->pool;
         int i;
-        if (HAVE_THREADS && avctx->thread_opaque)
+        if (HAVE_THREADS && avctx->internal->thread_ctx)
             ff_thread_free(avctx);
         if (avctx->codec && avctx->codec->close)
             avctx->codec->close(avctx);
diff --git a/libavcodec/version.h b/libavcodec/version.h
index 05ac33e033..eac9ee4c4a 100644
--- a/libavcodec/version.h
+++ b/libavcodec/version.h
@@ -88,5 +88,8 @@
 #ifndef FF_API_ASPECT_EXTENDED
 #define FF_API_ASPECT_EXTENDED   (LIBAVCODEC_VERSION_MAJOR < 56)
 #endif
+#ifndef FF_API_THREAD_OPAQUE
+#define FF_API_THREAD_OPAQUE     (LIBAVCODEC_VERSION_MAJOR < 56)
+#endif
 
 #endif /* AVCODEC_VERSION_H */