diff --git a/libswscale/utils.c b/libswscale/utils.c
index c6fa07f752..053c6bb76b 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -1317,11 +1317,6 @@ static int context_init_threaded(SwsContext *c,
         }
     }
 
-    c->frame_src = av_frame_alloc();
-    c->frame_dst = av_frame_alloc();
-    if (!c->frame_src || !c->frame_dst)
-        return AVERROR(ENOMEM);
-
     return 0;
 }
 
@@ -1581,11 +1576,6 @@ static av_cold int sws_init_single_context(SwsContext *c, SwsFilter *srcFilter,
     if (!FF_ALLOCZ_TYPED_ARRAY(c->formatConvBuffer, FFALIGN(srcW * 2 + 78, 16) * 2))
         goto nomem;
 
-    c->frame_src = av_frame_alloc();
-    c->frame_dst = av_frame_alloc();
-    if (!c->frame_src || !c->frame_dst)
-        goto nomem;
-
     c->srcBpc = desc_src->comp[0].depth;
     if (c->srcBpc < 8)
         c->srcBpc = 8;
@@ -2055,6 +2045,11 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
 {
     int ret;
 
+    c->frame_src = av_frame_alloc();
+    c->frame_dst = av_frame_alloc();
+    if (!c->frame_src || !c->frame_dst)
+        return AVERROR(ENOMEM);
+
     if (c->nb_threads != 1) {
         ret = context_init_threaded(c, srcFilter, dstFilter);
         if (ret < 0 || c->nb_threads > 1)