From 09ca2f1950f9cdd50a8a4f7d2b911de53e3e1dce Mon Sep 17 00:00:00 2001
From: Anton Khirnov <anton@khirnov.net>
Date: Wed, 19 Jan 2022 14:37:51 +0100
Subject: [PATCH] lavu/hwcontext: clarify behavior on av_hwframe_map() failure

Clear anything that av_hwframe_map() might have done to the destination
frame, but leave caller-provided fields unchanged.
---
 libavutil/hwcontext.c | 27 +++++++++++++++++++++++++--
 libavutil/hwcontext.h |  4 ++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c
index 31c7840dba..ab9ad3703e 100644
--- a/libavutil/hwcontext.c
+++ b/libavutil/hwcontext.c
@@ -18,6 +18,7 @@
 
 #include "config.h"
 
+#include "avassert.h"
 #include "buffer.h"
 #include "common.h"
 #include "hwcontext.h"
@@ -788,6 +789,8 @@ fail:
 
 int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
 {
+    AVBufferRef    *orig_dst_frames = dst->hw_frames_ctx;
+    enum AVPixelFormat orig_dst_fmt = dst->format;
     AVHWFramesContext *src_frames, *dst_frames;
     HWMapDescriptor *hwmap;
     int ret;
@@ -824,8 +827,10 @@ int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
             src_frames->internal->hw_type->map_from) {
             ret = src_frames->internal->hw_type->map_from(src_frames,
                                                           dst, src, flags);
-            if (ret != AVERROR(ENOSYS))
+            if (ret >= 0)
                 return ret;
+            else if (ret != AVERROR(ENOSYS))
+                goto fail;
         }
     }
 
@@ -836,12 +841,30 @@ int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
             dst_frames->internal->hw_type->map_to) {
             ret = dst_frames->internal->hw_type->map_to(dst_frames,
                                                         dst, src, flags);
-            if (ret != AVERROR(ENOSYS))
+            if (ret >= 0)
                 return ret;
+            else if (ret != AVERROR(ENOSYS))
+                goto fail;
         }
     }
 
     return AVERROR(ENOSYS);
+
+fail:
+    // if the caller provided dst frames context, it should be preserved
+    // by this function
+    av_assert0(orig_dst_frames == NULL ||
+               orig_dst_frames == dst->hw_frames_ctx);
+
+    // preserve user-provided dst frame fields, but clean
+    // anything we might have set
+    dst->hw_frames_ctx = NULL;
+    av_frame_unref(dst);
+
+    dst->hw_frames_ctx = orig_dst_frames;
+    dst->format        = orig_dst_fmt;
+
+    return ret;
 }
 
 int av_hwframe_ctx_create_derived(AVBufferRef **derived_frame_ctx,
diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h
index 04d19d89c2..c18b7e1e8b 100644
--- a/libavutil/hwcontext.h
+++ b/libavutil/hwcontext.h
@@ -571,6 +571,10 @@ enum {
  * possible with the given arguments and hwframe setup, while other return
  * values indicate that it failed somehow.
  *
+ * On failure, the destination frame will be left blank, except for the
+ * hw_frames_ctx/format fields thay may have been set by the caller - those will
+ * be preserved as they were.
+ *
  * @param dst Destination frame, to contain the mapping.
  * @param src Source frame, to be mapped.
  * @param flags Some combination of AV_HWFRAME_MAP_* flags.