From e3c7b22451799a2890062b756b645cc5f2e8752e Mon Sep 17 00:00:00 2001 From: Lynne Date: Tue, 21 Apr 2020 18:55:24 +0100 Subject: [PATCH] hwcontext_vulkan: correctly download and upload flipped images We derive the destination buffer stride from the input stride, which meant if the image was flipped with a negative stride, we'd be FFALIGNING a negative number which ends up being huge, thus making the Vulkan buffer allocation fail and the whole image transfer fail. Only found out about this as OpenGL compositors can copy an entire image with a single call if its flipped, rather than iterate over each line. --- libavutil/hwcontext_vulkan.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c index e4546f67ca..fa53d9d121 100644 --- a/libavutil/hwcontext_vulkan.c +++ b/libavutil/hwcontext_vulkan.c @@ -2638,7 +2638,7 @@ static int vulkan_transfer_data_from_mem(AVHWFramesContext *hwfc, AVFrame *dst, int h = src->height; int p_height = i > 0 ? AV_CEIL_RSHIFT(h, log2_chroma) : h; - tmp.linesize[i] = src->linesize[i]; + tmp.linesize[i] = FFABS(src->linesize[i]); err = create_buf(dev_ctx, &buf[i], p_height, &tmp.linesize[i], VK_BUFFER_USAGE_TRANSFER_SRC_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, NULL, NULL); @@ -2793,7 +2793,7 @@ static int vulkan_transfer_data_to_mem(AVHWFramesContext *hwfc, AVFrame *dst, int h = dst->height; int p_height = i > 0 ? AV_CEIL_RSHIFT(h, log2_chroma) : h; - tmp.linesize[i] = dst->linesize[i]; + tmp.linesize[i] = FFABS(dst->linesize[i]); err = create_buf(dev_ctx, &buf[i], p_height, &tmp.linesize[i], VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, NULL, NULL);