From 41ecb203c58145a7de06108737e7b85d3f7cc1c7 Mon Sep 17 00:00:00 2001
From: Lynne <dev@lynne.ee>
Date: Sun, 26 Oct 2025 20:12:15 +0000
Subject: [PATCH] hwcontext_vulkan: fix 3-plane 444 10 and 12-bit formats using
 the new MSB formats

We previously tried to fudge this somehow, but the pixel formats
are simply broken and we cannot use them without declaring them as MSB.
---
 libavcodec/vulkan_decode.c   |  6 ++++++
 libavcodec/vulkan_video.c    | 12 ++++++------
 libavutil/hwcontext_vulkan.c | 12 ++++++------
 libavutil/vulkan.c           | 12 ++++++------
 4 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c
index 0e03df1ccb..760913b0e5 100644
--- a/libavcodec/vulkan_decode.c
+++ b/libavcodec/vulkan_decode.c
@@ -1116,6 +1116,12 @@ int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
         /* Vulkan's formats are all LSB-padded */
         case AV_PIX_FMT_GRAY10: frames_ctx->sw_format = AV_PIX_FMT_GRAY10MSB; break;
         case AV_PIX_FMT_GRAY12: frames_ctx->sw_format = AV_PIX_FMT_GRAY12MSB; break;
+        case AV_PIX_FMT_YUV420P10: frames_ctx->sw_format = AV_PIX_FMT_YUV420P10MSB; break;
+        case AV_PIX_FMT_YUV420P12: frames_ctx->sw_format = AV_PIX_FMT_YUV420P12MSB; break;
+        case AV_PIX_FMT_YUV422P10: frames_ctx->sw_format = AV_PIX_FMT_YUV422P10MSB; break;
+        case AV_PIX_FMT_YUV422P12: frames_ctx->sw_format = AV_PIX_FMT_YUV422P12MSB; break;
+        case AV_PIX_FMT_YUV444P10: frames_ctx->sw_format = AV_PIX_FMT_YUV444P10MSB; break;
+        case AV_PIX_FMT_YUV444P12: frames_ctx->sw_format = AV_PIX_FMT_YUV444P12MSB; break;
         case AV_PIX_FMT_GBRAP16:
             /* This should be more efficient for downloading and using */
             frames_ctx->sw_format = AV_PIX_FMT_RGBA64;
diff --git a/libavcodec/vulkan_video.c b/libavcodec/vulkan_video.c
index 471cfd5ad5..8c86882a55 100644
--- a/libavcodec/vulkan_video.c
+++ b/libavcodec/vulkan_video.c
@@ -73,16 +73,16 @@ static const struct FFVkFormatMapEntry {
 
     /* Three-plane 420, 422, 444 at 8, 10, 12 and 16 bits */
     { VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM,    AV_PIX_FMT_YUV420P,   ASPECT_3PLANE },
-    { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P10, ASPECT_3PLANE },
-    { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P12, ASPECT_3PLANE },
+    { VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_420_UNORM_3PACK16, AV_PIX_FMT_YUV420P10MSB, ASPECT_3PLANE },
+    { VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16, AV_PIX_FMT_YUV420P12MSB, ASPECT_3PLANE },
     { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P16, ASPECT_3PLANE },
     { VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM,    AV_PIX_FMT_YUV422P,   ASPECT_3PLANE },
-    { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P10, ASPECT_3PLANE },
-    { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P12, ASPECT_3PLANE },
+    { VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_422_UNORM_3PACK16, AV_PIX_FMT_YUV422P10MSB, ASPECT_3PLANE },
+    { VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16, AV_PIX_FMT_YUV422P12MSB, ASPECT_3PLANE },
     { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P16, ASPECT_3PLANE },
     { VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM,    AV_PIX_FMT_YUV444P,   ASPECT_3PLANE },
-    { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P10, ASPECT_3PLANE },
-    { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P12, ASPECT_3PLANE },
+    { VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_444_UNORM_3PACK16, AV_PIX_FMT_YUV444P10MSB, ASPECT_3PLANE },
+    { VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_444_UNORM_3PACK16, AV_PIX_FMT_YUV444P12MSB, ASPECT_3PLANE },
     { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P16, ASPECT_3PLANE },
 
     /* Single plane 422 at 8, 10, 12 and 16 bits */
diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
index 8af99fda54..519dea379b 100644
--- a/libavutil/hwcontext_vulkan.c
+++ b/libavutil/hwcontext_vulkan.c
@@ -424,16 +424,16 @@ static const struct FFVkFormatEntry {
 
     /* Three-plane 420, 422, 444 at 8, 10, 12 and 16 bits */
     { VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM,    AV_PIX_FMT_YUV420P,   ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM,  VK_FORMAT_R8_UNORM,  VK_FORMAT_R8_UNORM  } },
-    { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
-    { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
+    { VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_420_UNORM_3PACK16, AV_PIX_FMT_YUV420P10MSB, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
+    { VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16, AV_PIX_FMT_YUV420P12MSB, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
     { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
     { VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM,    AV_PIX_FMT_YUV422P,   ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM,  VK_FORMAT_R8_UNORM,  VK_FORMAT_R8_UNORM  } },
-    { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
-    { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
+    { VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_422_UNORM_3PACK16, AV_PIX_FMT_YUV422P10MSB, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
+    { VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_422_UNORM_3PACK16, AV_PIX_FMT_YUV422P12MSB, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
     { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
     { VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM,    AV_PIX_FMT_YUV444P,   ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM,  VK_FORMAT_R8_UNORM,  VK_FORMAT_R8_UNORM  } },
-    { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
-    { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
+    { VK_FORMAT_G10X6_B10X6_R10X6_3PLANE_444_UNORM_3PACK16, AV_PIX_FMT_YUV444P10MSB, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
+    { VK_FORMAT_G12X4_B12X4_R12X4_3PLANE_444_UNORM_3PACK16, AV_PIX_FMT_YUV444P12MSB, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
     { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
 
     /* Single plane 422 at 8, 10, 12 and 16 bits */
diff --git a/libavutil/vulkan.c b/libavutil/vulkan.c
index 8a14f8a6c3..7975ed27d2 100644
--- a/libavutil/vulkan.c
+++ b/libavutil/vulkan.c
@@ -1684,14 +1684,14 @@ const char *ff_vk_shader_rep_fmt(enum AVPixelFormat pix_fmt,
     case AV_PIX_FMT_GBRP12:
     case AV_PIX_FMT_GBRP14:
     case AV_PIX_FMT_GBRP16:
-    case AV_PIX_FMT_YUV420P10:
-    case AV_PIX_FMT_YUV420P12:
+    case AV_PIX_FMT_YUV420P10MSB:
+    case AV_PIX_FMT_YUV420P12MSB:
     case AV_PIX_FMT_YUV420P16:
-    case AV_PIX_FMT_YUV422P10:
-    case AV_PIX_FMT_YUV422P12:
+    case AV_PIX_FMT_YUV422P10MSB:
+    case AV_PIX_FMT_YUV422P12MSB:
     case AV_PIX_FMT_YUV422P16:
-    case AV_PIX_FMT_YUV444P10:
-    case AV_PIX_FMT_YUV444P12:
+    case AV_PIX_FMT_YUV444P10MSB:
+    case AV_PIX_FMT_YUV444P12MSB:
     case AV_PIX_FMT_YUV444P16:
     case AV_PIX_FMT_YUVA420P10:
     case AV_PIX_FMT_YUVA420P16: