vulkan: unify handling of BGR and simplify ffv1_rct

libavcodec/ffv1_vulkan.h

@@ -37,6 +37,7 @@ int ff_ffv1_vk_init_crc_table_data(FFVulkanContext *s,
                                    FFVkBuffer *vkb, FFV1Context *f);
 
 typedef struct FFv1VkRCTParameters {
+    int fmt_lut[4];
     int offset;
     uint8_t bits;
     uint8_t planar_rgb;

libavcodec/ffv1enc_vulkan.c

@@ -264,6 +264,15 @@ static int run_rct(AVCodecContext *avctx, FFVkExecContext *exec,
                       (ff_vk_count_images((AVVkFrame *)enc_in->data[0]) > 1),
         .transparency = f->transparency,
     };
+
+    /* For some reason the C FFv1 encoder/decoder treats these differently */
+    if (src_hwfc->sw_format == AV_PIX_FMT_GBRP10 ||
+        src_hwfc->sw_format == AV_PIX_FMT_GBRP12 ||
+        src_hwfc->sw_format == AV_PIX_FMT_GBRP14)
+        memcpy(pd.fmt_lut, (int [4]) { 2, 1, 0, 3 }, 4*sizeof(int));
+    else
+        ff_vk_set_perm(src_hwfc->sw_format, pd.fmt_lut, 1);
+
     ff_vk_shader_update_push_const(&fv->s, exec, &fv->rct,
                                    VK_SHADER_STAGE_COMPUTE_BIT,
                                    0, sizeof(pd), &pd);
@@ -1157,6 +1166,7 @@ static int init_rct_shader(AVCodecContext *avctx, FFVkSPIRVCompiler *spv)
     GLSLD(ff_source_common_comp);
 
     GLSLC(0, layout(push_constant, scalar) uniform pushConstants {             );
+    GLSLC(1,    ivec4 fmt_lut;                                                 );
     GLSLC(1,    int offset;                                                    );
     GLSLC(1,    uint8_t bits;                                                  );
     GLSLC(1,    uint8_t planar_rgb;                                            );

libavcodec/vulkan/ffv1_enc_rct.comp

@@ -22,17 +22,14 @@
 
 ivec4 load_components(ivec2 pos)
 {
-    if (planar_rgb == 0)
+    ivec4 pix = ivec4(imageLoad(src[0], pos));
-        return ivec4(imageLoad(src[0], pos));
+    if (planar_rgb != 0) {
-
+        for (int i = 1; i < (3 + transparency); i++)
-    ivec4 pix;
-    for (int i = 0; i < (3 + transparency); i++)
             pix[i] = int(imageLoad(src[i], pos)[0]);
+    }
 
-    /* Swizzle out the difference */
+    return ivec4(pix[fmt_lut[0]], pix[fmt_lut[1]],
-    if (bits > 8 && bits < 16 && transparency == 0)
+                 pix[fmt_lut[2]], pix[fmt_lut[3]]);
-        return pix.bgra;
-    return pix.brga;
 }
 
 void bypass_sample(ivec2 pos)

libavutil/vulkan.c

@@ -1451,6 +1451,52 @@ int ff_vk_mt_is_np_rgb(enum AVPixelFormat pix_fmt)
     return 0;
 }
 
+void ff_vk_set_perm(enum AVPixelFormat pix_fmt, int lut[4], int inv)
+{
+    switch (pix_fmt) {
+    case AV_PIX_FMT_BGRA:
+    case AV_PIX_FMT_BGR0:
+    case AV_PIX_FMT_BGR565:
+    case AV_PIX_FMT_X2BGR10:
+        lut[0] = 2;
+        lut[1] = 1;
+        lut[2] = 0;
+        lut[3] = 3;
+        break;
+    case AV_PIX_FMT_GBRAP:
+    case AV_PIX_FMT_GBRP:
+    case AV_PIX_FMT_GBRAP10:
+    case AV_PIX_FMT_GBRAP12:
+    case AV_PIX_FMT_GBRAP14:
+    case AV_PIX_FMT_GBRAP16:
+    case AV_PIX_FMT_GBRP10:
+    case AV_PIX_FMT_GBRP12:
+    case AV_PIX_FMT_GBRP14:
+    case AV_PIX_FMT_GBRP16:
+    case AV_PIX_FMT_GBRPF32:
+    case AV_PIX_FMT_GBRAPF32:
+        lut[0] = 1;
+        lut[1] = 2;
+        lut[2] = 0;
+        lut[3] = 3;
+        break;
+    default:
+        lut[0] = 0;
+        lut[1] = 1;
+        lut[2] = 2;
+        lut[3] = 3;
+        break;
+    }
+
+    if (inv) {
+        int lut_tmp[4] = { lut[0], lut[1], lut[2], lut[3] };
+        for (int i = 0; i < 4; i++)
+            lut[lut_tmp[i]] = i;
+    }
+
+    return;
+}
+
 const char *ff_vk_shader_rep_fmt(enum AVPixelFormat pix_fmt,
                                  enum FFVkShaderRepFormat rep_fmt)
 {

libavutil/vulkan.h

@@ -371,6 +371,12 @@ const char *ff_vk_ret2str(VkResult res);
  */
 int ff_vk_mt_is_np_rgb(enum AVPixelFormat pix_fmt);
 
+/**
+ * Since storage images may not be swizzled, we have to do this in the
+ * shader itself. This fills in a lookup table to do it.
+ */
+void ff_vk_set_perm(enum AVPixelFormat pix_fmt, int lut[4], int inv);
+
 /**
  * Get the aspect flag for a plane from an image.
  */