hwcontext_vulkan: dlopen libvulkan

While Vulkan itself went more or less the way it was expected to go,
libvulkan didn't quite solve all of the opengl loader issues. It's multi-vendor,
yes, but unfortunately, the code is Google/Khronos QUALITY, so suffers from
big static linking issues (static linking on anything but OSX is unsupported),
has bugs, and due to the prefix system used, there are 3 or so ways to type out
functions.

Just solve all of those problems by dlopening it. We even have nice emulation
for it on Windows.

configure

@@ -1451,6 +1451,23 @@ test_pkg_config(){
         set_sanitized "${name}_extralibs" $pkg_libs
 }
 
+test_pkg_config_cpp(){
+    log test_pkg_config_cpp "$@"
+    name="$1"
+    pkg_version="$2"
+    pkg="${2%% *}"
+    headers="$3"
+    cond="$4"
+    shift 4
+    disable $name
+    test_cmd $pkg_config --exists --print-errors $pkg_version || return
+    pkg_cflags=$($pkg_config --cflags $pkg_config_flags $pkg)
+    pkg_incdir=$($pkg_config --variable=includedir $pkg_config_flags $pkg)
+    test_cpp_condition "$pkg_incdir/$headers" "$cond" "$@" &&
+        enable $name &&
+        set_sanitized "${name}_cflags" $pkg_cflags
+}
+
 check_pkg_config(){
     log check_pkg_config "$@"
     name="$1"
@@ -1458,6 +1475,13 @@ check_pkg_config(){
         eval add_cflags \$${name}_cflags
 }
 
+check_pkg_config_cpp(){
+    log check_pkg_config_cpp "$@"
+    name="$1"
+    test_pkg_config_cpp "$@" &&
+        eval add_cflags \$${name}_cflags
+}
+
 test_exec(){
     test_ld "cc" "$@" && { enabled cross_compile || $TMPE >> $logfile 2>&1; }
 }
@@ -1581,6 +1605,12 @@ require_pkg_config(){
     check_pkg_config "$@" || die "ERROR: $pkg_version not found using pkg-config$pkg_config_fail_message"
 }
 
+require_pkg_config_cpp(){
+    log require_pkg_config_cpp "$@"
+    pkg_version="$2"
+    check_pkg_config_cpp "$@" || die "ERROR: $pkg_version not found using pkg-config$pkg_config_fail_message"
+}
+
 test_host_cc(){
     log test_host_cc "$@"
     cat > $TMPC
@@ -2920,6 +2950,7 @@ nvdec_deps="ffnvcodec"
 vaapi_x11_deps="xlib"
 videotoolbox_hwaccel_deps="videotoolbox pthreads"
 videotoolbox_hwaccel_extralibs="-framework QuartzCore"
+vulkan_deps_any="libdl LoadLibrary"
 xvmc_deps="X11_extensions_XvMClib_h"
 
 av1_d3d11va_hwaccel_deps="d3d11va DXVA_PicParams_AV1"
@@ -6739,7 +6770,8 @@ enabled vdpau &&
 enabled crystalhd && check_lib crystalhd "stdint.h libcrystalhd/libcrystalhd_if.h" DtsCrystalHDVersion -lcrystalhd
 
 enabled vulkan &&
-    require_pkg_config vulkan "vulkan >= 1.1.97" "vulkan/vulkan.h" vkCreateInstance
+    { require_pkg_config_cpp vulkan "vulkan >= 1.1.97" "vulkan/vulkan.h" "defined VK_VERSION_1_1" ||
+      require_cpp_condition vulkan "vulkan/vulkan.h" "defined VK_VERSION_1_1"; }
 
 if enabled x86; then
     case $target_os in

libavutil/hwcontext_vulkan.c

@@ -26,8 +26,11 @@
 #include "hwcontext_internal.h"
 #include "hwcontext_vulkan.h"
 
-VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,
+#ifdef _WIN32
-                                                               const char *name);
+#include "compat/w32dlfcn.h"
+#else
+#include <dlfcn.h>
+#endif
 
 #if CONFIG_LIBDRM
 #include <unistd.h>
@@ -186,7 +189,8 @@ typedef struct VulkanExecCtx {
 } VulkanExecCtx;
 
 typedef struct VulkanDevicePriv {
-    /* Vulkan loader functions */
+    /* Vulkan library and loader functions */
+    void *libvulkan;
     VulkanFunctions vkfn;
 
     /* Properties */
@@ -369,6 +373,40 @@ static int pixfmt_is_supported(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
     return 1;
 }
 
+static int load_libvulkan(AVHWDeviceContext *ctx)
+{
+    AVVulkanDeviceContext *hwctx = ctx->hwctx;
+    VulkanDevicePriv *p = ctx->internal->priv;
+
+    static const char *lib_names[] = {
+#if defined(_WIN32)
+        "vulkan-1.dll",
+#elif defined(__APPLE__)
+        "libvulkan.dylib",
+        "libvulkan.1.dylib",
+        "libMoltenVK.dylib",
+#else
+        "libvulkan.so.1",
+        "libvulkan.so",
+#endif
+    };
+
+    for (int i = 0; i < FF_ARRAY_ELEMS(lib_names); i++) {
+        p->libvulkan = dlopen(lib_names[i], RTLD_NOW | RTLD_LOCAL);
+        if (p->libvulkan)
+            break;
+    }
+
+    if (!p->libvulkan) {
+        av_log(ctx, AV_LOG_ERROR, "Unable to open the libvulkan library!\n");
+        return AVERROR_UNKNOWN;
+    }
+
+    hwctx->get_proc_addr = (PFN_vkGetInstanceProcAddr)dlsym(p->libvulkan, "vkGetInstanceProcAddr");
+
+    return 0;
+}
+
 static int load_functions(AVHWDeviceContext *ctx, int has_inst, int has_dev)
 {
     AVVulkanDeviceContext *hwctx = ctx->hwctx;
@@ -649,7 +687,9 @@ static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts)
     };
 
     if (!hwctx->get_proc_addr) {
-        hwctx->get_proc_addr = vkGetInstanceProcAddr;
+        err = load_libvulkan(ctx);
+        if (err < 0)
+            return err;
     }
 
     err = load_functions(ctx, 0, 0);
@@ -1188,6 +1228,9 @@ static void vulkan_device_free(AVHWDeviceContext *ctx)
 
     vk->DestroyInstance(hwctx->inst, hwctx->alloc);
 
+    if (p->libvulkan)
+        dlclose(p->libvulkan);
+
     for (int i = 0; i < hwctx->nb_enabled_inst_extensions; i++)
         av_free((void *)hwctx->enabled_inst_extensions[i]);
     av_free((void *)hwctx->enabled_inst_extensions);