virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2024-11-26 19:01:44 +02:00
Code Issues Releases Activity

hwcontext_vulkan: do not segfault when failing to init a AVHWFramesContext

Browse Source 
frames_uninit is always called on failure, and the free_exec_ctx function
did not zero the pool when freeing it, so it resulted in a double free.
This commit is contained in:
Lynne 2020-11-25 12:06:00 +01:00
parent 18a6535b08
commit 7b274a9b89
No known key found for this signature in database
GPG Key ID: A2FEA5F03F034464
1 changed files with 34 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
libavutil/hwcontext_vulkan.c
View File

@ -752,14 +752,6 @@ static int create_exec_ctx(AVHWFramesContext *hwfc, VulkanExecCtx *cmd,
cmd->nb_queues = num_queues; cmd->nb_queues = num_queues;
cmd->queues = av_mallocz(num_queues * sizeof(*cmd->queues));
if (!cmd->queues)
return AVERROR(ENOMEM);
cmd->bufs = av_mallocz(num_queues * sizeof(*cmd->bufs));
if (!cmd->bufs)
return AVERROR(ENOMEM);
/* Create command pool */ /* Create command pool */
ret = vkCreateCommandPool(hwctx->act_dev, &cqueue_create, ret = vkCreateCommandPool(hwctx->act_dev, &cqueue_create,
hwctx->alloc, &cmd->pool); hwctx->alloc, &cmd->pool);
@ -769,6 +761,10 @@ static int create_exec_ctx(AVHWFramesContext *hwfc, VulkanExecCtx *cmd,
return AVERROR_EXTERNAL; return AVERROR_EXTERNAL;
} }
cmd->bufs = av_mallocz(num_queues * sizeof(*cmd->bufs));
if (!cmd->bufs)
return AVERROR(ENOMEM);
cbuf_create.commandPool = cmd->pool; cbuf_create.commandPool = cmd->pool;
/* Allocate command buffer */ /* Allocate command buffer */
@ -776,9 +772,14 @@ static int create_exec_ctx(AVHWFramesContext *hwfc, VulkanExecCtx *cmd,
if (ret != VK_SUCCESS) { if (ret != VK_SUCCESS) {
av_log(hwfc, AV_LOG_ERROR, "Command buffer alloc failure: %s\n", av_log(hwfc, AV_LOG_ERROR, "Command buffer alloc failure: %s\n",
vk_ret2str(ret)); vk_ret2str(ret));
av_freep(&cmd->bufs);
return AVERROR_EXTERNAL; return AVERROR_EXTERNAL;
} }
cmd->queues = av_mallocz(num_queues * sizeof(*cmd->queues));
if (!cmd->queues)
return AVERROR(ENOMEM);
for (int i = 0; i < num_queues; i++) { for (int i = 0; i < num_queues; i++) {
VulkanQueueCtx *q = &cmd->queues[i]; VulkanQueueCtx *q = &cmd->queues[i];
vkGetDeviceQueue(hwctx->act_dev, queue_family_index, i, &q->queue); vkGetDeviceQueue(hwctx->act_dev, queue_family_index, i, &q->queue);
@ -792,23 +793,25 @@ static void free_exec_ctx(AVHWFramesContext *hwfc, VulkanExecCtx *cmd)
{ {
AVVulkanDeviceContext *hwctx = hwfc->device_ctx->hwctx; AVVulkanDeviceContext *hwctx = hwfc->device_ctx->hwctx;
/* Make sure all queues have finished executing */ if (cmd->queues) {
for (int i = 0; i < cmd->nb_queues; i++) { for (int i = 0; i < cmd->nb_queues; i++) {
VulkanQueueCtx *q = &cmd->queues[i]; VulkanQueueCtx *q = &cmd->queues[i];
if (q->fence && !q->was_synchronous) { /* Make sure all queues have finished executing */
vkWaitForFences(hwctx->act_dev, 1, &q->fence, VK_TRUE, UINT64_MAX); if (q->fence && !q->was_synchronous) {
vkResetFences(hwctx->act_dev, 1, &q->fence); vkWaitForFences(hwctx->act_dev, 1, &q->fence, VK_TRUE, UINT64_MAX);
vkResetFences(hwctx->act_dev, 1, &q->fence);
}
/* Free the fence */
if (q->fence)
vkDestroyFence(hwctx->act_dev, q->fence, hwctx->alloc);
/* Free buffer dependencies */
for (int j = 0; j < q->nb_buf_deps; j++)
av_buffer_unref(&q->buf_deps[j]);
av_free(q->buf_deps);
} }
/* Free the fence */
if (q->fence)
vkDestroyFence(hwctx->act_dev, q->fence, hwctx->alloc);
/* Free buffer dependencies */
for (int j = 0; j < q->nb_buf_deps; j++)
av_buffer_unref(&q->buf_deps[j]);
av_free(q->buf_deps);
} }
if (cmd->bufs) if (cmd->bufs)
@ -816,8 +819,9 @@ static void free_exec_ctx(AVHWFramesContext *hwfc, VulkanExecCtx *cmd)
if (cmd->pool) if (cmd->pool)
vkDestroyCommandPool(hwctx->act_dev, cmd->pool, hwctx->alloc); vkDestroyCommandPool(hwctx->act_dev, cmd->pool, hwctx->alloc);
av_freep(&cmd->bufs);
av_freep(&cmd->queues); av_freep(&cmd->queues);
av_freep(&cmd->bufs);
cmd->pool = NULL;
} }
static VkCommandBuffer get_buf_exec_ctx(AVHWFramesContext *hwfc, VulkanExecCtx *cmd) static VkCommandBuffer get_buf_exec_ctx(AVHWFramesContext *hwfc, VulkanExecCtx *cmd)
@ -1719,24 +1723,24 @@ static int vulkan_frames_init(AVHWFramesContext *hwfc)
dev_hwctx->queue_family_comp_index, dev_hwctx->queue_family_comp_index,
GET_QUEUE_COUNT(dev_hwctx, 0, 1, 0)); GET_QUEUE_COUNT(dev_hwctx, 0, 1, 0));
if (err) if (err)
goto fail; return err;
err = create_exec_ctx(hwfc, &fp->upload_ctx, err = create_exec_ctx(hwfc, &fp->upload_ctx,
dev_hwctx->queue_family_tx_index, dev_hwctx->queue_family_tx_index,
GET_QUEUE_COUNT(dev_hwctx, 0, 0, 1)); GET_QUEUE_COUNT(dev_hwctx, 0, 0, 1));
if (err) if (err)
goto fail; return err;
err = create_exec_ctx(hwfc, &fp->download_ctx, err = create_exec_ctx(hwfc, &fp->download_ctx,
dev_hwctx->queue_family_tx_index, 1); dev_hwctx->queue_family_tx_index, 1);
if (err) if (err)
goto fail; return err;
/* Test to see if allocation will fail */ /* Test to see if allocation will fail */
err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage,
hwctx->create_pnext); hwctx->create_pnext);
if (err) if (err)
goto fail; return err;
vulkan_frame_free(hwfc, (uint8_t *)f); vulkan_frame_free(hwfc, (uint8_t *)f);
@ -1746,20 +1750,11 @@ static int vulkan_frames_init(AVHWFramesContext *hwfc)
hwfc->internal->pool_internal = av_buffer_pool_init2(sizeof(AVVkFrame), hwfc->internal->pool_internal = av_buffer_pool_init2(sizeof(AVVkFrame),
hwfc, vulkan_pool_alloc, hwfc, vulkan_pool_alloc,
NULL); NULL);
if (!hwfc->internal->pool_internal) { if (!hwfc->internal->pool_internal)
err = AVERROR(ENOMEM); return AVERROR(ENOMEM);
goto fail;
}
} }
return 0; return 0;
fail:
free_exec_ctx(hwfc, &fp->conv_ctx);
free_exec_ctx(hwfc, &fp->upload_ctx);
free_exec_ctx(hwfc, &fp->download_ctx);
return err;
} }
static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame) static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)