virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2025-02-04 06:08:26 +02:00
Code Issues Releases Activity

lavfi/dnn: Task-based Inference in Native Backend

Browse Source 
This commit rearranges the code in Native Backend to use the TaskItem
for inference.

Signed-off-by: Shubhanshu Saxena <shubhanshu.e01@gmail.com>
This commit is contained in:
Shubhanshu Saxena 2021-08-26 02:38:06 +05:30 committed by Guo Yejun
parent d91542e618
commit d39580ac11
2 changed files with 120 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
libavfilter/dnn/dnn_backend_native.c
View File

@ -45,9 +45,29 @@ static const AVClass dnn_native_class = {
.category = AV_CLASS_CATEGORY_FILTER, .category = AV_CLASS_CATEGORY_FILTER,
}; };
static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame, static DNNReturnType execute_model_native(Queue *inference_queue);
const char **output_names, uint32_t nb_output, AVFrame *out_frame,
int do_ioproc); static DNNReturnType extract_inference_from_task(TaskItem *task, Queue *inference_queue)
{
NativeModel *native_model = task->model;
NativeContext *ctx = &native_model->ctx;
InferenceItem *inference = av_malloc(sizeof(*inference));
if (!inference) {
av_log(ctx, AV_LOG_ERROR, "Unable to allocate space for InferenceItem\n");
return DNN_ERROR;
}
task->inference_todo = 1;
task->inference_done = 0;
inference->task = task;
if (ff_queue_push_back(inference_queue, inference) < 0) {
av_log(ctx, AV_LOG_ERROR, "Failed to push back inference_queue.\n");
av_freep(&inference);
return DNN_ERROR;
}
return DNN_SUCCESS;
}
static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name) static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
{ {
@ -78,34 +98,36 @@ static DNNReturnType get_input_native(void *model, DNNData *input, const char *i
static DNNReturnType get_output_native(void *model, const char *input_name, int input_width, int input_height, static DNNReturnType get_output_native(void *model, const char *input_name, int input_width, int input_height,
const char *output_name, int *output_width, int *output_height) const char *output_name, int *output_width, int *output_height)
{ {
DNNReturnType ret; DNNReturnType ret = 0;
NativeModel *native_model = model; NativeModel *native_model = model;
NativeContext *ctx = &native_model->ctx; NativeContext *ctx = &native_model->ctx;
AVFrame *in_frame = av_frame_alloc(); TaskItem task;
AVFrame *out_frame = NULL; DNNExecBaseParams exec_params = {
.input_name = input_name,
.output_names = &output_name,
.nb_output = 1,
.in_frame = NULL,
.out_frame = NULL,
};
if (!in_frame) { if (ff_dnn_fill_gettingoutput_task(&task, &exec_params, native_model, input_height, input_width, ctx) != DNN_SUCCESS) {
av_log(ctx, AV_LOG_ERROR, "Could not allocate memory for input frame\n"); ret = DNN_ERROR;
return DNN_ERROR; goto err;
} }
out_frame = av_frame_alloc(); if (extract_inference_from_task(&task, native_model->inference_queue) != DNN_SUCCESS) {
av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
if (!out_frame) { ret = DNN_ERROR;
av_log(ctx, AV_LOG_ERROR, "Could not allocate memory for output frame\n"); goto err;
av_frame_free(&in_frame);
return DNN_ERROR;
} }
in_frame->width = input_width; ret = execute_model_native(native_model->inference_queue);
in_frame->height = input_height; *output_width = task.out_frame->width;
*output_height = task.out_frame->height;
ret = execute_model_native(native_model->model, input_name, in_frame, &output_name, 1, out_frame, 0); err:
*output_width = out_frame->width; av_frame_free(&task.out_frame);
*output_height = out_frame->height; av_frame_free(&task.in_frame);
av_frame_free(&out_frame);
av_frame_free(&in_frame);
return ret; return ret;
} }
@ -190,6 +212,11 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType f
goto fail; goto fail;
} }
native_model->inference_queue = ff_queue_create();
if (!native_model->inference_queue) {
goto fail;
}
for (layer = 0; layer < native_model->layers_num; ++layer){ for (layer = 0; layer < native_model->layers_num; ++layer){
layer_type = (int32_t)avio_rl32(model_file_context); layer_type = (int32_t)avio_rl32(model_file_context);
dnn_size += 4; dnn_size += 4;
@ -259,50 +286,66 @@ fail:
return NULL; return NULL;
} }
static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame, static DNNReturnType execute_model_native(Queue *inference_queue)
const char **output_names, uint32_t nb_output, AVFrame *out_frame,
int do_ioproc)
{ {
NativeModel *native_model = model->model; NativeModel *native_model = NULL;
NativeContext *ctx = &native_model->ctx; NativeContext *ctx = NULL;
int32_t layer; int32_t layer;
DNNData input, output; DNNData input, output;
DnnOperand *oprd = NULL; DnnOperand *oprd = NULL;
InferenceItem *inference = NULL;
TaskItem *task = NULL;
DNNReturnType ret = 0;
inference = ff_queue_pop_front(inference_queue);
if (!inference) {
av_log(NULL, AV_LOG_ERROR, "Failed to get inference item\n");
ret = DNN_ERROR;
goto err;
}
task = inference->task;
native_model = task->model;
ctx = &native_model->ctx;
if (native_model->layers_num <= 0 || native_model->operands_num <= 0) { if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n"); av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
return DNN_ERROR; ret = DNN_ERROR;
goto err;
} }
for (int i = 0; i < native_model->operands_num; ++i) { for (int i = 0; i < native_model->operands_num; ++i) {
oprd = &native_model->operands[i]; oprd = &native_model->operands[i];
if (strcmp(oprd->name, input_name) == 0) { if (strcmp(oprd->name, task->input_name) == 0) {
if (oprd->type != DOT_INPUT) { if (oprd->type != DOT_INPUT) {
av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name); av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", task->input_name);
return DNN_ERROR; ret = DNN_ERROR;
goto err;
} }
break; break;
} }
oprd = NULL; oprd = NULL;
} }
if (!oprd) { if (!oprd) {
av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name); av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name);
return DNN_ERROR; ret = DNN_ERROR;
goto err;
} }
oprd->dims[1] = in_frame->height; oprd->dims[1] = task->in_frame->height;
oprd->dims[2] = in_frame->width; oprd->dims[2] = task->in_frame->width;
av_freep(&oprd->data); av_freep(&oprd->data);
oprd->length = ff_calculate_operand_data_length(oprd); oprd->length = ff_calculate_operand_data_length(oprd);
if (oprd->length <= 0) { if (oprd->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n"); av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n");
return DNN_ERROR; ret = DNN_ERROR;
goto err;
} }
oprd->data = av_malloc(oprd->length); oprd->data = av_malloc(oprd->length);
if (!oprd->data) { if (!oprd->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n"); av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n");
return DNN_ERROR; ret = DNN_ERROR;
goto err;
} }
input.height = oprd->dims[1]; input.height = oprd->dims[1];
@ -310,19 +353,20 @@ static DNNReturnType execute_model_native(const DNNModel *model, const char *inp
input.channels = oprd->dims[3]; input.channels = oprd->dims[3];
input.data = oprd->data; input.data = oprd->data;
input.dt = oprd->data_type; input.dt = oprd->data_type;
if (do_ioproc) { if (task->do_ioproc) {
if (native_model->model->frame_pre_proc != NULL) { if (native_model->model->frame_pre_proc != NULL) {
native_model->model->frame_pre_proc(in_frame, &input, native_model->model->filter_ctx); native_model->model->frame_pre_proc(task->in_frame, &input, native_model->model->filter_ctx);
} else { } else {
ff_proc_from_frame_to_dnn(in_frame, &input, ctx); ff_proc_from_frame_to_dnn(task->in_frame, &input, ctx);
} }
} }
if (nb_output != 1) { if (task->nb_output != 1) {
// currently, the filter does not need multiple outputs, // currently, the filter does not need multiple outputs,
// so we just pending the support until we really need it. // so we just pending the support until we really need it.
avpriv_report_missing_feature(ctx, "multiple outputs"); avpriv_report_missing_feature(ctx, "multiple outputs");
return DNN_ERROR; ret = DNN_ERROR;
goto err;
} }
for (layer = 0; layer < native_model->layers_num; ++layer){ for (layer = 0; layer < native_model->layers_num; ++layer){
@ -333,13 +377,14 @@ static DNNReturnType execute_model_native(const DNNModel *model, const char *inp
native_model->layers[layer].params, native_model->layers[layer].params,
&native_model->ctx) == DNN_ERROR) { &native_model->ctx) == DNN_ERROR) {
av_log(ctx, AV_LOG_ERROR, "Failed to execute model\n"); av_log(ctx, AV_LOG_ERROR, "Failed to execute model\n");
return DNN_ERROR; ret = DNN_ERROR;
goto err;
} }
} }
for (uint32_t i = 0; i < nb_output; ++i) { for (uint32_t i = 0; i < task->nb_output; ++i) {
DnnOperand *oprd = NULL; DnnOperand *oprd = NULL;
const char *output_name = output_names[i]; const char *output_name = task->output_names[i];
for (int j = 0; j < native_model->operands_num; ++j) { for (int j = 0; j < native_model->operands_num; ++j) {
if (strcmp(native_model->operands[j].name, output_name) == 0) { if (strcmp(native_model->operands[j].name, output_name) == 0) {
oprd = &native_model->operands[j]; oprd = &native_model->operands[j];
@ -349,7 +394,8 @@ static DNNReturnType execute_model_native(const DNNModel *model, const char *inp
if (oprd == NULL) { if (oprd == NULL) {
av_log(ctx, AV_LOG_ERROR, "Could not find output in model\n"); av_log(ctx, AV_LOG_ERROR, "Could not find output in model\n");
return DNN_ERROR; ret = DNN_ERROR;
goto err;
} }
output.data = oprd->data; output.data = oprd->data;
@ -358,32 +404,43 @@ static DNNReturnType execute_model_native(const DNNModel *model, const char *inp
output.channels = oprd->dims[3]; output.channels = oprd->dims[3];
output.dt = oprd->data_type; output.dt = oprd->data_type;
if (do_ioproc) { if (task->do_ioproc) {
if (native_model->model->frame_post_proc != NULL) { if (native_model->model->frame_post_proc != NULL) {
native_model->model->frame_post_proc(out_frame, &output, native_model->model->filter_ctx); native_model->model->frame_post_proc(task->out_frame, &output, native_model->model->filter_ctx);
} else { } else {
ff_proc_from_dnn_to_frame(out_frame, &output, ctx); ff_proc_from_dnn_to_frame(task->out_frame, &output, ctx);
} }
} else { } else {
out_frame->width = output.width; task->out_frame->width = output.width;
out_frame->height = output.height; task->out_frame->height = output.height;
} }
} }
task->inference_done++;
return DNN_SUCCESS; err:
av_freep(&inference);
return ret;
} }
DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params) DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params)
{ {
NativeModel *native_model = model->model; NativeModel *native_model = model->model;
NativeContext *ctx = &native_model->ctx; NativeContext *ctx = &native_model->ctx;
TaskItem task;
if (ff_check_exec_params(ctx, DNN_NATIVE, model->func_type, exec_params) != 0) { if (ff_check_exec_params(ctx, DNN_NATIVE, model->func_type, exec_params) != 0) {
return DNN_ERROR; return DNN_ERROR;
} }
return execute_model_native(model, exec_params->input_name, exec_params->in_frame, if (ff_dnn_fill_task(&task, exec_params, native_model, 0, 1) != DNN_SUCCESS) {
exec_params->output_names, exec_params->nb_output, exec_params->out_frame, 1); return DNN_ERROR;
}
if (extract_inference_from_task(&task, native_model->inference_queue) != DNN_SUCCESS) {
av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
return DNN_ERROR;
}
return execute_model_native(native_model->inference_queue);
} }
int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd) int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd)
@ -435,6 +492,11 @@ void ff_dnn_free_model_native(DNNModel **model)
av_freep(&native_model->operands); av_freep(&native_model->operands);
} }
while (ff_queue_size(native_model->inference_queue) != 0) {
InferenceItem *item = ff_queue_pop_front(native_model->inference_queue);
av_freep(&item);
}
ff_queue_destroy(native_model->inference_queue);
av_freep(&native_model); av_freep(&native_model);
} }
av_freep(model); av_freep(model);

2
libavfilter/dnn/dnn_backend_native.h
View File

@ -30,6 +30,7 @@
#include "../dnn_interface.h" #include "../dnn_interface.h"
#include "libavformat/avio.h" #include "libavformat/avio.h"
#include "libavutil/opt.h" #include "libavutil/opt.h"
#include "queue.h"
/** /**
* the enum value of DNNLayerType should not be changed, * the enum value of DNNLayerType should not be changed,
@ -126,6 +127,7 @@ typedef struct NativeModel{
int32_t layers_num; int32_t layers_num;
DnnOperand *operands; DnnOperand *operands;
int32_t operands_num; int32_t operands_num;
Queue *inference_queue;
} NativeModel; } NativeModel;
DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx); DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);