1
0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2024-11-21 10:55:51 +02:00

libavfiter/dnn_backend_openvino: Add multiple output support

Add multiple output support to openvino backend. You can use '&' to
split different output when you set output name using command line.

Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
Reviewed-by: Guo Yejun <yejun.guo@intel.com>
This commit is contained in:
Wenbin Chen 2023-12-12 10:33:31 +08:00 committed by Guo Yejun
parent 1f56bfc986
commit 22652b576c
3 changed files with 150 additions and 84 deletions

View File

@ -43,13 +43,6 @@ int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func
return AVERROR(EINVAL);
}
if (exec_params->nb_output != 1 && backend != DNN_TF) {
// currently, the filter does not need multiple outputs,
// so we just pending the support until we really need it.
avpriv_report_missing_feature(ctx, "multiple outputs");
return AVERROR(ENOSYS);
}
return 0;
}

View File

@ -64,7 +64,7 @@ typedef struct OVModel{
ov_compiled_model_t *compiled_model;
ov_output_const_port_t* input_port;
ov_preprocess_input_info_t* input_info;
ov_output_const_port_t* output_port;
ov_output_const_port_t** output_ports;
ov_preprocess_output_info_t* output_info;
ov_preprocess_prepostprocessor_t* preprocess;
#else
@ -77,6 +77,7 @@ typedef struct OVModel{
SafeQueue *request_queue; // holds OVRequestItem
Queue *task_queue; // holds TaskItem
Queue *lltask_queue; // holds LastLevelTaskItem
int nb_outputs;
} OVModel;
// one request for one call to openvino
@ -349,7 +350,7 @@ static void infer_completion_callback(void *args)
TaskItem *task = lltask->task;
OVModel *ov_model = task->model;
SafeQueue *requestq = ov_model->request_queue;
DNNData output;
DNNData *outputs;
OVContext *ctx = &ov_model->ctx;
#if HAVE_OPENVINO2
size_t* dims;
@ -358,45 +359,61 @@ static void infer_completion_callback(void *args)
ov_shape_t output_shape = {0};
ov_element_type_e precision;
memset(&output, 0, sizeof(output));
status = ov_infer_request_get_output_tensor_by_index(request->infer_request, 0, &output_tensor);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR,
"Failed to get output tensor.");
outputs = av_calloc(ov_model->nb_outputs, sizeof(*outputs));
if (!outputs) {
av_log(ctx, AV_LOG_ERROR, "Failed to alloc outputs.");
return;
}
status = ov_tensor_data(output_tensor, &output.data);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR,
"Failed to get output data.");
return;
}
for (int i = 0; i < ov_model->nb_outputs; i++) {
status = ov_infer_request_get_tensor_by_const_port(request->infer_request,
ov_model->output_ports[i],
&output_tensor);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR,
"Failed to get output tensor.");
goto end;
}
status = ov_tensor_get_shape(output_tensor, &output_shape);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get output port shape.\n");
return;
}
dims = output_shape.dims;
status = ov_tensor_data(output_tensor, &outputs[i].data);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR,
"Failed to get output data.");
goto end;
}
status = ov_port_get_element_type(ov_model->output_port, &precision);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get output port data type.\n");
status = ov_tensor_get_shape(output_tensor, &output_shape);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get output port shape.\n");
goto end;
}
dims = output_shape.dims;
status = ov_port_get_element_type(ov_model->output_ports[i], &precision);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get output port data type.\n");
goto end;
}
outputs[i].dt = precision_to_datatype(precision);
outputs[i].channels = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1;
outputs[i].height = output_shape.rank > 1 ? dims[output_shape.rank - 2] : 1;
outputs[i].width = output_shape.rank > 0 ? dims[output_shape.rank - 1] : 1;
av_assert0(request->lltask_count <= dims[0]);
outputs[i].layout = ctx->options.layout;
outputs[i].scale = ctx->options.scale;
outputs[i].mean = ctx->options.mean;
ov_shape_free(&output_shape);
return;
ov_tensor_free(output_tensor);
output_tensor = NULL;
}
output.channels = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1;
output.height = output_shape.rank > 1 ? dims[output_shape.rank - 2] : 1;
output.width = output_shape.rank > 0 ? dims[output_shape.rank - 1] : 1;
av_assert0(request->lltask_count <= dims[0]);
ov_shape_free(&output_shape);
#else
IEStatusCode status;
dimensions_t dims;
ie_blob_t *output_blob = NULL;
ie_blob_buffer_t blob_buffer;
precision_e precision;
DNNData output;
status = ie_infer_request_get_blob(request->infer_request, task->output_names[0], &output_blob);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR,
@ -424,11 +441,12 @@ static void infer_completion_callback(void *args)
output.height = dims.dims[2];
output.width = dims.dims[3];
av_assert0(request->lltask_count <= dims.dims[0]);
#endif
output.dt = precision_to_datatype(precision);
output.layout = ctx->options.layout;
output.scale = ctx->options.scale;
output.mean = ctx->options.mean;
outputs = &output;
#endif
av_assert0(request->lltask_count >= 1);
for (int i = 0; i < request->lltask_count; ++i) {
@ -438,28 +456,33 @@ static void infer_completion_callback(void *args)
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
if (ov_model->model->frame_post_proc != NULL) {
ov_model->model->frame_post_proc(task->out_frame, &output, ov_model->model->filter_ctx);
ov_model->model->frame_post_proc(task->out_frame, outputs, ov_model->model->filter_ctx);
} else {
ff_proc_from_dnn_to_frame(task->out_frame, &output, ctx);
ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
} else {
task->out_frame->width = output.width;
task->out_frame->height = output.height;
task->out_frame->width = outputs[0].width;
task->out_frame->height = outputs[0].height;
}
break;
case DFT_ANALYTICS_DETECT:
if (!ov_model->model->detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post proc\n");
return;
goto end;
}
ov_model->model->detect_post_proc(task->in_frame, &output, 1, ov_model->model->filter_ctx);
ov_model->model->detect_post_proc(task->in_frame, outputs,
ov_model->nb_outputs,
ov_model->model->filter_ctx);
break;
case DFT_ANALYTICS_CLASSIFY:
if (!ov_model->model->classify_post_proc) {
av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide post proc\n");
return;
goto end;
}
ov_model->model->classify_post_proc(task->in_frame, &output, request->lltasks[i]->bbox_index, ov_model->model->filter_ctx);
for (int output_i = 0; output_i < ov_model->nb_outputs; output_i++)
ov_model->model->classify_post_proc(task->in_frame, outputs,
request->lltasks[i]->bbox_index,
ov_model->model->filter_ctx);
break;
default:
av_assert0(!"should not reach here");
@ -468,10 +491,17 @@ static void infer_completion_callback(void *args)
task->inference_done++;
av_freep(&request->lltasks[i]);
output.data = (uint8_t *)output.data
+ output.width * output.height * output.channels * get_datatype_size(output.dt);
for (int i = 0; i < ov_model->nb_outputs; i++)
outputs[i].data = (uint8_t *)outputs[i].data +
outputs[i].width * outputs[i].height * outputs[i].channels * get_datatype_size(outputs[i].dt);
}
#if !HAVE_OPENVINO2
end:
#if HAVE_OPENVINO2
av_freep(&outputs);
ov_shape_free(&output_shape);
if (output_tensor)
ov_tensor_free(output_tensor);
#else
ie_blob_free(&output_blob);
#endif
request->lltask_count = 0;
@ -525,8 +555,10 @@ static void dnn_free_model_ov(DNNModel **model)
#if HAVE_OPENVINO2
if (ov_model->input_port)
ov_output_const_port_free(ov_model->input_port);
if (ov_model->output_port)
ov_output_const_port_free(ov_model->output_port);
for (int i = 0; i < ov_model->nb_outputs; i++)
if (ov_model->output_ports[i])
ov_output_const_port_free(ov_model->output_ports[i]);
av_freep(&ov_model->output_ports);
if (ov_model->preprocess)
ov_preprocess_prepostprocessor_free(ov_model->preprocess);
if (ov_model->compiled_model)
@ -551,7 +583,7 @@ static void dnn_free_model_ov(DNNModel **model)
}
static int init_model_ov(OVModel *ov_model, const char *input_name, const char *output_name)
static int init_model_ov(OVModel *ov_model, const char *input_name, const char **output_names, int nb_outputs)
{
int ret = 0;
OVContext *ctx = &ov_model->ctx;
@ -594,17 +626,15 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
}
status = ov_preprocess_prepostprocessor_get_input_info_by_name(ov_model->preprocess, input_name, &ov_model->input_info);
status |= ov_preprocess_prepostprocessor_get_output_info_by_name(ov_model->preprocess, output_name, &ov_model->output_info);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get input/output info from preprocess.\n");
av_log(ctx, AV_LOG_ERROR, "Failed to get input info from preprocess.\n");
ret = ov2_map_error(status, NULL);
goto err;
}
status = ov_preprocess_input_info_get_tensor_info(ov_model->input_info, &input_tensor_info);
status |= ov_preprocess_output_info_get_tensor_info(ov_model->output_info, &output_tensor_info);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input/output.\n");
av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input.\n");
ret = ov2_map_error(status, NULL);
goto err;
}
@ -642,17 +672,43 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
}
status = ov_preprocess_input_tensor_info_set_element_type(input_tensor_info, U8);
if (ov_model->model->func_type != DFT_PROCESS_FRAME)
status |= ov_preprocess_output_set_element_type(output_tensor_info, F32);
else if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f)
status |= ov_preprocess_output_set_element_type(output_tensor_info, F32);
else
status |= ov_preprocess_output_set_element_type(output_tensor_info, U8);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to set input/output element type\n");
av_log(ctx, AV_LOG_ERROR, "Failed to set input element type\n");
ret = ov2_map_error(status, NULL);
goto err;
}
ov_model->nb_outputs = nb_outputs;
for (int i = 0; i < nb_outputs; i++) {
status = ov_preprocess_prepostprocessor_get_output_info_by_name(
ov_model->preprocess, output_names[i], &ov_model->output_info);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get output info from preprocess.\n");
ret = ov2_map_error(status, NULL);
goto err;
}
status |= ov_preprocess_output_info_get_tensor_info(ov_model->output_info, &output_tensor_info);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input/output.\n");
ret = ov2_map_error(status, NULL);
goto err;
}
if (ov_model->model->func_type != DFT_PROCESS_FRAME)
status |= ov_preprocess_output_set_element_type(output_tensor_info, F32);
else if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f)
status |= ov_preprocess_output_set_element_type(output_tensor_info, F32);
else
status |= ov_preprocess_output_set_element_type(output_tensor_info, U8);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to set output element type\n");
ret = ov2_map_error(status, NULL);
goto err;
}
ov_preprocess_output_tensor_info_free(output_tensor_info);
output_tensor_info = NULL;
ov_preprocess_output_info_free(ov_model->output_info);
ov_model->output_info = NULL;
}
// set preprocess steps.
if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f) {
ov_preprocess_preprocess_steps_t* input_process_steps = NULL;
@ -667,11 +723,18 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
status |= ov_preprocess_preprocess_steps_scale(input_process_steps, ctx->options.scale);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to set preprocess steps\n");
ov_preprocess_preprocess_steps_free(input_process_steps);
input_process_steps = NULL;
ret = ov2_map_error(status, NULL);
goto err;
}
ov_preprocess_preprocess_steps_free(input_process_steps);
input_process_steps = NULL;
}
ov_preprocess_input_tensor_info_free(input_tensor_info);
input_tensor_info = NULL;
ov_preprocess_input_info_free(ov_model->input_info);
ov_model->input_info = NULL;
//update model
if(ov_model->ov_model)
@ -679,20 +742,33 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
status = ov_preprocess_prepostprocessor_build(ov_model->preprocess, &ov_model->ov_model);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to update OV model\n");
ov_model_free(tmp_ov_model);
tmp_ov_model = NULL;
ret = ov2_map_error(status, NULL);
goto err;
}
ov_model_free(tmp_ov_model);
//update output_port
if (ov_model->output_port) {
ov_output_const_port_free(ov_model->output_port);
ov_model->output_port = NULL;
}
status = ov_model_const_output_by_name(ov_model->ov_model, output_name, &ov_model->output_port);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get output port.\n");
goto err;
if (!ov_model->output_ports) {
ov_model->output_ports = av_calloc(nb_outputs, sizeof(*ov_model->output_ports));
if (!ov_model->output_ports) {
ret = AVERROR(ENOMEM);
goto err;
}
} else
for (int i = 0; i < nb_outputs; i++) {
ov_output_const_port_free(ov_model->output_ports[i]);
ov_model->output_ports[i] = NULL;
}
for (int i = 0; i < nb_outputs; i++) {
status = ov_model_const_output_by_name(ov_model->ov_model, output_names[i],
&ov_model->output_ports[i]);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get output port %s.\n", output_names[i]);
goto err;
}
}
//compile network
status = ov_core_compile_model(ov_model->core, ov_model->ov_model, device, 0, &ov_model->compiled_model);
@ -701,6 +777,7 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
goto err;
}
ov_preprocess_input_model_info_free(input_model_info);
input_model_info = NULL;
ov_layout_free(NCHW_layout);
ov_layout_free(NHWC_layout);
#else
@ -745,6 +822,7 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
ret = DNN_GENERIC_ERROR;
goto err;
}
ov_model->nb_outputs = 1;
// all models in openvino open model zoo use BGR with range [0.0f, 255.0f] as input,
// we don't have a AVPixelFormat to describe it, so we'll use AV_PIX_FMT_BGR24 and
@ -848,6 +926,10 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
err:
#if HAVE_OPENVINO2
if (output_tensor_info)
ov_preprocess_output_tensor_info_free(output_tensor_info);
if (ov_model->output_info)
ov_preprocess_output_info_free(ov_model->output_info);
if (NCHW_layout)
ov_layout_free(NCHW_layout);
if (NHWC_layout)
@ -1204,11 +1286,6 @@ static int get_output_ov(void *model, const char *input_name, int input_width, i
}
}
status = ov_model_const_output_by_name(ov_model->ov_model, output_name, &ov_model->output_port);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to get output port.\n");
return ov2_map_error(status, NULL);
}
if (!ov_model->compiled_model) {
#else
if (ctx->options.input_resizable) {
@ -1224,7 +1301,7 @@ static int get_output_ov(void *model, const char *input_name, int input_width, i
}
if (!ov_model->exe_network) {
#endif
ret = init_model_ov(ov_model, input_name, output_name);
ret = init_model_ov(ov_model, input_name, &output_name, 1);
if (ret != 0) {
av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
return ret;
@ -1397,7 +1474,8 @@ static int dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams *exec_p
#else
if (!ov_model->exe_network) {
#endif
ret = init_model_ov(ov_model, exec_params->input_name, exec_params->output_names[0]);
ret = init_model_ov(ov_model, exec_params->input_name,
exec_params->output_names, exec_params->nb_output);
if (ret != 0) {
av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
return ret;

View File

@ -360,11 +360,11 @@ static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, AVFilterCon
break;
}
}
return 0;
}
static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx)
static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, int nb_outputs,
AVFilterContext *filter_ctx)
{
AVFrameSideData *sd;
DnnDetectContext *ctx = filter_ctx->priv;
@ -472,7 +472,7 @@ static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AV
DnnContext *dnn_ctx = &ctx->dnnctx;
switch (dnn_ctx->backend_type) {
case DNN_OV:
return dnn_detect_post_proc_ov(frame, output, filter_ctx);
return dnn_detect_post_proc_ov(frame, output, nb, filter_ctx);
case DNN_TF:
return dnn_detect_post_proc_tf(frame, output, filter_ctx);
default:
@ -559,11 +559,6 @@ static int check_output_nb(DnnDetectContext *ctx, DNNBackendType backend_type, i
}
return 0;
case DNN_OV:
if (output_nb != 1) {
av_log(ctx, AV_LOG_ERROR, "Dnn detect filter with openvino backend needs 1 output only, \
but get %d instead\n", output_nb);
return AVERROR(EINVAL);
}
return 0;
default:
avpriv_report_missing_feature(ctx, "Dnn detect filter does not support current backend\n");