FFmpeg
dnn_interface.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN inference engine interface.
24  */
25 
26 #ifndef AVFILTER_DNN_INTERFACE_H
27 #define AVFILTER_DNN_INTERFACE_H
28 
29 #include <stdint.h>
30 #include "libavutil/frame.h"
31 #include "avfilter.h"
32 
33 #define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!')
34 
35 typedef enum {DNN_TF = 1, DNN_OV, DNN_TH} DNNBackendType;
36 
37 typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
38 
39 typedef enum {
44 
45 typedef enum {
46  DAST_FAIL, // something wrong
47  DAST_EMPTY_QUEUE, // no more inference result to get
48  DAST_NOT_READY, // all queued inferences are not finished
49  DAST_SUCCESS // got a result frame successfully
51 
52 typedef enum {
54  DFT_PROCESS_FRAME, // process the whole frame
55  DFT_ANALYTICS_DETECT, // detect from the whole frame
56  DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
58 
59 typedef enum {
63 } DNNLayout;
64 
65 typedef struct DNNData{
66  void *data;
67  int dims[4];
68  // dt and order together decide the color format
72  float scale;
73  float mean;
74 } DNNData;
75 
76 typedef struct DNNExecBaseParams {
77  const char *input_name;
78  const char **output_names;
79  uint32_t nb_output;
83 
86  const char *target;
88 
91 typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx);
92 
93 typedef struct DNNModel{
94  // Stores model that can be different for different backends.
95  void *model;
96  // Stores options when the model is executed by the backend
97  const char *options;
98  // Stores FilterContext used for the interaction between AVFrame and DNNData
100  // Stores function type of the model
102  // Gets model input information
103  // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
104  int (*get_input)(void *model, DNNData *input, const char *input_name);
105  // Gets model output width/height with given input w/h
106  int (*get_output)(void *model, const char *input_name, int input_width, int input_height,
107  const char *output_name, int *output_width, int *output_height);
108  // set the pre process to transfer data from AVFrame to DNNData
109  // the default implementation within DNN is used if it is not provided by the filter
111  // set the post process to transfer data from DNNData to AVFrame
112  // the default implementation within DNN is used if it is not provided by the filter
114  // set the post process to interpret detect result from DNNData
116  // set the post process to interpret classify result from DNNData
118 } DNNModel;
119 
120 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
121 typedef struct DNNModule{
122  // Loads model and parameters from given file. Returns NULL if it is not possible.
123  DNNModel *(*load_model)(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
124  // Executes model with specified input and output. Returns the error code otherwise.
125  int (*execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params);
126  // Retrieve inference result.
128  // Flush all the pending tasks.
129  int (*flush)(const DNNModel *model);
130  // Frees memory allocated for model.
131  void (*free_model)(DNNModel **model);
132 } DNNModule;
133 
134 // Initializes DNNModule depending on chosen backend.
135 const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx);
136 
138 {
139  return layout == DL_NHWC ? 2 : 3;
140 }
141 
143 {
144  return layout == DL_NHWC ? 1 : 2;
145 }
146 
148 {
149  return layout == DL_NHWC ? 3 : 1;
150 }
151 
152 #endif
DNNColorOrder
DNNColorOrder
Definition: dnn_interface.h:39
out
FILE * out
Definition: movenc.c:54
DNNModule::get_result
DNNAsyncStatusType(* get_result)(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_interface.h:127
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
DNNData::data
void * data
Definition: dnn_interface.h:66
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:375
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:110
DetectPostProc
int(* DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:90
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:77
DNNExecBaseParams::in_frame
AVFrame * in_frame
Definition: dnn_interface.h:80
DFT_NONE
@ DFT_NONE
Definition: dnn_interface.h:53
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:99
dnn_get_width_idx_by_layout
static int dnn_get_width_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:137
filter_ctx
static FilteringContext * filter_ctx
Definition: transcode.c:51
DAST_FAIL
@ DAST_FAIL
Definition: dnn_interface.h:46
DL_NHWC
@ DL_NHWC
Definition: dnn_interface.h:62
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:35
DCO_NONE
@ DCO_NONE
Definition: dnn_interface.h:40
DNNExecClassificationParams
Definition: dnn_interface.h:84
DNNData::order
DNNColorOrder order
Definition: dnn_interface.h:70
DNNData
Definition: dnn_interface.h:65
DL_NCHW
@ DL_NCHW
Definition: dnn_interface.h:61
DNN_OV
@ DNN_OV
Definition: dnn_interface.h:35
DNNExecClassificationParams::target
const char * target
Definition: dnn_interface.h:86
DNNExecClassificationParams::base
DNNExecBaseParams base
Definition: dnn_interface.h:85
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:113
ff_get_dnn_module
const DNNModule * ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
Definition: dnn_interface.c:33
options
const OptionDef options[]
ClassifyPostProc
int(* ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:91
DAST_SUCCESS
@ DAST_SUCCESS
Definition: dnn_interface.h:49
DNNBackendType
DNNBackendType
Definition: dnn_interface.h:35
DAST_EMPTY_QUEUE
@ DAST_EMPTY_QUEUE
Definition: dnn_interface.h:47
DNNLayout
DNNLayout
Definition: dnn_interface.h:59
DNNModel::detect_post_proc
DetectPostProc detect_post_proc
Definition: dnn_interface.h:115
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:101
DNNDataType
DNNDataType
Definition: dnn_interface.h:37
DNNData::dt
DNNDataType dt
Definition: dnn_interface.h:69
frame.h
DNNData::scale
float scale
Definition: dnn_interface.h:72
DNNData::layout
DNNLayout layout
Definition: dnn_interface.h:71
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
DNNExecBaseParams::out_frame
AVFrame * out_frame
Definition: dnn_interface.h:81
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:55
DNNModel::classify_post_proc
ClassifyPostProc classify_post_proc
Definition: dnn_interface.h:117
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
DNNData::mean
float mean
Definition: dnn_interface.h:73
DNNModel::get_input
int(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:104
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:37
DFT_ANALYTICS_CLASSIFY
@ DFT_ANALYTICS_CLASSIFY
Definition: dnn_interface.h:56
DNNModule::free_model
void(* free_model)(DNNModel **model)
Definition: dnn_interface.h:131
avfilter.h
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:42
DNNExecBaseParams::output_names
const char ** output_names
Definition: dnn_interface.h:78
FramePrePostProc
int(* FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:89
DL_NONE
@ DL_NONE
Definition: dnn_interface.h:60
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
DNNModel
Definition: dnn_interface.h:93
DNNData::dims
int dims[4]
Definition: dnn_interface.h:67
DNN_TH
@ DNN_TH
Definition: dnn_interface.h:35
dnn_get_height_idx_by_layout
static int dnn_get_height_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:142
DNNModel::options
const char * options
Definition: dnn_interface.h:97
dnn_get_channel_idx_by_layout
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:147
DNNExecBaseParams
Definition: dnn_interface.h:76
DNNModel::get_output
int(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:106
DCO_BGR
@ DCO_BGR
Definition: dnn_interface.h:41
DAST_NOT_READY
@ DAST_NOT_READY
Definition: dnn_interface.h:48
int
int
Definition: ffmpeg_filter.c:410
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
DNNModule
Definition: dnn_interface.h:121
DNNModule::flush
int(* flush)(const DNNModel *model)
Definition: dnn_interface.h:129
DNNExecBaseParams::nb_output
uint32_t nb_output
Definition: dnn_interface.h:79
DNNModel::model
void * model
Definition: dnn_interface.h:95
DNNModule::execute_model
int(* execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_interface.h:125