FFmpeg  4.4.4
vulkan.h
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #ifndef AVFILTER_VULKAN_H
20 #define AVFILTER_VULKAN_H
21 
22 #include "avfilter.h"
23 #include "libavutil/pixdesc.h"
24 #include "libavutil/bprint.h"
25 #include "libavutil/hwcontext.h"
27 
28 /* GLSL management macros */
29 #define INDENT(N) INDENT_##N
30 #define INDENT_0
31 #define INDENT_1 INDENT_0 " "
32 #define INDENT_2 INDENT_1 INDENT_1
33 #define INDENT_3 INDENT_2 INDENT_1
34 #define INDENT_4 INDENT_3 INDENT_1
35 #define INDENT_5 INDENT_4 INDENT_1
36 #define INDENT_6 INDENT_5 INDENT_1
37 #define C(N, S) INDENT(N) #S "\n"
38 #define GLSLC(N, S) av_bprintf(&shd->src, C(N, S))
39 #define GLSLA(...) av_bprintf(&shd->src, __VA_ARGS__)
40 #define GLSLF(N, S, ...) av_bprintf(&shd->src, C(N, S), __VA_ARGS__)
41 #define GLSLD(D) GLSLC(0, ); \
42  av_bprint_append_data(&shd->src, D, strlen(D)); \
43  GLSLC(0, )
44 
45 /* Helper, pretty much every Vulkan return value needs to be checked */
46 #define RET(x) \
47  do { \
48  if ((err = (x)) < 0) \
49  goto fail; \
50  } while (0)
51 
52 /* Gets the queues count for a single queue family */
53 #define GET_QUEUE_COUNT(hwctx, graph, comp, tx) ( \
54  graph ? hwctx->nb_graphics_queues : \
55  comp ? (hwctx->nb_comp_queues ? \
56  hwctx->nb_comp_queues : hwctx->nb_graphics_queues) : \
57  tx ? (hwctx->nb_tx_queues ? hwctx->nb_tx_queues : \
58  (hwctx->nb_comp_queues ? \
59  hwctx->nb_comp_queues : hwctx->nb_graphics_queues)) : \
60  0 \
61 )
62 
63 /* Useful for attaching immutable samplers to arrays */
64 #define DUP_SAMPLER_ARRAY4(x) (VkSampler []){ x, x, x, x, }
65 
66 typedef struct SPIRVShader {
67  const char *name; /* Name for id/debugging purposes */
68  AVBPrint src;
69  int local_size[3]; /* Compute shader workgroup sizes */
70  VkPipelineShaderStageCreateInfo shader;
71 } SPIRVShader;
72 
74  const char *name;
75  VkDescriptorType type;
76  const char *mem_layout; /* Storage images (rgba8, etc.) and buffers (std430, etc.) */
77  const char *mem_quali; /* readonly, writeonly, etc. */
78  const char *buf_content; /* For buffers */
79  uint32_t dimensions; /* Needed for e.g. sampler%iD */
80  uint32_t elems; /* 0 - scalar, 1 or more - vector */
81  VkShaderStageFlags stages;
82  const VkSampler *samplers; /* Immutable samplers, length - #elems */
83  void *updater; /* Pointer to VkDescriptor*Info */
85 
86 typedef struct FFVkBuffer {
87  VkBuffer buf;
88  VkDeviceMemory mem;
89  VkMemoryPropertyFlagBits flags;
90 } FFVkBuffer;
91 
92 typedef struct VulkanPipeline {
93  VkPipelineBindPoint bind_point;
94 
95  /* Contexts */
96  VkPipelineLayout pipeline_layout;
97  VkPipeline pipeline;
98 
99  /* Shaders */
102 
103  /* Push consts */
104  VkPushConstantRange *push_consts;
106 
107  /* Descriptors */
108  VkDescriptorSetLayout *desc_layout;
109  VkDescriptorPool desc_pool;
110  VkDescriptorSet *desc_set;
111  VkDescriptorUpdateTemplate *desc_template;
115 
116  /* Temporary, used to store data in between initialization stages */
117  VkDescriptorUpdateTemplateCreateInfo *desc_template_info;
118  VkDescriptorPoolSize *pool_size_desc;
120 
121 typedef struct FFVkQueueCtx {
122  VkFence fence;
123  VkQueue queue;
124 
125  /* Buffer dependencies */
129 
130  /* Frame dependencies */
134 } FFVkQueueCtx;
135 
136 typedef struct FFVkExecContext {
137  VkCommandPool pool;
138  VkCommandBuffer *bufs;
140 
142  int *nb_deps;
144 
146 
147  VkSemaphore *sem_wait;
148  int sem_wait_alloc; /* Allocated sem_wait */
150 
151  VkPipelineStageFlagBits *sem_wait_dst;
152  int sem_wait_dst_alloc; /* Allocated sem_wait_dst */
153 
154  VkSemaphore *sem_sig;
155  int sem_sig_alloc; /* Allocated sem_sig */
158 
159 typedef struct VulkanFilterContext {
160  const AVClass *class;
161 
163  AVBufferRef *frames_ref; /* For in-place filtering */
166 
167  /* State - mirrored with the exec ctx */
171 
172  /* Properties */
177 
178  /* Samplers */
179  VkSampler **samplers;
181 
182  /* Exec contexts */
185 
186  /* Pipelines (each can have 1 shader of each type) */
189 
190  void *scratch; /* Scratch memory used only in functions */
191  unsigned int scratch_size;
193 
194 /* Identity mapping - r = r, b = b, g = g, a = a */
195 extern const VkComponentMapping ff_comp_identity_map;
196 
197 /**
198  * General lavfi IO functions
199  */
206 
207 /**
208  * Converts Vulkan return values to strings
209  */
210 const char *ff_vk_ret2str(VkResult res);
211 
212 /**
213  * Returns 1 if the image is any sort of supported RGB
214  */
216 
217 /**
218  * Gets the glsl format string for a pixel format
219  */
220 const char *ff_vk_shader_rep_fmt(enum AVPixelFormat pixfmt);
221 
222 /**
223  * Create a Vulkan sampler, will be auto-freed in ff_vk_filter_uninit()
224  */
225 VkSampler *ff_vk_init_sampler(AVFilterContext *avctx, int unnorm_coords,
226  VkFilter filt);
227 
228 /**
229  * Create an imageview.
230  * Guaranteed to remain alive until the queue submission has finished executing,
231  * and will be destroyed after that.
232  */
234  VkImageView *v, VkImage img, VkFormat fmt,
235  const VkComponentMapping map);
236 
237 /**
238  * Define a push constant for a given stage into a pipeline.
239  * Must be called before the pipeline layout has been initialized.
240  */
242  int offset, int size, VkShaderStageFlagBits stage);
243 
244 /**
245  * Inits a pipeline. Everything in it will be auto-freed when calling
246  * ff_vk_filter_uninit().
247  */
249 
250 /**
251  * Inits a shader for a specific pipeline. Will be auto-freed on uninit.
252  */
254  const char *name, VkShaderStageFlags stage);
255 
256 /**
257  * Writes the workgroup size for a shader.
258  */
260  int local_size[3]);
261 
262 /**
263  * Adds a descriptor set to the shader and registers them in the pipeline.
264  */
267  int num, int only_print_to_shader);
268 
269 /**
270  * Compiles the shader, entrypoint must be set to "main".
271  */
273  const char *entrypoint);
274 
275 /**
276  * Initializes the pipeline layout after all shaders and descriptor sets have
277  * been finished.
278  */
280 
281 /**
282  * Initializes a compute pipeline. Will pick the first shader with the
283  * COMPUTE flag set.
284  */
286 
287 /**
288  * Updates a descriptor set via the updaters defined.
289  * Can be called immediately after pipeline creation, but must be called
290  * at least once before queue submission.
291  */
293  int set_id);
294 
295 /**
296  * Init an execution context for command recording and queue submission.
297  * WIll be auto-freed on uninit.
298  */
300 
301 /**
302  * Begin recording to the command buffer. Previous execution must have been
303  * completed, which ff_vk_submit_exec_queue() will ensure.
304  */
306 
307 /**
308  * Add a command to bind the completed pipeline and its descriptor sets.
309  * Must be called after ff_vk_start_exec_recording() and before submission.
310  */
312  VulkanPipeline *pl);
313 
314 /**
315  * Updates push constants.
316  * Must be called after binding a pipeline if any push constants were defined.
317  */
319  VkShaderStageFlagBits stage, int offset,
320  size_t size, void *src);
321 
322 /**
323  * Gets the command buffer to use for this submission from the exe context.
324  */
325 VkCommandBuffer ff_vk_get_exec_buf(AVFilterContext *avctx, FFVkExecContext *e);
326 
327 /**
328  * Adds a generic AVBufferRef as a queue depenency.
329  */
331  AVBufferRef **deps, int nb_deps);
332 
333 /**
334  * Discards all queue dependencies
335  */
337 
338 /**
339  * Adds a frame as a queue dependency. This also manages semaphore signalling.
340  * Must be called before submission.
341  */
343  AVFrame *frame, VkPipelineStageFlagBits in_wait_dst_flag);
344 
345 /**
346  * Submits a command buffer to the queue for execution.
347  * Will block until execution has finished in order to simplify resource
348  * management.
349  */
351 
352 /**
353  * Create a VkBuffer with the specified parameters.
354  */
355 int ff_vk_create_buf(AVFilterContext *avctx, FFVkBuffer *buf, size_t size,
356  VkBufferUsageFlags usage, VkMemoryPropertyFlagBits flags);
357 
358 /**
359  * Maps the buffer to userspace. Set invalidate to 1 if reading the contents
360  * is necessary.
361  */
362 int ff_vk_map_buffers(AVFilterContext *avctx, FFVkBuffer *buf, uint8_t *mem[],
363  int nb_buffers, int invalidate);
364 
365 /**
366  * Unmaps the buffer from userspace. Set flush to 1 to write and sync.
367  */
368 int ff_vk_unmap_buffers(AVFilterContext *avctx, FFVkBuffer *buf, int nb_buffers,
369  int flush);
370 
371 /**
372  * Frees a buffer.
373  */
374 void ff_vk_free_buf(AVFilterContext *avctx, FFVkBuffer *buf);
375 
376 #endif /* AVFILTER_VULKAN_H */
static void flush(AVCodecContext *avctx)
static const int8_t filt[NUMTAPS *2]
Definition: af_earwax.c:39
uint8_t
Main libavfilter public API header.
#define flags(name, subs,...)
Definition: cbs_av1.c:561
static enum AVPixelFormat pix_fmt
static AVFrame * frame
const char * usage
Definition: floatimg_cmp.c:60
const VDPAUPixFmtMap * map
API-specific header for AV_HWDEVICE_TYPE_VULKAN.
enum AVPixelFormat pixfmt
Definition: kmsgrab.c:365
const char * desc
Definition: libsvtav1.c:79
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
const char * name
Definition: qsvenc.c:46
A reference to a data buffer.
Definition: buffer.h:84
Describe the class of an AVClass context structure.
Definition: log.h:67
An instance of a filter.
Definition: avfilter.h:341
This structure describes decoded (raw) audio or video data.
Definition: frame.h:318
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:61
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
VkBuffer buf
Definition: vulkan.h:87
VkMemoryPropertyFlagBits flags
Definition: vulkan.h:89
VkDeviceMemory mem
Definition: vulkan.h:88
VulkanPipeline * bound_pl
Definition: vulkan.h:145
VkSemaphore * sem_wait
Definition: vulkan.h:147
int sem_wait_cnt
Definition: vulkan.h:149
int sem_sig_cnt
Definition: vulkan.h:156
FFVkQueueCtx * queues
Definition: vulkan.h:139
int sem_sig_alloc
Definition: vulkan.h:155
int sem_wait_alloc
Definition: vulkan.h:148
VkCommandBuffer * bufs
Definition: vulkan.h:138
AVBufferRef *** deps
Definition: vulkan.h:141
VkSemaphore * sem_sig
Definition: vulkan.h:154
int * nb_deps
Definition: vulkan.h:142
int * dep_alloc_size
Definition: vulkan.h:143
VkPipelineStageFlagBits * sem_wait_dst
Definition: vulkan.h:151
int sem_wait_dst_alloc
Definition: vulkan.h:152
VkCommandPool pool
Definition: vulkan.h:137
int frame_deps_alloc_size
Definition: vulkan.h:133
AVFrame ** frame_deps
Definition: vulkan.h:131
VkQueue queue
Definition: vulkan.h:123
int buf_deps_alloc_size
Definition: vulkan.h:128
VkFence fence
Definition: vulkan.h:122
int nb_buf_deps
Definition: vulkan.h:127
int nb_frame_deps
Definition: vulkan.h:132
AVBufferRef ** buf_deps
Definition: vulkan.h:126
int local_size[3]
Definition: vulkan.h:69
VkPipelineShaderStageCreateInfo shader
Definition: vulkan.h:70
const char * name
Definition: vulkan.h:67
AVBPrint src
Definition: vulkan.h:68
const char * mem_quali
Definition: vulkan.h:77
VkDescriptorType type
Definition: vulkan.h:75
const char * name
Definition: vulkan.h:74
const char * mem_layout
Definition: vulkan.h:76
const char * buf_content
Definition: vulkan.h:78
const VkSampler * samplers
Definition: vulkan.h:82
VkShaderStageFlags stages
Definition: vulkan.h:81
VulkanPipeline ** pipelines
Definition: vulkan.h:187
AVHWDeviceContext * device
Definition: vulkan.h:164
unsigned int scratch_size
Definition: vulkan.h:191
FFVkExecContext ** exec_ctx
Definition: vulkan.h:183
VkSampler ** samplers
Definition: vulkan.h:179
AVVulkanDeviceContext * hwctx
Definition: vulkan.h:165
AVBufferRef * frames_ref
Definition: vulkan.h:163
enum AVPixelFormat input_format
Definition: vulkan.h:176
AVBufferRef * device_ref
Definition: vulkan.h:162
enum AVPixelFormat output_format
Definition: vulkan.h:175
VkPushConstantRange * push_consts
Definition: vulkan.h:104
VkDescriptorUpdateTemplateCreateInfo * desc_template_info
Definition: vulkan.h:117
VkDescriptorPool desc_pool
Definition: vulkan.h:109
VkPipelineLayout pipeline_layout
Definition: vulkan.h:96
int pool_size_desc_num
Definition: vulkan.h:114
VkDescriptorSetLayout * desc_layout
Definition: vulkan.h:108
VkPipelineBindPoint bind_point
Definition: vulkan.h:93
VkDescriptorPoolSize * pool_size_desc
Definition: vulkan.h:118
VkPipeline pipeline
Definition: vulkan.h:97
int descriptor_sets_num
Definition: vulkan.h:113
VkDescriptorUpdateTemplate * desc_template
Definition: vulkan.h:111
VkDescriptorSet * desc_set
Definition: vulkan.h:110
int desc_layout_num
Definition: vulkan.h:112
int shaders_num
Definition: vulkan.h:101
int push_consts_num
Definition: vulkan.h:105
SPIRVShader ** shaders
Definition: vulkan.h:100
#define src
Definition: vp8dsp.c:255
AVFormatContext * ctx
Definition: movenc.c:48
int size
#define img
static const uint8_t offset[127][2]
Definition: vf_spp.c:107
int ff_vk_unmap_buffers(AVFilterContext *avctx, FFVkBuffer *buf, int nb_buffers, int flush)
Unmaps the buffer from userspace.
Definition: vulkan.c:264
VkSampler * ff_vk_init_sampler(AVFilterContext *avctx, int unnorm_coords, VkFilter filt)
Create a Vulkan sampler, will be auto-freed in ff_vk_filter_uninit()
Definition: vulkan.c:769
int ff_vk_map_buffers(AVFilterContext *avctx, FFVkBuffer *buf, uint8_t *mem[], int nb_buffers, int invalidate)
Maps the buffer to userspace.
Definition: vulkan.c:215
int ff_vk_add_exec_dep(AVFilterContext *avctx, FFVkExecContext *e, AVFrame *frame, VkPipelineStageFlagBits in_wait_dst_flag)
Adds a frame as a queue dependency.
Definition: vulkan.c:464
int ff_vk_filter_config_output_inplace(AVFilterLink *outlink)
Definition: vulkan.c:675
int ff_vk_create_buf(AVFilterContext *avctx, FFVkBuffer *buf, size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlagBits flags)
Create a VkBuffer with the specified parameters.
Definition: vulkan.c:150
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:52
int ff_vk_add_descriptor_set(AVFilterContext *avctx, VulkanPipeline *pl, SPIRVShader *shd, VulkanDescriptorSetBinding *desc, int num, int only_print_to_shader)
Adds a descriptor set to the shader and registers them in the pipeline.
Definition: vulkan.c:1020
const char * ff_vk_shader_rep_fmt(enum AVPixelFormat pixfmt)
Gets the glsl format string for a pixel format.
Definition: vulkan.c:817
void ff_vk_discard_exec_deps(AVFilterContext *avctx, FFVkExecContext *e)
Discards all queue dependencies.
Definition: vulkan.c:400
void ff_vk_set_compute_shader_sizes(AVFilterContext *avctx, SPIRVShader *shd, int local_size[3])
Writes the workgroup size for a shader.
Definition: vulkan.c:909
int ff_vk_add_dep_exec_ctx(AVFilterContext *avctx, FFVkExecContext *e, AVBufferRef **deps, int nb_deps)
Adds a generic AVBufferRef as a queue depenency.
Definition: vulkan.c:561
int ff_vk_add_push_constant(AVFilterContext *avctx, VulkanPipeline *pl, int offset, int size, VkShaderStageFlagBits stage)
Define a push constant for a given stage into a pipeline.
Definition: vulkan.c:318
int ff_vk_init_compute_pipeline(AVFilterContext *avctx, VulkanPipeline *pl)
Initializes a compute pipeline.
Definition: vulkan.c:1281
int ff_vk_filter_config_input(AVFilterLink *inlink)
Definition: vulkan.c:635
void ff_vk_update_push_exec(AVFilterContext *avctx, FFVkExecContext *e, VkShaderStageFlagBits stage, int offset, size_t size, void *src)
Updates push constants.
Definition: vulkan.c:1171
int ff_vk_filter_config_output(AVFilterLink *outlink)
Definition: vulkan.c:705
const VkComponentMapping ff_comp_identity_map
Definition: vulkan.c:44
void ff_vk_filter_uninit(AVFilterContext *avctx)
Definition: vulkan.c:1415
SPIRVShader * ff_vk_init_shader(AVFilterContext *avctx, VulkanPipeline *pl, const char *name, VkShaderStageFlags stage)
Inits a shader for a specific pipeline.
Definition: vulkan.c:888
int ff_vk_compile_shader(AVFilterContext *avctx, SPIRVShader *shd, const char *entrypoint)
Compiles the shader, entrypoint must be set to "main".
Definition: vulkan.c:942
int ff_vk_create_exec_ctx(AVFilterContext *avctx, FFVkExecContext **ctx)
Init an execution context for command recording and queue submission.
Definition: vulkan.c:339
VkCommandBuffer ff_vk_get_exec_buf(AVFilterContext *avctx, FFVkExecContext *e)
Gets the command buffer to use for this submission from the exe context.
Definition: vulkan.c:458
void ff_vk_update_descriptor_set(AVFilterContext *avctx, VulkanPipeline *pl, int set_id)
Updates a descriptor set via the updaters defined.
Definition: vulkan.c:1160
void ff_vk_bind_pipeline_exec(AVFilterContext *avctx, FFVkExecContext *e, VulkanPipeline *pl)
Add a command to bind the completed pipeline and its descriptor sets.
Definition: vulkan.c:1316
int ff_vk_start_exec_recording(AVFilterContext *avctx, FFVkExecContext *e)
Begin recording to the command buffer.
Definition: vulkan.c:417
int ff_vk_mt_is_np_rgb(enum AVPixelFormat pix_fmt)
Returns 1 if the image is any sort of supported RGB.
Definition: vulkan.c:805
int ff_vk_init_pipeline_layout(AVFilterContext *avctx, VulkanPipeline *pl)
Initializes the pipeline layout after all shaders and descriptor sets have been finished.
Definition: vulkan.c:1180
void ff_vk_free_buf(AVFilterContext *avctx, FFVkBuffer *buf)
Frees a buffer.
Definition: vulkan.c:306
int ff_vk_submit_exec_queue(AVFilterContext *avctx, FFVkExecContext *e)
Submits a command buffer to the queue for execution.
Definition: vulkan.c:522
int ff_vk_filter_query_formats(AVFilterContext *avctx)
General lavfi IO functions.
Definition: vulkan.c:592
int ff_vk_filter_init(AVFilterContext *avctx)
Definition: vulkan.c:756
VulkanPipeline * ff_vk_create_pipeline(AVFilterContext *avctx)
Inits a pipeline.
Definition: vulkan.c:1276
int ff_vk_create_imageview(AVFilterContext *avctx, FFVkExecContext *e, VkImageView *v, VkImage img, VkFormat fmt, const VkComponentMapping map)
Create an imageview.
Definition: vulkan.c:836