FFmpeg
hwcontext_vulkan.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) Lynne
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #define VK_NO_PROTOTYPES
22 #define VK_ENABLE_BETA_EXTENSIONS
23 
24 #ifdef _WIN32
25 #include <windows.h> /* Included to prevent conflicts with CreateSemaphore */
26 #include <versionhelpers.h>
27 #include "compat/w32dlfcn.h"
28 #else
29 #include <dlfcn.h>
30 #include <unistd.h>
31 #endif
32 
33 #include "thread.h"
34 
35 #include "config.h"
36 #include "pixdesc.h"
37 #include "avstring.h"
38 #include "imgutils.h"
39 #include "hwcontext.h"
40 #include "hwcontext_internal.h"
41 #include "hwcontext_vulkan.h"
42 #include "mem.h"
43 
44 #include "vulkan.h"
45 #include "vulkan_loader.h"
46 
47 #if CONFIG_VAAPI
48 #include "hwcontext_vaapi.h"
49 #endif
50 
51 #if CONFIG_LIBDRM
52 #if CONFIG_VAAPI
53 #include <va/va_drmcommon.h>
54 #endif
55 #ifdef __linux__
56 #include <sys/sysmacros.h>
57 #endif
58 #include <sys/stat.h>
59 #include <xf86drm.h>
60 #include <drm_fourcc.h>
61 #include "hwcontext_drm.h"
62 #endif
63 
64 #if HAVE_LINUX_DMA_BUF_H
65 #include <sys/ioctl.h>
66 #include <linux/dma-buf.h>
67 #endif
68 
69 #if CONFIG_CUDA
70 #include "hwcontext_cuda_internal.h"
71 #include "cuda_check.h"
72 #define CHECK_CU(x) FF_CUDA_CHECK_DL(cuda_cu, cu, x)
73 #endif
74 
75 typedef struct VulkanDeviceFeatures {
76  VkPhysicalDeviceFeatures2 device;
77 
78  VkPhysicalDeviceVulkan11Features vulkan_1_1;
79  VkPhysicalDeviceVulkan12Features vulkan_1_2;
80  VkPhysicalDeviceVulkan13Features vulkan_1_3;
81  VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore;
82  VkPhysicalDeviceShaderSubgroupRotateFeaturesKHR subgroup_rotate;
83  VkPhysicalDeviceHostImageCopyFeaturesEXT host_image_copy;
84  VkPhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR explicit_mem_layout;
85 
86 #ifdef VK_EXT_shader_long_vector
87  VkPhysicalDeviceShaderLongVectorFeaturesEXT long_vector;
88 #endif
89 
90 #ifdef VK_EXT_shader_replicated_composites
91  VkPhysicalDeviceShaderReplicatedCompositesFeaturesEXT replicated_composites;
92 #endif
93 
94 #ifdef VK_EXT_zero_initialize_device_memory
95  VkPhysicalDeviceZeroInitializeDeviceMemoryFeaturesEXT zero_initialize;
96 #endif
97 
98 #ifdef VK_KHR_shader_expect_assume
99  VkPhysicalDeviceShaderExpectAssumeFeaturesKHR expect_assume;
100 #endif
101 
102  VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1;
103 #ifdef VK_KHR_video_maintenance2
104  VkPhysicalDeviceVideoMaintenance2FeaturesKHR video_maintenance_2;
105 #endif
106 #ifdef VK_KHR_video_decode_vp9
107  VkPhysicalDeviceVideoDecodeVP9FeaturesKHR vp9_decode;
108 #endif
109 #ifdef VK_KHR_video_encode_av1
110  VkPhysicalDeviceVideoEncodeAV1FeaturesKHR av1_encode;
111 #endif
112 
113  VkPhysicalDeviceShaderObjectFeaturesEXT shader_object;
114  VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix;
115  VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float;
116 
117 #ifdef VK_KHR_shader_relaxed_extended_instruction
118  VkPhysicalDeviceShaderRelaxedExtendedInstructionFeaturesKHR relaxed_extended_instruction;
119 #endif
120 } VulkanDeviceFeatures;
121 
122 typedef struct VulkanDevicePriv {
123  /**
124  * The public AVVulkanDeviceContext. See hwcontext_vulkan.h for it.
125  */
126  AVVulkanDeviceContext p;
127 
128  /* Vulkan library and loader functions */
129  void *libvulkan;
130 
131  FFVulkanContext vkctx;
132  AVVulkanDeviceQueueFamily *compute_qf;
133  AVVulkanDeviceQueueFamily *transfer_qf;
134 
135  /* Properties */
136  VkPhysicalDeviceProperties2 props;
137  VkPhysicalDeviceMemoryProperties mprops;
138  VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops;
139  VkPhysicalDeviceDriverProperties dprops;
140 
141  /* Opaque FD external semaphore properties */
142  VkExternalSemaphoreProperties ext_sem_props_opaque;
143 
144  /* Enabled features */
145  VulkanDeviceFeatures feats;
146 
147  /* Queues */
148  pthread_mutex_t **qf_mutex;
149  uint32_t nb_tot_qfs;
150  uint32_t img_qfs[64];
151  uint32_t nb_img_qfs;
152 
153  /* Debug callback */
154  VkDebugUtilsMessengerEXT debug_ctx;
155 
156  /* Settings */
157  int use_linear_images;
158 
159  /* Option to allocate all image planes in a single allocation */
160  int contiguous_planes;
161 
162  /* Disable multiplane images */
163  int disable_multiplane;
164 
165  /* Prefer memcpy over dynamic host pointer imports */
166  int avoid_host_import;
167 
168  /* Maximum queues */
169  int limit_queues;
170 } VulkanDevicePriv;
171 
172 typedef struct VulkanFramesPriv {
173  /**
174  * The public AVVulkanFramesContext. See hwcontext_vulkan.h for it.
175  */
176  AVVulkanFramesContext p;
177 
178  /* Image conversions */
179  FFVkExecPool compute_exec;
180 
181  /* Image transfers */
182  FFVkExecPool upload_exec;
183  FFVkExecPool download_exec;
184 
185  /* Temporary buffer pools */
186  AVBufferPool *tmp;
187 
188  /* Modifier info list to free at uninit */
189  VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
190 
191  /* Properties for DRM modifier for each plane in the image */
192  VkDrmFormatModifierPropertiesEXT drm_format_modifier_properties[5];
193 } VulkanFramesPriv;
194 
195 typedef struct AVVkFrameInternal {
196  pthread_mutex_t update_mutex;
197 
198 #if CONFIG_CUDA
199  /* Importing external memory into cuda is really expensive so we keep the
200  * memory imported all the time */
201  AVBufferRef *cuda_fc_ref; /* Need to keep it around for uninit */
202  CUexternalMemory ext_mem[AV_NUM_DATA_POINTERS];
203  CUmipmappedArray cu_mma[AV_NUM_DATA_POINTERS];
204  CUarray cu_array[AV_NUM_DATA_POINTERS];
205  CUexternalSemaphore cu_sem[AV_NUM_DATA_POINTERS];
206 #ifdef _WIN32
207  HANDLE ext_mem_handle[AV_NUM_DATA_POINTERS];
208  HANDLE ext_sem_handle[AV_NUM_DATA_POINTERS];
209 #endif
210 #endif
211 } AVVkFrameInternal;
212 
213 /* Initialize all structs in VulkanDeviceFeatures */
214 static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
215 {
216  VulkanDevicePriv *p = ctx->hwctx;
217  FFVulkanContext *s = &p->vkctx;
218 
219  feats->device = (VkPhysicalDeviceFeatures2) {
220  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
221  };
222 
223  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_1, FF_VK_EXT_NO_FLAG,
224  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES);
225  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_2, FF_VK_EXT_NO_FLAG,
226  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES);
227  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_3, FF_VK_EXT_NO_FLAG,
228  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES);
229 
230  FF_VK_STRUCT_EXT(s, &feats->device, &feats->timeline_semaphore, FF_VK_EXT_PORTABILITY_SUBSET,
231  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES);
232  FF_VK_STRUCT_EXT(s, &feats->device, &feats->subgroup_rotate, FF_VK_EXT_SUBGROUP_ROTATE,
233  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SUBGROUP_ROTATE_FEATURES_KHR);
234  FF_VK_STRUCT_EXT(s, &feats->device, &feats->host_image_copy, FF_VK_EXT_HOST_IMAGE_COPY,
235  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_IMAGE_COPY_FEATURES_EXT);
236 
237 #ifdef VK_EXT_shader_long_vector
238  FF_VK_STRUCT_EXT(s, &feats->device, &feats->long_vector, FF_VK_EXT_LONG_VECTOR,
239  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_LONG_VECTOR_FEATURES_EXT);
240 #endif
241 
242 #ifdef VK_EXT_shader_replicated_composites
243  FF_VK_STRUCT_EXT(s, &feats->device, &feats->replicated_composites, FF_VK_EXT_REPLICATED_COMPOSITES,
244  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_REPLICATED_COMPOSITES_FEATURES_EXT);
245 #endif
246 
247 #ifdef VK_EXT_zero_initialize_device_memory
248  FF_VK_STRUCT_EXT(s, &feats->device, &feats->zero_initialize, FF_VK_EXT_ZERO_INITIALIZE,
249  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ZERO_INITIALIZE_DEVICE_MEMORY_FEATURES_EXT);
250 #endif
251 
252 #ifdef VK_KHR_shader_expect_assume
253  FF_VK_STRUCT_EXT(s, &feats->device, &feats->expect_assume, FF_VK_EXT_EXPECT_ASSUME,
254  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_EXPECT_ASSUME_FEATURES_KHR);
255 #endif
256 
257  FF_VK_STRUCT_EXT(s, &feats->device, &feats->video_maintenance_1, FF_VK_EXT_VIDEO_MAINTENANCE_1,
258  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_1_FEATURES_KHR);
259 #ifdef VK_KHR_video_maintenance2
260  FF_VK_STRUCT_EXT(s, &feats->device, &feats->video_maintenance_2, FF_VK_EXT_VIDEO_MAINTENANCE_2,
261  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_2_FEATURES_KHR);
262 #endif
263 #ifdef VK_KHR_video_decode_vp9
264  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vp9_decode, FF_VK_EXT_VIDEO_DECODE_VP9,
265  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_DECODE_VP9_FEATURES_KHR);
266 #endif
267 #ifdef VK_KHR_video_encode_av1
268  FF_VK_STRUCT_EXT(s, &feats->device, &feats->av1_encode, FF_VK_EXT_VIDEO_ENCODE_AV1,
269  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_ENCODE_AV1_FEATURES_KHR);
270 #endif
271 
272  FF_VK_STRUCT_EXT(s, &feats->device, &feats->shader_object, FF_VK_EXT_SHADER_OBJECT,
273  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OBJECT_FEATURES_EXT);
274  FF_VK_STRUCT_EXT(s, &feats->device, &feats->cooperative_matrix, FF_VK_EXT_COOP_MATRIX,
275  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR);
276  FF_VK_STRUCT_EXT(s, &feats->device, &feats->atomic_float, FF_VK_EXT_ATOMIC_FLOAT,
277  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT);
278  FF_VK_STRUCT_EXT(s, &feats->device, &feats->explicit_mem_layout, FF_VK_EXT_EXPLICIT_MEM_LAYOUT,
279  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_FEATURES_KHR);
280 
281 #ifdef VK_KHR_shader_relaxed_extended_instruction
282  FF_VK_STRUCT_EXT(s, &feats->device, &feats->relaxed_extended_instruction, FF_VK_EXT_RELAXED_EXTENDED_INSTR,
283  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_RELAXED_EXTENDED_INSTRUCTION_FEATURES_KHR);
284 #endif
285 }
286 
287 /* Copy all needed device features */
288 static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
289 {
290 #define COPY_VAL(VAL) \
291  do { \
292  dst->VAL = src->VAL; \
293  } while (0) \
294 
295  COPY_VAL(device.features.shaderImageGatherExtended);
296  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
297  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
298  COPY_VAL(device.features.fragmentStoresAndAtomics);
299  COPY_VAL(device.features.vertexPipelineStoresAndAtomics);
300  COPY_VAL(device.features.shaderInt64);
301  COPY_VAL(device.features.shaderInt16);
302  COPY_VAL(device.features.shaderFloat64);
303  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
304  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
305 
306  COPY_VAL(vulkan_1_1.samplerYcbcrConversion);
307  COPY_VAL(vulkan_1_1.storagePushConstant16);
308  COPY_VAL(vulkan_1_1.storageBuffer16BitAccess);
309  COPY_VAL(vulkan_1_1.uniformAndStorageBuffer16BitAccess);
310 
311  COPY_VAL(vulkan_1_2.timelineSemaphore);
312  COPY_VAL(vulkan_1_2.scalarBlockLayout);
313  COPY_VAL(vulkan_1_2.bufferDeviceAddress);
314  COPY_VAL(vulkan_1_2.hostQueryReset);
315  COPY_VAL(vulkan_1_2.storagePushConstant8);
316  COPY_VAL(vulkan_1_2.shaderInt8);
317  COPY_VAL(vulkan_1_2.storageBuffer8BitAccess);
318  COPY_VAL(vulkan_1_2.uniformAndStorageBuffer8BitAccess);
319  COPY_VAL(vulkan_1_2.shaderFloat16);
320  COPY_VAL(vulkan_1_2.shaderBufferInt64Atomics);
321  COPY_VAL(vulkan_1_2.shaderSharedInt64Atomics);
322  COPY_VAL(vulkan_1_2.vulkanMemoryModel);
323  COPY_VAL(vulkan_1_2.vulkanMemoryModelDeviceScope);
324  COPY_VAL(vulkan_1_2.vulkanMemoryModelAvailabilityVisibilityChains);
325  COPY_VAL(vulkan_1_2.uniformBufferStandardLayout);
326  COPY_VAL(vulkan_1_2.runtimeDescriptorArray);
327  COPY_VAL(vulkan_1_2.shaderSubgroupExtendedTypes);
328  COPY_VAL(vulkan_1_2.shaderUniformBufferArrayNonUniformIndexing);
329  COPY_VAL(vulkan_1_2.shaderSampledImageArrayNonUniformIndexing);
330  COPY_VAL(vulkan_1_2.shaderStorageBufferArrayNonUniformIndexing);
331  COPY_VAL(vulkan_1_2.shaderStorageImageArrayNonUniformIndexing);
332 
333  COPY_VAL(vulkan_1_3.dynamicRendering);
334  COPY_VAL(vulkan_1_3.maintenance4);
335  COPY_VAL(vulkan_1_3.synchronization2);
336  COPY_VAL(vulkan_1_3.computeFullSubgroups);
337  COPY_VAL(vulkan_1_3.subgroupSizeControl);
338  COPY_VAL(vulkan_1_3.shaderZeroInitializeWorkgroupMemory);
339  COPY_VAL(vulkan_1_3.dynamicRendering);
340 
341  COPY_VAL(timeline_semaphore.timelineSemaphore);
342  COPY_VAL(subgroup_rotate.shaderSubgroupRotate);
343  COPY_VAL(host_image_copy.hostImageCopy);
344 
345 #ifdef VK_EXT_shader_long_vector
346  COPY_VAL(long_vector.longVector);
347 #endif
348 
349 #ifdef VK_EXT_shader_replicated_composites
350  COPY_VAL(replicated_composites.shaderReplicatedComposites);
351 #endif
352 
353 #ifdef VK_EXT_zero_initialize_device_memory
354  COPY_VAL(zero_initialize.zeroInitializeDeviceMemory);
355 #endif
356 
357  COPY_VAL(video_maintenance_1.videoMaintenance1);
358 #ifdef VK_KHR_video_maintenance2
359  COPY_VAL(video_maintenance_2.videoMaintenance2);
360 #endif
361 
362 #ifdef VK_KHR_video_decode_vp9
363  COPY_VAL(vp9_decode.videoDecodeVP9);
364 #endif
365 
366 #ifdef VK_KHR_video_encode_av1
367  COPY_VAL(av1_encode.videoEncodeAV1);
368 #endif
369 
370  COPY_VAL(shader_object.shaderObject);
371 
372  COPY_VAL(cooperative_matrix.cooperativeMatrix);
373 
374  COPY_VAL(atomic_float.shaderBufferFloat32Atomics);
375  COPY_VAL(atomic_float.shaderBufferFloat32AtomicAdd);
376 
377  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayout);
378  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayoutScalarBlockLayout);
379  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayout8BitAccess);
380  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayout16BitAccess);
381 
382 #ifdef VK_KHR_shader_relaxed_extended_instruction
383  COPY_VAL(relaxed_extended_instruction.shaderRelaxedExtendedInstruction);
384 #endif
385 
386 #ifdef VK_KHR_shader_expect_assume
387  COPY_VAL(expect_assume.shaderExpectAssume);
388 #endif
389 
390 #undef COPY_VAL
391 }
392 
393 #define ASPECT_2PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT)
394 #define ASPECT_3PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT | VK_IMAGE_ASPECT_PLANE_2_BIT)
395 
396 static const struct FFVkFormatEntry {
397  VkFormat vkf;
398  enum AVPixelFormat pixfmt;
399  VkImageAspectFlags aspect;
400  int vk_planes;
401  int nb_images;
402  int nb_images_fallback;
403  const VkFormat fallback[5];
404 } vk_formats_list[] = {
405  /* Gray formats */
406  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GRAY8, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8_UNORM } },
407  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
408  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY12, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
409  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY14, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
410  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
411  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GRAY32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_UINT } },
412  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GRAYF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_SFLOAT } },
413 
414  /* RGB formats */
415  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGRA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
416  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGBA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
417  { VK_FORMAT_R8G8B8_UNORM, AV_PIX_FMT_RGB24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8_UNORM } },
418  { VK_FORMAT_B8G8R8_UNORM, AV_PIX_FMT_BGR24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8_UNORM } },
419  { VK_FORMAT_R16G16B16_UNORM, AV_PIX_FMT_RGB48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16_UNORM } },
420  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_RGBA64, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
421  { VK_FORMAT_R5G6B5_UNORM_PACK16, AV_PIX_FMT_RGB565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R5G6B5_UNORM_PACK16 } },
422  { VK_FORMAT_B5G6R5_UNORM_PACK16, AV_PIX_FMT_BGR565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B5G6R5_UNORM_PACK16 } },
423  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGR0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
424  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGB0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
425  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_X2RGB10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2R10G10B10_UNORM_PACK32 } },
426  { VK_FORMAT_A2B10G10R10_UNORM_PACK32, AV_PIX_FMT_X2BGR10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2B10G10R10_UNORM_PACK32 } },
427  { VK_FORMAT_R32G32B32_SFLOAT, AV_PIX_FMT_RGBF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_SFLOAT } },
428  { VK_FORMAT_R32G32B32A32_SFLOAT, AV_PIX_FMT_RGBAF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_SFLOAT } },
429  { VK_FORMAT_R32G32B32_UINT, AV_PIX_FMT_RGB96, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_UINT } },
430  { VK_FORMAT_R32G32B32A32_UINT, AV_PIX_FMT_RGBA128, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_UINT } },
431 
432  /* Planar RGB */
433  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRP, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
434  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP10, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
435  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP12, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
436  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP14, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
437  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP16, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
438  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRPF32, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
439 
440  /* Planar RGB + Alpha */
441  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRAP, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
442  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
443  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
444  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP14, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
445  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
446  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GBRAP32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT } },
447  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRAPF32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
448 
449  /* Bayer */
450  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_BAYER_RGGB16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
451 
452  /* Two-plane 420 YUV at 8, 10, 12 and 16 bits */
453  { VK_FORMAT_G8_B8R8_2PLANE_420_UNORM, AV_PIX_FMT_NV12, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
454  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P010, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
455  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P012, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
456  { VK_FORMAT_G16_B16R16_2PLANE_420_UNORM, AV_PIX_FMT_P016, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
457 
458  /* Two-plane 422 YUV at 8, 10 and 16 bits */
459  { VK_FORMAT_G8_B8R8_2PLANE_422_UNORM, AV_PIX_FMT_NV16, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
460  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P210, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
461  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P212, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
462  { VK_FORMAT_G16_B16R16_2PLANE_422_UNORM, AV_PIX_FMT_P216, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
463 
464  /* Two-plane 444 YUV at 8, 10 and 16 bits */
465  { VK_FORMAT_G8_B8R8_2PLANE_444_UNORM, AV_PIX_FMT_NV24, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
466  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P410, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
467  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P412, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
468  { VK_FORMAT_G16_B16R16_2PLANE_444_UNORM, AV_PIX_FMT_P416, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
469 
470  /* Three-plane 420, 422, 444 at 8, 10, 12 and 16 bits */
471  { VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
472  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
473  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
474  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
475  { VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
476  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
477  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
478  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
479  { VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
480  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
481  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
482  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
483 
484  /* Single plane 422 at 8, 10, 12 and 16 bits */
485  { VK_FORMAT_G8B8G8R8_422_UNORM, AV_PIX_FMT_YUYV422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
486  { VK_FORMAT_B8G8R8G8_422_UNORM, AV_PIX_FMT_UYVY422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
487  { VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16, AV_PIX_FMT_Y210, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
488  { VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16, AV_PIX_FMT_Y212, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
489  { VK_FORMAT_G16B16G16R16_422_UNORM, AV_PIX_FMT_Y216, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
490 
491  /* Planar YUVA 420 at 8, 10 and 16 bits */
492  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA420P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
493  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA420P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
494  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA420P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
495 
496  /* Planar YUVA 422 at 8, 10, 12 and 16 bits */
497  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA422P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
498  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
499  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
500  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
501 
502  /* Planar YUVA 444 at 8, 10, 12 and 16 bits */
503  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA444P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
504  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
505  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
506  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
507 
508  /* Single plane 444 at 8, 10, 12 and 16 bits */
509  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_UYVA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
510  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_XV30, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
511  { VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16, AV_PIX_FMT_XV36, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
512  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_XV48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
513 };
514 static const int nb_vk_formats_list = FF_ARRAY_ELEMS(vk_formats_list);
515 
516 const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p)
517 {
518  for (int i = 0; i < nb_vk_formats_list; i++)
519  if (vk_formats_list[i].pixfmt == p)
520  return vk_formats_list[i].fallback;
521  return NULL;
522 }
523 
524 static const struct FFVkFormatEntry *vk_find_format_entry(enum AVPixelFormat p)
525 {
526  for (int i = 0; i < nb_vk_formats_list; i++)
527  if (vk_formats_list[i].pixfmt == p)
528  return &vk_formats_list[i];
529  return NULL;
530 }
531 
532 static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
533  VkImageTiling tiling,
534  VkFormat fmts[AV_NUM_DATA_POINTERS], /* Output format list */
535  int *nb_images, /* Output number of images */
536  VkImageAspectFlags *aspect, /* Output aspect */
537  VkImageUsageFlags *supported_usage, /* Output supported usage */
538  int disable_multiplane, int need_storage)
539 {
540  VulkanDevicePriv *priv = dev_ctx->hwctx;
541  AVVulkanDeviceContext *hwctx = &priv->p;
542  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
543 
544  const VkFormatFeatureFlagBits2 basic_flags = VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT |
545  VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
546  VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
547 
548  for (int i = 0; i < nb_vk_formats_list; i++) {
549  if (vk_formats_list[i].pixfmt == p) {
550  VkFormatProperties3 fprops = {
551  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3,
552  };
553  VkFormatProperties2 prop = {
554  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
555  .pNext = &fprops,
556  };
557  VkFormatFeatureFlagBits2 feats_primary, feats_secondary;
558  int basics_primary = 0, basics_secondary = 0;
559  int storage_primary = 0, storage_secondary = 0;
560 
561  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
562  vk_formats_list[i].vkf,
563  &prop);
564 
565  feats_primary = tiling == VK_IMAGE_TILING_LINEAR ?
566  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
567  basics_primary = (feats_primary & basic_flags) == basic_flags;
568  storage_primary = !!(feats_primary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
569 
570  if (vk_formats_list[i].vkf != vk_formats_list[i].fallback[0]) {
571  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
572  vk_formats_list[i].fallback[0],
573  &prop);
574  feats_secondary = tiling == VK_IMAGE_TILING_LINEAR ?
575  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
576  basics_secondary = (feats_secondary & basic_flags) == basic_flags;
577  storage_secondary = !!(feats_secondary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
578  } else {
579  basics_secondary = basics_primary;
580  storage_secondary = storage_primary;
581  }
582 
583  if (basics_primary &&
584  !(disable_multiplane && vk_formats_list[i].vk_planes > 1) &&
585  (!need_storage || (need_storage && (storage_primary | storage_secondary)))) {
586  if (fmts) {
587  if (vk_formats_list[i].nb_images > 1) {
588  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
589  fmts[j] = vk_formats_list[i].fallback[j];
590  } else {
591  fmts[0] = vk_formats_list[i].vkf;
592  }
593  }
594  if (nb_images)
595  *nb_images = 1;
596  if (aspect)
597  *aspect = vk_formats_list[i].aspect;
598  if (supported_usage)
599  *supported_usage = ff_vk_map_feats_to_usage(feats_primary) |
600  ((need_storage && (storage_primary | storage_secondary)) ?
601  VK_IMAGE_USAGE_STORAGE_BIT : 0);
602  return 0;
603  } else if (basics_secondary &&
604  (!need_storage || (need_storage && storage_secondary))) {
605  if (fmts) {
606  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
607  fmts[j] = vk_formats_list[i].fallback[j];
608  }
609  if (nb_images)
610  *nb_images = vk_formats_list[i].nb_images_fallback;
611  if (aspect)
612  *aspect = vk_formats_list[i].aspect;
613  if (supported_usage)
614  *supported_usage = ff_vk_map_feats_to_usage(feats_secondary);
615  return 0;
616  } else {
617  return AVERROR(ENOTSUP);
618  }
619  }
620  }
621 
622  return AVERROR(EINVAL);
623 }
624 
625 #if CONFIG_VULKAN_STATIC
626 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,
627  const char *pName);
628 #endif
629 
630 static int load_libvulkan(AVHWDeviceContext *ctx)
631 {
632  VulkanDevicePriv *p = ctx->hwctx;
633  AVVulkanDeviceContext *hwctx = &p->p;
634 
635 #if CONFIG_VULKAN_STATIC
636  hwctx->get_proc_addr = vkGetInstanceProcAddr;
637 #else
638  static const char *lib_names[] = {
639 #if defined(_WIN32)
640  "vulkan-1.dll",
641 #elif defined(__APPLE__)
642  "libvulkan.dylib",
643  "libvulkan.1.dylib",
644  "libMoltenVK.dylib",
645 #else
646  "libvulkan.so.1",
647  "libvulkan.so",
648 #endif
649  };
650 
651  for (int i = 0; i < FF_ARRAY_ELEMS(lib_names); i++) {
652  p->libvulkan = dlopen(lib_names[i], RTLD_NOW | RTLD_LOCAL);
653  if (p->libvulkan)
654  break;
655  }
656 
657  if (!p->libvulkan) {
658  av_log(ctx, AV_LOG_ERROR, "Unable to open the libvulkan library!\n");
659  return AVERROR_UNKNOWN;
660  }
661 
662  hwctx->get_proc_addr = (PFN_vkGetInstanceProcAddr)dlsym(p->libvulkan, "vkGetInstanceProcAddr");
663 #endif /* CONFIG_VULKAN_STATIC */
664 
665  return 0;
666 }
667 
668 typedef struct VulkanOptExtension {
669  const char *name;
670  FFVulkanExtensions flag;
671 } VulkanOptExtension;
672 
673 static const VulkanOptExtension optional_instance_exts[] = {
674 #ifdef __APPLE__
675  { VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME, FF_VK_EXT_NO_FLAG },
676 #endif
677  { 0 },
678 };
679 
680 static const VulkanOptExtension optional_device_exts[] = {
681  /* Misc or required by other extensions */
682  { VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME, FF_VK_EXT_PORTABILITY_SUBSET },
683  { VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, FF_VK_EXT_PUSH_DESCRIPTOR },
684  { VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME, FF_VK_EXT_DEVICE_DRM },
685  { VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, FF_VK_EXT_ATOMIC_FLOAT },
686  { VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME, FF_VK_EXT_COOP_MATRIX },
687  { VK_EXT_SHADER_OBJECT_EXTENSION_NAME, FF_VK_EXT_SHADER_OBJECT },
688  { VK_KHR_SHADER_SUBGROUP_ROTATE_EXTENSION_NAME, FF_VK_EXT_SUBGROUP_ROTATE },
689  { VK_EXT_HOST_IMAGE_COPY_EXTENSION_NAME, FF_VK_EXT_HOST_IMAGE_COPY },
690  { VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME, FF_VK_EXT_EXPLICIT_MEM_LAYOUT },
691 #ifdef VK_KHR_shader_relaxed_extended_instruction
692  { VK_KHR_SHADER_RELAXED_EXTENDED_INSTRUCTION_EXTENSION_NAME, FF_VK_EXT_RELAXED_EXTENDED_INSTR },
693 #endif
694 #ifdef VK_EXT_shader_long_vector
695  { VK_EXT_SHADER_LONG_VECTOR_EXTENSION_NAME, FF_VK_EXT_LONG_VECTOR },
696 #endif
697 #ifdef VK_EXT_shader_replicated_composites
698  { VK_EXT_SHADER_REPLICATED_COMPOSITES_EXTENSION_NAME, FF_VK_EXT_REPLICATED_COMPOSITES },
699 #endif
700 #ifdef VK_EXT_zero_initialize_device_memory
701  { VK_EXT_ZERO_INITIALIZE_DEVICE_MEMORY_EXTENSION_NAME, FF_VK_EXT_ZERO_INITIALIZE },
702 #endif
703 #ifdef VK_KHR_shader_expect_assume
704  { VK_KHR_SHADER_EXPECT_ASSUME_EXTENSION_NAME, FF_VK_EXT_EXPECT_ASSUME },
705 #endif
706  { VK_KHR_VIDEO_MAINTENANCE_1_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_1 },
707 #ifdef VK_KHR_video_maintenance2
708  { VK_KHR_VIDEO_MAINTENANCE_2_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_2 },
709 #endif
710 
711  /* Imports/exports */
712  { VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_MEMORY },
713  { VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_DMABUF_MEMORY },
714  { VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, FF_VK_EXT_DRM_MODIFIER_FLAGS },
715  { VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_SEM },
716  { VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_HOST_MEMORY },
717 #ifdef _WIN32
718  { VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_MEMORY },
719  { VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_SEM },
720 #endif
721 
722  /* Video encoding/decoding */
723  { VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_QUEUE },
724  { VK_KHR_VIDEO_ENCODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_QUEUE },
725  { VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_QUEUE },
726  { VK_KHR_VIDEO_ENCODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H264 },
727  { VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H264 },
728  { VK_KHR_VIDEO_ENCODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H265 },
729  { VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H265 },
730 #ifdef VK_KHR_video_decode_vp9
731  { VK_KHR_VIDEO_DECODE_VP9_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_VP9 },
732 #endif
733 #ifdef VK_KHR_video_encode_av1
734  { VK_KHR_VIDEO_ENCODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_AV1 },
735 #endif
736  { VK_KHR_VIDEO_DECODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_AV1 },
737 };
738 
739 const char **av_vk_get_optional_instance_extensions(int *count)
740 {
741  const char **exts = av_malloc_array(sizeof(*exts),
742  FF_ARRAY_ELEMS(optional_instance_exts) - 1);
743  if (!exts)
744  return NULL;
745 
746  for (int i = 0; i < FF_ARRAY_ELEMS(optional_instance_exts) - 1; i++)
747  exts[i] = optional_instance_exts[i].name;
748 
749  *count = FF_ARRAY_ELEMS(optional_instance_exts) - 1;
750  return exts;
751 }
752 
753 const char **av_vk_get_optional_device_extensions(int *count)
754 {
755  const char **exts = av_malloc_array(sizeof(*exts),
756  FF_ARRAY_ELEMS(optional_device_exts));
757  if (!exts)
758  return NULL;
759 
760  for (int i = 0; i < FF_ARRAY_ELEMS(optional_device_exts); i++)
761  exts[i] = optional_device_exts[i].name;
762 
763  *count = FF_ARRAY_ELEMS(optional_device_exts);
764  return exts;
765 }
766 
767 static VKAPI_ATTR
768 VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
769  VkDebugUtilsMessageTypeFlagsEXT messageType,
770  const VkDebugUtilsMessengerCallbackDataEXT *data,
771  void *priv)
772 {
773  int l;
774  AVHWDeviceContext *ctx = priv;
775 
776  /* Ignore false positives */
777  switch (data->messageIdNumber) {
778  case 0x086974c1: /* BestPractices-vkCreateCommandPool-command-buffer-reset */
779  case 0xfd92477a: /* BestPractices-vkAllocateMemory-small-allocation */
780  return VK_FALSE;
781  default:
782  break;
783  }
784 
785  switch (severity) {
786  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT: l = AV_LOG_VERBOSE; break;
787  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT: l = AV_LOG_INFO; break;
788  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT: l = AV_LOG_WARNING; break;
789  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT: l = AV_LOG_ERROR; break;
790  default: l = AV_LOG_DEBUG; break;
791  }
792 
793  av_log(ctx, l, "%s\n", data->pMessage);
794  for (int i = 0; i < data->cmdBufLabelCount; i++)
795  av_log(ctx, l, "\t%i: %s\n", i, data->pCmdBufLabels[i].pLabelName);
796 
797  return VK_FALSE;
798 }
799 
800 #define ADD_VAL_TO_LIST(list, count, val) \
801  do { \
802  list = av_realloc_array(list, ++count, sizeof(*list)); \
803  if (!list) { \
804  err = AVERROR(ENOMEM); \
805  goto fail; \
806  } \
807  list[count - 1] = av_strdup(val); \
808  if (!list[count - 1]) { \
809  err = AVERROR(ENOMEM); \
810  goto fail; \
811  } \
812  } while(0)
813 
814 #define RELEASE_PROPS(props, count) \
815  if (props) { \
816  for (int i = 0; i < count; i++) \
817  av_free((void *)((props)[i])); \
818  av_free((void *)props); \
819  }
820 
821 static int vulkan_device_has_rebar(AVHWDeviceContext *ctx)
822 {
823  VulkanDevicePriv *p = ctx->hwctx;
824  VkDeviceSize max_vram = 0, max_visible_vram = 0;
825 
826  /* Get device memory properties */
827  av_assert0(p->mprops.memoryTypeCount);
828  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
829  const VkMemoryType type = p->mprops.memoryTypes[i];
830  const VkMemoryHeap heap = p->mprops.memoryHeaps[type.heapIndex];
831  if (!(type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
832  continue;
833  max_vram = FFMAX(max_vram, heap.size);
834  if (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
835  max_visible_vram = FFMAX(max_visible_vram, heap.size);
836  }
837 
838  return max_vram - max_visible_vram < 1024; /* 1 kB tolerance */
839 }
840 
841 enum FFVulkanDebugMode {
842  FF_VULKAN_DEBUG_NONE = 0,
843  /* Standard GPU-assisted validation */
844  FF_VULKAN_DEBUG_VALIDATE = 1,
845  /* Passes printfs in shaders to the debug callback */
846  FF_VULKAN_DEBUG_PRINTF = 2,
847  /* Enables extra printouts */
848  FF_VULKAN_DEBUG_PRACTICES = 3,
849 
850  FF_VULKAN_DEBUG_NB,
851 };
852 
853 static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts,
854  const char * const **dst, uint32_t *num,
855  enum FFVulkanDebugMode debug_mode)
856 {
857  const char *tstr;
858  const char **extension_names = NULL;
859  VulkanDevicePriv *p = ctx->hwctx;
860  AVVulkanDeviceContext *hwctx = &p->p;
861  FFVulkanFunctions *vk = &p->vkctx.vkfn;
862  int err = 0, found, extensions_found = 0;
863 
864  const char *mod;
865  int optional_exts_num;
866  uint32_t sup_ext_count;
867  char *user_exts_str = NULL;
868  AVDictionaryEntry *user_exts;
869  VkExtensionProperties *sup_ext;
870  const VulkanOptExtension *optional_exts;
871 
872  if (!dev) {
873  mod = "instance";
874  optional_exts = optional_instance_exts;
875  optional_exts_num = FF_ARRAY_ELEMS(optional_instance_exts) - 1;
876  user_exts = av_dict_get(opts, "instance_extensions", NULL, 0);
877  if (user_exts) {
878  user_exts_str = av_strdup(user_exts->value);
879  if (!user_exts_str) {
880  err = AVERROR(ENOMEM);
881  goto fail;
882  }
883  }
884  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, NULL);
885  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
886  if (!sup_ext)
887  return AVERROR(ENOMEM);
888  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, sup_ext);
889  } else {
890  mod = "device";
891  optional_exts = optional_device_exts;
892  optional_exts_num = FF_ARRAY_ELEMS(optional_device_exts);
893  user_exts = av_dict_get(opts, "device_extensions", NULL, 0);
894  if (user_exts) {
895  user_exts_str = av_strdup(user_exts->value);
896  if (!user_exts_str) {
897  err = AVERROR(ENOMEM);
898  goto fail;
899  }
900  }
901  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
902  &sup_ext_count, NULL);
903  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
904  if (!sup_ext)
905  return AVERROR(ENOMEM);
906  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
907  &sup_ext_count, sup_ext);
908  }
909 
910  for (int i = 0; i < optional_exts_num; i++) {
911  tstr = optional_exts[i].name;
912  found = 0;
913 
914  /* Check if the device has ReBAR for host image copies */
915  if (!strcmp(tstr, VK_EXT_HOST_IMAGE_COPY_EXTENSION_NAME) &&
916  !vulkan_device_has_rebar(ctx))
917  continue;
918 
919  if (dev &&
920  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
921  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
922  (debug_mode == FF_VULKAN_DEBUG_PRACTICES)) &&
923  (!strcmp(tstr, VK_EXT_SHADER_OBJECT_EXTENSION_NAME))) {
924  continue;
925  }
926 
927  for (int j = 0; j < sup_ext_count; j++) {
928  if (!strcmp(tstr, sup_ext[j].extensionName)) {
929  found = 1;
930  break;
931  }
932  }
933  if (!found)
934  continue;
935 
936  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
937  p->vkctx.extensions |= optional_exts[i].flag;
938  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
939  }
940 
941  if (!dev &&
942  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
943  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
944  (debug_mode == FF_VULKAN_DEBUG_PRACTICES))) {
945  tstr = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
946  found = 0;
947  for (int j = 0; j < sup_ext_count; j++) {
948  if (!strcmp(tstr, sup_ext[j].extensionName)) {
949  found = 1;
950  break;
951  }
952  }
953  if (found) {
954  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
955  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
956  } else {
957  av_log(ctx, AV_LOG_ERROR, "Debug extension \"%s\" not found!\n",
958  tstr);
959  err = AVERROR(EINVAL);
960  goto fail;
961  }
962  }
963 
964 #ifdef VK_KHR_shader_relaxed_extended_instruction
965  if ((debug_mode == FF_VULKAN_DEBUG_PRINTF) && dev) {
966  tstr = VK_KHR_SHADER_RELAXED_EXTENDED_INSTRUCTION_EXTENSION_NAME;
967  found = 0;
968  for (int j = 0; j < sup_ext_count; j++) {
969  if (!strcmp(tstr, sup_ext[j].extensionName)) {
970  found = 1;
971  break;
972  }
973  }
974  if (!found) {
975  av_log(ctx, AV_LOG_ERROR, "Debug_printf/profile enabled, but extension \"%s\" not found!\n",
976  tstr);
977  err = AVERROR(EINVAL);
978  goto fail;
979  }
980  }
981 #endif
982 
983  if (user_exts_str) {
984  char *save, *token = av_strtok(user_exts_str, "+", &save);
985  while (token) {
986  found = 0;
987  for (int j = 0; j < sup_ext_count; j++) {
988  if (!strcmp(token, sup_ext[j].extensionName)) {
989  found = 1;
990  break;
991  }
992  }
993  if (found) {
994  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension \"%s\"\n", mod, token);
995  ADD_VAL_TO_LIST(extension_names, extensions_found, token);
996  } else {
997  av_log(ctx, AV_LOG_WARNING, "%s extension \"%s\" not found, excluding.\n",
998  mod, token);
999  }
1000  token = av_strtok(NULL, "+", &save);
1001  }
1002  }
1003 
1004  *dst = extension_names;
1005  *num = extensions_found;
1006 
1007  av_free(user_exts_str);
1008  av_free(sup_ext);
1009  return 0;
1010 
1011 fail:
1012  RELEASE_PROPS(extension_names, extensions_found);
1013  av_free(user_exts_str);
1014  av_free(sup_ext);
1015  return err;
1016 }
1017 
1018 static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts,
1019  const char * const **dst, uint32_t *num,
1020  enum FFVulkanDebugMode *debug_mode)
1021 {
1022  int err = 0;
1023  VulkanDevicePriv *priv = ctx->hwctx;
1024  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
1025 
1026  static const char layer_standard_validation[] = { "VK_LAYER_KHRONOS_validation" };
1027  int layer_standard_validation_found = 0;
1028 
1029  uint32_t sup_layer_count;
1030  VkLayerProperties *sup_layers;
1031 
1032  AVDictionaryEntry *user_layers = av_dict_get(opts, "layers", NULL, 0);
1033  char *user_layers_str = NULL;
1034  char *save, *token;
1035 
1036  const char **enabled_layers = NULL;
1037  uint32_t enabled_layers_count = 0;
1038 
1039  AVDictionaryEntry *debug_opt = av_dict_get(opts, "debug", NULL, 0);
1040  enum FFVulkanDebugMode mode;
1041 
1042  *debug_mode = mode = FF_VULKAN_DEBUG_NONE;
1043 
1044  /* Get a list of all layers */
1045  vk->EnumerateInstanceLayerProperties(&sup_layer_count, NULL);
1046  sup_layers = av_malloc_array(sup_layer_count, sizeof(VkLayerProperties));
1047  if (!sup_layers)
1048  return AVERROR(ENOMEM);
1049  vk->EnumerateInstanceLayerProperties(&sup_layer_count, sup_layers);
1050 
1051  av_log(ctx, AV_LOG_VERBOSE, "Supported layers:\n");
1052  for (int i = 0; i < sup_layer_count; i++)
1053  av_log(ctx, AV_LOG_VERBOSE, "\t%s\n", sup_layers[i].layerName);
1054 
1055  /* If no user layers or debug layers are given, return */
1056  if (!debug_opt && !user_layers)
1057  goto end;
1058 
1059  /* Check for any properly supported validation layer */
1060  if (debug_opt) {
1061  if (!strcmp(debug_opt->value, "printf")) {
1062  mode = FF_VULKAN_DEBUG_PRINTF;
1063  } else if (!strcmp(debug_opt->value, "validate")) {
1064  mode = FF_VULKAN_DEBUG_VALIDATE;
1065  } else if (!strcmp(debug_opt->value, "practices")) {
1066  mode = FF_VULKAN_DEBUG_PRACTICES;
1067  } else {
1068  char *end_ptr = NULL;
1069  int idx = strtol(debug_opt->value, &end_ptr, 10);
1070  if (end_ptr == debug_opt->value || end_ptr[0] != '\0' ||
1071  idx < 0 || idx >= FF_VULKAN_DEBUG_NB) {
1072  av_log(ctx, AV_LOG_ERROR, "Invalid debugging mode \"%s\"\n",
1073  debug_opt->value);
1074  err = AVERROR(EINVAL);
1075  goto end;
1076  }
1077  mode = idx;
1078  }
1079  }
1080 
1081  /* If mode is VALIDATE or PRINTF, try to find the standard validation layer extension */
1082  if ((mode == FF_VULKAN_DEBUG_VALIDATE) ||
1083  (mode == FF_VULKAN_DEBUG_PRINTF) ||
1084  (mode == FF_VULKAN_DEBUG_PRACTICES)) {
1085  for (int i = 0; i < sup_layer_count; i++) {
1086  if (!strcmp(layer_standard_validation, sup_layers[i].layerName)) {
1087  av_log(ctx, AV_LOG_VERBOSE, "Standard validation layer %s is enabled\n",
1088  layer_standard_validation);
1089  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, layer_standard_validation);
1090  *debug_mode = mode;
1091  layer_standard_validation_found = 1;
1092  break;
1093  }
1094  }
1095  if (!layer_standard_validation_found) {
1096  av_log(ctx, AV_LOG_ERROR,
1097  "Validation Layer \"%s\" not supported\n", layer_standard_validation);
1098  err = AVERROR(ENOTSUP);
1099  goto end;
1100  }
1101  }
1102 
1103  /* Process any custom layers enabled */
1104  if (user_layers) {
1105  int found;
1106 
1107  user_layers_str = av_strdup(user_layers->value);
1108  if (!user_layers_str) {
1109  err = AVERROR(ENOMEM);
1110  goto fail;
1111  }
1112 
1113  token = av_strtok(user_layers_str, "+", &save);
1114  while (token) {
1115  found = 0;
1116 
1117  /* If debug=1/2 was specified as an option, skip this layer */
1118  if (!strcmp(layer_standard_validation, token) && layer_standard_validation_found) {
1119  token = av_strtok(NULL, "+", &save);
1120  break;
1121  }
1122 
1123  /* Try to find the layer in the list of supported layers */
1124  for (int j = 0; j < sup_layer_count; j++) {
1125  if (!strcmp(token, sup_layers[j].layerName)) {
1126  found = 1;
1127  break;
1128  }
1129  }
1130 
1131  if (found) {
1132  av_log(ctx, AV_LOG_VERBOSE, "Using layer: %s\n", token);
1133  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, token);
1134 
1135  /* If debug was not set as an option, force it */
1136  if (!strcmp(layer_standard_validation, token))
1137  *debug_mode = FF_VULKAN_DEBUG_VALIDATE;
1138  } else {
1139  av_log(ctx, AV_LOG_ERROR,
1140  "Layer \"%s\" not supported\n", token);
1141  err = AVERROR(EINVAL);
1142  goto end;
1143  }
1144 
1145  token = av_strtok(NULL, "+", &save);
1146  }
1147  }
1148 
1149 fail:
1150 end:
1151  av_free(sup_layers);
1152  av_free(user_layers_str);
1153 
1154  if (err < 0) {
1155  RELEASE_PROPS(enabled_layers, enabled_layers_count);
1156  } else {
1157  *dst = enabled_layers;
1158  *num = enabled_layers_count;
1159  }
1160 
1161  return err;
1162 }
1163 
1164 /* Creates a VkInstance */
1165 static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts,
1166  enum FFVulkanDebugMode *debug_mode)
1167 {
1168  int err = 0;
1169  VkResult ret;
1170  VulkanDevicePriv *p = ctx->hwctx;
1171  AVVulkanDeviceContext *hwctx = &p->p;
1172  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1173  VkApplicationInfo application_info = {
1174  .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
1175  .pApplicationName = "ffmpeg",
1176  .applicationVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1177  LIBAVUTIL_VERSION_MINOR,
1178  LIBAVUTIL_VERSION_MICRO),
1179  .pEngineName = "libavutil",
1180  .apiVersion = VK_API_VERSION_1_3,
1181  .engineVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1182  LIBAVUTIL_VERSION_MINOR,
1183  LIBAVUTIL_VERSION_MICRO),
1184  };
1185  VkValidationFeaturesEXT validation_features = {
1186  .sType = VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT,
1187  };
1188  VkInstanceCreateInfo inst_props = {
1189  .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
1190  .pApplicationInfo = &application_info,
1191  };
1192 
1193  if (!hwctx->get_proc_addr) {
1194  err = load_libvulkan(ctx);
1195  if (err < 0)
1196  return err;
1197  }
1198 
1199  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 0, 0);
1200  if (err < 0) {
1201  av_log(ctx, AV_LOG_ERROR, "Unable to load instance enumeration functions!\n");
1202  return err;
1203  }
1204 
1205  err = check_layers(ctx, opts, &inst_props.ppEnabledLayerNames,
1206  &inst_props.enabledLayerCount, debug_mode);
1207  if (err)
1208  goto fail;
1209 
1210  /* Check for present/missing extensions */
1211  err = check_extensions(ctx, 0, opts, &inst_props.ppEnabledExtensionNames,
1212  &inst_props.enabledExtensionCount, *debug_mode);
1213  hwctx->enabled_inst_extensions = inst_props.ppEnabledExtensionNames;
1214  hwctx->nb_enabled_inst_extensions = inst_props.enabledExtensionCount;
1215  if (err < 0)
1216  goto fail;
1217 
1218  /* Enable debug features if needed */
1219  if (*debug_mode == FF_VULKAN_DEBUG_VALIDATE) {
1220  static const VkValidationFeatureEnableEXT feat_list_validate[] = {
1221  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1222  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1223  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_EXT,
1224  };
1225  validation_features.pEnabledValidationFeatures = feat_list_validate;
1226  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_validate);
1227  inst_props.pNext = &validation_features;
1228  } else if (*debug_mode == FF_VULKAN_DEBUG_PRINTF) {
1229  static const VkValidationFeatureEnableEXT feat_list_debug[] = {
1230  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1231  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1232  VK_VALIDATION_FEATURE_ENABLE_DEBUG_PRINTF_EXT,
1233  };
1234  validation_features.pEnabledValidationFeatures = feat_list_debug;
1235  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_debug);
1236  inst_props.pNext = &validation_features;
1237  } else if (*debug_mode == FF_VULKAN_DEBUG_PRACTICES) {
1238  static const VkValidationFeatureEnableEXT feat_list_practices[] = {
1239  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1240  VK_VALIDATION_FEATURE_ENABLE_BEST_PRACTICES_EXT,
1241  };
1242  validation_features.pEnabledValidationFeatures = feat_list_practices;
1243  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_practices);
1244  inst_props.pNext = &validation_features;
1245  }
1246 
1247 #ifdef __APPLE__
1248  for (int i = 0; i < inst_props.enabledExtensionCount; i++) {
1249  if (!strcmp(VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME,
1250  inst_props.ppEnabledExtensionNames[i])) {
1251  inst_props.flags |= VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR;
1252  break;
1253  }
1254  }
1255 #endif
1256 
1257  /* Try to create the instance */
1258  ret = vk->CreateInstance(&inst_props, hwctx->alloc, &hwctx->inst);
1259 
1260  /* Check for errors */
1261  if (ret != VK_SUCCESS) {
1262  av_log(ctx, AV_LOG_ERROR, "Instance creation failure: %s\n",
1263  ff_vk_ret2str(ret));
1264  err = AVERROR_EXTERNAL;
1265  goto fail;
1266  }
1267 
1268  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 0);
1269  if (err < 0) {
1270  av_log(ctx, AV_LOG_ERROR, "Unable to load instance functions!\n");
1271  goto fail;
1272  }
1273 
1274  /* Setup debugging callback if needed */
1275  if ((*debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
1276  (*debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
1277  (*debug_mode == FF_VULKAN_DEBUG_PRACTICES)) {
1278  VkDebugUtilsMessengerCreateInfoEXT dbg = {
1279  .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
1280  .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT |
1281  VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
1282  VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
1283  VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
1284  .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
1285  VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
1286  VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
1287  .pfnUserCallback = vk_dbg_callback,
1288  .pUserData = ctx,
1289  };
1290 
1291  vk->CreateDebugUtilsMessengerEXT(hwctx->inst, &dbg,
1292  hwctx->alloc, &p->debug_ctx);
1293  }
1294 
1295  err = 0;
1296 
1297 fail:
1298  RELEASE_PROPS(inst_props.ppEnabledLayerNames, inst_props.enabledLayerCount);
1299  return err;
1300 }
1301 
1302 typedef struct VulkanDeviceSelection {
1303  uint8_t uuid[VK_UUID_SIZE]; /* Will use this first unless !has_uuid */
1304  int has_uuid;
1305  uint32_t drm_major; /* Will use this second unless !has_drm */
1306  uint32_t drm_minor; /* Will use this second unless !has_drm */
1307  uint32_t has_drm; /* has drm node info */
1308  const char *name; /* Will use this third unless NULL */
1309  uint32_t pci_device; /* Will use this fourth unless 0x0 */
1310  uint32_t vendor_id; /* Last resort to find something deterministic */
1311  int index; /* Finally fall back to index */
1312 } VulkanDeviceSelection;
1313 
1314 static const char *vk_dev_type(enum VkPhysicalDeviceType type)
1315 {
1316  switch (type) {
1317  case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "integrated";
1318  case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "discrete";
1319  case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "virtual";
1320  case VK_PHYSICAL_DEVICE_TYPE_CPU: return "software";
1321  default: return "unknown";
1322  }
1323 }
1324 
1325 /* Finds a device */
1326 static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
1327 {
1328  int err = 0, choice = -1;
1329  uint32_t num;
1330  VkResult ret;
1331  VulkanDevicePriv *p = ctx->hwctx;
1332  AVVulkanDeviceContext *hwctx = &p->p;
1333  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1334  VkPhysicalDevice *devices = NULL;
1335  VkPhysicalDeviceIDProperties *idp = NULL;
1336  VkPhysicalDeviceProperties2 *prop = NULL;
1337  VkPhysicalDeviceDriverProperties *driver_prop = NULL;
1338  VkPhysicalDeviceDrmPropertiesEXT *drm_prop = NULL;
1339 
1340  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, NULL);
1341  if (ret != VK_SUCCESS || !num) {
1342  av_log(ctx, AV_LOG_ERROR, "No devices found: %s!\n", ff_vk_ret2str(ret));
1343  return AVERROR(ENODEV);
1344  }
1345 
1346  devices = av_malloc_array(num, sizeof(VkPhysicalDevice));
1347  if (!devices)
1348  return AVERROR(ENOMEM);
1349 
1350  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, devices);
1351  if (ret != VK_SUCCESS) {
1352  av_log(ctx, AV_LOG_ERROR, "Failed enumerating devices: %s\n",
1353  ff_vk_ret2str(ret));
1354  err = AVERROR(ENODEV);
1355  goto end;
1356  }
1357 
1358  prop = av_calloc(num, sizeof(*prop));
1359  if (!prop) {
1360  err = AVERROR(ENOMEM);
1361  goto end;
1362  }
1363 
1364  idp = av_calloc(num, sizeof(*idp));
1365  if (!idp) {
1366  err = AVERROR(ENOMEM);
1367  goto end;
1368  }
1369 
1370  driver_prop = av_calloc(num, sizeof(*driver_prop));
1371  if (!driver_prop) {
1372  err = AVERROR(ENOMEM);
1373  goto end;
1374  }
1375 
1376  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1377  drm_prop = av_calloc(num, sizeof(*drm_prop));
1378  if (!drm_prop) {
1379  err = AVERROR(ENOMEM);
1380  goto end;
1381  }
1382  }
1383 
1384  av_log(ctx, AV_LOG_VERBOSE, "GPU listing:\n");
1385  for (int i = 0; i < num; i++) {
1386  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1387  drm_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT;
1388  driver_prop[i].pNext = &drm_prop[i];
1389  }
1390  driver_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
1391  idp[i].pNext = &driver_prop[i];
1392  idp[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES;
1393  prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1394  prop[i].pNext = &idp[i];
1395 
1396  vk->GetPhysicalDeviceProperties2(devices[i], &prop[i]);
1397  av_log(ctx, AV_LOG_VERBOSE, " %d: %s (%s) (0x%x)\n", i,
1398  prop[i].properties.deviceName,
1399  vk_dev_type(prop[i].properties.deviceType),
1400  prop[i].properties.deviceID);
1401  }
1402 
1403  if (select->has_uuid) {
1404  for (int i = 0; i < num; i++) {
1405  if (!strncmp(idp[i].deviceUUID, select->uuid, VK_UUID_SIZE)) {
1406  choice = i;
1407  goto end;
1408  }
1409  }
1410  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given UUID!\n");
1411  err = AVERROR(ENODEV);
1412  goto end;
1413  } else if ((p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) && select->has_drm) {
1414  for (int i = 0; i < num; i++) {
1415  if ((select->drm_major == drm_prop[i].primaryMajor &&
1416  select->drm_minor == drm_prop[i].primaryMinor) ||
1417  (select->drm_major == drm_prop[i].renderMajor &&
1418  select->drm_minor == drm_prop[i].renderMinor)) {
1419  choice = i;
1420  goto end;
1421  }
1422  }
1423  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given DRM node numbers %i:%i!\n",
1424  select->drm_major, select->drm_minor);
1425  err = AVERROR(ENODEV);
1426  goto end;
1427  } else if (select->name) {
1428  av_log(ctx, AV_LOG_VERBOSE, "Requested device: %s\n", select->name);
1429  for (int i = 0; i < num; i++) {
1430  if (strstr(prop[i].properties.deviceName, select->name)) {
1431  choice = i;
1432  goto end;
1433  }
1434  }
1435  av_log(ctx, AV_LOG_ERROR, "Unable to find device \"%s\"!\n",
1436  select->name);
1437  err = AVERROR(ENODEV);
1438  goto end;
1439  } else if (select->pci_device) {
1440  av_log(ctx, AV_LOG_VERBOSE, "Requested device: 0x%x\n", select->pci_device);
1441  for (int i = 0; i < num; i++) {
1442  if (select->pci_device == prop[i].properties.deviceID) {
1443  choice = i;
1444  goto end;
1445  }
1446  }
1447  av_log(ctx, AV_LOG_ERROR, "Unable to find device with PCI ID 0x%x!\n",
1448  select->pci_device);
1449  err = AVERROR(EINVAL);
1450  goto end;
1451  } else if (select->vendor_id) {
1452  av_log(ctx, AV_LOG_VERBOSE, "Requested vendor: 0x%x\n", select->vendor_id);
1453  for (int i = 0; i < num; i++) {
1454  if (select->vendor_id == prop[i].properties.vendorID) {
1455  choice = i;
1456  goto end;
1457  }
1458  }
1459  av_log(ctx, AV_LOG_ERROR, "Unable to find device with Vendor ID 0x%x!\n",
1460  select->vendor_id);
1461  err = AVERROR(ENODEV);
1462  goto end;
1463  } else {
1464  if (select->index < num) {
1465  choice = select->index;
1466  goto end;
1467  }
1468  av_log(ctx, AV_LOG_ERROR, "Unable to find device with index %i!\n",
1469  select->index);
1470  err = AVERROR(ENODEV);
1471  goto end;
1472  }
1473 
1474 end:
1475  if (choice > -1) {
1476  av_log(ctx, AV_LOG_VERBOSE, "Device %d selected: %s (%s) (0x%x)\n",
1477  choice, prop[choice].properties.deviceName,
1478  vk_dev_type(prop[choice].properties.deviceType),
1479  prop[choice].properties.deviceID);
1480  hwctx->phys_dev = devices[choice];
1481  p->props = prop[choice];
1482  p->props.pNext = NULL;
1483  p->dprops = driver_prop[choice];
1484  p->dprops.pNext = NULL;
1485  }
1486 
1487  av_free(devices);
1488  av_free(prop);
1489  av_free(idp);
1490  av_free(drm_prop);
1491  av_free(driver_prop);
1492 
1493  return err;
1494 }
1495 
1496 /* Picks the least used qf with the fewest unneeded flags, or -1 if none found */
1497 static inline int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf,
1498  VkQueueFlagBits flags)
1499 {
1500  int index = -1;
1501  uint32_t min_score = UINT32_MAX;
1502 
1503  for (int i = 0; i < num_qf; i++) {
1504  VkQueueFlagBits qflags = qf[i].queueFamilyProperties.queueFlags;
1505 
1506  /* Per the spec, reporting transfer caps is optional for these 2 types */
1507  if ((flags & VK_QUEUE_TRANSFER_BIT) &&
1508  (qflags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT)))
1509  qflags |= VK_QUEUE_TRANSFER_BIT;
1510 
1511  if (qflags & flags) {
1512  uint32_t score = av_popcount(qflags) + qf[i].queueFamilyProperties.timestampValidBits;
1513  if (score < min_score) {
1514  index = i;
1515  min_score = score;
1516  }
1517  }
1518  }
1519 
1520  if (index > -1)
1521  qf[index].queueFamilyProperties.timestampValidBits++;
1522 
1523  return index;
1524 }
1525 
1526 static inline int pick_video_queue_family(VkQueueFamilyProperties2 *qf,
1527  VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf,
1528  VkVideoCodecOperationFlagsKHR flags)
1529 {
1530  int index = -1;
1531  uint32_t min_score = UINT32_MAX;
1532 
1533  for (int i = 0; i < num_qf; i++) {
1534  const VkQueueFlags qflags = qf[i].queueFamilyProperties.queueFlags;
1535  const VkVideoCodecOperationFlagsKHR vflags = qf_vid[i].videoCodecOperations;
1536 
1537  if (!(qflags & (VK_QUEUE_VIDEO_ENCODE_BIT_KHR | VK_QUEUE_VIDEO_DECODE_BIT_KHR)))
1538  continue;
1539 
1540  if (vflags & flags) {
1541  uint32_t score = av_popcount(vflags) + qf[i].queueFamilyProperties.timestampValidBits;
1542  if (score < min_score) {
1543  index = i;
1544  min_score = score;
1545  }
1546  }
1547  }
1548 
1549  if (index > -1)
1550  qf[index].queueFamilyProperties.timestampValidBits++;
1551 
1552  return index;
1553 }
1554 
1555 static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
1556 {
1557  uint32_t num;
1558  VulkanDevicePriv *p = ctx->hwctx;
1559  AVVulkanDeviceContext *hwctx = &p->p;
1560  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1561 
1562  VkQueueFamilyProperties2 *qf = NULL;
1563  VkQueueFamilyVideoPropertiesKHR *qf_vid = NULL;
1564 
1565  /* First get the number of queue families */
1566  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &num, NULL);
1567  if (!num) {
1568  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1569  return AVERROR_EXTERNAL;
1570  }
1571 
1572  /* Then allocate memory */
1573  qf = av_malloc_array(num, sizeof(VkQueueFamilyProperties2));
1574  if (!qf)
1575  return AVERROR(ENOMEM);
1576 
1577  qf_vid = av_malloc_array(num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1578  if (!qf_vid)
1579  return AVERROR(ENOMEM);
1580 
1581  for (uint32_t i = 0; i < num; i++) {
1582  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1583  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1584  };
1585  qf[i] = (VkQueueFamilyProperties2) {
1586  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1587  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1588  };
1589  }
1590 
1591  /* Finally retrieve the queue families */
1592  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &num, qf);
1593 
1594  av_log(ctx, AV_LOG_VERBOSE, "Queue families:\n");
1595  for (int i = 0; i < num; i++) {
1596  av_log(ctx, AV_LOG_VERBOSE, " %i:%s%s%s%s%s%s%s%s (queues: %i)\n", i,
1597  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_GRAPHICS_BIT) ? " graphics" : "",
1598  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_COMPUTE_BIT) ? " compute" : "",
1599  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_TRANSFER_BIT) ? " transfer" : "",
1600  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_ENCODE_BIT_KHR) ? " encode" : "",
1601  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_DECODE_BIT_KHR) ? " decode" : "",
1602  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_SPARSE_BINDING_BIT) ? " sparse" : "",
1603  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_OPTICAL_FLOW_BIT_NV) ? " optical_flow" : "",
1604  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_PROTECTED_BIT) ? " protected" : "",
1605  qf[i].queueFamilyProperties.queueCount);
1606 
1607  /* We use this field to keep a score of how many times we've used that
1608  * queue family in order to make better choices. */
1609  qf[i].queueFamilyProperties.timestampValidBits = 0;
1610  }
1611 
1612  hwctx->nb_qf = 0;
1613 
1614  /* Pick each queue family to use. */
1615 #define PICK_QF(type, vid_op) \
1616  do { \
1617  uint32_t i; \
1618  uint32_t idx; \
1619  \
1620  if (vid_op) \
1621  idx = pick_video_queue_family(qf, qf_vid, num, vid_op); \
1622  else \
1623  idx = pick_queue_family(qf, num, type); \
1624  \
1625  if (idx == -1) \
1626  continue; \
1627  \
1628  for (i = 0; i < hwctx->nb_qf; i++) { \
1629  if (hwctx->qf[i].idx == idx) { \
1630  hwctx->qf[i].flags |= type; \
1631  hwctx->qf[i].video_caps |= vid_op; \
1632  break; \
1633  } \
1634  } \
1635  if (i == hwctx->nb_qf) { \
1636  hwctx->qf[i].idx = idx; \
1637  hwctx->qf[i].num = qf[idx].queueFamilyProperties.queueCount; \
1638  if (p->limit_queues || \
1639  p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY) { \
1640  int max = p->limit_queues; \
1641  if (type == VK_QUEUE_GRAPHICS_BIT) \
1642  hwctx->qf[i].num = FFMIN(hwctx->qf[i].num, \
1643  max ? max : 1); \
1644  else if (max) \
1645  hwctx->qf[i].num = FFMIN(hwctx->qf[i].num, max); \
1646  } \
1647  hwctx->qf[i].flags = type; \
1648  hwctx->qf[i].video_caps = vid_op; \
1649  hwctx->nb_qf++; \
1650  } \
1651  } while (0)
1652 
1653  PICK_QF(VK_QUEUE_GRAPHICS_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1654  PICK_QF(VK_QUEUE_COMPUTE_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1655  PICK_QF(VK_QUEUE_TRANSFER_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1656 
1657  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR);
1658  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR);
1659 
1660  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR);
1661  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR);
1662 
1663 #ifdef VK_KHR_video_decode_vp9
1664  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_VP9_BIT_KHR);
1665 #endif
1666 
1667 #ifdef VK_KHR_video_encode_av1
1668  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_AV1_BIT_KHR);
1669 #endif
1670  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR);
1671 
1672  av_free(qf);
1673  av_free(qf_vid);
1674 
1675 #undef PICK_QF
1676 
1677  cd->pQueueCreateInfos = av_malloc_array(hwctx->nb_qf,
1678  sizeof(VkDeviceQueueCreateInfo));
1679  if (!cd->pQueueCreateInfos)
1680  return AVERROR(ENOMEM);
1681 
1682  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1683  int dup = 0;
1684  float *weights = NULL;
1685  VkDeviceQueueCreateInfo *pc;
1686  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++) {
1687  if (hwctx->qf[i].idx == cd->pQueueCreateInfos[j].queueFamilyIndex) {
1688  dup = 1;
1689  break;
1690  }
1691  }
1692  if (dup)
1693  continue;
1694 
1695  weights = av_malloc_array(hwctx->qf[i].num, sizeof(float));
1696  if (!weights) {
1697  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++)
1698  av_free((void *)cd->pQueueCreateInfos[i].pQueuePriorities);
1699  av_free((void *)cd->pQueueCreateInfos);
1700  return AVERROR(ENOMEM);
1701  }
1702 
1703  for (uint32_t j = 0; j < hwctx->qf[i].num; j++)
1704  weights[j] = 1.0;
1705 
1706  pc = (VkDeviceQueueCreateInfo *)cd->pQueueCreateInfos;
1707  pc[cd->queueCreateInfoCount++] = (VkDeviceQueueCreateInfo) {
1708  .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
1709  .queueFamilyIndex = hwctx->qf[i].idx,
1710  .queueCount = hwctx->qf[i].num,
1711  .pQueuePriorities = weights,
1712  };
1713  }
1714 
1715 #if FF_API_VULKAN_FIXED_QUEUES
1716 FF_DISABLE_DEPRECATION_WARNINGS
1717  /* Setup deprecated fields */
1718  hwctx->queue_family_index = -1;
1719  hwctx->queue_family_comp_index = -1;
1720  hwctx->queue_family_tx_index = -1;
1721  hwctx->queue_family_encode_index = -1;
1722  hwctx->queue_family_decode_index = -1;
1723 
1724 #define SET_OLD_QF(field, nb_field, type) \
1725  do { \
1726  if (field < 0 && hwctx->qf[i].flags & type) { \
1727  field = hwctx->qf[i].idx; \
1728  nb_field = hwctx->qf[i].num; \
1729  } \
1730  } while (0)
1731 
1732  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1733  SET_OLD_QF(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
1734  SET_OLD_QF(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
1735  SET_OLD_QF(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
1736  SET_OLD_QF(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
1737  SET_OLD_QF(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
1738  }
1739 
1740 #undef SET_OLD_QF
1741 FF_ENABLE_DEPRECATION_WARNINGS
1742 #endif
1743 
1744  return 0;
1745 }
1746 
1747 /* Only resources created by vulkan_device_create should be released here,
1748  * resources created by vulkan_device_init should be released by
1749  * vulkan_device_uninit, to make sure we don't free user provided resources,
1750  * and there is no leak.
1751  */
1752 static void vulkan_device_free(AVHWDeviceContext *ctx)
1753 {
1754  VulkanDevicePriv *p = ctx->hwctx;
1755  AVVulkanDeviceContext *hwctx = &p->p;
1756  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1757 
1758  if (hwctx->act_dev)
1759  vk->DestroyDevice(hwctx->act_dev, hwctx->alloc);
1760 
1761  if (p->debug_ctx)
1762  vk->DestroyDebugUtilsMessengerEXT(hwctx->inst, p->debug_ctx,
1763  hwctx->alloc);
1764 
1765  if (hwctx->inst)
1766  vk->DestroyInstance(hwctx->inst, hwctx->alloc);
1767 
1768  if (p->libvulkan)
1769  dlclose(p->libvulkan);
1770 
1771  RELEASE_PROPS(hwctx->enabled_inst_extensions, hwctx->nb_enabled_inst_extensions);
1772  RELEASE_PROPS(hwctx->enabled_dev_extensions, hwctx->nb_enabled_dev_extensions);
1773 }
1774 
1775 static void vulkan_device_uninit(AVHWDeviceContext *ctx)
1776 {
1777  VulkanDevicePriv *p = ctx->hwctx;
1778 
1779  for (uint32_t i = 0; i < p->nb_tot_qfs; i++) {
1780  pthread_mutex_destroy(p->qf_mutex[i]);
1781  av_freep(&p->qf_mutex[i]);
1782  }
1783  av_freep(&p->qf_mutex);
1784 
1785  ff_vk_uninit(&p->vkctx);
1786 }
1787 
1788 static int vulkan_device_create_internal(AVHWDeviceContext *ctx,
1789  VulkanDeviceSelection *dev_select,
1790  int disable_multiplane,
1791  AVDictionary *opts, int flags)
1792 {
1793  int err = 0;
1794  VkResult ret;
1795  AVDictionaryEntry *opt_d;
1796  VulkanDevicePriv *p = ctx->hwctx;
1797  AVVulkanDeviceContext *hwctx = &p->p;
1798  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1799  enum FFVulkanDebugMode debug_mode = FF_VULKAN_DEBUG_NONE;
1800  VulkanDeviceFeatures supported_feats = { 0 };
1801  VkDeviceCreateInfo dev_info = {
1802  .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
1803  };
1804 
1805  /* Create an instance if not given one */
1806  if ((err = create_instance(ctx, opts, &debug_mode)))
1807  goto end;
1808 
1809  /* Find a physical device (if not given one) */
1810  if ((err = find_device(ctx, dev_select)))
1811  goto end;
1812 
1813  /* Get supported memory types */
1814  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
1815 
1816  /* Find and enable extensions for the physical device */
1817  if ((err = check_extensions(ctx, 1, opts, &dev_info.ppEnabledExtensionNames,
1818  &dev_info.enabledExtensionCount, debug_mode))) {
1819  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1820  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1821  av_free((void *)dev_info.pQueueCreateInfos);
1822  goto end;
1823  }
1824 
1825  /* Get all supported features for the physical device */
1826  device_features_init(ctx, &supported_feats);
1827  vk->GetPhysicalDeviceFeatures2(hwctx->phys_dev, &supported_feats.device);
1828 
1829  /* Copy all needed features from those supported and activate them */
1830  device_features_init(ctx, &p->feats);
1831  device_features_copy_needed(&p->feats, &supported_feats);
1832  dev_info.pNext = p->feats.device.pNext;
1833  dev_info.pEnabledFeatures = &p->feats.device.features;
1834 
1835  /* Limit queues to a given number if needed */
1836  opt_d = av_dict_get(opts, "limit_queues", NULL, 0);
1837  if (opt_d)
1838  p->limit_queues = strtol(opt_d->value, NULL, 10);
1839 
1840  /* Setup enabled queue families */
1841  if ((err = setup_queue_families(ctx, &dev_info)))
1842  goto end;
1843 
1844  /* Finally create the device */
1845  ret = vk->CreateDevice(hwctx->phys_dev, &dev_info, hwctx->alloc,
1846  &hwctx->act_dev);
1847 
1848  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1849  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1850  av_free((void *)dev_info.pQueueCreateInfos);
1851 
1852  if (ret != VK_SUCCESS) {
1853  av_log(ctx, AV_LOG_ERROR, "Device creation failure: %s\n",
1854  ff_vk_ret2str(ret));
1855  for (int i = 0; i < dev_info.enabledExtensionCount; i++)
1856  av_free((void *)dev_info.ppEnabledExtensionNames[i]);
1857  av_free((void *)dev_info.ppEnabledExtensionNames);
1858  err = AVERROR_EXTERNAL;
1859  goto end;
1860  }
1861 
1862  /* Tiled images setting, use them by default */
1863  opt_d = av_dict_get(opts, "linear_images", NULL, 0);
1864  if (opt_d)
1865  p->use_linear_images = strtol(opt_d->value, NULL, 10);
1866 
1867  /* The disable_multiplane argument takes precedent over the option */
1868  p->disable_multiplane = disable_multiplane;
1869  if (!p->disable_multiplane) {
1870  opt_d = av_dict_get(opts, "disable_multiplane", NULL, 0);
1871  if (opt_d)
1872  p->disable_multiplane = strtol(opt_d->value, NULL, 10);
1873  }
1874 
1875  /* Disable host pointer imports (by default on nvidia) */
1876  p->avoid_host_import = p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY;
1877  opt_d = av_dict_get(opts, "avoid_host_import", NULL, 0);
1878  if (opt_d)
1879  p->avoid_host_import = strtol(opt_d->value, NULL, 10);
1880 
1881  /* Set the public device feature struct and its pNext chain */
1882  hwctx->device_features = p->feats.device;
1883 
1884  /* Set the list of all active extensions */
1885  hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames;
1886  hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount;
1887 
1888  /* The extension lists need to be freed */
1889  ctx->free = vulkan_device_free;
1890 
1891 end:
1892  return err;
1893 }
1894 
1895 static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1896 {
1897  VulkanDevicePriv *p = ctx->hwctx;
1898  pthread_mutex_lock(&p->qf_mutex[queue_family][index]);
1899 }
1900 
1901 static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1902 {
1903  VulkanDevicePriv *p = ctx->hwctx;
1904  pthread_mutex_unlock(&p->qf_mutex[queue_family][index]);
1905 }
1906 
1907 static int vulkan_device_init(AVHWDeviceContext *ctx)
1908 {
1909  int err = 0;
1910  uint32_t qf_num;
1911  VulkanDevicePriv *p = ctx->hwctx;
1912  AVVulkanDeviceContext *hwctx = &p->p;
1913  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1914  VkQueueFamilyProperties2 *qf;
1915  VkQueueFamilyVideoPropertiesKHR *qf_vid;
1916  VkPhysicalDeviceExternalSemaphoreInfo ext_sem_props_info;
1917  int graph_index, comp_index, tx_index, enc_index, dec_index;
1918 
1919  /* Set device extension flags */
1920  for (int i = 0; i < hwctx->nb_enabled_dev_extensions; i++) {
1921  for (int j = 0; j < FF_ARRAY_ELEMS(optional_device_exts); j++) {
1922  if (!strcmp(hwctx->enabled_dev_extensions[i],
1923  optional_device_exts[j].name)) {
1924  p->vkctx.extensions |= optional_device_exts[j].flag;
1925  break;
1926  }
1927  }
1928  }
1929 
1930  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 1);
1931  if (err < 0) {
1932  av_log(ctx, AV_LOG_ERROR, "Unable to load functions!\n");
1933  return err;
1934  }
1935 
1936  p->props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1937  p->props.pNext = &p->hprops;
1938  p->hprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT;
1939  p->hprops.pNext = &p->dprops;
1940  p->dprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
1941 
1942  vk->GetPhysicalDeviceProperties2(hwctx->phys_dev, &p->props);
1943  av_log(ctx, AV_LOG_VERBOSE, "Using device: %s\n",
1944  p->props.properties.deviceName);
1945  av_log(ctx, AV_LOG_VERBOSE, "Alignments:\n");
1946  av_log(ctx, AV_LOG_VERBOSE, " optimalBufferCopyRowPitchAlignment: %"PRIu64"\n",
1947  p->props.properties.limits.optimalBufferCopyRowPitchAlignment);
1948  av_log(ctx, AV_LOG_VERBOSE, " minMemoryMapAlignment: %zu\n",
1949  p->props.properties.limits.minMemoryMapAlignment);
1950  av_log(ctx, AV_LOG_VERBOSE, " nonCoherentAtomSize: %"PRIu64"\n",
1951  p->props.properties.limits.nonCoherentAtomSize);
1952  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY)
1953  av_log(ctx, AV_LOG_VERBOSE, " minImportedHostPointerAlignment: %"PRIu64"\n",
1954  p->hprops.minImportedHostPointerAlignment);
1955 
1956  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &qf_num, NULL);
1957  if (!qf_num) {
1958  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1959  return AVERROR_EXTERNAL;
1960  }
1961 
1962  ext_sem_props_info = (VkPhysicalDeviceExternalSemaphoreInfo) {
1963  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO,
1964  };
1965 
1966  /* Opaque FD semaphore properties */
1967  ext_sem_props_info.handleType =
1968 #ifdef _WIN32
1969  IsWindows8OrGreater()
1970  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
1971  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT;
1972 #else
1973  VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
1974 #endif
1975  p->ext_sem_props_opaque.sType = VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES;
1976  vk->GetPhysicalDeviceExternalSemaphoreProperties(hwctx->phys_dev,
1977  &ext_sem_props_info,
1978  &p->ext_sem_props_opaque);
1979 
1980  qf = av_malloc_array(qf_num, sizeof(VkQueueFamilyProperties2));
1981  if (!qf)
1982  return AVERROR(ENOMEM);
1983 
1984  qf_vid = av_malloc_array(qf_num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1985  if (!qf_vid) {
1986  av_free(qf);
1987  return AVERROR(ENOMEM);
1988  }
1989 
1990  for (uint32_t i = 0; i < qf_num; i++) {
1991  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1992  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1993  };
1994  qf[i] = (VkQueueFamilyProperties2) {
1995  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1996  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1997  };
1998  }
1999 
2000  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &qf_num, qf);
2001 
2002  p->qf_mutex = av_calloc(qf_num, sizeof(*p->qf_mutex));
2003  if (!p->qf_mutex) {
2004  err = AVERROR(ENOMEM);
2005  goto end;
2006  }
2007  p->nb_tot_qfs = qf_num;
2008 
2009  for (uint32_t i = 0; i < qf_num; i++) {
2010  p->qf_mutex[i] = av_calloc(qf[i].queueFamilyProperties.queueCount,
2011  sizeof(**p->qf_mutex));
2012  if (!p->qf_mutex[i]) {
2013  err = AVERROR(ENOMEM);
2014  goto end;
2015  }
2016  for (uint32_t j = 0; j < qf[i].queueFamilyProperties.queueCount; j++) {
2017  err = pthread_mutex_init(&p->qf_mutex[i][j], NULL);
2018  if (err != 0) {
2019  av_log(ctx, AV_LOG_ERROR, "pthread_mutex_init failed : %s\n",
2020  av_err2str(err));
2021  err = AVERROR(err);
2022  goto end;
2023  }
2024  }
2025  }
2026 
2027 #if FF_API_VULKAN_FIXED_QUEUES
2028 FF_DISABLE_DEPRECATION_WARNINGS
2029  graph_index = hwctx->nb_graphics_queues ? hwctx->queue_family_index : -1;
2030  comp_index = hwctx->nb_comp_queues ? hwctx->queue_family_comp_index : -1;
2031  tx_index = hwctx->nb_tx_queues ? hwctx->queue_family_tx_index : -1;
2032  dec_index = hwctx->nb_decode_queues ? hwctx->queue_family_decode_index : -1;
2033  enc_index = hwctx->nb_encode_queues ? hwctx->queue_family_encode_index : -1;
2034 
2035 #define CHECK_QUEUE(type, required, fidx, ctx_qf, qc) \
2036  do { \
2037  if (ctx_qf < 0 && required) { \
2038  av_log(ctx, AV_LOG_ERROR, "%s queue family is required, but marked as missing" \
2039  " in the context!\n", type); \
2040  err = AVERROR(EINVAL); \
2041  goto end; \
2042  } else if (fidx < 0 || ctx_qf < 0) { \
2043  break; \
2044  } else if (ctx_qf >= qf_num) { \
2045  av_log(ctx, AV_LOG_ERROR, "Invalid %s family index %i (device has %i families)!\n", \
2046  type, ctx_qf, qf_num); \
2047  err = AVERROR(EINVAL); \
2048  goto end; \
2049  } \
2050  \
2051  av_log(ctx, AV_LOG_VERBOSE, "Using queue family %i (queues: %i)" \
2052  " for%s%s%s%s%s\n", \
2053  ctx_qf, qc, \
2054  ctx_qf == graph_index ? " graphics" : "", \
2055  ctx_qf == comp_index ? " compute" : "", \
2056  ctx_qf == tx_index ? " transfers" : "", \
2057  ctx_qf == enc_index ? " encode" : "", \
2058  ctx_qf == dec_index ? " decode" : ""); \
2059  graph_index = (ctx_qf == graph_index) ? -1 : graph_index; \
2060  comp_index = (ctx_qf == comp_index) ? -1 : comp_index; \
2061  tx_index = (ctx_qf == tx_index) ? -1 : tx_index; \
2062  enc_index = (ctx_qf == enc_index) ? -1 : enc_index; \
2063  dec_index = (ctx_qf == dec_index) ? -1 : dec_index; \
2064  } while (0)
2065 
2066  CHECK_QUEUE("graphics", 0, graph_index, hwctx->queue_family_index, hwctx->nb_graphics_queues);
2067  CHECK_QUEUE("compute", 1, comp_index, hwctx->queue_family_comp_index, hwctx->nb_comp_queues);
2068  CHECK_QUEUE("upload", 1, tx_index, hwctx->queue_family_tx_index, hwctx->nb_tx_queues);
2069  CHECK_QUEUE("decode", 0, dec_index, hwctx->queue_family_decode_index, hwctx->nb_decode_queues);
2070  CHECK_QUEUE("encode", 0, enc_index, hwctx->queue_family_encode_index, hwctx->nb_encode_queues);
2071 
2072 #undef CHECK_QUEUE
2073 
2074  /* Update the new queue family fields. If non-zero already,
2075  * it means API users have set it. */
2076  if (!hwctx->nb_qf) {
2077 #define ADD_QUEUE(ctx_qf, qc, flag) \
2078  do { \
2079  if (ctx_qf != -1) { \
2080  hwctx->qf[hwctx->nb_qf++] = (AVVulkanDeviceQueueFamily) { \
2081  .idx = ctx_qf, \
2082  .num = qc, \
2083  .flags = flag, \
2084  }; \
2085  } \
2086  } while (0)
2087 
2088  ADD_QUEUE(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
2089  ADD_QUEUE(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
2090  ADD_QUEUE(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
2091  ADD_QUEUE(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
2092  ADD_QUEUE(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
2093 #undef ADD_QUEUE
2094  }
2095 FF_ENABLE_DEPRECATION_WARNINGS
2096 #endif
2097 
2098  for (int i = 0; i < hwctx->nb_qf; i++) {
2099  if (!hwctx->qf[i].video_caps &&
2100  hwctx->qf[i].flags & (VK_QUEUE_VIDEO_DECODE_BIT_KHR |
2101  VK_QUEUE_VIDEO_ENCODE_BIT_KHR)) {
2102  hwctx->qf[i].video_caps = qf_vid[hwctx->qf[i].idx].videoCodecOperations;
2103  }
2104  }
2105 
2106  /* Setup array for pQueueFamilyIndices with used queue families */
2107  p->nb_img_qfs = 0;
2108  for (int i = 0; i < hwctx->nb_qf; i++) {
2109  int seen = 0;
2110  /* Make sure each entry is unique
2111  * (VUID-VkBufferCreateInfo-sharingMode-01419) */
2112  for (int j = (i - 1); j >= 0; j--) {
2113  if (hwctx->qf[i].idx == hwctx->qf[j].idx) {
2114  seen = 1;
2115  break;
2116  }
2117  }
2118  if (!seen)
2119  p->img_qfs[p->nb_img_qfs++] = hwctx->qf[i].idx;
2120  }
2121 
2122 #if FF_API_VULKAN_SYNC_QUEUES
2123 FF_DISABLE_DEPRECATION_WARNINGS
2124  if (!hwctx->lock_queue)
2125  hwctx->lock_queue = lock_queue;
2126  if (!hwctx->unlock_queue)
2127  hwctx->unlock_queue = unlock_queue;
2128 FF_ENABLE_DEPRECATION_WARNINGS
2129 #endif
2130 
2131  /* Re-query device capabilities, in case the device was created externally */
2132  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
2133 
2134  p->vkctx.device = ctx;
2135  p->vkctx.hwctx = hwctx;
2136 
2137  ff_vk_load_props(&p->vkctx);
2138  p->compute_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_COMPUTE_BIT, 0);
2139  p->transfer_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_TRANSFER_BIT, 0);
2140 
2141  /* Re-query device capabilities, in case the device was created externally */
2142  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
2143 
2144 end:
2145  av_free(qf_vid);
2146  av_free(qf);
2147  return err;
2148 }
2149 
2150 static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device,
2151  AVDictionary *opts, int flags)
2152 {
2153  VulkanDeviceSelection dev_select = { 0 };
2154  if (device && device[0]) {
2155  char *end = NULL;
2156  dev_select.index = strtol(device, &end, 10);
2157  if (end == device) {
2158  dev_select.index = 0;
2159  dev_select.name = device;
2160  }
2161  }
2162 
2163  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2164 }
2165 
2166 static int vulkan_device_derive(AVHWDeviceContext *ctx,
2167  AVHWDeviceContext *src_ctx,
2168  AVDictionary *opts, int flags)
2169 {
2170  av_unused VulkanDeviceSelection dev_select = { 0 };
2171 
2172  /* If there's only one device on the system, then even if its not covered
2173  * by the following checks (e.g. non-PCIe ARM GPU), having an empty
2174  * dev_select will mean it'll get picked. */
2175  switch(src_ctx->type) {
2176 #if CONFIG_VAAPI
2177  case AV_HWDEVICE_TYPE_VAAPI: {
2178  AVVAAPIDeviceContext *src_hwctx = src_ctx->hwctx;
2179  VADisplay dpy = src_hwctx->display;
2180 #if VA_CHECK_VERSION(1, 15, 0)
2181  VAStatus vas;
2182  VADisplayAttribute attr = {
2183  .type = VADisplayPCIID,
2184  };
2185 #endif
2186  const char *vendor;
2187 
2188 #if VA_CHECK_VERSION(1, 15, 0)
2189  vas = vaGetDisplayAttributes(dpy, &attr, 1);
2190  if (vas == VA_STATUS_SUCCESS && attr.flags != VA_DISPLAY_ATTRIB_NOT_SUPPORTED)
2191  dev_select.pci_device = (attr.value & 0xFFFF);
2192 #endif
2193 
2194  if (!dev_select.pci_device) {
2195  vendor = vaQueryVendorString(dpy);
2196  if (!vendor) {
2197  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from VAAPI!\n");
2198  return AVERROR_EXTERNAL;
2199  }
2200 
2201  if (strstr(vendor, "AMD"))
2202  dev_select.vendor_id = 0x1002;
2203  }
2204 
2205  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2206  }
2207 #endif
2208 #if CONFIG_LIBDRM
2209  case AV_HWDEVICE_TYPE_DRM: {
2210  int err;
2211  struct stat drm_node_info;
2212  drmDevice *drm_dev_info;
2213  AVDRMDeviceContext *src_hwctx = src_ctx->hwctx;
2214 
2215  err = fstat(src_hwctx->fd, &drm_node_info);
2216  if (err) {
2217  av_log(ctx, AV_LOG_ERROR, "Unable to get node info from DRM fd: %s!\n",
2218  av_err2str(AVERROR(errno)));
2219  return AVERROR_EXTERNAL;
2220  }
2221 
2222  dev_select.drm_major = major(drm_node_info.st_dev);
2223  dev_select.drm_minor = minor(drm_node_info.st_dev);
2224  dev_select.has_drm = 1;
2225 
2226  err = drmGetDevice(src_hwctx->fd, &drm_dev_info);
2227  if (err) {
2228  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from DRM fd: %s!\n",
2229  av_err2str(AVERROR(errno)));
2230  return AVERROR_EXTERNAL;
2231  }
2232 
2233  if (drm_dev_info->bustype == DRM_BUS_PCI)
2234  dev_select.pci_device = drm_dev_info->deviceinfo.pci->device_id;
2235 
2236  drmFreeDevice(&drm_dev_info);
2237 
2238  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2239  }
2240 #endif
2241 #if CONFIG_CUDA
2242  case AV_HWDEVICE_TYPE_CUDA: {
2243  AVHWDeviceContext *cuda_cu = src_ctx;
2244  AVCUDADeviceContext *src_hwctx = src_ctx->hwctx;
2245  AVCUDADeviceContextInternal *cu_internal = src_hwctx->internal;
2246  CudaFunctions *cu = cu_internal->cuda_dl;
2247 
2248  int ret = CHECK_CU(cu->cuDeviceGetUuid((CUuuid *)&dev_select.uuid,
2249  cu_internal->cuda_device));
2250  if (ret < 0) {
2251  av_log(ctx, AV_LOG_ERROR, "Unable to get UUID from CUDA!\n");
2252  return AVERROR_EXTERNAL;
2253  }
2254 
2255  dev_select.has_uuid = 1;
2256 
2257  /*
2258  * CUDA is not able to import multiplane images, so always derive a
2259  * Vulkan device with multiplane disabled.
2260  */
2261  return vulkan_device_create_internal(ctx, &dev_select, 1, opts, flags);
2262  }
2263 #endif
2264  default:
2265  return AVERROR(ENOSYS);
2266  }
2267 }
2268 
2269 static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx,
2270  const void *hwconfig,
2271  AVHWFramesConstraints *constraints)
2272 {
2273  int count = 0;
2274  VulkanDevicePriv *p = ctx->hwctx;
2275 
2276  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2277  count += vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2278  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2279  VK_IMAGE_TILING_OPTIMAL,
2280  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0;
2281  }
2282 
2283  constraints->valid_sw_formats = av_malloc_array(count + 1,
2284  sizeof(enum AVPixelFormat));
2285  if (!constraints->valid_sw_formats)
2286  return AVERROR(ENOMEM);
2287 
2288  count = 0;
2289  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2290  if (vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2291  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2292  VK_IMAGE_TILING_OPTIMAL,
2293  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0) {
2294  constraints->valid_sw_formats[count++] = vk_formats_list[i].pixfmt;
2295  }
2296  }
2297 
2298  constraints->valid_sw_formats[count++] = AV_PIX_FMT_NONE;
2299 
2300  constraints->min_width = 1;
2301  constraints->min_height = 1;
2302  constraints->max_width = p->props.properties.limits.maxImageDimension2D;
2303  constraints->max_height = p->props.properties.limits.maxImageDimension2D;
2304 
2305  constraints->valid_hw_formats = av_malloc_array(2, sizeof(enum AVPixelFormat));
2306  if (!constraints->valid_hw_formats)
2307  return AVERROR(ENOMEM);
2308 
2309  constraints->valid_hw_formats[0] = AV_PIX_FMT_VULKAN;
2310  constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
2311 
2312  return 0;
2313 }
2314 
2315 static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req,
2316  VkMemoryPropertyFlagBits req_flags, const void *alloc_extension,
2317  VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
2318 {
2319  VkResult ret;
2320  int index = -1;
2321  VulkanDevicePriv *p = ctx->hwctx;
2322  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2323  AVVulkanDeviceContext *dev_hwctx = &p->p;
2324  VkMemoryAllocateInfo alloc_info = {
2325  .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
2326  .pNext = alloc_extension,
2327  .allocationSize = req->size,
2328  };
2329 
2330  /* The vulkan spec requires memory types to be sorted in the "optimal"
2331  * order, so the first matching type we find will be the best/fastest one */
2332  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
2333  const VkMemoryType *type = &p->mprops.memoryTypes[i];
2334 
2335  /* The memory type must be supported by the requirements (bitfield) */
2336  if (!(req->memoryTypeBits & (1 << i)))
2337  continue;
2338 
2339  /* The memory type flags must include our properties */
2340  if ((type->propertyFlags & req_flags) != req_flags)
2341  continue;
2342 
2343  /* The memory type must be large enough */
2344  if (req->size > p->mprops.memoryHeaps[type->heapIndex].size)
2345  continue;
2346 
2347  /* Found a suitable memory type */
2348  index = i;
2349  break;
2350  }
2351 
2352  if (index < 0) {
2353  av_log(ctx, AV_LOG_ERROR, "No memory type found for flags 0x%x\n",
2354  req_flags);
2355  return AVERROR(EINVAL);
2356  }
2357 
2358  alloc_info.memoryTypeIndex = index;
2359 
2360  ret = vk->AllocateMemory(dev_hwctx->act_dev, &alloc_info,
2361  dev_hwctx->alloc, mem);
2362  if (ret != VK_SUCCESS) {
2363  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory: %s\n",
2364  ff_vk_ret2str(ret));
2365  return AVERROR(ENOMEM);
2366  }
2367 
2368  *mem_flags |= p->mprops.memoryTypes[index].propertyFlags;
2369 
2370  return 0;
2371 }
2372 
2373 static void vulkan_free_internal(AVVkFrame *f)
2374 {
2375  av_unused AVVkFrameInternal *internal = f->internal;
2376 
2377 #if CONFIG_CUDA
2378  if (internal->cuda_fc_ref) {
2379  AVHWFramesContext *cuda_fc = (AVHWFramesContext *)internal->cuda_fc_ref->data;
2380  int planes = av_pix_fmt_count_planes(cuda_fc->sw_format);
2381  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
2382  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
2383  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
2384  CudaFunctions *cu = cu_internal->cuda_dl;
2385 
2386  for (int i = 0; i < planes; i++) {
2387  if (internal->cu_sem[i])
2388  CHECK_CU(cu->cuDestroyExternalSemaphore(internal->cu_sem[i]));
2389  if (internal->cu_mma[i])
2390  CHECK_CU(cu->cuMipmappedArrayDestroy(internal->cu_mma[i]));
2391  if (internal->ext_mem[i])
2392  CHECK_CU(cu->cuDestroyExternalMemory(internal->ext_mem[i]));
2393 #ifdef _WIN32
2394  if (internal->ext_sem_handle[i])
2395  CloseHandle(internal->ext_sem_handle[i]);
2396  if (internal->ext_mem_handle[i])
2397  CloseHandle(internal->ext_mem_handle[i]);
2398 #endif
2399  }
2400 
2401  av_buffer_unref(&internal->cuda_fc_ref);
2402  }
2403 #endif
2404 
2405  pthread_mutex_destroy(&internal->update_mutex);
2406  av_freep(&f->internal);
2407 }
2408 
2409 static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
2410 {
2411  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2412  AVVulkanDeviceContext *hwctx = &p->p;
2413  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2414  int nb_images = ff_vk_count_images(f);
2415  int nb_sems = 0;
2416 
2417  while (nb_sems < FF_ARRAY_ELEMS(f->sem) && f->sem[nb_sems])
2418  nb_sems++;
2419 
2420  if (nb_sems) {
2421  VkSemaphoreWaitInfo sem_wait = {
2422  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
2423  .flags = 0x0,
2424  .pSemaphores = f->sem,
2425  .pValues = f->sem_value,
2426  .semaphoreCount = nb_sems,
2427  };
2428 
2429  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
2430  }
2431 
2432  vulkan_free_internal(f);
2433 
2434  for (int i = 0; i < nb_images; i++) {
2435  vk->DestroyImage(hwctx->act_dev, f->img[i], hwctx->alloc);
2436  vk->FreeMemory(hwctx->act_dev, f->mem[i], hwctx->alloc);
2437  vk->DestroySemaphore(hwctx->act_dev, f->sem[i], hwctx->alloc);
2438  }
2439 
2440  av_free(f);
2441 }
2442 
2443 static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
2444 {
2445  vulkan_frame_free(opaque, (AVVkFrame*)data);
2446 }
2447 
2448 static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f,
2449  void *alloc_pnext, size_t alloc_pnext_stride)
2450 {
2451  int img_cnt = 0, err;
2452  VkResult ret;
2453  AVHWDeviceContext *ctx = hwfc->device_ctx;
2454  VulkanDevicePriv *p = ctx->hwctx;
2455  AVVulkanDeviceContext *hwctx = &p->p;
2456  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2457  VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } };
2458 
2459  while (f->img[img_cnt]) {
2460  int use_ded_mem;
2461  VkImageMemoryRequirementsInfo2 req_desc = {
2462  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
2463  .image = f->img[img_cnt],
2464  };
2465  VkMemoryDedicatedAllocateInfo ded_alloc = {
2466  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
2467  .pNext = (void *)(((uint8_t *)alloc_pnext) + img_cnt*alloc_pnext_stride),
2468  };
2469  VkMemoryDedicatedRequirements ded_req = {
2470  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
2471  };
2472  VkMemoryRequirements2 req = {
2473  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
2474  .pNext = &ded_req,
2475  };
2476 
2477  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req);
2478 
2479  if (f->tiling == VK_IMAGE_TILING_LINEAR)
2480  req.memoryRequirements.size = FFALIGN(req.memoryRequirements.size,
2481  p->props.properties.limits.minMemoryMapAlignment);
2482 
2483  /* In case the implementation prefers/requires dedicated allocation */
2484  use_ded_mem = ded_req.prefersDedicatedAllocation |
2485  ded_req.requiresDedicatedAllocation;
2486  if (use_ded_mem)
2487  ded_alloc.image = f->img[img_cnt];
2488 
2489  /* Allocate memory */
2490  if ((err = alloc_mem(ctx, &req.memoryRequirements,
2491  f->tiling == VK_IMAGE_TILING_LINEAR ?
2492  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT :
2493  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
2494  use_ded_mem ? &ded_alloc : (void *)ded_alloc.pNext,
2495  &f->flags, &f->mem[img_cnt])))
2496  return err;
2497 
2498  f->size[img_cnt] = req.memoryRequirements.size;
2499  bind_info[img_cnt].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
2500  bind_info[img_cnt].image = f->img[img_cnt];
2501  bind_info[img_cnt].memory = f->mem[img_cnt];
2502 
2503  img_cnt++;
2504  }
2505 
2506  /* Bind the allocated memory to the images */
2507  ret = vk->BindImageMemory2(hwctx->act_dev, img_cnt, bind_info);
2508  if (ret != VK_SUCCESS) {
2509  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
2510  ff_vk_ret2str(ret));
2511  return AVERROR_EXTERNAL;
2512  }
2513 
2514  return 0;
2515 }
2516 
2517 enum PrepMode {
2518  PREP_MODE_GENERAL,
2519  PREP_MODE_WRITE,
2520  PREP_MODE_EXTERNAL_EXPORT,
2521  PREP_MODE_EXTERNAL_IMPORT,
2522  PREP_MODE_DECODING_DST,
2523  PREP_MODE_DECODING_DPB,
2524  PREP_MODE_ENCODING_DPB,
2525 };
2526 
2527 static void switch_new_props(enum PrepMode pmode, VkImageLayout *new_layout,
2528  VkAccessFlags2 *new_access)
2529 {
2530  switch (pmode) {
2531  case PREP_MODE_GENERAL:
2532  *new_layout = VK_IMAGE_LAYOUT_GENERAL;
2533  *new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2534  break;
2535  case PREP_MODE_WRITE:
2536  *new_layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
2537  *new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2538  break;
2539  case PREP_MODE_EXTERNAL_IMPORT:
2540  *new_layout = VK_IMAGE_LAYOUT_GENERAL;
2541  *new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2542  break;
2543  case PREP_MODE_EXTERNAL_EXPORT:
2544  *new_layout = VK_IMAGE_LAYOUT_GENERAL;
2545  *new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2546  break;
2547  case PREP_MODE_DECODING_DST:
2548  *new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR;
2549  *new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2550  break;
2551  case PREP_MODE_DECODING_DPB:
2552  *new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
2553  *new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2554  break;
2555  case PREP_MODE_ENCODING_DPB:
2556  *new_layout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
2557  *new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2558  break;
2559  }
2560 }
2561 
2562 static int switch_layout(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2563  AVVkFrame *frame, enum PrepMode pmode)
2564 {
2565  int err;
2566  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2567  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2568  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
2569  int nb_img_bar = 0;
2570 
2571  VkImageLayout new_layout;
2572  VkAccessFlags2 new_access;
2573  switch_new_props(pmode, &new_layout, &new_access);
2574 
2575  uint32_t dst_qf = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2576  VkPipelineStageFlagBits2 src_stage = VK_PIPELINE_STAGE_2_NONE;
2577  if (pmode == PREP_MODE_EXTERNAL_EXPORT) {
2578  dst_qf = VK_QUEUE_FAMILY_EXTERNAL_KHR;
2579  src_stage = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
2580  }
2581 
2582  /* This is dirty - but it works. The vulkan.c dependency system doesn't
2583  * free non-refcounted frames, and non-refcounted hardware frames cannot
2584  * happen anywhere outside of here. */
2585  AVBufferRef tmp_ref = {
2586  .data = (uint8_t *)hwfc,
2587  };
2588  AVFrame tmp_frame = {
2589  .data[0] = (uint8_t *)frame,
2590  .hw_frames_ctx = &tmp_ref,
2591  };
2592 
2593  VkCommandBuffer cmd_buf;
2594  FFVkExecContext *exec = ff_vk_exec_get(&p->vkctx, ectx);
2595  cmd_buf = exec->buf;
2596  ff_vk_exec_start(&p->vkctx, exec);
2597 
2598  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, &tmp_frame,
2599  VK_PIPELINE_STAGE_2_NONE,
2600  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
2601  if (err < 0)
2602  return err;
2603 
2604  ff_vk_frame_barrier(&p->vkctx, exec, &tmp_frame, img_bar, &nb_img_bar,
2605  src_stage,
2606  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
2607  new_access, new_layout, dst_qf);
2608 
2609  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
2610  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
2611  .pImageMemoryBarriers = img_bar,
2612  .imageMemoryBarrierCount = nb_img_bar,
2613  });
2614 
2615  err = ff_vk_exec_submit(&p->vkctx, exec);
2616  if (err < 0)
2617  return err;
2618 
2619  /* We can do this because there are no real dependencies */
2620  ff_vk_exec_discard_deps(&p->vkctx, exec);
2621 
2622  return 0;
2623 }
2624 
2625 static int switch_layout_host(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2626  AVVkFrame *frame, enum PrepMode pmode)
2627 {
2628  VkResult ret;
2629  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2630  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2631  VkHostImageLayoutTransitionInfoEXT layout_change[AV_NUM_DATA_POINTERS];
2632  int nb_images = ff_vk_count_images(frame);
2633 
2634  VkImageLayout new_layout;
2635  VkAccessFlags2 new_access;
2636  switch_new_props(pmode, &new_layout, &new_access);
2637 
2638  int i;
2639  for (i = 0; i < p->vkctx.host_image_props.copyDstLayoutCount; i++) {
2640  if (p->vkctx.host_image_props.pCopyDstLayouts[i] == new_layout)
2641  break;
2642  }
2643  if (i == p->vkctx.host_image_props.copyDstLayoutCount)
2644  return AVERROR(ENOTSUP);
2645 
2646  for (i = 0; i < nb_images; i++) {
2647  layout_change[i] = (VkHostImageLayoutTransitionInfoEXT) {
2648  .sType = VK_STRUCTURE_TYPE_HOST_IMAGE_LAYOUT_TRANSITION_INFO_EXT,
2649  .image = frame->img[i],
2650  .oldLayout = frame->layout[i],
2651  .newLayout = new_layout,
2652  .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
2653  .subresourceRange.layerCount = 1,
2654  .subresourceRange.levelCount = 1,
2655  };
2656  frame->layout[i] = new_layout;
2657  }
2658 
2659  ret = vk->TransitionImageLayoutEXT(p->vkctx.hwctx->act_dev,
2660  nb_images, layout_change);
2661  if (ret != VK_SUCCESS) {
2662  av_log(hwfc, AV_LOG_ERROR, "Unable to prepare frame: %s\n",
2663  ff_vk_ret2str(ret));
2664  return AVERROR_EXTERNAL;
2665  }
2666 
2667  return 0;
2668 }
2669 
2670 static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2671  AVVkFrame *frame, enum PrepMode pmode)
2672 {
2673  int err = 0;
2674  AVVulkanFramesContext *hwfc_vk = hwfc->hwctx;
2675  if (hwfc_vk->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT &&
2676  (pmode != PREP_MODE_EXTERNAL_EXPORT) &&
2677  (pmode != PREP_MODE_EXTERNAL_IMPORT))
2678  err = switch_layout_host(hwfc, ectx, frame, pmode);
2679 
2680  if (err != AVERROR(ENOTSUP))
2681  return err;
2682 
2683  return switch_layout(hwfc, ectx, frame, pmode);
2684 }
2685 
2686 static inline void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format,
2687  int frame_w, int frame_h, int plane)
2688 {
2689  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(format);
2690 
2691  /* Currently always true unless gray + alpha support is added */
2692  if (!plane || (plane == 3) || desc->flags & AV_PIX_FMT_FLAG_RGB ||
2693  !(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) {
2694  *w = frame_w;
2695  *h = frame_h;
2696  return;
2697  }
2698 
2699  *w = AV_CEIL_RSHIFT(frame_w, desc->log2_chroma_w);
2700  *h = AV_CEIL_RSHIFT(frame_h, desc->log2_chroma_h);
2701 }
2702 
2703 static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame,
2704  VkImageTiling tiling, VkImageUsageFlagBits usage,
2705  VkImageCreateFlags flags, int nb_layers,
2706  void *create_pnext)
2707 {
2708  int err;
2709  VkResult ret;
2710  AVVulkanFramesContext *hwfc_vk = hwfc->hwctx;
2711  AVHWDeviceContext *ctx = hwfc->device_ctx;
2712  VulkanDevicePriv *p = ctx->hwctx;
2713  AVVulkanDeviceContext *hwctx = &p->p;
2714  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2715  AVVkFrame *f;
2716 
2717  VkSemaphoreTypeCreateInfo sem_type_info = {
2718  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
2719  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
2720  .initialValue = 0,
2721  };
2722  VkSemaphoreCreateInfo sem_spawn = {
2723  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
2724  .pNext = &sem_type_info,
2725  };
2726 
2727  VkExportSemaphoreCreateInfo ext_sem_info_opaque = {
2728  .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
2729 #ifdef _WIN32
2730  .handleTypes = IsWindows8OrGreater()
2731  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
2732  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
2733 #else
2734  .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
2735 #endif
2736  };
2737 
2738  /* Check if exporting is supported before chaining any structs */
2739  if (p->ext_sem_props_opaque.externalSemaphoreFeatures & VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT) {
2740  if (p->vkctx.extensions & (FF_VK_EXT_EXTERNAL_WIN32_SEM | FF_VK_EXT_EXTERNAL_FD_SEM))
2741  ff_vk_link_struct(&sem_type_info, &ext_sem_info_opaque);
2742  }
2743 
2744  f = av_vk_frame_alloc();
2745  if (!f) {
2746  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
2747  return AVERROR(ENOMEM);
2748  }
2749 
2750  // TODO: check width and height for alignment in case of multiplanar (must be mod-2 if subsampled)
2751 
2752  /* Create the images */
2753  for (int i = 0; (hwfc_vk->format[i] != VK_FORMAT_UNDEFINED); i++) {
2754  VkImageCreateInfo create_info = {
2755  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
2756  .pNext = create_pnext,
2757  .imageType = VK_IMAGE_TYPE_2D,
2758  .format = hwfc_vk->format[i],
2759  .extent.depth = 1,
2760  .mipLevels = 1,
2761  .arrayLayers = nb_layers,
2762  .flags = flags,
2763  .tiling = tiling,
2764  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
2765  .usage = usage,
2766  .samples = VK_SAMPLE_COUNT_1_BIT,
2767  .pQueueFamilyIndices = p->img_qfs,
2768  .queueFamilyIndexCount = p->nb_img_qfs,
2769  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2770  VK_SHARING_MODE_EXCLUSIVE,
2771  };
2772 
2773  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
2774  hwfc->sw_format, hwfc->width, hwfc->height, i);
2775 
2776  ret = vk->CreateImage(hwctx->act_dev, &create_info,
2777  hwctx->alloc, &f->img[i]);
2778  if (ret != VK_SUCCESS) {
2779  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
2780  ff_vk_ret2str(ret));
2781  err = AVERROR(EINVAL);
2782  goto fail;
2783  }
2784 
2785  /* Create semaphore */
2786  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
2787  hwctx->alloc, &f->sem[i]);
2788  if (ret != VK_SUCCESS) {
2789  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
2790  ff_vk_ret2str(ret));
2791  err = AVERROR_EXTERNAL;
2792  goto fail;
2793  }
2794 
2795  f->queue_family[i] = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2796  f->layout[i] = create_info.initialLayout;
2797  f->access[i] = 0x0;
2798  f->sem_value[i] = 0;
2799  }
2800 
2801  f->flags = 0x0;
2802  f->tiling = tiling;
2803 
2804  *frame = f;
2805  return 0;
2806 
2807 fail:
2808  vulkan_frame_free(hwfc, f);
2809  return err;
2810 }
2811 
2812 /* Checks if an export flag is enabled, and if it is ORs it with *iexp */
2813 static void try_export_flags(AVHWFramesContext *hwfc,
2814  VkExternalMemoryHandleTypeFlags *comp_handle_types,
2815  VkExternalMemoryHandleTypeFlags *iexp,
2816  VkExternalMemoryHandleTypeFlagBits exp)
2817 {
2818  VkResult ret;
2819  AVVulkanFramesContext *hwctx = hwfc->hwctx;
2820  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2821  AVVulkanDeviceContext *dev_hwctx = &p->p;
2822  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2823 
2824  const VkImageDrmFormatModifierListCreateInfoEXT *drm_mod_info =
2825  ff_vk_find_struct(hwctx->create_pnext,
2826  VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
2827  int has_mods = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT && drm_mod_info;
2828  int nb_mods;
2829 
2830  VkExternalImageFormatProperties eprops = {
2831  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
2832  };
2833  VkImageFormatProperties2 props = {
2834  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
2835  .pNext = &eprops,
2836  };
2837  VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
2838  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
2839  .pNext = NULL,
2840  .pQueueFamilyIndices = p->img_qfs,
2841  .queueFamilyIndexCount = p->nb_img_qfs,
2842  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2843  VK_SHARING_MODE_EXCLUSIVE,
2844  };
2845  VkPhysicalDeviceExternalImageFormatInfo enext = {
2846  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
2847  .handleType = exp,
2848  .pNext = has_mods ? &phy_dev_mod_info : NULL,
2849  };
2850  VkPhysicalDeviceImageFormatInfo2 pinfo = {
2851  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
2852  .pNext = !exp ? NULL : &enext,
2853  .format = vk_find_format_entry(hwfc->sw_format)->vkf,
2854  .type = VK_IMAGE_TYPE_2D,
2855  .tiling = hwctx->tiling,
2856  .usage = hwctx->usage,
2857  .flags = VK_IMAGE_CREATE_ALIAS_BIT,
2858  };
2859 
2860  nb_mods = has_mods ? drm_mod_info->drmFormatModifierCount : 1;
2861  for (int i = 0; i < nb_mods; i++) {
2862  if (has_mods)
2863  phy_dev_mod_info.drmFormatModifier = drm_mod_info->pDrmFormatModifiers[i];
2864 
2865  ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
2866  &pinfo, &props);
2867 
2868  if (ret == VK_SUCCESS) {
2869  *iexp |= exp;
2870  *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
2871  }
2872  }
2873 }
2874 
2875 static AVBufferRef *vulkan_pool_alloc(void *opaque, size_t size)
2876 {
2877  int err;
2878  AVVkFrame *f;
2879  AVBufferRef *avbuf = NULL;
2880  AVHWFramesContext *hwfc = opaque;
2881  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2882  VulkanFramesPriv *fp = hwfc->hwctx;
2883  AVVulkanFramesContext *hwctx = &fp->p;
2884  VkExternalMemoryHandleTypeFlags e = 0x0;
2885  VkExportMemoryAllocateInfo eminfo[AV_NUM_DATA_POINTERS];
2886 
2887  VkExternalMemoryImageCreateInfo eiinfo = {
2888  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
2889  .pNext = hwctx->create_pnext,
2890  };
2891 
2892 #ifdef _WIN32
2893  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY)
2894  try_export_flags(hwfc, &eiinfo.handleTypes, &e, IsWindows8OrGreater()
2895  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
2896  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT);
2897 #else
2898  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY)
2899  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2900  VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT);
2901 
2902  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_DMABUF_MEMORY &&
2903  hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
2904  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2905  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
2906 #endif
2907 
2908  for (int i = 0; i < av_pix_fmt_count_planes(hwfc->sw_format); i++) {
2909  eminfo[i].sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
2910  eminfo[i].pNext = hwctx->alloc_pnext[i];
2911  eminfo[i].handleTypes = e;
2912  }
2913 
2914  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
2915  hwctx->nb_layers,
2916  eiinfo.handleTypes ? &eiinfo : hwctx->create_pnext);
2917  if (err)
2918  return NULL;
2919 
2920  err = alloc_bind_mem(hwfc, f, eminfo, sizeof(*eminfo));
2921  if (err)
2922  goto fail;
2923 
2924  if ( (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2925  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR))
2926  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DPB);
2927  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)
2928  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DST);
2929  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR)
2930  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_ENCODING_DPB);
2931  else if (hwctx->usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
2932  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_WRITE);
2933  else
2934  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_GENERAL);
2935  if (err)
2936  goto fail;
2937 
2938  avbuf = av_buffer_create((uint8_t *)f, sizeof(AVVkFrame),
2939  vulkan_frame_free_cb, hwfc, 0);
2940  if (!avbuf)
2941  goto fail;
2942 
2943  return avbuf;
2944 
2945 fail:
2946  vulkan_frame_free(hwfc, f);
2947  return NULL;
2948 }
2949 
2950 static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2951 {
2952  pthread_mutex_lock(&vkf->internal->update_mutex);
2953 }
2954 
2955 static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2956 {
2957  pthread_mutex_unlock(&vkf->internal->update_mutex);
2958 }
2959 
2960 static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
2961 {
2962  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2963  VulkanFramesPriv *fp = hwfc->hwctx;
2964 
2965  if (fp->modifier_info) {
2966  if (fp->modifier_info->pDrmFormatModifiers)
2967  av_freep(&fp->modifier_info->pDrmFormatModifiers);
2968  av_freep(&fp->modifier_info);
2969  }
2970 
2971  ff_vk_exec_pool_free(&p->vkctx, &fp->compute_exec);
2972  ff_vk_exec_pool_free(&p->vkctx, &fp->upload_exec);
2973  ff_vk_exec_pool_free(&p->vkctx, &fp->download_exec);
2974 
2975  av_buffer_pool_uninit(&fp->tmp);
2976 }
2977 
2978 static int vulkan_frames_init(AVHWFramesContext *hwfc)
2979 {
2980  int err;
2981  AVVkFrame *f;
2982  VulkanFramesPriv *fp = hwfc->hwctx;
2983  AVVulkanFramesContext *hwctx = &fp->p;
2984  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2985  AVVulkanDeviceContext *dev_hwctx = &p->p;
2986  VkImageUsageFlags supported_usage;
2987  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2988  const struct FFVkFormatEntry *fmt;
2989  int disable_multiplane = p->disable_multiplane ||
2990  (hwctx->flags & AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE);
2991  int is_lone_dpb = ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) ||
2992  ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2993  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)));
2994 
2995  /* Defaults */
2996  if (!hwctx->nb_layers)
2997  hwctx->nb_layers = 1;
2998 
2999  /* VK_IMAGE_TILING_OPTIMAL == 0, can't check for it really */
3000  if (p->use_linear_images &&
3001  (hwctx->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT))
3002  hwctx->tiling = VK_IMAGE_TILING_LINEAR;
3003 
3004 
3005  fmt = vk_find_format_entry(hwfc->sw_format);
3006  if (!fmt) {
3007  av_log(hwfc, AV_LOG_ERROR, "Unsupported pixel format: %s!\n",
3008  av_get_pix_fmt_name(hwfc->sw_format));
3009  return AVERROR(EINVAL);
3010  }
3011 
3012  if (hwctx->format[0] != VK_FORMAT_UNDEFINED) {
3013  if (hwctx->format[0] != fmt->vkf) {
3014  for (int i = 0; i < fmt->nb_images_fallback; i++) {
3015  if (hwctx->format[i] != fmt->fallback[i]) {
3016  av_log(hwfc, AV_LOG_ERROR, "Incompatible Vulkan format given "
3017  "for the current sw_format %s!\n",
3018  av_get_pix_fmt_name(hwfc->sw_format));
3019  return AVERROR(EINVAL);
3020  }
3021  }
3022  }
3023 
3024  /* Check if the sw_format itself is supported */
3025  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
3026  hwctx->tiling, NULL,
3027  NULL, NULL, &supported_usage, 0,
3028  !hwctx->usage ||
3029  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
3030  if (err < 0) {
3031  av_log(hwfc, AV_LOG_ERROR, "Unsupported sw format: %s!\n",
3032  av_get_pix_fmt_name(hwfc->sw_format));
3033  return AVERROR(EINVAL);
3034  }
3035  } else {
3036  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
3037  hwctx->tiling, hwctx->format, NULL,
3038  NULL, &supported_usage,
3039  disable_multiplane,
3040  !hwctx->usage ||
3041  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
3042  if (err < 0)
3043  return err;
3044  }
3045 
3046  /* Lone DPB images do not need additional flags. */
3047  if (!is_lone_dpb) {
3048  /* Image usage flags */
3049  hwctx->usage |= supported_usage & (VK_IMAGE_USAGE_TRANSFER_DST_BIT |
3050  VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
3051  VK_IMAGE_USAGE_STORAGE_BIT |
3052  VK_IMAGE_USAGE_SAMPLED_BIT);
3053 
3054  if (p->vkctx.extensions & FF_VK_EXT_HOST_IMAGE_COPY &&
3055  !(p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY) &&
3056  !(p->dprops.driverID == VK_DRIVER_ID_MOLTENVK))
3057  hwctx->usage |= supported_usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT;
3058 
3059  /* Enables encoding of images, if supported by format and extensions */
3060  if ((supported_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
3061  (p->vkctx.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
3062  (p->vkctx.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1))
3063  hwctx->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
3064 
3065  /* Image creation flags.
3066  * Only fill them in automatically if the image is not going to be used as
3067  * a DPB-only image, and we have SAMPLED/STORAGE bits set. */
3068  if (!hwctx->img_flags) {
3069  int sampleable = hwctx->usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
3070  VK_IMAGE_USAGE_STORAGE_BIT);
3071  hwctx->img_flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
3072  if (sampleable) {
3073  hwctx->img_flags |= VK_IMAGE_CREATE_ALIAS_BIT;
3074  if ((fmt->vk_planes > 1) && (hwctx->format[0] == fmt->vkf))
3075  hwctx->img_flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
3076  }
3077  }
3078  }
3079 
3080  /* If the image has an ENCODE_SRC usage, and the maintenance1
3081  * extension is supported, check if it has a profile list.
3082  * If there's no profile list, or it has no encode operations,
3083  * then allow creating the image with no specific profile. */
3084  if ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
3085  (p->vkctx.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
3086  (p->vkctx.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1)) {
3087  const VkVideoProfileListInfoKHR *pl;
3088  pl = ff_vk_find_struct(hwctx->create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
3089  if (!pl) {
3090  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
3091  } else {
3092  uint32_t i;
3093  for (i = 0; i < pl->profileCount; i++) {
3094  /* Video ops start at exactly 0x00010000 */
3095  if (pl->pProfiles[i].videoCodecOperation & 0xFFFF0000)
3096  break;
3097  }
3098  if (i == pl->profileCount)
3099  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
3100  }
3101  }
3102 
3103  if (!hwctx->lock_frame)
3104  hwctx->lock_frame = lock_frame;
3105 
3106  if (!hwctx->unlock_frame)
3107  hwctx->unlock_frame = unlock_frame;
3108 
3109  err = ff_vk_exec_pool_init(&p->vkctx, p->compute_qf, &fp->compute_exec,
3110  p->compute_qf->num, 0, 0, 0, NULL);
3111  if (err)
3112  return err;
3113 
3114  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->upload_exec,
3115  p->transfer_qf->num*2, 0, 0, 0, NULL);
3116  if (err)
3117  return err;
3118 
3119  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->download_exec,
3120  p->transfer_qf->num, 0, 0, 0, NULL);
3121  if (err)
3122  return err;
3123 
3124  /* Test to see if allocation will fail */
3125  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
3126  hwctx->nb_layers, hwctx->create_pnext);
3127  if (err)
3128  return err;
3129 
3130  /* Collect `VkDrmFormatModifierPropertiesEXT` for each plane. Required for DRM export. */
3131  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS && hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
3132  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
3133  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
3134  };
3135  err = vk->GetImageDrmFormatModifierPropertiesEXT(dev_hwctx->act_dev, f->img[0],
3136  &drm_mod);
3137  if (err != VK_SUCCESS) {
3138  av_log(hwfc, AV_LOG_ERROR, "Failed to get image DRM format modifier properties");
3139  vulkan_frame_free(hwfc, f);
3140  return AVERROR_EXTERNAL;
3141  }
3142  for (int i = 0; i < fmt->vk_planes; ++i) {
3143  VkDrmFormatModifierPropertiesListEXT modp;
3144  VkFormatProperties2 fmtp;
3145  VkDrmFormatModifierPropertiesEXT *mod_props = NULL;
3146 
3147  modp = (VkDrmFormatModifierPropertiesListEXT) {
3148  .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
3149  };
3150  fmtp = (VkFormatProperties2) {
3151  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
3152  .pNext = &modp,
3153  };
3154 
3155  /* query drmFormatModifierCount by keeping pDrmFormatModifierProperties NULL */
3156  vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt->fallback[i], &fmtp);
3157 
3158  modp.pDrmFormatModifierProperties =
3159  av_calloc(modp.drmFormatModifierCount, sizeof(*modp.pDrmFormatModifierProperties));
3160  if (!modp.pDrmFormatModifierProperties) {
3161  vulkan_frame_free(hwfc, f);
3162  return AVERROR(ENOMEM);
3163  }
3164  vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt->fallback[i], &fmtp);
3165 
3166  for (uint32_t i = 0; i < modp.drmFormatModifierCount; ++i) {
3167  VkDrmFormatModifierPropertiesEXT *m = &modp.pDrmFormatModifierProperties[i];
3168  if (m->drmFormatModifier == drm_mod.drmFormatModifier) {
3169  mod_props = m;
3170  break;
3171  }
3172  }
3173 
3174  if (mod_props == NULL) {
3175  av_log(hwfc, AV_LOG_ERROR, "No DRM format modifier properties found for modifier 0x%016"PRIx64"\n",
3176  drm_mod.drmFormatModifier);
3177  av_free(modp.pDrmFormatModifierProperties);
3178  vulkan_frame_free(hwfc, f);
3179  return AVERROR_EXTERNAL;
3180  }
3181 
3182  fp->drm_format_modifier_properties[i] = *mod_props;
3183  av_free(modp.pDrmFormatModifierProperties);
3184  }
3185  }
3186 
3187  vulkan_frame_free(hwfc, f);
3188 
3189  /* If user did not specify a pool, hwfc->pool will be set to the internal one
3190  * in hwcontext.c just after this gets called */
3191  if (!hwfc->pool) {
3192  ffhwframesctx(hwfc)->pool_internal = av_buffer_pool_init2(sizeof(AVVkFrame),
3193  hwfc, vulkan_pool_alloc,
3194  NULL);
3195  if (!ffhwframesctx(hwfc)->pool_internal)
3196  return AVERROR(ENOMEM);
3197  }
3198 
3199  return 0;
3200 }
3201 
3202 static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
3203 {
3204  frame->buf[0] = av_buffer_pool_get(hwfc->pool);
3205  if (!frame->buf[0])
3206  return AVERROR(ENOMEM);
3207 
3208  frame->data[0] = frame->buf[0]->data;
3209  frame->format = AV_PIX_FMT_VULKAN;
3210  frame->width = hwfc->width;
3211  frame->height = hwfc->height;
3212 
3213  return 0;
3214 }
3215 
3216 static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc,
3217  enum AVHWFrameTransferDirection dir,
3218  enum AVPixelFormat **formats)
3219 {
3220  enum AVPixelFormat *fmts;
3221  int n = 2;
3222 
3223 #if CONFIG_CUDA
3224  n++;
3225 #endif
3226  fmts = av_malloc_array(n, sizeof(*fmts));
3227  if (!fmts)
3228  return AVERROR(ENOMEM);
3229 
3230  n = 0;
3231  fmts[n++] = hwfc->sw_format;
3232 #if CONFIG_CUDA
3233  fmts[n++] = AV_PIX_FMT_CUDA;
3234 #endif
3235  fmts[n++] = AV_PIX_FMT_NONE;
3236 
3237  *formats = fmts;
3238  return 0;
3239 }
3240 
3241 #if CONFIG_LIBDRM
3242 static void vulkan_unmap_from_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
3243 {
3244  vulkan_frame_free(hwfc, hwmap->priv);
3245 }
3246 
3247 static const struct {
3248  uint32_t drm_fourcc;
3249  VkFormat vk_format;
3250 } vulkan_drm_format_map[] = {
3251  { DRM_FORMAT_R8, VK_FORMAT_R8_UNORM },
3252  { DRM_FORMAT_R16, VK_FORMAT_R16_UNORM },
3253  { DRM_FORMAT_GR88, VK_FORMAT_R8G8_UNORM },
3254  { DRM_FORMAT_RG88, VK_FORMAT_R8G8_UNORM },
3255  { DRM_FORMAT_GR1616, VK_FORMAT_R16G16_UNORM },
3256  { DRM_FORMAT_RG1616, VK_FORMAT_R16G16_UNORM },
3257  { DRM_FORMAT_ARGB8888, VK_FORMAT_B8G8R8A8_UNORM },
3258  { DRM_FORMAT_XRGB8888, VK_FORMAT_B8G8R8A8_UNORM },
3259  { DRM_FORMAT_ABGR8888, VK_FORMAT_R8G8B8A8_UNORM },
3260  { DRM_FORMAT_XBGR8888, VK_FORMAT_R8G8B8A8_UNORM },
3261  { DRM_FORMAT_ARGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
3262  { DRM_FORMAT_ABGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
3263  { DRM_FORMAT_XRGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
3264  { DRM_FORMAT_XBGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
3265 
3266  // All these DRM_FORMATs were added in the same libdrm commit.
3267 #ifdef DRM_FORMAT_XYUV8888
3268  { DRM_FORMAT_XYUV8888, VK_FORMAT_R8G8B8A8_UNORM },
3269  { DRM_FORMAT_XVYU2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 } ,
3270  { DRM_FORMAT_XVYU12_16161616, VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16 } ,
3271  { DRM_FORMAT_XVYU16161616, VK_FORMAT_R16G16B16A16_UNORM } ,
3272 #endif
3273 };
3274 
3275 static inline VkFormat drm_to_vulkan_fmt(uint32_t drm_fourcc)
3276 {
3277  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
3278  if (vulkan_drm_format_map[i].drm_fourcc == drm_fourcc)
3279  return vulkan_drm_format_map[i].vk_format;
3280  return VK_FORMAT_UNDEFINED;
3281 }
3282 
3283 static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **frame,
3284  const AVFrame *src, int flags)
3285 {
3286  int err = 0;
3287  VkResult ret;
3288  AVVkFrame *f;
3289  int bind_counts = 0;
3290  AVHWDeviceContext *ctx = hwfc->device_ctx;
3291  VulkanDevicePriv *p = ctx->hwctx;
3292  AVVulkanDeviceContext *hwctx = &p->p;
3293  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3294  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3295  VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES];
3296  VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES];
3297 
3298  for (int i = 0; i < desc->nb_layers; i++) {
3299  if (drm_to_vulkan_fmt(desc->layers[i].format) == VK_FORMAT_UNDEFINED) {
3300  av_log(ctx, AV_LOG_ERROR, "Unsupported DMABUF layer format %#08x!\n",
3301  desc->layers[i].format);
3302  return AVERROR(EINVAL);
3303  }
3304  }
3305 
3306  if (!(f = av_vk_frame_alloc())) {
3307  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
3308  err = AVERROR(ENOMEM);
3309  goto fail;
3310  }
3311 
3312  f->tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
3313 
3314  for (int i = 0; i < desc->nb_layers; i++) {
3315  const int planes = desc->layers[i].nb_planes;
3316 
3317  /* Semaphore */
3318  VkSemaphoreTypeCreateInfo sem_type_info = {
3319  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3320  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
3321  .initialValue = 0,
3322  };
3323  VkSemaphoreCreateInfo sem_spawn = {
3324  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3325  .pNext = &sem_type_info,
3326  };
3327 
3328  /* Image creation */
3329  VkSubresourceLayout ext_img_layouts[AV_DRM_MAX_PLANES];
3330  VkImageDrmFormatModifierExplicitCreateInfoEXT ext_img_mod_spec = {
3331  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT,
3332  .drmFormatModifier = desc->objects[0].format_modifier,
3333  .drmFormatModifierPlaneCount = planes,
3334  .pPlaneLayouts = (const VkSubresourceLayout *)&ext_img_layouts,
3335  };
3336  VkExternalMemoryImageCreateInfo ext_img_spec = {
3337  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
3338  .pNext = &ext_img_mod_spec,
3339  .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3340  };
3341  VkImageCreateInfo create_info = {
3342  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
3343  .pNext = &ext_img_spec,
3344  .imageType = VK_IMAGE_TYPE_2D,
3345  .format = drm_to_vulkan_fmt(desc->layers[i].format),
3346  .extent.depth = 1,
3347  .mipLevels = 1,
3348  .arrayLayers = 1,
3349  .flags = 0x0,
3350  .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
3351  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, /* specs say so */
3352  .usage = 0x0, /* filled in below */
3353  .samples = VK_SAMPLE_COUNT_1_BIT,
3354  .pQueueFamilyIndices = p->img_qfs,
3355  .queueFamilyIndexCount = p->nb_img_qfs,
3356  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
3357  VK_SHARING_MODE_EXCLUSIVE,
3358  };
3359 
3360  /* Image format verification */
3361  VkExternalImageFormatProperties ext_props = {
3362  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
3363  };
3364  VkImageFormatProperties2 props_ret = {
3365  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
3366  .pNext = &ext_props,
3367  };
3368  VkPhysicalDeviceImageDrmFormatModifierInfoEXT props_drm_mod = {
3369  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
3370  .drmFormatModifier = ext_img_mod_spec.drmFormatModifier,
3371  .pQueueFamilyIndices = create_info.pQueueFamilyIndices,
3372  .queueFamilyIndexCount = create_info.queueFamilyIndexCount,
3373  .sharingMode = create_info.sharingMode,
3374  };
3375  VkPhysicalDeviceExternalImageFormatInfo props_ext = {
3376  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
3377  .pNext = &props_drm_mod,
3378  .handleType = ext_img_spec.handleTypes,
3379  };
3380  VkPhysicalDeviceImageFormatInfo2 fmt_props;
3381 
3382  if (flags & AV_HWFRAME_MAP_READ)
3383  create_info.usage |= VK_IMAGE_USAGE_SAMPLED_BIT |
3384  VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
3385  if (flags & AV_HWFRAME_MAP_WRITE)
3386  create_info.usage |= VK_IMAGE_USAGE_STORAGE_BIT |
3387  VK_IMAGE_USAGE_TRANSFER_DST_BIT;
3388 
3389  fmt_props = (VkPhysicalDeviceImageFormatInfo2) {
3390  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
3391  .pNext = &props_ext,
3392  .format = create_info.format,
3393  .type = create_info.imageType,
3394  .tiling = create_info.tiling,
3395  .usage = create_info.usage,
3396  .flags = create_info.flags,
3397  };
3398 
3399  /* Check if importing is possible for this combination of parameters */
3400  ret = vk->GetPhysicalDeviceImageFormatProperties2(hwctx->phys_dev,
3401  &fmt_props, &props_ret);
3402  if (ret != VK_SUCCESS) {
3403  av_log(ctx, AV_LOG_ERROR, "Cannot map DRM frame to Vulkan: %s\n",
3404  ff_vk_ret2str(ret));
3405  err = AVERROR_EXTERNAL;
3406  goto fail;
3407  }
3408 
3409  /* Set the image width/height */
3410  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
3411  hwfc->sw_format, src->width, src->height, i);
3412 
3413  /* Set the subresource layout based on the layer properties */
3414  for (int j = 0; j < planes; j++) {
3415  ext_img_layouts[j].offset = desc->layers[i].planes[j].offset;
3416  ext_img_layouts[j].rowPitch = desc->layers[i].planes[j].pitch;
3417  ext_img_layouts[j].size = 0; /* The specs say so for all 3 */
3418  ext_img_layouts[j].arrayPitch = 0;
3419  ext_img_layouts[j].depthPitch = 0;
3420  }
3421 
3422  /* Create image */
3423  ret = vk->CreateImage(hwctx->act_dev, &create_info,
3424  hwctx->alloc, &f->img[i]);
3425  if (ret != VK_SUCCESS) {
3426  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
3427  ff_vk_ret2str(ret));
3428  err = AVERROR(EINVAL);
3429  goto fail;
3430  }
3431 
3432  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3433  hwctx->alloc, &f->sem[i]);
3434  if (ret != VK_SUCCESS) {
3435  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3436  ff_vk_ret2str(ret));
3437  err = AVERROR_EXTERNAL;
3438  goto fail;
3439  }
3440 
3441  f->queue_family[i] = VK_QUEUE_FAMILY_EXTERNAL;
3442  f->layout[i] = create_info.initialLayout;
3443  f->access[i] = 0x0;
3444  f->sem_value[i] = 0;
3445  }
3446 
3447  for (int i = 0; i < desc->nb_layers; i++) {
3448  /* Memory requirements */
3449  VkImageMemoryRequirementsInfo2 req_desc = {
3450  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
3451  .image = f->img[i],
3452  };
3453  VkMemoryDedicatedRequirements ded_req = {
3454  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
3455  };
3456  VkMemoryRequirements2 req2 = {
3457  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
3458  .pNext = &ded_req,
3459  };
3460 
3461  /* Allocation/importing */
3462  VkMemoryFdPropertiesKHR fdmp = {
3463  .sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR,
3464  };
3465  /* This assumes that a layer will never be constructed from multiple
3466  * objects. If that was to happen in the real world, this code would
3467  * need to import each plane separately.
3468  */
3469  VkImportMemoryFdInfoKHR idesc = {
3470  .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
3471  .fd = dup(desc->objects[desc->layers[i].planes[0].object_index].fd),
3472  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3473  };
3474  VkMemoryDedicatedAllocateInfo ded_alloc = {
3475  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
3476  .pNext = &idesc,
3477  .image = req_desc.image,
3478  };
3479 
3480  /* Get object properties */
3481  ret = vk->GetMemoryFdPropertiesKHR(hwctx->act_dev,
3482  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3483  idesc.fd, &fdmp);
3484  if (ret != VK_SUCCESS) {
3485  av_log(hwfc, AV_LOG_ERROR, "Failed to get FD properties: %s\n",
3486  ff_vk_ret2str(ret));
3487  err = AVERROR_EXTERNAL;
3488  close(idesc.fd);
3489  goto fail;
3490  }
3491 
3492  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req2);
3493 
3494  /* Only a single bit must be set, not a range, and it must match */
3495  req2.memoryRequirements.memoryTypeBits = fdmp.memoryTypeBits;
3496 
3497  err = alloc_mem(ctx, &req2.memoryRequirements,
3498  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
3499  (ded_req.prefersDedicatedAllocation ||
3500  ded_req.requiresDedicatedAllocation) ?
3501  &ded_alloc : ded_alloc.pNext,
3502  &f->flags, &f->mem[i]);
3503  if (err) {
3504  close(idesc.fd);
3505  return err;
3506  }
3507 
3508  f->size[i] = req2.memoryRequirements.size;
3509  }
3510 
3511  for (int i = 0; i < desc->nb_layers; i++) {
3512  const int planes = desc->layers[i].nb_planes;
3513  for (int j = 0; j < planes; j++) {
3514  VkImageAspectFlagBits aspect = j == 0 ? VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT :
3515  j == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3516  VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
3517 
3518  plane_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO;
3519  plane_info[bind_counts].pNext = NULL;
3520  plane_info[bind_counts].planeAspect = aspect;
3521 
3522  bind_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
3523  bind_info[bind_counts].pNext = planes > 1 ? &plane_info[bind_counts] : NULL;
3524  bind_info[bind_counts].image = f->img[i];
3525  bind_info[bind_counts].memory = f->mem[i];
3526 
3527  /* Offset is already signalled via pPlaneLayouts above */
3528  bind_info[bind_counts].memoryOffset = 0;
3529 
3530  bind_counts++;
3531  }
3532  }
3533 
3534  /* Bind the allocated memory to the images */
3535  ret = vk->BindImageMemory2(hwctx->act_dev, bind_counts, bind_info);
3536  if (ret != VK_SUCCESS) {
3537  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
3538  ff_vk_ret2str(ret));
3539  err = AVERROR_EXTERNAL;
3540  goto fail;
3541  }
3542 
3543  *frame = f;
3544 
3545  return 0;
3546 
3547 fail:
3548  vulkan_frame_free(hwfc, f);
3549 
3550  return err;
3551 }
3552 
3553 static int vulkan_map_from_drm_frame_sync(AVHWFramesContext *hwfc, AVFrame *dst,
3554  const AVDRMFrameDescriptor *desc, int flags)
3555 {
3556  int err;
3557  VkResult ret;
3558  AVHWDeviceContext *ctx = hwfc->device_ctx;
3559  VulkanDevicePriv *p = ctx->hwctx;
3560  VulkanFramesPriv *fp = hwfc->hwctx;
3561  AVVulkanDeviceContext *hwctx = &p->p;
3562  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3563 
3564 #ifdef DMA_BUF_IOCTL_EXPORT_SYNC_FILE
3565  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM) {
3566  VkCommandBuffer cmd_buf;
3567  FFVkExecContext *exec;
3568  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
3569  VkSemaphore drm_sync_sem[AV_DRM_MAX_PLANES] = { 0 };
3570  int nb_img_bar = 0;
3571 
3572  for (int i = 0; i < desc->nb_objects; i++) {
3573  VkSemaphoreTypeCreateInfo sem_type_info = {
3574  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3575  .semaphoreType = VK_SEMAPHORE_TYPE_BINARY,
3576  };
3577  VkSemaphoreCreateInfo sem_spawn = {
3578  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3579  .pNext = &sem_type_info,
3580  };
3581  VkImportSemaphoreFdInfoKHR import_info;
3582  struct dma_buf_export_sync_file implicit_fd_info = {
3583  .flags = DMA_BUF_SYNC_READ,
3584  .fd = -1,
3585  };
3586 
3587  if (ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE,
3588  &implicit_fd_info)) {
3589  err = AVERROR(errno);
3590  av_log(hwctx, AV_LOG_ERROR, "Failed to retrieve implicit DRM sync file: %s\n",
3591  av_err2str(err));
3592  for (; i >= 0; i--)
3593  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3594  return err;
3595  }
3596 
3597  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3598  hwctx->alloc, &drm_sync_sem[i]);
3599  if (ret != VK_SUCCESS) {
3600  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3601  ff_vk_ret2str(ret));
3602  err = AVERROR_EXTERNAL;
3603  for (; i >= 0; i--)
3604  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3605  return err;
3606  }
3607 
3608  import_info = (VkImportSemaphoreFdInfoKHR) {
3609  .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
3610  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3611  .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT,
3612  .semaphore = drm_sync_sem[i],
3613  .fd = implicit_fd_info.fd,
3614  };
3615 
3616  ret = vk->ImportSemaphoreFdKHR(hwctx->act_dev, &import_info);
3617  if (ret != VK_SUCCESS) {
3618  av_log(hwctx, AV_LOG_ERROR, "Failed to import semaphore: %s\n",
3619  ff_vk_ret2str(ret));
3620  err = AVERROR_EXTERNAL;
3621  for (; i >= 0; i--)
3622  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3623  return err;
3624  }
3625  }
3626 
3627  exec = ff_vk_exec_get(&p->vkctx, &fp->compute_exec);
3628  cmd_buf = exec->buf;
3629 
3630  ff_vk_exec_start(&p->vkctx, exec);
3631 
3632  /* Ownership of semaphores is passed */
3633  err = ff_vk_exec_add_dep_bool_sem(&p->vkctx, exec,
3634  drm_sync_sem, desc->nb_objects,
3635  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 1);
3636  if (err < 0)
3637  return err;
3638 
3639  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, dst,
3640  VK_PIPELINE_STAGE_2_NONE,
3641  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
3642  if (err < 0)
3643  return err;
3644 
3645  ff_vk_frame_barrier(&p->vkctx, exec, dst, img_bar, &nb_img_bar,
3646  VK_PIPELINE_STAGE_2_NONE,
3647  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
3648  ((flags & AV_HWFRAME_MAP_READ) ?
3649  VK_ACCESS_2_SHADER_SAMPLED_READ_BIT : 0x0) |
3650  ((flags & AV_HWFRAME_MAP_WRITE) ?
3651  VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0x0),
3652  VK_IMAGE_LAYOUT_GENERAL,
3653  p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0]);
3654 
3655  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
3656  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3657  .pImageMemoryBarriers = img_bar,
3658  .imageMemoryBarrierCount = nb_img_bar,
3659  });
3660 
3661  err = ff_vk_exec_submit(&p->vkctx, exec);
3662  if (err < 0)
3663  return err;
3664  } else
3665 #endif
3666  {
3667  AVVkFrame *f = (AVVkFrame *)dst->data[0];
3668  av_log(hwctx, AV_LOG_WARNING, "No support for synchronization when importing DMA-BUFs, "
3669  "image may be corrupted.\n");
3670  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3671  if (err)
3672  return err;
3673  }
3674 
3675  return 0;
3676 }
3677 
3678 static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3679  const AVFrame *src, int flags)
3680 {
3681  int err = 0;
3682  AVVkFrame *f;
3683  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3684 
3685  if ((err = vulkan_map_from_drm_frame_desc(hwfc, &f, src, flags)))
3686  return err;
3687 
3688  /* The unmapping function will free this */
3689  dst->data[0] = (uint8_t *)f;
3690  dst->width = src->width;
3691  dst->height = src->height;
3692 
3693  err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
3694  &vulkan_unmap_from_drm, f);
3695  if (err < 0)
3696  goto fail;
3697 
3698  err = vulkan_map_from_drm_frame_sync(hwfc, dst, desc, flags);
3699  if (err < 0)
3700  return err;
3701 
3702  av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n");
3703 
3704  return 0;
3705 
3706 fail:
3707  vulkan_frame_free(hwfc->device_ctx->hwctx, f);
3708  dst->data[0] = NULL;
3709  return err;
3710 }
3711 
3712 #if CONFIG_VAAPI
3713 static int vulkan_map_from_vaapi(AVHWFramesContext *dst_fc,
3714  AVFrame *dst, const AVFrame *src,
3715  int flags)
3716 {
3717  int err;
3718  AVFrame *tmp = av_frame_alloc();
3719  AVHWFramesContext *vaapi_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3720  AVVAAPIDeviceContext *vaapi_ctx = vaapi_fc->device_ctx->hwctx;
3721  VASurfaceID surface_id = (VASurfaceID)(uintptr_t)src->data[3];
3722 
3723  if (!tmp)
3724  return AVERROR(ENOMEM);
3725 
3726  /* We have to sync since like the previous comment said, no semaphores */
3727  vaSyncSurface(vaapi_ctx->display, surface_id);
3728 
3729  tmp->format = AV_PIX_FMT_DRM_PRIME;
3730 
3731  err = av_hwframe_map(tmp, src, flags);
3732  if (err < 0)
3733  goto fail;
3734 
3735  err = vulkan_map_from_drm(dst_fc, dst, tmp, flags);
3736  if (err < 0)
3737  goto fail;
3738 
3739  err = ff_hwframe_map_replace(dst, src);
3740 
3741 fail:
3742  av_frame_free(&tmp);
3743  return err;
3744 }
3745 #endif
3746 #endif
3747 
3748 #if CONFIG_CUDA
3749 static int export_mem_to_cuda(AVHWDeviceContext *ctx,
3750  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3751  AVVkFrameInternal *dst_int, int idx,
3752  VkDeviceMemory mem, size_t size)
3753 {
3754  VkResult ret;
3755  VulkanDevicePriv *p = ctx->hwctx;
3756  AVVulkanDeviceContext *hwctx = &p->p;
3757  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3758 
3759 #ifdef _WIN32
3760  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3761  .type = IsWindows8OrGreater()
3762  ? CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32
3763  : CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT,
3764  .size = size,
3765  };
3766  VkMemoryGetWin32HandleInfoKHR export_info = {
3767  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR,
3768  .memory = mem,
3769  .handleType = IsWindows8OrGreater()
3770  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
3771  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3772  };
3773 
3774  ret = vk->GetMemoryWin32HandleKHR(hwctx->act_dev, &export_info,
3775  &ext_desc.handle.win32.handle);
3776  if (ret != VK_SUCCESS) {
3777  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a Win32 Handle: %s!\n",
3778  ff_vk_ret2str(ret));
3779  return AVERROR_EXTERNAL;
3780  }
3781  dst_int->ext_mem_handle[idx] = ext_desc.handle.win32.handle;
3782 #else
3783  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3784  .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD,
3785  .size = size,
3786  };
3787  VkMemoryGetFdInfoKHR export_info = {
3788  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
3789  .memory = mem,
3790  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR,
3791  };
3792 
3793  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
3794  &ext_desc.handle.fd);
3795  if (ret != VK_SUCCESS) {
3796  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a FD: %s!\n",
3797  ff_vk_ret2str(ret));
3798  return AVERROR_EXTERNAL;
3799  }
3800 #endif
3801 
3802  ret = CHECK_CU(cu->cuImportExternalMemory(&dst_int->ext_mem[idx], &ext_desc));
3803  if (ret < 0) {
3804 #ifndef _WIN32
3805  close(ext_desc.handle.fd);
3806 #endif
3807  return AVERROR_EXTERNAL;
3808  }
3809 
3810  return 0;
3811 }
3812 
3813 static int export_sem_to_cuda(AVHWDeviceContext *ctx,
3814  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3815  AVVkFrameInternal *dst_int, int idx,
3816  VkSemaphore sem)
3817 {
3818  VkResult ret;
3819  VulkanDevicePriv *p = ctx->hwctx;
3820  AVVulkanDeviceContext *hwctx = &p->p;
3821  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3822 
3823 #ifdef _WIN32
3824  VkSemaphoreGetWin32HandleInfoKHR sem_export = {
3825  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR,
3826  .semaphore = sem,
3827  .handleType = IsWindows8OrGreater()
3828  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
3829  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3830  };
3831  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3832  .type = 10 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32 */,
3833  };
3834 #else
3835  VkSemaphoreGetFdInfoKHR sem_export = {
3836  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
3837  .semaphore = sem,
3838  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
3839  };
3840  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3841  .type = 9 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD */,
3842  };
3843 #endif
3844 
3845 #ifdef _WIN32
3846  ret = vk->GetSemaphoreWin32HandleKHR(hwctx->act_dev, &sem_export,
3847  &ext_sem_desc.handle.win32.handle);
3848 #else
3849  ret = vk->GetSemaphoreFdKHR(hwctx->act_dev, &sem_export,
3850  &ext_sem_desc.handle.fd);
3851 #endif
3852  if (ret != VK_SUCCESS) {
3853  av_log(ctx, AV_LOG_ERROR, "Failed to export semaphore: %s\n",
3854  ff_vk_ret2str(ret));
3855  return AVERROR_EXTERNAL;
3856  }
3857 #ifdef _WIN32
3858  dst_int->ext_sem_handle[idx] = ext_sem_desc.handle.win32.handle;
3859 #endif
3860 
3861  ret = CHECK_CU(cu->cuImportExternalSemaphore(&dst_int->cu_sem[idx],
3862  &ext_sem_desc));
3863  if (ret < 0) {
3864 #ifndef _WIN32
3865  close(ext_sem_desc.handle.fd);
3866 #endif
3867  return AVERROR_EXTERNAL;
3868  }
3869 
3870  return 0;
3871 }
3872 
3873 static int vulkan_export_to_cuda(AVHWFramesContext *hwfc,
3874  AVBufferRef *cuda_hwfc,
3875  const AVFrame *frame)
3876 {
3877  int err;
3878  VkResult ret;
3879  AVVkFrame *dst_f;
3880  AVVkFrameInternal *dst_int;
3881  AVHWDeviceContext *ctx = hwfc->device_ctx;
3882  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3883  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3884  VulkanDevicePriv *p = ctx->hwctx;
3885  AVVulkanDeviceContext *hwctx = &p->p;
3886  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3887  int nb_images;
3888 
3889  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)cuda_hwfc->data;
3890  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3891  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3892  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3893  CudaFunctions *cu = cu_internal->cuda_dl;
3894  CUarray_format cufmt = desc->comp[0].depth > 8 ? CU_AD_FORMAT_UNSIGNED_INT16 :
3895  CU_AD_FORMAT_UNSIGNED_INT8;
3896 
3897  dst_f = (AVVkFrame *)frame->data[0];
3898  dst_int = dst_f->internal;
3899 
3900  if (!dst_int->cuda_fc_ref) {
3901  size_t offsets[3] = { 0 };
3902 
3903  dst_int->cuda_fc_ref = av_buffer_ref(cuda_hwfc);
3904  if (!dst_int->cuda_fc_ref)
3905  return AVERROR(ENOMEM);
3906 
3907  nb_images = ff_vk_count_images(dst_f);
3908  for (int i = 0; i < nb_images; i++) {
3909  err = export_mem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3910  dst_f->mem[i], dst_f->size[i]);
3911  if (err < 0)
3912  goto fail;
3913 
3914  err = export_sem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3915  dst_f->sem[i]);
3916  if (err < 0)
3917  goto fail;
3918  }
3919 
3920  if (nb_images != planes) {
3921  for (int i = 0; i < planes; i++) {
3922  VkImageSubresource subres = {
3923  .aspectMask = i == 2 ? VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT :
3924  i == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3925  VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT
3926  };
3927  VkSubresourceLayout layout = { 0 };
3928  vk->GetImageSubresourceLayout(hwctx->act_dev, dst_f->img[FFMIN(i, nb_images - 1)],
3929  &subres, &layout);
3930  offsets[i] = layout.offset;
3931  }
3932  }
3933 
3934  for (int i = 0; i < planes; i++) {
3935  CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC tex_desc = {
3936  .offset = offsets[i],
3937  .arrayDesc = {
3938  .Depth = 0,
3939  .Format = cufmt,
3940  .NumChannels = 1 + ((planes == 2) && i),
3941  .Flags = 0,
3942  },
3943  .numLevels = 1,
3944  };
3945  int p_w, p_h;
3946 
3947  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
3948  tex_desc.arrayDesc.Width = p_w;
3949  tex_desc.arrayDesc.Height = p_h;
3950 
3951  ret = CHECK_CU(cu->cuExternalMemoryGetMappedMipmappedArray(&dst_int->cu_mma[i],
3952  dst_int->ext_mem[FFMIN(i, nb_images - 1)],
3953  &tex_desc));
3954  if (ret < 0) {
3955  err = AVERROR_EXTERNAL;
3956  goto fail;
3957  }
3958 
3959  ret = CHECK_CU(cu->cuMipmappedArrayGetLevel(&dst_int->cu_array[i],
3960  dst_int->cu_mma[i], 0));
3961  if (ret < 0) {
3962  err = AVERROR_EXTERNAL;
3963  goto fail;
3964  }
3965 
3966  }
3967  }
3968 
3969  return 0;
3970 
3971 fail:
3972  vulkan_free_internal(dst_f);
3973  return err;
3974 }
3975 
3976 static int vulkan_transfer_data_from_cuda(AVHWFramesContext *hwfc,
3977  AVFrame *dst, const AVFrame *src)
3978 {
3979  int err;
3980  CUcontext dummy;
3981  AVVkFrame *dst_f;
3982  AVVkFrameInternal *dst_int;
3983  VulkanFramesPriv *fp = hwfc->hwctx;
3984  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3985  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3986 
3987  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3988  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3989  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3990  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3991  CudaFunctions *cu = cu_internal->cuda_dl;
3992  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
3993  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
3994 
3995  dst_f = (AVVkFrame *)dst->data[0];
3996 
3997  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
3998  if (err < 0)
3999  return err;
4000 
4001  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
4002  if (err < 0)
4003  return err;
4004 
4005  err = vulkan_export_to_cuda(hwfc, src->hw_frames_ctx, dst);
4006  if (err < 0) {
4007  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4008  return err;
4009  }
4010 
4011  dst_int = dst_f->internal;
4012 
4013  for (int i = 0; i < planes; i++) {
4014  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
4015  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
4016  }
4017 
4018  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
4019  planes, cuda_dev->stream));
4020  if (err < 0)
4021  goto fail;
4022 
4023  for (int i = 0; i < planes; i++) {
4024  CUDA_MEMCPY2D cpy = {
4025  .srcMemoryType = CU_MEMORYTYPE_DEVICE,
4026  .srcDevice = (CUdeviceptr)src->data[i],
4027  .srcPitch = src->linesize[i],
4028  .srcY = 0,
4029 
4030  .dstMemoryType = CU_MEMORYTYPE_ARRAY,
4031  .dstArray = dst_int->cu_array[i],
4032  };
4033 
4034  int p_w, p_h;
4035  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
4036 
4037  cpy.WidthInBytes = p_w * desc->comp[i].step;
4038  cpy.Height = p_h;
4039 
4040  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
4041  if (err < 0)
4042  goto fail;
4043  }
4044 
4045  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
4046  planes, cuda_dev->stream));
4047  if (err < 0)
4048  goto fail;
4049 
4050  for (int i = 0; i < planes; i++)
4051  dst_f->sem_value[i]++;
4052 
4053  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4054 
4055  av_log(hwfc, AV_LOG_VERBOSE, "Transferred CUDA image to Vulkan!\n");
4056 
4057  return err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
4058 
4059 fail:
4060  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4061  vulkan_free_internal(dst_f);
4062  av_buffer_unref(&dst->buf[0]);
4063  return err;
4064 }
4065 #endif
4066 
4067 static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
4068  const AVFrame *src, int flags)
4069 {
4070  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4071 
4072  switch (src->format) {
4073 #if CONFIG_LIBDRM
4074 #if CONFIG_VAAPI
4075  case AV_PIX_FMT_VAAPI:
4076  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4077  return vulkan_map_from_vaapi(hwfc, dst, src, flags);
4078  else
4079  return AVERROR(ENOSYS);
4080 #endif
4081  case AV_PIX_FMT_DRM_PRIME:
4082  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4083  return vulkan_map_from_drm(hwfc, dst, src, flags);
4084  else
4085  return AVERROR(ENOSYS);
4086 #endif
4087  default:
4088  return AVERROR(ENOSYS);
4089  }
4090 }
4091 
4092 #if CONFIG_LIBDRM
4093 typedef struct VulkanDRMMapping {
4094  AVDRMFrameDescriptor drm_desc;
4095  AVVkFrame *source;
4096 } VulkanDRMMapping;
4097 
4098 static void vulkan_unmap_to_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
4099 {
4100  AVDRMFrameDescriptor *drm_desc = hwmap->priv;
4101 
4102  /* on unmap from DRM, make sure to import sync objects so that we are sync'd with any work that was
4103  * done on the buffer while exported. We don't know if who used the dmabuf did reads or writes, so protect against both */
4104  vulkan_map_from_drm_frame_sync(hwfc, hwmap->source, drm_desc, AV_HWFRAME_MAP_READ | AV_HWFRAME_MAP_WRITE);
4105 
4106  for (int i = 0; i < drm_desc->nb_objects; i++)
4107  close(drm_desc->objects[i].fd);
4108 
4109  av_free(drm_desc);
4110 }
4111 
4112 static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt)
4113 {
4114  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
4115  if (vulkan_drm_format_map[i].vk_format == vkfmt)
4116  return vulkan_drm_format_map[i].drm_fourcc;
4117  return DRM_FORMAT_INVALID;
4118 }
4119 
4120 #define MAX_MEMORY_PLANES 4
4121 static VkImageAspectFlags plane_index_to_aspect(int plane) {
4122  if (plane == 0) return VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
4123  if (plane == 1) return VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT;
4124  if (plane == 2) return VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
4125  if (plane == 3) return VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT;
4126 
4127  av_assert2 (0 && "Invalid plane index");
4128  return VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
4129 }
4130 
4131 static int vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
4132  const AVFrame *src, int flags)
4133 {
4134  int err = 0;
4135  VkResult ret;
4136  AVVkFrame *f = (AVVkFrame *)src->data[0];
4137  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4138  AVVulkanDeviceContext *hwctx = &p->p;
4139  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4140  VulkanFramesPriv *fp = hwfc->hwctx;
4141  const int nb_images = ff_vk_count_images(f);
4142  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
4143  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
4144  };
4145  VkSemaphoreWaitInfo wait_info = {
4146  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
4147  .flags = 0x0,
4148  .semaphoreCount = nb_images,
4149  };
4150 
4151  AVDRMFrameDescriptor *drm_desc = av_mallocz(sizeof(*drm_desc));
4152  if (!drm_desc)
4153  return AVERROR(ENOMEM);
4154 
4155  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_EXPORT);
4156  if (err < 0)
4157  goto end;
4158 
4159  /* Wait for the operation to finish so we can cleanly export it. */
4160  wait_info.pSemaphores = f->sem;
4161  wait_info.pValues = f->sem_value;
4162 
4163  vk->WaitSemaphores(hwctx->act_dev, &wait_info, UINT64_MAX);
4164 
4165  err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, &vulkan_unmap_to_drm, drm_desc);
4166  if (err < 0)
4167  goto end;
4168 
4169  ret = vk->GetImageDrmFormatModifierPropertiesEXT(hwctx->act_dev, f->img[0],
4170  &drm_mod);
4171  if (ret != VK_SUCCESS) {
4172  av_log(hwfc, AV_LOG_ERROR, "Failed to retrieve DRM format modifier!\n");
4173  err = AVERROR_EXTERNAL;
4174  goto end;
4175  }
4176 
4177  for (int i = 0; (i < nb_images) && (f->mem[i]); i++) {
4178  VkMemoryGetFdInfoKHR export_info = {
4179  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
4180  .memory = f->mem[i],
4181  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
4182  };
4183 
4184  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
4185  &drm_desc->objects[i].fd);
4186  if (ret != VK_SUCCESS) {
4187  av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a FD!\n");
4188  err = AVERROR_EXTERNAL;
4189  goto end;
4190  }
4191 
4192  drm_desc->nb_objects++;
4193  drm_desc->objects[i].size = f->size[i];
4194  drm_desc->objects[i].format_modifier = drm_mod.drmFormatModifier;
4195  }
4196 
4197  drm_desc->nb_layers = nb_images;
4198  for (int i = 0; i < drm_desc->nb_layers; i++) {
4199  VkFormat plane_vkfmt = av_vkfmt_from_pixfmt(hwfc->sw_format)[i];
4200 
4201  drm_desc->layers[i].format = vulkan_fmt_to_drm(plane_vkfmt);
4202  drm_desc->layers[i].nb_planes = fp->drm_format_modifier_properties[i].drmFormatModifierPlaneCount;
4203 
4204  if (drm_desc->layers[i].nb_planes > MAX_MEMORY_PLANES) {
4205  av_log(hwfc, AV_LOG_ERROR, "Too many memory planes for DRM format!\n");
4206  err = AVERROR_EXTERNAL;
4207  goto end;
4208  }
4209 
4210  for (int j = 0; j < drm_desc->layers[i].nb_planes; j++) {
4211  VkSubresourceLayout layout;
4212  VkImageSubresource sub = {
4213  .aspectMask = plane_index_to_aspect(j),
4214  };
4215 
4216  drm_desc->layers[i].planes[j].object_index = FFMIN(i, drm_desc->nb_objects - 1);
4217 
4218  vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, &layout);
4219  drm_desc->layers[i].planes[j].offset = layout.offset;
4220  drm_desc->layers[i].planes[j].pitch = layout.rowPitch;
4221  }
4222 
4223  if (drm_desc->layers[i].format == DRM_FORMAT_INVALID) {
4224  av_log(hwfc, AV_LOG_ERROR, "Cannot map to DRM layer, unsupported!\n");
4225  err = AVERROR_PATCHWELCOME;
4226  goto end;
4227  }
4228 
4229 
4230  if (f->tiling == VK_IMAGE_TILING_OPTIMAL)
4231  continue;
4232 
4233  }
4234 
4235  dst->width = src->width;
4236  dst->height = src->height;
4237  dst->data[0] = (uint8_t *)drm_desc;
4238 
4239  av_log(hwfc, AV_LOG_VERBOSE, "Mapped AVVkFrame to a DRM object!\n");
4240 
4241  return 0;
4242 
4243 end:
4244  av_free(drm_desc);
4245  return err;
4246 }
4247 
4248 #if CONFIG_VAAPI
4249 static int vulkan_map_to_vaapi(AVHWFramesContext *hwfc, AVFrame *dst,
4250  const AVFrame *src, int flags)
4251 {
4252  int err;
4253  AVFrame *tmp = av_frame_alloc();
4254  if (!tmp)
4255  return AVERROR(ENOMEM);
4256 
4257  tmp->format = AV_PIX_FMT_DRM_PRIME;
4258 
4259  err = vulkan_map_to_drm(hwfc, tmp, src, flags);
4260  if (err < 0)
4261  goto fail;
4262 
4263  err = av_hwframe_map(dst, tmp, flags);
4264  if (err < 0)
4265  goto fail;
4266 
4267  err = ff_hwframe_map_replace(dst, src);
4268 
4269 fail:
4270  av_frame_free(&tmp);
4271  return err;
4272 }
4273 #endif
4274 #endif
4275 
4276 static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
4277  const AVFrame *src, int flags)
4278 {
4279  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4280 
4281  switch (dst->format) {
4282 #if CONFIG_LIBDRM
4283  case AV_PIX_FMT_DRM_PRIME:
4284  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4285  return vulkan_map_to_drm(hwfc, dst, src, flags);
4286  else
4287  return AVERROR(ENOSYS);
4288 #if CONFIG_VAAPI
4289  case AV_PIX_FMT_VAAPI:
4290  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4291  return vulkan_map_to_vaapi(hwfc, dst, src, flags);
4292  else
4293  return AVERROR(ENOSYS);
4294 #endif
4295 #endif
4296  default:
4297  break;
4298  }
4299  return AVERROR(ENOSYS);
4300 }
4301 
4302 static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf,
4303  AVFrame *swf, VkBufferImageCopy *region,
4304  int planes, int upload)
4305 {
4306  int err;
4307  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4308  FFVkBuffer *vkbuf = (FFVkBuffer *)buf->data;
4309 
4310  if (upload) {
4311  for (int i = 0; i < planes; i++)
4312  av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset,
4313  region[i].bufferRowLength,
4314  swf->data[i],
4315  swf->linesize[i],
4316  swf->linesize[i],
4317  region[i].imageExtent.height);
4318 
4319  err = ff_vk_flush_buffer(&p->vkctx, vkbuf, 0, VK_WHOLE_SIZE, 1);
4320  if (err != VK_SUCCESS) {
4321  av_log(hwfc, AV_LOG_ERROR, "Failed to flush buffer data: %s\n",
4322  av_err2str(err));
4323  return AVERROR_EXTERNAL;
4324  }
4325  } else {
4326  err = ff_vk_flush_buffer(&p->vkctx, vkbuf, 0, VK_WHOLE_SIZE, 0);
4327  if (err != VK_SUCCESS) {
4328  av_log(hwfc, AV_LOG_ERROR, "Failed to invalidate buffer data: %s\n",
4329  av_err2str(err));
4330  return AVERROR_EXTERNAL;
4331  }
4332 
4333  for (int i = 0; i < planes; i++)
4334  av_image_copy_plane(swf->data[i],
4335  swf->linesize[i],
4336  vkbuf->mapped_mem + region[i].bufferOffset,
4337  region[i].bufferRowLength,
4338  swf->linesize[i],
4339  region[i].imageExtent.height);
4340  }
4341 
4342  return 0;
4343 }
4344 
4345 static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst,
4346  AVFrame *swf, VkBufferImageCopy *region, int upload)
4347 {
4348  int err;
4349  uint32_t p_w, p_h;
4350  VulkanFramesPriv *fp = hwfc->hwctx;
4351  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4352  const int planes = av_pix_fmt_count_planes(swf->format);
4353  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4354  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4355 
4356  size_t buf_offset = 0;
4357  for (int i = 0; i < planes; i++) {
4358  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4359 
4360  region[i] = (VkBufferImageCopy) {
4361  .bufferOffset = buf_offset,
4362  .bufferRowLength = FFALIGN(swf->linesize[i],
4363  p->props.properties.limits.optimalBufferCopyRowPitchAlignment),
4364  .bufferImageHeight = p_h,
4365  .imageSubresource.layerCount = 1,
4366  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4367  /* Rest of the fields adjusted/filled in later */
4368  };
4369 
4370  buf_offset += FFALIGN(p_h*region[i].bufferRowLength,
4371  p->props.properties.limits.optimalBufferCopyOffsetAlignment);
4372  }
4373 
4374  err = ff_vk_get_pooled_buffer(&p->vkctx, &fp->tmp, dst, buf_usage,
4375  NULL, buf_offset,
4376  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
4377  p->vkctx.host_cached_flag);
4378  if (err < 0)
4379  return err;
4380 
4381  return 0;
4382 }
4383 
4384 static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs,
4385  AVFrame *swf, VkBufferImageCopy *region, int upload)
4386 {
4387  int err;
4388  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4389 
4390  int nb_src_bufs;
4391  const int planes = av_pix_fmt_count_planes(swf->format);
4392  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4393  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4394 
4395  /* We can't host map images with negative strides */
4396  for (int i = 0; i < planes; i++)
4397  if (swf->linesize[i] < 0)
4398  return AVERROR(EINVAL);
4399 
4400  /* Count the number of buffers in the software frame */
4401  nb_src_bufs = 0;
4402  while (swf->buf[nb_src_bufs])
4403  nb_src_bufs++;
4404 
4405  /* Single buffer contains all planes */
4406  if (nb_src_bufs == 1) {
4407  err = ff_vk_host_map_buffer(&p->vkctx, &dst[0],
4408  swf->data[0], swf->buf[0],
4409  buf_usage);
4410  if (err < 0)
4411  return err;
4412  (*nb_bufs)++;
4413 
4414  for (int i = 0; i < planes; i++)
4415  region[i].bufferOffset = ((FFVkBuffer *)dst[0]->data)->virtual_offset +
4416  swf->data[i] - swf->data[0];
4417  } else if (nb_src_bufs == planes) { /* One buffer per plane */
4418  for (int i = 0; i < planes; i++) {
4419  err = ff_vk_host_map_buffer(&p->vkctx, &dst[i],
4420  swf->data[i], swf->buf[i],
4421  buf_usage);
4422  if (err < 0)
4423  goto fail;
4424  (*nb_bufs)++;
4425 
4426  region[i].bufferOffset = ((FFVkBuffer *)dst[i]->data)->virtual_offset;
4427  }
4428  } else {
4429  /* Weird layout (3 planes, 2 buffers), patch welcome, fallback to copy */
4430  return AVERROR_PATCHWELCOME;
4431  }
4432 
4433  return 0;
4434 
4435 fail:
4436  for (int i = 0; i < (*nb_bufs); i++)
4437  av_buffer_unref(&dst[i]);
4438  return err;
4439 }
4440 
4441 static int vulkan_transfer_host(AVHWFramesContext *hwfc, AVFrame *hwf,
4442  AVFrame *swf, int upload)
4443 {
4444  VulkanFramesPriv *fp = hwfc->hwctx;
4445  AVVulkanFramesContext *hwfc_vk = &fp->p;
4446  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4447  AVVulkanDeviceContext *hwctx = &p->p;
4448  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4449 
4450  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4451  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format);
4452  const int planes = av_pix_fmt_count_planes(swf->format);
4453  const int nb_images = ff_vk_count_images(hwf_vk);
4454 
4455  VkSemaphoreWaitInfo sem_wait;
4456  VkHostImageLayoutTransitionInfoEXT layout_ch_info[AV_NUM_DATA_POINTERS];
4457  int nb_layout_ch = 0;
4458 
4459  hwfc_vk->lock_frame(hwfc, hwf_vk);
4460 
4461  for (int i = 0; i < nb_images; i++) {
4462  int compat = 0;
4463  for (int j = 0; j < p->vkctx.host_image_props.copySrcLayoutCount; j++) {
4464  if (hwf_vk->layout[i] == p->vkctx.host_image_props.pCopySrcLayouts[j]) {
4465  compat = 1;
4466  break;
4467  }
4468  }
4469  if (compat)
4470  continue;
4471 
4472  layout_ch_info[nb_layout_ch] = (VkHostImageLayoutTransitionInfoEXT) {
4473  .sType = VK_STRUCTURE_TYPE_HOST_IMAGE_LAYOUT_TRANSITION_INFO_EXT,
4474  .image = hwf_vk->img[i],
4475  .oldLayout = hwf_vk->layout[i],
4476  .newLayout = VK_IMAGE_LAYOUT_GENERAL,
4477  .subresourceRange = {
4478  .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
4479  .levelCount = 1,
4480  .layerCount = 1,
4481  },
4482  };
4483 
4484  hwf_vk->layout[i] = layout_ch_info[nb_layout_ch].newLayout;
4485  nb_layout_ch++;
4486  }
4487 
4488  sem_wait = (VkSemaphoreWaitInfo) {
4489  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
4490  .pSemaphores = hwf_vk->sem,
4491  .pValues = hwf_vk->sem_value,
4492  .semaphoreCount = nb_images,
4493  };
4494 
4495  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
4496 
4497  if (nb_layout_ch)
4498  vk->TransitionImageLayoutEXT(hwctx->act_dev,
4499  nb_layout_ch, layout_ch_info);
4500 
4501  if (upload) {
4502  VkMemoryToImageCopyEXT region_info = {
4503  .sType = VK_STRUCTURE_TYPE_MEMORY_TO_IMAGE_COPY_EXT,
4504  .imageSubresource = {
4505  .layerCount = 1,
4506  },
4507  };
4508  VkCopyMemoryToImageInfoEXT copy_info = {
4509  .sType = VK_STRUCTURE_TYPE_COPY_MEMORY_TO_IMAGE_INFO_EXT,
4510  .regionCount = 1,
4511  .pRegions = &region_info,
4512  };
4513  for (int i = 0; i < planes; i++) {
4514  int img_idx = FFMIN(i, (nb_images - 1));
4515  uint32_t p_w, p_h;
4516  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4517 
4518  region_info.pHostPointer = swf->data[i];
4519  region_info.memoryRowLength = swf->linesize[i] / desc->comp[i].step;
4520  region_info.imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4521  region_info.imageExtent = (VkExtent3D){ p_w, p_h, 1 };
4522  copy_info.dstImage = hwf_vk->img[img_idx];
4523  copy_info.dstImageLayout = hwf_vk->layout[img_idx];
4524 
4525  vk->CopyMemoryToImageEXT(hwctx->act_dev, &copy_info);
4526  }
4527  } else {
4528  VkImageToMemoryCopyEXT region_info = {
4529  .sType = VK_STRUCTURE_TYPE_IMAGE_TO_MEMORY_COPY_EXT,
4530  .imageSubresource = {
4531  .layerCount = 1,
4532  },
4533  };
4534  VkCopyImageToMemoryInfoEXT copy_info = {
4535  .sType = VK_STRUCTURE_TYPE_COPY_IMAGE_TO_MEMORY_INFO_EXT,
4536  .regionCount = 1,
4537  .pRegions = &region_info,
4538  };
4539  for (int i = 0; i < planes; i++) {
4540  int img_idx = FFMIN(i, (nb_images - 1));
4541  uint32_t p_w, p_h;
4542  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4543 
4544  region_info.pHostPointer = swf->data[i];
4545  region_info.memoryRowLength = swf->linesize[i] / desc->comp[i].step;
4546  region_info.imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4547  region_info.imageExtent = (VkExtent3D){ p_w, p_h, 1 };
4548  copy_info.srcImage = hwf_vk->img[img_idx];
4549  copy_info.srcImageLayout = hwf_vk->layout[img_idx];
4550 
4551  vk->CopyImageToMemoryEXT(hwctx->act_dev, &copy_info);
4552  }
4553  }
4554 
4555  hwfc_vk->unlock_frame(hwfc, hwf_vk);
4556 
4557  return 0;
4558 }
4559 
4560 static int vulkan_transfer_frame(AVHWFramesContext *hwfc,
4561  AVFrame *swf, AVFrame *hwf,
4562  int upload)
4563 {
4564  int err;
4565  VulkanFramesPriv *fp = hwfc->hwctx;
4566  AVVulkanFramesContext *hwctx = &fp->p;
4567  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4568  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4569 
4570  int host_mapped = 0;
4571 
4572  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4573  VkBufferImageCopy region[AV_NUM_DATA_POINTERS]; // always one per plane
4574 
4575  const int planes = av_pix_fmt_count_planes(swf->format);
4576  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format);
4577  const int nb_images = ff_vk_count_images(hwf_vk);
4578 
4579  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
4580  int nb_img_bar = 0;
4581 
4582  AVBufferRef *bufs[AV_NUM_DATA_POINTERS];
4583  int nb_bufs = 0;
4584 
4585  VkCommandBuffer cmd_buf;
4586  FFVkExecContext *exec;
4587 
4588  /* Sanity checking */
4589  if ((swf->format != AV_PIX_FMT_NONE && !av_vkfmt_from_pixfmt(swf->format))) {
4590  av_log(hwfc, AV_LOG_ERROR, "Unsupported software frame pixel format!\n");
4591  return AVERROR(EINVAL);
4592  }
4593 
4594  if (swf->width > hwfc->width || swf->height > hwfc->height)
4595  return AVERROR(EINVAL);
4596 
4597  if (hwctx->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT &&
4598  !(p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY))
4599  return vulkan_transfer_host(hwfc, hwf, swf, upload);
4600 
4601  for (int i = 0; i < av_pix_fmt_count_planes(swf->format); i++) {
4602  uint32_t p_w, p_h;
4603  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4604 
4605  /* Buffer region for this plane */
4606  region[i] = (VkBufferImageCopy) {
4607  .bufferOffset = 0,
4608  .bufferRowLength = swf->linesize[i],
4609  .bufferImageHeight = p_h,
4610  .imageSubresource.layerCount = 1,
4611  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4612  /* Rest of the fields adjusted/filled in later */
4613  };
4614  }
4615 
4616  /* Setup buffers first */
4617  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY && !p->avoid_host_import) {
4618  err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload);
4619  if (err >= 0)
4620  host_mapped = 1;
4621  }
4622 
4623  if (!host_mapped) {
4624  err = get_plane_buf(hwfc, &bufs[0], swf, region, upload);
4625  if (err < 0)
4626  goto end;
4627  nb_bufs = 1;
4628 
4629  if (upload) {
4630  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 1);
4631  if (err < 0)
4632  goto end;
4633  }
4634  }
4635 
4636  exec = ff_vk_exec_get(&p->vkctx, &fp->upload_exec);
4637  cmd_buf = exec->buf;
4638 
4639  ff_vk_exec_start(&p->vkctx, exec);
4640 
4641  /* Prep destination Vulkan frame */
4642  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, hwf,
4643  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4644  VK_PIPELINE_STAGE_2_TRANSFER_BIT);
4645  if (err < 0)
4646  goto end;
4647 
4648  /* No need to declare buf deps for synchronous transfers (downloads) */
4649  if (upload) {
4650  /* Add the software frame backing the buffers if we're host mapping */
4651  if (host_mapped) {
4652  err = ff_vk_exec_add_dep_sw_frame(&p->vkctx, exec, swf);
4653  if (err < 0) {
4654  ff_vk_exec_discard_deps(&p->vkctx, exec);
4655  goto end;
4656  }
4657  }
4658 
4659  /* Add the buffers as a dependency */
4660  err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, nb_bufs, 1);
4661  if (err < 0) {
4662  ff_vk_exec_discard_deps(&p->vkctx, exec);
4663  goto end;
4664  }
4665  }
4666 
4667  ff_vk_frame_barrier(&p->vkctx, exec, hwf, img_bar, &nb_img_bar,
4668  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4669  VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR,
4670  upload ? VK_ACCESS_TRANSFER_WRITE_BIT :
4671  VK_ACCESS_TRANSFER_READ_BIT,
4672  upload ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL :
4673  VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
4674  p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0]);
4675 
4676  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
4677  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
4678  .pImageMemoryBarriers = img_bar,
4679  .imageMemoryBarrierCount = nb_img_bar,
4680  });
4681 
4682  for (int i = 0; i < planes; i++) {
4683  int buf_idx = FFMIN(i, (nb_bufs - 1));
4684  int img_idx = FFMIN(i, (nb_images - 1));
4685  FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[buf_idx]->data;
4686 
4687  uint32_t orig_stride = region[i].bufferRowLength;
4688  region[i].bufferRowLength /= desc->comp[i].step;
4689  region[i].imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4690 
4691  if (upload)
4692  vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf,
4693  hwf_vk->img[img_idx],
4694  img_bar[img_idx].newLayout,
4695  1, &region[i]);
4696  else
4697  vk->CmdCopyImageToBuffer(cmd_buf, hwf_vk->img[img_idx],
4698  img_bar[img_idx].newLayout,
4699  vkbuf->buf,
4700  1, &region[i]);
4701 
4702  region[i].bufferRowLength = orig_stride;
4703  }
4704 
4705  err = ff_vk_exec_submit(&p->vkctx, exec);
4706  if (err < 0) {
4707  ff_vk_exec_discard_deps(&p->vkctx, exec);
4708  } else if (!upload) {
4709  ff_vk_exec_wait(&p->vkctx, exec);
4710  if (!host_mapped)
4711  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 0);
4712  }
4713 
4714 end:
4715  for (int i = 0; i < nb_bufs; i++)
4716  av_buffer_unref(&bufs[i]);
4717 
4718  return err;
4719 }
4720 
4721 static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst,
4722  const AVFrame *src)
4723 {
4724  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4725 
4726  switch (src->format) {
4727 #if CONFIG_CUDA
4728  case AV_PIX_FMT_CUDA:
4729 #ifdef _WIN32
4730  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4731  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4732 #else
4733  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4734  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4735 #endif
4736  return vulkan_transfer_data_from_cuda(hwfc, dst, src);
4737 #endif
4738  default:
4739  if (src->hw_frames_ctx)
4740  return AVERROR(ENOSYS);
4741  else
4742  return vulkan_transfer_frame(hwfc, (AVFrame *)src, dst, 1);
4743  }
4744 }
4745 
4746 #if CONFIG_CUDA
4747 static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst,
4748  const AVFrame *src)
4749 {
4750  int err;
4751  CUcontext dummy;
4752  AVVkFrame *dst_f;
4753  AVVkFrameInternal *dst_int;
4754  VulkanFramesPriv *fp = hwfc->hwctx;
4755  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
4756  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
4757  int nb_images;
4758 
4759  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)dst->hw_frames_ctx->data;
4760  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
4761  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
4762  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
4763  CudaFunctions *cu = cu_internal->cuda_dl;
4764  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
4765  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
4766 
4767  dst_f = (AVVkFrame *)src->data[0];
4768  nb_images = ff_vk_count_images(dst_f);
4769 
4770  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
4771  if (err < 0)
4772  return err;
4773 
4774  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
4775  if (err < 0)
4776  return err;
4777 
4778  err = vulkan_export_to_cuda(hwfc, dst->hw_frames_ctx, src);
4779  if (err < 0) {
4780  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4781  return err;
4782  }
4783 
4784  dst_int = dst_f->internal;
4785 
4786  for (int i = 0; i < planes; i++) {
4787  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
4788  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
4789  }
4790 
4791  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
4792  nb_images, cuda_dev->stream));
4793  if (err < 0)
4794  goto fail;
4795 
4796  for (int i = 0; i < planes; i++) {
4797  CUDA_MEMCPY2D cpy = {
4798  .dstMemoryType = CU_MEMORYTYPE_DEVICE,
4799  .dstDevice = (CUdeviceptr)dst->data[i],
4800  .dstPitch = dst->linesize[i],
4801  .dstY = 0,
4802 
4803  .srcMemoryType = CU_MEMORYTYPE_ARRAY,
4804  .srcArray = dst_int->cu_array[i],
4805  };
4806 
4807  int w, h;
4808  get_plane_wh(&w, &h, hwfc->sw_format, hwfc->width, hwfc->height, i);
4809 
4810  cpy.WidthInBytes = w * desc->comp[i].step;
4811  cpy.Height = h;
4812 
4813  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
4814  if (err < 0)
4815  goto fail;
4816  }
4817 
4818  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
4819  nb_images, cuda_dev->stream));
4820  if (err < 0)
4821  goto fail;
4822 
4823  for (int i = 0; i < planes; i++)
4824  dst_f->sem_value[i]++;
4825 
4826  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4827 
4828  av_log(hwfc, AV_LOG_VERBOSE, "Transferred Vulkan image to CUDA!\n");
4829 
4830  return prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
4831 
4832 fail:
4833  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4834  vulkan_free_internal(dst_f);
4835  av_buffer_unref(&dst->buf[0]);
4836  return err;
4837 }
4838 #endif
4839 
4840 static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst,
4841  const AVFrame *src)
4842 {
4843  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4844 
4845  switch (dst->format) {
4846 #if CONFIG_CUDA
4847  case AV_PIX_FMT_CUDA:
4848 #ifdef _WIN32
4849  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4850  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4851 #else
4852  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4853  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4854 #endif
4855  return vulkan_transfer_data_to_cuda(hwfc, dst, src);
4856 #endif
4857  default:
4858  if (dst->hw_frames_ctx)
4859  return AVERROR(ENOSYS);
4860  else
4861  return vulkan_transfer_frame(hwfc, dst, (AVFrame *)src, 0);
4862  }
4863 }
4864 
4865 static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc,
4866  AVHWFramesContext *src_fc, int flags)
4867 {
4868  return vulkan_frames_init(dst_fc);
4869 }
4870 
4871 AVVkFrame *av_vk_frame_alloc(void)
4872 {
4873  int err;
4874  AVVkFrame *f = av_mallocz(sizeof(AVVkFrame));
4875  if (!f)
4876  return NULL;
4877 
4878  f->internal = av_mallocz(sizeof(*f->internal));
4879  if (!f->internal) {
4880  av_free(f);
4881  return NULL;
4882  }
4883 
4884  err = pthread_mutex_init(&f->internal->update_mutex, NULL);
4885  if (err != 0) {
4886  av_free(f->internal);
4887  av_free(f);
4888  return NULL;
4889  }
4890 
4891  return f;
4892 }
4893 
4894 const HWContextType ff_hwcontext_type_vulkan = {
4895  .type = AV_HWDEVICE_TYPE_VULKAN,
4896  .name = "Vulkan",
4897 
4898  .device_hwctx_size = sizeof(VulkanDevicePriv),
4899  .frames_hwctx_size = sizeof(VulkanFramesPriv),
4900 
4901  .device_init = &vulkan_device_init,
4902  .device_uninit = &vulkan_device_uninit,
4903  .device_create = &vulkan_device_create,
4904  .device_derive = &vulkan_device_derive,
4905 
4906  .frames_get_constraints = &vulkan_frames_get_constraints,
4907  .frames_init = vulkan_frames_init,
4908  .frames_get_buffer = vulkan_get_buffer,
4909  .frames_uninit = vulkan_frames_uninit,
4910 
4911  .transfer_get_formats = vulkan_transfer_get_formats,
4912  .transfer_data_to = vulkan_transfer_data_to,
4913  .transfer_data_from = vulkan_transfer_data_from,
4914 
4915  .map_to = vulkan_map_to,
4916  .map_from = vulkan_map_from,
4917  .frames_derive_to = &vulkan_frames_derive_to,
4918 
4919  .pix_fmts = (const enum AVPixelFormat []) {
4920  AV_PIX_FMT_VULKAN,
4921  AV_PIX_FMT_NONE
4922  },
4923 };
flags
const SwsFlags flags[]
Definition: swscale.c:71
vulkan_loader.h
formats
formats
Definition: signature.h:47
AV_PIX_FMT_YUVA422P16
#define AV_PIX_FMT_YUVA422P16
Definition: pixfmt.h:596
load_libvulkan
static int load_libvulkan(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:630
pthread_mutex_t
_fmutex pthread_mutex_t
Definition: os2threads.h:53
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:88
FFHWFramesContext::pool_internal
AVBufferPool * pool_internal
Definition: hwcontext_internal.h:101
vulkan_device_init
static int vulkan_device_init(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1907
AV_PIX_FMT_GBRAP16
#define AV_PIX_FMT_GBRAP16
Definition: pixfmt.h:565
FF_ENABLE_DEPRECATION_WARNINGS
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:73
ff_vk_load_props
int ff_vk_load_props(FFVulkanContext *s)
Loads props/mprops/driver_props.
Definition: vulkan.c:147
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:216
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:79
AV_PIX_FMT_CUDA
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition: pixfmt.h:260
VulkanDeviceFeatures::vulkan_1_2
VkPhysicalDeviceVulkan12Features vulkan_1_2
Definition: hwcontext_vulkan.c:79
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
VulkanDevicePriv::libvulkan
void * libvulkan
Definition: hwcontext_vulkan.c:129
name
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option name
Definition: writing_filters.txt:88
VulkanOptExtension::name
const char * name
Definition: hwcontext_vulkan.c:669
FFVkFormatEntry::nb_images
int nb_images
Definition: hwcontext_vulkan.c:401
host_map_frame
static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4384
AVCUDADeviceContextInternal
Definition: hwcontext_cuda_internal.h:31
AV_PIX_FMT_GRAY32
#define AV_PIX_FMT_GRAY32
Definition: pixfmt.h:523
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:214
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
hwcontext_cuda_internal.h
HWMapDescriptor::source
AVFrame * source
A reference to the original source of the mapping.
Definition: hwcontext_internal.h:124
FFVulkanExtensions
uint64_t FFVulkanExtensions
Definition: vulkan_functions.h:29
AVBufferPool
The buffer pool.
Definition: buffer_internal.h:88
SET_OLD_QF
#define SET_OLD_QF(field, nb_field, type)
vulkan_transfer_data_to
static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4721
FF_VK_EXT_EXTERNAL_WIN32_MEMORY
#define FF_VK_EXT_EXTERNAL_WIN32_MEMORY
Definition: vulkan_functions.h:39
FF_VK_EXT_VIDEO_QUEUE
#define FF_VK_EXT_VIDEO_QUEUE
Definition: vulkan_functions.h:58
thread.h
vk_dbg_callback
static VKAPI_ATTR VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT messageType, const VkDebugUtilsMessengerCallbackDataEXT *data, void *priv)
Definition: hwcontext_vulkan.c:768
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3456
CHECK_QUEUE
#define CHECK_QUEUE(type, required, fidx, ctx_qf, qc)
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
pthread_mutex_init
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, AVVulkanDeviceQueueFamily *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:357
FF_VK_EXT_PORTABILITY_SUBSET
#define FF_VK_EXT_PORTABILITY_SUBSET
Definition: vulkan_functions.h:73
vulkan_frames_get_constraints
static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx, const void *hwconfig, AVHWFramesConstraints *constraints)
Definition: hwcontext_vulkan.c:2269
FF_VK_EXT_VIDEO_MAINTENANCE_2
#define FF_VK_EXT_VIDEO_MAINTENANCE_2
Definition: vulkan_functions.h:60
AVVkFrameInternal::update_mutex
pthread_mutex_t update_mutex
Definition: hwcontext_vulkan.c:196
av_unused
#define av_unused
Definition: attributes.h:156
vulkan_frames_derive_to
static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc, AVHWFramesContext *src_fc, int flags)
Definition: hwcontext_vulkan.c:4865
VulkanDeviceFeatures::explicit_mem_layout
VkPhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR explicit_mem_layout
Definition: hwcontext_vulkan.c:84
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:64
PICK_QF
#define PICK_QF(type, vid_op)
FF_VK_EXT_VIDEO_DECODE_H265
#define FF_VK_EXT_VIDEO_DECODE_H265
Definition: vulkan_functions.h:64
mode
Definition: swscale.c:59
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:427
pixdesc.h
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to a vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:69
optional_device_exts
static const VulkanOptExtension optional_device_exts[]
Definition: hwcontext_vulkan.c:680
AVFrame::width
int width
Definition: frame.h:499
AV_PIX_FMT_YUVA420P16
#define AV_PIX_FMT_YUVA420P16
Definition: pixfmt.h:595
AV_PIX_FMT_Y216
#define AV_PIX_FMT_Y216
Definition: pixfmt.h:608
create_frame
static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame, VkImageTiling tiling, VkImageUsageFlagBits usage, VkImageCreateFlags flags, int nb_layers, void *create_pnext)
Definition: hwcontext_vulkan.c:2703
AVDRMFrameDescriptor::nb_layers
int nb_layers
Number of layers in the frame.
Definition: hwcontext_drm.h:145
AV_PIX_FMT_DRM_PRIME
@ AV_PIX_FMT_DRM_PRIME
DRM-managed buffers exposed through PRIME buffer sharing.
Definition: pixfmt.h:351
AV_PIX_FMT_YUVA420P10
#define AV_PIX_FMT_YUVA420P10
Definition: pixfmt.h:590
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:251
ff_vk_find_struct
static const void * ff_vk_find_struct(const void *chain, VkStructureType stype)
Definition: vulkan.h:375
pthread_mutex_lock
static av_always_inline int pthread_mutex_lock(pthread_mutex_t *mutex)
Definition: os2threads.h:119
av_hwframe_map
int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
Map a hardware frame.
Definition: hwcontext.c:793
COPY_VAL
#define COPY_VAL(VAL)
nb_vk_formats_list
static const int nb_vk_formats_list
Definition: hwcontext_vulkan.c:514
data
const char data[16]
Definition: mxf.c:149
AVVulkanDeviceContext::queue_family_encode_index
attribute_deprecated int queue_family_encode_index
Queue family index for video encode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:156
AV_PIX_FMT_RGBA128
#define AV_PIX_FMT_RGBA128
Definition: pixfmt.h:630
AV_PIX_FMT_YUV420P10
#define AV_PIX_FMT_YUV420P10
Definition: pixfmt.h:539
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:74
AV_PIX_FMT_XV30
#define AV_PIX_FMT_XV30
Definition: pixfmt.h:609
ff_vk_flush_buffer
int ff_vk_flush_buffer(FFVulkanContext *s, FFVkBuffer *buf, VkDeviceSize offset, VkDeviceSize mem_size, int flush)
Flush or invalidate a single buffer, with a given size and offset.
Definition: vulkan.c:1171
AVVulkanFramesContext::lock_frame
void(* lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Locks a frame, preventing other threads from changing frame properties.
Definition: hwcontext_vulkan.h:296
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:226
AVVAAPIDeviceContext::display
VADisplay display
The VADisplay handle, to be filled by the user.
Definition: hwcontext_vaapi.h:72
switch_new_props
static void switch_new_props(enum PrepMode pmode, VkImageLayout *new_layout, VkAccessFlags2 *new_access)
Definition: hwcontext_vulkan.c:2527
FF_VULKAN_DEBUG_PRACTICES
@ FF_VULKAN_DEBUG_PRACTICES
Definition: hwcontext_vulkan.c:848
vulkan_map_from
static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:4276
AV_PIX_FMT_BGR24
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:76
AV_PIX_FMT_BGRA
@ AV_PIX_FMT_BGRA
packed BGRA 8:8:8:8, 32bpp, BGRABGRA...
Definition: pixfmt.h:102
FFVkFormatEntry::vkf
VkFormat vkf
Definition: hwcontext_vulkan.c:397
ff_vk_exec_get
FFVkExecContext * ff_vk_exec_get(FFVulkanContext *s, FFVkExecPool *pool)
Retrieve an execution pool.
Definition: vulkan.c:548
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2825
AVDictionary
Definition: dict.c:32
ff_hwframe_map_create
int ff_hwframe_map_create(AVBufferRef *hwframe_ref, AVFrame *dst, const AVFrame *src, void(*unmap)(AVHWFramesContext *ctx, HWMapDescriptor *hwmap), void *priv)
Definition: hwcontext.c:741
FFMAX
#define FFMAX(a, b)
Definition: macros.h:47
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
HWMapDescriptor::priv
void * priv
Hardware-specific private data associated with the mapping.
Definition: hwcontext_internal.h:139
FF_VK_EXT_COOP_MATRIX
#define FF_VK_EXT_COOP_MATRIX
Definition: vulkan_functions.h:44
av_popcount
#define av_popcount
Definition: common.h:154
AVDRMFrameDescriptor
DRM frame descriptor.
Definition: hwcontext_drm.h:133
AVHWFramesConstraints::valid_hw_formats
enum AVPixelFormat * valid_hw_formats
A list of possible values for format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:449
AVERROR_UNKNOWN
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:73
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:220
AVFrame::buf
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:604
AV_PIX_FMT_YUVA422P10
#define AV_PIX_FMT_YUVA422P10
Definition: pixfmt.h:591
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
VulkanDeviceSelection::uuid
uint8_t uuid[VK_UUID_SIZE]
Definition: hwcontext_vulkan.c:1303
ff_vk_exec_add_dep_frame
int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage)
Definition: vulkan.c:780
FF_VULKAN_DEBUG_PRINTF
@ FF_VULKAN_DEBUG_PRINTF
Definition: hwcontext_vulkan.c:846
AV_PIX_FMT_P212
#define AV_PIX_FMT_P212
Definition: pixfmt.h:618
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:448
AVVulkanDeviceContext::unlock_queue
attribute_deprecated void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:189
av_vk_get_optional_device_extensions
const char ** av_vk_get_optional_device_extensions(int *count)
Returns an array of optional Vulkan device extensions that FFmpeg may use if enabled.
Definition: hwcontext_vulkan.c:753
FFVkBuffer::buf
VkBuffer buf
Definition: vulkan.h:126
VulkanDeviceFeatures::host_image_copy
VkPhysicalDeviceHostImageCopyFeaturesEXT host_image_copy
Definition: hwcontext_vulkan.c:83
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:374
alloc_mem
static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req, VkMemoryPropertyFlagBits req_flags, const void *alloc_extension, VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
Definition: hwcontext_vulkan.c:2315
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
VulkanDevicePriv::compute_qf
AVVulkanDeviceQueueFamily * compute_qf
Definition: hwcontext_vulkan.c:132
AVHWFramesConstraints
This struct describes the constraints on hardware frames attached to a given device with a hardware-s...
Definition: hwcontext.h:444
FF_VK_EXT_HOST_IMAGE_COPY
#define FF_VK_EXT_HOST_IMAGE_COPY
Definition: vulkan_functions.h:51
AV_HWDEVICE_TYPE_CUDA
@ AV_HWDEVICE_TYPE_CUDA
Definition: hwcontext.h:30
FF_VK_EXT_EXPECT_ASSUME
#define FF_VK_EXT_EXPECT_ASSUME
Definition: vulkan_functions.h:49
AVDRMDeviceContext::fd
int fd
File descriptor of DRM device.
Definition: hwcontext_drm.h:166
PREP_MODE_DECODING_DPB
@ PREP_MODE_DECODING_DPB
Definition: hwcontext_vulkan.c:2523
FF_VK_EXT_EXTERNAL_FD_SEM
#define FF_VK_EXT_EXTERNAL_FD_SEM
Definition: vulkan_functions.h:35
VulkanDeviceFeatures::device
VkPhysicalDeviceFeatures2 device
Definition: hwcontext_vulkan.c:76
VulkanDeviceFeatures::video_maintenance_1
VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1
Definition: hwcontext_vulkan.c:102
close
static av_cold void close(AVCodecParserContext *s)
Definition: apv_parser.c:197
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3496
ASPECT_3PLANE
#define ASPECT_3PLANE
Definition: hwcontext_vulkan.c:394
FF_VK_EXT_LONG_VECTOR
#define FF_VK_EXT_LONG_VECTOR
Definition: vulkan_functions.h:55
VulkanDevicePriv::hprops
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops
Definition: hwcontext_vulkan.c:138
vulkan_device_derive
static int vulkan_device_derive(AVHWDeviceContext *ctx, AVHWDeviceContext *src_ctx, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:2166
VulkanOptExtension::flag
FFVulkanExtensions flag
Definition: hwcontext_vulkan.c:670
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AV_PIX_FMT_GBRP14
#define AV_PIX_FMT_GBRP14
Definition: pixfmt.h:560
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:315
AV_PIX_FMT_GBRAP
@ AV_PIX_FMT_GBRAP
planar GBRA 4:4:4:4 32bpp
Definition: pixfmt.h:212
VulkanDeviceSelection::drm_minor
uint32_t drm_minor
Definition: hwcontext_vulkan.c:1306
AVVulkanDeviceContext::lock_queue
attribute_deprecated void(* lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Locks a queue, preventing other threads from submitting any command buffers to this queue.
Definition: hwcontext_vulkan.h:181
lock_frame
static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2950
fail
#define fail()
Definition: checkasm.h:220
AVDRMLayerDescriptor::nb_planes
int nb_planes
Number of planes in the layer.
Definition: hwcontext_drm.h:106
ff_vk_exec_add_dep_bool_sem
int ff_vk_exec_add_dep_bool_sem(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore *sem, int nb, VkPipelineStageFlagBits2 stage, int wait)
Definition: vulkan.c:714
AV_DRM_MAX_PLANES
@ AV_DRM_MAX_PLANES
The maximum number of layers/planes in a DRM frame.
Definition: hwcontext_drm.h:39
AV_PIX_FMT_GBRP10
#define AV_PIX_FMT_GBRP10
Definition: pixfmt.h:558
AV_PIX_FMT_YUVA444P16
#define AV_PIX_FMT_YUVA444P16
Definition: pixfmt.h:597
AVVulkanFramesContext::flags
AVVkFrameFlags flags
A combination of AVVkFrameFlags.
Definition: hwcontext_vulkan.h:267
VulkanDevicePriv
Definition: hwcontext_vulkan.c:122
AVVulkanDeviceContext::nb_tx_queues
attribute_deprecated int nb_tx_queues
Definition: hwcontext_vulkan.h:139
AVDRMLayerDescriptor::planes
AVDRMPlaneDescriptor planes[AV_DRM_MAX_PLANES]
Array of planes in this layer.
Definition: hwcontext_drm.h:110
dummy
int dummy
Definition: motion.c:64
AVVkFrame::mem
VkDeviceMemory mem[AV_NUM_DATA_POINTERS]
Memory backing the images.
Definition: hwcontext_vulkan.h:328
switch_layout
static int switch_layout(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2562
device_features_copy_needed
static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
Definition: hwcontext_vulkan.c:288
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:220
ff_vk_frame_barrier
void ff_vk_frame_barrier(FFVulkanContext *s, FFVkExecContext *e, AVFrame *pic, VkImageMemoryBarrier2 *bar, int *nb_bar, VkPipelineStageFlags2 src_stage, VkPipelineStageFlags2 dst_stage, VkAccessFlagBits2 new_access, VkImageLayout new_layout, uint32_t new_qf)
Definition: vulkan.c:2039
av_buffer_pool_init2
AVBufferPool * av_buffer_pool_init2(size_t size, void *opaque, AVBufferRef *(*alloc)(void *opaque, size_t size), void(*pool_free)(void *opaque))
Allocate and initialize a buffer pool with a more complex allocator.
Definition: buffer.c:259
AVHWFramesConstraints::min_width
int min_width
The minimum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:462
lock_queue
static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1895
AVCUDADeviceContextInternal::cuda_device
CUdevice cuda_device
Definition: hwcontext_cuda_internal.h:34
VulkanDevicePriv::limit_queues
int limit_queues
Definition: hwcontext_vulkan.c:169
VulkanDevicePriv::vkctx
FFVulkanContext vkctx
Definition: hwcontext_vulkan.c:131
AV_PIX_FMT_XV48
#define AV_PIX_FMT_XV48
Definition: pixfmt.h:611
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
FF_VK_EXT_VIDEO_ENCODE_H265
#define FF_VK_EXT_VIDEO_ENCODE_H265
Definition: vulkan_functions.h:70
ff_vk_host_map_buffer
int ff_vk_host_map_buffer(FFVulkanContext *s, AVBufferRef **dst, uint8_t *src_data, const AVBufferRef *src_buf, VkBufferUsageFlags usage)
Maps a system RAM buffer into a Vulkan buffer.
Definition: vulkan.c:1391
ff_vk_ret2str
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:40
AV_PIX_FMT_GRAY16
#define AV_PIX_FMT_GRAY16
Definition: pixfmt.h:522
VulkanDeviceFeatures::vulkan_1_3
VkPhysicalDeviceVulkan13Features vulkan_1_3
Definition: hwcontext_vulkan.c:80
FF_VK_EXT_EXTERNAL_WIN32_SEM
#define FF_VK_EXT_EXTERNAL_WIN32_SEM
Definition: vulkan_functions.h:40
AVVulkanDeviceContext::queue_family_decode_index
attribute_deprecated int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:166
VulkanDevicePriv::props
VkPhysicalDeviceProperties2 props
Definition: hwcontext_vulkan.c:136
ff_vk_aspect_flag
VkImageAspectFlags ff_vk_aspect_flag(AVFrame *f, int p)
Get the aspect flag for a plane from an image.
Definition: vulkan.c:1532
VulkanFramesPriv::drm_format_modifier_properties
VkDrmFormatModifierPropertiesEXT drm_format_modifier_properties[5]
Definition: hwcontext_vulkan.c:192
vk_find_format_entry
static const struct FFVkFormatEntry * vk_find_format_entry(enum AVPixelFormat p)
Definition: hwcontext_vulkan.c:524
AVDRMPlaneDescriptor::offset
ptrdiff_t offset
Offset within that object of this plane.
Definition: hwcontext_drm.h:83
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:63
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:52
AV_PIX_FMT_YUV444P10
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:542
AV_PIX_FMT_Y210
#define AV_PIX_FMT_Y210
Definition: pixfmt.h:606
AVVulkanDeviceQueueFamily::num
int num
Definition: hwcontext_vulkan.h:37
HWContextType::type
enum AVHWDeviceType type
Definition: hwcontext_internal.h:30
ffhwframesctx
static FFHWFramesContext * ffhwframesctx(AVHWFramesContext *ctx)
Definition: hwcontext_internal.h:115
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:210
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
ff_vk_link_struct
static void ff_vk_link_struct(void *chain, const void *in)
Definition: vulkan.h:388
check_layers
static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:1018
AV_PIX_FMT_YUV422P16
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:551
AVVulkanDeviceContext::nb_encode_queues
attribute_deprecated int nb_encode_queues
Definition: hwcontext_vulkan.h:158
AVHWFramesContext::height
int height
Definition: hwcontext.h:220
AVHWFramesConstraints::valid_sw_formats
enum AVPixelFormat * valid_sw_formats
A list of possible values for sw_format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:456
vulkan_map_to
static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:4067
av_dict_get
AVDictionaryEntry * av_dict_get(const AVDictionary *m, const char *key, const AVDictionaryEntry *prev, int flags)
Get a dictionary entry with matching key.
Definition: dict.c:60
av_buffer_pool_get
AVBufferRef * av_buffer_pool_get(AVBufferPool *pool)
Allocate a new AVBuffer, reusing an old buffer from the pool when available.
Definition: buffer.c:390
AV_PIX_FMT_GBRAP10
#define AV_PIX_FMT_GBRAP10
Definition: pixfmt.h:562
AVHWFramesContext::pool
AVBufferPool * pool
A pool from which the frames are allocated by av_hwframe_get_buffer().
Definition: hwcontext.h:181
VulkanDeviceSelection::pci_device
uint32_t pci_device
Definition: hwcontext_vulkan.c:1309
s
#define s(width, name)
Definition: cbs_vp9.c:198
AV_PIX_FMT_GBRAP14
#define AV_PIX_FMT_GBRAP14
Definition: pixfmt.h:564
AV_PIX_FMT_GBRAP12
#define AV_PIX_FMT_GBRAP12
Definition: pixfmt.h:563
AV_PIX_FMT_YUVA420P
@ AV_PIX_FMT_YUVA420P
planar YUV 4:2:0, 20bpp, (1 Cr & Cb sample per 2x2 Y & A samples)
Definition: pixfmt.h:108
AV_PIX_FMT_RGB96
#define AV_PIX_FMT_RGB96
Definition: pixfmt.h:629
pthread_mutex_unlock
static av_always_inline int pthread_mutex_unlock(pthread_mutex_t *mutex)
Definition: os2threads.h:126
AV_PIX_FMT_YUV444P16
#define AV_PIX_FMT_YUV444P16
Definition: pixfmt.h:552
offsets
static const int offsets[]
Definition: hevc_pel.c:34
AV_CEIL_RSHIFT
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:60
ff_vk_load_functions
static int ff_vk_load_functions(AVHWDeviceContext *ctx, FFVulkanFunctions *vk, uint64_t extensions_mask, int has_inst, int has_dev)
Function loader.
Definition: vulkan_loader.h:128
VulkanDevicePriv::ext_sem_props_opaque
VkExternalSemaphoreProperties ext_sem_props_opaque
Definition: hwcontext_vulkan.c:142
prepare_frame
static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2670
vulkan_transfer_frame
static int vulkan_transfer_frame(AVHWFramesContext *hwfc, AVFrame *swf, AVFrame *hwf, int upload)
Definition: hwcontext_vulkan.c:4560
FF_VK_EXT_DEVICE_DRM
#define FF_VK_EXT_DEVICE_DRM
Definition: vulkan_functions.h:42
ff_vk_exec_wait
void ff_vk_exec_wait(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:553
av_strtok
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
Definition: avstring.c:179
ASPECT_2PLANE
#define ASPECT_2PLANE
Definition: hwcontext_vulkan.c:393
vulkan_device_uninit
static void vulkan_device_uninit(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1775
fc
#define fc(width, name, range_min, range_max)
Definition: cbs_av1.c:494
AVVulkanFramesContext::unlock_frame
void(* unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Similar to lock_frame(), unlocks a frame.
Definition: hwcontext_vulkan.h:301
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:42
AVVulkanFramesContext::img_flags
VkImageCreateFlags img_flags
Flags to set during image creation.
Definition: hwcontext_vulkan.h:273
AV_PIX_FMT_GBRAP32
#define AV_PIX_FMT_GBRAP32
Definition: pixfmt.h:566
AV_PIX_FMT_YUVA444P12
#define AV_PIX_FMT_YUVA444P12
Definition: pixfmt.h:594
vulkan_frames_uninit
static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2960
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:231
AV_PIX_FMT_YUV420P16
#define AV_PIX_FMT_YUV420P16
Definition: pixfmt.h:550
ctx
static AVFormatContext * ctx
Definition: movenc.c:49
AVDRMObjectDescriptor::fd
int fd
DRM PRIME fd for the object.
Definition: hwcontext_drm.h:52
AV_PIX_FMT_GRAY14
#define AV_PIX_FMT_GRAY14
Definition: pixfmt.h:521
ff_vk_exec_add_dep_buf
int ff_vk_exec_add_dep_buf(FFVulkanContext *s, FFVkExecContext *e, AVBufferRef **deps, int nb_deps, int ref)
Execution dependency management.
Definition: vulkan.c:620
VulkanDeviceSelection::index
int index
Definition: hwcontext_vulkan.c:1311
AV_PIX_FMT_RGBF32
#define AV_PIX_FMT_RGBF32
Definition: pixfmt.h:626
VulkanFramesPriv::p
AVVulkanFramesContext p
The public AVVulkanFramesContext.
Definition: hwcontext_vulkan.c:176
vulkan_transfer_get_formats
static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats)
Definition: hwcontext_vulkan.c:3216
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:73
VulkanDevicePriv::avoid_host_import
int avoid_host_import
Definition: hwcontext_vulkan.c:166
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:299
av_mallocz
#define av_mallocz(s)
Definition: tableprint_vlc.h:31
FFVulkanDebugMode
FFVulkanDebugMode
Definition: hwcontext_vulkan.c:841
AV_PIX_FMT_GRAYF32
#define AV_PIX_FMT_GRAYF32
Definition: pixfmt.h:582
LIBAVUTIL_VERSION_MINOR
#define LIBAVUTIL_VERSION_MINOR
Definition: version.h:82
tmp
static uint8_t tmp[40]
Definition: aes_ctr.c:52
AV_PIX_FMT_RGBA
@ AV_PIX_FMT_RGBA
packed RGBA 8:8:8:8, 32bpp, RGBARGBA...
Definition: pixfmt.h:100
FFVkFormatEntry::nb_images_fallback
int nb_images_fallback
Definition: hwcontext_vulkan.c:402
vulkan_frame_free_cb
static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
Definition: hwcontext_vulkan.c:2443
AV_PIX_FMT_GRAY10
#define AV_PIX_FMT_GRAY10
Definition: pixfmt.h:519
AV_HWFRAME_MAP_READ
@ AV_HWFRAME_MAP_READ
The mapping must be readable.
Definition: hwcontext.h:515
if
if(ret)
Definition: filter_design.txt:179
av_vkfmt_from_pixfmt
const VkFormat * av_vkfmt_from_pixfmt(enum AVPixelFormat p)
Returns the optimal per-plane Vulkan format for a given sw_format, one for each plane.
Definition: hwcontext_vulkan.c:516
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:59
VulkanDevicePriv::qf_mutex
pthread_mutex_t ** qf_mutex
Definition: hwcontext_vulkan.c:148
AV_PIX_FMT_GBRP16
#define AV_PIX_FMT_GBRP16
Definition: pixfmt.h:561
opts
static AVDictionary * opts
Definition: movenc.c:51
PREP_MODE_WRITE
@ PREP_MODE_WRITE
Definition: hwcontext_vulkan.c:2519
AV_PIX_FMT_RGBA64
#define AV_PIX_FMT_RGBA64
Definition: pixfmt.h:529
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:213
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
format
New swscale design to change SwsGraph is what coordinates multiple passes These can include cascaded scaling error diffusion and so on Or we could have separate passes for the vertical and horizontal scaling In between each SwsPass lies a fully allocated image buffer Graph passes may have different levels of e g we can have a single threaded error diffusion pass following a multi threaded scaling pass SwsGraph is internally recreated whenever the image format
Definition: swscale-v2.txt:14
FF_VK_EXT_DRM_MODIFIER_FLAGS
#define FF_VK_EXT_DRM_MODIFIER_FLAGS
Definition: vulkan_functions.h:33
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:117
FFVkFormatEntry
Definition: hwcontext_vulkan.c:396
FF_VK_EXT_SHADER_OBJECT
#define FF_VK_EXT_SHADER_OBJECT
Definition: vulkan_functions.h:46
AV_PIX_FMT_YUYV422
@ AV_PIX_FMT_YUYV422
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:74
AVVkFrameInternal
Definition: hwcontext_vulkan.c:195
FF_VK_EXT_VIDEO_DECODE_VP9
#define FF_VK_EXT_VIDEO_DECODE_VP9
Definition: vulkan_functions.h:65
FF_VK_EXT_SUBGROUP_ROTATE
#define FF_VK_EXT_SUBGROUP_ROTATE
Definition: vulkan_functions.h:50
FF_VK_EXT_VIDEO_ENCODE_QUEUE
#define FF_VK_EXT_VIDEO_ENCODE_QUEUE
Definition: vulkan_functions.h:68
VulkanDevicePriv::debug_ctx
VkDebugUtilsMessengerEXT debug_ctx
Definition: hwcontext_vulkan.c:154
FF_VULKAN_DEBUG_NONE
@ FF_VULKAN_DEBUG_NONE
Definition: hwcontext_vulkan.c:842
AVVulkanFramesContext::alloc_pnext
void * alloc_pnext[AV_NUM_DATA_POINTERS]
Extension data for memory allocation.
Definition: hwcontext_vulkan.h:260
setup_queue_families
static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
Definition: hwcontext_vulkan.c:1555
LIBAVUTIL_VERSION_MAJOR
#define LIBAVUTIL_VERSION_MAJOR
Definition: version.h:81
AVVulkanDeviceContext::nb_decode_queues
attribute_deprecated int nb_decode_queues
Definition: hwcontext_vulkan.h:168
AV_PIX_FMT_P410
#define AV_PIX_FMT_P410
Definition: pixfmt.h:617
av_buffer_pool_uninit
void av_buffer_pool_uninit(AVBufferPool **ppool)
Mark the pool as being available for freeing.
Definition: buffer.c:328
AVVulkanDeviceContext::nb_qf
int nb_qf
Definition: hwcontext_vulkan.h:201
ff_hwcontext_type_vulkan
const HWContextType ff_hwcontext_type_vulkan
Definition: hwcontext_vulkan.c:4894
hwcontext_vulkan.h
AVVulkanDeviceContext::queue_family_tx_index
attribute_deprecated int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:137
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:103
vk_formats_list
static const struct FFVkFormatEntry vk_formats_list[]
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:281
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:240
AV_PIX_FMT_BGR0
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:265
AV_PIX_FMT_YUV422P10
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:540
planes
static const struct @561 planes[]
alloc_bind_mem
static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f, void *alloc_pnext, size_t alloc_pnext_stride)
Definition: hwcontext_vulkan.c:2448
FFVkBuffer::mapped_mem
uint8_t * mapped_mem
Definition: vulkan.h:134
AVVulkanDeviceContext::qf
AVVulkanDeviceQueueFamily qf[64]
Queue families used.
Definition: hwcontext_vulkan.h:200
AV_PIX_FMT_GRAY8
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
Definition: pixfmt.h:81
FFVulkanContext
Definition: vulkan.h:312
exp
int8_t exp
Definition: eval.c:76
FF_VK_EXT_REPLICATED_COMPOSITES
#define FF_VK_EXT_REPLICATED_COMPOSITES
Definition: vulkan_functions.h:54
VulkanFramesPriv
Definition: hwcontext_vulkan.c:172
vulkan_frame_free
static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
Definition: hwcontext_vulkan.c:2409
pick_video_queue_family
static int pick_video_queue_family(VkQueueFamilyProperties2 *qf, VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf, VkVideoCodecOperationFlagsKHR flags)
Definition: hwcontext_vulkan.c:1526
index
int index
Definition: gxfenc.c:90
av_buffer_create
AVBufferRef * av_buffer_create(uint8_t *data, size_t size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:55
vkfmt_from_pixfmt2
static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p, VkImageTiling tiling, VkFormat fmts[AV_NUM_DATA_POINTERS], int *nb_images, VkImageAspectFlags *aspect, VkImageUsageFlags *supported_usage, int disable_multiplane, int need_storage)
Definition: hwcontext_vulkan.c:532
VulkanDeviceSelection
Definition: hwcontext_vulkan.c:1302
source
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a source
Definition: filter_design.txt:256
VulkanDevicePriv::nb_tot_qfs
uint32_t nb_tot_qfs
Definition: hwcontext_vulkan.c:149
VulkanDeviceFeatures
Definition: hwcontext_vulkan.c:75
AVDRMFrameDescriptor::layers
AVDRMLayerDescriptor layers[AV_DRM_MAX_PLANES]
Array of layers in the frame.
Definition: hwcontext_drm.h:149
usage
const char * usage
Definition: floatimg_cmp.c:62
PREP_MODE_DECODING_DST
@ PREP_MODE_DECODING_DST
Definition: hwcontext_vulkan.c:2522
AVVkFrame::size
size_t size[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:329
vulkan_pool_alloc
static AVBufferRef * vulkan_pool_alloc(void *opaque, size_t size)
Definition: hwcontext_vulkan.c:2875
VulkanDevicePriv::dprops
VkPhysicalDeviceDriverProperties dprops
Definition: hwcontext_vulkan.c:139
AV_PIX_FMT_X2BGR10
#define AV_PIX_FMT_X2BGR10
Definition: pixfmt.h:614
PrepMode
PrepMode
Definition: hwcontext_vulkan.c:2517
FF_VK_EXT_VIDEO_MAINTENANCE_1
#define FF_VK_EXT_VIDEO_MAINTENANCE_1
Definition: vulkan_functions.h:59
VulkanDeviceSelection::has_drm
uint32_t has_drm
Definition: hwcontext_vulkan.c:1307
f
f
Definition: af_crystalizer.c:122
AVCUDADeviceContext::internal
AVCUDADeviceContextInternal * internal
Definition: hwcontext_cuda.h:45
AV_PIX_FMT_RGB24
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:75
VulkanDeviceFeatures::vulkan_1_1
VkPhysicalDeviceVulkan11Features vulkan_1_1
Definition: hwcontext_vulkan.c:78
sem_wait
#define sem_wait(psem)
Definition: semaphore.h:27
AV_PIX_FMT_P012
#define AV_PIX_FMT_P012
Definition: pixfmt.h:603
AV_PIX_FMT_FLAG_RGB
#define AV_PIX_FMT_FLAG_RGB
The pixel format contains RGB-like data (as opposed to YUV/grayscale).
Definition: pixdesc.h:136
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition: dsp.h:87
AVVkFrame
Definition: hwcontext_vulkan.h:310
i
#define i(width, name, range_min, range_max)
Definition: cbs_h264.c:63
av_vk_frame_alloc
AVVkFrame * av_vk_frame_alloc(void)
Allocates a single AVVkFrame and initializes everything as 0.
Definition: hwcontext_vulkan.c:4871
FF_VULKAN_DEBUG_VALIDATE
@ FF_VULKAN_DEBUG_VALIDATE
Definition: hwcontext_vulkan.c:844
vulkan.h
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:122
FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
#define FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
Definition: vulkan_functions.h:32
vulkan_device_free
static void vulkan_device_free(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1752
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:424
FF_VK_EXT_NO_FLAG
#define FF_VK_EXT_NO_FLAG
Definition: vulkan_functions.h:74
AV_PIX_FMT_GBRPF32
#define AV_PIX_FMT_GBRPF32
Definition: pixfmt.h:578
AV_PIX_FMT_YUV422P12
#define AV_PIX_FMT_YUV422P12
Definition: pixfmt.h:544
VulkanDeviceFeatures::cooperative_matrix
VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix
Definition: hwcontext_vulkan.c:114
AV_PIX_FMT_RGB48
#define AV_PIX_FMT_RGB48
Definition: pixfmt.h:525
size
int size
Definition: twinvq_data.h:10344
ff_vk_exec_add_dep_sw_frame
int ff_vk_exec_add_dep_sw_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f)
Definition: vulkan.c:647
vulkan_transfer_data_from
static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4840
AV_NUM_DATA_POINTERS
#define AV_NUM_DATA_POINTERS
Definition: frame.h:428
FF_VK_EXT_PUSH_DESCRIPTOR
#define FF_VK_EXT_PUSH_DESCRIPTOR
Definition: vulkan_functions.h:47
VulkanDevicePriv::use_linear_images
int use_linear_images
Definition: hwcontext_vulkan.c:157
AV_PIX_FMT_YUV444P12
#define AV_PIX_FMT_YUV444P12
Definition: pixfmt.h:546
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:514
AV_PIX_FMT_NV16
@ AV_PIX_FMT_NV16
interleaved chroma YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:198
VulkanDeviceSelection::vendor_id
uint32_t vendor_id
Definition: hwcontext_vulkan.c:1310
PREP_MODE_GENERAL
@ PREP_MODE_GENERAL
Definition: hwcontext_vulkan.c:2518
AV_PIX_FMT_Y212
#define AV_PIX_FMT_Y212
Definition: pixfmt.h:607
AVVulkanDeviceContext::queue_family_index
attribute_deprecated int queue_family_index
Queue family index for graphics operations, and the number of queues enabled for it.
Definition: hwcontext_vulkan.h:128
AVDRMObjectDescriptor::size
size_t size
Total size of the object.
Definition: hwcontext_drm.h:58
AV_PIX_FMT_YUVA444P
@ AV_PIX_FMT_YUVA444P
planar YUV 4:4:4 32bpp, (1 Cr & Cb sample per 1x1 Y & A samples)
Definition: pixfmt.h:174
VulkanFramesPriv::upload_exec
FFVkExecPool upload_exec
Definition: hwcontext_vulkan.c:182
AVVulkanDeviceQueueFamily::idx
int idx
Definition: hwcontext_vulkan.h:35
AV_PIX_FMT_YUVA444P10
#define AV_PIX_FMT_YUVA444P10
Definition: pixfmt.h:592
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
AVHWFramesConstraints::max_width
int max_width
The maximum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:469
FFVkExecContext
Definition: vulkan.h:145
VulkanOptExtension
Definition: hwcontext_vulkan.c:668
AV_PIX_FMT_RGB0
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:263
AV_PIX_FMT_P216
#define AV_PIX_FMT_P216
Definition: pixfmt.h:620
CHECK_CU
#define CHECK_CU(x)
Definition: cuviddec.c:117
AV_PIX_FMT_P210
#define AV_PIX_FMT_P210
Definition: pixfmt.h:616
AV_PIX_FMT_VAAPI
@ AV_PIX_FMT_VAAPI
Hardware acceleration through VA-API, data[3] contains a VASurfaceID.
Definition: pixfmt.h:126
VulkanDeviceFeatures::subgroup_rotate
VkPhysicalDeviceShaderSubgroupRotateFeaturesKHR subgroup_rotate
Definition: hwcontext_vulkan.c:82
FF_VK_EXT_VIDEO_DECODE_QUEUE
#define FF_VK_EXT_VIDEO_DECODE_QUEUE
Definition: vulkan_functions.h:62
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:221
vulkan_free_internal
static void vulkan_free_internal(AVVkFrame *f)
Definition: hwcontext_vulkan.c:2373
VulkanDeviceFeatures::timeline_semaphore
VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore
Definition: hwcontext_vulkan.c:81
AV_HWDEVICE_TYPE_VAAPI
@ AV_HWDEVICE_TYPE_VAAPI
Definition: hwcontext.h:31
pthread_mutex_destroy
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
FF_VK_EXT_EXTERNAL_HOST_MEMORY
#define FF_VK_EXT_EXTERNAL_HOST_MEMORY
Definition: vulkan_functions.h:36
FF_VK_EXT_EXPLICIT_MEM_LAYOUT
#define FF_VK_EXT_EXPLICIT_MEM_LAYOUT
Definition: vulkan_functions.h:53
AV_PIX_FMT_UYVA
@ AV_PIX_FMT_UYVA
packed UYVA 4:4:4:4, 32bpp (1 Cr & Cb sample per 1x1 Y & A samples), UYVAUYVA...
Definition: pixfmt.h:444
AVDRMFrameDescriptor::objects
AVDRMObjectDescriptor objects[AV_DRM_MAX_PLANES]
Array of objects making up the frame.
Definition: hwcontext_drm.h:141
AVCUDADeviceContextInternal::cuda_dl
CudaFunctions * cuda_dl
Definition: hwcontext_cuda_internal.h:32
av_assert2
#define av_assert2(cond)
assert() equivalent, that does lie in speed critical code.
Definition: avassert.h:68
ff_vk_exec_start
int ff_vk_exec_start(FFVulkanContext *s, FFVkExecContext *e)
Start/submit/wait an execution.
Definition: vulkan.c:560
FF_VK_EXT_RELAXED_EXTENDED_INSTR
#define FF_VK_EXT_RELAXED_EXTENDED_INSTR
Definition: vulkan_functions.h:48
FF_VK_EXT_VIDEO_DECODE_H264
#define FF_VK_EXT_VIDEO_DECODE_H264
Definition: vulkan_functions.h:63
VulkanFramesPriv::compute_exec
FFVkExecPool compute_exec
Definition: hwcontext_vulkan.c:179
AVVulkanDeviceContext::queue_family_comp_index
attribute_deprecated int queue_family_comp_index
Queue family index for compute operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:146
AVDRMObjectDescriptor::format_modifier
uint64_t format_modifier
Format modifier applied to the object (DRM_FORMAT_MOD_*).
Definition: hwcontext_drm.h:65
VkFormat
enum VkFormat VkFormat
Definition: hwcontext_stub.c:25
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:32
AV_PIX_FMT_GBRP12
#define AV_PIX_FMT_GBRP12
Definition: pixfmt.h:559
weights
static const int weights[]
Definition: hevc_pel.c:32
AV_PIX_FMT_NV24
@ AV_PIX_FMT_NV24
planar YUV 4:4:4, 24bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:371
plane_info
Definition: vf_edgedetect.c:53
VulkanDevicePriv::nb_img_qfs
uint32_t nb_img_qfs
Definition: hwcontext_vulkan.c:151
vulkan_frames_init
static int vulkan_frames_init(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2978
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
AV_PIX_FMT_X2RGB10
#define AV_PIX_FMT_X2RGB10
Definition: pixfmt.h:613
VulkanDeviceSelection::has_uuid
int has_uuid
Definition: hwcontext_vulkan.c:1304
hwcontext_drm.h
AVDRMPlaneDescriptor::object_index
int object_index
Index of the object containing this plane in the objects array of the enclosing frame descriptor.
Definition: hwcontext_drm.h:79
AV_PIX_FMT_BGR565
#define AV_PIX_FMT_BGR565
Definition: pixfmt.h:531
ff_hwframe_map_replace
int ff_hwframe_map_replace(AVFrame *dst, const AVFrame *src)
Replace the current hwmap of dst with the one from src, used for indirect mappings like VAAPI->(DRM)-...
Definition: hwcontext.c:948
VulkanDevicePriv::img_qfs
uint32_t img_qfs[64]
Definition: hwcontext_vulkan.c:150
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:264
mod
static int mod(int a, int b)
Modulo operation with only positive remainders.
Definition: vf_v360.c:755
AV_PIX_FMT_P016
#define AV_PIX_FMT_P016
Definition: pixfmt.h:604
AV_PIX_FMT_RGB565
#define AV_PIX_FMT_RGB565
Definition: pixfmt.h:526
AVHWFrameTransferDirection
AVHWFrameTransferDirection
Definition: hwcontext.h:406
AVVkFrame::sem
VkSemaphore sem[AV_NUM_DATA_POINTERS]
Synchronization timeline semaphores, one for each VkImage.
Definition: hwcontext_vulkan.h:348
create_instance
static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:1165
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:118
FF_VK_EXT_EXTERNAL_FD_MEMORY
#define FF_VK_EXT_EXTERNAL_FD_MEMORY
Definition: vulkan_functions.h:34
AVCUDADeviceContext
This struct is allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_cuda.h:42
hwcontext_vaapi.h
AVDRMLayerDescriptor::format
uint32_t format
Format of the layer (DRM_FORMAT_*).
Definition: hwcontext_drm.h:100
VulkanDevicePriv::transfer_qf
AVVulkanDeviceQueueFamily * transfer_qf
Definition: hwcontext_vulkan.c:133
ret
ret
Definition: filter_design.txt:187
pixfmt
enum AVPixelFormat pixfmt
Definition: kmsgrab.c:367
AVHWDeviceContext::type
enum AVHWDeviceType type
This field identifies the underlying API used for hardware access.
Definition: hwcontext.h:75
AV_PIX_FMT_NV12
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:96
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:265
AVHWFramesContext::device_ctx
AVHWDeviceContext * device_ctx
The parent AVHWDeviceContext.
Definition: hwcontext.h:137
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:316
cuda_check.h
AVHWFramesContext::hwctx
void * hwctx
The format-specific data, allocated and freed automatically along with this context.
Definition: hwcontext.h:153
VulkanFramesPriv::tmp
AVBufferPool * tmp
Definition: hwcontext_vulkan.c:186
vulkan_get_buffer
static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
Definition: hwcontext_vulkan.c:3202
pick_queue_family
static int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf, VkQueueFlagBits flags)
Definition: hwcontext_vulkan.c:1497
FFVkExecPool
Definition: vulkan.h:290
vulkan_transfer_host
static int vulkan_transfer_host(AVHWFramesContext *hwfc, AVFrame *hwf, AVFrame *swf, int upload)
Definition: hwcontext_vulkan.c:4441
unlock_queue
static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1901
AVHWFramesConstraints::max_height
int max_height
Definition: hwcontext.h:470
AVVkFrame::internal
struct AVVkFrameInternal * internal
Internal data.
Definition: hwcontext_vulkan.h:361
ff_vk_qf_find
AVVulkanDeviceQueueFamily * ff_vk_qf_find(FFVulkanContext *s, VkQueueFlagBits dev_family, VkVideoCodecOperationFlagBitsKHR vid_ops)
Chooses an appropriate QF.
Definition: vulkan.c:286
AV_PIX_FMT_YUV420P12
#define AV_PIX_FMT_YUV420P12
Definition: pixfmt.h:543
AV_PIX_FMT_UYVY422
@ AV_PIX_FMT_UYVY422
packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1
Definition: pixfmt.h:88
check_extensions
static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode debug_mode)
Definition: hwcontext_vulkan.c:853
FFVkExecContext::buf
VkCommandBuffer buf
Definition: vulkan.h:156
vulkan_device_create
static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:2150
AVFrame::height
int height
Definition: frame.h:499
PREP_MODE_ENCODING_DPB
@ PREP_MODE_ENCODING_DPB
Definition: hwcontext_vulkan.c:2524
FFVkFormatEntry::pixfmt
enum AVPixelFormat pixfmt
Definition: hwcontext_vulkan.c:398
AVHWFramesConstraints::min_height
int min_height
Definition: hwcontext.h:463
RELEASE_PROPS
#define RELEASE_PROPS(props, count)
Definition: hwcontext_vulkan.c:814
mode
mode
Definition: ebur128.h:83
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
AV_PIX_FMT_YUVA422P12
#define AV_PIX_FMT_YUVA422P12
Definition: pixfmt.h:593
AV_PIX_FMT_GBRAPF32
#define AV_PIX_FMT_GBRAPF32
Definition: pixfmt.h:579
VulkanDevicePriv::feats
VulkanDeviceFeatures feats
Definition: hwcontext_vulkan.c:145
switch_layout_host
static int switch_layout_host(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2625
LIBAVUTIL_VERSION_MICRO
#define LIBAVUTIL_VERSION_MICRO
Definition: version.h:83
find_device
static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
Definition: hwcontext_vulkan.c:1326
AVVulkanDeviceContext::nb_graphics_queues
attribute_deprecated int nb_graphics_queues
Definition: hwcontext_vulkan.h:130
FF_VK_STRUCT_EXT
#define FF_VK_STRUCT_EXT(CTX, BASE, STRUCT_P, EXT_FLAG, TYPE)
Definition: vulkan.h:397
optional_instance_exts
static const VulkanOptExtension optional_instance_exts[]
Definition: hwcontext_vulkan.c:673
AV_PIX_FMT_FLAG_PLANAR
#define AV_PIX_FMT_FLAG_PLANAR
At least one pixel component is not in the first data plane.
Definition: pixdesc.h:132
ff_vk_map_feats_to_usage
VkImageUsageFlags ff_vk_map_feats_to_usage(VkFormatFeatureFlagBits2 feats)
Map between usage and features.
FF_VK_EXT_ATOMIC_FLOAT
#define FF_VK_EXT_ATOMIC_FLOAT
Definition: vulkan_functions.h:43
FFVkFormatEntry::fallback
const VkFormat fallback[5]
Definition: hwcontext_vulkan.c:403
Windows::Graphics::DirectX::Direct3D11::p
IDirect3DDxgiInterfaceAccess _COM_Outptr_ void ** p
Definition: vsrc_gfxcapture_winrt.hpp:53
PREP_MODE_EXTERNAL_IMPORT
@ PREP_MODE_EXTERNAL_IMPORT
Definition: hwcontext_vulkan.c:2521
AV_PIX_FMT_RGBAF32
#define AV_PIX_FMT_RGBAF32
Definition: pixfmt.h:627
FF_VK_EXT_VIDEO_DECODE_AV1
#define FF_VK_EXT_VIDEO_DECODE_AV1
Definition: vulkan_functions.h:66
AV_PIX_FMT_YUV444P
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:78
AVVulkanFramesContext::tiling
VkImageTiling tiling
Controls the tiling of allocated frames.
Definition: hwcontext_vulkan.h:229
VulkanDeviceSelection::drm_major
uint32_t drm_major
Definition: hwcontext_vulkan.c:1305
get_plane_buf
static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4345
AV_PIX_FMT_P010
#define AV_PIX_FMT_P010
Definition: pixfmt.h:602
unlock_frame
static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2955
FF_DISABLE_DEPRECATION_WARNINGS
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:72
AVVkFrame::sem_value
uint64_t sem_value[AV_NUM_DATA_POINTERS]
Up to date semaphore value at which each image becomes accessible.
Definition: hwcontext_vulkan.h:356
AV_PIX_FMT_GBRP
@ AV_PIX_FMT_GBRP
planar GBR 4:4:4 24bpp
Definition: pixfmt.h:165
av_strdup
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:272
desc
const char * desc
Definition: libsvtav1.c:82
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:116
AVVulkanFramesContext::nb_layers
int nb_layers
Number of layers each image will have.
Definition: hwcontext_vulkan.h:286
AV_PIX_FMT_YUV422P
@ AV_PIX_FMT_YUV422P
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:77
VulkanFramesPriv::download_exec
FFVkExecPool download_exec
Definition: hwcontext_vulkan.c:183
mem.h
AVVkFrame::layout
VkImageLayout layout[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:340
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
w
uint8_t w
Definition: llvidencdsp.c:39
hwcontext_internal.h
FF_VK_EXT_VIDEO_ENCODE_H264
#define FF_VK_EXT_VIDEO_ENCODE_H264
Definition: vulkan_functions.h:69
AV_HWFRAME_MAP_WRITE
@ AV_HWFRAME_MAP_WRITE
The mapping must be writeable.
Definition: hwcontext.h:519
ADD_QUEUE
#define ADD_QUEUE(ctx_qf, qc, flag)
AVVulkanDeviceContext::nb_enabled_inst_extensions
int nb_enabled_inst_extensions
Definition: hwcontext_vulkan.h:104
av_free
#define av_free(p)
Definition: tableprint_vlc.h:34
AVDictionaryEntry
Definition: dict.h:90
get_plane_wh
static void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format, int frame_w, int frame_h, int plane)
Definition: hwcontext_vulkan.c:2686
ff_vk_count_images
static int ff_vk_count_images(AVVkFrame *f)
Definition: vulkan.h:366
ff_vk_exec_discard_deps
void ff_vk_exec_discard_deps(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:592
PREP_MODE_EXTERNAL_EXPORT
@ PREP_MODE_EXTERNAL_EXPORT
Definition: hwcontext_vulkan.c:2520
FFALIGN
#define FFALIGN(x, a)
Definition: macros.h:78
AV_PIX_FMT_P416
#define AV_PIX_FMT_P416
Definition: pixfmt.h:621
VulkanDevicePriv::mprops
VkPhysicalDeviceMemoryProperties mprops
Definition: hwcontext_vulkan.c:137
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
FF_VULKAN_DEBUG_NB
@ FF_VULKAN_DEBUG_NB
Definition: hwcontext_vulkan.c:850
FFVkBuffer
Definition: vulkan.h:125
vk_dev_type
static const char * vk_dev_type(enum VkPhysicalDeviceType type)
Definition: hwcontext_vulkan.c:1314
VulkanDevicePriv::disable_multiplane
int disable_multiplane
Definition: hwcontext_vulkan.c:163
imgutils.h
ff_vk_exec_submit
int ff_vk_exec_submit(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:905
AV_PIX_FMT_XV36
#define AV_PIX_FMT_XV36
Definition: pixfmt.h:610
hwcontext.h
AVDRMPlaneDescriptor::pitch
ptrdiff_t pitch
Pitch (linesize) of this plane.
Definition: hwcontext_drm.h:87
AVFrame::linesize
int linesize[AV_NUM_DATA_POINTERS]
For video, a positive or negative value, which is typically indicating the size in bytes of each pict...
Definition: frame.h:472
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
FFVkFormatEntry::vk_planes
int vk_planes
Definition: hwcontext_vulkan.c:400
AVVulkanDeviceQueueFamily
Definition: hwcontext_vulkan.h:33
HWContextType
Definition: hwcontext_internal.h:29
FF_VK_EXT_VIDEO_ENCODE_AV1
#define FF_VK_EXT_VIDEO_ENCODE_AV1
Definition: vulkan_functions.h:71
AV_PIX_FMT_P412
#define AV_PIX_FMT_P412
Definition: pixfmt.h:619
device_features_init
static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
Definition: hwcontext_vulkan.c:214
VulkanDevicePriv::contiguous_planes
int contiguous_planes
Definition: hwcontext_vulkan.c:160
AVVAAPIDeviceContext
VAAPI connection details.
Definition: hwcontext_vaapi.h:68
h
h
Definition: vp9dsp_template.c:2070
AVVulkanDeviceContext::device_features
VkPhysicalDeviceFeatures2 device_features
This structure should be set to the set of features that present and enabled during device creation.
Definition: hwcontext_vulkan.h:92
AVDictionaryEntry::value
char * value
Definition: dict.h:92
avstring.h
AVDRMDeviceContext
DRM device.
Definition: hwcontext_drm.h:157
AV_PIX_FMT_GRAY12
#define AV_PIX_FMT_GRAY12
Definition: pixfmt.h:520
copy_buffer_data
static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf, AVFrame *swf, VkBufferImageCopy *region, int planes, int upload)
Definition: hwcontext_vulkan.c:4302
VulkanDeviceFeatures::atomic_float
VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float
Definition: hwcontext_vulkan.c:115
ADD_VAL_TO_LIST
#define ADD_VAL_TO_LIST(list, count, val)
Definition: hwcontext_vulkan.c:800
AV_PIX_FMT_BAYER_RGGB16
#define AV_PIX_FMT_BAYER_RGGB16
Definition: pixfmt.h:572
AVDRMFrameDescriptor::nb_objects
int nb_objects
Number of DRM objects making up this frame.
Definition: hwcontext_drm.h:137
VulkanDeviceFeatures::shader_object
VkPhysicalDeviceShaderObjectFeaturesEXT shader_object
Definition: hwcontext_vulkan.c:113
HWMapDescriptor
Definition: hwcontext_internal.h:120
AVVulkanDeviceQueueFamily::flags
VkQueueFlagBits flags
Definition: hwcontext_vulkan.h:41
AVVulkanDeviceQueueFamily::video_caps
VkVideoCodecOperationFlagBitsKHR video_caps
Definition: hwcontext_vulkan.h:44
FF_VK_EXT_ZERO_INITIALIZE
#define FF_VK_EXT_ZERO_INITIALIZE
Definition: vulkan_functions.h:52
FFVulkanFunctions
Definition: vulkan_functions.h:274
VulkanDevicePriv::p
AVVulkanDeviceContext p
The public AVVulkanDeviceContext.
Definition: hwcontext_vulkan.c:126
VulkanFramesPriv::modifier_info
VkImageDrmFormatModifierListCreateInfoEXT * modifier_info
Definition: hwcontext_vulkan.c:189
FFVkFormatEntry::aspect
VkImageAspectFlags aspect
Definition: hwcontext_vulkan.c:399
ff_vk_get_pooled_buffer
int ff_vk_get_pooled_buffer(FFVulkanContext *ctx, AVBufferPool **buf_pool, AVBufferRef **buf, VkBufferUsageFlags usage, void *create_pNext, size_t size, VkMemoryPropertyFlagBits mem_props)
Initialize a pool and create AVBufferRefs containing FFVkBuffer.
Definition: vulkan.c:1286
VulkanDeviceSelection::name
const char * name
Definition: hwcontext_vulkan.c:1308
src
#define src
Definition: vp8dsp.c:248
AV_HWDEVICE_TYPE_DRM
@ AV_HWDEVICE_TYPE_DRM
Definition: hwcontext.h:36
AVVulkanDeviceContext::nb_comp_queues
attribute_deprecated int nb_comp_queues
Definition: hwcontext_vulkan.h:148
AV_PIX_FMT_YUVA422P
@ AV_PIX_FMT_YUVA422P
planar YUV 4:2:2 24bpp, (1 Cr & Cb sample per 2x1 Y & A samples)
Definition: pixfmt.h:173
vulkan_device_create_internal
static int vulkan_device_create_internal(AVHWDeviceContext *ctx, VulkanDeviceSelection *dev_select, int disable_multiplane, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:1788
w32dlfcn.h
av_vk_get_optional_instance_extensions
const char ** av_vk_get_optional_instance_extensions(int *count)
Returns an array of optional Vulkan instance extensions that FFmpeg may use if enabled.
Definition: hwcontext_vulkan.c:739
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:3376
vulkan_device_has_rebar
static int vulkan_device_has_rebar(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:821
try_export_flags
static void try_export_flags(AVHWFramesContext *hwfc, VkExternalMemoryHandleTypeFlags *comp_handle_types, VkExternalMemoryHandleTypeFlags *iexp, VkExternalMemoryHandleTypeFlagBits exp)
Definition: hwcontext_vulkan.c:2813
Generated on Wed Mar 18 2026 19:23:55 for FFmpeg by   doxygen 1.8.17