FFmpeg
hwcontext_vulkan.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) Lynne
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #define VK_NO_PROTOTYPES
22 #define VK_ENABLE_BETA_EXTENSIONS
23 
24 #ifdef _WIN32
25 #include <windows.h> /* Included to prevent conflicts with CreateSemaphore */
26 #include <versionhelpers.h>
27 #include "compat/w32dlfcn.h"
28 #else
29 #include <dlfcn.h>
30 #include <unistd.h>
31 #endif
32 
33 #include "thread.h"
34 
35 #include "config.h"
36 #include "pixdesc.h"
37 #include "avstring.h"
38 #include "imgutils.h"
39 #include "hwcontext.h"
40 #include "hwcontext_internal.h"
41 #include "hwcontext_vulkan.h"
42 #include "mem.h"
43 
44 #include "vulkan.h"
45 #include "vulkan_loader.h"
46 
47 #if CONFIG_VAAPI
48 #include "hwcontext_vaapi.h"
49 #endif
50 
51 #if CONFIG_LIBDRM
52 #if CONFIG_VAAPI
53 #include <va/va_drmcommon.h>
54 #endif
55 #ifdef __linux__
56 #include <sys/sysmacros.h>
57 #endif
58 #include <sys/stat.h>
59 #include <xf86drm.h>
60 #include <drm_fourcc.h>
61 #include "hwcontext_drm.h"
62 #endif
63 
64 #if HAVE_LINUX_DMA_BUF_H
65 #include <sys/ioctl.h>
66 #include <linux/dma-buf.h>
67 #endif
68 
69 #if CONFIG_CUDA
70 #include "hwcontext_cuda_internal.h"
71 #include "cuda_check.h"
72 #define CHECK_CU(x) FF_CUDA_CHECK_DL(cuda_cu, cu, x)
73 #endif
74 
75 typedef struct VulkanDeviceFeatures {
76  VkPhysicalDeviceFeatures2 device;
77 
78  VkPhysicalDeviceVulkan11Features vulkan_1_1;
79  VkPhysicalDeviceVulkan12Features vulkan_1_2;
80  VkPhysicalDeviceVulkan13Features vulkan_1_3;
81  VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore;
82  VkPhysicalDeviceShaderSubgroupRotateFeaturesKHR subgroup_rotate;
83  VkPhysicalDeviceHostImageCopyFeaturesEXT host_image_copy;
84  VkPhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR explicit_mem_layout;
85 
86 #ifdef VK_EXT_shader_long_vector
87  VkPhysicalDeviceShaderLongVectorFeaturesEXT long_vector;
88 #endif
89 
90 #ifdef VK_EXT_shader_replicated_composites
91  VkPhysicalDeviceShaderReplicatedCompositesFeaturesEXT replicated_composites;
92 #endif
93 
94 #ifdef VK_EXT_zero_initialize_device_memory
95  VkPhysicalDeviceZeroInitializeDeviceMemoryFeaturesEXT zero_initialize;
96 #endif
97 
98 #ifdef VK_KHR_shader_expect_assume
99  VkPhysicalDeviceShaderExpectAssumeFeaturesKHR expect_assume;
100 #endif
101 
102  VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1;
103 #ifdef VK_KHR_video_maintenance2
104  VkPhysicalDeviceVideoMaintenance2FeaturesKHR video_maintenance_2;
105 #endif
106 #ifdef VK_KHR_video_decode_vp9
107  VkPhysicalDeviceVideoDecodeVP9FeaturesKHR vp9_decode;
108 #endif
109 #ifdef VK_KHR_video_encode_av1
110  VkPhysicalDeviceVideoEncodeAV1FeaturesKHR av1_encode;
111 #endif
112 
113  VkPhysicalDeviceShaderObjectFeaturesEXT shader_object;
114  VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix;
115  VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float;
116 
117 #ifdef VK_KHR_shader_relaxed_extended_instruction
118  VkPhysicalDeviceShaderRelaxedExtendedInstructionFeaturesKHR relaxed_extended_instruction;
119 #endif
120 } VulkanDeviceFeatures;
121 
122 typedef struct VulkanDevicePriv {
123  /**
124  * The public AVVulkanDeviceContext. See hwcontext_vulkan.h for it.
125  */
126  AVVulkanDeviceContext p;
127 
128  /* Vulkan library and loader functions */
129  void *libvulkan;
130 
131  FFVulkanContext vkctx;
132  AVVulkanDeviceQueueFamily *compute_qf;
133  AVVulkanDeviceQueueFamily *transfer_qf;
134 
135  /* Properties */
136  VkPhysicalDeviceProperties2 props;
137  VkPhysicalDeviceMemoryProperties mprops;
138  VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops;
139  VkPhysicalDeviceDriverProperties dprops;
140 
141  /* Opaque FD external semaphore properties */
142  VkExternalSemaphoreProperties ext_sem_props_opaque;
143 
144  /* Enabled features */
145  VulkanDeviceFeatures feats;
146 
147  /* Queues */
148  pthread_mutex_t **qf_mutex;
149  uint32_t nb_tot_qfs;
150  uint32_t img_qfs[64];
151  uint32_t nb_img_qfs;
152 
153  /* Debug callback */
154  VkDebugUtilsMessengerEXT debug_ctx;
155 
156  /* Settings */
157  int use_linear_images;
158 
159  /* Option to allocate all image planes in a single allocation */
160  int contiguous_planes;
161 
162  /* Disable multiplane images */
163  int disable_multiplane;
164 
165  /* Prefer memcpy over dynamic host pointer imports */
166  int avoid_host_import;
167 
168  /* Maximum queues */
169  int limit_queues;
170 } VulkanDevicePriv;
171 
172 typedef struct VulkanFramesPriv {
173  /**
174  * The public AVVulkanFramesContext. See hwcontext_vulkan.h for it.
175  */
176  AVVulkanFramesContext p;
177 
178  /* Image conversions */
179  FFVkExecPool compute_exec;
180 
181  /* Image transfers */
182  FFVkExecPool upload_exec;
183  FFVkExecPool download_exec;
184 
185  /* Temporary buffer pools */
186  AVBufferPool *tmp;
187 
188  /* Modifier info list to free at uninit */
189  VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
190 
191  /* Properties for DRM modifier for each plane in the image */
192  VkDrmFormatModifierPropertiesEXT drm_format_modifier_properties[5];
193 } VulkanFramesPriv;
194 
195 typedef struct AVVkFrameInternal {
196  pthread_mutex_t update_mutex;
197 
198 #if CONFIG_CUDA
199  /* Importing external memory into cuda is really expensive so we keep the
200  * memory imported all the time */
201  AVBufferRef *cuda_fc_ref; /* Need to keep it around for uninit */
202  CUexternalMemory ext_mem[AV_NUM_DATA_POINTERS];
203  CUmipmappedArray cu_mma[AV_NUM_DATA_POINTERS];
204  CUarray cu_array[AV_NUM_DATA_POINTERS];
205  CUexternalSemaphore cu_sem[AV_NUM_DATA_POINTERS];
206 #ifdef _WIN32
207  HANDLE ext_mem_handle[AV_NUM_DATA_POINTERS];
208  HANDLE ext_sem_handle[AV_NUM_DATA_POINTERS];
209 #endif
210 #endif
211 } AVVkFrameInternal;
212 
213 /* Initialize all structs in VulkanDeviceFeatures */
214 static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
215 {
216  VulkanDevicePriv *p = ctx->hwctx;
217  FFVulkanContext *s = &p->vkctx;
218 
219  feats->device = (VkPhysicalDeviceFeatures2) {
220  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
221  };
222 
223  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_1, FF_VK_EXT_NO_FLAG,
224  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES);
225  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_2, FF_VK_EXT_NO_FLAG,
226  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES);
227  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_3, FF_VK_EXT_NO_FLAG,
228  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES);
229 
230  FF_VK_STRUCT_EXT(s, &feats->device, &feats->timeline_semaphore, FF_VK_EXT_PORTABILITY_SUBSET,
231  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES);
232  FF_VK_STRUCT_EXT(s, &feats->device, &feats->subgroup_rotate, FF_VK_EXT_SUBGROUP_ROTATE,
233  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SUBGROUP_ROTATE_FEATURES_KHR);
234  FF_VK_STRUCT_EXT(s, &feats->device, &feats->host_image_copy, FF_VK_EXT_HOST_IMAGE_COPY,
235  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_IMAGE_COPY_FEATURES_EXT);
236 
237 #ifdef VK_EXT_shader_long_vector
238  FF_VK_STRUCT_EXT(s, &feats->device, &feats->long_vector, FF_VK_EXT_LONG_VECTOR,
239  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_LONG_VECTOR_FEATURES_EXT);
240 #endif
241 
242 #ifdef VK_EXT_shader_replicated_composites
243  FF_VK_STRUCT_EXT(s, &feats->device, &feats->replicated_composites, FF_VK_EXT_REPLICATED_COMPOSITES,
244  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_REPLICATED_COMPOSITES_FEATURES_EXT);
245 #endif
246 
247 #ifdef VK_EXT_zero_initialize_device_memory
248  FF_VK_STRUCT_EXT(s, &feats->device, &feats->zero_initialize, FF_VK_EXT_ZERO_INITIALIZE,
249  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ZERO_INITIALIZE_DEVICE_MEMORY_FEATURES_EXT);
250 #endif
251 
252 #ifdef VK_KHR_shader_expect_assume
253  FF_VK_STRUCT_EXT(s, &feats->device, &feats->expect_assume, FF_VK_EXT_EXPECT_ASSUME,
254  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_EXPECT_ASSUME_FEATURES_KHR);
255 #endif
256 
257  FF_VK_STRUCT_EXT(s, &feats->device, &feats->video_maintenance_1, FF_VK_EXT_VIDEO_MAINTENANCE_1,
258  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_1_FEATURES_KHR);
259 #ifdef VK_KHR_video_maintenance2
260  FF_VK_STRUCT_EXT(s, &feats->device, &feats->video_maintenance_2, FF_VK_EXT_VIDEO_MAINTENANCE_2,
261  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_2_FEATURES_KHR);
262 #endif
263 #ifdef VK_KHR_video_decode_vp9
264  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vp9_decode, FF_VK_EXT_VIDEO_DECODE_VP9,
265  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_DECODE_VP9_FEATURES_KHR);
266 #endif
267 #ifdef VK_KHR_video_encode_av1
268  FF_VK_STRUCT_EXT(s, &feats->device, &feats->av1_encode, FF_VK_EXT_VIDEO_ENCODE_AV1,
269  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_ENCODE_AV1_FEATURES_KHR);
270 #endif
271 
272  FF_VK_STRUCT_EXT(s, &feats->device, &feats->shader_object, FF_VK_EXT_SHADER_OBJECT,
273  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OBJECT_FEATURES_EXT);
274  FF_VK_STRUCT_EXT(s, &feats->device, &feats->cooperative_matrix, FF_VK_EXT_COOP_MATRIX,
275  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR);
276  FF_VK_STRUCT_EXT(s, &feats->device, &feats->atomic_float, FF_VK_EXT_ATOMIC_FLOAT,
277  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT);
278  FF_VK_STRUCT_EXT(s, &feats->device, &feats->explicit_mem_layout, FF_VK_EXT_EXPLICIT_MEM_LAYOUT,
279  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_FEATURES_KHR);
280 
281 #ifdef VK_KHR_shader_relaxed_extended_instruction
282  FF_VK_STRUCT_EXT(s, &feats->device, &feats->relaxed_extended_instruction, FF_VK_EXT_RELAXED_EXTENDED_INSTR,
283  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_RELAXED_EXTENDED_INSTRUCTION_FEATURES_KHR);
284 #endif
285 }
286 
287 /* Copy all needed device features */
288 static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
289 {
290 #define COPY_VAL(VAL) \
291  do { \
292  dst->VAL = src->VAL; \
293  } while (0) \
294 
295  COPY_VAL(device.features.shaderImageGatherExtended);
296  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
297  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
298  COPY_VAL(device.features.fragmentStoresAndAtomics);
299  COPY_VAL(device.features.vertexPipelineStoresAndAtomics);
300  COPY_VAL(device.features.shaderInt64);
301  COPY_VAL(device.features.shaderInt16);
302  COPY_VAL(device.features.shaderFloat64);
303  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
304  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
305 
306  COPY_VAL(vulkan_1_1.samplerYcbcrConversion);
307  COPY_VAL(vulkan_1_1.storagePushConstant16);
308  COPY_VAL(vulkan_1_1.storageBuffer16BitAccess);
309  COPY_VAL(vulkan_1_1.uniformAndStorageBuffer16BitAccess);
310 
311  COPY_VAL(vulkan_1_2.timelineSemaphore);
312  COPY_VAL(vulkan_1_2.scalarBlockLayout);
313  COPY_VAL(vulkan_1_2.bufferDeviceAddress);
314  COPY_VAL(vulkan_1_2.hostQueryReset);
315  COPY_VAL(vulkan_1_2.storagePushConstant8);
316  COPY_VAL(vulkan_1_2.shaderInt8);
317  COPY_VAL(vulkan_1_2.storageBuffer8BitAccess);
318  COPY_VAL(vulkan_1_2.uniformAndStorageBuffer8BitAccess);
319  COPY_VAL(vulkan_1_2.shaderFloat16);
320  COPY_VAL(vulkan_1_2.shaderBufferInt64Atomics);
321  COPY_VAL(vulkan_1_2.shaderSharedInt64Atomics);
322  COPY_VAL(vulkan_1_2.vulkanMemoryModel);
323  COPY_VAL(vulkan_1_2.vulkanMemoryModelDeviceScope);
324  COPY_VAL(vulkan_1_2.vulkanMemoryModelAvailabilityVisibilityChains);
325  COPY_VAL(vulkan_1_2.uniformBufferStandardLayout);
326  COPY_VAL(vulkan_1_2.runtimeDescriptorArray);
327  COPY_VAL(vulkan_1_2.shaderSubgroupExtendedTypes);
328  COPY_VAL(vulkan_1_2.shaderUniformBufferArrayNonUniformIndexing);
329  COPY_VAL(vulkan_1_2.shaderSampledImageArrayNonUniformIndexing);
330  COPY_VAL(vulkan_1_2.shaderStorageBufferArrayNonUniformIndexing);
331  COPY_VAL(vulkan_1_2.shaderStorageImageArrayNonUniformIndexing);
332 
333  COPY_VAL(vulkan_1_3.dynamicRendering);
334  COPY_VAL(vulkan_1_3.maintenance4);
335  COPY_VAL(vulkan_1_3.synchronization2);
336  COPY_VAL(vulkan_1_3.computeFullSubgroups);
337  COPY_VAL(vulkan_1_3.subgroupSizeControl);
338  COPY_VAL(vulkan_1_3.shaderZeroInitializeWorkgroupMemory);
339  COPY_VAL(vulkan_1_3.dynamicRendering);
340 
341  COPY_VAL(timeline_semaphore.timelineSemaphore);
342  COPY_VAL(subgroup_rotate.shaderSubgroupRotate);
343  COPY_VAL(host_image_copy.hostImageCopy);
344 
345 #ifdef VK_EXT_shader_long_vector
346  COPY_VAL(long_vector.longVector);
347 #endif
348 
349 #ifdef VK_EXT_shader_replicated_composites
350  COPY_VAL(replicated_composites.shaderReplicatedComposites);
351 #endif
352 
353 #ifdef VK_EXT_zero_initialize_device_memory
354  COPY_VAL(zero_initialize.zeroInitializeDeviceMemory);
355 #endif
356 
357  COPY_VAL(video_maintenance_1.videoMaintenance1);
358 #ifdef VK_KHR_video_maintenance2
359  COPY_VAL(video_maintenance_2.videoMaintenance2);
360 #endif
361 
362 #ifdef VK_KHR_video_decode_vp9
363  COPY_VAL(vp9_decode.videoDecodeVP9);
364 #endif
365 
366 #ifdef VK_KHR_video_encode_av1
367  COPY_VAL(av1_encode.videoEncodeAV1);
368 #endif
369 
370  COPY_VAL(shader_object.shaderObject);
371 
372  COPY_VAL(cooperative_matrix.cooperativeMatrix);
373 
374  COPY_VAL(atomic_float.shaderBufferFloat32Atomics);
375  COPY_VAL(atomic_float.shaderBufferFloat32AtomicAdd);
376 
377  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayout);
378  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayoutScalarBlockLayout);
379  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayout8BitAccess);
380  COPY_VAL(explicit_mem_layout.workgroupMemoryExplicitLayout16BitAccess);
381 
382 #ifdef VK_KHR_shader_relaxed_extended_instruction
383  COPY_VAL(relaxed_extended_instruction.shaderRelaxedExtendedInstruction);
384 #endif
385 
386 #ifdef VK_KHR_shader_expect_assume
387  COPY_VAL(expect_assume.shaderExpectAssume);
388 #endif
389 
390 #undef COPY_VAL
391 }
392 
393 #define ASPECT_2PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT)
394 #define ASPECT_3PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT | VK_IMAGE_ASPECT_PLANE_2_BIT)
395 
396 static const struct FFVkFormatEntry {
397  VkFormat vkf;
398  enum AVPixelFormat pixfmt;
399  VkImageAspectFlags aspect;
400  int vk_planes;
401  int nb_images;
402  int nb_images_fallback;
403  const VkFormat fallback[5];
404 } vk_formats_list[] = {
405  /* Gray formats */
406  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GRAY8, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8_UNORM } },
407  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
408  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY12, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
409  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY14, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
410  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
411  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GRAY32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_UINT } },
412  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GRAYF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_SFLOAT } },
413 
414  /* RGB formats */
415  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGRA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
416  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGBA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
417  { VK_FORMAT_R8G8B8_UNORM, AV_PIX_FMT_RGB24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8_UNORM } },
418  { VK_FORMAT_B8G8R8_UNORM, AV_PIX_FMT_BGR24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8_UNORM } },
419  { VK_FORMAT_R16G16B16_UNORM, AV_PIX_FMT_RGB48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16_UNORM } },
420  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_RGBA64, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
421  { VK_FORMAT_R5G6B5_UNORM_PACK16, AV_PIX_FMT_RGB565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R5G6B5_UNORM_PACK16 } },
422  { VK_FORMAT_B5G6R5_UNORM_PACK16, AV_PIX_FMT_BGR565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B5G6R5_UNORM_PACK16 } },
423  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGR0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
424  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGB0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
425  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_X2RGB10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2R10G10B10_UNORM_PACK32 } },
426  { VK_FORMAT_A2B10G10R10_UNORM_PACK32, AV_PIX_FMT_X2BGR10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2B10G10R10_UNORM_PACK32 } },
427  { VK_FORMAT_R32G32B32_SFLOAT, AV_PIX_FMT_RGBF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_SFLOAT } },
428  { VK_FORMAT_R32G32B32A32_SFLOAT, AV_PIX_FMT_RGBAF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_SFLOAT } },
429  { VK_FORMAT_R32G32B32_UINT, AV_PIX_FMT_RGB96, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_UINT } },
430  { VK_FORMAT_R32G32B32A32_UINT, AV_PIX_FMT_RGBA128, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_UINT } },
431 
432  /* Planar RGB */
433  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRP, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
434  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP10, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
435  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP12, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
436  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP14, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
437  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP16, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
438  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRPF32, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
439 
440  /* Planar RGB + Alpha */
441  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRAP, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
442  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
443  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
444  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP14, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
445  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
446  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GBRAP32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT } },
447  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRAPF32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
448 
449  /* Bayer */
450  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_BAYER_RGGB16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
451 
452  /* Two-plane 420 YUV at 8, 10, 12 and 16 bits */
453  { VK_FORMAT_G8_B8R8_2PLANE_420_UNORM, AV_PIX_FMT_NV12, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
454  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P010, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
455  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P012, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
456  { VK_FORMAT_G16_B16R16_2PLANE_420_UNORM, AV_PIX_FMT_P016, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
457 
458  /* Two-plane 422 YUV at 8, 10 and 16 bits */
459  { VK_FORMAT_G8_B8R8_2PLANE_422_UNORM, AV_PIX_FMT_NV16, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
460  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P210, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
461  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P212, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
462  { VK_FORMAT_G16_B16R16_2PLANE_422_UNORM, AV_PIX_FMT_P216, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
463 
464  /* Two-plane 444 YUV at 8, 10 and 16 bits */
465  { VK_FORMAT_G8_B8R8_2PLANE_444_UNORM, AV_PIX_FMT_NV24, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
466  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P410, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
467  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P412, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
468  { VK_FORMAT_G16_B16R16_2PLANE_444_UNORM, AV_PIX_FMT_P416, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
469 
470  /* Three-plane 420, 422, 444 at 8, 10, 12 and 16 bits */
471  { VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
472  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
473  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
474  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
475  { VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
476  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
477  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
478  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
479  { VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
480  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
481  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
482  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
483 
484  /* Single plane 422 at 8, 10, 12 and 16 bits */
485  { VK_FORMAT_G8B8G8R8_422_UNORM, AV_PIX_FMT_YUYV422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
486  { VK_FORMAT_B8G8R8G8_422_UNORM, AV_PIX_FMT_UYVY422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
487  { VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16, AV_PIX_FMT_Y210, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
488  { VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16, AV_PIX_FMT_Y212, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
489  { VK_FORMAT_G16B16G16R16_422_UNORM, AV_PIX_FMT_Y216, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
490 
491  /* Planar YUVA 420 at 8, 10 and 16 bits */
492  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA420P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
493  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA420P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
494  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA420P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
495 
496  /* Planar YUVA 422 at 8, 10, 12 and 16 bits */
497  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA422P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
498  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
499  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
500  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
501 
502  /* Planar YUVA 444 at 8, 10, 12 and 16 bits */
503  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA444P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
504  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
505  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
506  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
507 
508  /* Single plane 444 at 8, 10, 12 and 16 bits */
509  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_UYVA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
510  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_XV30, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
511  { VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16, AV_PIX_FMT_XV36, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
512  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_XV48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
513 };
514 static const int nb_vk_formats_list = FF_ARRAY_ELEMS(vk_formats_list);
515 
516 const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p)
517 {
518  for (int i = 0; i < nb_vk_formats_list; i++)
519  if (vk_formats_list[i].pixfmt == p)
520  return vk_formats_list[i].fallback;
521  return NULL;
522 }
523 
524 static const struct FFVkFormatEntry *vk_find_format_entry(enum AVPixelFormat p)
525 {
526  for (int i = 0; i < nb_vk_formats_list; i++)
527  if (vk_formats_list[i].pixfmt == p)
528  return &vk_formats_list[i];
529  return NULL;
530 }
531 
532 static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
533  VkImageTiling tiling,
534  VkFormat fmts[AV_NUM_DATA_POINTERS], /* Output format list */
535  int *nb_images, /* Output number of images */
536  VkImageAspectFlags *aspect, /* Output aspect */
537  VkImageUsageFlags *supported_usage, /* Output supported usage */
538  int disable_multiplane, int need_storage)
539 {
540  VulkanDevicePriv *priv = dev_ctx->hwctx;
541  AVVulkanDeviceContext *hwctx = &priv->p;
542  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
543 
544  const VkFormatFeatureFlagBits2 basic_flags = VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT |
545  VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
546  VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
547 
548  for (int i = 0; i < nb_vk_formats_list; i++) {
549  if (vk_formats_list[i].pixfmt == p) {
550  VkFormatProperties3 fprops = {
551  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3,
552  };
553  VkFormatProperties2 prop = {
554  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
555  .pNext = &fprops,
556  };
557  VkFormatFeatureFlagBits2 feats_primary, feats_secondary;
558  int basics_primary = 0, basics_secondary = 0;
559  int storage_primary = 0, storage_secondary = 0;
560 
561  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
562  vk_formats_list[i].vkf,
563  &prop);
564 
565  feats_primary = tiling == VK_IMAGE_TILING_LINEAR ?
566  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
567  basics_primary = (feats_primary & basic_flags) == basic_flags;
568  storage_primary = !!(feats_primary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
569 
570  if (vk_formats_list[i].vkf != vk_formats_list[i].fallback[0]) {
571  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
572  vk_formats_list[i].fallback[0],
573  &prop);
574  feats_secondary = tiling == VK_IMAGE_TILING_LINEAR ?
575  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
576  basics_secondary = (feats_secondary & basic_flags) == basic_flags;
577  storage_secondary = !!(feats_secondary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
578  } else {
579  basics_secondary = basics_primary;
580  storage_secondary = storage_primary;
581  }
582 
583  if (basics_primary &&
584  !(disable_multiplane && vk_formats_list[i].vk_planes > 1) &&
585  (!need_storage || (need_storage && (storage_primary | storage_secondary)))) {
586  if (fmts) {
587  if (vk_formats_list[i].nb_images > 1) {
588  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
589  fmts[j] = vk_formats_list[i].fallback[j];
590  } else {
591  fmts[0] = vk_formats_list[i].vkf;
592  }
593  }
594  if (nb_images)
595  *nb_images = 1;
596  if (aspect)
597  *aspect = vk_formats_list[i].aspect;
598  if (supported_usage)
599  *supported_usage = ff_vk_map_feats_to_usage(feats_primary) |
600  ((need_storage && (storage_primary | storage_secondary)) ?
601  VK_IMAGE_USAGE_STORAGE_BIT : 0);
602  return 0;
603  } else if (basics_secondary &&
604  (!need_storage || (need_storage && storage_secondary))) {
605  if (fmts) {
606  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
607  fmts[j] = vk_formats_list[i].fallback[j];
608  }
609  if (nb_images)
610  *nb_images = vk_formats_list[i].nb_images_fallback;
611  if (aspect)
612  *aspect = vk_formats_list[i].aspect;
613  if (supported_usage)
614  *supported_usage = ff_vk_map_feats_to_usage(feats_secondary);
615  return 0;
616  } else {
617  return AVERROR(ENOTSUP);
618  }
619  }
620  }
621 
622  return AVERROR(EINVAL);
623 }
624 
625 #if CONFIG_VULKAN_STATIC
626 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,
627  const char *pName);
628 #endif
629 
630 static int load_libvulkan(AVHWDeviceContext *ctx)
631 {
632  VulkanDevicePriv *p = ctx->hwctx;
633  AVVulkanDeviceContext *hwctx = &p->p;
634 
635 #if CONFIG_VULKAN_STATIC
636  hwctx->get_proc_addr = vkGetInstanceProcAddr;
637 #else
638  static const char *lib_names[] = {
639 #if defined(_WIN32)
640  "vulkan-1.dll",
641 #elif defined(__APPLE__)
642  "libvulkan.dylib",
643  "libvulkan.1.dylib",
644  "libMoltenVK.dylib",
645 #else
646  "libvulkan.so.1",
647  "libvulkan.so",
648 #endif
649  };
650 
651  for (int i = 0; i < FF_ARRAY_ELEMS(lib_names); i++) {
652  p->libvulkan = dlopen(lib_names[i], RTLD_NOW | RTLD_LOCAL);
653  if (p->libvulkan)
654  break;
655  }
656 
657  if (!p->libvulkan) {
658  av_log(ctx, AV_LOG_ERROR, "Unable to open the libvulkan library!\n");
659  return AVERROR_UNKNOWN;
660  }
661 
662  hwctx->get_proc_addr = (PFN_vkGetInstanceProcAddr)dlsym(p->libvulkan, "vkGetInstanceProcAddr");
663 #endif /* CONFIG_VULKAN_STATIC */
664 
665  return 0;
666 }
667 
668 typedef struct VulkanOptExtension {
669  const char *name;
670  FFVulkanExtensions flag;
671 } VulkanOptExtension;
672 
673 static const VulkanOptExtension optional_instance_exts[] = {
674 #ifdef __APPLE__
675  { VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME, FF_VK_EXT_NO_FLAG },
676 #endif
677  { 0 },
678 };
679 
680 static const VulkanOptExtension optional_device_exts[] = {
681  /* Misc or required by other extensions */
682  { VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME, FF_VK_EXT_PORTABILITY_SUBSET },
683  { VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, FF_VK_EXT_PUSH_DESCRIPTOR },
684  { VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME, FF_VK_EXT_DEVICE_DRM },
685  { VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, FF_VK_EXT_ATOMIC_FLOAT },
686  { VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME, FF_VK_EXT_COOP_MATRIX },
687  { VK_EXT_SHADER_OBJECT_EXTENSION_NAME, FF_VK_EXT_SHADER_OBJECT },
688  { VK_KHR_SHADER_SUBGROUP_ROTATE_EXTENSION_NAME, FF_VK_EXT_SUBGROUP_ROTATE },
689  { VK_EXT_HOST_IMAGE_COPY_EXTENSION_NAME, FF_VK_EXT_HOST_IMAGE_COPY },
690  { VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME, FF_VK_EXT_EXPLICIT_MEM_LAYOUT },
691 #ifdef VK_KHR_shader_relaxed_extended_instruction
692  { VK_KHR_SHADER_RELAXED_EXTENDED_INSTRUCTION_EXTENSION_NAME, FF_VK_EXT_RELAXED_EXTENDED_INSTR },
693 #endif
694 #ifdef VK_EXT_shader_long_vector
695  { VK_EXT_SHADER_LONG_VECTOR_EXTENSION_NAME, FF_VK_EXT_LONG_VECTOR },
696 #endif
697 #ifdef VK_EXT_shader_replicated_composites
698  { VK_EXT_SHADER_REPLICATED_COMPOSITES_EXTENSION_NAME, FF_VK_EXT_REPLICATED_COMPOSITES },
699 #endif
700 #ifdef VK_EXT_zero_initialize_device_memory
701  { VK_EXT_ZERO_INITIALIZE_DEVICE_MEMORY_EXTENSION_NAME, FF_VK_EXT_ZERO_INITIALIZE },
702 #endif
703 #ifdef VK_KHR_shader_expect_assume
704  { VK_KHR_SHADER_EXPECT_ASSUME_EXTENSION_NAME, FF_VK_EXT_EXPECT_ASSUME },
705 #endif
706  { VK_KHR_VIDEO_MAINTENANCE_1_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_1 },
707 #ifdef VK_KHR_video_maintenance2
708  { VK_KHR_VIDEO_MAINTENANCE_2_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_2 },
709 #endif
710 
711  /* Imports/exports */
712  { VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_MEMORY },
713  { VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_DMABUF_MEMORY },
714  { VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, FF_VK_EXT_DRM_MODIFIER_FLAGS },
715  { VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_SEM },
716  { VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_HOST_MEMORY },
717 #ifdef _WIN32
718  { VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_MEMORY },
719  { VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_SEM },
720 #endif
721 
722  /* Video encoding/decoding */
723  { VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_QUEUE },
724  { VK_KHR_VIDEO_ENCODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_QUEUE },
725  { VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_QUEUE },
726  { VK_KHR_VIDEO_ENCODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H264 },
727  { VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H264 },
728  { VK_KHR_VIDEO_ENCODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H265 },
729  { VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H265 },
730 #ifdef VK_KHR_video_decode_vp9
731  { VK_KHR_VIDEO_DECODE_VP9_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_VP9 },
732 #endif
733 #ifdef VK_KHR_video_encode_av1
734  { VK_KHR_VIDEO_ENCODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_AV1 },
735 #endif
736  { VK_KHR_VIDEO_DECODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_AV1 },
737 };
738 
739 const char **av_vk_get_optional_instance_extensions(int *count)
740 {
741  const char **exts = av_malloc_array(sizeof(*exts),
742  FF_ARRAY_ELEMS(optional_instance_exts) - 1);
743  if (!exts)
744  return NULL;
745 
746  for (int i = 0; i < FF_ARRAY_ELEMS(optional_instance_exts) - 1; i++)
747  exts[i] = optional_instance_exts[i].name;
748 
749  *count = FF_ARRAY_ELEMS(optional_instance_exts) - 1;
750  return exts;
751 }
752 
753 const char **av_vk_get_optional_device_extensions(int *count)
754 {
755  const char **exts = av_malloc_array(sizeof(*exts),
756  FF_ARRAY_ELEMS(optional_device_exts));
757  if (!exts)
758  return NULL;
759 
760  for (int i = 0; i < FF_ARRAY_ELEMS(optional_device_exts); i++)
761  exts[i] = optional_device_exts[i].name;
762 
763  *count = FF_ARRAY_ELEMS(optional_device_exts);
764  return exts;
765 }
766 
767 static VKAPI_ATTR
768 VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
769  VkDebugUtilsMessageTypeFlagsEXT messageType,
770  const VkDebugUtilsMessengerCallbackDataEXT *data,
771  void *priv)
772 {
773  int l;
774  AVHWDeviceContext *ctx = priv;
775 
776  /* Ignore false positives */
777  switch (data->messageIdNumber) {
778  case 0x086974c1: /* BestPractices-vkCreateCommandPool-command-buffer-reset */
779  case 0xfd92477a: /* BestPractices-vkAllocateMemory-small-allocation */
780  return VK_FALSE;
781  default:
782  break;
783  }
784 
785  switch (severity) {
786  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT: l = AV_LOG_VERBOSE; break;
787  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT: l = AV_LOG_INFO; break;
788  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT: l = AV_LOG_WARNING; break;
789  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT: l = AV_LOG_ERROR; break;
790  default: l = AV_LOG_DEBUG; break;
791  }
792 
793  av_log(ctx, l, "%s\n", data->pMessage);
794  for (int i = 0; i < data->cmdBufLabelCount; i++)
795  av_log(ctx, l, "\t%i: %s\n", i, data->pCmdBufLabels[i].pLabelName);
796 
797  return VK_FALSE;
798 }
799 
800 #define ADD_VAL_TO_LIST(list, count, val) \
801  do { \
802  list = av_realloc_array(list, ++count, sizeof(*list)); \
803  if (!list) { \
804  err = AVERROR(ENOMEM); \
805  goto fail; \
806  } \
807  list[count - 1] = av_strdup(val); \
808  if (!list[count - 1]) { \
809  err = AVERROR(ENOMEM); \
810  goto fail; \
811  } \
812  } while(0)
813 
814 #define RELEASE_PROPS(props, count) \
815  if (props) { \
816  for (int i = 0; i < count; i++) \
817  av_free((void *)((props)[i])); \
818  av_free((void *)props); \
819  }
820 
821 static int vulkan_device_has_rebar(AVHWDeviceContext *ctx)
822 {
823  VulkanDevicePriv *p = ctx->hwctx;
824  VkDeviceSize max_vram = 0, max_visible_vram = 0;
825 
826  /* Get device memory properties */
827  av_assert0(p->mprops.memoryTypeCount);
828  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
829  const VkMemoryType type = p->mprops.memoryTypes[i];
830  const VkMemoryHeap heap = p->mprops.memoryHeaps[type.heapIndex];
831  if (!(type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
832  continue;
833  max_vram = FFMAX(max_vram, heap.size);
834  if (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
835  max_visible_vram = FFMAX(max_visible_vram, heap.size);
836  }
837 
838  return max_vram - max_visible_vram < 1024; /* 1 kB tolerance */
839 }
840 
841 enum FFVulkanDebugMode {
842  FF_VULKAN_DEBUG_NONE = 0,
843  /* Standard GPU-assisted validation */
844  FF_VULKAN_DEBUG_VALIDATE = 1,
845  /* Passes printfs in shaders to the debug callback */
846  FF_VULKAN_DEBUG_PRINTF = 2,
847  /* Enables extra printouts */
848  FF_VULKAN_DEBUG_PRACTICES = 3,
849 
850  FF_VULKAN_DEBUG_NB,
851 };
852 
853 static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts,
854  const char * const **dst, uint32_t *num,
855  enum FFVulkanDebugMode debug_mode)
856 {
857  const char *tstr;
858  const char **extension_names = NULL;
859  VulkanDevicePriv *p = ctx->hwctx;
860  AVVulkanDeviceContext *hwctx = &p->p;
861  FFVulkanFunctions *vk = &p->vkctx.vkfn;
862  int err = 0, found, extensions_found = 0;
863 
864  const char *mod;
865  int optional_exts_num;
866  uint32_t sup_ext_count;
867  char *user_exts_str = NULL;
868  AVDictionaryEntry *user_exts;
869  VkExtensionProperties *sup_ext;
870  const VulkanOptExtension *optional_exts;
871 
872  if (!dev) {
873  mod = "instance";
874  optional_exts = optional_instance_exts;
875  optional_exts_num = FF_ARRAY_ELEMS(optional_instance_exts) - 1;
876  user_exts = av_dict_get(opts, "instance_extensions", NULL, 0);
877  if (user_exts) {
878  user_exts_str = av_strdup(user_exts->value);
879  if (!user_exts_str) {
880  err = AVERROR(ENOMEM);
881  goto fail;
882  }
883  }
884  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, NULL);
885  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
886  if (!sup_ext)
887  return AVERROR(ENOMEM);
888  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, sup_ext);
889  } else {
890  mod = "device";
891  optional_exts = optional_device_exts;
892  optional_exts_num = FF_ARRAY_ELEMS(optional_device_exts);
893  user_exts = av_dict_get(opts, "device_extensions", NULL, 0);
894  if (user_exts) {
895  user_exts_str = av_strdup(user_exts->value);
896  if (!user_exts_str) {
897  err = AVERROR(ENOMEM);
898  goto fail;
899  }
900  }
901  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
902  &sup_ext_count, NULL);
903  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
904  if (!sup_ext)
905  return AVERROR(ENOMEM);
906  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
907  &sup_ext_count, sup_ext);
908  }
909 
910  for (int i = 0; i < optional_exts_num; i++) {
911  tstr = optional_exts[i].name;
912  found = 0;
913 
914  /* Check if the device has ReBAR for host image copies */
915  if (!strcmp(tstr, VK_EXT_HOST_IMAGE_COPY_EXTENSION_NAME) &&
916  !vulkan_device_has_rebar(ctx))
917  continue;
918 
919  if (dev &&
920  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
921  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
922  (debug_mode == FF_VULKAN_DEBUG_PRACTICES)) &&
923  (!strcmp(tstr, VK_EXT_SHADER_OBJECT_EXTENSION_NAME))) {
924  continue;
925  }
926 
927  for (int j = 0; j < sup_ext_count; j++) {
928  if (!strcmp(tstr, sup_ext[j].extensionName)) {
929  found = 1;
930  break;
931  }
932  }
933  if (!found)
934  continue;
935 
936  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
937  p->vkctx.extensions |= optional_exts[i].flag;
938  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
939  }
940 
941  if (!dev &&
942  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
943  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
944  (debug_mode == FF_VULKAN_DEBUG_PRACTICES))) {
945  tstr = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
946  found = 0;
947  for (int j = 0; j < sup_ext_count; j++) {
948  if (!strcmp(tstr, sup_ext[j].extensionName)) {
949  found = 1;
950  break;
951  }
952  }
953  if (found) {
954  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
955  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
956  } else {
957  av_log(ctx, AV_LOG_ERROR, "Debug extension \"%s\" not found!\n",
958  tstr);
959  err = AVERROR(EINVAL);
960  goto fail;
961  }
962  }
963 
964 #ifdef VK_KHR_shader_relaxed_extended_instruction
965  if ((debug_mode == FF_VULKAN_DEBUG_PRINTF) && dev) {
966  tstr = VK_KHR_SHADER_RELAXED_EXTENDED_INSTRUCTION_EXTENSION_NAME;
967  found = 0;
968  for (int j = 0; j < sup_ext_count; j++) {
969  if (!strcmp(tstr, sup_ext[j].extensionName)) {
970  found = 1;
971  break;
972  }
973  }
974  if (!found) {
975  av_log(ctx, AV_LOG_ERROR, "Debug_printf/profile enabled, but extension \"%s\" not found!\n",
976  tstr);
977  err = AVERROR(EINVAL);
978  goto fail;
979  }
980  }
981 #endif
982 
983  if (user_exts_str) {
984  char *save, *token = av_strtok(user_exts_str, "+", &save);
985  while (token) {
986  found = 0;
987  for (int j = 0; j < sup_ext_count; j++) {
988  if (!strcmp(token, sup_ext[j].extensionName)) {
989  found = 1;
990  break;
991  }
992  }
993  if (found) {
994  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension \"%s\"\n", mod, token);
995  ADD_VAL_TO_LIST(extension_names, extensions_found, token);
996  } else {
997  av_log(ctx, AV_LOG_WARNING, "%s extension \"%s\" not found, excluding.\n",
998  mod, token);
999  }
1000  token = av_strtok(NULL, "+", &save);
1001  }
1002  }
1003 
1004  *dst = extension_names;
1005  *num = extensions_found;
1006 
1007  av_free(user_exts_str);
1008  av_free(sup_ext);
1009  return 0;
1010 
1011 fail:
1012  RELEASE_PROPS(extension_names, extensions_found);
1013  av_free(user_exts_str);
1014  av_free(sup_ext);
1015  return err;
1016 }
1017 
1018 static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts,
1019  const char * const **dst, uint32_t *num,
1020  enum FFVulkanDebugMode *debug_mode)
1021 {
1022  int err = 0;
1023  VulkanDevicePriv *priv = ctx->hwctx;
1024  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
1025 
1026  static const char layer_standard_validation[] = { "VK_LAYER_KHRONOS_validation" };
1027  int layer_standard_validation_found = 0;
1028 
1029  uint32_t sup_layer_count;
1030  VkLayerProperties *sup_layers;
1031 
1032  AVDictionaryEntry *user_layers = av_dict_get(opts, "layers", NULL, 0);
1033  char *user_layers_str = NULL;
1034  char *save, *token;
1035 
1036  const char **enabled_layers = NULL;
1037  uint32_t enabled_layers_count = 0;
1038 
1039  AVDictionaryEntry *debug_opt = av_dict_get(opts, "debug", NULL, 0);
1040  enum FFVulkanDebugMode mode;
1041 
1042  *debug_mode = mode = FF_VULKAN_DEBUG_NONE;
1043 
1044  /* Get a list of all layers */
1045  vk->EnumerateInstanceLayerProperties(&sup_layer_count, NULL);
1046  sup_layers = av_malloc_array(sup_layer_count, sizeof(VkLayerProperties));
1047  if (!sup_layers)
1048  return AVERROR(ENOMEM);
1049  vk->EnumerateInstanceLayerProperties(&sup_layer_count, sup_layers);
1050 
1051  av_log(ctx, AV_LOG_VERBOSE, "Supported layers:\n");
1052  for (int i = 0; i < sup_layer_count; i++)
1053  av_log(ctx, AV_LOG_VERBOSE, "\t%s\n", sup_layers[i].layerName);
1054 
1055  /* If no user layers or debug layers are given, return */
1056  if (!debug_opt && !user_layers)
1057  goto end;
1058 
1059  /* Check for any properly supported validation layer */
1060  if (debug_opt) {
1061  if (!strcmp(debug_opt->value, "printf")) {
1062  mode = FF_VULKAN_DEBUG_PRINTF;
1063  } else if (!strcmp(debug_opt->value, "validate")) {
1064  mode = FF_VULKAN_DEBUG_VALIDATE;
1065  } else if (!strcmp(debug_opt->value, "practices")) {
1066  mode = FF_VULKAN_DEBUG_PRACTICES;
1067  } else {
1068  char *end_ptr = NULL;
1069  int idx = strtol(debug_opt->value, &end_ptr, 10);
1070  if (end_ptr == debug_opt->value || end_ptr[0] != '\0' ||
1071  idx < 0 || idx >= FF_VULKAN_DEBUG_NB) {
1072  av_log(ctx, AV_LOG_ERROR, "Invalid debugging mode \"%s\"\n",
1073  debug_opt->value);
1074  err = AVERROR(EINVAL);
1075  goto end;
1076  }
1077  mode = idx;
1078  }
1079  }
1080 
1081  /* If mode is VALIDATE or PRINTF, try to find the standard validation layer extension */
1082  if ((mode == FF_VULKAN_DEBUG_VALIDATE) ||
1083  (mode == FF_VULKAN_DEBUG_PRINTF) ||
1084  (mode == FF_VULKAN_DEBUG_PRACTICES)) {
1085  for (int i = 0; i < sup_layer_count; i++) {
1086  if (!strcmp(layer_standard_validation, sup_layers[i].layerName)) {
1087  av_log(ctx, AV_LOG_VERBOSE, "Standard validation layer %s is enabled\n",
1088  layer_standard_validation);
1089  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, layer_standard_validation);
1090  *debug_mode = mode;
1091  layer_standard_validation_found = 1;
1092  break;
1093  }
1094  }
1095  if (!layer_standard_validation_found) {
1096  av_log(ctx, AV_LOG_ERROR,
1097  "Validation Layer \"%s\" not supported\n", layer_standard_validation);
1098  err = AVERROR(ENOTSUP);
1099  goto end;
1100  }
1101  }
1102 
1103  /* Process any custom layers enabled */
1104  if (user_layers) {
1105  int found;
1106 
1107  user_layers_str = av_strdup(user_layers->value);
1108  if (!user_layers_str) {
1109  err = AVERROR(ENOMEM);
1110  goto fail;
1111  }
1112 
1113  token = av_strtok(user_layers_str, "+", &save);
1114  while (token) {
1115  found = 0;
1116 
1117  /* If debug=1/2 was specified as an option, skip this layer */
1118  if (!strcmp(layer_standard_validation, token) && layer_standard_validation_found) {
1119  token = av_strtok(NULL, "+", &save);
1120  break;
1121  }
1122 
1123  /* Try to find the layer in the list of supported layers */
1124  for (int j = 0; j < sup_layer_count; j++) {
1125  if (!strcmp(token, sup_layers[j].layerName)) {
1126  found = 1;
1127  break;
1128  }
1129  }
1130 
1131  if (found) {
1132  av_log(ctx, AV_LOG_VERBOSE, "Using layer: %s\n", token);
1133  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, token);
1134 
1135  /* If debug was not set as an option, force it */
1136  if (!strcmp(layer_standard_validation, token))
1137  *debug_mode = FF_VULKAN_DEBUG_VALIDATE;
1138  } else {
1139  av_log(ctx, AV_LOG_ERROR,
1140  "Layer \"%s\" not supported\n", token);
1141  err = AVERROR(EINVAL);
1142  goto end;
1143  }
1144 
1145  token = av_strtok(NULL, "+", &save);
1146  }
1147  }
1148 
1149 fail:
1150 end:
1151  av_free(sup_layers);
1152  av_free(user_layers_str);
1153 
1154  if (err < 0) {
1155  RELEASE_PROPS(enabled_layers, enabled_layers_count);
1156  } else {
1157  *dst = enabled_layers;
1158  *num = enabled_layers_count;
1159  }
1160 
1161  return err;
1162 }
1163 
1164 /* Creates a VkInstance */
1165 static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts,
1166  enum FFVulkanDebugMode *debug_mode)
1167 {
1168  int err = 0;
1169  VkResult ret;
1170  VulkanDevicePriv *p = ctx->hwctx;
1171  AVVulkanDeviceContext *hwctx = &p->p;
1172  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1173  VkApplicationInfo application_info = {
1174  .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
1175  .pApplicationName = "ffmpeg",
1176  .applicationVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1177  LIBAVUTIL_VERSION_MINOR,
1178  LIBAVUTIL_VERSION_MICRO),
1179  .pEngineName = "libavutil",
1180  .apiVersion = VK_API_VERSION_1_3,
1181  .engineVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1182  LIBAVUTIL_VERSION_MINOR,
1183  LIBAVUTIL_VERSION_MICRO),
1184  };
1185  VkValidationFeaturesEXT validation_features = {
1186  .sType = VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT,
1187  };
1188  VkInstanceCreateInfo inst_props = {
1189  .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
1190  .pApplicationInfo = &application_info,
1191  };
1192 
1193  if (!hwctx->get_proc_addr) {
1194  err = load_libvulkan(ctx);
1195  if (err < 0)
1196  return err;
1197  }
1198 
1199  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 0, 0);
1200  if (err < 0) {
1201  av_log(ctx, AV_LOG_ERROR, "Unable to load instance enumeration functions!\n");
1202  return err;
1203  }
1204 
1205  err = check_layers(ctx, opts, &inst_props.ppEnabledLayerNames,
1206  &inst_props.enabledLayerCount, debug_mode);
1207  if (err)
1208  goto fail;
1209 
1210  /* Check for present/missing extensions */
1211  err = check_extensions(ctx, 0, opts, &inst_props.ppEnabledExtensionNames,
1212  &inst_props.enabledExtensionCount, *debug_mode);
1213  hwctx->enabled_inst_extensions = inst_props.ppEnabledExtensionNames;
1214  hwctx->nb_enabled_inst_extensions = inst_props.enabledExtensionCount;
1215  if (err < 0)
1216  goto fail;
1217 
1218  /* Enable debug features if needed */
1219  if (*debug_mode == FF_VULKAN_DEBUG_VALIDATE) {
1220  static const VkValidationFeatureEnableEXT feat_list_validate[] = {
1221  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1222  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1223  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_EXT,
1224  };
1225  validation_features.pEnabledValidationFeatures = feat_list_validate;
1226  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_validate);
1227  inst_props.pNext = &validation_features;
1228  } else if (*debug_mode == FF_VULKAN_DEBUG_PRINTF) {
1229  static const VkValidationFeatureEnableEXT feat_list_debug[] = {
1230  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1231  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1232  VK_VALIDATION_FEATURE_ENABLE_DEBUG_PRINTF_EXT,
1233  };
1234  validation_features.pEnabledValidationFeatures = feat_list_debug;
1235  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_debug);
1236  inst_props.pNext = &validation_features;
1237  } else if (*debug_mode == FF_VULKAN_DEBUG_PRACTICES) {
1238  static const VkValidationFeatureEnableEXT feat_list_practices[] = {
1239  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1240  VK_VALIDATION_FEATURE_ENABLE_BEST_PRACTICES_EXT,
1241  };
1242  validation_features.pEnabledValidationFeatures = feat_list_practices;
1243  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_practices);
1244  inst_props.pNext = &validation_features;
1245  }
1246 
1247 #ifdef __APPLE__
1248  for (int i = 0; i < inst_props.enabledExtensionCount; i++) {
1249  if (!strcmp(VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME,
1250  inst_props.ppEnabledExtensionNames[i])) {
1251  inst_props.flags |= VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR;
1252  break;
1253  }
1254  }
1255 #endif
1256 
1257  /* Try to create the instance */
1258  ret = vk->CreateInstance(&inst_props, hwctx->alloc, &hwctx->inst);
1259 
1260  /* Check for errors */
1261  if (ret != VK_SUCCESS) {
1262  av_log(ctx, AV_LOG_ERROR, "Instance creation failure: %s\n",
1263  ff_vk_ret2str(ret));
1264  err = AVERROR_EXTERNAL;
1265  goto fail;
1266  }
1267 
1268  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 0);
1269  if (err < 0) {
1270  av_log(ctx, AV_LOG_ERROR, "Unable to load instance functions!\n");
1271  goto fail;
1272  }
1273 
1274  /* Setup debugging callback if needed */
1275  if ((*debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
1276  (*debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
1277  (*debug_mode == FF_VULKAN_DEBUG_PRACTICES)) {
1278  VkDebugUtilsMessengerCreateInfoEXT dbg = {
1279  .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
1280  .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT |
1281  VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
1282  VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
1283  VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
1284  .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
1285  VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
1286  VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
1287  .pfnUserCallback = vk_dbg_callback,
1288  .pUserData = ctx,
1289  };
1290 
1291  vk->CreateDebugUtilsMessengerEXT(hwctx->inst, &dbg,
1292  hwctx->alloc, &p->debug_ctx);
1293  }
1294 
1295  err = 0;
1296 
1297 fail:
1298  RELEASE_PROPS(inst_props.ppEnabledLayerNames, inst_props.enabledLayerCount);
1299  return err;
1300 }
1301 
1302 typedef struct VulkanDeviceSelection {
1303  uint8_t uuid[VK_UUID_SIZE]; /* Will use this first unless !has_uuid */
1304  int has_uuid;
1305  uint32_t drm_major; /* Will use this second unless !has_drm */
1306  uint32_t drm_minor; /* Will use this second unless !has_drm */
1307  uint32_t has_drm; /* has drm node info */
1308  const char *name; /* Will use this third unless NULL */
1309  uint32_t pci_device; /* Will use this fourth unless 0x0 */
1310  uint32_t vendor_id; /* Last resort to find something deterministic */
1311  int index; /* Finally fall back to index */
1312 } VulkanDeviceSelection;
1313 
1314 static const char *vk_dev_type(enum VkPhysicalDeviceType type)
1315 {
1316  switch (type) {
1317  case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "integrated";
1318  case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "discrete";
1319  case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "virtual";
1320  case VK_PHYSICAL_DEVICE_TYPE_CPU: return "software";
1321  default: return "unknown";
1322  }
1323 }
1324 
1325 /* Finds a device */
1326 static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
1327 {
1328  int err = 0, choice = -1;
1329  uint32_t num;
1330  VkResult ret;
1331  VulkanDevicePriv *p = ctx->hwctx;
1332  AVVulkanDeviceContext *hwctx = &p->p;
1333  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1334  VkPhysicalDevice *devices = NULL;
1335  VkPhysicalDeviceIDProperties *idp = NULL;
1336  VkPhysicalDeviceProperties2 *prop = NULL;
1337  VkPhysicalDeviceDriverProperties *driver_prop = NULL;
1338  VkPhysicalDeviceDrmPropertiesEXT *drm_prop = NULL;
1339 
1340  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, NULL);
1341  if (ret != VK_SUCCESS || !num) {
1342  av_log(ctx, AV_LOG_ERROR, "No devices found: %s!\n", ff_vk_ret2str(ret));
1343  return AVERROR(ENODEV);
1344  }
1345 
1346  devices = av_malloc_array(num, sizeof(VkPhysicalDevice));
1347  if (!devices)
1348  return AVERROR(ENOMEM);
1349 
1350  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, devices);
1351  if (ret != VK_SUCCESS) {
1352  av_log(ctx, AV_LOG_ERROR, "Failed enumerating devices: %s\n",
1353  ff_vk_ret2str(ret));
1354  err = AVERROR(ENODEV);
1355  goto end;
1356  }
1357 
1358  prop = av_calloc(num, sizeof(*prop));
1359  if (!prop) {
1360  err = AVERROR(ENOMEM);
1361  goto end;
1362  }
1363 
1364  idp = av_calloc(num, sizeof(*idp));
1365  if (!idp) {
1366  err = AVERROR(ENOMEM);
1367  goto end;
1368  }
1369 
1370  driver_prop = av_calloc(num, sizeof(*driver_prop));
1371  if (!driver_prop) {
1372  err = AVERROR(ENOMEM);
1373  goto end;
1374  }
1375 
1376  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1377  drm_prop = av_calloc(num, sizeof(*drm_prop));
1378  if (!drm_prop) {
1379  err = AVERROR(ENOMEM);
1380  goto end;
1381  }
1382  }
1383 
1384  av_log(ctx, AV_LOG_VERBOSE, "GPU listing:\n");
1385  for (int i = 0; i < num; i++) {
1386  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1387  drm_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT;
1388  driver_prop[i].pNext = &drm_prop[i];
1389  }
1390  driver_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
1391  idp[i].pNext = &driver_prop[i];
1392  idp[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES;
1393  prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1394  prop[i].pNext = &idp[i];
1395 
1396  vk->GetPhysicalDeviceProperties2(devices[i], &prop[i]);
1397  av_log(ctx, AV_LOG_VERBOSE, " %d: %s (%s) (0x%x)\n", i,
1398  prop[i].properties.deviceName,
1399  vk_dev_type(prop[i].properties.deviceType),
1400  prop[i].properties.deviceID);
1401  }
1402 
1403  if (select->has_uuid) {
1404  for (int i = 0; i < num; i++) {
1405  if (!strncmp(idp[i].deviceUUID, select->uuid, VK_UUID_SIZE)) {
1406  choice = i;
1407  goto end;
1408  }
1409  }
1410  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given UUID!\n");
1411  err = AVERROR(ENODEV);
1412  goto end;
1413  } else if ((p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) && select->has_drm) {
1414  for (int i = 0; i < num; i++) {
1415  if ((select->drm_major == drm_prop[i].primaryMajor &&
1416  select->drm_minor == drm_prop[i].primaryMinor) ||
1417  (select->drm_major == drm_prop[i].renderMajor &&
1418  select->drm_minor == drm_prop[i].renderMinor)) {
1419  choice = i;
1420  goto end;
1421  }
1422  }
1423  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given DRM node numbers %i:%i!\n",
1424  select->drm_major, select->drm_minor);
1425  err = AVERROR(ENODEV);
1426  goto end;
1427  } else if (select->name) {
1428  av_log(ctx, AV_LOG_VERBOSE, "Requested device: %s\n", select->name);
1429  for (int i = 0; i < num; i++) {
1430  if (strstr(prop[i].properties.deviceName, select->name)) {
1431  choice = i;
1432  goto end;
1433  }
1434  }
1435  av_log(ctx, AV_LOG_ERROR, "Unable to find device \"%s\"!\n",
1436  select->name);
1437  err = AVERROR(ENODEV);
1438  goto end;
1439  } else if (select->pci_device) {
1440  av_log(ctx, AV_LOG_VERBOSE, "Requested device: 0x%x\n", select->pci_device);
1441  for (int i = 0; i < num; i++) {
1442  if (select->pci_device == prop[i].properties.deviceID) {
1443  choice = i;
1444  goto end;
1445  }
1446  }
1447  av_log(ctx, AV_LOG_ERROR, "Unable to find device with PCI ID 0x%x!\n",
1448  select->pci_device);
1449  err = AVERROR(EINVAL);
1450  goto end;
1451  } else if (select->vendor_id) {
1452  av_log(ctx, AV_LOG_VERBOSE, "Requested vendor: 0x%x\n", select->vendor_id);
1453  for (int i = 0; i < num; i++) {
1454  if (select->vendor_id == prop[i].properties.vendorID) {
1455  choice = i;
1456  goto end;
1457  }
1458  }
1459  av_log(ctx, AV_LOG_ERROR, "Unable to find device with Vendor ID 0x%x!\n",
1460  select->vendor_id);
1461  err = AVERROR(ENODEV);
1462  goto end;
1463  } else {
1464  if (select->index < num) {
1465  choice = select->index;
1466  goto end;
1467  }
1468  av_log(ctx, AV_LOG_ERROR, "Unable to find device with index %i!\n",
1469  select->index);
1470  err = AVERROR(ENODEV);
1471  goto end;
1472  }
1473 
1474 end:
1475  if (choice > -1) {
1476  av_log(ctx, AV_LOG_VERBOSE, "Device %d selected: %s (%s) (0x%x)\n",
1477  choice, prop[choice].properties.deviceName,
1478  vk_dev_type(prop[choice].properties.deviceType),
1479  prop[choice].properties.deviceID);
1480  hwctx->phys_dev = devices[choice];
1481  p->props = prop[choice];
1482  p->props.pNext = NULL;
1483  p->dprops = driver_prop[choice];
1484  p->dprops.pNext = NULL;
1485  }
1486 
1487  av_free(devices);
1488  av_free(prop);
1489  av_free(idp);
1490  av_free(drm_prop);
1491  av_free(driver_prop);
1492 
1493  return err;
1494 }
1495 
1496 /* Picks the least used qf with the fewest unneeded flags, or -1 if none found */
1497 static inline int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf,
1498  VkQueueFlagBits flags)
1499 {
1500  int index = -1;
1501  uint32_t min_score = UINT32_MAX;
1502 
1503  for (int i = 0; i < num_qf; i++) {
1504  VkQueueFlagBits qflags = qf[i].queueFamilyProperties.queueFlags;
1505 
1506  /* Per the spec, reporting transfer caps is optional for these 2 types */
1507  if ((flags & VK_QUEUE_TRANSFER_BIT) &&
1508  (qflags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT)))
1509  qflags |= VK_QUEUE_TRANSFER_BIT;
1510 
1511  if (qflags & flags) {
1512  uint32_t score = av_popcount(qflags) + qf[i].queueFamilyProperties.timestampValidBits;
1513  if (score < min_score) {
1514  index = i;
1515  min_score = score;
1516  }
1517  }
1518  }
1519 
1520  if (index > -1)
1521  qf[index].queueFamilyProperties.timestampValidBits++;
1522 
1523  return index;
1524 }
1525 
1526 static inline int pick_video_queue_family(VkQueueFamilyProperties2 *qf,
1527  VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf,
1528  VkVideoCodecOperationFlagsKHR flags)
1529 {
1530  int index = -1;
1531  uint32_t min_score = UINT32_MAX;
1532 
1533  for (int i = 0; i < num_qf; i++) {
1534  const VkQueueFlags qflags = qf[i].queueFamilyProperties.queueFlags;
1535  const VkVideoCodecOperationFlagsKHR vflags = qf_vid[i].videoCodecOperations;
1536 
1537  if (!(qflags & (VK_QUEUE_VIDEO_ENCODE_BIT_KHR | VK_QUEUE_VIDEO_DECODE_BIT_KHR)))
1538  continue;
1539 
1540  if (vflags & flags) {
1541  uint32_t score = av_popcount(vflags) + qf[i].queueFamilyProperties.timestampValidBits;
1542  if (score < min_score) {
1543  index = i;
1544  min_score = score;
1545  }
1546  }
1547  }
1548 
1549  if (index > -1)
1550  qf[index].queueFamilyProperties.timestampValidBits++;
1551 
1552  return index;
1553 }
1554 
1555 static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
1556 {
1557  uint32_t num;
1558  VulkanDevicePriv *p = ctx->hwctx;
1559  AVVulkanDeviceContext *hwctx = &p->p;
1560  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1561 
1562  VkQueueFamilyProperties2 *qf = NULL;
1563  VkQueueFamilyVideoPropertiesKHR *qf_vid = NULL;
1564 
1565  /* First get the number of queue families */
1566  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &num, NULL);
1567  if (!num) {
1568  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1569  return AVERROR_EXTERNAL;
1570  }
1571 
1572  /* Then allocate memory */
1573  qf = av_malloc_array(num, sizeof(VkQueueFamilyProperties2));
1574  if (!qf)
1575  return AVERROR(ENOMEM);
1576 
1577  qf_vid = av_malloc_array(num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1578  if (!qf_vid)
1579  return AVERROR(ENOMEM);
1580 
1581  for (uint32_t i = 0; i < num; i++) {
1582  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1583  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1584  };
1585  qf[i] = (VkQueueFamilyProperties2) {
1586  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1587  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1588  };
1589  }
1590 
1591  /* Finally retrieve the queue families */
1592  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &num, qf);
1593 
1594  av_log(ctx, AV_LOG_VERBOSE, "Queue families:\n");
1595  for (int i = 0; i < num; i++) {
1596  av_log(ctx, AV_LOG_VERBOSE, " %i:%s%s%s%s%s%s%s%s (queues: %i)\n", i,
1597  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_GRAPHICS_BIT) ? " graphics" : "",
1598  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_COMPUTE_BIT) ? " compute" : "",
1599  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_TRANSFER_BIT) ? " transfer" : "",
1600  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_ENCODE_BIT_KHR) ? " encode" : "",
1601  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_DECODE_BIT_KHR) ? " decode" : "",
1602  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_SPARSE_BINDING_BIT) ? " sparse" : "",
1603  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_OPTICAL_FLOW_BIT_NV) ? " optical_flow" : "",
1604  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_PROTECTED_BIT) ? " protected" : "",
1605  qf[i].queueFamilyProperties.queueCount);
1606 
1607  /* We use this field to keep a score of how many times we've used that
1608  * queue family in order to make better choices. */
1609  qf[i].queueFamilyProperties.timestampValidBits = 0;
1610  }
1611 
1612  hwctx->nb_qf = 0;
1613 
1614  /* Pick each queue family to use. */
1615 #define PICK_QF(type, vid_op) \
1616  do { \
1617  uint32_t i; \
1618  uint32_t idx; \
1619  \
1620  if (vid_op) \
1621  idx = pick_video_queue_family(qf, qf_vid, num, vid_op); \
1622  else \
1623  idx = pick_queue_family(qf, num, type); \
1624  \
1625  if (idx == -1) \
1626  continue; \
1627  \
1628  for (i = 0; i < hwctx->nb_qf; i++) { \
1629  if (hwctx->qf[i].idx == idx) { \
1630  hwctx->qf[i].flags |= type; \
1631  hwctx->qf[i].video_caps |= vid_op; \
1632  break; \
1633  } \
1634  } \
1635  if (i == hwctx->nb_qf) { \
1636  hwctx->qf[i].idx = idx; \
1637  hwctx->qf[i].num = qf[idx].queueFamilyProperties.queueCount; \
1638  if (p->limit_queues || \
1639  p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY) { \
1640  int max = p->limit_queues; \
1641  if (type == VK_QUEUE_GRAPHICS_BIT) \
1642  hwctx->qf[i].num = FFMIN(hwctx->qf[i].num, \
1643  max ? max : 1); \
1644  else if (max) \
1645  hwctx->qf[i].num = FFMIN(hwctx->qf[i].num, max); \
1646  } \
1647  hwctx->qf[i].flags = type; \
1648  hwctx->qf[i].video_caps = vid_op; \
1649  hwctx->nb_qf++; \
1650  } \
1651  } while (0)
1652 
1653  PICK_QF(VK_QUEUE_GRAPHICS_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1654  PICK_QF(VK_QUEUE_COMPUTE_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1655  PICK_QF(VK_QUEUE_TRANSFER_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1656 
1657  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR);
1658  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR);
1659 
1660  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR);
1661  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR);
1662 
1663 #ifdef VK_KHR_video_decode_vp9
1664  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_VP9_BIT_KHR);
1665 #endif
1666 
1667 #ifdef VK_KHR_video_encode_av1
1668  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_AV1_BIT_KHR);
1669 #endif
1670  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR);
1671 
1672  av_free(qf);
1673  av_free(qf_vid);
1674 
1675 #undef PICK_QF
1676 
1677  cd->pQueueCreateInfos = av_malloc_array(hwctx->nb_qf,
1678  sizeof(VkDeviceQueueCreateInfo));
1679  if (!cd->pQueueCreateInfos)
1680  return AVERROR(ENOMEM);
1681 
1682  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1683  int dup = 0;
1684  float *weights = NULL;
1685  VkDeviceQueueCreateInfo *pc;
1686  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++) {
1687  if (hwctx->qf[i].idx == cd->pQueueCreateInfos[j].queueFamilyIndex) {
1688  dup = 1;
1689  break;
1690  }
1691  }
1692  if (dup)
1693  continue;
1694 
1695  weights = av_malloc_array(hwctx->qf[i].num, sizeof(float));
1696  if (!weights) {
1697  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++)
1698  av_free((void *)cd->pQueueCreateInfos[i].pQueuePriorities);
1699  av_free((void *)cd->pQueueCreateInfos);
1700  return AVERROR(ENOMEM);
1701  }
1702 
1703  for (uint32_t j = 0; j < hwctx->qf[i].num; j++)
1704  weights[j] = 1.0;
1705 
1706  pc = (VkDeviceQueueCreateInfo *)cd->pQueueCreateInfos;
1707  pc[cd->queueCreateInfoCount++] = (VkDeviceQueueCreateInfo) {
1708  .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
1709  .queueFamilyIndex = hwctx->qf[i].idx,
1710  .queueCount = hwctx->qf[i].num,
1711  .pQueuePriorities = weights,
1712  };
1713  }
1714 
1715 #if FF_API_VULKAN_FIXED_QUEUES
1716 FF_DISABLE_DEPRECATION_WARNINGS
1717  /* Setup deprecated fields */
1718  hwctx->queue_family_index = -1;
1719  hwctx->queue_family_comp_index = -1;
1720  hwctx->queue_family_tx_index = -1;
1721  hwctx->queue_family_encode_index = -1;
1722  hwctx->queue_family_decode_index = -1;
1723 
1724 #define SET_OLD_QF(field, nb_field, type) \
1725  do { \
1726  if (field < 0 && hwctx->qf[i].flags & type) { \
1727  field = hwctx->qf[i].idx; \
1728  nb_field = hwctx->qf[i].num; \
1729  } \
1730  } while (0)
1731 
1732  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1733  SET_OLD_QF(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
1734  SET_OLD_QF(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
1735  SET_OLD_QF(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
1736  SET_OLD_QF(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
1737  SET_OLD_QF(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
1738  }
1739 
1740 #undef SET_OLD_QF
1741 FF_ENABLE_DEPRECATION_WARNINGS
1742 #endif
1743 
1744  return 0;
1745 }
1746 
1747 /* Only resources created by vulkan_device_create should be released here,
1748  * resources created by vulkan_device_init should be released by
1749  * vulkan_device_uninit, to make sure we don't free user provided resources,
1750  * and there is no leak.
1751  */
1752 static void vulkan_device_free(AVHWDeviceContext *ctx)
1753 {
1754  VulkanDevicePriv *p = ctx->hwctx;
1755  AVVulkanDeviceContext *hwctx = &p->p;
1756  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1757 
1758  if (hwctx->act_dev)
1759  vk->DestroyDevice(hwctx->act_dev, hwctx->alloc);
1760 
1761  if (p->debug_ctx)
1762  vk->DestroyDebugUtilsMessengerEXT(hwctx->inst, p->debug_ctx,
1763  hwctx->alloc);
1764 
1765  if (hwctx->inst)
1766  vk->DestroyInstance(hwctx->inst, hwctx->alloc);
1767 
1768  if (p->libvulkan)
1769  dlclose(p->libvulkan);
1770 
1771  RELEASE_PROPS(hwctx->enabled_inst_extensions, hwctx->nb_enabled_inst_extensions);
1772  RELEASE_PROPS(hwctx->enabled_dev_extensions, hwctx->nb_enabled_dev_extensions);
1773 }
1774 
1775 static void vulkan_device_uninit(AVHWDeviceContext *ctx)
1776 {
1777  VulkanDevicePriv *p = ctx->hwctx;
1778 
1779  for (uint32_t i = 0; i < p->nb_tot_qfs; i++) {
1780  pthread_mutex_destroy(p->qf_mutex[i]);
1781  av_freep(&p->qf_mutex[i]);
1782  }
1783  av_freep(&p->qf_mutex);
1784 
1785  ff_vk_uninit(&p->vkctx);
1786 }
1787 
1788 static int vulkan_device_create_internal(AVHWDeviceContext *ctx,
1789  VulkanDeviceSelection *dev_select,
1790  int disable_multiplane,
1791  AVDictionary *opts, int flags)
1792 {
1793  int err = 0;
1794  VkResult ret;
1795  AVDictionaryEntry *opt_d;
1796  VulkanDevicePriv *p = ctx->hwctx;
1797  AVVulkanDeviceContext *hwctx = &p->p;
1798  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1799  enum FFVulkanDebugMode debug_mode = FF_VULKAN_DEBUG_NONE;
1800  VulkanDeviceFeatures supported_feats = { 0 };
1801  VkDeviceCreateInfo dev_info = {
1802  .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
1803  };
1804 
1805  /* Create an instance if not given one */
1806  if ((err = create_instance(ctx, opts, &debug_mode)))
1807  goto end;
1808 
1809  /* Find a physical device (if not given one) */
1810  if ((err = find_device(ctx, dev_select)))
1811  goto end;
1812 
1813  /* Get supported memory types */
1814  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
1815 
1816  /* Find and enable extensions for the physical device */
1817  if ((err = check_extensions(ctx, 1, opts, &dev_info.ppEnabledExtensionNames,
1818  &dev_info.enabledExtensionCount, debug_mode))) {
1819  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1820  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1821  av_free((void *)dev_info.pQueueCreateInfos);
1822  goto end;
1823  }
1824 
1825  /* Get all supported features for the physical device */
1826  device_features_init(ctx, &supported_feats);
1827  vk->GetPhysicalDeviceFeatures2(hwctx->phys_dev, &supported_feats.device);
1828 
1829  /* Copy all needed features from those supported and activate them */
1830  device_features_init(ctx, &p->feats);
1831  device_features_copy_needed(&p->feats, &supported_feats);
1832  dev_info.pNext = p->feats.device.pNext;
1833  dev_info.pEnabledFeatures = &p->feats.device.features;
1834 
1835  /* Limit queues to a given number if needed */
1836  opt_d = av_dict_get(opts, "limit_queues", NULL, 0);
1837  if (opt_d)
1838  p->limit_queues = strtol(opt_d->value, NULL, 10);
1839 
1840  /* Setup enabled queue families */
1841  if ((err = setup_queue_families(ctx, &dev_info)))
1842  goto end;
1843 
1844  /* Finally create the device */
1845  ret = vk->CreateDevice(hwctx->phys_dev, &dev_info, hwctx->alloc,
1846  &hwctx->act_dev);
1847 
1848  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1849  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1850  av_free((void *)dev_info.pQueueCreateInfos);
1851 
1852  if (ret != VK_SUCCESS) {
1853  av_log(ctx, AV_LOG_ERROR, "Device creation failure: %s\n",
1854  ff_vk_ret2str(ret));
1855  for (int i = 0; i < dev_info.enabledExtensionCount; i++)
1856  av_free((void *)dev_info.ppEnabledExtensionNames[i]);
1857  av_free((void *)dev_info.ppEnabledExtensionNames);
1858  err = AVERROR_EXTERNAL;
1859  goto end;
1860  }
1861 
1862  /* Tiled images setting, use them by default */
1863  opt_d = av_dict_get(opts, "linear_images", NULL, 0);
1864  if (opt_d)
1865  p->use_linear_images = strtol(opt_d->value, NULL, 10);
1866 
1867  /* The disable_multiplane argument takes precedent over the option */
1868  p->disable_multiplane = disable_multiplane;
1869  if (!p->disable_multiplane) {
1870  opt_d = av_dict_get(opts, "disable_multiplane", NULL, 0);
1871  if (opt_d)
1872  p->disable_multiplane = strtol(opt_d->value, NULL, 10);
1873  }
1874 
1875  /* Disable host pointer imports (by default on nvidia) */
1876  p->avoid_host_import = p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY;
1877  opt_d = av_dict_get(opts, "avoid_host_import", NULL, 0);
1878  if (opt_d)
1879  p->avoid_host_import = strtol(opt_d->value, NULL, 10);
1880 
1881  /* Set the public device feature struct and its pNext chain */
1882  hwctx->device_features = p->feats.device;
1883 
1884  /* Set the list of all active extensions */
1885  hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames;
1886  hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount;
1887 
1888  /* The extension lists need to be freed */
1889  ctx->free = vulkan_device_free;
1890 
1891 end:
1892  return err;
1893 }
1894 
1895 static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1896 {
1897  VulkanDevicePriv *p = ctx->hwctx;
1898  pthread_mutex_lock(&p->qf_mutex[queue_family][index]);
1899 }
1900 
1901 static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1902 {
1903  VulkanDevicePriv *p = ctx->hwctx;
1904  pthread_mutex_unlock(&p->qf_mutex[queue_family][index]);
1905 }
1906 
1907 static int vulkan_device_init(AVHWDeviceContext *ctx)
1908 {
1909  int err = 0;
1910  uint32_t qf_num;
1911  VulkanDevicePriv *p = ctx->hwctx;
1912  AVVulkanDeviceContext *hwctx = &p->p;
1913  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1914  VkQueueFamilyProperties2 *qf;
1915  VkQueueFamilyVideoPropertiesKHR *qf_vid;
1916  VkPhysicalDeviceExternalSemaphoreInfo ext_sem_props_info;
1917  int graph_index, comp_index, tx_index, enc_index, dec_index;
1918 
1919  /* Set device extension flags */
1920  for (int i = 0; i < hwctx->nb_enabled_dev_extensions; i++) {
1921  for (int j = 0; j < FF_ARRAY_ELEMS(optional_device_exts); j++) {
1922  if (!strcmp(hwctx->enabled_dev_extensions[i],
1923  optional_device_exts[j].name)) {
1924  p->vkctx.extensions |= optional_device_exts[j].flag;
1925  break;
1926  }
1927  }
1928  }
1929 
1930  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 1);
1931  if (err < 0) {
1932  av_log(ctx, AV_LOG_ERROR, "Unable to load functions!\n");
1933  return err;
1934  }
1935 
1936  p->props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1937  p->props.pNext = &p->hprops;
1938  p->hprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT;
1939  p->hprops.pNext = &p->dprops;
1940  p->dprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
1941 
1942  vk->GetPhysicalDeviceProperties2(hwctx->phys_dev, &p->props);
1943  av_log(ctx, AV_LOG_VERBOSE, "Using device: %s\n",
1944  p->props.properties.deviceName);
1945  av_log(ctx, AV_LOG_VERBOSE, "Alignments:\n");
1946  av_log(ctx, AV_LOG_VERBOSE, " optimalBufferCopyRowPitchAlignment: %"PRIu64"\n",
1947  p->props.properties.limits.optimalBufferCopyRowPitchAlignment);
1948  av_log(ctx, AV_LOG_VERBOSE, " minMemoryMapAlignment: %zu\n",
1949  p->props.properties.limits.minMemoryMapAlignment);
1950  av_log(ctx, AV_LOG_VERBOSE, " nonCoherentAtomSize: %"PRIu64"\n",
1951  p->props.properties.limits.nonCoherentAtomSize);
1952  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY)
1953  av_log(ctx, AV_LOG_VERBOSE, " minImportedHostPointerAlignment: %"PRIu64"\n",
1954  p->hprops.minImportedHostPointerAlignment);
1955 
1956  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &qf_num, NULL);
1957  if (!qf_num) {
1958  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1959  return AVERROR_EXTERNAL;
1960  }
1961 
1962  ext_sem_props_info = (VkPhysicalDeviceExternalSemaphoreInfo) {
1963  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO,
1964  };
1965 
1966  /* Opaque FD semaphore properties */
1967  ext_sem_props_info.handleType =
1968 #ifdef _WIN32
1969  IsWindows8OrGreater()
1970  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
1971  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT;
1972 #else
1973  VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
1974 #endif
1975  p->ext_sem_props_opaque.sType = VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES;
1976  vk->GetPhysicalDeviceExternalSemaphoreProperties(hwctx->phys_dev,
1977  &ext_sem_props_info,
1978  &p->ext_sem_props_opaque);
1979 
1980  qf = av_malloc_array(qf_num, sizeof(VkQueueFamilyProperties2));
1981  if (!qf)
1982  return AVERROR(ENOMEM);
1983 
1984  qf_vid = av_malloc_array(qf_num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1985  if (!qf_vid) {
1986  av_free(qf);
1987  return AVERROR(ENOMEM);
1988  }
1989 
1990  for (uint32_t i = 0; i < qf_num; i++) {
1991  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1992  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1993  };
1994  qf[i] = (VkQueueFamilyProperties2) {
1995  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1996  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1997  };
1998  }
1999 
2000  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &qf_num, qf);
2001 
2002  p->qf_mutex = av_calloc(qf_num, sizeof(*p->qf_mutex));
2003  if (!p->qf_mutex) {
2004  err = AVERROR(ENOMEM);
2005  goto end;
2006  }
2007  p->nb_tot_qfs = qf_num;
2008 
2009  for (uint32_t i = 0; i < qf_num; i++) {
2010  p->qf_mutex[i] = av_calloc(qf[i].queueFamilyProperties.queueCount,
2011  sizeof(**p->qf_mutex));
2012  if (!p->qf_mutex[i]) {
2013  err = AVERROR(ENOMEM);
2014  goto end;
2015  }
2016  for (uint32_t j = 0; j < qf[i].queueFamilyProperties.queueCount; j++) {
2017  err = pthread_mutex_init(&p->qf_mutex[i][j], NULL);
2018  if (err != 0) {
2019  av_log(ctx, AV_LOG_ERROR, "pthread_mutex_init failed : %s\n",
2020  av_err2str(err));
2021  err = AVERROR(err);
2022  goto end;
2023  }
2024  }
2025  }
2026 
2027 #if FF_API_VULKAN_FIXED_QUEUES
2028 FF_DISABLE_DEPRECATION_WARNINGS
2029  graph_index = hwctx->nb_graphics_queues ? hwctx->queue_family_index : -1;
2030  comp_index = hwctx->nb_comp_queues ? hwctx->queue_family_comp_index : -1;
2031  tx_index = hwctx->nb_tx_queues ? hwctx->queue_family_tx_index : -1;
2032  dec_index = hwctx->nb_decode_queues ? hwctx->queue_family_decode_index : -1;
2033  enc_index = hwctx->nb_encode_queues ? hwctx->queue_family_encode_index : -1;
2034 
2035 #define CHECK_QUEUE(type, required, fidx, ctx_qf, qc) \
2036  do { \
2037  if (ctx_qf < 0 && required) { \
2038  av_log(ctx, AV_LOG_ERROR, "%s queue family is required, but marked as missing" \
2039  " in the context!\n", type); \
2040  err = AVERROR(EINVAL); \
2041  goto end; \
2042  } else if (fidx < 0 || ctx_qf < 0) { \
2043  break; \
2044  } else if (ctx_qf >= qf_num) { \
2045  av_log(ctx, AV_LOG_ERROR, "Invalid %s family index %i (device has %i families)!\n", \
2046  type, ctx_qf, qf_num); \
2047  err = AVERROR(EINVAL); \
2048  goto end; \
2049  } \
2050  \
2051  av_log(ctx, AV_LOG_VERBOSE, "Using queue family %i (queues: %i)" \
2052  " for%s%s%s%s%s\n", \
2053  ctx_qf, qc, \
2054  ctx_qf == graph_index ? " graphics" : "", \
2055  ctx_qf == comp_index ? " compute" : "", \
2056  ctx_qf == tx_index ? " transfers" : "", \
2057  ctx_qf == enc_index ? " encode" : "", \
2058  ctx_qf == dec_index ? " decode" : ""); \
2059  graph_index = (ctx_qf == graph_index) ? -1 : graph_index; \
2060  comp_index = (ctx_qf == comp_index) ? -1 : comp_index; \
2061  tx_index = (ctx_qf == tx_index) ? -1 : tx_index; \
2062  enc_index = (ctx_qf == enc_index) ? -1 : enc_index; \
2063  dec_index = (ctx_qf == dec_index) ? -1 : dec_index; \
2064  } while (0)
2065 
2066  CHECK_QUEUE("graphics", 0, graph_index, hwctx->queue_family_index, hwctx->nb_graphics_queues);
2067  CHECK_QUEUE("compute", 1, comp_index, hwctx->queue_family_comp_index, hwctx->nb_comp_queues);
2068  CHECK_QUEUE("upload", 1, tx_index, hwctx->queue_family_tx_index, hwctx->nb_tx_queues);
2069  CHECK_QUEUE("decode", 0, dec_index, hwctx->queue_family_decode_index, hwctx->nb_decode_queues);
2070  CHECK_QUEUE("encode", 0, enc_index, hwctx->queue_family_encode_index, hwctx->nb_encode_queues);
2071 
2072 #undef CHECK_QUEUE
2073 
2074  /* Update the new queue family fields. If non-zero already,
2075  * it means API users have set it. */
2076  if (!hwctx->nb_qf) {
2077 #define ADD_QUEUE(ctx_qf, qc, flag) \
2078  do { \
2079  if (ctx_qf != -1) { \
2080  hwctx->qf[hwctx->nb_qf++] = (AVVulkanDeviceQueueFamily) { \
2081  .idx = ctx_qf, \
2082  .num = qc, \
2083  .flags = flag, \
2084  }; \
2085  } \
2086  } while (0)
2087 
2088  ADD_QUEUE(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
2089  ADD_QUEUE(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
2090  ADD_QUEUE(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
2091  ADD_QUEUE(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
2092  ADD_QUEUE(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
2093 #undef ADD_QUEUE
2094  }
2095 FF_ENABLE_DEPRECATION_WARNINGS
2096 #endif
2097 
2098  for (int i = 0; i < hwctx->nb_qf; i++) {
2099  if (!hwctx->qf[i].video_caps &&
2100  hwctx->qf[i].flags & (VK_QUEUE_VIDEO_DECODE_BIT_KHR |
2101  VK_QUEUE_VIDEO_ENCODE_BIT_KHR)) {
2102  hwctx->qf[i].video_caps = qf_vid[hwctx->qf[i].idx].videoCodecOperations;
2103  }
2104  }
2105 
2106  /* Setup array for pQueueFamilyIndices with used queue families */
2107  p->nb_img_qfs = 0;
2108  for (int i = 0; i < hwctx->nb_qf; i++) {
2109  int seen = 0;
2110  /* Make sure each entry is unique
2111  * (VUID-VkBufferCreateInfo-sharingMode-01419) */
2112  for (int j = (i - 1); j >= 0; j--) {
2113  if (hwctx->qf[i].idx == hwctx->qf[j].idx) {
2114  seen = 1;
2115  break;
2116  }
2117  }
2118  if (!seen)
2119  p->img_qfs[p->nb_img_qfs++] = hwctx->qf[i].idx;
2120  }
2121 
2122  if (!hwctx->lock_queue)
2123  hwctx->lock_queue = lock_queue;
2124  if (!hwctx->unlock_queue)
2125  hwctx->unlock_queue = unlock_queue;
2126 
2127  /* Re-query device capabilities, in case the device was created externally */
2128  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
2129 
2130  p->vkctx.device = ctx;
2131  p->vkctx.hwctx = hwctx;
2132 
2133  ff_vk_load_props(&p->vkctx);
2134  p->compute_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_COMPUTE_BIT, 0);
2135  p->transfer_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_TRANSFER_BIT, 0);
2136 
2137  /* Re-query device capabilities, in case the device was created externally */
2138  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
2139 
2140 end:
2141  av_free(qf_vid);
2142  av_free(qf);
2143  return err;
2144 }
2145 
2146 static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device,
2147  AVDictionary *opts, int flags)
2148 {
2149  VulkanDeviceSelection dev_select = { 0 };
2150  if (device && device[0]) {
2151  char *end = NULL;
2152  dev_select.index = strtol(device, &end, 10);
2153  if (end == device) {
2154  dev_select.index = 0;
2155  dev_select.name = device;
2156  }
2157  }
2158 
2159  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2160 }
2161 
2162 static int vulkan_device_derive(AVHWDeviceContext *ctx,
2163  AVHWDeviceContext *src_ctx,
2164  AVDictionary *opts, int flags)
2165 {
2166  av_unused VulkanDeviceSelection dev_select = { 0 };
2167 
2168  /* If there's only one device on the system, then even if its not covered
2169  * by the following checks (e.g. non-PCIe ARM GPU), having an empty
2170  * dev_select will mean it'll get picked. */
2171  switch(src_ctx->type) {
2172 #if CONFIG_VAAPI
2173  case AV_HWDEVICE_TYPE_VAAPI: {
2174  AVVAAPIDeviceContext *src_hwctx = src_ctx->hwctx;
2175  VADisplay dpy = src_hwctx->display;
2176 #if VA_CHECK_VERSION(1, 15, 0)
2177  VAStatus vas;
2178  VADisplayAttribute attr = {
2179  .type = VADisplayPCIID,
2180  };
2181 #endif
2182  const char *vendor;
2183 
2184 #if VA_CHECK_VERSION(1, 15, 0)
2185  vas = vaGetDisplayAttributes(dpy, &attr, 1);
2186  if (vas == VA_STATUS_SUCCESS && attr.flags != VA_DISPLAY_ATTRIB_NOT_SUPPORTED)
2187  dev_select.pci_device = (attr.value & 0xFFFF);
2188 #endif
2189 
2190  if (!dev_select.pci_device) {
2191  vendor = vaQueryVendorString(dpy);
2192  if (!vendor) {
2193  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from VAAPI!\n");
2194  return AVERROR_EXTERNAL;
2195  }
2196 
2197  if (strstr(vendor, "AMD"))
2198  dev_select.vendor_id = 0x1002;
2199  }
2200 
2201  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2202  }
2203 #endif
2204 #if CONFIG_LIBDRM
2205  case AV_HWDEVICE_TYPE_DRM: {
2206  int err;
2207  struct stat drm_node_info;
2208  drmDevice *drm_dev_info;
2209  AVDRMDeviceContext *src_hwctx = src_ctx->hwctx;
2210 
2211  err = fstat(src_hwctx->fd, &drm_node_info);
2212  if (err) {
2213  av_log(ctx, AV_LOG_ERROR, "Unable to get node info from DRM fd: %s!\n",
2214  av_err2str(AVERROR(errno)));
2215  return AVERROR_EXTERNAL;
2216  }
2217 
2218  dev_select.drm_major = major(drm_node_info.st_dev);
2219  dev_select.drm_minor = minor(drm_node_info.st_dev);
2220  dev_select.has_drm = 1;
2221 
2222  err = drmGetDevice(src_hwctx->fd, &drm_dev_info);
2223  if (err) {
2224  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from DRM fd: %s!\n",
2225  av_err2str(AVERROR(errno)));
2226  return AVERROR_EXTERNAL;
2227  }
2228 
2229  if (drm_dev_info->bustype == DRM_BUS_PCI)
2230  dev_select.pci_device = drm_dev_info->deviceinfo.pci->device_id;
2231 
2232  drmFreeDevice(&drm_dev_info);
2233 
2234  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2235  }
2236 #endif
2237 #if CONFIG_CUDA
2238  case AV_HWDEVICE_TYPE_CUDA: {
2239  AVHWDeviceContext *cuda_cu = src_ctx;
2240  AVCUDADeviceContext *src_hwctx = src_ctx->hwctx;
2241  AVCUDADeviceContextInternal *cu_internal = src_hwctx->internal;
2242  CudaFunctions *cu = cu_internal->cuda_dl;
2243 
2244  int ret = CHECK_CU(cu->cuDeviceGetUuid((CUuuid *)&dev_select.uuid,
2245  cu_internal->cuda_device));
2246  if (ret < 0) {
2247  av_log(ctx, AV_LOG_ERROR, "Unable to get UUID from CUDA!\n");
2248  return AVERROR_EXTERNAL;
2249  }
2250 
2251  dev_select.has_uuid = 1;
2252 
2253  /*
2254  * CUDA is not able to import multiplane images, so always derive a
2255  * Vulkan device with multiplane disabled.
2256  */
2257  return vulkan_device_create_internal(ctx, &dev_select, 1, opts, flags);
2258  }
2259 #endif
2260  default:
2261  return AVERROR(ENOSYS);
2262  }
2263 }
2264 
2265 static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx,
2266  const void *hwconfig,
2267  AVHWFramesConstraints *constraints)
2268 {
2269  int count = 0;
2270  VulkanDevicePriv *p = ctx->hwctx;
2271 
2272  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2273  count += vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2274  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2275  VK_IMAGE_TILING_OPTIMAL,
2276  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0;
2277  }
2278 
2279  constraints->valid_sw_formats = av_malloc_array(count + 1,
2280  sizeof(enum AVPixelFormat));
2281  if (!constraints->valid_sw_formats)
2282  return AVERROR(ENOMEM);
2283 
2284  count = 0;
2285  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2286  if (vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2287  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2288  VK_IMAGE_TILING_OPTIMAL,
2289  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0) {
2290  constraints->valid_sw_formats[count++] = vk_formats_list[i].pixfmt;
2291  }
2292  }
2293 
2294  constraints->valid_sw_formats[count++] = AV_PIX_FMT_NONE;
2295 
2296  constraints->min_width = 1;
2297  constraints->min_height = 1;
2298  constraints->max_width = p->props.properties.limits.maxImageDimension2D;
2299  constraints->max_height = p->props.properties.limits.maxImageDimension2D;
2300 
2301  constraints->valid_hw_formats = av_malloc_array(2, sizeof(enum AVPixelFormat));
2302  if (!constraints->valid_hw_formats)
2303  return AVERROR(ENOMEM);
2304 
2305  constraints->valid_hw_formats[0] = AV_PIX_FMT_VULKAN;
2306  constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
2307 
2308  return 0;
2309 }
2310 
2311 static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req,
2312  VkMemoryPropertyFlagBits req_flags, const void *alloc_extension,
2313  VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
2314 {
2315  VkResult ret;
2316  int index = -1;
2317  VulkanDevicePriv *p = ctx->hwctx;
2318  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2319  AVVulkanDeviceContext *dev_hwctx = &p->p;
2320  VkMemoryAllocateInfo alloc_info = {
2321  .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
2322  .pNext = alloc_extension,
2323  .allocationSize = req->size,
2324  };
2325 
2326  /* The vulkan spec requires memory types to be sorted in the "optimal"
2327  * order, so the first matching type we find will be the best/fastest one */
2328  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
2329  const VkMemoryType *type = &p->mprops.memoryTypes[i];
2330 
2331  /* The memory type must be supported by the requirements (bitfield) */
2332  if (!(req->memoryTypeBits & (1 << i)))
2333  continue;
2334 
2335  /* The memory type flags must include our properties */
2336  if ((type->propertyFlags & req_flags) != req_flags)
2337  continue;
2338 
2339  /* The memory type must be large enough */
2340  if (req->size > p->mprops.memoryHeaps[type->heapIndex].size)
2341  continue;
2342 
2343  /* Found a suitable memory type */
2344  index = i;
2345  break;
2346  }
2347 
2348  if (index < 0) {
2349  av_log(ctx, AV_LOG_ERROR, "No memory type found for flags 0x%x\n",
2350  req_flags);
2351  return AVERROR(EINVAL);
2352  }
2353 
2354  alloc_info.memoryTypeIndex = index;
2355 
2356  ret = vk->AllocateMemory(dev_hwctx->act_dev, &alloc_info,
2357  dev_hwctx->alloc, mem);
2358  if (ret != VK_SUCCESS) {
2359  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory: %s\n",
2360  ff_vk_ret2str(ret));
2361  return AVERROR(ENOMEM);
2362  }
2363 
2364  *mem_flags |= p->mprops.memoryTypes[index].propertyFlags;
2365 
2366  return 0;
2367 }
2368 
2369 static void vulkan_free_internal(AVVkFrame *f)
2370 {
2371  av_unused AVVkFrameInternal *internal = f->internal;
2372 
2373 #if CONFIG_CUDA
2374  if (internal->cuda_fc_ref) {
2375  AVHWFramesContext *cuda_fc = (AVHWFramesContext *)internal->cuda_fc_ref->data;
2376  int planes = av_pix_fmt_count_planes(cuda_fc->sw_format);
2377  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
2378  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
2379  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
2380  CudaFunctions *cu = cu_internal->cuda_dl;
2381 
2382  for (int i = 0; i < planes; i++) {
2383  if (internal->cu_sem[i])
2384  CHECK_CU(cu->cuDestroyExternalSemaphore(internal->cu_sem[i]));
2385  if (internal->cu_mma[i])
2386  CHECK_CU(cu->cuMipmappedArrayDestroy(internal->cu_mma[i]));
2387  if (internal->ext_mem[i])
2388  CHECK_CU(cu->cuDestroyExternalMemory(internal->ext_mem[i]));
2389 #ifdef _WIN32
2390  if (internal->ext_sem_handle[i])
2391  CloseHandle(internal->ext_sem_handle[i]);
2392  if (internal->ext_mem_handle[i])
2393  CloseHandle(internal->ext_mem_handle[i]);
2394 #endif
2395  }
2396 
2397  av_buffer_unref(&internal->cuda_fc_ref);
2398  }
2399 #endif
2400 
2401  pthread_mutex_destroy(&internal->update_mutex);
2402  av_freep(&f->internal);
2403 }
2404 
2405 static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
2406 {
2407  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2408  AVVulkanDeviceContext *hwctx = &p->p;
2409  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2410  int nb_images = ff_vk_count_images(f);
2411  int nb_sems = 0;
2412 
2413  while (nb_sems < FF_ARRAY_ELEMS(f->sem) && f->sem[nb_sems])
2414  nb_sems++;
2415 
2416  if (nb_sems) {
2417  VkSemaphoreWaitInfo sem_wait = {
2418  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
2419  .flags = 0x0,
2420  .pSemaphores = f->sem,
2421  .pValues = f->sem_value,
2422  .semaphoreCount = nb_sems,
2423  };
2424 
2425  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
2426  }
2427 
2428  vulkan_free_internal(f);
2429 
2430  for (int i = 0; i < nb_images; i++) {
2431  vk->DestroyImage(hwctx->act_dev, f->img[i], hwctx->alloc);
2432  vk->FreeMemory(hwctx->act_dev, f->mem[i], hwctx->alloc);
2433  vk->DestroySemaphore(hwctx->act_dev, f->sem[i], hwctx->alloc);
2434  }
2435 
2436  av_free(f);
2437 }
2438 
2439 static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
2440 {
2441  vulkan_frame_free(opaque, (AVVkFrame*)data);
2442 }
2443 
2444 static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f,
2445  void *alloc_pnext, size_t alloc_pnext_stride)
2446 {
2447  int img_cnt = 0, err;
2448  VkResult ret;
2449  AVHWDeviceContext *ctx = hwfc->device_ctx;
2450  VulkanDevicePriv *p = ctx->hwctx;
2451  AVVulkanDeviceContext *hwctx = &p->p;
2452  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2453  VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } };
2454 
2455  while (f->img[img_cnt]) {
2456  int use_ded_mem;
2457  VkImageMemoryRequirementsInfo2 req_desc = {
2458  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
2459  .image = f->img[img_cnt],
2460  };
2461  VkMemoryDedicatedAllocateInfo ded_alloc = {
2462  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
2463  .pNext = (void *)(((uint8_t *)alloc_pnext) + img_cnt*alloc_pnext_stride),
2464  };
2465  VkMemoryDedicatedRequirements ded_req = {
2466  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
2467  };
2468  VkMemoryRequirements2 req = {
2469  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
2470  .pNext = &ded_req,
2471  };
2472 
2473  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req);
2474 
2475  if (f->tiling == VK_IMAGE_TILING_LINEAR)
2476  req.memoryRequirements.size = FFALIGN(req.memoryRequirements.size,
2477  p->props.properties.limits.minMemoryMapAlignment);
2478 
2479  /* In case the implementation prefers/requires dedicated allocation */
2480  use_ded_mem = ded_req.prefersDedicatedAllocation |
2481  ded_req.requiresDedicatedAllocation;
2482  if (use_ded_mem)
2483  ded_alloc.image = f->img[img_cnt];
2484 
2485  /* Allocate memory */
2486  if ((err = alloc_mem(ctx, &req.memoryRequirements,
2487  f->tiling == VK_IMAGE_TILING_LINEAR ?
2488  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT :
2489  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
2490  use_ded_mem ? &ded_alloc : (void *)ded_alloc.pNext,
2491  &f->flags, &f->mem[img_cnt])))
2492  return err;
2493 
2494  f->size[img_cnt] = req.memoryRequirements.size;
2495  bind_info[img_cnt].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
2496  bind_info[img_cnt].image = f->img[img_cnt];
2497  bind_info[img_cnt].memory = f->mem[img_cnt];
2498 
2499  img_cnt++;
2500  }
2501 
2502  /* Bind the allocated memory to the images */
2503  ret = vk->BindImageMemory2(hwctx->act_dev, img_cnt, bind_info);
2504  if (ret != VK_SUCCESS) {
2505  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
2506  ff_vk_ret2str(ret));
2507  return AVERROR_EXTERNAL;
2508  }
2509 
2510  return 0;
2511 }
2512 
2513 enum PrepMode {
2514  PREP_MODE_GENERAL,
2515  PREP_MODE_WRITE,
2516  PREP_MODE_EXTERNAL_EXPORT,
2517  PREP_MODE_EXTERNAL_IMPORT,
2518  PREP_MODE_DECODING_DST,
2519  PREP_MODE_DECODING_DPB,
2520  PREP_MODE_ENCODING_DPB,
2521 };
2522 
2523 static void switch_new_props(enum PrepMode pmode, VkImageLayout *new_layout,
2524  VkAccessFlags2 *new_access)
2525 {
2526  switch (pmode) {
2527  case PREP_MODE_GENERAL:
2528  *new_layout = VK_IMAGE_LAYOUT_GENERAL;
2529  *new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2530  break;
2531  case PREP_MODE_WRITE:
2532  *new_layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
2533  *new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2534  break;
2535  case PREP_MODE_EXTERNAL_IMPORT:
2536  *new_layout = VK_IMAGE_LAYOUT_GENERAL;
2537  *new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2538  break;
2539  case PREP_MODE_EXTERNAL_EXPORT:
2540  *new_layout = VK_IMAGE_LAYOUT_GENERAL;
2541  *new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2542  break;
2543  case PREP_MODE_DECODING_DST:
2544  *new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR;
2545  *new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2546  break;
2547  case PREP_MODE_DECODING_DPB:
2548  *new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
2549  *new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2550  break;
2551  case PREP_MODE_ENCODING_DPB:
2552  *new_layout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
2553  *new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2554  break;
2555  }
2556 }
2557 
2558 static int switch_layout(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2559  AVVkFrame *frame, enum PrepMode pmode)
2560 {
2561  int err;
2562  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2563  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2564  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
2565  int nb_img_bar = 0;
2566 
2567  VkImageLayout new_layout;
2568  VkAccessFlags2 new_access;
2569  switch_new_props(pmode, &new_layout, &new_access);
2570 
2571  uint32_t dst_qf = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2572  VkPipelineStageFlagBits2 src_stage = VK_PIPELINE_STAGE_2_NONE;
2573  if (pmode == PREP_MODE_EXTERNAL_EXPORT) {
2574  dst_qf = VK_QUEUE_FAMILY_EXTERNAL_KHR;
2575  src_stage = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
2576  }
2577 
2578  /* This is dirty - but it works. The vulkan.c dependency system doesn't
2579  * free non-refcounted frames, and non-refcounted hardware frames cannot
2580  * happen anywhere outside of here. */
2581  AVBufferRef tmp_ref = {
2582  .data = (uint8_t *)hwfc,
2583  };
2584  AVFrame tmp_frame = {
2585  .data[0] = (uint8_t *)frame,
2586  .hw_frames_ctx = &tmp_ref,
2587  };
2588 
2589  VkCommandBuffer cmd_buf;
2590  FFVkExecContext *exec = ff_vk_exec_get(&p->vkctx, ectx);
2591  cmd_buf = exec->buf;
2592  ff_vk_exec_start(&p->vkctx, exec);
2593 
2594  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, &tmp_frame,
2595  VK_PIPELINE_STAGE_2_NONE,
2596  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
2597  if (err < 0)
2598  return err;
2599 
2600  ff_vk_frame_barrier(&p->vkctx, exec, &tmp_frame, img_bar, &nb_img_bar,
2601  src_stage,
2602  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
2603  new_access, new_layout, dst_qf);
2604 
2605  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
2606  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
2607  .pImageMemoryBarriers = img_bar,
2608  .imageMemoryBarrierCount = nb_img_bar,
2609  });
2610 
2611  err = ff_vk_exec_submit(&p->vkctx, exec);
2612  if (err < 0)
2613  return err;
2614 
2615  /* We can do this because there are no real dependencies */
2616  ff_vk_exec_discard_deps(&p->vkctx, exec);
2617 
2618  return 0;
2619 }
2620 
2621 static int switch_layout_host(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2622  AVVkFrame *frame, enum PrepMode pmode)
2623 {
2624  VkResult ret;
2625  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2626  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2627  VkHostImageLayoutTransitionInfoEXT layout_change[AV_NUM_DATA_POINTERS];
2628  int nb_images = ff_vk_count_images(frame);
2629 
2630  VkImageLayout new_layout;
2631  VkAccessFlags2 new_access;
2632  switch_new_props(pmode, &new_layout, &new_access);
2633 
2634  int i;
2635  for (i = 0; i < p->vkctx.host_image_props.copyDstLayoutCount; i++) {
2636  if (p->vkctx.host_image_props.pCopyDstLayouts[i] == new_layout)
2637  break;
2638  }
2639  if (i == p->vkctx.host_image_props.copyDstLayoutCount)
2640  return AVERROR(ENOTSUP);
2641 
2642  for (i = 0; i < nb_images; i++) {
2643  layout_change[i] = (VkHostImageLayoutTransitionInfoEXT) {
2644  .sType = VK_STRUCTURE_TYPE_HOST_IMAGE_LAYOUT_TRANSITION_INFO_EXT,
2645  .image = frame->img[i],
2646  .oldLayout = frame->layout[i],
2647  .newLayout = new_layout,
2648  .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
2649  .subresourceRange.layerCount = 1,
2650  .subresourceRange.levelCount = 1,
2651  };
2652  frame->layout[i] = new_layout;
2653  }
2654 
2655  ret = vk->TransitionImageLayoutEXT(p->vkctx.hwctx->act_dev,
2656  nb_images, layout_change);
2657  if (ret != VK_SUCCESS) {
2658  av_log(hwfc, AV_LOG_ERROR, "Unable to prepare frame: %s\n",
2659  ff_vk_ret2str(ret));
2660  return AVERROR_EXTERNAL;
2661  }
2662 
2663  return 0;
2664 }
2665 
2666 static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2667  AVVkFrame *frame, enum PrepMode pmode)
2668 {
2669  int err = 0;
2670  AVVulkanFramesContext *hwfc_vk = hwfc->hwctx;
2671  if (hwfc_vk->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT &&
2672  (pmode != PREP_MODE_EXTERNAL_EXPORT) &&
2673  (pmode != PREP_MODE_EXTERNAL_IMPORT))
2674  err = switch_layout_host(hwfc, ectx, frame, pmode);
2675 
2676  if (err != AVERROR(ENOTSUP))
2677  return err;
2678 
2679  return switch_layout(hwfc, ectx, frame, pmode);
2680 }
2681 
2682 static inline void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format,
2683  int frame_w, int frame_h, int plane)
2684 {
2685  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(format);
2686 
2687  /* Currently always true unless gray + alpha support is added */
2688  if (!plane || (plane == 3) || desc->flags & AV_PIX_FMT_FLAG_RGB ||
2689  !(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) {
2690  *w = frame_w;
2691  *h = frame_h;
2692  return;
2693  }
2694 
2695  *w = AV_CEIL_RSHIFT(frame_w, desc->log2_chroma_w);
2696  *h = AV_CEIL_RSHIFT(frame_h, desc->log2_chroma_h);
2697 }
2698 
2699 static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame,
2700  VkImageTiling tiling, VkImageUsageFlagBits usage,
2701  VkImageCreateFlags flags, int nb_layers,
2702  void *create_pnext)
2703 {
2704  int err;
2705  VkResult ret;
2706  AVVulkanFramesContext *hwfc_vk = hwfc->hwctx;
2707  AVHWDeviceContext *ctx = hwfc->device_ctx;
2708  VulkanDevicePriv *p = ctx->hwctx;
2709  AVVulkanDeviceContext *hwctx = &p->p;
2710  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2711  AVVkFrame *f;
2712 
2713  VkSemaphoreTypeCreateInfo sem_type_info = {
2714  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
2715  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
2716  .initialValue = 0,
2717  };
2718  VkSemaphoreCreateInfo sem_spawn = {
2719  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
2720  .pNext = &sem_type_info,
2721  };
2722 
2723  VkExportSemaphoreCreateInfo ext_sem_info_opaque = {
2724  .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
2725 #ifdef _WIN32
2726  .handleTypes = IsWindows8OrGreater()
2727  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
2728  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
2729 #else
2730  .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
2731 #endif
2732  };
2733 
2734  /* Check if exporting is supported before chaining any structs */
2735  if (p->ext_sem_props_opaque.externalSemaphoreFeatures & VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT) {
2736  if (p->vkctx.extensions & (FF_VK_EXT_EXTERNAL_WIN32_SEM | FF_VK_EXT_EXTERNAL_FD_SEM))
2737  ff_vk_link_struct(&sem_type_info, &ext_sem_info_opaque);
2738  }
2739 
2740  f = av_vk_frame_alloc();
2741  if (!f) {
2742  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
2743  return AVERROR(ENOMEM);
2744  }
2745 
2746  // TODO: check width and height for alignment in case of multiplanar (must be mod-2 if subsampled)
2747 
2748  /* Create the images */
2749  for (int i = 0; (hwfc_vk->format[i] != VK_FORMAT_UNDEFINED); i++) {
2750  VkImageCreateInfo create_info = {
2751  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
2752  .pNext = create_pnext,
2753  .imageType = VK_IMAGE_TYPE_2D,
2754  .format = hwfc_vk->format[i],
2755  .extent.depth = 1,
2756  .mipLevels = 1,
2757  .arrayLayers = nb_layers,
2758  .flags = flags,
2759  .tiling = tiling,
2760  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
2761  .usage = usage,
2762  .samples = VK_SAMPLE_COUNT_1_BIT,
2763  .pQueueFamilyIndices = p->img_qfs,
2764  .queueFamilyIndexCount = p->nb_img_qfs,
2765  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2766  VK_SHARING_MODE_EXCLUSIVE,
2767  };
2768 
2769  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
2770  hwfc->sw_format, hwfc->width, hwfc->height, i);
2771 
2772  ret = vk->CreateImage(hwctx->act_dev, &create_info,
2773  hwctx->alloc, &f->img[i]);
2774  if (ret != VK_SUCCESS) {
2775  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
2776  ff_vk_ret2str(ret));
2777  err = AVERROR(EINVAL);
2778  goto fail;
2779  }
2780 
2781  /* Create semaphore */
2782  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
2783  hwctx->alloc, &f->sem[i]);
2784  if (ret != VK_SUCCESS) {
2785  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
2786  ff_vk_ret2str(ret));
2787  err = AVERROR_EXTERNAL;
2788  goto fail;
2789  }
2790 
2791  f->queue_family[i] = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2792  f->layout[i] = create_info.initialLayout;
2793  f->access[i] = 0x0;
2794  f->sem_value[i] = 0;
2795  }
2796 
2797  f->flags = 0x0;
2798  f->tiling = tiling;
2799 
2800  *frame = f;
2801  return 0;
2802 
2803 fail:
2804  vulkan_frame_free(hwfc, f);
2805  return err;
2806 }
2807 
2808 /* Checks if an export flag is enabled, and if it is ORs it with *iexp */
2809 static void try_export_flags(AVHWFramesContext *hwfc,
2810  VkExternalMemoryHandleTypeFlags *comp_handle_types,
2811  VkExternalMemoryHandleTypeFlags *iexp,
2812  VkExternalMemoryHandleTypeFlagBits exp)
2813 {
2814  VkResult ret;
2815  AVVulkanFramesContext *hwctx = hwfc->hwctx;
2816  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2817  AVVulkanDeviceContext *dev_hwctx = &p->p;
2818  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2819 
2820  const VkImageDrmFormatModifierListCreateInfoEXT *drm_mod_info =
2821  ff_vk_find_struct(hwctx->create_pnext,
2822  VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
2823  int has_mods = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT && drm_mod_info;
2824  int nb_mods;
2825 
2826  VkExternalImageFormatProperties eprops = {
2827  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
2828  };
2829  VkImageFormatProperties2 props = {
2830  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
2831  .pNext = &eprops,
2832  };
2833  VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
2834  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
2835  .pNext = NULL,
2836  .pQueueFamilyIndices = p->img_qfs,
2837  .queueFamilyIndexCount = p->nb_img_qfs,
2838  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2839  VK_SHARING_MODE_EXCLUSIVE,
2840  };
2841  VkPhysicalDeviceExternalImageFormatInfo enext = {
2842  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
2843  .handleType = exp,
2844  .pNext = has_mods ? &phy_dev_mod_info : NULL,
2845  };
2846  VkPhysicalDeviceImageFormatInfo2 pinfo = {
2847  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
2848  .pNext = !exp ? NULL : &enext,
2849  .format = vk_find_format_entry(hwfc->sw_format)->vkf,
2850  .type = VK_IMAGE_TYPE_2D,
2851  .tiling = hwctx->tiling,
2852  .usage = hwctx->usage,
2853  .flags = VK_IMAGE_CREATE_ALIAS_BIT,
2854  };
2855 
2856  nb_mods = has_mods ? drm_mod_info->drmFormatModifierCount : 1;
2857  for (int i = 0; i < nb_mods; i++) {
2858  if (has_mods)
2859  phy_dev_mod_info.drmFormatModifier = drm_mod_info->pDrmFormatModifiers[i];
2860 
2861  ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
2862  &pinfo, &props);
2863 
2864  if (ret == VK_SUCCESS) {
2865  *iexp |= exp;
2866  *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
2867  }
2868  }
2869 }
2870 
2871 static AVBufferRef *vulkan_pool_alloc(void *opaque, size_t size)
2872 {
2873  int err;
2874  AVVkFrame *f;
2875  AVBufferRef *avbuf = NULL;
2876  AVHWFramesContext *hwfc = opaque;
2877  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2878  VulkanFramesPriv *fp = hwfc->hwctx;
2879  AVVulkanFramesContext *hwctx = &fp->p;
2880  VkExternalMemoryHandleTypeFlags e = 0x0;
2881  VkExportMemoryAllocateInfo eminfo[AV_NUM_DATA_POINTERS];
2882 
2883  VkExternalMemoryImageCreateInfo eiinfo = {
2884  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
2885  .pNext = hwctx->create_pnext,
2886  };
2887 
2888 #ifdef _WIN32
2889  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY)
2890  try_export_flags(hwfc, &eiinfo.handleTypes, &e, IsWindows8OrGreater()
2891  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
2892  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT);
2893 #else
2894  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY)
2895  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2896  VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT);
2897 
2898  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_DMABUF_MEMORY &&
2899  hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
2900  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2901  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
2902 #endif
2903 
2904  for (int i = 0; i < av_pix_fmt_count_planes(hwfc->sw_format); i++) {
2905  eminfo[i].sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
2906  eminfo[i].pNext = hwctx->alloc_pnext[i];
2907  eminfo[i].handleTypes = e;
2908  }
2909 
2910  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
2911  hwctx->nb_layers,
2912  eiinfo.handleTypes ? &eiinfo : hwctx->create_pnext);
2913  if (err)
2914  return NULL;
2915 
2916  err = alloc_bind_mem(hwfc, f, eminfo, sizeof(*eminfo));
2917  if (err)
2918  goto fail;
2919 
2920  if ( (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2921  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR))
2922  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DPB);
2923  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)
2924  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DST);
2925  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR)
2926  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_ENCODING_DPB);
2927  else if (hwctx->usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
2928  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_WRITE);
2929  else
2930  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_GENERAL);
2931  if (err)
2932  goto fail;
2933 
2934  avbuf = av_buffer_create((uint8_t *)f, sizeof(AVVkFrame),
2935  vulkan_frame_free_cb, hwfc, 0);
2936  if (!avbuf)
2937  goto fail;
2938 
2939  return avbuf;
2940 
2941 fail:
2942  vulkan_frame_free(hwfc, f);
2943  return NULL;
2944 }
2945 
2946 static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2947 {
2948  pthread_mutex_lock(&vkf->internal->update_mutex);
2949 }
2950 
2951 static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2952 {
2953  pthread_mutex_unlock(&vkf->internal->update_mutex);
2954 }
2955 
2956 static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
2957 {
2958  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2959  VulkanFramesPriv *fp = hwfc->hwctx;
2960 
2961  if (fp->modifier_info) {
2962  if (fp->modifier_info->pDrmFormatModifiers)
2963  av_freep(&fp->modifier_info->pDrmFormatModifiers);
2964  av_freep(&fp->modifier_info);
2965  }
2966 
2967  ff_vk_exec_pool_free(&p->vkctx, &fp->compute_exec);
2968  ff_vk_exec_pool_free(&p->vkctx, &fp->upload_exec);
2969  ff_vk_exec_pool_free(&p->vkctx, &fp->download_exec);
2970 
2971  av_buffer_pool_uninit(&fp->tmp);
2972 }
2973 
2974 static int vulkan_frames_init(AVHWFramesContext *hwfc)
2975 {
2976  int err;
2977  AVVkFrame *f;
2978  VulkanFramesPriv *fp = hwfc->hwctx;
2979  AVVulkanFramesContext *hwctx = &fp->p;
2980  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2981  AVVulkanDeviceContext *dev_hwctx = &p->p;
2982  VkImageUsageFlags supported_usage;
2983  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2984  const struct FFVkFormatEntry *fmt;
2985  int disable_multiplane = p->disable_multiplane ||
2986  (hwctx->flags & AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE);
2987  int is_lone_dpb = ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) ||
2988  ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2989  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)));
2990 
2991  /* Defaults */
2992  if (!hwctx->nb_layers)
2993  hwctx->nb_layers = 1;
2994 
2995  /* VK_IMAGE_TILING_OPTIMAL == 0, can't check for it really */
2996  if (p->use_linear_images &&
2997  (hwctx->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT))
2998  hwctx->tiling = VK_IMAGE_TILING_LINEAR;
2999 
3000 
3001  fmt = vk_find_format_entry(hwfc->sw_format);
3002  if (!fmt) {
3003  av_log(hwfc, AV_LOG_ERROR, "Unsupported pixel format: %s!\n",
3004  av_get_pix_fmt_name(hwfc->sw_format));
3005  return AVERROR(EINVAL);
3006  }
3007 
3008  if (hwctx->format[0] != VK_FORMAT_UNDEFINED) {
3009  if (hwctx->format[0] != fmt->vkf) {
3010  for (int i = 0; i < fmt->nb_images_fallback; i++) {
3011  if (hwctx->format[i] != fmt->fallback[i]) {
3012  av_log(hwfc, AV_LOG_ERROR, "Incompatible Vulkan format given "
3013  "for the current sw_format %s!\n",
3014  av_get_pix_fmt_name(hwfc->sw_format));
3015  return AVERROR(EINVAL);
3016  }
3017  }
3018  }
3019 
3020  /* Check if the sw_format itself is supported */
3021  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
3022  hwctx->tiling, NULL,
3023  NULL, NULL, &supported_usage, 0,
3024  !hwctx->usage ||
3025  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
3026  if (err < 0) {
3027  av_log(hwfc, AV_LOG_ERROR, "Unsupported sw format: %s!\n",
3028  av_get_pix_fmt_name(hwfc->sw_format));
3029  return AVERROR(EINVAL);
3030  }
3031  } else {
3032  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
3033  hwctx->tiling, hwctx->format, NULL,
3034  NULL, &supported_usage,
3035  disable_multiplane,
3036  !hwctx->usage ||
3037  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
3038  if (err < 0)
3039  return err;
3040  }
3041 
3042  /* Lone DPB images do not need additional flags. */
3043  if (!is_lone_dpb) {
3044  /* Image usage flags */
3045  hwctx->usage |= supported_usage & (VK_IMAGE_USAGE_TRANSFER_DST_BIT |
3046  VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
3047  VK_IMAGE_USAGE_STORAGE_BIT |
3048  VK_IMAGE_USAGE_SAMPLED_BIT);
3049 
3050  if (p->vkctx.extensions & FF_VK_EXT_HOST_IMAGE_COPY &&
3051  !(p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY) &&
3052  !(p->dprops.driverID == VK_DRIVER_ID_MOLTENVK))
3053  hwctx->usage |= supported_usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT;
3054 
3055  /* Enables encoding of images, if supported by format and extensions */
3056  if ((supported_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
3057  (p->vkctx.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
3058  (p->vkctx.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1))
3059  hwctx->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
3060 
3061  /* Image creation flags.
3062  * Only fill them in automatically if the image is not going to be used as
3063  * a DPB-only image, and we have SAMPLED/STORAGE bits set. */
3064  if (!hwctx->img_flags) {
3065  int sampleable = hwctx->usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
3066  VK_IMAGE_USAGE_STORAGE_BIT);
3067  hwctx->img_flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
3068  if (sampleable) {
3069  hwctx->img_flags |= VK_IMAGE_CREATE_ALIAS_BIT;
3070  if ((fmt->vk_planes > 1) && (hwctx->format[0] == fmt->vkf))
3071  hwctx->img_flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
3072  }
3073  }
3074  }
3075 
3076  /* If the image has an ENCODE_SRC usage, and the maintenance1
3077  * extension is supported, check if it has a profile list.
3078  * If there's no profile list, or it has no encode operations,
3079  * then allow creating the image with no specific profile. */
3080  if ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
3081  (p->vkctx.extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE) &&
3082  (p->vkctx.extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1)) {
3083  const VkVideoProfileListInfoKHR *pl;
3084  pl = ff_vk_find_struct(hwctx->create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
3085  if (!pl) {
3086  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
3087  } else {
3088  uint32_t i;
3089  for (i = 0; i < pl->profileCount; i++) {
3090  /* Video ops start at exactly 0x00010000 */
3091  if (pl->pProfiles[i].videoCodecOperation & 0xFFFF0000)
3092  break;
3093  }
3094  if (i == pl->profileCount)
3095  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
3096  }
3097  }
3098 
3099  if (!hwctx->lock_frame)
3100  hwctx->lock_frame = lock_frame;
3101 
3102  if (!hwctx->unlock_frame)
3103  hwctx->unlock_frame = unlock_frame;
3104 
3105  err = ff_vk_exec_pool_init(&p->vkctx, p->compute_qf, &fp->compute_exec,
3106  p->compute_qf->num, 0, 0, 0, NULL);
3107  if (err)
3108  return err;
3109 
3110  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->upload_exec,
3111  p->transfer_qf->num*2, 0, 0, 0, NULL);
3112  if (err)
3113  return err;
3114 
3115  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->download_exec,
3116  p->transfer_qf->num, 0, 0, 0, NULL);
3117  if (err)
3118  return err;
3119 
3120  /* Test to see if allocation will fail */
3121  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
3122  hwctx->nb_layers, hwctx->create_pnext);
3123  if (err)
3124  return err;
3125 
3126  /* Collect `VkDrmFormatModifierPropertiesEXT` for each plane. Required for DRM export. */
3127  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS && hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
3128  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
3129  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
3130  };
3131  err = vk->GetImageDrmFormatModifierPropertiesEXT(dev_hwctx->act_dev, f->img[0],
3132  &drm_mod);
3133  if (err != VK_SUCCESS) {
3134  av_log(hwfc, AV_LOG_ERROR, "Failed to get image DRM format modifier properties");
3135  vulkan_frame_free(hwfc, f);
3136  return AVERROR_EXTERNAL;
3137  }
3138  for (int i = 0; i < fmt->vk_planes; ++i) {
3139  VkDrmFormatModifierPropertiesListEXT modp;
3140  VkFormatProperties2 fmtp;
3141  VkDrmFormatModifierPropertiesEXT *mod_props = NULL;
3142 
3143  modp = (VkDrmFormatModifierPropertiesListEXT) {
3144  .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
3145  };
3146  fmtp = (VkFormatProperties2) {
3147  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
3148  .pNext = &modp,
3149  };
3150 
3151  /* query drmFormatModifierCount by keeping pDrmFormatModifierProperties NULL */
3152  vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt->fallback[i], &fmtp);
3153 
3154  modp.pDrmFormatModifierProperties =
3155  av_calloc(modp.drmFormatModifierCount, sizeof(*modp.pDrmFormatModifierProperties));
3156  if (!modp.pDrmFormatModifierProperties) {
3157  vulkan_frame_free(hwfc, f);
3158  return AVERROR(ENOMEM);
3159  }
3160  vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt->fallback[i], &fmtp);
3161 
3162  for (uint32_t i = 0; i < modp.drmFormatModifierCount; ++i) {
3163  VkDrmFormatModifierPropertiesEXT *m = &modp.pDrmFormatModifierProperties[i];
3164  if (m->drmFormatModifier == drm_mod.drmFormatModifier) {
3165  mod_props = m;
3166  break;
3167  }
3168  }
3169 
3170  if (mod_props == NULL) {
3171  av_log(hwfc, AV_LOG_ERROR, "No DRM format modifier properties found for modifier 0x%016"PRIx64"\n",
3172  drm_mod.drmFormatModifier);
3173  av_free(modp.pDrmFormatModifierProperties);
3174  vulkan_frame_free(hwfc, f);
3175  return AVERROR_EXTERNAL;
3176  }
3177 
3178  fp->drm_format_modifier_properties[i] = *mod_props;
3179  av_free(modp.pDrmFormatModifierProperties);
3180  }
3181  }
3182 
3183  vulkan_frame_free(hwfc, f);
3184 
3185  /* If user did not specify a pool, hwfc->pool will be set to the internal one
3186  * in hwcontext.c just after this gets called */
3187  if (!hwfc->pool) {
3188  ffhwframesctx(hwfc)->pool_internal = av_buffer_pool_init2(sizeof(AVVkFrame),
3189  hwfc, vulkan_pool_alloc,
3190  NULL);
3191  if (!ffhwframesctx(hwfc)->pool_internal)
3192  return AVERROR(ENOMEM);
3193  }
3194 
3195  return 0;
3196 }
3197 
3198 static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
3199 {
3200  frame->buf[0] = av_buffer_pool_get(hwfc->pool);
3201  if (!frame->buf[0])
3202  return AVERROR(ENOMEM);
3203 
3204  frame->data[0] = frame->buf[0]->data;
3205  frame->format = AV_PIX_FMT_VULKAN;
3206  frame->width = hwfc->width;
3207  frame->height = hwfc->height;
3208 
3209  return 0;
3210 }
3211 
3212 static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc,
3213  enum AVHWFrameTransferDirection dir,
3214  enum AVPixelFormat **formats)
3215 {
3216  enum AVPixelFormat *fmts;
3217  int n = 2;
3218 
3219 #if CONFIG_CUDA
3220  n++;
3221 #endif
3222  fmts = av_malloc_array(n, sizeof(*fmts));
3223  if (!fmts)
3224  return AVERROR(ENOMEM);
3225 
3226  n = 0;
3227  fmts[n++] = hwfc->sw_format;
3228 #if CONFIG_CUDA
3229  fmts[n++] = AV_PIX_FMT_CUDA;
3230 #endif
3231  fmts[n++] = AV_PIX_FMT_NONE;
3232 
3233  *formats = fmts;
3234  return 0;
3235 }
3236 
3237 #if CONFIG_LIBDRM
3238 static void vulkan_unmap_from_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
3239 {
3240  vulkan_frame_free(hwfc, hwmap->priv);
3241 }
3242 
3243 static const struct {
3244  uint32_t drm_fourcc;
3245  VkFormat vk_format;
3246 } vulkan_drm_format_map[] = {
3247  { DRM_FORMAT_R8, VK_FORMAT_R8_UNORM },
3248  { DRM_FORMAT_R16, VK_FORMAT_R16_UNORM },
3249  { DRM_FORMAT_GR88, VK_FORMAT_R8G8_UNORM },
3250  { DRM_FORMAT_RG88, VK_FORMAT_R8G8_UNORM },
3251  { DRM_FORMAT_GR1616, VK_FORMAT_R16G16_UNORM },
3252  { DRM_FORMAT_RG1616, VK_FORMAT_R16G16_UNORM },
3253  { DRM_FORMAT_ARGB8888, VK_FORMAT_B8G8R8A8_UNORM },
3254  { DRM_FORMAT_XRGB8888, VK_FORMAT_B8G8R8A8_UNORM },
3255  { DRM_FORMAT_ABGR8888, VK_FORMAT_R8G8B8A8_UNORM },
3256  { DRM_FORMAT_XBGR8888, VK_FORMAT_R8G8B8A8_UNORM },
3257  { DRM_FORMAT_ARGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
3258  { DRM_FORMAT_ABGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
3259  { DRM_FORMAT_XRGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
3260  { DRM_FORMAT_XBGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
3261 
3262  // All these DRM_FORMATs were added in the same libdrm commit.
3263 #ifdef DRM_FORMAT_XYUV8888
3264  { DRM_FORMAT_XYUV8888, VK_FORMAT_R8G8B8A8_UNORM },
3265  { DRM_FORMAT_XVYU2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 } ,
3266  { DRM_FORMAT_XVYU12_16161616, VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16 } ,
3267  { DRM_FORMAT_XVYU16161616, VK_FORMAT_R16G16B16A16_UNORM } ,
3268 #endif
3269 };
3270 
3271 static inline VkFormat drm_to_vulkan_fmt(uint32_t drm_fourcc)
3272 {
3273  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
3274  if (vulkan_drm_format_map[i].drm_fourcc == drm_fourcc)
3275  return vulkan_drm_format_map[i].vk_format;
3276  return VK_FORMAT_UNDEFINED;
3277 }
3278 
3279 static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **frame,
3280  const AVFrame *src, int flags)
3281 {
3282  int err = 0;
3283  VkResult ret;
3284  AVVkFrame *f;
3285  int bind_counts = 0;
3286  AVHWDeviceContext *ctx = hwfc->device_ctx;
3287  VulkanDevicePriv *p = ctx->hwctx;
3288  AVVulkanDeviceContext *hwctx = &p->p;
3289  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3290  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3291  VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES];
3292  VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES];
3293 
3294  for (int i = 0; i < desc->nb_layers; i++) {
3295  if (drm_to_vulkan_fmt(desc->layers[i].format) == VK_FORMAT_UNDEFINED) {
3296  av_log(ctx, AV_LOG_ERROR, "Unsupported DMABUF layer format %#08x!\n",
3297  desc->layers[i].format);
3298  return AVERROR(EINVAL);
3299  }
3300  }
3301 
3302  if (!(f = av_vk_frame_alloc())) {
3303  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
3304  err = AVERROR(ENOMEM);
3305  goto fail;
3306  }
3307 
3308  f->tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
3309 
3310  for (int i = 0; i < desc->nb_layers; i++) {
3311  const int planes = desc->layers[i].nb_planes;
3312 
3313  /* Semaphore */
3314  VkSemaphoreTypeCreateInfo sem_type_info = {
3315  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3316  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
3317  .initialValue = 0,
3318  };
3319  VkSemaphoreCreateInfo sem_spawn = {
3320  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3321  .pNext = &sem_type_info,
3322  };
3323 
3324  /* Image creation */
3325  VkSubresourceLayout ext_img_layouts[AV_DRM_MAX_PLANES];
3326  VkImageDrmFormatModifierExplicitCreateInfoEXT ext_img_mod_spec = {
3327  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT,
3328  .drmFormatModifier = desc->objects[0].format_modifier,
3329  .drmFormatModifierPlaneCount = planes,
3330  .pPlaneLayouts = (const VkSubresourceLayout *)&ext_img_layouts,
3331  };
3332  VkExternalMemoryImageCreateInfo ext_img_spec = {
3333  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
3334  .pNext = &ext_img_mod_spec,
3335  .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3336  };
3337  VkImageCreateInfo create_info = {
3338  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
3339  .pNext = &ext_img_spec,
3340  .imageType = VK_IMAGE_TYPE_2D,
3341  .format = drm_to_vulkan_fmt(desc->layers[i].format),
3342  .extent.depth = 1,
3343  .mipLevels = 1,
3344  .arrayLayers = 1,
3345  .flags = 0x0,
3346  .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
3347  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, /* specs say so */
3348  .usage = 0x0, /* filled in below */
3349  .samples = VK_SAMPLE_COUNT_1_BIT,
3350  .pQueueFamilyIndices = p->img_qfs,
3351  .queueFamilyIndexCount = p->nb_img_qfs,
3352  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
3353  VK_SHARING_MODE_EXCLUSIVE,
3354  };
3355 
3356  /* Image format verification */
3357  VkExternalImageFormatProperties ext_props = {
3358  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
3359  };
3360  VkImageFormatProperties2 props_ret = {
3361  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
3362  .pNext = &ext_props,
3363  };
3364  VkPhysicalDeviceImageDrmFormatModifierInfoEXT props_drm_mod = {
3365  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
3366  .drmFormatModifier = ext_img_mod_spec.drmFormatModifier,
3367  .pQueueFamilyIndices = create_info.pQueueFamilyIndices,
3368  .queueFamilyIndexCount = create_info.queueFamilyIndexCount,
3369  .sharingMode = create_info.sharingMode,
3370  };
3371  VkPhysicalDeviceExternalImageFormatInfo props_ext = {
3372  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
3373  .pNext = &props_drm_mod,
3374  .handleType = ext_img_spec.handleTypes,
3375  };
3376  VkPhysicalDeviceImageFormatInfo2 fmt_props;
3377 
3378  if (flags & AV_HWFRAME_MAP_READ)
3379  create_info.usage |= VK_IMAGE_USAGE_SAMPLED_BIT |
3380  VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
3381  if (flags & AV_HWFRAME_MAP_WRITE)
3382  create_info.usage |= VK_IMAGE_USAGE_STORAGE_BIT |
3383  VK_IMAGE_USAGE_TRANSFER_DST_BIT;
3384 
3385  fmt_props = (VkPhysicalDeviceImageFormatInfo2) {
3386  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
3387  .pNext = &props_ext,
3388  .format = create_info.format,
3389  .type = create_info.imageType,
3390  .tiling = create_info.tiling,
3391  .usage = create_info.usage,
3392  .flags = create_info.flags,
3393  };
3394 
3395  /* Check if importing is possible for this combination of parameters */
3396  ret = vk->GetPhysicalDeviceImageFormatProperties2(hwctx->phys_dev,
3397  &fmt_props, &props_ret);
3398  if (ret != VK_SUCCESS) {
3399  av_log(ctx, AV_LOG_ERROR, "Cannot map DRM frame to Vulkan: %s\n",
3400  ff_vk_ret2str(ret));
3401  err = AVERROR_EXTERNAL;
3402  goto fail;
3403  }
3404 
3405  /* Set the image width/height */
3406  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
3407  hwfc->sw_format, src->width, src->height, i);
3408 
3409  /* Set the subresource layout based on the layer properties */
3410  for (int j = 0; j < planes; j++) {
3411  ext_img_layouts[j].offset = desc->layers[i].planes[j].offset;
3412  ext_img_layouts[j].rowPitch = desc->layers[i].planes[j].pitch;
3413  ext_img_layouts[j].size = 0; /* The specs say so for all 3 */
3414  ext_img_layouts[j].arrayPitch = 0;
3415  ext_img_layouts[j].depthPitch = 0;
3416  }
3417 
3418  /* Create image */
3419  ret = vk->CreateImage(hwctx->act_dev, &create_info,
3420  hwctx->alloc, &f->img[i]);
3421  if (ret != VK_SUCCESS) {
3422  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
3423  ff_vk_ret2str(ret));
3424  err = AVERROR(EINVAL);
3425  goto fail;
3426  }
3427 
3428  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3429  hwctx->alloc, &f->sem[i]);
3430  if (ret != VK_SUCCESS) {
3431  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3432  ff_vk_ret2str(ret));
3433  err = AVERROR_EXTERNAL;
3434  goto fail;
3435  }
3436 
3437  f->queue_family[i] = VK_QUEUE_FAMILY_EXTERNAL;
3438  f->layout[i] = create_info.initialLayout;
3439  f->access[i] = 0x0;
3440  f->sem_value[i] = 0;
3441  }
3442 
3443  for (int i = 0; i < desc->nb_layers; i++) {
3444  /* Memory requirements */
3445  VkImageMemoryRequirementsInfo2 req_desc = {
3446  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
3447  .image = f->img[i],
3448  };
3449  VkMemoryDedicatedRequirements ded_req = {
3450  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
3451  };
3452  VkMemoryRequirements2 req2 = {
3453  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
3454  .pNext = &ded_req,
3455  };
3456 
3457  /* Allocation/importing */
3458  VkMemoryFdPropertiesKHR fdmp = {
3459  .sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR,
3460  };
3461  /* This assumes that a layer will never be constructed from multiple
3462  * objects. If that was to happen in the real world, this code would
3463  * need to import each plane separately.
3464  */
3465  VkImportMemoryFdInfoKHR idesc = {
3466  .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
3467  .fd = dup(desc->objects[desc->layers[i].planes[0].object_index].fd),
3468  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3469  };
3470  VkMemoryDedicatedAllocateInfo ded_alloc = {
3471  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
3472  .pNext = &idesc,
3473  .image = req_desc.image,
3474  };
3475 
3476  /* Get object properties */
3477  ret = vk->GetMemoryFdPropertiesKHR(hwctx->act_dev,
3478  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3479  idesc.fd, &fdmp);
3480  if (ret != VK_SUCCESS) {
3481  av_log(hwfc, AV_LOG_ERROR, "Failed to get FD properties: %s\n",
3482  ff_vk_ret2str(ret));
3483  err = AVERROR_EXTERNAL;
3484  close(idesc.fd);
3485  goto fail;
3486  }
3487 
3488  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req2);
3489 
3490  /* Only a single bit must be set, not a range, and it must match */
3491  req2.memoryRequirements.memoryTypeBits = fdmp.memoryTypeBits;
3492 
3493  err = alloc_mem(ctx, &req2.memoryRequirements,
3494  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
3495  (ded_req.prefersDedicatedAllocation ||
3496  ded_req.requiresDedicatedAllocation) ?
3497  &ded_alloc : ded_alloc.pNext,
3498  &f->flags, &f->mem[i]);
3499  if (err) {
3500  close(idesc.fd);
3501  return err;
3502  }
3503 
3504  f->size[i] = req2.memoryRequirements.size;
3505  }
3506 
3507  for (int i = 0; i < desc->nb_layers; i++) {
3508  const int planes = desc->layers[i].nb_planes;
3509  for (int j = 0; j < planes; j++) {
3510  VkImageAspectFlagBits aspect = j == 0 ? VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT :
3511  j == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3512  VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
3513 
3514  plane_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO;
3515  plane_info[bind_counts].pNext = NULL;
3516  plane_info[bind_counts].planeAspect = aspect;
3517 
3518  bind_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
3519  bind_info[bind_counts].pNext = planes > 1 ? &plane_info[bind_counts] : NULL;
3520  bind_info[bind_counts].image = f->img[i];
3521  bind_info[bind_counts].memory = f->mem[i];
3522 
3523  /* Offset is already signalled via pPlaneLayouts above */
3524  bind_info[bind_counts].memoryOffset = 0;
3525 
3526  bind_counts++;
3527  }
3528  }
3529 
3530  /* Bind the allocated memory to the images */
3531  ret = vk->BindImageMemory2(hwctx->act_dev, bind_counts, bind_info);
3532  if (ret != VK_SUCCESS) {
3533  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
3534  ff_vk_ret2str(ret));
3535  err = AVERROR_EXTERNAL;
3536  goto fail;
3537  }
3538 
3539  *frame = f;
3540 
3541  return 0;
3542 
3543 fail:
3544  vulkan_frame_free(hwfc, f);
3545 
3546  return err;
3547 }
3548 
3549 static int vulkan_map_from_drm_frame_sync(AVHWFramesContext *hwfc, AVFrame *dst,
3550  const AVDRMFrameDescriptor *desc, int flags)
3551 {
3552  int err;
3553  VkResult ret;
3554  AVHWDeviceContext *ctx = hwfc->device_ctx;
3555  VulkanDevicePriv *p = ctx->hwctx;
3556  VulkanFramesPriv *fp = hwfc->hwctx;
3557  AVVulkanDeviceContext *hwctx = &p->p;
3558  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3559 
3560 #ifdef DMA_BUF_IOCTL_EXPORT_SYNC_FILE
3561  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM) {
3562  VkCommandBuffer cmd_buf;
3563  FFVkExecContext *exec;
3564  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
3565  VkSemaphore drm_sync_sem[AV_DRM_MAX_PLANES] = { 0 };
3566  int nb_img_bar = 0;
3567 
3568  for (int i = 0; i < desc->nb_objects; i++) {
3569  VkSemaphoreTypeCreateInfo sem_type_info = {
3570  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3571  .semaphoreType = VK_SEMAPHORE_TYPE_BINARY,
3572  };
3573  VkSemaphoreCreateInfo sem_spawn = {
3574  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3575  .pNext = &sem_type_info,
3576  };
3577  VkImportSemaphoreFdInfoKHR import_info;
3578  struct dma_buf_export_sync_file implicit_fd_info = {
3579  .flags = DMA_BUF_SYNC_READ,
3580  .fd = -1,
3581  };
3582 
3583  if (ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE,
3584  &implicit_fd_info)) {
3585  err = AVERROR(errno);
3586  av_log(hwctx, AV_LOG_ERROR, "Failed to retrieve implicit DRM sync file: %s\n",
3587  av_err2str(err));
3588  for (; i >= 0; i--)
3589  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3590  return err;
3591  }
3592 
3593  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3594  hwctx->alloc, &drm_sync_sem[i]);
3595  if (ret != VK_SUCCESS) {
3596  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3597  ff_vk_ret2str(ret));
3598  err = AVERROR_EXTERNAL;
3599  for (; i >= 0; i--)
3600  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3601  return err;
3602  }
3603 
3604  import_info = (VkImportSemaphoreFdInfoKHR) {
3605  .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
3606  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3607  .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT,
3608  .semaphore = drm_sync_sem[i],
3609  .fd = implicit_fd_info.fd,
3610  };
3611 
3612  ret = vk->ImportSemaphoreFdKHR(hwctx->act_dev, &import_info);
3613  if (ret != VK_SUCCESS) {
3614  av_log(hwctx, AV_LOG_ERROR, "Failed to import semaphore: %s\n",
3615  ff_vk_ret2str(ret));
3616  err = AVERROR_EXTERNAL;
3617  for (; i >= 0; i--)
3618  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3619  return err;
3620  }
3621  }
3622 
3623  exec = ff_vk_exec_get(&p->vkctx, &fp->compute_exec);
3624  cmd_buf = exec->buf;
3625 
3626  ff_vk_exec_start(&p->vkctx, exec);
3627 
3628  /* Ownership of semaphores is passed */
3629  err = ff_vk_exec_add_dep_bool_sem(&p->vkctx, exec,
3630  drm_sync_sem, desc->nb_objects,
3631  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 1);
3632  if (err < 0)
3633  return err;
3634 
3635  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, dst,
3636  VK_PIPELINE_STAGE_2_NONE,
3637  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
3638  if (err < 0)
3639  return err;
3640 
3641  ff_vk_frame_barrier(&p->vkctx, exec, dst, img_bar, &nb_img_bar,
3642  VK_PIPELINE_STAGE_2_NONE,
3643  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
3644  ((flags & AV_HWFRAME_MAP_READ) ?
3645  VK_ACCESS_2_SHADER_SAMPLED_READ_BIT : 0x0) |
3646  ((flags & AV_HWFRAME_MAP_WRITE) ?
3647  VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0x0),
3648  VK_IMAGE_LAYOUT_GENERAL,
3649  p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0]);
3650 
3651  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
3652  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3653  .pImageMemoryBarriers = img_bar,
3654  .imageMemoryBarrierCount = nb_img_bar,
3655  });
3656 
3657  err = ff_vk_exec_submit(&p->vkctx, exec);
3658  if (err < 0)
3659  return err;
3660  } else
3661 #endif
3662  {
3663  AVVkFrame *f = (AVVkFrame *)dst->data[0];
3664  av_log(hwctx, AV_LOG_WARNING, "No support for synchronization when importing DMA-BUFs, "
3665  "image may be corrupted.\n");
3666  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3667  if (err)
3668  return err;
3669  }
3670 
3671  return 0;
3672 }
3673 
3674 static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3675  const AVFrame *src, int flags)
3676 {
3677  int err = 0;
3678  AVVkFrame *f;
3679  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3680 
3681  if ((err = vulkan_map_from_drm_frame_desc(hwfc, &f, src, flags)))
3682  return err;
3683 
3684  /* The unmapping function will free this */
3685  dst->data[0] = (uint8_t *)f;
3686  dst->width = src->width;
3687  dst->height = src->height;
3688 
3689  err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
3690  &vulkan_unmap_from_drm, f);
3691  if (err < 0)
3692  goto fail;
3693 
3694  err = vulkan_map_from_drm_frame_sync(hwfc, dst, desc, flags);
3695  if (err < 0)
3696  return err;
3697 
3698  av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n");
3699 
3700  return 0;
3701 
3702 fail:
3703  vulkan_frame_free(hwfc->device_ctx->hwctx, f);
3704  dst->data[0] = NULL;
3705  return err;
3706 }
3707 
3708 #if CONFIG_VAAPI
3709 static int vulkan_map_from_vaapi(AVHWFramesContext *dst_fc,
3710  AVFrame *dst, const AVFrame *src,
3711  int flags)
3712 {
3713  int err;
3714  AVFrame *tmp = av_frame_alloc();
3715  AVHWFramesContext *vaapi_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3716  AVVAAPIDeviceContext *vaapi_ctx = vaapi_fc->device_ctx->hwctx;
3717  VASurfaceID surface_id = (VASurfaceID)(uintptr_t)src->data[3];
3718 
3719  if (!tmp)
3720  return AVERROR(ENOMEM);
3721 
3722  /* We have to sync since like the previous comment said, no semaphores */
3723  vaSyncSurface(vaapi_ctx->display, surface_id);
3724 
3725  tmp->format = AV_PIX_FMT_DRM_PRIME;
3726 
3727  err = av_hwframe_map(tmp, src, flags);
3728  if (err < 0)
3729  goto fail;
3730 
3731  err = vulkan_map_from_drm(dst_fc, dst, tmp, flags);
3732  if (err < 0)
3733  goto fail;
3734 
3735  err = ff_hwframe_map_replace(dst, src);
3736 
3737 fail:
3738  av_frame_free(&tmp);
3739  return err;
3740 }
3741 #endif
3742 #endif
3743 
3744 #if CONFIG_CUDA
3745 static int export_mem_to_cuda(AVHWDeviceContext *ctx,
3746  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3747  AVVkFrameInternal *dst_int, int idx,
3748  VkDeviceMemory mem, size_t size)
3749 {
3750  VkResult ret;
3751  VulkanDevicePriv *p = ctx->hwctx;
3752  AVVulkanDeviceContext *hwctx = &p->p;
3753  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3754 
3755 #ifdef _WIN32
3756  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3757  .type = IsWindows8OrGreater()
3758  ? CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32
3759  : CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT,
3760  .size = size,
3761  };
3762  VkMemoryGetWin32HandleInfoKHR export_info = {
3763  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR,
3764  .memory = mem,
3765  .handleType = IsWindows8OrGreater()
3766  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
3767  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3768  };
3769 
3770  ret = vk->GetMemoryWin32HandleKHR(hwctx->act_dev, &export_info,
3771  &ext_desc.handle.win32.handle);
3772  if (ret != VK_SUCCESS) {
3773  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a Win32 Handle: %s!\n",
3774  ff_vk_ret2str(ret));
3775  return AVERROR_EXTERNAL;
3776  }
3777  dst_int->ext_mem_handle[idx] = ext_desc.handle.win32.handle;
3778 #else
3779  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3780  .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD,
3781  .size = size,
3782  };
3783  VkMemoryGetFdInfoKHR export_info = {
3784  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
3785  .memory = mem,
3786  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR,
3787  };
3788 
3789  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
3790  &ext_desc.handle.fd);
3791  if (ret != VK_SUCCESS) {
3792  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a FD: %s!\n",
3793  ff_vk_ret2str(ret));
3794  return AVERROR_EXTERNAL;
3795  }
3796 #endif
3797 
3798  ret = CHECK_CU(cu->cuImportExternalMemory(&dst_int->ext_mem[idx], &ext_desc));
3799  if (ret < 0) {
3800 #ifndef _WIN32
3801  close(ext_desc.handle.fd);
3802 #endif
3803  return AVERROR_EXTERNAL;
3804  }
3805 
3806  return 0;
3807 }
3808 
3809 static int export_sem_to_cuda(AVHWDeviceContext *ctx,
3810  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3811  AVVkFrameInternal *dst_int, int idx,
3812  VkSemaphore sem)
3813 {
3814  VkResult ret;
3815  VulkanDevicePriv *p = ctx->hwctx;
3816  AVVulkanDeviceContext *hwctx = &p->p;
3817  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3818 
3819 #ifdef _WIN32
3820  VkSemaphoreGetWin32HandleInfoKHR sem_export = {
3821  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR,
3822  .semaphore = sem,
3823  .handleType = IsWindows8OrGreater()
3824  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
3825  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3826  };
3827  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3828  .type = 10 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32 */,
3829  };
3830 #else
3831  VkSemaphoreGetFdInfoKHR sem_export = {
3832  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
3833  .semaphore = sem,
3834  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
3835  };
3836  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3837  .type = 9 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD */,
3838  };
3839 #endif
3840 
3841 #ifdef _WIN32
3842  ret = vk->GetSemaphoreWin32HandleKHR(hwctx->act_dev, &sem_export,
3843  &ext_sem_desc.handle.win32.handle);
3844 #else
3845  ret = vk->GetSemaphoreFdKHR(hwctx->act_dev, &sem_export,
3846  &ext_sem_desc.handle.fd);
3847 #endif
3848  if (ret != VK_SUCCESS) {
3849  av_log(ctx, AV_LOG_ERROR, "Failed to export semaphore: %s\n",
3850  ff_vk_ret2str(ret));
3851  return AVERROR_EXTERNAL;
3852  }
3853 #ifdef _WIN32
3854  dst_int->ext_sem_handle[idx] = ext_sem_desc.handle.win32.handle;
3855 #endif
3856 
3857  ret = CHECK_CU(cu->cuImportExternalSemaphore(&dst_int->cu_sem[idx],
3858  &ext_sem_desc));
3859  if (ret < 0) {
3860 #ifndef _WIN32
3861  close(ext_sem_desc.handle.fd);
3862 #endif
3863  return AVERROR_EXTERNAL;
3864  }
3865 
3866  return 0;
3867 }
3868 
3869 static int vulkan_export_to_cuda(AVHWFramesContext *hwfc,
3870  AVBufferRef *cuda_hwfc,
3871  const AVFrame *frame)
3872 {
3873  int err;
3874  VkResult ret;
3875  AVVkFrame *dst_f;
3876  AVVkFrameInternal *dst_int;
3877  AVHWDeviceContext *ctx = hwfc->device_ctx;
3878  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3879  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3880  VulkanDevicePriv *p = ctx->hwctx;
3881  AVVulkanDeviceContext *hwctx = &p->p;
3882  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3883  int nb_images;
3884 
3885  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)cuda_hwfc->data;
3886  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3887  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3888  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3889  CudaFunctions *cu = cu_internal->cuda_dl;
3890  CUarray_format cufmt = desc->comp[0].depth > 8 ? CU_AD_FORMAT_UNSIGNED_INT16 :
3891  CU_AD_FORMAT_UNSIGNED_INT8;
3892 
3893  dst_f = (AVVkFrame *)frame->data[0];
3894  dst_int = dst_f->internal;
3895 
3896  if (!dst_int->cuda_fc_ref) {
3897  size_t offsets[3] = { 0 };
3898 
3899  dst_int->cuda_fc_ref = av_buffer_ref(cuda_hwfc);
3900  if (!dst_int->cuda_fc_ref)
3901  return AVERROR(ENOMEM);
3902 
3903  nb_images = ff_vk_count_images(dst_f);
3904  for (int i = 0; i < nb_images; i++) {
3905  err = export_mem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3906  dst_f->mem[i], dst_f->size[i]);
3907  if (err < 0)
3908  goto fail;
3909 
3910  err = export_sem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3911  dst_f->sem[i]);
3912  if (err < 0)
3913  goto fail;
3914  }
3915 
3916  if (nb_images != planes) {
3917  for (int i = 0; i < planes; i++) {
3918  VkImageSubresource subres = {
3919  .aspectMask = i == 2 ? VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT :
3920  i == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3921  VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT
3922  };
3923  VkSubresourceLayout layout = { 0 };
3924  vk->GetImageSubresourceLayout(hwctx->act_dev, dst_f->img[FFMIN(i, nb_images - 1)],
3925  &subres, &layout);
3926  offsets[i] = layout.offset;
3927  }
3928  }
3929 
3930  for (int i = 0; i < planes; i++) {
3931  CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC tex_desc = {
3932  .offset = offsets[i],
3933  .arrayDesc = {
3934  .Depth = 0,
3935  .Format = cufmt,
3936  .NumChannels = 1 + ((planes == 2) && i),
3937  .Flags = 0,
3938  },
3939  .numLevels = 1,
3940  };
3941  int p_w, p_h;
3942 
3943  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
3944  tex_desc.arrayDesc.Width = p_w;
3945  tex_desc.arrayDesc.Height = p_h;
3946 
3947  ret = CHECK_CU(cu->cuExternalMemoryGetMappedMipmappedArray(&dst_int->cu_mma[i],
3948  dst_int->ext_mem[FFMIN(i, nb_images - 1)],
3949  &tex_desc));
3950  if (ret < 0) {
3951  err = AVERROR_EXTERNAL;
3952  goto fail;
3953  }
3954 
3955  ret = CHECK_CU(cu->cuMipmappedArrayGetLevel(&dst_int->cu_array[i],
3956  dst_int->cu_mma[i], 0));
3957  if (ret < 0) {
3958  err = AVERROR_EXTERNAL;
3959  goto fail;
3960  }
3961 
3962  }
3963  }
3964 
3965  return 0;
3966 
3967 fail:
3968  vulkan_free_internal(dst_f);
3969  return err;
3970 }
3971 
3972 static int vulkan_transfer_data_from_cuda(AVHWFramesContext *hwfc,
3973  AVFrame *dst, const AVFrame *src)
3974 {
3975  int err;
3976  CUcontext dummy;
3977  AVVkFrame *dst_f;
3978  AVVkFrameInternal *dst_int;
3979  VulkanFramesPriv *fp = hwfc->hwctx;
3980  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3981  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3982 
3983  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3984  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3985  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3986  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3987  CudaFunctions *cu = cu_internal->cuda_dl;
3988  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
3989  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
3990 
3991  dst_f = (AVVkFrame *)dst->data[0];
3992 
3993  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
3994  if (err < 0)
3995  return err;
3996 
3997  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
3998  if (err < 0)
3999  return err;
4000 
4001  err = vulkan_export_to_cuda(hwfc, src->hw_frames_ctx, dst);
4002  if (err < 0) {
4003  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4004  return err;
4005  }
4006 
4007  dst_int = dst_f->internal;
4008 
4009  for (int i = 0; i < planes; i++) {
4010  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
4011  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
4012  }
4013 
4014  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
4015  planes, cuda_dev->stream));
4016  if (err < 0)
4017  goto fail;
4018 
4019  for (int i = 0; i < planes; i++) {
4020  CUDA_MEMCPY2D cpy = {
4021  .srcMemoryType = CU_MEMORYTYPE_DEVICE,
4022  .srcDevice = (CUdeviceptr)src->data[i],
4023  .srcPitch = src->linesize[i],
4024  .srcY = 0,
4025 
4026  .dstMemoryType = CU_MEMORYTYPE_ARRAY,
4027  .dstArray = dst_int->cu_array[i],
4028  };
4029 
4030  int p_w, p_h;
4031  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
4032 
4033  cpy.WidthInBytes = p_w * desc->comp[i].step;
4034  cpy.Height = p_h;
4035 
4036  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
4037  if (err < 0)
4038  goto fail;
4039  }
4040 
4041  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
4042  planes, cuda_dev->stream));
4043  if (err < 0)
4044  goto fail;
4045 
4046  for (int i = 0; i < planes; i++)
4047  dst_f->sem_value[i]++;
4048 
4049  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4050 
4051  av_log(hwfc, AV_LOG_VERBOSE, "Transferred CUDA image to Vulkan!\n");
4052 
4053  return err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
4054 
4055 fail:
4056  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4057  vulkan_free_internal(dst_f);
4058  av_buffer_unref(&dst->buf[0]);
4059  return err;
4060 }
4061 #endif
4062 
4063 static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
4064  const AVFrame *src, int flags)
4065 {
4066  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4067 
4068  switch (src->format) {
4069 #if CONFIG_LIBDRM
4070 #if CONFIG_VAAPI
4071  case AV_PIX_FMT_VAAPI:
4072  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4073  return vulkan_map_from_vaapi(hwfc, dst, src, flags);
4074  else
4075  return AVERROR(ENOSYS);
4076 #endif
4077  case AV_PIX_FMT_DRM_PRIME:
4078  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4079  return vulkan_map_from_drm(hwfc, dst, src, flags);
4080  else
4081  return AVERROR(ENOSYS);
4082 #endif
4083  default:
4084  return AVERROR(ENOSYS);
4085  }
4086 }
4087 
4088 #if CONFIG_LIBDRM
4089 typedef struct VulkanDRMMapping {
4090  AVDRMFrameDescriptor drm_desc;
4091  AVVkFrame *source;
4092 } VulkanDRMMapping;
4093 
4094 static void vulkan_unmap_to_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
4095 {
4096  AVDRMFrameDescriptor *drm_desc = hwmap->priv;
4097 
4098  /* on unmap from DRM, make sure to import sync objects so that we are sync'd with any work that was
4099  * done on the buffer while exported. We don't know if who used the dmabuf did reads or writes, so protect against both */
4100  vulkan_map_from_drm_frame_sync(hwfc, hwmap->source, drm_desc, AV_HWFRAME_MAP_READ | AV_HWFRAME_MAP_WRITE);
4101 
4102  for (int i = 0; i < drm_desc->nb_objects; i++)
4103  close(drm_desc->objects[i].fd);
4104 
4105  av_free(drm_desc);
4106 }
4107 
4108 static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt)
4109 {
4110  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
4111  if (vulkan_drm_format_map[i].vk_format == vkfmt)
4112  return vulkan_drm_format_map[i].drm_fourcc;
4113  return DRM_FORMAT_INVALID;
4114 }
4115 
4116 #define MAX_MEMORY_PLANES 4
4117 static VkImageAspectFlags plane_index_to_aspect(int plane) {
4118  if (plane == 0) return VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
4119  if (plane == 1) return VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT;
4120  if (plane == 2) return VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
4121  if (plane == 3) return VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT;
4122 
4123  av_assert2 (0 && "Invalid plane index");
4124  return VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
4125 }
4126 
4127 static int vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
4128  const AVFrame *src, int flags)
4129 {
4130  int err = 0;
4131  VkResult ret;
4132  AVVkFrame *f = (AVVkFrame *)src->data[0];
4133  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4134  AVVulkanDeviceContext *hwctx = &p->p;
4135  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4136  VulkanFramesPriv *fp = hwfc->hwctx;
4137  const int nb_images = ff_vk_count_images(f);
4138  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
4139  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
4140  };
4141  VkSemaphoreWaitInfo wait_info = {
4142  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
4143  .flags = 0x0,
4144  .semaphoreCount = nb_images,
4145  };
4146 
4147  AVDRMFrameDescriptor *drm_desc = av_mallocz(sizeof(*drm_desc));
4148  if (!drm_desc)
4149  return AVERROR(ENOMEM);
4150 
4151  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_EXPORT);
4152  if (err < 0)
4153  goto end;
4154 
4155  /* Wait for the operation to finish so we can cleanly export it. */
4156  wait_info.pSemaphores = f->sem;
4157  wait_info.pValues = f->sem_value;
4158 
4159  vk->WaitSemaphores(hwctx->act_dev, &wait_info, UINT64_MAX);
4160 
4161  err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, &vulkan_unmap_to_drm, drm_desc);
4162  if (err < 0)
4163  goto end;
4164 
4165  ret = vk->GetImageDrmFormatModifierPropertiesEXT(hwctx->act_dev, f->img[0],
4166  &drm_mod);
4167  if (ret != VK_SUCCESS) {
4168  av_log(hwfc, AV_LOG_ERROR, "Failed to retrieve DRM format modifier!\n");
4169  err = AVERROR_EXTERNAL;
4170  goto end;
4171  }
4172 
4173  for (int i = 0; (i < nb_images) && (f->mem[i]); i++) {
4174  VkMemoryGetFdInfoKHR export_info = {
4175  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
4176  .memory = f->mem[i],
4177  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
4178  };
4179 
4180  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
4181  &drm_desc->objects[i].fd);
4182  if (ret != VK_SUCCESS) {
4183  av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a FD!\n");
4184  err = AVERROR_EXTERNAL;
4185  goto end;
4186  }
4187 
4188  drm_desc->nb_objects++;
4189  drm_desc->objects[i].size = f->size[i];
4190  drm_desc->objects[i].format_modifier = drm_mod.drmFormatModifier;
4191  }
4192 
4193  drm_desc->nb_layers = nb_images;
4194  for (int i = 0; i < drm_desc->nb_layers; i++) {
4195  VkFormat plane_vkfmt = av_vkfmt_from_pixfmt(hwfc->sw_format)[i];
4196 
4197  drm_desc->layers[i].format = vulkan_fmt_to_drm(plane_vkfmt);
4198  drm_desc->layers[i].nb_planes = fp->drm_format_modifier_properties[i].drmFormatModifierPlaneCount;
4199 
4200  if (drm_desc->layers[i].nb_planes > MAX_MEMORY_PLANES) {
4201  av_log(hwfc, AV_LOG_ERROR, "Too many memory planes for DRM format!\n");
4202  err = AVERROR_EXTERNAL;
4203  goto end;
4204  }
4205 
4206  for (int j = 0; j < drm_desc->layers[i].nb_planes; j++) {
4207  VkSubresourceLayout layout;
4208  VkImageSubresource sub = {
4209  .aspectMask = plane_index_to_aspect(j),
4210  };
4211 
4212  drm_desc->layers[i].planes[j].object_index = FFMIN(i, drm_desc->nb_objects - 1);
4213 
4214  vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, &layout);
4215  drm_desc->layers[i].planes[j].offset = layout.offset;
4216  drm_desc->layers[i].planes[j].pitch = layout.rowPitch;
4217  }
4218 
4219  if (drm_desc->layers[i].format == DRM_FORMAT_INVALID) {
4220  av_log(hwfc, AV_LOG_ERROR, "Cannot map to DRM layer, unsupported!\n");
4221  err = AVERROR_PATCHWELCOME;
4222  goto end;
4223  }
4224 
4225 
4226  if (f->tiling == VK_IMAGE_TILING_OPTIMAL)
4227  continue;
4228 
4229  }
4230 
4231  dst->width = src->width;
4232  dst->height = src->height;
4233  dst->data[0] = (uint8_t *)drm_desc;
4234 
4235  av_log(hwfc, AV_LOG_VERBOSE, "Mapped AVVkFrame to a DRM object!\n");
4236 
4237  return 0;
4238 
4239 end:
4240  av_free(drm_desc);
4241  return err;
4242 }
4243 
4244 #if CONFIG_VAAPI
4245 static int vulkan_map_to_vaapi(AVHWFramesContext *hwfc, AVFrame *dst,
4246  const AVFrame *src, int flags)
4247 {
4248  int err;
4249  AVFrame *tmp = av_frame_alloc();
4250  if (!tmp)
4251  return AVERROR(ENOMEM);
4252 
4253  tmp->format = AV_PIX_FMT_DRM_PRIME;
4254 
4255  err = vulkan_map_to_drm(hwfc, tmp, src, flags);
4256  if (err < 0)
4257  goto fail;
4258 
4259  err = av_hwframe_map(dst, tmp, flags);
4260  if (err < 0)
4261  goto fail;
4262 
4263  err = ff_hwframe_map_replace(dst, src);
4264 
4265 fail:
4266  av_frame_free(&tmp);
4267  return err;
4268 }
4269 #endif
4270 #endif
4271 
4272 static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
4273  const AVFrame *src, int flags)
4274 {
4275  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4276 
4277  switch (dst->format) {
4278 #if CONFIG_LIBDRM
4279  case AV_PIX_FMT_DRM_PRIME:
4280  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4281  return vulkan_map_to_drm(hwfc, dst, src, flags);
4282  else
4283  return AVERROR(ENOSYS);
4284 #if CONFIG_VAAPI
4285  case AV_PIX_FMT_VAAPI:
4286  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4287  return vulkan_map_to_vaapi(hwfc, dst, src, flags);
4288  else
4289  return AVERROR(ENOSYS);
4290 #endif
4291 #endif
4292  default:
4293  break;
4294  }
4295  return AVERROR(ENOSYS);
4296 }
4297 
4298 static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf,
4299  AVFrame *swf, VkBufferImageCopy *region,
4300  int planes, int upload)
4301 {
4302  int err;
4303  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4304  FFVkBuffer *vkbuf = (FFVkBuffer *)buf->data;
4305 
4306  if (upload) {
4307  for (int i = 0; i < planes; i++)
4308  av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset,
4309  region[i].bufferRowLength,
4310  swf->data[i],
4311  swf->linesize[i],
4312  swf->linesize[i],
4313  region[i].imageExtent.height);
4314 
4315  err = ff_vk_flush_buffer(&p->vkctx, vkbuf, 0, VK_WHOLE_SIZE, 1);
4316  if (err != VK_SUCCESS) {
4317  av_log(hwfc, AV_LOG_ERROR, "Failed to flush buffer data: %s\n",
4318  av_err2str(err));
4319  return AVERROR_EXTERNAL;
4320  }
4321  } else {
4322  err = ff_vk_flush_buffer(&p->vkctx, vkbuf, 0, VK_WHOLE_SIZE, 0);
4323  if (err != VK_SUCCESS) {
4324  av_log(hwfc, AV_LOG_ERROR, "Failed to invalidate buffer data: %s\n",
4325  av_err2str(err));
4326  return AVERROR_EXTERNAL;
4327  }
4328 
4329  for (int i = 0; i < planes; i++)
4330  av_image_copy_plane(swf->data[i],
4331  swf->linesize[i],
4332  vkbuf->mapped_mem + region[i].bufferOffset,
4333  region[i].bufferRowLength,
4334  swf->linesize[i],
4335  region[i].imageExtent.height);
4336  }
4337 
4338  return 0;
4339 }
4340 
4341 static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst,
4342  AVFrame *swf, VkBufferImageCopy *region, int upload)
4343 {
4344  int err;
4345  uint32_t p_w, p_h;
4346  VulkanFramesPriv *fp = hwfc->hwctx;
4347  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4348  const int planes = av_pix_fmt_count_planes(swf->format);
4349  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4350  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4351 
4352  size_t buf_offset = 0;
4353  for (int i = 0; i < planes; i++) {
4354  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4355 
4356  region[i] = (VkBufferImageCopy) {
4357  .bufferOffset = buf_offset,
4358  .bufferRowLength = FFALIGN(swf->linesize[i],
4359  p->props.properties.limits.optimalBufferCopyRowPitchAlignment),
4360  .bufferImageHeight = p_h,
4361  .imageSubresource.layerCount = 1,
4362  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4363  /* Rest of the fields adjusted/filled in later */
4364  };
4365 
4366  buf_offset += FFALIGN(p_h*region[i].bufferRowLength,
4367  p->props.properties.limits.optimalBufferCopyOffsetAlignment);
4368  }
4369 
4370  err = ff_vk_get_pooled_buffer(&p->vkctx, &fp->tmp, dst, buf_usage,
4371  NULL, buf_offset,
4372  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
4373  p->vkctx.host_cached_flag);
4374  if (err < 0)
4375  return err;
4376 
4377  return 0;
4378 }
4379 
4380 static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs,
4381  AVFrame *swf, VkBufferImageCopy *region, int upload)
4382 {
4383  int err;
4384  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4385 
4386  int nb_src_bufs;
4387  const int planes = av_pix_fmt_count_planes(swf->format);
4388  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4389  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4390 
4391  /* We can't host map images with negative strides */
4392  for (int i = 0; i < planes; i++)
4393  if (swf->linesize[i] < 0)
4394  return AVERROR(EINVAL);
4395 
4396  /* Count the number of buffers in the software frame */
4397  nb_src_bufs = 0;
4398  while (swf->buf[nb_src_bufs])
4399  nb_src_bufs++;
4400 
4401  /* Single buffer contains all planes */
4402  if (nb_src_bufs == 1) {
4403  err = ff_vk_host_map_buffer(&p->vkctx, &dst[0],
4404  swf->data[0], swf->buf[0],
4405  buf_usage);
4406  if (err < 0)
4407  return err;
4408  (*nb_bufs)++;
4409 
4410  for (int i = 0; i < planes; i++)
4411  region[i].bufferOffset = ((FFVkBuffer *)dst[0]->data)->virtual_offset +
4412  swf->data[i] - swf->data[0];
4413  } else if (nb_src_bufs == planes) { /* One buffer per plane */
4414  for (int i = 0; i < planes; i++) {
4415  err = ff_vk_host_map_buffer(&p->vkctx, &dst[i],
4416  swf->data[i], swf->buf[i],
4417  buf_usage);
4418  if (err < 0)
4419  goto fail;
4420  (*nb_bufs)++;
4421 
4422  region[i].bufferOffset = ((FFVkBuffer *)dst[i]->data)->virtual_offset;
4423  }
4424  } else {
4425  /* Weird layout (3 planes, 2 buffers), patch welcome, fallback to copy */
4426  return AVERROR_PATCHWELCOME;
4427  }
4428 
4429  return 0;
4430 
4431 fail:
4432  for (int i = 0; i < (*nb_bufs); i++)
4433  av_buffer_unref(&dst[i]);
4434  return err;
4435 }
4436 
4437 static int vulkan_transfer_host(AVHWFramesContext *hwfc, AVFrame *hwf,
4438  AVFrame *swf, int upload)
4439 {
4440  VulkanFramesPriv *fp = hwfc->hwctx;
4441  AVVulkanFramesContext *hwfc_vk = &fp->p;
4442  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4443  AVVulkanDeviceContext *hwctx = &p->p;
4444  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4445 
4446  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4447  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format);
4448  const int planes = av_pix_fmt_count_planes(swf->format);
4449  const int nb_images = ff_vk_count_images(hwf_vk);
4450 
4451  VkSemaphoreWaitInfo sem_wait;
4452  VkHostImageLayoutTransitionInfoEXT layout_ch_info[AV_NUM_DATA_POINTERS];
4453  int nb_layout_ch = 0;
4454 
4455  hwfc_vk->lock_frame(hwfc, hwf_vk);
4456 
4457  for (int i = 0; i < nb_images; i++) {
4458  int compat = 0;
4459  for (int j = 0; j < p->vkctx.host_image_props.copySrcLayoutCount; j++) {
4460  if (hwf_vk->layout[i] == p->vkctx.host_image_props.pCopySrcLayouts[j]) {
4461  compat = 1;
4462  break;
4463  }
4464  }
4465  if (compat)
4466  continue;
4467 
4468  layout_ch_info[nb_layout_ch] = (VkHostImageLayoutTransitionInfoEXT) {
4469  .sType = VK_STRUCTURE_TYPE_HOST_IMAGE_LAYOUT_TRANSITION_INFO_EXT,
4470  .image = hwf_vk->img[i],
4471  .oldLayout = hwf_vk->layout[i],
4472  .newLayout = VK_IMAGE_LAYOUT_GENERAL,
4473  .subresourceRange = {
4474  .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
4475  .levelCount = 1,
4476  .layerCount = 1,
4477  },
4478  };
4479 
4480  hwf_vk->layout[i] = layout_ch_info[nb_layout_ch].newLayout;
4481  nb_layout_ch++;
4482  }
4483 
4484  sem_wait = (VkSemaphoreWaitInfo) {
4485  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
4486  .pSemaphores = hwf_vk->sem,
4487  .pValues = hwf_vk->sem_value,
4488  .semaphoreCount = nb_images,
4489  };
4490 
4491  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
4492 
4493  if (nb_layout_ch)
4494  vk->TransitionImageLayoutEXT(hwctx->act_dev,
4495  nb_layout_ch, layout_ch_info);
4496 
4497  if (upload) {
4498  VkMemoryToImageCopyEXT region_info = {
4499  .sType = VK_STRUCTURE_TYPE_MEMORY_TO_IMAGE_COPY_EXT,
4500  .imageSubresource = {
4501  .layerCount = 1,
4502  },
4503  };
4504  VkCopyMemoryToImageInfoEXT copy_info = {
4505  .sType = VK_STRUCTURE_TYPE_COPY_MEMORY_TO_IMAGE_INFO_EXT,
4506  .regionCount = 1,
4507  .pRegions = &region_info,
4508  };
4509  for (int i = 0; i < planes; i++) {
4510  int img_idx = FFMIN(i, (nb_images - 1));
4511  uint32_t p_w, p_h;
4512  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4513 
4514  region_info.pHostPointer = swf->data[i];
4515  region_info.memoryRowLength = swf->linesize[i] / desc->comp[i].step;
4516  region_info.imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4517  region_info.imageExtent = (VkExtent3D){ p_w, p_h, 1 };
4518  copy_info.dstImage = hwf_vk->img[img_idx];
4519  copy_info.dstImageLayout = hwf_vk->layout[img_idx];
4520 
4521  vk->CopyMemoryToImageEXT(hwctx->act_dev, &copy_info);
4522  }
4523  } else {
4524  VkImageToMemoryCopyEXT region_info = {
4525  .sType = VK_STRUCTURE_TYPE_IMAGE_TO_MEMORY_COPY_EXT,
4526  .imageSubresource = {
4527  .layerCount = 1,
4528  },
4529  };
4530  VkCopyImageToMemoryInfoEXT copy_info = {
4531  .sType = VK_STRUCTURE_TYPE_COPY_IMAGE_TO_MEMORY_INFO_EXT,
4532  .regionCount = 1,
4533  .pRegions = &region_info,
4534  };
4535  for (int i = 0; i < planes; i++) {
4536  int img_idx = FFMIN(i, (nb_images - 1));
4537  uint32_t p_w, p_h;
4538  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4539 
4540  region_info.pHostPointer = swf->data[i];
4541  region_info.memoryRowLength = swf->linesize[i] / desc->comp[i].step;
4542  region_info.imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4543  region_info.imageExtent = (VkExtent3D){ p_w, p_h, 1 };
4544  copy_info.srcImage = hwf_vk->img[img_idx];
4545  copy_info.srcImageLayout = hwf_vk->layout[img_idx];
4546 
4547  vk->CopyImageToMemoryEXT(hwctx->act_dev, &copy_info);
4548  }
4549  }
4550 
4551  hwfc_vk->unlock_frame(hwfc, hwf_vk);
4552 
4553  return 0;
4554 }
4555 
4556 static int vulkan_transfer_frame(AVHWFramesContext *hwfc,
4557  AVFrame *swf, AVFrame *hwf,
4558  int upload)
4559 {
4560  int err;
4561  VulkanFramesPriv *fp = hwfc->hwctx;
4562  AVVulkanFramesContext *hwctx = &fp->p;
4563  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4564  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4565 
4566  int host_mapped = 0;
4567 
4568  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4569  VkBufferImageCopy region[AV_NUM_DATA_POINTERS]; // always one per plane
4570 
4571  const int planes = av_pix_fmt_count_planes(swf->format);
4572  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format);
4573  const int nb_images = ff_vk_count_images(hwf_vk);
4574 
4575  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
4576  int nb_img_bar = 0;
4577 
4578  AVBufferRef *bufs[AV_NUM_DATA_POINTERS];
4579  int nb_bufs = 0;
4580 
4581  VkCommandBuffer cmd_buf;
4582  FFVkExecContext *exec;
4583 
4584  /* Sanity checking */
4585  if ((swf->format != AV_PIX_FMT_NONE && !av_vkfmt_from_pixfmt(swf->format))) {
4586  av_log(hwfc, AV_LOG_ERROR, "Unsupported software frame pixel format!\n");
4587  return AVERROR(EINVAL);
4588  }
4589 
4590  if (swf->width > hwfc->width || swf->height > hwfc->height)
4591  return AVERROR(EINVAL);
4592 
4593  if (hwctx->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT &&
4594  !(p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY))
4595  return vulkan_transfer_host(hwfc, hwf, swf, upload);
4596 
4597  for (int i = 0; i < av_pix_fmt_count_planes(swf->format); i++) {
4598  uint32_t p_w, p_h;
4599  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4600 
4601  /* Buffer region for this plane */
4602  region[i] = (VkBufferImageCopy) {
4603  .bufferOffset = 0,
4604  .bufferRowLength = swf->linesize[i],
4605  .bufferImageHeight = p_h,
4606  .imageSubresource.layerCount = 1,
4607  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4608  /* Rest of the fields adjusted/filled in later */
4609  };
4610  }
4611 
4612  /* Setup buffers first */
4613  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY && !p->avoid_host_import) {
4614  err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload);
4615  if (err >= 0)
4616  host_mapped = 1;
4617  }
4618 
4619  if (!host_mapped) {
4620  err = get_plane_buf(hwfc, &bufs[0], swf, region, upload);
4621  if (err < 0)
4622  goto end;
4623  nb_bufs = 1;
4624 
4625  if (upload) {
4626  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 1);
4627  if (err < 0)
4628  goto end;
4629  }
4630  }
4631 
4632  exec = ff_vk_exec_get(&p->vkctx, &fp->upload_exec);
4633  cmd_buf = exec->buf;
4634 
4635  ff_vk_exec_start(&p->vkctx, exec);
4636 
4637  /* Prep destination Vulkan frame */
4638  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, hwf,
4639  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4640  VK_PIPELINE_STAGE_2_TRANSFER_BIT);
4641  if (err < 0)
4642  goto end;
4643 
4644  /* No need to declare buf deps for synchronous transfers (downloads) */
4645  if (upload) {
4646  /* Add the software frame backing the buffers if we're host mapping */
4647  if (host_mapped) {
4648  err = ff_vk_exec_add_dep_sw_frame(&p->vkctx, exec, swf);
4649  if (err < 0) {
4650  ff_vk_exec_discard_deps(&p->vkctx, exec);
4651  goto end;
4652  }
4653  }
4654 
4655  /* Add the buffers as a dependency */
4656  err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, nb_bufs, 1);
4657  if (err < 0) {
4658  ff_vk_exec_discard_deps(&p->vkctx, exec);
4659  goto end;
4660  }
4661  }
4662 
4663  ff_vk_frame_barrier(&p->vkctx, exec, hwf, img_bar, &nb_img_bar,
4664  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4665  VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR,
4666  upload ? VK_ACCESS_TRANSFER_WRITE_BIT :
4667  VK_ACCESS_TRANSFER_READ_BIT,
4668  upload ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL :
4669  VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
4670  p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0]);
4671 
4672  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
4673  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
4674  .pImageMemoryBarriers = img_bar,
4675  .imageMemoryBarrierCount = nb_img_bar,
4676  });
4677 
4678  for (int i = 0; i < planes; i++) {
4679  int buf_idx = FFMIN(i, (nb_bufs - 1));
4680  int img_idx = FFMIN(i, (nb_images - 1));
4681  FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[buf_idx]->data;
4682 
4683  uint32_t orig_stride = region[i].bufferRowLength;
4684  region[i].bufferRowLength /= desc->comp[i].step;
4685  region[i].imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4686 
4687  if (upload)
4688  vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf,
4689  hwf_vk->img[img_idx],
4690  img_bar[img_idx].newLayout,
4691  1, &region[i]);
4692  else
4693  vk->CmdCopyImageToBuffer(cmd_buf, hwf_vk->img[img_idx],
4694  img_bar[img_idx].newLayout,
4695  vkbuf->buf,
4696  1, &region[i]);
4697 
4698  region[i].bufferRowLength = orig_stride;
4699  }
4700 
4701  err = ff_vk_exec_submit(&p->vkctx, exec);
4702  if (err < 0) {
4703  ff_vk_exec_discard_deps(&p->vkctx, exec);
4704  } else if (!upload) {
4705  ff_vk_exec_wait(&p->vkctx, exec);
4706  if (!host_mapped)
4707  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 0);
4708  }
4709 
4710 end:
4711  for (int i = 0; i < nb_bufs; i++)
4712  av_buffer_unref(&bufs[i]);
4713 
4714  return err;
4715 }
4716 
4717 static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst,
4718  const AVFrame *src)
4719 {
4720  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4721 
4722  switch (src->format) {
4723 #if CONFIG_CUDA
4724  case AV_PIX_FMT_CUDA:
4725 #ifdef _WIN32
4726  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4727  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4728 #else
4729  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4730  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4731 #endif
4732  return vulkan_transfer_data_from_cuda(hwfc, dst, src);
4733 #endif
4734  default:
4735  if (src->hw_frames_ctx)
4736  return AVERROR(ENOSYS);
4737  else
4738  return vulkan_transfer_frame(hwfc, (AVFrame *)src, dst, 1);
4739  }
4740 }
4741 
4742 #if CONFIG_CUDA
4743 static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst,
4744  const AVFrame *src)
4745 {
4746  int err;
4747  CUcontext dummy;
4748  AVVkFrame *dst_f;
4749  AVVkFrameInternal *dst_int;
4750  VulkanFramesPriv *fp = hwfc->hwctx;
4751  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
4752  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
4753  int nb_images;
4754 
4755  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)dst->hw_frames_ctx->data;
4756  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
4757  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
4758  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
4759  CudaFunctions *cu = cu_internal->cuda_dl;
4760  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
4761  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
4762 
4763  dst_f = (AVVkFrame *)src->data[0];
4764  nb_images = ff_vk_count_images(dst_f);
4765 
4766  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
4767  if (err < 0)
4768  return err;
4769 
4770  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
4771  if (err < 0)
4772  return err;
4773 
4774  err = vulkan_export_to_cuda(hwfc, dst->hw_frames_ctx, src);
4775  if (err < 0) {
4776  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4777  return err;
4778  }
4779 
4780  dst_int = dst_f->internal;
4781 
4782  for (int i = 0; i < planes; i++) {
4783  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
4784  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
4785  }
4786 
4787  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
4788  nb_images, cuda_dev->stream));
4789  if (err < 0)
4790  goto fail;
4791 
4792  for (int i = 0; i < planes; i++) {
4793  CUDA_MEMCPY2D cpy = {
4794  .dstMemoryType = CU_MEMORYTYPE_DEVICE,
4795  .dstDevice = (CUdeviceptr)dst->data[i],
4796  .dstPitch = dst->linesize[i],
4797  .dstY = 0,
4798 
4799  .srcMemoryType = CU_MEMORYTYPE_ARRAY,
4800  .srcArray = dst_int->cu_array[i],
4801  };
4802 
4803  int w, h;
4804  get_plane_wh(&w, &h, hwfc->sw_format, hwfc->width, hwfc->height, i);
4805 
4806  cpy.WidthInBytes = w * desc->comp[i].step;
4807  cpy.Height = h;
4808 
4809  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
4810  if (err < 0)
4811  goto fail;
4812  }
4813 
4814  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
4815  nb_images, cuda_dev->stream));
4816  if (err < 0)
4817  goto fail;
4818 
4819  for (int i = 0; i < planes; i++)
4820  dst_f->sem_value[i]++;
4821 
4822  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4823 
4824  av_log(hwfc, AV_LOG_VERBOSE, "Transferred Vulkan image to CUDA!\n");
4825 
4826  return prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
4827 
4828 fail:
4829  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4830  vulkan_free_internal(dst_f);
4831  av_buffer_unref(&dst->buf[0]);
4832  return err;
4833 }
4834 #endif
4835 
4836 static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst,
4837  const AVFrame *src)
4838 {
4839  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4840 
4841  switch (dst->format) {
4842 #if CONFIG_CUDA
4843  case AV_PIX_FMT_CUDA:
4844 #ifdef _WIN32
4845  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4846  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4847 #else
4848  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4849  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4850 #endif
4851  return vulkan_transfer_data_to_cuda(hwfc, dst, src);
4852 #endif
4853  default:
4854  if (dst->hw_frames_ctx)
4855  return AVERROR(ENOSYS);
4856  else
4857  return vulkan_transfer_frame(hwfc, dst, (AVFrame *)src, 0);
4858  }
4859 }
4860 
4861 static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc,
4862  AVHWFramesContext *src_fc, int flags)
4863 {
4864  return vulkan_frames_init(dst_fc);
4865 }
4866 
4867 AVVkFrame *av_vk_frame_alloc(void)
4868 {
4869  int err;
4870  AVVkFrame *f = av_mallocz(sizeof(AVVkFrame));
4871  if (!f)
4872  return NULL;
4873 
4874  f->internal = av_mallocz(sizeof(*f->internal));
4875  if (!f->internal) {
4876  av_free(f);
4877  return NULL;
4878  }
4879 
4880  err = pthread_mutex_init(&f->internal->update_mutex, NULL);
4881  if (err != 0) {
4882  av_free(f->internal);
4883  av_free(f);
4884  return NULL;
4885  }
4886 
4887  return f;
4888 }
4889 
4890 const HWContextType ff_hwcontext_type_vulkan = {
4891  .type = AV_HWDEVICE_TYPE_VULKAN,
4892  .name = "Vulkan",
4893 
4894  .device_hwctx_size = sizeof(VulkanDevicePriv),
4895  .frames_hwctx_size = sizeof(VulkanFramesPriv),
4896 
4897  .device_init = &vulkan_device_init,
4898  .device_uninit = &vulkan_device_uninit,
4899  .device_create = &vulkan_device_create,
4900  .device_derive = &vulkan_device_derive,
4901 
4902  .frames_get_constraints = &vulkan_frames_get_constraints,
4903  .frames_init = vulkan_frames_init,
4904  .frames_get_buffer = vulkan_get_buffer,
4905  .frames_uninit = vulkan_frames_uninit,
4906 
4907  .transfer_get_formats = vulkan_transfer_get_formats,
4908  .transfer_data_to = vulkan_transfer_data_to,
4909  .transfer_data_from = vulkan_transfer_data_from,
4910 
4911  .map_to = vulkan_map_to,
4912  .map_from = vulkan_map_from,
4913  .frames_derive_to = &vulkan_frames_derive_to,
4914 
4915  .pix_fmts = (const enum AVPixelFormat []) {
4916  AV_PIX_FMT_VULKAN,
4917  AV_PIX_FMT_NONE
4918  },
4919 };
flags
const SwsFlags flags[]
Definition: swscale.c:61
vulkan_loader.h
formats
formats
Definition: signature.h:47
AV_PIX_FMT_YUVA422P16
#define AV_PIX_FMT_YUVA422P16
Definition: pixfmt.h:596
load_libvulkan
static int load_libvulkan(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:630
pthread_mutex_t
_fmutex pthread_mutex_t
Definition: os2threads.h:53
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:88
FFHWFramesContext::pool_internal
AVBufferPool * pool_internal
Definition: hwcontext_internal.h:101
vulkan_device_init
static int vulkan_device_init(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1907
AV_PIX_FMT_GBRAP16
#define AV_PIX_FMT_GBRAP16
Definition: pixfmt.h:565
FF_ENABLE_DEPRECATION_WARNINGS
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:73
ff_vk_load_props
int ff_vk_load_props(FFVulkanContext *s)
Loads props/mprops/driver_props.
Definition: vulkan.c:147
AV_DRM_MAX_PLANES
@ AV_DRM_MAX_PLANES
The maximum number of layers/planes in a DRM frame.
Definition: hwcontext_drm.h:39
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:216
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:79
AV_PIX_FMT_CUDA
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition: pixfmt.h:260
VulkanDeviceFeatures::vulkan_1_2
VkPhysicalDeviceVulkan12Features vulkan_1_2
Definition: hwcontext_vulkan.c:79
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
VulkanDevicePriv::libvulkan
void * libvulkan
Definition: hwcontext_vulkan.c:129
name
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option name
Definition: writing_filters.txt:88
VulkanOptExtension::name
const char * name
Definition: hwcontext_vulkan.c:669
FFVkFormatEntry::nb_images
int nb_images
Definition: hwcontext_vulkan.c:401
host_map_frame
static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4380
AVCUDADeviceContextInternal
Definition: hwcontext_cuda_internal.h:31
AV_PIX_FMT_GRAY32
#define AV_PIX_FMT_GRAY32
Definition: pixfmt.h:523
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:206
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
hwcontext_cuda_internal.h
HWMapDescriptor::source
AVFrame * source
A reference to the original source of the mapping.
Definition: hwcontext_internal.h:124
FFVulkanExtensions
uint64_t FFVulkanExtensions
Definition: vulkan_functions.h:29
AVBufferPool
The buffer pool.
Definition: buffer_internal.h:88
SET_OLD_QF
#define SET_OLD_QF(field, nb_field, type)
vulkan_transfer_data_to
static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4717
FF_VK_EXT_EXTERNAL_WIN32_MEMORY
#define FF_VK_EXT_EXTERNAL_WIN32_MEMORY
Definition: vulkan_functions.h:39
FF_VK_EXT_VIDEO_QUEUE
#define FF_VK_EXT_VIDEO_QUEUE
Definition: vulkan_functions.h:58
thread.h
vk_dbg_callback
static VKAPI_ATTR VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT messageType, const VkDebugUtilsMessengerCallbackDataEXT *data, void *priv)
Definition: hwcontext_vulkan.c:768
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3456
CHECK_QUEUE
#define CHECK_QUEUE(type, required, fidx, ctx_qf, qc)
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
pthread_mutex_init
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, AVVulkanDeviceQueueFamily *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:355
FF_VK_EXT_PORTABILITY_SUBSET
#define FF_VK_EXT_PORTABILITY_SUBSET
Definition: vulkan_functions.h:73
vulkan_frames_get_constraints
static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx, const void *hwconfig, AVHWFramesConstraints *constraints)
Definition: hwcontext_vulkan.c:2265
FF_VK_EXT_VIDEO_MAINTENANCE_2
#define FF_VK_EXT_VIDEO_MAINTENANCE_2
Definition: vulkan_functions.h:60
AVVkFrameInternal::update_mutex
pthread_mutex_t update_mutex
Definition: hwcontext_vulkan.c:196
av_unused
#define av_unused
Definition: attributes.h:151
vulkan_frames_derive_to
static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc, AVHWFramesContext *src_fc, int flags)
Definition: hwcontext_vulkan.c:4861
VulkanDeviceFeatures::explicit_mem_layout
VkPhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR explicit_mem_layout
Definition: hwcontext_vulkan.c:84
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:64
PICK_QF
#define PICK_QF(type, vid_op)
FF_VK_EXT_VIDEO_DECODE_H265
#define FF_VK_EXT_VIDEO_DECODE_H265
Definition: vulkan_functions.h:64
mode
Definition: swscale.c:56
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:427
pixdesc.h
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to a vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:69
optional_device_exts
static const VulkanOptExtension optional_device_exts[]
Definition: hwcontext_vulkan.c:680
AVFrame::width
int width
Definition: frame.h:499
AV_PIX_FMT_YUVA420P16
#define AV_PIX_FMT_YUVA420P16
Definition: pixfmt.h:595
AV_PIX_FMT_Y216
#define AV_PIX_FMT_Y216
Definition: pixfmt.h:608
create_frame
static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame, VkImageTiling tiling, VkImageUsageFlagBits usage, VkImageCreateFlags flags, int nb_layers, void *create_pnext)
Definition: hwcontext_vulkan.c:2699
AVDRMFrameDescriptor::nb_layers
int nb_layers
Number of layers in the frame.
Definition: hwcontext_drm.h:145
AV_PIX_FMT_DRM_PRIME
@ AV_PIX_FMT_DRM_PRIME
DRM-managed buffers exposed through PRIME buffer sharing.
Definition: pixfmt.h:351
AV_PIX_FMT_YUVA420P10
#define AV_PIX_FMT_YUVA420P10
Definition: pixfmt.h:590
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:243
ff_vk_find_struct
static const void * ff_vk_find_struct(const void *chain, VkStructureType stype)
Definition: vulkan.h:375
pthread_mutex_lock
static av_always_inline int pthread_mutex_lock(pthread_mutex_t *mutex)
Definition: os2threads.h:119
av_hwframe_map
int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
Map a hardware frame.
Definition: hwcontext.c:793
COPY_VAL
#define COPY_VAL(VAL)
nb_vk_formats_list
static const int nb_vk_formats_list
Definition: hwcontext_vulkan.c:514
data
const char data[16]
Definition: mxf.c:149
AVVulkanDeviceContext::queue_family_encode_index
attribute_deprecated int queue_family_encode_index
Queue family index for video encode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:156
AV_PIX_FMT_RGBA128
#define AV_PIX_FMT_RGBA128
Definition: pixfmt.h:630
AV_PIX_FMT_YUV420P10
#define AV_PIX_FMT_YUV420P10
Definition: pixfmt.h:539
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:74
AV_PIX_FMT_XV30
#define AV_PIX_FMT_XV30
Definition: pixfmt.h:609
ff_vk_flush_buffer
int ff_vk_flush_buffer(FFVulkanContext *s, FFVkBuffer *buf, VkDeviceSize offset, VkDeviceSize mem_size, int flush)
Flush or invalidate a single buffer, with a given size and offset.
Definition: vulkan.c:1161
AVVulkanFramesContext::lock_frame
void(* lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Locks a frame, preventing other threads from changing frame properties.
Definition: hwcontext_vulkan.h:288
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:226
AVVAAPIDeviceContext::display
VADisplay display
The VADisplay handle, to be filled by the user.
Definition: hwcontext_vaapi.h:72
switch_new_props
static void switch_new_props(enum PrepMode pmode, VkImageLayout *new_layout, VkAccessFlags2 *new_access)
Definition: hwcontext_vulkan.c:2523
FF_VULKAN_DEBUG_PRACTICES
@ FF_VULKAN_DEBUG_PRACTICES
Definition: hwcontext_vulkan.c:848
vulkan_map_from
static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:4272
AV_PIX_FMT_BGR24
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:76
AV_PIX_FMT_BGRA
@ AV_PIX_FMT_BGRA
packed BGRA 8:8:8:8, 32bpp, BGRABGRA...
Definition: pixfmt.h:102
FFVkFormatEntry::vkf
VkFormat vkf
Definition: hwcontext_vulkan.c:397
ff_vk_exec_get
FFVkExecContext * ff_vk_exec_get(FFVulkanContext *s, FFVkExecPool *pool)
Retrieve an execution pool.
Definition: vulkan.c:546
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2815
AVDictionary
Definition: dict.c:32
ff_hwframe_map_create
int ff_hwframe_map_create(AVBufferRef *hwframe_ref, AVFrame *dst, const AVFrame *src, void(*unmap)(AVHWFramesContext *ctx, HWMapDescriptor *hwmap), void *priv)
Definition: hwcontext.c:741
FFMAX
#define FFMAX(a, b)
Definition: macros.h:47
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
HWMapDescriptor::priv
void * priv
Hardware-specific private data associated with the mapping.
Definition: hwcontext_internal.h:139
FF_VK_EXT_COOP_MATRIX
#define FF_VK_EXT_COOP_MATRIX
Definition: vulkan_functions.h:44
av_popcount
#define av_popcount
Definition: common.h:154
AVDRMFrameDescriptor
DRM frame descriptor.
Definition: hwcontext_drm.h:133
AVHWFramesConstraints::valid_hw_formats
enum AVPixelFormat * valid_hw_formats
A list of possible values for format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:449
AVERROR_UNKNOWN
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:73
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:220
AVFrame::buf
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:604
AV_PIX_FMT_YUVA422P10
#define AV_PIX_FMT_YUVA422P10
Definition: pixfmt.h:591
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
VulkanDeviceSelection::uuid
uint8_t uuid[VK_UUID_SIZE]
Definition: hwcontext_vulkan.c:1303
ff_vk_exec_add_dep_frame
int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage)
Definition: vulkan.c:778
FF_VULKAN_DEBUG_PRINTF
@ FF_VULKAN_DEBUG_PRINTF
Definition: hwcontext_vulkan.c:846
AV_PIX_FMT_P212
#define AV_PIX_FMT_P212
Definition: pixfmt.h:618
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:448
av_vk_get_optional_device_extensions
const char ** av_vk_get_optional_device_extensions(int *count)
Returns an array of optional Vulkan device extensions that FFmpeg may use if enabled.
Definition: hwcontext_vulkan.c:753
FFVkBuffer::buf
VkBuffer buf
Definition: vulkan.h:126
VulkanDeviceFeatures::host_image_copy
VkPhysicalDeviceHostImageCopyFeaturesEXT host_image_copy
Definition: hwcontext_vulkan.c:83
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:374
alloc_mem
static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req, VkMemoryPropertyFlagBits req_flags, const void *alloc_extension, VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
Definition: hwcontext_vulkan.c:2311
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
VulkanDevicePriv::compute_qf
AVVulkanDeviceQueueFamily * compute_qf
Definition: hwcontext_vulkan.c:132
AVHWFramesConstraints
This struct describes the constraints on hardware frames attached to a given device with a hardware-s...
Definition: hwcontext.h:444
FF_VK_EXT_HOST_IMAGE_COPY
#define FF_VK_EXT_HOST_IMAGE_COPY
Definition: vulkan_functions.h:51
AV_HWDEVICE_TYPE_CUDA
@ AV_HWDEVICE_TYPE_CUDA
Definition: hwcontext.h:30
FF_VK_EXT_EXPECT_ASSUME
#define FF_VK_EXT_EXPECT_ASSUME
Definition: vulkan_functions.h:49
AVDRMDeviceContext::fd
int fd
File descriptor of DRM device.
Definition: hwcontext_drm.h:166
PREP_MODE_DECODING_DPB
@ PREP_MODE_DECODING_DPB
Definition: hwcontext_vulkan.c:2519
FF_VK_EXT_EXTERNAL_FD_SEM
#define FF_VK_EXT_EXTERNAL_FD_SEM
Definition: vulkan_functions.h:35
VulkanDeviceFeatures::device
VkPhysicalDeviceFeatures2 device
Definition: hwcontext_vulkan.c:76
VulkanDeviceFeatures::video_maintenance_1
VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1
Definition: hwcontext_vulkan.c:102
close
static av_cold void close(AVCodecParserContext *s)
Definition: apv_parser.c:197
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3496
ASPECT_3PLANE
#define ASPECT_3PLANE
Definition: hwcontext_vulkan.c:394
FF_VK_EXT_LONG_VECTOR
#define FF_VK_EXT_LONG_VECTOR
Definition: vulkan_functions.h:55
VulkanDevicePriv::hprops
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops
Definition: hwcontext_vulkan.c:138
vulkan_device_derive
static int vulkan_device_derive(AVHWDeviceContext *ctx, AVHWDeviceContext *src_ctx, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:2162
VulkanOptExtension::flag
FFVulkanExtensions flag
Definition: hwcontext_vulkan.c:670
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AV_PIX_FMT_GBRP14
#define AV_PIX_FMT_GBRP14
Definition: pixfmt.h:560
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:307
AV_PIX_FMT_GBRAP
@ AV_PIX_FMT_GBRAP
planar GBRA 4:4:4:4 32bpp
Definition: pixfmt.h:212
VulkanDeviceSelection::drm_minor
uint32_t drm_minor
Definition: hwcontext_vulkan.c:1306
lock_frame
static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2946
fail
#define fail()
Definition: checkasm.h:218
AVDRMLayerDescriptor::nb_planes
int nb_planes
Number of planes in the layer.
Definition: hwcontext_drm.h:106
ff_vk_exec_add_dep_bool_sem
int ff_vk_exec_add_dep_bool_sem(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore *sem, int nb, VkPipelineStageFlagBits2 stage, int wait)
Definition: vulkan.c:712
AV_PIX_FMT_GBRP10
#define AV_PIX_FMT_GBRP10
Definition: pixfmt.h:558
AV_PIX_FMT_YUVA444P16
#define AV_PIX_FMT_YUVA444P16
Definition: pixfmt.h:597
AVVulkanFramesContext::flags
AVVkFrameFlags flags
A combination of AVVkFrameFlags.
Definition: hwcontext_vulkan.h:259
VulkanDevicePriv
Definition: hwcontext_vulkan.c:122
AVVulkanDeviceContext::nb_tx_queues
attribute_deprecated int nb_tx_queues
Definition: hwcontext_vulkan.h:139
AVDRMLayerDescriptor::planes
AVDRMPlaneDescriptor planes[AV_DRM_MAX_PLANES]
Array of planes in this layer.
Definition: hwcontext_drm.h:110
dummy
int dummy
Definition: motion.c:64
AVVkFrame::mem
VkDeviceMemory mem[AV_NUM_DATA_POINTERS]
Memory backing the images.
Definition: hwcontext_vulkan.h:320
switch_layout
static int switch_layout(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2558
device_features_copy_needed
static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
Definition: hwcontext_vulkan.c:288
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:212
ff_vk_frame_barrier
void ff_vk_frame_barrier(FFVulkanContext *s, FFVkExecContext *e, AVFrame *pic, VkImageMemoryBarrier2 *bar, int *nb_bar, VkPipelineStageFlags2 src_stage, VkPipelineStageFlags2 dst_stage, VkAccessFlagBits2 new_access, VkImageLayout new_layout, uint32_t new_qf)
Definition: vulkan.c:2029
av_buffer_pool_init2
AVBufferPool * av_buffer_pool_init2(size_t size, void *opaque, AVBufferRef *(*alloc)(void *opaque, size_t size), void(*pool_free)(void *opaque))
Allocate and initialize a buffer pool with a more complex allocator.
Definition: buffer.c:259
AVHWFramesConstraints::min_width
int min_width
The minimum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:462
lock_queue
static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1895
AVCUDADeviceContextInternal::cuda_device
CUdevice cuda_device
Definition: hwcontext_cuda_internal.h:34
VulkanDevicePriv::limit_queues
int limit_queues
Definition: hwcontext_vulkan.c:169
VulkanDevicePriv::vkctx
FFVulkanContext vkctx
Definition: hwcontext_vulkan.c:131
AV_PIX_FMT_XV48
#define AV_PIX_FMT_XV48
Definition: pixfmt.h:611
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
FF_VK_EXT_VIDEO_ENCODE_H265
#define FF_VK_EXT_VIDEO_ENCODE_H265
Definition: vulkan_functions.h:70
ff_vk_host_map_buffer
int ff_vk_host_map_buffer(FFVulkanContext *s, AVBufferRef **dst, uint8_t *src_data, const AVBufferRef *src_buf, VkBufferUsageFlags usage)
Maps a system RAM buffer into a Vulkan buffer.
Definition: vulkan.c:1381
ff_vk_ret2str
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:40
AV_PIX_FMT_GRAY16
#define AV_PIX_FMT_GRAY16
Definition: pixfmt.h:522
AV_HWFRAME_MAP_WRITE
@ AV_HWFRAME_MAP_WRITE
The mapping must be writeable.
Definition: hwcontext.h:519
VulkanDeviceFeatures::vulkan_1_3
VkPhysicalDeviceVulkan13Features vulkan_1_3
Definition: hwcontext_vulkan.c:80
FF_VK_EXT_EXTERNAL_WIN32_SEM
#define FF_VK_EXT_EXTERNAL_WIN32_SEM
Definition: vulkan_functions.h:40
AVVulkanDeviceContext::queue_family_decode_index
attribute_deprecated int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:166
VulkanDevicePriv::props
VkPhysicalDeviceProperties2 props
Definition: hwcontext_vulkan.c:136
ff_vk_aspect_flag
VkImageAspectFlags ff_vk_aspect_flag(AVFrame *f, int p)
Get the aspect flag for a plane from an image.
Definition: vulkan.c:1522
VulkanFramesPriv::drm_format_modifier_properties
VkDrmFormatModifierPropertiesEXT drm_format_modifier_properties[5]
Definition: hwcontext_vulkan.c:192
vk_find_format_entry
static const struct FFVkFormatEntry * vk_find_format_entry(enum AVPixelFormat p)
Definition: hwcontext_vulkan.c:524
AVDRMPlaneDescriptor::offset
ptrdiff_t offset
Offset within that object of this plane.
Definition: hwcontext_drm.h:83
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:63
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:52
AV_PIX_FMT_YUV444P10
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:542
AV_PIX_FMT_Y210
#define AV_PIX_FMT_Y210
Definition: pixfmt.h:606
AVVulkanDeviceQueueFamily::num
int num
Definition: hwcontext_vulkan.h:37
HWContextType::type
enum AVHWDeviceType type
Definition: hwcontext_internal.h:30
ffhwframesctx
static FFHWFramesContext * ffhwframesctx(AVHWFramesContext *ctx)
Definition: hwcontext_internal.h:115
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:210
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
ff_vk_link_struct
static void ff_vk_link_struct(void *chain, const void *in)
Definition: vulkan.h:388
check_layers
static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:1018
AV_PIX_FMT_YUV422P16
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:551
AVVulkanDeviceContext::nb_encode_queues
attribute_deprecated int nb_encode_queues
Definition: hwcontext_vulkan.h:158
AVHWFramesContext::height
int height
Definition: hwcontext.h:220
AVHWFramesConstraints::valid_sw_formats
enum AVPixelFormat * valid_sw_formats
A list of possible values for sw_format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:456
vulkan_map_to
static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:4063
av_dict_get
AVDictionaryEntry * av_dict_get(const AVDictionary *m, const char *key, const AVDictionaryEntry *prev, int flags)
Get a dictionary entry with matching key.
Definition: dict.c:60
av_buffer_pool_get
AVBufferRef * av_buffer_pool_get(AVBufferPool *pool)
Allocate a new AVBuffer, reusing an old buffer from the pool when available.
Definition: buffer.c:390
AV_PIX_FMT_GBRAP10
#define AV_PIX_FMT_GBRAP10
Definition: pixfmt.h:562
AVHWFramesContext::pool
AVBufferPool * pool
A pool from which the frames are allocated by av_hwframe_get_buffer().
Definition: hwcontext.h:181
VulkanDeviceSelection::pci_device
uint32_t pci_device
Definition: hwcontext_vulkan.c:1309
s
#define s(width, name)
Definition: cbs_vp9.c:198
AV_PIX_FMT_GBRAP14
#define AV_PIX_FMT_GBRAP14
Definition: pixfmt.h:564
AV_PIX_FMT_GBRAP12
#define AV_PIX_FMT_GBRAP12
Definition: pixfmt.h:563
AV_PIX_FMT_YUVA420P
@ AV_PIX_FMT_YUVA420P
planar YUV 4:2:0, 20bpp, (1 Cr & Cb sample per 2x2 Y & A samples)
Definition: pixfmt.h:108
AV_PIX_FMT_RGB96
#define AV_PIX_FMT_RGB96
Definition: pixfmt.h:629
pthread_mutex_unlock
static av_always_inline int pthread_mutex_unlock(pthread_mutex_t *mutex)
Definition: os2threads.h:126
AV_PIX_FMT_YUV444P16
#define AV_PIX_FMT_YUV444P16
Definition: pixfmt.h:552
offsets
static const int offsets[]
Definition: hevc_pel.c:34
AV_CEIL_RSHIFT
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:60
ff_vk_load_functions
static int ff_vk_load_functions(AVHWDeviceContext *ctx, FFVulkanFunctions *vk, uint64_t extensions_mask, int has_inst, int has_dev)
Function loader.
Definition: vulkan_loader.h:128
VulkanDevicePriv::ext_sem_props_opaque
VkExternalSemaphoreProperties ext_sem_props_opaque
Definition: hwcontext_vulkan.c:142
prepare_frame
static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2666
vulkan_transfer_frame
static int vulkan_transfer_frame(AVHWFramesContext *hwfc, AVFrame *swf, AVFrame *hwf, int upload)
Definition: hwcontext_vulkan.c:4556
FF_VK_EXT_DEVICE_DRM
#define FF_VK_EXT_DEVICE_DRM
Definition: vulkan_functions.h:42
ff_vk_exec_wait
void ff_vk_exec_wait(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:551
av_strtok
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
Definition: avstring.c:179
ASPECT_2PLANE
#define ASPECT_2PLANE
Definition: hwcontext_vulkan.c:393
vulkan_device_uninit
static void vulkan_device_uninit(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1775
fc
#define fc(width, name, range_min, range_max)
Definition: cbs_av1.c:494
AVVulkanFramesContext::unlock_frame
void(* unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Similar to lock_frame(), unlocks a frame.
Definition: hwcontext_vulkan.h:293
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:42
AVVulkanFramesContext::img_flags
VkImageCreateFlags img_flags
Flags to set during image creation.
Definition: hwcontext_vulkan.h:265
AV_PIX_FMT_GBRAP32
#define AV_PIX_FMT_GBRAP32
Definition: pixfmt.h:566
AV_PIX_FMT_YUVA444P12
#define AV_PIX_FMT_YUVA444P12
Definition: pixfmt.h:594
vulkan_frames_uninit
static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2956
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:231
AV_PIX_FMT_YUV420P16
#define AV_PIX_FMT_YUV420P16
Definition: pixfmt.h:550
ctx
static AVFormatContext * ctx
Definition: movenc.c:49
AVDRMObjectDescriptor::fd
int fd
DRM PRIME fd for the object.
Definition: hwcontext_drm.h:52
AV_PIX_FMT_GRAY14
#define AV_PIX_FMT_GRAY14
Definition: pixfmt.h:521
ff_vk_exec_add_dep_buf
int ff_vk_exec_add_dep_buf(FFVulkanContext *s, FFVkExecContext *e, AVBufferRef **deps, int nb_deps, int ref)
Execution dependency management.
Definition: vulkan.c:618
VulkanDeviceSelection::index
int index
Definition: hwcontext_vulkan.c:1311
AV_PIX_FMT_RGBF32
#define AV_PIX_FMT_RGBF32
Definition: pixfmt.h:626
VulkanFramesPriv::p
AVVulkanFramesContext p
The public AVVulkanFramesContext.
Definition: hwcontext_vulkan.c:176
vulkan_transfer_get_formats
static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats)
Definition: hwcontext_vulkan.c:3212
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:73
VulkanDevicePriv::avoid_host_import
int avoid_host_import
Definition: hwcontext_vulkan.c:166
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:299
av_mallocz
#define av_mallocz(s)
Definition: tableprint_vlc.h:31
FFVulkanDebugMode
FFVulkanDebugMode
Definition: hwcontext_vulkan.c:841
AV_PIX_FMT_GRAYF32
#define AV_PIX_FMT_GRAYF32
Definition: pixfmt.h:582
LIBAVUTIL_VERSION_MINOR
#define LIBAVUTIL_VERSION_MINOR
Definition: version.h:82
tmp
static uint8_t tmp[40]
Definition: aes_ctr.c:52
AV_PIX_FMT_RGBA
@ AV_PIX_FMT_RGBA
packed RGBA 8:8:8:8, 32bpp, RGBARGBA...
Definition: pixfmt.h:100
FFVkFormatEntry::nb_images_fallback
int nb_images_fallback
Definition: hwcontext_vulkan.c:402
vulkan_frame_free_cb
static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
Definition: hwcontext_vulkan.c:2439
AV_PIX_FMT_GRAY10
#define AV_PIX_FMT_GRAY10
Definition: pixfmt.h:519
if
if(ret)
Definition: filter_design.txt:179
av_vkfmt_from_pixfmt
const VkFormat * av_vkfmt_from_pixfmt(enum AVPixelFormat p)
Returns the optimal per-plane Vulkan format for a given sw_format, one for each plane.
Definition: hwcontext_vulkan.c:516
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:59
VulkanDevicePriv::qf_mutex
pthread_mutex_t ** qf_mutex
Definition: hwcontext_vulkan.c:148
AV_PIX_FMT_GBRP16
#define AV_PIX_FMT_GBRP16
Definition: pixfmt.h:561
opts
static AVDictionary * opts
Definition: movenc.c:51
PREP_MODE_WRITE
@ PREP_MODE_WRITE
Definition: hwcontext_vulkan.c:2515
AV_PIX_FMT_RGBA64
#define AV_PIX_FMT_RGBA64
Definition: pixfmt.h:529
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:213
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
format
New swscale design to change SwsGraph is what coordinates multiple passes These can include cascaded scaling error diffusion and so on Or we could have separate passes for the vertical and horizontal scaling In between each SwsPass lies a fully allocated image buffer Graph passes may have different levels of e g we can have a single threaded error diffusion pass following a multi threaded scaling pass SwsGraph is internally recreated whenever the image format
Definition: swscale-v2.txt:14
FF_VK_EXT_DRM_MODIFIER_FLAGS
#define FF_VK_EXT_DRM_MODIFIER_FLAGS
Definition: vulkan_functions.h:33
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:117
FFVkFormatEntry
Definition: hwcontext_vulkan.c:396
FF_VK_EXT_SHADER_OBJECT
#define FF_VK_EXT_SHADER_OBJECT
Definition: vulkan_functions.h:46
AV_PIX_FMT_YUYV422
@ AV_PIX_FMT_YUYV422
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:74
AVVkFrameInternal
Definition: hwcontext_vulkan.c:195
FF_VK_EXT_VIDEO_DECODE_VP9
#define FF_VK_EXT_VIDEO_DECODE_VP9
Definition: vulkan_functions.h:65
FF_VK_EXT_SUBGROUP_ROTATE
#define FF_VK_EXT_SUBGROUP_ROTATE
Definition: vulkan_functions.h:50
FF_VK_EXT_VIDEO_ENCODE_QUEUE
#define FF_VK_EXT_VIDEO_ENCODE_QUEUE
Definition: vulkan_functions.h:68
VulkanDevicePriv::debug_ctx
VkDebugUtilsMessengerEXT debug_ctx
Definition: hwcontext_vulkan.c:154
FF_VULKAN_DEBUG_NONE
@ FF_VULKAN_DEBUG_NONE
Definition: hwcontext_vulkan.c:842
AVVulkanFramesContext::alloc_pnext
void * alloc_pnext[AV_NUM_DATA_POINTERS]
Extension data for memory allocation.
Definition: hwcontext_vulkan.h:252
setup_queue_families
static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
Definition: hwcontext_vulkan.c:1555
AVVulkanDeviceContext::unlock_queue
void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:182
LIBAVUTIL_VERSION_MAJOR
#define LIBAVUTIL_VERSION_MAJOR
Definition: version.h:81
AVVulkanDeviceContext::nb_decode_queues
attribute_deprecated int nb_decode_queues
Definition: hwcontext_vulkan.h:168
AV_PIX_FMT_P410
#define AV_PIX_FMT_P410
Definition: pixfmt.h:617
av_buffer_pool_uninit
void av_buffer_pool_uninit(AVBufferPool **ppool)
Mark the pool as being available for freeing.
Definition: buffer.c:328
AVVulkanDeviceContext::nb_qf
int nb_qf
Definition: hwcontext_vulkan.h:193
ff_hwcontext_type_vulkan
const HWContextType ff_hwcontext_type_vulkan
Definition: hwcontext_vulkan.c:4890
hwcontext_vulkan.h
AVVulkanDeviceContext::queue_family_tx_index
attribute_deprecated int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:137
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:103
vk_formats_list
static const struct FFVkFormatEntry vk_formats_list[]
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:273
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:232
AV_PIX_FMT_BGR0
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:265
AV_PIX_FMT_YUV422P10
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:540
alloc_bind_mem
static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f, void *alloc_pnext, size_t alloc_pnext_stride)
Definition: hwcontext_vulkan.c:2444
FFVkBuffer::mapped_mem
uint8_t * mapped_mem
Definition: vulkan.h:134
AVVulkanDeviceContext::qf
AVVulkanDeviceQueueFamily qf[64]
Queue families used.
Definition: hwcontext_vulkan.h:192
AV_PIX_FMT_GRAY8
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
Definition: pixfmt.h:81
FFVulkanContext
Definition: vulkan.h:312
exp
int8_t exp
Definition: eval.c:73
FF_VK_EXT_REPLICATED_COMPOSITES
#define FF_VK_EXT_REPLICATED_COMPOSITES
Definition: vulkan_functions.h:54
VulkanFramesPriv
Definition: hwcontext_vulkan.c:172
vulkan_frame_free
static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
Definition: hwcontext_vulkan.c:2405
pick_video_queue_family
static int pick_video_queue_family(VkQueueFamilyProperties2 *qf, VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf, VkVideoCodecOperationFlagsKHR flags)
Definition: hwcontext_vulkan.c:1526
index
int index
Definition: gxfenc.c:90
av_buffer_create
AVBufferRef * av_buffer_create(uint8_t *data, size_t size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:55
vkfmt_from_pixfmt2
static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p, VkImageTiling tiling, VkFormat fmts[AV_NUM_DATA_POINTERS], int *nb_images, VkImageAspectFlags *aspect, VkImageUsageFlags *supported_usage, int disable_multiplane, int need_storage)
Definition: hwcontext_vulkan.c:532
VulkanDeviceSelection
Definition: hwcontext_vulkan.c:1302
source
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a source
Definition: filter_design.txt:256
VulkanDevicePriv::nb_tot_qfs
uint32_t nb_tot_qfs
Definition: hwcontext_vulkan.c:149
VulkanDeviceFeatures
Definition: hwcontext_vulkan.c:75
AVDRMFrameDescriptor::layers
AVDRMLayerDescriptor layers[AV_DRM_MAX_PLANES]
Array of layers in the frame.
Definition: hwcontext_drm.h:149
usage
const char * usage
Definition: floatimg_cmp.c:62
PREP_MODE_DECODING_DST
@ PREP_MODE_DECODING_DST
Definition: hwcontext_vulkan.c:2518
AVVkFrame::size
size_t size[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:321
vulkan_pool_alloc
static AVBufferRef * vulkan_pool_alloc(void *opaque, size_t size)
Definition: hwcontext_vulkan.c:2871
VulkanDevicePriv::dprops
VkPhysicalDeviceDriverProperties dprops
Definition: hwcontext_vulkan.c:139
AV_PIX_FMT_X2BGR10
#define AV_PIX_FMT_X2BGR10
Definition: pixfmt.h:614
PrepMode
PrepMode
Definition: hwcontext_vulkan.c:2513
FF_VK_EXT_VIDEO_MAINTENANCE_1
#define FF_VK_EXT_VIDEO_MAINTENANCE_1
Definition: vulkan_functions.h:59
VulkanDeviceSelection::has_drm
uint32_t has_drm
Definition: hwcontext_vulkan.c:1307
f
f
Definition: af_crystalizer.c:122
AVCUDADeviceContext::internal
AVCUDADeviceContextInternal * internal
Definition: hwcontext_cuda.h:45
AV_PIX_FMT_RGB24
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:75
VulkanDeviceFeatures::vulkan_1_1
VkPhysicalDeviceVulkan11Features vulkan_1_1
Definition: hwcontext_vulkan.c:78
sem_wait
#define sem_wait(psem)
Definition: semaphore.h:27
AV_PIX_FMT_P012
#define AV_PIX_FMT_P012
Definition: pixfmt.h:603
AV_PIX_FMT_FLAG_RGB
#define AV_PIX_FMT_FLAG_RGB
The pixel format contains RGB-like data (as opposed to YUV/grayscale).
Definition: pixdesc.h:136
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition: dsp.h:87
AVVkFrame
Definition: hwcontext_vulkan.h:302
av_vk_frame_alloc
AVVkFrame * av_vk_frame_alloc(void)
Allocates a single AVVkFrame and initializes everything as 0.
Definition: hwcontext_vulkan.c:4867
FF_VULKAN_DEBUG_VALIDATE
@ FF_VULKAN_DEBUG_VALIDATE
Definition: hwcontext_vulkan.c:844
vulkan.h
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:122
FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
#define FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
Definition: vulkan_functions.h:32
vulkan_device_free
static void vulkan_device_free(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1752
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:424
FF_VK_EXT_NO_FLAG
#define FF_VK_EXT_NO_FLAG
Definition: vulkan_functions.h:74
AV_PIX_FMT_GBRPF32
#define AV_PIX_FMT_GBRPF32
Definition: pixfmt.h:578
AV_PIX_FMT_YUV422P12
#define AV_PIX_FMT_YUV422P12
Definition: pixfmt.h:544
VulkanDeviceFeatures::cooperative_matrix
VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix
Definition: hwcontext_vulkan.c:114
AV_PIX_FMT_RGB48
#define AV_PIX_FMT_RGB48
Definition: pixfmt.h:525
size
int size
Definition: twinvq_data.h:10344
ff_vk_exec_add_dep_sw_frame
int ff_vk_exec_add_dep_sw_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f)
Definition: vulkan.c:645
vulkan_transfer_data_from
static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4836
AV_NUM_DATA_POINTERS
#define AV_NUM_DATA_POINTERS
Definition: frame.h:428
FF_VK_EXT_PUSH_DESCRIPTOR
#define FF_VK_EXT_PUSH_DESCRIPTOR
Definition: vulkan_functions.h:47
VulkanDevicePriv::use_linear_images
int use_linear_images
Definition: hwcontext_vulkan.c:157
AV_PIX_FMT_YUV444P12
#define AV_PIX_FMT_YUV444P12
Definition: pixfmt.h:546
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:514
AV_PIX_FMT_NV16
@ AV_PIX_FMT_NV16
interleaved chroma YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:198
VulkanDeviceSelection::vendor_id
uint32_t vendor_id
Definition: hwcontext_vulkan.c:1310
PREP_MODE_GENERAL
@ PREP_MODE_GENERAL
Definition: hwcontext_vulkan.c:2514
AV_PIX_FMT_Y212
#define AV_PIX_FMT_Y212
Definition: pixfmt.h:607
AVVulkanDeviceContext::queue_family_index
attribute_deprecated int queue_family_index
Queue family index for graphics operations, and the number of queues enabled for it.
Definition: hwcontext_vulkan.h:128
AVDRMObjectDescriptor::size
size_t size
Total size of the object.
Definition: hwcontext_drm.h:58
AV_PIX_FMT_YUVA444P
@ AV_PIX_FMT_YUVA444P
planar YUV 4:4:4 32bpp, (1 Cr & Cb sample per 1x1 Y & A samples)
Definition: pixfmt.h:174
VulkanFramesPriv::upload_exec
FFVkExecPool upload_exec
Definition: hwcontext_vulkan.c:182
AVVulkanDeviceQueueFamily::idx
int idx
Definition: hwcontext_vulkan.h:35
AV_PIX_FMT_YUVA444P10
#define AV_PIX_FMT_YUVA444P10
Definition: pixfmt.h:592
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
AVHWFramesConstraints::max_width
int max_width
The maximum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:469
FFVkExecContext
Definition: vulkan.h:145
VulkanOptExtension
Definition: hwcontext_vulkan.c:668
AV_PIX_FMT_RGB0
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:263
AV_PIX_FMT_P216
#define AV_PIX_FMT_P216
Definition: pixfmt.h:620
CHECK_CU
#define CHECK_CU(x)
Definition: cuviddec.c:117
AV_PIX_FMT_P210
#define AV_PIX_FMT_P210
Definition: pixfmt.h:616
AV_PIX_FMT_VAAPI
@ AV_PIX_FMT_VAAPI
Hardware acceleration through VA-API, data[3] contains a VASurfaceID.
Definition: pixfmt.h:126
VulkanDeviceFeatures::subgroup_rotate
VkPhysicalDeviceShaderSubgroupRotateFeaturesKHR subgroup_rotate
Definition: hwcontext_vulkan.c:82
FF_VK_EXT_VIDEO_DECODE_QUEUE
#define FF_VK_EXT_VIDEO_DECODE_QUEUE
Definition: vulkan_functions.h:62
AVVulkanDeviceContext::lock_queue
void(* lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Locks a queue, preventing other threads from submitting any command buffers to this queue.
Definition: hwcontext_vulkan.h:177
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:221
vulkan_free_internal
static void vulkan_free_internal(AVVkFrame *f)
Definition: hwcontext_vulkan.c:2369
VulkanDeviceFeatures::timeline_semaphore
VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore
Definition: hwcontext_vulkan.c:81
AV_HWDEVICE_TYPE_VAAPI
@ AV_HWDEVICE_TYPE_VAAPI
Definition: hwcontext.h:31
pthread_mutex_destroy
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
FF_VK_EXT_EXTERNAL_HOST_MEMORY
#define FF_VK_EXT_EXTERNAL_HOST_MEMORY
Definition: vulkan_functions.h:36
FF_VK_EXT_EXPLICIT_MEM_LAYOUT
#define FF_VK_EXT_EXPLICIT_MEM_LAYOUT
Definition: vulkan_functions.h:53
AV_PIX_FMT_UYVA
@ AV_PIX_FMT_UYVA
packed UYVA 4:4:4:4, 32bpp (1 Cr & Cb sample per 1x1 Y & A samples), UYVAUYVA...
Definition: pixfmt.h:444
AVDRMFrameDescriptor::objects
AVDRMObjectDescriptor objects[AV_DRM_MAX_PLANES]
Array of objects making up the frame.
Definition: hwcontext_drm.h:141
AVCUDADeviceContextInternal::cuda_dl
CudaFunctions * cuda_dl
Definition: hwcontext_cuda_internal.h:32
av_assert2
#define av_assert2(cond)
assert() equivalent, that does lie in speed critical code.
Definition: avassert.h:68
ff_vk_exec_start
int ff_vk_exec_start(FFVulkanContext *s, FFVkExecContext *e)
Start/submit/wait an execution.
Definition: vulkan.c:558
FF_VK_EXT_RELAXED_EXTENDED_INSTR
#define FF_VK_EXT_RELAXED_EXTENDED_INSTR
Definition: vulkan_functions.h:48
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
FF_VK_EXT_VIDEO_DECODE_H264
#define FF_VK_EXT_VIDEO_DECODE_H264
Definition: vulkan_functions.h:63
VulkanFramesPriv::compute_exec
FFVkExecPool compute_exec
Definition: hwcontext_vulkan.c:179
AVVulkanDeviceContext::queue_family_comp_index
attribute_deprecated int queue_family_comp_index
Queue family index for compute operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:146
AVDRMObjectDescriptor::format_modifier
uint64_t format_modifier
Format modifier applied to the object (DRM_FORMAT_MOD_*).
Definition: hwcontext_drm.h:65
VkFormat
enum VkFormat VkFormat
Definition: hwcontext_stub.c:25
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:32
AV_PIX_FMT_GBRP12
#define AV_PIX_FMT_GBRP12
Definition: pixfmt.h:559
weights
static const int weights[]
Definition: hevc_pel.c:32
AV_PIX_FMT_NV24
@ AV_PIX_FMT_NV24
planar YUV 4:4:4, 24bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:371
plane_info
Definition: vf_edgedetect.c:53
VulkanDevicePriv::nb_img_qfs
uint32_t nb_img_qfs
Definition: hwcontext_vulkan.c:151
vulkan_frames_init
static int vulkan_frames_init(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2974
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
AV_PIX_FMT_X2RGB10
#define AV_PIX_FMT_X2RGB10
Definition: pixfmt.h:613
VulkanDeviceSelection::has_uuid
int has_uuid
Definition: hwcontext_vulkan.c:1304
hwcontext_drm.h
AVDRMPlaneDescriptor::object_index
int object_index
Index of the object containing this plane in the objects array of the enclosing frame descriptor.
Definition: hwcontext_drm.h:79
AV_PIX_FMT_BGR565
#define AV_PIX_FMT_BGR565
Definition: pixfmt.h:531
ff_hwframe_map_replace
int ff_hwframe_map_replace(AVFrame *dst, const AVFrame *src)
Replace the current hwmap of dst with the one from src, used for indirect mappings like VAAPI->(DRM)-...
Definition: hwcontext.c:948
VulkanDevicePriv::img_qfs
uint32_t img_qfs[64]
Definition: hwcontext_vulkan.c:150
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:264
mod
static int mod(int a, int b)
Modulo operation with only positive remainders.
Definition: vf_v360.c:755
AV_PIX_FMT_P016
#define AV_PIX_FMT_P016
Definition: pixfmt.h:604
AV_PIX_FMT_RGB565
#define AV_PIX_FMT_RGB565
Definition: pixfmt.h:526
AVHWFrameTransferDirection
AVHWFrameTransferDirection
Definition: hwcontext.h:406
AVVkFrame::sem
VkSemaphore sem[AV_NUM_DATA_POINTERS]
Synchronization timeline semaphores, one for each VkImage.
Definition: hwcontext_vulkan.h:340
create_instance
static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:1165
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:118
FF_VK_EXT_EXTERNAL_FD_MEMORY
#define FF_VK_EXT_EXTERNAL_FD_MEMORY
Definition: vulkan_functions.h:34
AVCUDADeviceContext
This struct is allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_cuda.h:42
hwcontext_vaapi.h
AVDRMLayerDescriptor::format
uint32_t format
Format of the layer (DRM_FORMAT_*).
Definition: hwcontext_drm.h:100
VulkanDevicePriv::transfer_qf
AVVulkanDeviceQueueFamily * transfer_qf
Definition: hwcontext_vulkan.c:133
ret
ret
Definition: filter_design.txt:187
pixfmt
enum AVPixelFormat pixfmt
Definition: kmsgrab.c:367
AVHWDeviceContext::type
enum AVHWDeviceType type
This field identifies the underlying API used for hardware access.
Definition: hwcontext.h:75
AV_PIX_FMT_NV12
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:96
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:265
AVHWFramesContext::device_ctx
AVHWDeviceContext * device_ctx
The parent AVHWDeviceContext.
Definition: hwcontext.h:137
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:316
cuda_check.h
AVHWFramesContext::hwctx
void * hwctx
The format-specific data, allocated and freed automatically along with this context.
Definition: hwcontext.h:153
VulkanFramesPriv::tmp
AVBufferPool * tmp
Definition: hwcontext_vulkan.c:186
vulkan_get_buffer
static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
Definition: hwcontext_vulkan.c:3198
pick_queue_family
static int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf, VkQueueFlagBits flags)
Definition: hwcontext_vulkan.c:1497
FFVkExecPool
Definition: vulkan.h:290
vulkan_transfer_host
static int vulkan_transfer_host(AVHWFramesContext *hwfc, AVFrame *hwf, AVFrame *swf, int upload)
Definition: hwcontext_vulkan.c:4437
unlock_queue
static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1901
AVHWFramesConstraints::max_height
int max_height
Definition: hwcontext.h:470
AVVkFrame::internal
struct AVVkFrameInternal * internal
Internal data.
Definition: hwcontext_vulkan.h:353
ff_vk_qf_find
AVVulkanDeviceQueueFamily * ff_vk_qf_find(FFVulkanContext *s, VkQueueFlagBits dev_family, VkVideoCodecOperationFlagBitsKHR vid_ops)
Chooses an appropriate QF.
Definition: vulkan.c:286
AV_PIX_FMT_YUV420P12
#define AV_PIX_FMT_YUV420P12
Definition: pixfmt.h:543
AV_PIX_FMT_UYVY422
@ AV_PIX_FMT_UYVY422
packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1
Definition: pixfmt.h:88
check_extensions
static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode debug_mode)
Definition: hwcontext_vulkan.c:853
FFVkExecContext::buf
VkCommandBuffer buf
Definition: vulkan.h:156
vulkan_device_create
static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:2146
AVFrame::height
int height
Definition: frame.h:499
PREP_MODE_ENCODING_DPB
@ PREP_MODE_ENCODING_DPB
Definition: hwcontext_vulkan.c:2520
FFVkFormatEntry::pixfmt
enum AVPixelFormat pixfmt
Definition: hwcontext_vulkan.c:398
AVHWFramesConstraints::min_height
int min_height
Definition: hwcontext.h:463
RELEASE_PROPS
#define RELEASE_PROPS(props, count)
Definition: hwcontext_vulkan.c:814
mode
mode
Definition: ebur128.h:83
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
AV_PIX_FMT_YUVA422P12
#define AV_PIX_FMT_YUVA422P12
Definition: pixfmt.h:593
AV_PIX_FMT_GBRAPF32
#define AV_PIX_FMT_GBRAPF32
Definition: pixfmt.h:579
VulkanDevicePriv::feats
VulkanDeviceFeatures feats
Definition: hwcontext_vulkan.c:145
switch_layout_host
static int switch_layout_host(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2621
LIBAVUTIL_VERSION_MICRO
#define LIBAVUTIL_VERSION_MICRO
Definition: version.h:83
find_device
static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
Definition: hwcontext_vulkan.c:1326
AVVulkanDeviceContext::nb_graphics_queues
attribute_deprecated int nb_graphics_queues
Definition: hwcontext_vulkan.h:130
FF_VK_STRUCT_EXT
#define FF_VK_STRUCT_EXT(CTX, BASE, STRUCT_P, EXT_FLAG, TYPE)
Definition: vulkan.h:397
optional_instance_exts
static const VulkanOptExtension optional_instance_exts[]
Definition: hwcontext_vulkan.c:673
AV_PIX_FMT_FLAG_PLANAR
#define AV_PIX_FMT_FLAG_PLANAR
At least one pixel component is not in the first data plane.
Definition: pixdesc.h:132
ff_vk_map_feats_to_usage
VkImageUsageFlags ff_vk_map_feats_to_usage(VkFormatFeatureFlagBits2 feats)
Map between usage and features.
FF_VK_EXT_ATOMIC_FLOAT
#define FF_VK_EXT_ATOMIC_FLOAT
Definition: vulkan_functions.h:43
FFVkFormatEntry::fallback
const VkFormat fallback[5]
Definition: hwcontext_vulkan.c:403
planes
static const struct @554 planes[]
Windows::Graphics::DirectX::Direct3D11::p
IDirect3DDxgiInterfaceAccess _COM_Outptr_ void ** p
Definition: vsrc_gfxcapture_winrt.hpp:53
PREP_MODE_EXTERNAL_IMPORT
@ PREP_MODE_EXTERNAL_IMPORT
Definition: hwcontext_vulkan.c:2517
AV_PIX_FMT_RGBAF32
#define AV_PIX_FMT_RGBAF32
Definition: pixfmt.h:627
FF_VK_EXT_VIDEO_DECODE_AV1
#define FF_VK_EXT_VIDEO_DECODE_AV1
Definition: vulkan_functions.h:66
AV_PIX_FMT_YUV444P
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:78
AVVulkanFramesContext::tiling
VkImageTiling tiling
Controls the tiling of allocated frames.
Definition: hwcontext_vulkan.h:221
VulkanDeviceSelection::drm_major
uint32_t drm_major
Definition: hwcontext_vulkan.c:1305
get_plane_buf
static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4341
AV_PIX_FMT_P010
#define AV_PIX_FMT_P010
Definition: pixfmt.h:602
unlock_frame
static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2951
FF_DISABLE_DEPRECATION_WARNINGS
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:72
AVVkFrame::sem_value
uint64_t sem_value[AV_NUM_DATA_POINTERS]
Up to date semaphore value at which each image becomes accessible.
Definition: hwcontext_vulkan.h:348
AV_PIX_FMT_GBRP
@ AV_PIX_FMT_GBRP
planar GBR 4:4:4 24bpp
Definition: pixfmt.h:165
av_strdup
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:272
desc
const char * desc
Definition: libsvtav1.c:78
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:116
AVVulkanFramesContext::nb_layers
int nb_layers
Number of layers each image will have.
Definition: hwcontext_vulkan.h:278
AV_PIX_FMT_YUV422P
@ AV_PIX_FMT_YUV422P
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:77
AV_HWFRAME_MAP_READ
@ AV_HWFRAME_MAP_READ
The mapping must be readable.
Definition: hwcontext.h:515
VulkanFramesPriv::download_exec
FFVkExecPool download_exec
Definition: hwcontext_vulkan.c:183
mem.h
AVVkFrame::layout
VkImageLayout layout[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:332
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
w
uint8_t w
Definition: llvidencdsp.c:39
hwcontext_internal.h
FF_VK_EXT_VIDEO_ENCODE_H264
#define FF_VK_EXT_VIDEO_ENCODE_H264
Definition: vulkan_functions.h:69
ADD_QUEUE
#define ADD_QUEUE(ctx_qf, qc, flag)
AVVulkanDeviceContext::nb_enabled_inst_extensions
int nb_enabled_inst_extensions
Definition: hwcontext_vulkan.h:104
av_free
#define av_free(p)
Definition: tableprint_vlc.h:34
AVDictionaryEntry
Definition: dict.h:90
get_plane_wh
static void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format, int frame_w, int frame_h, int plane)
Definition: hwcontext_vulkan.c:2682
ff_vk_count_images
static int ff_vk_count_images(AVVkFrame *f)
Definition: vulkan.h:366
ff_vk_exec_discard_deps
void ff_vk_exec_discard_deps(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:590
PREP_MODE_EXTERNAL_EXPORT
@ PREP_MODE_EXTERNAL_EXPORT
Definition: hwcontext_vulkan.c:2516
FFALIGN
#define FFALIGN(x, a)
Definition: macros.h:78
AV_PIX_FMT_P416
#define AV_PIX_FMT_P416
Definition: pixfmt.h:621
VulkanDevicePriv::mprops
VkPhysicalDeviceMemoryProperties mprops
Definition: hwcontext_vulkan.c:137
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
FF_VULKAN_DEBUG_NB
@ FF_VULKAN_DEBUG_NB
Definition: hwcontext_vulkan.c:850
FFVkBuffer
Definition: vulkan.h:125
vk_dev_type
static const char * vk_dev_type(enum VkPhysicalDeviceType type)
Definition: hwcontext_vulkan.c:1314
VulkanDevicePriv::disable_multiplane
int disable_multiplane
Definition: hwcontext_vulkan.c:163
imgutils.h
ff_vk_exec_submit
int ff_vk_exec_submit(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:903
AV_PIX_FMT_XV36
#define AV_PIX_FMT_XV36
Definition: pixfmt.h:610
hwcontext.h
AVDRMPlaneDescriptor::pitch
ptrdiff_t pitch
Pitch (linesize) of this plane.
Definition: hwcontext_drm.h:87
AVFrame::linesize
int linesize[AV_NUM_DATA_POINTERS]
For video, a positive or negative value, which is typically indicating the size in bytes of each pict...
Definition: frame.h:472
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
FFVkFormatEntry::vk_planes
int vk_planes
Definition: hwcontext_vulkan.c:400
AVVulkanDeviceQueueFamily
Definition: hwcontext_vulkan.h:33
HWContextType
Definition: hwcontext_internal.h:29
FF_VK_EXT_VIDEO_ENCODE_AV1
#define FF_VK_EXT_VIDEO_ENCODE_AV1
Definition: vulkan_functions.h:71
AV_PIX_FMT_P412
#define AV_PIX_FMT_P412
Definition: pixfmt.h:619
device_features_init
static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
Definition: hwcontext_vulkan.c:214
VulkanDevicePriv::contiguous_planes
int contiguous_planes
Definition: hwcontext_vulkan.c:160
AVVAAPIDeviceContext
VAAPI connection details.
Definition: hwcontext_vaapi.h:68
h
h
Definition: vp9dsp_template.c:2070
AVVulkanDeviceContext::device_features
VkPhysicalDeviceFeatures2 device_features
This structure should be set to the set of features that present and enabled during device creation.
Definition: hwcontext_vulkan.h:92
AVDictionaryEntry::value
char * value
Definition: dict.h:92
avstring.h
AVDRMDeviceContext
DRM device.
Definition: hwcontext_drm.h:157
AV_PIX_FMT_GRAY12
#define AV_PIX_FMT_GRAY12
Definition: pixfmt.h:520
copy_buffer_data
static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf, AVFrame *swf, VkBufferImageCopy *region, int planes, int upload)
Definition: hwcontext_vulkan.c:4298
VulkanDeviceFeatures::atomic_float
VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float
Definition: hwcontext_vulkan.c:115
ADD_VAL_TO_LIST
#define ADD_VAL_TO_LIST(list, count, val)
Definition: hwcontext_vulkan.c:800
AV_PIX_FMT_BAYER_RGGB16
#define AV_PIX_FMT_BAYER_RGGB16
Definition: pixfmt.h:572
AVDRMFrameDescriptor::nb_objects
int nb_objects
Number of DRM objects making up this frame.
Definition: hwcontext_drm.h:137
VulkanDeviceFeatures::shader_object
VkPhysicalDeviceShaderObjectFeaturesEXT shader_object
Definition: hwcontext_vulkan.c:113
HWMapDescriptor
Definition: hwcontext_internal.h:120
AVVulkanDeviceQueueFamily::flags
VkQueueFlagBits flags
Definition: hwcontext_vulkan.h:41
AVVulkanDeviceQueueFamily::video_caps
VkVideoCodecOperationFlagBitsKHR video_caps
Definition: hwcontext_vulkan.h:44
FF_VK_EXT_ZERO_INITIALIZE
#define FF_VK_EXT_ZERO_INITIALIZE
Definition: vulkan_functions.h:52
FFVulkanFunctions
Definition: vulkan_functions.h:274
VulkanDevicePriv::p
AVVulkanDeviceContext p
The public AVVulkanDeviceContext.
Definition: hwcontext_vulkan.c:126
VulkanFramesPriv::modifier_info
VkImageDrmFormatModifierListCreateInfoEXT * modifier_info
Definition: hwcontext_vulkan.c:189
FFVkFormatEntry::aspect
VkImageAspectFlags aspect
Definition: hwcontext_vulkan.c:399
ff_vk_get_pooled_buffer
int ff_vk_get_pooled_buffer(FFVulkanContext *ctx, AVBufferPool **buf_pool, AVBufferRef **buf, VkBufferUsageFlags usage, void *create_pNext, size_t size, VkMemoryPropertyFlagBits mem_props)
Initialize a pool and create AVBufferRefs containing FFVkBuffer.
Definition: vulkan.c:1276
VulkanDeviceSelection::name
const char * name
Definition: hwcontext_vulkan.c:1308
src
#define src
Definition: vp8dsp.c:248
AV_HWDEVICE_TYPE_DRM
@ AV_HWDEVICE_TYPE_DRM
Definition: hwcontext.h:36
AVVulkanDeviceContext::nb_comp_queues
attribute_deprecated int nb_comp_queues
Definition: hwcontext_vulkan.h:148
AV_PIX_FMT_YUVA422P
@ AV_PIX_FMT_YUVA422P
planar YUV 4:2:2 24bpp, (1 Cr & Cb sample per 2x1 Y & A samples)
Definition: pixfmt.h:173
vulkan_device_create_internal
static int vulkan_device_create_internal(AVHWDeviceContext *ctx, VulkanDeviceSelection *dev_select, int disable_multiplane, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:1788
w32dlfcn.h
av_vk_get_optional_instance_extensions
const char ** av_vk_get_optional_instance_extensions(int *count)
Returns an array of optional Vulkan instance extensions that FFmpeg may use if enabled.
Definition: hwcontext_vulkan.c:739
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:3376
vulkan_device_has_rebar
static int vulkan_device_has_rebar(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:821
try_export_flags
static void try_export_flags(AVHWFramesContext *hwfc, VkExternalMemoryHandleTypeFlags *comp_handle_types, VkExternalMemoryHandleTypeFlags *iexp, VkExternalMemoryHandleTypeFlagBits exp)
Definition: hwcontext_vulkan.c:2809
Generated on Fri Feb 20 2026 19:25:08 for FFmpeg by   doxygen 1.8.17