FFmpeg
hwcontext_vulkan.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) Lynne
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #define VK_NO_PROTOTYPES
22 #define VK_ENABLE_BETA_EXTENSIONS
23 
24 #ifdef _WIN32
25 #include <windows.h> /* Included to prevent conflicts with CreateSemaphore */
26 #include <versionhelpers.h>
27 #include "compat/w32dlfcn.h"
28 #else
29 #include <dlfcn.h>
30 #include <unistd.h>
31 #endif
32 
33 #include "thread.h"
34 
35 #include "config.h"
36 #include "pixdesc.h"
37 #include "avstring.h"
38 #include "imgutils.h"
39 #include "hwcontext.h"
40 #include "hwcontext_internal.h"
41 #include "hwcontext_vulkan.h"
42 #include "mem.h"
43 
44 #include "vulkan.h"
45 #include "vulkan_loader.h"
46 
47 #if CONFIG_VAAPI
48 #include "hwcontext_vaapi.h"
49 #endif
50 
51 #if CONFIG_LIBDRM
52 #if CONFIG_VAAPI
53 #include <va/va_drmcommon.h>
54 #endif
55 #ifdef __linux__
56 #include <sys/sysmacros.h>
57 #endif
58 #include <sys/stat.h>
59 #include <xf86drm.h>
60 #include <drm_fourcc.h>
61 #include "hwcontext_drm.h"
62 #endif
63 
64 #if HAVE_LINUX_DMA_BUF_H
65 #include <sys/ioctl.h>
66 #include <linux/dma-buf.h>
67 #endif
68 
69 #if CONFIG_CUDA
70 #include "hwcontext_cuda_internal.h"
71 #include "cuda_check.h"
72 #define CHECK_CU(x) FF_CUDA_CHECK_DL(cuda_cu, cu, x)
73 #endif
74 
75 typedef struct VulkanDeviceFeatures {
76  VkPhysicalDeviceFeatures2 device;
77 
78  VkPhysicalDeviceVulkan11Features vulkan_1_1;
79  VkPhysicalDeviceVulkan12Features vulkan_1_2;
80  VkPhysicalDeviceVulkan13Features vulkan_1_3;
81  VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore;
82  VkPhysicalDeviceShaderSubgroupRotateFeaturesKHR subgroup_rotate;
83  VkPhysicalDeviceHostImageCopyFeaturesEXT host_image_copy;
84 
85 #ifdef VK_KHR_shader_expect_assume
86  VkPhysicalDeviceShaderExpectAssumeFeaturesKHR expect_assume;
87 #endif
88 
89  VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1;
90 #ifdef VK_KHR_video_maintenance2
91  VkPhysicalDeviceVideoMaintenance2FeaturesKHR video_maintenance_2;
92 #endif
93 #ifdef VK_KHR_video_decode_vp9
94  VkPhysicalDeviceVideoDecodeVP9FeaturesKHR vp9_decode;
95 #endif
96 #ifdef VK_KHR_video_encode_av1
97  VkPhysicalDeviceVideoEncodeAV1FeaturesKHR av1_encode;
98 #endif
99 
100  VkPhysicalDeviceShaderObjectFeaturesEXT shader_object;
101  VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix;
102  VkPhysicalDeviceDescriptorBufferFeaturesEXT descriptor_buffer;
103  VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float;
104 
105 #ifdef VK_KHR_shader_relaxed_extended_instruction
106  VkPhysicalDeviceShaderRelaxedExtendedInstructionFeaturesKHR relaxed_extended_instruction;
107 #endif
108 } VulkanDeviceFeatures;
109 
110 typedef struct VulkanDevicePriv {
111  /**
112  * The public AVVulkanDeviceContext. See hwcontext_vulkan.h for it.
113  */
114  AVVulkanDeviceContext p;
115 
116  /* Vulkan library and loader functions */
117  void *libvulkan;
118 
119  FFVulkanContext vkctx;
120  AVVulkanDeviceQueueFamily *compute_qf;
121  AVVulkanDeviceQueueFamily *transfer_qf;
122 
123  /* Properties */
124  VkPhysicalDeviceProperties2 props;
125  VkPhysicalDeviceMemoryProperties mprops;
126  VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops;
127  VkPhysicalDeviceDriverProperties dprops;
128 
129  /* Opaque FD external semaphore properties */
130  VkExternalSemaphoreProperties ext_sem_props_opaque;
131 
132  /* Enabled features */
133  VulkanDeviceFeatures feats;
134 
135  /* Queues */
136  pthread_mutex_t **qf_mutex;
137  uint32_t nb_tot_qfs;
138  uint32_t img_qfs[64];
139  uint32_t nb_img_qfs;
140 
141  /* Debug callback */
142  VkDebugUtilsMessengerEXT debug_ctx;
143 
144  /* Settings */
145  int use_linear_images;
146 
147  /* Option to allocate all image planes in a single allocation */
148  int contiguous_planes;
149 
150  /* Disable multiplane images */
151  int disable_multiplane;
152 
153  /* Disable host image transfer */
154  int disable_host_transfer;
155 
156  /* Prefer memcpy over dynamic host pointer imports */
157  int avoid_host_import;
158 
159  /* Maximum queues */
160  int limit_queues;
161 } VulkanDevicePriv;
162 
163 typedef struct VulkanFramesPriv {
164  /**
165  * The public AVVulkanFramesContext. See hwcontext_vulkan.h for it.
166  */
167  AVVulkanFramesContext p;
168 
169  /* Image conversions */
170  FFVkExecPool compute_exec;
171 
172  /* Image transfers */
173  FFVkExecPool upload_exec;
174  FFVkExecPool download_exec;
175 
176  /* Temporary buffer pools */
177  AVBufferPool *tmp;
178 
179  /* Modifier info list to free at uninit */
180  VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
181 
182  /* Properties for DRM modifier for each plane in the image */
183  VkDrmFormatModifierPropertiesEXT drm_format_modifier_properties[5];
184 } VulkanFramesPriv;
185 
186 typedef struct AVVkFrameInternal {
187  pthread_mutex_t update_mutex;
188 
189 #if CONFIG_CUDA
190  /* Importing external memory into cuda is really expensive so we keep the
191  * memory imported all the time */
192  AVBufferRef *cuda_fc_ref; /* Need to keep it around for uninit */
193  CUexternalMemory ext_mem[AV_NUM_DATA_POINTERS];
194  CUmipmappedArray cu_mma[AV_NUM_DATA_POINTERS];
195  CUarray cu_array[AV_NUM_DATA_POINTERS];
196  CUexternalSemaphore cu_sem[AV_NUM_DATA_POINTERS];
197 #ifdef _WIN32
198  HANDLE ext_mem_handle[AV_NUM_DATA_POINTERS];
199  HANDLE ext_sem_handle[AV_NUM_DATA_POINTERS];
200 #endif
201 #endif
202 } AVVkFrameInternal;
203 
204 /* Initialize all structs in VulkanDeviceFeatures */
205 static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
206 {
207  VulkanDevicePriv *p = ctx->hwctx;
208  FFVulkanContext *s = &p->vkctx;
209 
210  feats->device = (VkPhysicalDeviceFeatures2) {
211  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
212  };
213 
214  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_1, FF_VK_EXT_NO_FLAG,
215  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES);
216  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_2, FF_VK_EXT_NO_FLAG,
217  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES);
218  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vulkan_1_3, FF_VK_EXT_NO_FLAG,
219  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES);
220 
221  FF_VK_STRUCT_EXT(s, &feats->device, &feats->timeline_semaphore, FF_VK_EXT_PORTABILITY_SUBSET,
222  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES);
223  FF_VK_STRUCT_EXT(s, &feats->device, &feats->subgroup_rotate, FF_VK_EXT_SUBGROUP_ROTATE,
224  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SUBGROUP_ROTATE_FEATURES_KHR);
225  FF_VK_STRUCT_EXT(s, &feats->device, &feats->host_image_copy, FF_VK_EXT_HOST_IMAGE_COPY,
226  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_IMAGE_COPY_FEATURES_EXT);
227 
228 #ifdef VK_KHR_shader_expect_assume
229  FF_VK_STRUCT_EXT(s, &feats->device, &feats->expect_assume, FF_VK_EXT_EXPECT_ASSUME,
230  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_EXPECT_ASSUME_FEATURES_KHR);
231 #endif
232 
233  FF_VK_STRUCT_EXT(s, &feats->device, &feats->video_maintenance_1, FF_VK_EXT_VIDEO_MAINTENANCE_1,
234  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_1_FEATURES_KHR);
235 #ifdef VK_KHR_video_maintenance2
236  FF_VK_STRUCT_EXT(s, &feats->device, &feats->video_maintenance_2, FF_VK_EXT_VIDEO_MAINTENANCE_2,
237  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_2_FEATURES_KHR);
238 #endif
239 #ifdef VK_KHR_video_decode_vp9
240  FF_VK_STRUCT_EXT(s, &feats->device, &feats->vp9_decode, FF_VK_EXT_VIDEO_DECODE_VP9,
241  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_DECODE_VP9_FEATURES_KHR);
242 #endif
243 #ifdef VK_KHR_video_encode_av1
244  FF_VK_STRUCT_EXT(s, &feats->device, &feats->av1_encode, FF_VK_EXT_VIDEO_ENCODE_AV1,
245  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_ENCODE_AV1_FEATURES_KHR);
246 #endif
247 
248  FF_VK_STRUCT_EXT(s, &feats->device, &feats->shader_object, FF_VK_EXT_SHADER_OBJECT,
249  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OBJECT_FEATURES_EXT);
250  FF_VK_STRUCT_EXT(s, &feats->device, &feats->cooperative_matrix, FF_VK_EXT_COOP_MATRIX,
251  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR);
252  FF_VK_STRUCT_EXT(s, &feats->device, &feats->descriptor_buffer, FF_VK_EXT_DESCRIPTOR_BUFFER,
253  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_BUFFER_FEATURES_EXT);
254  FF_VK_STRUCT_EXT(s, &feats->device, &feats->atomic_float, FF_VK_EXT_ATOMIC_FLOAT,
255  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT);
256 
257 #ifdef VK_KHR_shader_relaxed_extended_instruction
258  FF_VK_STRUCT_EXT(s, &feats->device, &feats->relaxed_extended_instruction, FF_VK_EXT_RELAXED_EXTENDED_INSTR,
259  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_RELAXED_EXTENDED_INSTRUCTION_FEATURES_KHR);
260 #endif
261 }
262 
263 /* Copy all needed device features */
264 static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
265 {
266 #define COPY_VAL(VAL) \
267  do { \
268  dst->VAL = src->VAL; \
269  } while (0) \
270 
271  COPY_VAL(device.features.shaderImageGatherExtended);
272  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
273  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
274  COPY_VAL(device.features.fragmentStoresAndAtomics);
275  COPY_VAL(device.features.vertexPipelineStoresAndAtomics);
276  COPY_VAL(device.features.shaderInt64);
277  COPY_VAL(device.features.shaderInt16);
278  COPY_VAL(device.features.shaderFloat64);
279  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
280  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
281 
282  COPY_VAL(vulkan_1_1.samplerYcbcrConversion);
283  COPY_VAL(vulkan_1_1.storagePushConstant16);
284  COPY_VAL(vulkan_1_1.storageBuffer16BitAccess);
285  COPY_VAL(vulkan_1_1.uniformAndStorageBuffer16BitAccess);
286 
287  COPY_VAL(vulkan_1_2.timelineSemaphore);
288  COPY_VAL(vulkan_1_2.scalarBlockLayout);
289  COPY_VAL(vulkan_1_2.bufferDeviceAddress);
290  COPY_VAL(vulkan_1_2.hostQueryReset);
291  COPY_VAL(vulkan_1_2.storagePushConstant8);
292  COPY_VAL(vulkan_1_2.shaderInt8);
293  COPY_VAL(vulkan_1_2.storageBuffer8BitAccess);
294  COPY_VAL(vulkan_1_2.uniformAndStorageBuffer8BitAccess);
295  COPY_VAL(vulkan_1_2.shaderFloat16);
296  COPY_VAL(vulkan_1_2.shaderBufferInt64Atomics);
297  COPY_VAL(vulkan_1_2.shaderSharedInt64Atomics);
298  COPY_VAL(vulkan_1_2.vulkanMemoryModel);
299  COPY_VAL(vulkan_1_2.vulkanMemoryModelDeviceScope);
300  COPY_VAL(vulkan_1_2.uniformBufferStandardLayout);
301 
302  COPY_VAL(vulkan_1_3.dynamicRendering);
303  COPY_VAL(vulkan_1_3.maintenance4);
304  COPY_VAL(vulkan_1_3.synchronization2);
305  COPY_VAL(vulkan_1_3.computeFullSubgroups);
306  COPY_VAL(vulkan_1_3.subgroupSizeControl);
307  COPY_VAL(vulkan_1_3.shaderZeroInitializeWorkgroupMemory);
308  COPY_VAL(vulkan_1_3.dynamicRendering);
309 
310  COPY_VAL(timeline_semaphore.timelineSemaphore);
311  COPY_VAL(subgroup_rotate.shaderSubgroupRotate);
312  COPY_VAL(host_image_copy.hostImageCopy);
313 
314  COPY_VAL(video_maintenance_1.videoMaintenance1);
315 #ifdef VK_KHR_video_maintenance2
316  COPY_VAL(video_maintenance_2.videoMaintenance2);
317 #endif
318 
319 #ifdef VK_KHR_video_decode_vp9
320  COPY_VAL(vp9_decode.videoDecodeVP9);
321 #endif
322 
323 #ifdef VK_KHR_video_encode_av1
324  COPY_VAL(av1_encode.videoEncodeAV1);
325 #endif
326 
327  COPY_VAL(shader_object.shaderObject);
328 
329  COPY_VAL(cooperative_matrix.cooperativeMatrix);
330 
331  COPY_VAL(descriptor_buffer.descriptorBuffer);
332  COPY_VAL(descriptor_buffer.descriptorBufferPushDescriptors);
333 
334  COPY_VAL(atomic_float.shaderBufferFloat32Atomics);
335  COPY_VAL(atomic_float.shaderBufferFloat32AtomicAdd);
336 
337 #ifdef VK_KHR_shader_relaxed_extended_instruction
338  COPY_VAL(relaxed_extended_instruction.shaderRelaxedExtendedInstruction);
339 #endif
340 
341 #ifdef VK_KHR_shader_expect_assume
342  COPY_VAL(expect_assume.shaderExpectAssume);
343 #endif
344 
345 #undef COPY_VAL
346 }
347 
348 #define ASPECT_2PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT)
349 #define ASPECT_3PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT | VK_IMAGE_ASPECT_PLANE_2_BIT)
350 
351 static const struct FFVkFormatEntry {
352  VkFormat vkf;
353  enum AVPixelFormat pixfmt;
354  VkImageAspectFlags aspect;
355  int vk_planes;
356  int nb_images;
357  int nb_images_fallback;
358  const VkFormat fallback[5];
359 } vk_formats_list[] = {
360  /* Gray formats */
361  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GRAY8, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8_UNORM } },
362  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
363  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY12, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
364  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY14, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
365  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
366  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GRAY32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_UINT } },
367  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GRAYF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_SFLOAT } },
368 
369  /* RGB formats */
370  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGRA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
371  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGBA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
372  { VK_FORMAT_R8G8B8_UNORM, AV_PIX_FMT_RGB24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8_UNORM } },
373  { VK_FORMAT_B8G8R8_UNORM, AV_PIX_FMT_BGR24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8_UNORM } },
374  { VK_FORMAT_R16G16B16_UNORM, AV_PIX_FMT_RGB48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16_UNORM } },
375  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_RGBA64, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
376  { VK_FORMAT_R5G6B5_UNORM_PACK16, AV_PIX_FMT_RGB565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R5G6B5_UNORM_PACK16 } },
377  { VK_FORMAT_B5G6R5_UNORM_PACK16, AV_PIX_FMT_BGR565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B5G6R5_UNORM_PACK16 } },
378  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGR0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
379  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGB0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
380  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_X2RGB10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2R10G10B10_UNORM_PACK32 } },
381  { VK_FORMAT_A2B10G10R10_UNORM_PACK32, AV_PIX_FMT_X2BGR10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2B10G10R10_UNORM_PACK32 } },
382  { VK_FORMAT_R32G32B32_SFLOAT, AV_PIX_FMT_RGBF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_SFLOAT } },
383  { VK_FORMAT_R32G32B32A32_SFLOAT, AV_PIX_FMT_RGBAF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_SFLOAT } },
384  { VK_FORMAT_R32G32B32_UINT, AV_PIX_FMT_RGB96, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_UINT } },
385  { VK_FORMAT_R32G32B32A32_UINT, AV_PIX_FMT_RGBA128, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_UINT } },
386 
387  /* Planar RGB */
388  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRP, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
389  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP10, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
390  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP12, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
391  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP14, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
392  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP16, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
393  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRPF32, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
394 
395  /* Planar RGB + Alpha */
396  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRAP, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
397  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
398  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
399  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP14, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
400  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
401  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GBRAP32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT } },
402  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRAPF32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
403 
404  /* Bayer */
405  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_BAYER_RGGB16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
406 
407  /* Two-plane 420 YUV at 8, 10, 12 and 16 bits */
408  { VK_FORMAT_G8_B8R8_2PLANE_420_UNORM, AV_PIX_FMT_NV12, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
409  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P010, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
410  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P012, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
411  { VK_FORMAT_G16_B16R16_2PLANE_420_UNORM, AV_PIX_FMT_P016, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
412 
413  /* Two-plane 422 YUV at 8, 10 and 16 bits */
414  { VK_FORMAT_G8_B8R8_2PLANE_422_UNORM, AV_PIX_FMT_NV16, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
415  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P210, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
416  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P212, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
417  { VK_FORMAT_G16_B16R16_2PLANE_422_UNORM, AV_PIX_FMT_P216, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
418 
419  /* Two-plane 444 YUV at 8, 10 and 16 bits */
420  { VK_FORMAT_G8_B8R8_2PLANE_444_UNORM, AV_PIX_FMT_NV24, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
421  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P410, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
422  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P412, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
423  { VK_FORMAT_G16_B16R16_2PLANE_444_UNORM, AV_PIX_FMT_P416, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
424 
425  /* Three-plane 420, 422, 444 at 8, 10, 12 and 16 bits */
426  { VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
427  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
428  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
429  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
430  { VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
431  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
432  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
433  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
434  { VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
435  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
436  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
437  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
438 
439  /* Single plane 422 at 8, 10, 12 and 16 bits */
440  { VK_FORMAT_G8B8G8R8_422_UNORM, AV_PIX_FMT_YUYV422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
441  { VK_FORMAT_B8G8R8G8_422_UNORM, AV_PIX_FMT_UYVY422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
442  { VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16, AV_PIX_FMT_Y210, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
443  { VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16, AV_PIX_FMT_Y212, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
444  { VK_FORMAT_G16B16G16R16_422_UNORM, AV_PIX_FMT_Y216, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
445 
446  /* Planar YUVA 420 at 8, 10 and 16 bits */
447  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA420P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
448  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA420P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
449  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA420P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
450 
451  /* Planar YUVA 422 at 8, 10, 12 and 16 bits */
452  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA422P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
453  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
454  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
455  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA422P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
456 
457  /* Planar YUVA 444 at 8, 10, 12 and 16 bits */
458  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_YUVA444P, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
459  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
460  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
461  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_YUVA444P16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
462 
463  /* Single plane 444 at 8, 10, 12 and 16 bits */
464  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_UYVA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
465  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_XV30, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
466  { VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16, AV_PIX_FMT_XV36, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
467  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_XV48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
468 };
469 static const int nb_vk_formats_list = FF_ARRAY_ELEMS(vk_formats_list);
470 
471 const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p)
472 {
473  for (int i = 0; i < nb_vk_formats_list; i++)
474  if (vk_formats_list[i].pixfmt == p)
475  return vk_formats_list[i].fallback;
476  return NULL;
477 }
478 
479 static const struct FFVkFormatEntry *vk_find_format_entry(enum AVPixelFormat p)
480 {
481  for (int i = 0; i < nb_vk_formats_list; i++)
482  if (vk_formats_list[i].pixfmt == p)
483  return &vk_formats_list[i];
484  return NULL;
485 }
486 
487 static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
488  VkImageTiling tiling,
489  VkFormat fmts[AV_NUM_DATA_POINTERS], /* Output format list */
490  int *nb_images, /* Output number of images */
491  VkImageAspectFlags *aspect, /* Output aspect */
492  VkImageUsageFlags *supported_usage, /* Output supported usage */
493  int disable_multiplane, int need_storage)
494 {
495  VulkanDevicePriv *priv = dev_ctx->hwctx;
496  AVVulkanDeviceContext *hwctx = &priv->p;
497  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
498 
499  const VkFormatFeatureFlagBits2 basic_flags = VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT |
500  VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
501  VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
502 
503  for (int i = 0; i < nb_vk_formats_list; i++) {
504  if (vk_formats_list[i].pixfmt == p) {
505  VkFormatProperties3 fprops = {
506  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3,
507  };
508  VkFormatProperties2 prop = {
509  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
510  .pNext = &fprops,
511  };
512  VkFormatFeatureFlagBits2 feats_primary, feats_secondary;
513  int basics_primary = 0, basics_secondary = 0;
514  int storage_primary = 0, storage_secondary = 0;
515 
516  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
517  vk_formats_list[i].vkf,
518  &prop);
519 
520  feats_primary = tiling == VK_IMAGE_TILING_LINEAR ?
521  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
522  basics_primary = (feats_primary & basic_flags) == basic_flags;
523  storage_primary = !!(feats_primary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
524 
525  if (vk_formats_list[i].vkf != vk_formats_list[i].fallback[0]) {
526  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
527  vk_formats_list[i].fallback[0],
528  &prop);
529  feats_secondary = tiling == VK_IMAGE_TILING_LINEAR ?
530  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
531  basics_secondary = (feats_secondary & basic_flags) == basic_flags;
532  storage_secondary = !!(feats_secondary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
533  } else {
534  basics_secondary = basics_primary;
535  storage_secondary = storage_primary;
536  }
537 
538  if (basics_primary &&
539  !(disable_multiplane && vk_formats_list[i].vk_planes > 1) &&
540  (!need_storage || (need_storage && (storage_primary | storage_secondary)))) {
541  if (fmts) {
542  if (vk_formats_list[i].nb_images > 1) {
543  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
544  fmts[j] = vk_formats_list[i].fallback[j];
545  } else {
546  fmts[0] = vk_formats_list[i].vkf;
547  }
548  }
549  if (nb_images)
550  *nb_images = 1;
551  if (aspect)
552  *aspect = vk_formats_list[i].aspect;
553  if (supported_usage)
554  *supported_usage = ff_vk_map_feats_to_usage(feats_primary) |
555  ((need_storage && (storage_primary | storage_secondary)) ?
556  VK_IMAGE_USAGE_STORAGE_BIT : 0);
557  return 0;
558  } else if (basics_secondary &&
559  (!need_storage || (need_storage && storage_secondary))) {
560  if (fmts) {
561  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
562  fmts[j] = vk_formats_list[i].fallback[j];
563  }
564  if (nb_images)
565  *nb_images = vk_formats_list[i].nb_images_fallback;
566  if (aspect)
567  *aspect = vk_formats_list[i].aspect;
568  if (supported_usage)
569  *supported_usage = ff_vk_map_feats_to_usage(feats_secondary);
570  return 0;
571  } else {
572  return AVERROR(ENOTSUP);
573  }
574  }
575  }
576 
577  return AVERROR(EINVAL);
578 }
579 
580 #if CONFIG_VULKAN_STATIC
581 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,
582  const char *pName);
583 #endif
584 
585 static int load_libvulkan(AVHWDeviceContext *ctx)
586 {
587  VulkanDevicePriv *p = ctx->hwctx;
588  AVVulkanDeviceContext *hwctx = &p->p;
589 
590 #if CONFIG_VULKAN_STATIC
591  hwctx->get_proc_addr = vkGetInstanceProcAddr;
592 #else
593  static const char *lib_names[] = {
594 #if defined(_WIN32)
595  "vulkan-1.dll",
596 #elif defined(__APPLE__)
597  "libvulkan.dylib",
598  "libvulkan.1.dylib",
599  "libMoltenVK.dylib",
600 #else
601  "libvulkan.so.1",
602  "libvulkan.so",
603 #endif
604  };
605 
606  for (int i = 0; i < FF_ARRAY_ELEMS(lib_names); i++) {
607  p->libvulkan = dlopen(lib_names[i], RTLD_NOW | RTLD_LOCAL);
608  if (p->libvulkan)
609  break;
610  }
611 
612  if (!p->libvulkan) {
613  av_log(ctx, AV_LOG_ERROR, "Unable to open the libvulkan library!\n");
614  return AVERROR_UNKNOWN;
615  }
616 
617  hwctx->get_proc_addr = (PFN_vkGetInstanceProcAddr)dlsym(p->libvulkan, "vkGetInstanceProcAddr");
618 #endif /* CONFIG_VULKAN_STATIC */
619 
620  return 0;
621 }
622 
623 typedef struct VulkanOptExtension {
624  const char *name;
625  FFVulkanExtensions flag;
626 } VulkanOptExtension;
627 
628 static const VulkanOptExtension optional_instance_exts[] = {
629  { VK_EXT_LAYER_SETTINGS_EXTENSION_NAME, FF_VK_EXT_NO_FLAG },
630 #ifdef __APPLE__
631  { VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME, FF_VK_EXT_NO_FLAG },
632 #endif
633 };
634 
635 static const VulkanOptExtension optional_device_exts[] = {
636  /* Misc or required by other extensions */
637  { VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME, FF_VK_EXT_PORTABILITY_SUBSET },
638  { VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, FF_VK_EXT_PUSH_DESCRIPTOR },
639  { VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME, FF_VK_EXT_DESCRIPTOR_BUFFER },
640  { VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME, FF_VK_EXT_DEVICE_DRM },
641  { VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, FF_VK_EXT_ATOMIC_FLOAT },
642  { VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME, FF_VK_EXT_COOP_MATRIX },
643  { VK_EXT_SHADER_OBJECT_EXTENSION_NAME, FF_VK_EXT_SHADER_OBJECT },
644  { VK_KHR_SHADER_SUBGROUP_ROTATE_EXTENSION_NAME, FF_VK_EXT_SUBGROUP_ROTATE },
645 #ifdef VK_KHR_shader_expect_assume
646  { VK_KHR_SHADER_EXPECT_ASSUME_EXTENSION_NAME, FF_VK_EXT_EXPECT_ASSUME },
647 #endif
648  { VK_KHR_VIDEO_MAINTENANCE_1_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_1 },
649 #ifdef VK_KHR_video_maintenance2
650  { VK_KHR_VIDEO_MAINTENANCE_2_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_2 },
651 #endif
652 
653  /* Imports/exports */
654  { VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_MEMORY },
655  { VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_DMABUF_MEMORY },
656  { VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, FF_VK_EXT_DRM_MODIFIER_FLAGS },
657  { VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_SEM },
658  { VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_HOST_MEMORY },
659 #ifdef _WIN32
660  { VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_MEMORY },
661  { VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_SEM },
662 #endif
663 
664  /* Video encoding/decoding */
665  { VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_QUEUE },
666  { VK_KHR_VIDEO_ENCODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_QUEUE },
667  { VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_QUEUE },
668  { VK_KHR_VIDEO_ENCODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H264 },
669  { VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H264 },
670  { VK_KHR_VIDEO_ENCODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H265 },
671  { VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H265 },
672 #ifdef VK_KHR_video_decode_vp9
673  { VK_KHR_VIDEO_DECODE_VP9_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_VP9 },
674 #endif
675 #ifdef VK_KHR_video_encode_av1
676  { VK_KHR_VIDEO_ENCODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_AV1 },
677 #endif
678  { VK_KHR_VIDEO_DECODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_AV1 },
679 };
680 
681 static VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
682  VkDebugUtilsMessageTypeFlagsEXT messageType,
683  const VkDebugUtilsMessengerCallbackDataEXT *data,
684  void *priv)
685 {
686  int l;
687  AVHWDeviceContext *ctx = priv;
688 
689  /* Ignore false positives */
690  switch (data->messageIdNumber) {
691  case 0x086974c1: /* BestPractices-vkCreateCommandPool-command-buffer-reset */
692  case 0xfd92477a: /* BestPractices-vkAllocateMemory-small-allocation */
693  case 0x618ab1e7: /* VUID-VkImageViewCreateInfo-usage-02275 */
694  case 0x30f4ac70: /* VUID-VkImageCreateInfo-pNext-06811 */
695  return VK_FALSE;
696  default:
697  break;
698  }
699 
700  switch (severity) {
701  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT: l = AV_LOG_VERBOSE; break;
702  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT: l = AV_LOG_INFO; break;
703  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT: l = AV_LOG_WARNING; break;
704  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT: l = AV_LOG_ERROR; break;
705  default: l = AV_LOG_DEBUG; break;
706  }
707 
708  av_log(ctx, l, "%s\n", data->pMessage);
709  for (int i = 0; i < data->cmdBufLabelCount; i++)
710  av_log(ctx, l, "\t%i: %s\n", i, data->pCmdBufLabels[i].pLabelName);
711 
712  return VK_FALSE;
713 }
714 
715 #define ADD_VAL_TO_LIST(list, count, val) \
716  do { \
717  list = av_realloc_array(list, sizeof(*list), ++count); \
718  if (!list) { \
719  err = AVERROR(ENOMEM); \
720  goto fail; \
721  } \
722  list[count - 1] = av_strdup(val); \
723  if (!list[count - 1]) { \
724  err = AVERROR(ENOMEM); \
725  goto fail; \
726  } \
727  } while(0)
728 
729 #define RELEASE_PROPS(props, count) \
730  if (props) { \
731  for (int i = 0; i < count; i++) \
732  av_free((void *)((props)[i])); \
733  av_free((void *)props); \
734  }
735 
736 enum FFVulkanDebugMode {
737  FF_VULKAN_DEBUG_NONE = 0,
738  /* Standard GPU-assisted validation */
739  FF_VULKAN_DEBUG_VALIDATE = 1,
740  /* Passes printfs in shaders to the debug callback */
741  FF_VULKAN_DEBUG_PRINTF = 2,
742  /* Enables extra printouts */
743  FF_VULKAN_DEBUG_PRACTICES = 3,
744  /* Disables validation but keeps shader debug info and optimizations */
745  FF_VULKAN_DEBUG_PROFILE = 4,
746 
747  FF_VULKAN_DEBUG_NB,
748 };
749 
750 static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts,
751  const char * const **dst, uint32_t *num,
752  enum FFVulkanDebugMode debug_mode)
753 {
754  const char *tstr;
755  const char **extension_names = NULL;
756  VulkanDevicePriv *p = ctx->hwctx;
757  AVVulkanDeviceContext *hwctx = &p->p;
758  FFVulkanFunctions *vk = &p->vkctx.vkfn;
759  int err = 0, found, extensions_found = 0;
760 
761  const char *mod;
762  int optional_exts_num;
763  uint32_t sup_ext_count;
764  char *user_exts_str = NULL;
765  AVDictionaryEntry *user_exts;
766  VkExtensionProperties *sup_ext;
767  const VulkanOptExtension *optional_exts;
768 
769  if (!dev) {
770  mod = "instance";
771  optional_exts = optional_instance_exts;
772  optional_exts_num = FF_ARRAY_ELEMS(optional_instance_exts);
773  user_exts = av_dict_get(opts, "instance_extensions", NULL, 0);
774  if (user_exts) {
775  user_exts_str = av_strdup(user_exts->value);
776  if (!user_exts_str) {
777  err = AVERROR(ENOMEM);
778  goto fail;
779  }
780  }
781  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, NULL);
782  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
783  if (!sup_ext)
784  return AVERROR(ENOMEM);
785  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, sup_ext);
786  } else {
787  mod = "device";
788  optional_exts = optional_device_exts;
789  optional_exts_num = FF_ARRAY_ELEMS(optional_device_exts);
790  user_exts = av_dict_get(opts, "device_extensions", NULL, 0);
791  if (user_exts) {
792  user_exts_str = av_strdup(user_exts->value);
793  if (!user_exts_str) {
794  err = AVERROR(ENOMEM);
795  goto fail;
796  }
797  }
798  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
799  &sup_ext_count, NULL);
800  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
801  if (!sup_ext)
802  return AVERROR(ENOMEM);
803  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
804  &sup_ext_count, sup_ext);
805  }
806 
807  for (int i = 0; i < optional_exts_num; i++) {
808  tstr = optional_exts[i].name;
809  found = 0;
810 
811  /* Intel has had a bad descriptor buffer implementation for a while */
812  if (p->dprops.driverID == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA &&
813  !strcmp(tstr, VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME))
814  continue;
815 
816  if (dev &&
817  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
818  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
819  (debug_mode == FF_VULKAN_DEBUG_PRACTICES)) &&
820  !strcmp(tstr, VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME)) {
821  continue;
822  }
823 
824  for (int j = 0; j < sup_ext_count; j++) {
825  if (!strcmp(tstr, sup_ext[j].extensionName)) {
826  found = 1;
827  break;
828  }
829  }
830  if (!found)
831  continue;
832 
833  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
834  p->vkctx.extensions |= optional_exts[i].flag;
835  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
836  }
837 
838  if (!dev &&
839  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
840  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
841  (debug_mode == FF_VULKAN_DEBUG_PRACTICES))) {
842  tstr = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
843  found = 0;
844  for (int j = 0; j < sup_ext_count; j++) {
845  if (!strcmp(tstr, sup_ext[j].extensionName)) {
846  found = 1;
847  break;
848  }
849  }
850  if (found) {
851  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
852  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
853  } else {
854  av_log(ctx, AV_LOG_ERROR, "Debug extension \"%s\" not found!\n",
855  tstr);
856  err = AVERROR(EINVAL);
857  goto fail;
858  }
859  }
860 
861 #ifdef VK_KHR_shader_relaxed_extended_instruction
862  if (((debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
863  (debug_mode == FF_VULKAN_DEBUG_PROFILE)) && dev) {
864  tstr = VK_KHR_SHADER_RELAXED_EXTENDED_INSTRUCTION_EXTENSION_NAME;
865  found = 0;
866  for (int j = 0; j < sup_ext_count; j++) {
867  if (!strcmp(tstr, sup_ext[j].extensionName)) {
868  found = 1;
869  break;
870  }
871  }
872  if (found) {
873  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
874  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
875  } else {
876  av_log(ctx, AV_LOG_ERROR, "Debug_printf/profile enabled, but extension \"%s\" not found!\n",
877  tstr);
878  err = AVERROR(EINVAL);
879  goto fail;
880  }
881  }
882 #endif
883 
884  if (user_exts_str) {
885  char *save, *token = av_strtok(user_exts_str, "+", &save);
886  while (token) {
887  found = 0;
888  for (int j = 0; j < sup_ext_count; j++) {
889  if (!strcmp(token, sup_ext[j].extensionName)) {
890  found = 1;
891  break;
892  }
893  }
894  if (found) {
895  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension \"%s\"\n", mod, token);
896  ADD_VAL_TO_LIST(extension_names, extensions_found, token);
897  } else {
898  av_log(ctx, AV_LOG_WARNING, "%s extension \"%s\" not found, excluding.\n",
899  mod, token);
900  }
901  token = av_strtok(NULL, "+", &save);
902  }
903  }
904 
905  *dst = extension_names;
906  *num = extensions_found;
907 
908  av_free(user_exts_str);
909  av_free(sup_ext);
910  return 0;
911 
912 fail:
913  RELEASE_PROPS(extension_names, extensions_found);
914  av_free(user_exts_str);
915  av_free(sup_ext);
916  return err;
917 }
918 
919 static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts,
920  const char * const **dst, uint32_t *num,
921  enum FFVulkanDebugMode *debug_mode)
922 {
923  int err = 0;
924  VulkanDevicePriv *priv = ctx->hwctx;
925  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
926 
927  static const char layer_standard_validation[] = { "VK_LAYER_KHRONOS_validation" };
928  int layer_standard_validation_found = 0;
929 
930  uint32_t sup_layer_count;
931  VkLayerProperties *sup_layers;
932 
933  AVDictionaryEntry *user_layers = av_dict_get(opts, "layers", NULL, 0);
934  char *user_layers_str = NULL;
935  char *save, *token;
936 
937  const char **enabled_layers = NULL;
938  uint32_t enabled_layers_count = 0;
939 
940  AVDictionaryEntry *debug_opt = av_dict_get(opts, "debug", NULL, 0);
941  enum FFVulkanDebugMode mode;
942 
943  *debug_mode = mode = FF_VULKAN_DEBUG_NONE;
944 
945  /* Get a list of all layers */
946  vk->EnumerateInstanceLayerProperties(&sup_layer_count, NULL);
947  sup_layers = av_malloc_array(sup_layer_count, sizeof(VkLayerProperties));
948  if (!sup_layers)
949  return AVERROR(ENOMEM);
950  vk->EnumerateInstanceLayerProperties(&sup_layer_count, sup_layers);
951 
952  av_log(ctx, AV_LOG_VERBOSE, "Supported layers:\n");
953  for (int i = 0; i < sup_layer_count; i++)
954  av_log(ctx, AV_LOG_VERBOSE, "\t%s\n", sup_layers[i].layerName);
955 
956  /* If no user layers or debug layers are given, return */
957  if (!debug_opt && !user_layers)
958  goto end;
959 
960  /* Check for any properly supported validation layer */
961  if (debug_opt) {
962  if (!strcmp(debug_opt->value, "profile")) {
963  mode = FF_VULKAN_DEBUG_PROFILE;
964  } else if (!strcmp(debug_opt->value, "printf")) {
965  mode = FF_VULKAN_DEBUG_PRINTF;
966  } else if (!strcmp(debug_opt->value, "validate")) {
967  mode = FF_VULKAN_DEBUG_VALIDATE;
968  } else if (!strcmp(debug_opt->value, "practices")) {
969  mode = FF_VULKAN_DEBUG_PRACTICES;
970  } else {
971  char *end_ptr = NULL;
972  int idx = strtol(debug_opt->value, &end_ptr, 10);
973  if (end_ptr == debug_opt->value || end_ptr[0] != '\0' ||
974  idx < 0 || idx >= FF_VULKAN_DEBUG_NB) {
975  av_log(ctx, AV_LOG_ERROR, "Invalid debugging mode \"%s\"\n",
976  debug_opt->value);
977  err = AVERROR(EINVAL);
978  goto end;
979  }
980  mode = idx;
981  }
982  }
983 
984  /* If mode is VALIDATE or PRINTF, try to find the standard validation layer extension */
985  if ((mode == FF_VULKAN_DEBUG_VALIDATE) ||
986  (mode == FF_VULKAN_DEBUG_PRINTF) ||
987  (mode == FF_VULKAN_DEBUG_PRACTICES)) {
988  for (int i = 0; i < sup_layer_count; i++) {
989  if (!strcmp(layer_standard_validation, sup_layers[i].layerName)) {
990  av_log(ctx, AV_LOG_VERBOSE, "Standard validation layer %s is enabled\n",
991  layer_standard_validation);
992  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, layer_standard_validation);
993  *debug_mode = mode;
994  layer_standard_validation_found = 1;
995  break;
996  }
997  }
998  if (!layer_standard_validation_found) {
999  av_log(ctx, AV_LOG_ERROR,
1000  "Validation Layer \"%s\" not supported\n", layer_standard_validation);
1001  err = AVERROR(ENOTSUP);
1002  goto end;
1003  }
1004  } else if (mode == FF_VULKAN_DEBUG_PROFILE) {
1005  *debug_mode = mode;
1006  }
1007 
1008  /* Process any custom layers enabled */
1009  if (user_layers) {
1010  int found;
1011 
1012  user_layers_str = av_strdup(user_layers->value);
1013  if (!user_layers_str) {
1014  err = AVERROR(ENOMEM);
1015  goto fail;
1016  }
1017 
1018  token = av_strtok(user_layers_str, "+", &save);
1019  while (token) {
1020  found = 0;
1021 
1022  /* If debug=1/2 was specified as an option, skip this layer */
1023  if (!strcmp(layer_standard_validation, token) && layer_standard_validation_found) {
1024  token = av_strtok(NULL, "+", &save);
1025  break;
1026  }
1027 
1028  /* Try to find the layer in the list of supported layers */
1029  for (int j = 0; j < sup_layer_count; j++) {
1030  if (!strcmp(token, sup_layers[j].layerName)) {
1031  found = 1;
1032  break;
1033  }
1034  }
1035 
1036  if (found) {
1037  av_log(ctx, AV_LOG_VERBOSE, "Using layer: %s\n", token);
1038  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, token);
1039 
1040  /* If debug was not set as an option, force it */
1041  if (!strcmp(layer_standard_validation, token))
1042  *debug_mode = FF_VULKAN_DEBUG_VALIDATE;
1043  } else {
1044  av_log(ctx, AV_LOG_ERROR,
1045  "Layer \"%s\" not supported\n", token);
1046  err = AVERROR(EINVAL);
1047  goto end;
1048  }
1049 
1050  token = av_strtok(NULL, "+", &save);
1051  }
1052  }
1053 
1054 fail:
1055 end:
1056  av_free(sup_layers);
1057  av_free(user_layers_str);
1058 
1059  if (err < 0) {
1060  RELEASE_PROPS(enabled_layers, enabled_layers_count);
1061  } else {
1062  *dst = enabled_layers;
1063  *num = enabled_layers_count;
1064  }
1065 
1066  return err;
1067 }
1068 
1069 /* Creates a VkInstance */
1070 static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts,
1071  enum FFVulkanDebugMode *debug_mode)
1072 {
1073  int err = 0;
1074  VkResult ret;
1075  VulkanDevicePriv *p = ctx->hwctx;
1076  AVVulkanDeviceContext *hwctx = &p->p;
1077  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1078  VkApplicationInfo application_info = {
1079  .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
1080  .pApplicationName = "ffmpeg",
1081  .applicationVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1082  LIBAVUTIL_VERSION_MINOR,
1083  LIBAVUTIL_VERSION_MICRO),
1084  .pEngineName = "libavutil",
1085  .apiVersion = VK_API_VERSION_1_3,
1086  .engineVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1087  LIBAVUTIL_VERSION_MINOR,
1088  LIBAVUTIL_VERSION_MICRO),
1089  };
1090  VkValidationFeaturesEXT validation_features = {
1091  .sType = VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT,
1092  };
1093  VkInstanceCreateInfo inst_props = {
1094  .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
1095  .pApplicationInfo = &application_info,
1096  };
1097 
1098  if (!hwctx->get_proc_addr) {
1099  err = load_libvulkan(ctx);
1100  if (err < 0)
1101  return err;
1102  }
1103 
1104  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 0, 0);
1105  if (err < 0) {
1106  av_log(ctx, AV_LOG_ERROR, "Unable to load instance enumeration functions!\n");
1107  return err;
1108  }
1109 
1110  err = check_layers(ctx, opts, &inst_props.ppEnabledLayerNames,
1111  &inst_props.enabledLayerCount, debug_mode);
1112  if (err)
1113  goto fail;
1114 
1115  /* Check for present/missing extensions */
1116  err = check_extensions(ctx, 0, opts, &inst_props.ppEnabledExtensionNames,
1117  &inst_props.enabledExtensionCount, *debug_mode);
1118  hwctx->enabled_inst_extensions = inst_props.ppEnabledExtensionNames;
1119  hwctx->nb_enabled_inst_extensions = inst_props.enabledExtensionCount;
1120  if (err < 0)
1121  goto fail;
1122 
1123  /* Enable debug features if needed */
1124  if (*debug_mode == FF_VULKAN_DEBUG_VALIDATE) {
1125  static const VkValidationFeatureEnableEXT feat_list_validate[] = {
1126  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1127  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1128  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_EXT,
1129  };
1130  validation_features.pEnabledValidationFeatures = feat_list_validate;
1131  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_validate);
1132  inst_props.pNext = &validation_features;
1133  } else if (*debug_mode == FF_VULKAN_DEBUG_PRINTF) {
1134  static const VkValidationFeatureEnableEXT feat_list_debug[] = {
1135  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1136  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1137  VK_VALIDATION_FEATURE_ENABLE_DEBUG_PRINTF_EXT,
1138  };
1139  validation_features.pEnabledValidationFeatures = feat_list_debug;
1140  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_debug);
1141  inst_props.pNext = &validation_features;
1142  } else if (*debug_mode == FF_VULKAN_DEBUG_PRACTICES) {
1143  static const VkValidationFeatureEnableEXT feat_list_practices[] = {
1144  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1145  VK_VALIDATION_FEATURE_ENABLE_BEST_PRACTICES_EXT,
1146  };
1147  validation_features.pEnabledValidationFeatures = feat_list_practices;
1148  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_practices);
1149  inst_props.pNext = &validation_features;
1150  }
1151 
1152 #ifdef __APPLE__
1153  for (int i = 0; i < inst_props.enabledExtensionCount; i++) {
1154  if (!strcmp(VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME,
1155  inst_props.ppEnabledExtensionNames[i])) {
1156  inst_props.flags |= VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR;
1157  break;
1158  }
1159  }
1160 #endif
1161 
1162  /* Try to create the instance */
1163  ret = vk->CreateInstance(&inst_props, hwctx->alloc, &hwctx->inst);
1164 
1165  /* Check for errors */
1166  if (ret != VK_SUCCESS) {
1167  av_log(ctx, AV_LOG_ERROR, "Instance creation failure: %s\n",
1168  ff_vk_ret2str(ret));
1169  err = AVERROR_EXTERNAL;
1170  goto fail;
1171  }
1172 
1173  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 0);
1174  if (err < 0) {
1175  av_log(ctx, AV_LOG_ERROR, "Unable to load instance functions!\n");
1176  goto fail;
1177  }
1178 
1179  /* Setup debugging callback if needed */
1180  if ((*debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
1181  (*debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
1182  (*debug_mode == FF_VULKAN_DEBUG_PRACTICES)) {
1183  VkDebugUtilsMessengerCreateInfoEXT dbg = {
1184  .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
1185  .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT |
1186  VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
1187  VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
1188  VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
1189  .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
1190  VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
1191  VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
1192  .pfnUserCallback = vk_dbg_callback,
1193  .pUserData = ctx,
1194  };
1195 
1196  vk->CreateDebugUtilsMessengerEXT(hwctx->inst, &dbg,
1197  hwctx->alloc, &p->debug_ctx);
1198  }
1199 
1200  err = 0;
1201 
1202 fail:
1203  RELEASE_PROPS(inst_props.ppEnabledLayerNames, inst_props.enabledLayerCount);
1204  return err;
1205 }
1206 
1207 typedef struct VulkanDeviceSelection {
1208  uint8_t uuid[VK_UUID_SIZE]; /* Will use this first unless !has_uuid */
1209  int has_uuid;
1210  uint32_t drm_major; /* Will use this second unless !has_drm */
1211  uint32_t drm_minor; /* Will use this second unless !has_drm */
1212  uint32_t has_drm; /* has drm node info */
1213  const char *name; /* Will use this third unless NULL */
1214  uint32_t pci_device; /* Will use this fourth unless 0x0 */
1215  uint32_t vendor_id; /* Last resort to find something deterministic */
1216  int index; /* Finally fall back to index */
1217 } VulkanDeviceSelection;
1218 
1219 static const char *vk_dev_type(enum VkPhysicalDeviceType type)
1220 {
1221  switch (type) {
1222  case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "integrated";
1223  case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "discrete";
1224  case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "virtual";
1225  case VK_PHYSICAL_DEVICE_TYPE_CPU: return "software";
1226  default: return "unknown";
1227  }
1228 }
1229 
1230 /* Finds a device */
1231 static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
1232 {
1233  int err = 0, choice = -1;
1234  uint32_t num;
1235  VkResult ret;
1236  VulkanDevicePriv *p = ctx->hwctx;
1237  AVVulkanDeviceContext *hwctx = &p->p;
1238  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1239  VkPhysicalDevice *devices = NULL;
1240  VkPhysicalDeviceIDProperties *idp = NULL;
1241  VkPhysicalDeviceProperties2 *prop = NULL;
1242  VkPhysicalDeviceDriverProperties *driver_prop = NULL;
1243  VkPhysicalDeviceDrmPropertiesEXT *drm_prop = NULL;
1244 
1245  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, NULL);
1246  if (ret != VK_SUCCESS || !num) {
1247  av_log(ctx, AV_LOG_ERROR, "No devices found: %s!\n", ff_vk_ret2str(ret));
1248  return AVERROR(ENODEV);
1249  }
1250 
1251  devices = av_malloc_array(num, sizeof(VkPhysicalDevice));
1252  if (!devices)
1253  return AVERROR(ENOMEM);
1254 
1255  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, devices);
1256  if (ret != VK_SUCCESS) {
1257  av_log(ctx, AV_LOG_ERROR, "Failed enumerating devices: %s\n",
1258  ff_vk_ret2str(ret));
1259  err = AVERROR(ENODEV);
1260  goto end;
1261  }
1262 
1263  prop = av_calloc(num, sizeof(*prop));
1264  if (!prop) {
1265  err = AVERROR(ENOMEM);
1266  goto end;
1267  }
1268 
1269  idp = av_calloc(num, sizeof(*idp));
1270  if (!idp) {
1271  err = AVERROR(ENOMEM);
1272  goto end;
1273  }
1274 
1275  driver_prop = av_calloc(num, sizeof(*driver_prop));
1276  if (!driver_prop) {
1277  err = AVERROR(ENOMEM);
1278  goto end;
1279  }
1280 
1281  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1282  drm_prop = av_calloc(num, sizeof(*drm_prop));
1283  if (!drm_prop) {
1284  err = AVERROR(ENOMEM);
1285  goto end;
1286  }
1287  }
1288 
1289  av_log(ctx, AV_LOG_VERBOSE, "GPU listing:\n");
1290  for (int i = 0; i < num; i++) {
1291  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1292  drm_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT;
1293  driver_prop[i].pNext = &drm_prop[i];
1294  }
1295  driver_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
1296  idp[i].pNext = &driver_prop[i];
1297  idp[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES;
1298  prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1299  prop[i].pNext = &idp[i];
1300 
1301  vk->GetPhysicalDeviceProperties2(devices[i], &prop[i]);
1302  av_log(ctx, AV_LOG_VERBOSE, " %d: %s (%s) (0x%x)\n", i,
1303  prop[i].properties.deviceName,
1304  vk_dev_type(prop[i].properties.deviceType),
1305  prop[i].properties.deviceID);
1306  }
1307 
1308  if (select->has_uuid) {
1309  for (int i = 0; i < num; i++) {
1310  if (!strncmp(idp[i].deviceUUID, select->uuid, VK_UUID_SIZE)) {
1311  choice = i;
1312  goto end;
1313  }
1314  }
1315  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given UUID!\n");
1316  err = AVERROR(ENODEV);
1317  goto end;
1318  } else if ((p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) && select->has_drm) {
1319  for (int i = 0; i < num; i++) {
1320  if ((select->drm_major == drm_prop[i].primaryMajor &&
1321  select->drm_minor == drm_prop[i].primaryMinor) ||
1322  (select->drm_major == drm_prop[i].renderMajor &&
1323  select->drm_minor == drm_prop[i].renderMinor)) {
1324  choice = i;
1325  goto end;
1326  }
1327  }
1328  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given DRM node numbers %i:%i!\n",
1329  select->drm_major, select->drm_minor);
1330  err = AVERROR(ENODEV);
1331  goto end;
1332  } else if (select->name) {
1333  av_log(ctx, AV_LOG_VERBOSE, "Requested device: %s\n", select->name);
1334  for (int i = 0; i < num; i++) {
1335  if (strstr(prop[i].properties.deviceName, select->name)) {
1336  choice = i;
1337  goto end;
1338  }
1339  }
1340  av_log(ctx, AV_LOG_ERROR, "Unable to find device \"%s\"!\n",
1341  select->name);
1342  err = AVERROR(ENODEV);
1343  goto end;
1344  } else if (select->pci_device) {
1345  av_log(ctx, AV_LOG_VERBOSE, "Requested device: 0x%x\n", select->pci_device);
1346  for (int i = 0; i < num; i++) {
1347  if (select->pci_device == prop[i].properties.deviceID) {
1348  choice = i;
1349  goto end;
1350  }
1351  }
1352  av_log(ctx, AV_LOG_ERROR, "Unable to find device with PCI ID 0x%x!\n",
1353  select->pci_device);
1354  err = AVERROR(EINVAL);
1355  goto end;
1356  } else if (select->vendor_id) {
1357  av_log(ctx, AV_LOG_VERBOSE, "Requested vendor: 0x%x\n", select->vendor_id);
1358  for (int i = 0; i < num; i++) {
1359  if (select->vendor_id == prop[i].properties.vendorID) {
1360  choice = i;
1361  goto end;
1362  }
1363  }
1364  av_log(ctx, AV_LOG_ERROR, "Unable to find device with Vendor ID 0x%x!\n",
1365  select->vendor_id);
1366  err = AVERROR(ENODEV);
1367  goto end;
1368  } else {
1369  if (select->index < num) {
1370  choice = select->index;
1371  goto end;
1372  }
1373  av_log(ctx, AV_LOG_ERROR, "Unable to find device with index %i!\n",
1374  select->index);
1375  err = AVERROR(ENODEV);
1376  goto end;
1377  }
1378 
1379 end:
1380  if (choice > -1) {
1381  av_log(ctx, AV_LOG_VERBOSE, "Device %d selected: %s (%s) (0x%x)\n",
1382  choice, prop[choice].properties.deviceName,
1383  vk_dev_type(prop[choice].properties.deviceType),
1384  prop[choice].properties.deviceID);
1385  hwctx->phys_dev = devices[choice];
1386  p->props = prop[choice];
1387  p->props.pNext = NULL;
1388  p->dprops = driver_prop[choice];
1389  p->dprops.pNext = NULL;
1390  }
1391 
1392  av_free(devices);
1393  av_free(prop);
1394  av_free(idp);
1395  av_free(drm_prop);
1396  av_free(driver_prop);
1397 
1398  return err;
1399 }
1400 
1401 /* Picks the least used qf with the fewest unneeded flags, or -1 if none found */
1402 static inline int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf,
1403  VkQueueFlagBits flags)
1404 {
1405  int index = -1;
1406  uint32_t min_score = UINT32_MAX;
1407 
1408  for (int i = 0; i < num_qf; i++) {
1409  VkQueueFlagBits qflags = qf[i].queueFamilyProperties.queueFlags;
1410 
1411  /* Per the spec, reporting transfer caps is optional for these 2 types */
1412  if ((flags & VK_QUEUE_TRANSFER_BIT) &&
1413  (qflags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT)))
1414  qflags |= VK_QUEUE_TRANSFER_BIT;
1415 
1416  if (qflags & flags) {
1417  uint32_t score = av_popcount(qflags) + qf[i].queueFamilyProperties.timestampValidBits;
1418  if (score < min_score) {
1419  index = i;
1420  min_score = score;
1421  }
1422  }
1423  }
1424 
1425  if (index > -1)
1426  qf[index].queueFamilyProperties.timestampValidBits++;
1427 
1428  return index;
1429 }
1430 
1431 static inline int pick_video_queue_family(VkQueueFamilyProperties2 *qf,
1432  VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf,
1433  VkVideoCodecOperationFlagBitsKHR flags)
1434 {
1435  int index = -1;
1436  uint32_t min_score = UINT32_MAX;
1437 
1438  for (int i = 0; i < num_qf; i++) {
1439  const VkQueueFlagBits qflags = qf[i].queueFamilyProperties.queueFlags;
1440  const VkQueueFlagBits vflags = qf_vid[i].videoCodecOperations;
1441 
1442  if (!(qflags & (VK_QUEUE_VIDEO_ENCODE_BIT_KHR | VK_QUEUE_VIDEO_DECODE_BIT_KHR)))
1443  continue;
1444 
1445  if (vflags & flags) {
1446  uint32_t score = av_popcount(vflags) + qf[i].queueFamilyProperties.timestampValidBits;
1447  if (score < min_score) {
1448  index = i;
1449  min_score = score;
1450  }
1451  }
1452  }
1453 
1454  if (index > -1)
1455  qf[index].queueFamilyProperties.timestampValidBits++;
1456 
1457  return index;
1458 }
1459 
1460 static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
1461 {
1462  uint32_t num;
1463  VulkanDevicePriv *p = ctx->hwctx;
1464  AVVulkanDeviceContext *hwctx = &p->p;
1465  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1466 
1467  VkQueueFamilyProperties2 *qf = NULL;
1468  VkQueueFamilyVideoPropertiesKHR *qf_vid = NULL;
1469 
1470  /* First get the number of queue families */
1471  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &num, NULL);
1472  if (!num) {
1473  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1474  return AVERROR_EXTERNAL;
1475  }
1476 
1477  /* Then allocate memory */
1478  qf = av_malloc_array(num, sizeof(VkQueueFamilyProperties2));
1479  if (!qf)
1480  return AVERROR(ENOMEM);
1481 
1482  qf_vid = av_malloc_array(num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1483  if (!qf_vid)
1484  return AVERROR(ENOMEM);
1485 
1486  for (uint32_t i = 0; i < num; i++) {
1487  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1488  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1489  };
1490  qf[i] = (VkQueueFamilyProperties2) {
1491  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1492  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1493  };
1494  }
1495 
1496  /* Finally retrieve the queue families */
1497  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &num, qf);
1498 
1499  av_log(ctx, AV_LOG_VERBOSE, "Queue families:\n");
1500  for (int i = 0; i < num; i++) {
1501  av_log(ctx, AV_LOG_VERBOSE, " %i:%s%s%s%s%s%s%s%s (queues: %i)\n", i,
1502  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_GRAPHICS_BIT) ? " graphics" : "",
1503  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_COMPUTE_BIT) ? " compute" : "",
1504  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_TRANSFER_BIT) ? " transfer" : "",
1505  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_ENCODE_BIT_KHR) ? " encode" : "",
1506  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_DECODE_BIT_KHR) ? " decode" : "",
1507  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_SPARSE_BINDING_BIT) ? " sparse" : "",
1508  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_OPTICAL_FLOW_BIT_NV) ? " optical_flow" : "",
1509  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_PROTECTED_BIT) ? " protected" : "",
1510  qf[i].queueFamilyProperties.queueCount);
1511 
1512  /* We use this field to keep a score of how many times we've used that
1513  * queue family in order to make better choices. */
1514  qf[i].queueFamilyProperties.timestampValidBits = 0;
1515  }
1516 
1517  hwctx->nb_qf = 0;
1518 
1519  /* Pick each queue family to use. */
1520 #define PICK_QF(type, vid_op) \
1521  do { \
1522  uint32_t i; \
1523  uint32_t idx; \
1524  \
1525  if (vid_op) \
1526  idx = pick_video_queue_family(qf, qf_vid, num, vid_op); \
1527  else \
1528  idx = pick_queue_family(qf, num, type); \
1529  \
1530  if (idx == -1) \
1531  continue; \
1532  \
1533  for (i = 0; i < hwctx->nb_qf; i++) { \
1534  if (hwctx->qf[i].idx == idx) { \
1535  hwctx->qf[i].flags |= type; \
1536  hwctx->qf[i].video_caps |= vid_op; \
1537  break; \
1538  } \
1539  } \
1540  if (i == hwctx->nb_qf) { \
1541  hwctx->qf[i].idx = idx; \
1542  hwctx->qf[i].num = qf[idx].queueFamilyProperties.queueCount; \
1543  if (p->limit_queues || \
1544  p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY) { \
1545  int max = p->limit_queues; \
1546  if (type == VK_QUEUE_GRAPHICS_BIT) \
1547  hwctx->qf[i].num = FFMIN(hwctx->qf[i].num, \
1548  max ? max : 1); \
1549  else if (max) \
1550  hwctx->qf[i].num = FFMIN(hwctx->qf[i].num, max); \
1551  } \
1552  hwctx->qf[i].flags = type; \
1553  hwctx->qf[i].video_caps = vid_op; \
1554  hwctx->nb_qf++; \
1555  } \
1556  } while (0)
1557 
1558  PICK_QF(VK_QUEUE_GRAPHICS_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1559  PICK_QF(VK_QUEUE_COMPUTE_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1560  PICK_QF(VK_QUEUE_TRANSFER_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1561 
1562  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR);
1563  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR);
1564 
1565  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR);
1566  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR);
1567 
1568 #ifdef VK_KHR_video_decode_vp9
1569  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_VP9_BIT_KHR);
1570 #endif
1571 
1572 #ifdef VK_KHR_video_encode_av1
1573  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_AV1_BIT_KHR);
1574 #endif
1575  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR);
1576 
1577  av_free(qf);
1578  av_free(qf_vid);
1579 
1580 #undef PICK_QF
1581 
1582  cd->pQueueCreateInfos = av_malloc_array(hwctx->nb_qf,
1583  sizeof(VkDeviceQueueCreateInfo));
1584  if (!cd->pQueueCreateInfos)
1585  return AVERROR(ENOMEM);
1586 
1587  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1588  int dup = 0;
1589  float *weights = NULL;
1590  VkDeviceQueueCreateInfo *pc;
1591  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++) {
1592  if (hwctx->qf[i].idx == cd->pQueueCreateInfos[j].queueFamilyIndex) {
1593  dup = 1;
1594  break;
1595  }
1596  }
1597  if (dup)
1598  continue;
1599 
1600  weights = av_malloc_array(hwctx->qf[i].num, sizeof(float));
1601  if (!weights) {
1602  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++)
1603  av_free((void *)cd->pQueueCreateInfos[i].pQueuePriorities);
1604  av_free((void *)cd->pQueueCreateInfos);
1605  return AVERROR(ENOMEM);
1606  }
1607 
1608  for (uint32_t j = 0; j < hwctx->qf[i].num; j++)
1609  weights[j] = 1.0;
1610 
1611  pc = (VkDeviceQueueCreateInfo *)cd->pQueueCreateInfos;
1612  pc[cd->queueCreateInfoCount++] = (VkDeviceQueueCreateInfo) {
1613  .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
1614  .queueFamilyIndex = hwctx->qf[i].idx,
1615  .queueCount = hwctx->qf[i].num,
1616  .pQueuePriorities = weights,
1617  };
1618  }
1619 
1620 #if FF_API_VULKAN_FIXED_QUEUES
1621 FF_DISABLE_DEPRECATION_WARNINGS
1622  /* Setup deprecated fields */
1623  hwctx->queue_family_index = -1;
1624  hwctx->queue_family_comp_index = -1;
1625  hwctx->queue_family_tx_index = -1;
1626  hwctx->queue_family_encode_index = -1;
1627  hwctx->queue_family_decode_index = -1;
1628 
1629 #define SET_OLD_QF(field, nb_field, type) \
1630  do { \
1631  if (field < 0 && hwctx->qf[i].flags & type) { \
1632  field = hwctx->qf[i].idx; \
1633  nb_field = hwctx->qf[i].num; \
1634  } \
1635  } while (0)
1636 
1637  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1638  SET_OLD_QF(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
1639  SET_OLD_QF(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
1640  SET_OLD_QF(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
1641  SET_OLD_QF(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
1642  SET_OLD_QF(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
1643  }
1644 
1645 #undef SET_OLD_QF
1646 FF_ENABLE_DEPRECATION_WARNINGS
1647 #endif
1648 
1649  return 0;
1650 }
1651 
1652 /* Only resources created by vulkan_device_create should be released here,
1653  * resources created by vulkan_device_init should be released by
1654  * vulkan_device_uninit, to make sure we don't free user provided resources,
1655  * and there is no leak.
1656  */
1657 static void vulkan_device_free(AVHWDeviceContext *ctx)
1658 {
1659  VulkanDevicePriv *p = ctx->hwctx;
1660  AVVulkanDeviceContext *hwctx = &p->p;
1661  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1662 
1663  if (hwctx->act_dev)
1664  vk->DestroyDevice(hwctx->act_dev, hwctx->alloc);
1665 
1666  if (p->debug_ctx)
1667  vk->DestroyDebugUtilsMessengerEXT(hwctx->inst, p->debug_ctx,
1668  hwctx->alloc);
1669 
1670  if (hwctx->inst)
1671  vk->DestroyInstance(hwctx->inst, hwctx->alloc);
1672 
1673  if (p->libvulkan)
1674  dlclose(p->libvulkan);
1675 
1676  RELEASE_PROPS(hwctx->enabled_inst_extensions, hwctx->nb_enabled_inst_extensions);
1677  RELEASE_PROPS(hwctx->enabled_dev_extensions, hwctx->nb_enabled_dev_extensions);
1678 }
1679 
1680 static void vulkan_device_uninit(AVHWDeviceContext *ctx)
1681 {
1682  VulkanDevicePriv *p = ctx->hwctx;
1683 
1684  for (uint32_t i = 0; i < p->nb_tot_qfs; i++) {
1685  pthread_mutex_destroy(p->qf_mutex[i]);
1686  av_freep(&p->qf_mutex[i]);
1687  }
1688  av_freep(&p->qf_mutex);
1689 
1690  ff_vk_uninit(&p->vkctx);
1691 }
1692 
1693 static int vulkan_device_has_rebar(AVHWDeviceContext *ctx)
1694 {
1695  VulkanDevicePriv *p = ctx->hwctx;
1696  VkDeviceSize max_vram = 0, max_visible_vram = 0;
1697 
1698  /* Get device memory properties */
1699  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
1700  const VkMemoryType type = p->mprops.memoryTypes[i];
1701  const VkMemoryHeap heap = p->mprops.memoryHeaps[type.heapIndex];
1702  if (!(type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
1703  continue;
1704  max_vram = FFMAX(max_vram, heap.size);
1705  if (type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
1706  max_visible_vram = FFMAX(max_visible_vram, heap.size);
1707  }
1708 
1709  return max_vram - max_visible_vram < 1024; /* 1 kB tolerance */
1710 }
1711 
1712 static int vulkan_device_create_internal(AVHWDeviceContext *ctx,
1713  VulkanDeviceSelection *dev_select,
1714  int disable_multiplane,
1715  AVDictionary *opts, int flags)
1716 {
1717  int err = 0;
1718  VkResult ret;
1719  AVDictionaryEntry *opt_d;
1720  VulkanDevicePriv *p = ctx->hwctx;
1721  AVVulkanDeviceContext *hwctx = &p->p;
1722  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1723  enum FFVulkanDebugMode debug_mode = FF_VULKAN_DEBUG_NONE;
1724  VulkanDeviceFeatures supported_feats = { 0 };
1725  VkDeviceCreateInfo dev_info = {
1726  .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
1727  };
1728 
1729  /* Create an instance if not given one */
1730  if ((err = create_instance(ctx, opts, &debug_mode)))
1731  goto end;
1732 
1733  /* Find a physical device (if not given one) */
1734  if ((err = find_device(ctx, dev_select)))
1735  goto end;
1736 
1737  /* Find and enable extensions for the physical device */
1738  if ((err = check_extensions(ctx, 1, opts, &dev_info.ppEnabledExtensionNames,
1739  &dev_info.enabledExtensionCount, debug_mode))) {
1740  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1741  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1742  av_free((void *)dev_info.pQueueCreateInfos);
1743  goto end;
1744  }
1745 
1746  /* Get supported memory types */
1747  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
1748 
1749  /* Get all supported features for the physical device */
1750  device_features_init(ctx, &supported_feats);
1751  vk->GetPhysicalDeviceFeatures2(hwctx->phys_dev, &supported_feats.device);
1752 
1753  /* Copy all needed features from those supported and activate them */
1754  device_features_init(ctx, &p->feats);
1755  device_features_copy_needed(&p->feats, &supported_feats);
1756  dev_info.pNext = p->feats.device.pNext;
1757  dev_info.pEnabledFeatures = &p->feats.device.features;
1758 
1759  /* Setup enabled queue families */
1760  if ((err = setup_queue_families(ctx, &dev_info)))
1761  goto end;
1762 
1763  /* Finally create the device */
1764  ret = vk->CreateDevice(hwctx->phys_dev, &dev_info, hwctx->alloc,
1765  &hwctx->act_dev);
1766 
1767  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1768  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1769  av_free((void *)dev_info.pQueueCreateInfos);
1770 
1771  if (ret != VK_SUCCESS) {
1772  av_log(ctx, AV_LOG_ERROR, "Device creation failure: %s\n",
1773  ff_vk_ret2str(ret));
1774  for (int i = 0; i < dev_info.enabledExtensionCount; i++)
1775  av_free((void *)dev_info.ppEnabledExtensionNames[i]);
1776  av_free((void *)dev_info.ppEnabledExtensionNames);
1777  err = AVERROR_EXTERNAL;
1778  goto end;
1779  }
1780 
1781  /* Tiled images setting, use them by default */
1782  opt_d = av_dict_get(opts, "linear_images", NULL, 0);
1783  if (opt_d)
1784  p->use_linear_images = strtol(opt_d->value, NULL, 10);
1785 
1786  /* Limit queues to a given number if needed */
1787  opt_d = av_dict_get(opts, "limit_queues", NULL, 0);
1788  if (opt_d)
1789  p->limit_queues = strtol(opt_d->value, NULL, 10);
1790 
1791  /* The disable_multiplane argument takes precedent over the option */
1792  p->disable_multiplane = disable_multiplane;
1793  if (!p->disable_multiplane) {
1794  opt_d = av_dict_get(opts, "disable_multiplane", NULL, 0);
1795  if (opt_d)
1796  p->disable_multiplane = strtol(opt_d->value, NULL, 10);
1797  }
1798 
1799  /* Disable host pointer imports (by default on nvidia) */
1800  p->avoid_host_import = p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY;
1801  opt_d = av_dict_get(opts, "avoid_host_import", NULL, 0);
1802  if (opt_d)
1803  p->avoid_host_import = strtol(opt_d->value, NULL, 10);
1804 
1805  /* Set the public device feature struct and its pNext chain */
1806  hwctx->device_features = p->feats.device;
1807 
1808  /* Set the list of all active extensions */
1809  hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames;
1810  hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount;
1811 
1812  /* The extension lists need to be freed */
1813  ctx->free = vulkan_device_free;
1814 
1815 end:
1816  return err;
1817 }
1818 
1819 static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1820 {
1821  VulkanDevicePriv *p = ctx->hwctx;
1822  pthread_mutex_lock(&p->qf_mutex[queue_family][index]);
1823 }
1824 
1825 static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1826 {
1827  VulkanDevicePriv *p = ctx->hwctx;
1828  pthread_mutex_unlock(&p->qf_mutex[queue_family][index]);
1829 }
1830 
1831 static int vulkan_device_init(AVHWDeviceContext *ctx)
1832 {
1833  int err = 0;
1834  uint32_t qf_num;
1835  VulkanDevicePriv *p = ctx->hwctx;
1836  AVVulkanDeviceContext *hwctx = &p->p;
1837  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1838  VkQueueFamilyProperties2 *qf;
1839  VkQueueFamilyVideoPropertiesKHR *qf_vid;
1840  VkPhysicalDeviceExternalSemaphoreInfo ext_sem_props_info;
1841  int graph_index, comp_index, tx_index, enc_index, dec_index;
1842 
1843  /* Set device extension flags */
1844  for (int i = 0; i < hwctx->nb_enabled_dev_extensions; i++) {
1845  for (int j = 0; j < FF_ARRAY_ELEMS(optional_device_exts); j++) {
1846  if (!strcmp(hwctx->enabled_dev_extensions[i],
1847  optional_device_exts[j].name)) {
1848  p->vkctx.extensions |= optional_device_exts[j].flag;
1849  break;
1850  }
1851  }
1852  }
1853 
1854  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 1);
1855  if (err < 0) {
1856  av_log(ctx, AV_LOG_ERROR, "Unable to load functions!\n");
1857  return err;
1858  }
1859 
1860  p->props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1861  p->props.pNext = &p->hprops;
1862  p->hprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT;
1863  p->hprops.pNext = &p->dprops;
1864  p->dprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
1865 
1866  vk->GetPhysicalDeviceProperties2(hwctx->phys_dev, &p->props);
1867  av_log(ctx, AV_LOG_VERBOSE, "Using device: %s\n",
1868  p->props.properties.deviceName);
1869  av_log(ctx, AV_LOG_VERBOSE, "Alignments:\n");
1870  av_log(ctx, AV_LOG_VERBOSE, " optimalBufferCopyRowPitchAlignment: %"PRIu64"\n",
1871  p->props.properties.limits.optimalBufferCopyRowPitchAlignment);
1872  av_log(ctx, AV_LOG_VERBOSE, " minMemoryMapAlignment: %"SIZE_SPECIFIER"\n",
1873  p->props.properties.limits.minMemoryMapAlignment);
1874  av_log(ctx, AV_LOG_VERBOSE, " nonCoherentAtomSize: %"PRIu64"\n",
1875  p->props.properties.limits.nonCoherentAtomSize);
1876  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY)
1877  av_log(ctx, AV_LOG_VERBOSE, " minImportedHostPointerAlignment: %"PRIu64"\n",
1878  p->hprops.minImportedHostPointerAlignment);
1879 
1880  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &qf_num, NULL);
1881  if (!qf_num) {
1882  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1883  return AVERROR_EXTERNAL;
1884  }
1885 
1886  ext_sem_props_info = (VkPhysicalDeviceExternalSemaphoreInfo) {
1887  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO,
1888  };
1889 
1890  /* Opaque FD semaphore properties */
1891  ext_sem_props_info.handleType =
1892 #ifdef _WIN32
1893  IsWindows8OrGreater()
1894  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
1895  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT;
1896 #else
1897  VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
1898 #endif
1899  p->ext_sem_props_opaque.sType = VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES;
1900  vk->GetPhysicalDeviceExternalSemaphoreProperties(hwctx->phys_dev,
1901  &ext_sem_props_info,
1902  &p->ext_sem_props_opaque);
1903 
1904  qf = av_malloc_array(qf_num, sizeof(VkQueueFamilyProperties2));
1905  if (!qf)
1906  return AVERROR(ENOMEM);
1907 
1908  qf_vid = av_malloc_array(qf_num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1909  if (!qf_vid) {
1910  av_free(qf);
1911  return AVERROR(ENOMEM);
1912  }
1913 
1914  for (uint32_t i = 0; i < qf_num; i++) {
1915  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1916  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1917  };
1918  qf[i] = (VkQueueFamilyProperties2) {
1919  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1920  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1921  };
1922  }
1923 
1924  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &qf_num, qf);
1925 
1926  p->qf_mutex = av_calloc(qf_num, sizeof(*p->qf_mutex));
1927  if (!p->qf_mutex) {
1928  err = AVERROR(ENOMEM);
1929  goto end;
1930  }
1931  p->nb_tot_qfs = qf_num;
1932 
1933  for (uint32_t i = 0; i < qf_num; i++) {
1934  p->qf_mutex[i] = av_calloc(qf[i].queueFamilyProperties.queueCount,
1935  sizeof(**p->qf_mutex));
1936  if (!p->qf_mutex[i]) {
1937  err = AVERROR(ENOMEM);
1938  goto end;
1939  }
1940  for (uint32_t j = 0; j < qf[i].queueFamilyProperties.queueCount; j++) {
1941  err = pthread_mutex_init(&p->qf_mutex[i][j], NULL);
1942  if (err != 0) {
1943  av_log(ctx, AV_LOG_ERROR, "pthread_mutex_init failed : %s\n",
1944  av_err2str(err));
1945  err = AVERROR(err);
1946  goto end;
1947  }
1948  }
1949  }
1950 
1951 #if FF_API_VULKAN_FIXED_QUEUES
1952 FF_DISABLE_DEPRECATION_WARNINGS
1953  graph_index = hwctx->nb_graphics_queues ? hwctx->queue_family_index : -1;
1954  comp_index = hwctx->nb_comp_queues ? hwctx->queue_family_comp_index : -1;
1955  tx_index = hwctx->nb_tx_queues ? hwctx->queue_family_tx_index : -1;
1956  dec_index = hwctx->nb_decode_queues ? hwctx->queue_family_decode_index : -1;
1957  enc_index = hwctx->nb_encode_queues ? hwctx->queue_family_encode_index : -1;
1958 
1959 #define CHECK_QUEUE(type, required, fidx, ctx_qf, qc) \
1960  do { \
1961  if (ctx_qf < 0 && required) { \
1962  av_log(ctx, AV_LOG_ERROR, "%s queue family is required, but marked as missing" \
1963  " in the context!\n", type); \
1964  err = AVERROR(EINVAL); \
1965  goto end; \
1966  } else if (fidx < 0 || ctx_qf < 0) { \
1967  break; \
1968  } else if (ctx_qf >= qf_num) { \
1969  av_log(ctx, AV_LOG_ERROR, "Invalid %s family index %i (device has %i families)!\n", \
1970  type, ctx_qf, qf_num); \
1971  err = AVERROR(EINVAL); \
1972  goto end; \
1973  } \
1974  \
1975  av_log(ctx, AV_LOG_VERBOSE, "Using queue family %i (queues: %i)" \
1976  " for%s%s%s%s%s\n", \
1977  ctx_qf, qc, \
1978  ctx_qf == graph_index ? " graphics" : "", \
1979  ctx_qf == comp_index ? " compute" : "", \
1980  ctx_qf == tx_index ? " transfers" : "", \
1981  ctx_qf == enc_index ? " encode" : "", \
1982  ctx_qf == dec_index ? " decode" : ""); \
1983  graph_index = (ctx_qf == graph_index) ? -1 : graph_index; \
1984  comp_index = (ctx_qf == comp_index) ? -1 : comp_index; \
1985  tx_index = (ctx_qf == tx_index) ? -1 : tx_index; \
1986  enc_index = (ctx_qf == enc_index) ? -1 : enc_index; \
1987  dec_index = (ctx_qf == dec_index) ? -1 : dec_index; \
1988  } while (0)
1989 
1990  CHECK_QUEUE("graphics", 0, graph_index, hwctx->queue_family_index, hwctx->nb_graphics_queues);
1991  CHECK_QUEUE("compute", 1, comp_index, hwctx->queue_family_comp_index, hwctx->nb_comp_queues);
1992  CHECK_QUEUE("upload", 1, tx_index, hwctx->queue_family_tx_index, hwctx->nb_tx_queues);
1993  CHECK_QUEUE("decode", 0, dec_index, hwctx->queue_family_decode_index, hwctx->nb_decode_queues);
1994  CHECK_QUEUE("encode", 0, enc_index, hwctx->queue_family_encode_index, hwctx->nb_encode_queues);
1995 
1996 #undef CHECK_QUEUE
1997 
1998  /* Update the new queue family fields. If non-zero already,
1999  * it means API users have set it. */
2000  if (!hwctx->nb_qf) {
2001 #define ADD_QUEUE(ctx_qf, qc, flag) \
2002  do { \
2003  if (ctx_qf != -1) { \
2004  hwctx->qf[hwctx->nb_qf++] = (AVVulkanDeviceQueueFamily) { \
2005  .idx = ctx_qf, \
2006  .num = qc, \
2007  .flags = flag, \
2008  }; \
2009  } \
2010  } while (0)
2011 
2012  ADD_QUEUE(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
2013  ADD_QUEUE(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
2014  ADD_QUEUE(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
2015  ADD_QUEUE(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
2016  ADD_QUEUE(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
2017 #undef ADD_QUEUE
2018  }
2019 FF_ENABLE_DEPRECATION_WARNINGS
2020 #endif
2021 
2022  for (int i = 0; i < hwctx->nb_qf; i++) {
2023  if (!hwctx->qf[i].video_caps &&
2024  hwctx->qf[i].flags & (VK_QUEUE_VIDEO_DECODE_BIT_KHR |
2025  VK_QUEUE_VIDEO_ENCODE_BIT_KHR)) {
2026  hwctx->qf[i].video_caps = qf_vid[hwctx->qf[i].idx].videoCodecOperations;
2027  }
2028  }
2029 
2030  /* Setup array for pQueueFamilyIndices with used queue families */
2031  p->nb_img_qfs = 0;
2032  for (int i = 0; i < hwctx->nb_qf; i++) {
2033  int seen = 0;
2034  /* Make sure each entry is unique
2035  * (VUID-VkBufferCreateInfo-sharingMode-01419) */
2036  for (int j = (i - 1); j >= 0; j--) {
2037  if (hwctx->qf[i].idx == hwctx->qf[j].idx) {
2038  seen = 1;
2039  break;
2040  }
2041  }
2042  if (!seen)
2043  p->img_qfs[p->nb_img_qfs++] = hwctx->qf[i].idx;
2044  }
2045 
2046  if (!hwctx->lock_queue)
2047  hwctx->lock_queue = lock_queue;
2048  if (!hwctx->unlock_queue)
2049  hwctx->unlock_queue = unlock_queue;
2050 
2051  /* Re-query device capabilities, in case the device was created externally */
2052  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
2053 
2054  p->vkctx.device = ctx;
2055  p->vkctx.hwctx = hwctx;
2056 
2057  ff_vk_load_props(&p->vkctx);
2058  p->compute_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_COMPUTE_BIT, 0);
2059  p->transfer_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_TRANSFER_BIT, 0);
2060 
2061  /* Re-query device capabilities, in case the device was created externally */
2062  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
2063 
2064  /* Only use host image transfers if ReBAR is enabled */
2065  p->disable_host_transfer = !vulkan_device_has_rebar(ctx);
2066 
2067 end:
2068  av_free(qf_vid);
2069  av_free(qf);
2070  return err;
2071 }
2072 
2073 static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device,
2074  AVDictionary *opts, int flags)
2075 {
2076  VulkanDeviceSelection dev_select = { 0 };
2077  if (device && device[0]) {
2078  char *end = NULL;
2079  dev_select.index = strtol(device, &end, 10);
2080  if (end == device) {
2081  dev_select.index = 0;
2082  dev_select.name = device;
2083  }
2084  }
2085 
2086  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2087 }
2088 
2089 static int vulkan_device_derive(AVHWDeviceContext *ctx,
2090  AVHWDeviceContext *src_ctx,
2091  AVDictionary *opts, int flags)
2092 {
2093  av_unused VulkanDeviceSelection dev_select = { 0 };
2094 
2095  /* If there's only one device on the system, then even if its not covered
2096  * by the following checks (e.g. non-PCIe ARM GPU), having an empty
2097  * dev_select will mean it'll get picked. */
2098  switch(src_ctx->type) {
2099 #if CONFIG_VAAPI
2100  case AV_HWDEVICE_TYPE_VAAPI: {
2101  AVVAAPIDeviceContext *src_hwctx = src_ctx->hwctx;
2102  VADisplay dpy = src_hwctx->display;
2103 #if VA_CHECK_VERSION(1, 15, 0)
2104  VAStatus vas;
2105  VADisplayAttribute attr = {
2106  .type = VADisplayPCIID,
2107  };
2108 #endif
2109  const char *vendor;
2110 
2111 #if VA_CHECK_VERSION(1, 15, 0)
2112  vas = vaGetDisplayAttributes(dpy, &attr, 1);
2113  if (vas == VA_STATUS_SUCCESS && attr.flags != VA_DISPLAY_ATTRIB_NOT_SUPPORTED)
2114  dev_select.pci_device = (attr.value & 0xFFFF);
2115 #endif
2116 
2117  if (!dev_select.pci_device) {
2118  vendor = vaQueryVendorString(dpy);
2119  if (!vendor) {
2120  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from VAAPI!\n");
2121  return AVERROR_EXTERNAL;
2122  }
2123 
2124  if (strstr(vendor, "AMD"))
2125  dev_select.vendor_id = 0x1002;
2126  }
2127 
2128  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2129  }
2130 #endif
2131 #if CONFIG_LIBDRM
2132  case AV_HWDEVICE_TYPE_DRM: {
2133  int err;
2134  struct stat drm_node_info;
2135  drmDevice *drm_dev_info;
2136  AVDRMDeviceContext *src_hwctx = src_ctx->hwctx;
2137 
2138  err = fstat(src_hwctx->fd, &drm_node_info);
2139  if (err) {
2140  av_log(ctx, AV_LOG_ERROR, "Unable to get node info from DRM fd: %s!\n",
2141  av_err2str(AVERROR(errno)));
2142  return AVERROR_EXTERNAL;
2143  }
2144 
2145  dev_select.drm_major = major(drm_node_info.st_dev);
2146  dev_select.drm_minor = minor(drm_node_info.st_dev);
2147  dev_select.has_drm = 1;
2148 
2149  err = drmGetDevice(src_hwctx->fd, &drm_dev_info);
2150  if (err) {
2151  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from DRM fd: %s!\n",
2152  av_err2str(AVERROR(errno)));
2153  return AVERROR_EXTERNAL;
2154  }
2155 
2156  if (drm_dev_info->bustype == DRM_BUS_PCI)
2157  dev_select.pci_device = drm_dev_info->deviceinfo.pci->device_id;
2158 
2159  drmFreeDevice(&drm_dev_info);
2160 
2161  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2162  }
2163 #endif
2164 #if CONFIG_CUDA
2165  case AV_HWDEVICE_TYPE_CUDA: {
2166  AVHWDeviceContext *cuda_cu = src_ctx;
2167  AVCUDADeviceContext *src_hwctx = src_ctx->hwctx;
2168  AVCUDADeviceContextInternal *cu_internal = src_hwctx->internal;
2169  CudaFunctions *cu = cu_internal->cuda_dl;
2170 
2171  int ret = CHECK_CU(cu->cuDeviceGetUuid((CUuuid *)&dev_select.uuid,
2172  cu_internal->cuda_device));
2173  if (ret < 0) {
2174  av_log(ctx, AV_LOG_ERROR, "Unable to get UUID from CUDA!\n");
2175  return AVERROR_EXTERNAL;
2176  }
2177 
2178  dev_select.has_uuid = 1;
2179 
2180  /*
2181  * CUDA is not able to import multiplane images, so always derive a
2182  * Vulkan device with multiplane disabled.
2183  */
2184  return vulkan_device_create_internal(ctx, &dev_select, 1, opts, flags);
2185  }
2186 #endif
2187  default:
2188  return AVERROR(ENOSYS);
2189  }
2190 }
2191 
2192 static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx,
2193  const void *hwconfig,
2194  AVHWFramesConstraints *constraints)
2195 {
2196  int count = 0;
2197  VulkanDevicePriv *p = ctx->hwctx;
2198 
2199  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2200  count += vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2201  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2202  VK_IMAGE_TILING_OPTIMAL,
2203  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0;
2204  }
2205 
2206  constraints->valid_sw_formats = av_malloc_array(count + 1,
2207  sizeof(enum AVPixelFormat));
2208  if (!constraints->valid_sw_formats)
2209  return AVERROR(ENOMEM);
2210 
2211  count = 0;
2212  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2213  if (vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2214  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2215  VK_IMAGE_TILING_OPTIMAL,
2216  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0) {
2217  constraints->valid_sw_formats[count++] = vk_formats_list[i].pixfmt;
2218  }
2219  }
2220 
2221  constraints->valid_sw_formats[count++] = AV_PIX_FMT_NONE;
2222 
2223  constraints->min_width = 1;
2224  constraints->min_height = 1;
2225  constraints->max_width = p->props.properties.limits.maxImageDimension2D;
2226  constraints->max_height = p->props.properties.limits.maxImageDimension2D;
2227 
2228  constraints->valid_hw_formats = av_malloc_array(2, sizeof(enum AVPixelFormat));
2229  if (!constraints->valid_hw_formats)
2230  return AVERROR(ENOMEM);
2231 
2232  constraints->valid_hw_formats[0] = AV_PIX_FMT_VULKAN;
2233  constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
2234 
2235  return 0;
2236 }
2237 
2238 static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req,
2239  VkMemoryPropertyFlagBits req_flags, const void *alloc_extension,
2240  VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
2241 {
2242  VkResult ret;
2243  int index = -1;
2244  VulkanDevicePriv *p = ctx->hwctx;
2245  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2246  AVVulkanDeviceContext *dev_hwctx = &p->p;
2247  VkMemoryAllocateInfo alloc_info = {
2248  .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
2249  .pNext = alloc_extension,
2250  .allocationSize = req->size,
2251  };
2252 
2253  /* The vulkan spec requires memory types to be sorted in the "optimal"
2254  * order, so the first matching type we find will be the best/fastest one */
2255  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
2256  const VkMemoryType *type = &p->mprops.memoryTypes[i];
2257 
2258  /* The memory type must be supported by the requirements (bitfield) */
2259  if (!(req->memoryTypeBits & (1 << i)))
2260  continue;
2261 
2262  /* The memory type flags must include our properties */
2263  if ((type->propertyFlags & req_flags) != req_flags)
2264  continue;
2265 
2266  /* The memory type must be large enough */
2267  if (req->size > p->mprops.memoryHeaps[type->heapIndex].size)
2268  continue;
2269 
2270  /* Found a suitable memory type */
2271  index = i;
2272  break;
2273  }
2274 
2275  if (index < 0) {
2276  av_log(ctx, AV_LOG_ERROR, "No memory type found for flags 0x%x\n",
2277  req_flags);
2278  return AVERROR(EINVAL);
2279  }
2280 
2281  alloc_info.memoryTypeIndex = index;
2282 
2283  ret = vk->AllocateMemory(dev_hwctx->act_dev, &alloc_info,
2284  dev_hwctx->alloc, mem);
2285  if (ret != VK_SUCCESS) {
2286  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory: %s\n",
2287  ff_vk_ret2str(ret));
2288  return AVERROR(ENOMEM);
2289  }
2290 
2291  *mem_flags |= p->mprops.memoryTypes[index].propertyFlags;
2292 
2293  return 0;
2294 }
2295 
2296 static void vulkan_free_internal(AVVkFrame *f)
2297 {
2298  av_unused AVVkFrameInternal *internal = f->internal;
2299 
2300 #if CONFIG_CUDA
2301  if (internal->cuda_fc_ref) {
2302  AVHWFramesContext *cuda_fc = (AVHWFramesContext *)internal->cuda_fc_ref->data;
2303  int planes = av_pix_fmt_count_planes(cuda_fc->sw_format);
2304  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
2305  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
2306  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
2307  CudaFunctions *cu = cu_internal->cuda_dl;
2308 
2309  for (int i = 0; i < planes; i++) {
2310  if (internal->cu_sem[i])
2311  CHECK_CU(cu->cuDestroyExternalSemaphore(internal->cu_sem[i]));
2312  if (internal->cu_mma[i])
2313  CHECK_CU(cu->cuMipmappedArrayDestroy(internal->cu_mma[i]));
2314  if (internal->ext_mem[i])
2315  CHECK_CU(cu->cuDestroyExternalMemory(internal->ext_mem[i]));
2316 #ifdef _WIN32
2317  if (internal->ext_sem_handle[i])
2318  CloseHandle(internal->ext_sem_handle[i]);
2319  if (internal->ext_mem_handle[i])
2320  CloseHandle(internal->ext_mem_handle[i]);
2321 #endif
2322  }
2323 
2324  av_buffer_unref(&internal->cuda_fc_ref);
2325  }
2326 #endif
2327 
2328  pthread_mutex_destroy(&internal->update_mutex);
2329  av_freep(&f->internal);
2330 }
2331 
2332 static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
2333 {
2334  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2335  AVVulkanDeviceContext *hwctx = &p->p;
2336  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2337  int nb_images = ff_vk_count_images(f);
2338  int nb_sems = 0;
2339 
2340  while (nb_sems < FF_ARRAY_ELEMS(f->sem) && f->sem[nb_sems])
2341  nb_sems++;
2342 
2343  if (nb_sems) {
2344  VkSemaphoreWaitInfo sem_wait = {
2345  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
2346  .flags = 0x0,
2347  .pSemaphores = f->sem,
2348  .pValues = f->sem_value,
2349  .semaphoreCount = nb_sems,
2350  };
2351 
2352  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
2353  }
2354 
2355  vulkan_free_internal(f);
2356 
2357  for (int i = 0; i < nb_images; i++) {
2358  vk->DestroyImage(hwctx->act_dev, f->img[i], hwctx->alloc);
2359  vk->FreeMemory(hwctx->act_dev, f->mem[i], hwctx->alloc);
2360  vk->DestroySemaphore(hwctx->act_dev, f->sem[i], hwctx->alloc);
2361  }
2362 
2363  av_free(f);
2364 }
2365 
2366 static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
2367 {
2368  vulkan_frame_free(opaque, (AVVkFrame*)data);
2369 }
2370 
2371 static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f,
2372  void *alloc_pnext, size_t alloc_pnext_stride)
2373 {
2374  int img_cnt = 0, err;
2375  VkResult ret;
2376  AVHWDeviceContext *ctx = hwfc->device_ctx;
2377  VulkanDevicePriv *p = ctx->hwctx;
2378  AVVulkanDeviceContext *hwctx = &p->p;
2379  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2380  VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } };
2381 
2382  while (f->img[img_cnt]) {
2383  int use_ded_mem;
2384  VkImageMemoryRequirementsInfo2 req_desc = {
2385  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
2386  .image = f->img[img_cnt],
2387  };
2388  VkMemoryDedicatedAllocateInfo ded_alloc = {
2389  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
2390  .pNext = (void *)(((uint8_t *)alloc_pnext) + img_cnt*alloc_pnext_stride),
2391  };
2392  VkMemoryDedicatedRequirements ded_req = {
2393  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
2394  };
2395  VkMemoryRequirements2 req = {
2396  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
2397  .pNext = &ded_req,
2398  };
2399 
2400  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req);
2401 
2402  if (f->tiling == VK_IMAGE_TILING_LINEAR)
2403  req.memoryRequirements.size = FFALIGN(req.memoryRequirements.size,
2404  p->props.properties.limits.minMemoryMapAlignment);
2405 
2406  /* In case the implementation prefers/requires dedicated allocation */
2407  use_ded_mem = ded_req.prefersDedicatedAllocation |
2408  ded_req.requiresDedicatedAllocation;
2409  if (use_ded_mem)
2410  ded_alloc.image = f->img[img_cnt];
2411 
2412  /* Allocate memory */
2413  if ((err = alloc_mem(ctx, &req.memoryRequirements,
2414  f->tiling == VK_IMAGE_TILING_LINEAR ?
2415  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT :
2416  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
2417  use_ded_mem ? &ded_alloc : (void *)ded_alloc.pNext,
2418  &f->flags, &f->mem[img_cnt])))
2419  return err;
2420 
2421  f->size[img_cnt] = req.memoryRequirements.size;
2422  bind_info[img_cnt].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
2423  bind_info[img_cnt].image = f->img[img_cnt];
2424  bind_info[img_cnt].memory = f->mem[img_cnt];
2425 
2426  img_cnt++;
2427  }
2428 
2429  /* Bind the allocated memory to the images */
2430  ret = vk->BindImageMemory2(hwctx->act_dev, img_cnt, bind_info);
2431  if (ret != VK_SUCCESS) {
2432  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
2433  ff_vk_ret2str(ret));
2434  return AVERROR_EXTERNAL;
2435  }
2436 
2437  return 0;
2438 }
2439 
2440 enum PrepMode {
2441  PREP_MODE_GENERAL,
2442  PREP_MODE_WRITE,
2443  PREP_MODE_EXTERNAL_EXPORT,
2444  PREP_MODE_EXTERNAL_IMPORT,
2445  PREP_MODE_DECODING_DST,
2446  PREP_MODE_DECODING_DPB,
2447  PREP_MODE_ENCODING_DPB,
2448 };
2449 
2450 static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2451  AVVkFrame *frame, enum PrepMode pmode)
2452 {
2453  int err;
2454  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2455  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2456  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
2457  int nb_img_bar = 0;
2458 
2459  uint32_t dst_qf = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2460  VkImageLayout new_layout;
2461  VkAccessFlags2 new_access;
2462  VkPipelineStageFlagBits2 src_stage = VK_PIPELINE_STAGE_2_NONE;
2463 
2464  /* This is dirty - but it works. The vulkan.c dependency system doesn't
2465  * free non-refcounted frames, and non-refcounted hardware frames cannot
2466  * happen anywhere outside of here. */
2467  AVBufferRef tmp_ref = {
2468  .data = (uint8_t *)hwfc,
2469  };
2470  AVFrame tmp_frame = {
2471  .data[0] = (uint8_t *)frame,
2472  .hw_frames_ctx = &tmp_ref,
2473  };
2474 
2475  VkCommandBuffer cmd_buf;
2476  FFVkExecContext *exec = ff_vk_exec_get(&p->vkctx, ectx);
2477  cmd_buf = exec->buf;
2478  ff_vk_exec_start(&p->vkctx, exec);
2479 
2480  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, &tmp_frame,
2481  VK_PIPELINE_STAGE_2_NONE,
2482  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
2483  if (err < 0)
2484  return err;
2485 
2486  switch (pmode) {
2487  case PREP_MODE_GENERAL:
2488  new_layout = VK_IMAGE_LAYOUT_GENERAL;
2489  new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2490  break;
2491  case PREP_MODE_WRITE:
2492  new_layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
2493  new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2494  break;
2495  case PREP_MODE_EXTERNAL_IMPORT:
2496  new_layout = VK_IMAGE_LAYOUT_GENERAL;
2497  new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2498  break;
2499  case PREP_MODE_EXTERNAL_EXPORT:
2500  new_layout = VK_IMAGE_LAYOUT_GENERAL;
2501  new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2502  dst_qf = VK_QUEUE_FAMILY_EXTERNAL_KHR;
2503  src_stage = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
2504  break;
2505  case PREP_MODE_DECODING_DST:
2506  new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR;
2507  new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2508  break;
2509  case PREP_MODE_DECODING_DPB:
2510  new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
2511  new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2512  break;
2513  case PREP_MODE_ENCODING_DPB:
2514  new_layout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
2515  new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2516  break;
2517  }
2518 
2519  ff_vk_frame_barrier(&p->vkctx, exec, &tmp_frame, img_bar, &nb_img_bar,
2520  src_stage,
2521  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
2522  new_access, new_layout, dst_qf);
2523 
2524  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
2525  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
2526  .pImageMemoryBarriers = img_bar,
2527  .imageMemoryBarrierCount = nb_img_bar,
2528  });
2529 
2530  err = ff_vk_exec_submit(&p->vkctx, exec);
2531  if (err < 0)
2532  return err;
2533 
2534  /* We can do this because there are no real dependencies */
2535  ff_vk_exec_discard_deps(&p->vkctx, exec);
2536 
2537  return 0;
2538 }
2539 
2540 static inline void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format,
2541  int frame_w, int frame_h, int plane)
2542 {
2543  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(format);
2544 
2545  /* Currently always true unless gray + alpha support is added */
2546  if (!plane || (plane == 3) || desc->flags & AV_PIX_FMT_FLAG_RGB ||
2547  !(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) {
2548  *w = frame_w;
2549  *h = frame_h;
2550  return;
2551  }
2552 
2553  *w = AV_CEIL_RSHIFT(frame_w, desc->log2_chroma_w);
2554  *h = AV_CEIL_RSHIFT(frame_h, desc->log2_chroma_h);
2555 }
2556 
2557 static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame,
2558  VkImageTiling tiling, VkImageUsageFlagBits usage,
2559  VkImageCreateFlags flags, int nb_layers,
2560  void *create_pnext)
2561 {
2562  int err;
2563  VkResult ret;
2564  AVVulkanFramesContext *hwfc_vk = hwfc->hwctx;
2565  AVHWDeviceContext *ctx = hwfc->device_ctx;
2566  VulkanDevicePriv *p = ctx->hwctx;
2567  AVVulkanDeviceContext *hwctx = &p->p;
2568  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2569  AVVkFrame *f;
2570 
2571  VkSemaphoreTypeCreateInfo sem_type_info = {
2572  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
2573  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
2574  .initialValue = 0,
2575  };
2576  VkSemaphoreCreateInfo sem_spawn = {
2577  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
2578  .pNext = &sem_type_info,
2579  };
2580 
2581  VkExportSemaphoreCreateInfo ext_sem_info_opaque = {
2582  .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
2583 #ifdef _WIN32
2584  .handleTypes = IsWindows8OrGreater()
2585  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
2586  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
2587 #else
2588  .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
2589 #endif
2590  };
2591 
2592  /* Check if exporting is supported before chaining any structs */
2593  if (p->ext_sem_props_opaque.externalSemaphoreFeatures & VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT) {
2594  if (p->vkctx.extensions & (FF_VK_EXT_EXTERNAL_WIN32_SEM | FF_VK_EXT_EXTERNAL_FD_SEM))
2595  ff_vk_link_struct(&sem_type_info, &ext_sem_info_opaque);
2596  }
2597 
2598  f = av_vk_frame_alloc();
2599  if (!f) {
2600  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
2601  return AVERROR(ENOMEM);
2602  }
2603 
2604  // TODO: check width and height for alignment in case of multiplanar (must be mod-2 if subsampled)
2605 
2606  /* Create the images */
2607  for (int i = 0; (hwfc_vk->format[i] != VK_FORMAT_UNDEFINED); i++) {
2608  VkImageCreateInfo create_info = {
2609  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
2610  .pNext = create_pnext,
2611  .imageType = VK_IMAGE_TYPE_2D,
2612  .format = hwfc_vk->format[i],
2613  .extent.depth = 1,
2614  .mipLevels = 1,
2615  .arrayLayers = nb_layers,
2616  .flags = flags,
2617  .tiling = tiling,
2618  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
2619  .usage = usage,
2620  .samples = VK_SAMPLE_COUNT_1_BIT,
2621  .pQueueFamilyIndices = p->img_qfs,
2622  .queueFamilyIndexCount = p->nb_img_qfs,
2623  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2624  VK_SHARING_MODE_EXCLUSIVE,
2625  };
2626 
2627  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
2628  hwfc->sw_format, hwfc->width, hwfc->height, i);
2629 
2630  ret = vk->CreateImage(hwctx->act_dev, &create_info,
2631  hwctx->alloc, &f->img[i]);
2632  if (ret != VK_SUCCESS) {
2633  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
2634  ff_vk_ret2str(ret));
2635  err = AVERROR(EINVAL);
2636  goto fail;
2637  }
2638 
2639  /* Create semaphore */
2640  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
2641  hwctx->alloc, &f->sem[i]);
2642  if (ret != VK_SUCCESS) {
2643  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
2644  ff_vk_ret2str(ret));
2645  err = AVERROR_EXTERNAL;
2646  goto fail;
2647  }
2648 
2649  f->queue_family[i] = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2650  f->layout[i] = create_info.initialLayout;
2651  f->access[i] = 0x0;
2652  f->sem_value[i] = 0;
2653  }
2654 
2655  f->flags = 0x0;
2656  f->tiling = tiling;
2657 
2658  *frame = f;
2659  return 0;
2660 
2661 fail:
2662  vulkan_frame_free(hwfc, f);
2663  return err;
2664 }
2665 
2666 /* Checks if an export flag is enabled, and if it is ORs it with *iexp */
2667 static void try_export_flags(AVHWFramesContext *hwfc,
2668  VkExternalMemoryHandleTypeFlags *comp_handle_types,
2669  VkExternalMemoryHandleTypeFlagBits *iexp,
2670  VkExternalMemoryHandleTypeFlagBits exp)
2671 {
2672  VkResult ret;
2673  AVVulkanFramesContext *hwctx = hwfc->hwctx;
2674  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2675  AVVulkanDeviceContext *dev_hwctx = &p->p;
2676  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2677 
2678  const VkImageDrmFormatModifierListCreateInfoEXT *drm_mod_info =
2679  ff_vk_find_struct(hwctx->create_pnext,
2680  VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
2681  int has_mods = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT && drm_mod_info;
2682  int nb_mods;
2683 
2684  VkExternalImageFormatProperties eprops = {
2685  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
2686  };
2687  VkImageFormatProperties2 props = {
2688  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
2689  .pNext = &eprops,
2690  };
2691  VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
2692  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
2693  .pNext = NULL,
2694  .pQueueFamilyIndices = p->img_qfs,
2695  .queueFamilyIndexCount = p->nb_img_qfs,
2696  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2697  VK_SHARING_MODE_EXCLUSIVE,
2698  };
2699  VkPhysicalDeviceExternalImageFormatInfo enext = {
2700  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
2701  .handleType = exp,
2702  .pNext = has_mods ? &phy_dev_mod_info : NULL,
2703  };
2704  VkPhysicalDeviceImageFormatInfo2 pinfo = {
2705  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
2706  .pNext = !exp ? NULL : &enext,
2707  .format = vk_find_format_entry(hwfc->sw_format)->vkf,
2708  .type = VK_IMAGE_TYPE_2D,
2709  .tiling = hwctx->tiling,
2710  .usage = hwctx->usage,
2711  .flags = VK_IMAGE_CREATE_ALIAS_BIT,
2712  };
2713 
2714  nb_mods = has_mods ? drm_mod_info->drmFormatModifierCount : 1;
2715  for (int i = 0; i < nb_mods; i++) {
2716  if (has_mods)
2717  phy_dev_mod_info.drmFormatModifier = drm_mod_info->pDrmFormatModifiers[i];
2718 
2719  ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
2720  &pinfo, &props);
2721 
2722  if (ret == VK_SUCCESS) {
2723  *iexp |= exp;
2724  *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
2725  }
2726  }
2727 }
2728 
2729 static AVBufferRef *vulkan_pool_alloc(void *opaque, size_t size)
2730 {
2731  int err;
2732  AVVkFrame *f;
2733  AVBufferRef *avbuf = NULL;
2734  AVHWFramesContext *hwfc = opaque;
2735  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2736  VulkanFramesPriv *fp = hwfc->hwctx;
2737  AVVulkanFramesContext *hwctx = &fp->p;
2738  VkExternalMemoryHandleTypeFlags e = 0x0;
2739  VkExportMemoryAllocateInfo eminfo[AV_NUM_DATA_POINTERS];
2740 
2741  VkExternalMemoryImageCreateInfo eiinfo = {
2742  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
2743  .pNext = hwctx->create_pnext,
2744  };
2745 
2746 #ifdef _WIN32
2747  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY)
2748  try_export_flags(hwfc, &eiinfo.handleTypes, &e, IsWindows8OrGreater()
2749  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
2750  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT);
2751 #else
2752  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY)
2753  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2754  VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT);
2755 
2756  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_DMABUF_MEMORY &&
2757  hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT)
2758  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2759  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
2760 #endif
2761 
2762  for (int i = 0; i < av_pix_fmt_count_planes(hwfc->sw_format); i++) {
2763  eminfo[i].sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
2764  eminfo[i].pNext = hwctx->alloc_pnext[i];
2765  eminfo[i].handleTypes = e;
2766  }
2767 
2768  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
2769  hwctx->nb_layers,
2770  eiinfo.handleTypes ? &eiinfo : hwctx->create_pnext);
2771  if (err)
2772  return NULL;
2773 
2774  err = alloc_bind_mem(hwfc, f, eminfo, sizeof(*eminfo));
2775  if (err)
2776  goto fail;
2777 
2778  if ( (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2779  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR))
2780  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DPB);
2781  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)
2782  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DST);
2783  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR)
2784  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_ENCODING_DPB);
2785  else if (hwctx->usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
2786  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_WRITE);
2787  else
2788  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_GENERAL);
2789  if (err)
2790  goto fail;
2791 
2792  avbuf = av_buffer_create((uint8_t *)f, sizeof(AVVkFrame),
2793  vulkan_frame_free_cb, hwfc, 0);
2794  if (!avbuf)
2795  goto fail;
2796 
2797  return avbuf;
2798 
2799 fail:
2800  vulkan_frame_free(hwfc, f);
2801  return NULL;
2802 }
2803 
2804 static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2805 {
2806  pthread_mutex_lock(&vkf->internal->update_mutex);
2807 }
2808 
2809 static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2810 {
2811  pthread_mutex_unlock(&vkf->internal->update_mutex);
2812 }
2813 
2814 static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
2815 {
2816  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2817  VulkanFramesPriv *fp = hwfc->hwctx;
2818 
2819  if (fp->modifier_info) {
2820  if (fp->modifier_info->pDrmFormatModifiers)
2821  av_freep(&fp->modifier_info->pDrmFormatModifiers);
2822  av_freep(&fp->modifier_info);
2823  }
2824 
2825  ff_vk_exec_pool_free(&p->vkctx, &fp->compute_exec);
2826  ff_vk_exec_pool_free(&p->vkctx, &fp->upload_exec);
2827  ff_vk_exec_pool_free(&p->vkctx, &fp->download_exec);
2828 
2829  av_buffer_pool_uninit(&fp->tmp);
2830 }
2831 
2832 static int vulkan_frames_init(AVHWFramesContext *hwfc)
2833 {
2834  int err;
2835  AVVkFrame *f;
2836  VulkanFramesPriv *fp = hwfc->hwctx;
2837  AVVulkanFramesContext *hwctx = &fp->p;
2838  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2839  AVVulkanDeviceContext *dev_hwctx = &p->p;
2840  VkImageUsageFlagBits supported_usage;
2841  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2842  const struct FFVkFormatEntry *fmt;
2843  int disable_multiplane = p->disable_multiplane ||
2844  (hwctx->flags & AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE);
2845 
2846  /* Defaults */
2847  if (!hwctx->nb_layers)
2848  hwctx->nb_layers = 1;
2849 
2850  /* VK_IMAGE_TILING_OPTIMAL == 0, can't check for it really */
2851  if (p->use_linear_images &&
2852  (hwctx->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT))
2853  hwctx->tiling = VK_IMAGE_TILING_LINEAR;
2854 
2855 
2856  fmt = vk_find_format_entry(hwfc->sw_format);
2857  if (!fmt) {
2858  av_log(hwfc, AV_LOG_ERROR, "Unsupported pixel format: %s!\n",
2859  av_get_pix_fmt_name(hwfc->sw_format));
2860  return AVERROR(EINVAL);
2861  }
2862 
2863  if (hwctx->format[0] != VK_FORMAT_UNDEFINED) {
2864  if (hwctx->format[0] != fmt->vkf) {
2865  for (int i = 0; i < fmt->nb_images_fallback; i++) {
2866  if (hwctx->format[i] != fmt->fallback[i]) {
2867  av_log(hwfc, AV_LOG_ERROR, "Incompatible Vulkan format given "
2868  "for the current sw_format %s!\n",
2869  av_get_pix_fmt_name(hwfc->sw_format));
2870  return AVERROR(EINVAL);
2871  }
2872  }
2873  }
2874 
2875  /* Check if the sw_format itself is supported */
2876  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
2877  hwctx->tiling, NULL,
2878  NULL, NULL, &supported_usage, 0,
2879  !hwctx->usage ||
2880  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
2881  if (err < 0) {
2882  av_log(hwfc, AV_LOG_ERROR, "Unsupported sw format: %s!\n",
2883  av_get_pix_fmt_name(hwfc->sw_format));
2884  return AVERROR(EINVAL);
2885  }
2886  } else {
2887  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
2888  hwctx->tiling, hwctx->format, NULL,
2889  NULL, &supported_usage,
2890  disable_multiplane,
2891  !hwctx->usage ||
2892  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
2893  if (err < 0)
2894  return err;
2895  }
2896 
2897  /* Nvidia is violating the spec because they thought no one would use this. */
2898  if (p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY &&
2899  (((fmt->nb_images == 1) && (fmt->vk_planes > 1)) ||
2900  (av_pix_fmt_desc_get(hwfc->sw_format)->nb_components == 1)))
2901  supported_usage &= ~VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT;
2902 
2903  /* Image usage flags */
2904  if (!hwctx->usage) {
2905  hwctx->usage = supported_usage & (VK_IMAGE_USAGE_TRANSFER_DST_BIT |
2906  VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
2907  VK_IMAGE_USAGE_STORAGE_BIT |
2908  VK_IMAGE_USAGE_SAMPLED_BIT);
2909 
2910  if ((p->vkctx.extensions & FF_VK_EXT_HOST_IMAGE_COPY) && !p->disable_host_transfer)
2911  hwctx->usage |= supported_usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT;
2912 
2913  /* Enables encoding of images, if supported by format and extensions */
2914  if ((supported_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
2915  (p->vkctx.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
2916  FF_VK_EXT_VIDEO_MAINTENANCE_1)))
2917  hwctx->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
2918  }
2919 
2920  /* Image creation flags.
2921  * Only fill them in automatically if the image is not going to be used as
2922  * a DPB-only image, and we have SAMPLED/STORAGE bits set. */
2923  if (!hwctx->img_flags) {
2924  int is_lone_dpb = ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) ||
2925  ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2926  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)));
2927  int sampleable = hwctx->usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
2928  VK_IMAGE_USAGE_STORAGE_BIT);
2929  hwctx->img_flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
2930  if (sampleable && !is_lone_dpb) {
2931  hwctx->img_flags |= VK_IMAGE_CREATE_ALIAS_BIT;
2932  if ((fmt->vk_planes > 1) && (hwctx->format[0] == fmt->vkf))
2933  hwctx->img_flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
2934  }
2935  }
2936 
2937  /* If the image has an ENCODE_SRC usage, and the maintenance1
2938  * extension is supported, check if it has a profile list.
2939  * If there's no profile list, or it has no encode operations,
2940  * then allow creating the image with no specific profile. */
2941  if ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
2942  (p->vkctx.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
2943  FF_VK_EXT_VIDEO_MAINTENANCE_1))) {
2944  const VkVideoProfileListInfoKHR *pl;
2945  pl = ff_vk_find_struct(hwctx->create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
2946  if (!pl) {
2947  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
2948  } else {
2949  uint32_t i;
2950  for (i = 0; i < pl->profileCount; i++) {
2951  /* Video ops start at exactly 0x00010000 */
2952  if (pl->pProfiles[i].videoCodecOperation & 0xFFFF0000)
2953  break;
2954  }
2955  if (i == pl->profileCount)
2956  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
2957  }
2958  }
2959 
2960  if (!hwctx->lock_frame)
2961  hwctx->lock_frame = lock_frame;
2962 
2963  if (!hwctx->unlock_frame)
2964  hwctx->unlock_frame = unlock_frame;
2965 
2966  err = ff_vk_exec_pool_init(&p->vkctx, p->compute_qf, &fp->compute_exec,
2967  p->compute_qf->num, 0, 0, 0, NULL);
2968  if (err)
2969  return err;
2970 
2971  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->upload_exec,
2972  p->transfer_qf->num*2, 0, 0, 0, NULL);
2973  if (err)
2974  return err;
2975 
2976  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->download_exec,
2977  p->transfer_qf->num, 0, 0, 0, NULL);
2978  if (err)
2979  return err;
2980 
2981  /* Test to see if allocation will fail */
2982  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
2983  hwctx->nb_layers, hwctx->create_pnext);
2984  if (err)
2985  return err;
2986 
2987  /* Collect `VkDrmFormatModifierPropertiesEXT` for each plane. Required for DRM export. */
2988  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS && hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
2989  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
2990  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
2991  };
2992  err = vk->GetImageDrmFormatModifierPropertiesEXT(dev_hwctx->act_dev, f->img[0],
2993  &drm_mod);
2994  if (err != VK_SUCCESS) {
2995  av_log(hwfc, AV_LOG_ERROR, "Failed to get image DRM format modifier properties");
2996  vulkan_frame_free(hwfc, f);
2997  return AVERROR_EXTERNAL;
2998  }
2999  for (int i = 0; i < fmt->vk_planes; ++i) {
3000  VkDrmFormatModifierPropertiesListEXT modp;
3001  VkFormatProperties2 fmtp;
3002  VkDrmFormatModifierPropertiesEXT *mod_props = NULL;
3003 
3004  modp = (VkDrmFormatModifierPropertiesListEXT) {
3005  .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
3006  };
3007  fmtp = (VkFormatProperties2) {
3008  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
3009  .pNext = &modp,
3010  };
3011 
3012  /* query drmFormatModifierCount by keeping pDrmFormatModifierProperties NULL */
3013  vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt->fallback[i], &fmtp);
3014 
3015  modp.pDrmFormatModifierProperties =
3016  av_calloc(modp.drmFormatModifierCount, sizeof(*modp.pDrmFormatModifierProperties));
3017  if (!modp.pDrmFormatModifierProperties) {
3018  vulkan_frame_free(hwfc, f);
3019  return AVERROR(ENOMEM);
3020  }
3021  vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt->fallback[i], &fmtp);
3022 
3023  for (uint32_t i = 0; i < modp.drmFormatModifierCount; ++i) {
3024  VkDrmFormatModifierPropertiesEXT *m = &modp.pDrmFormatModifierProperties[i];
3025  if (m->drmFormatModifier == drm_mod.drmFormatModifier) {
3026  mod_props = m;
3027  break;
3028  }
3029  }
3030 
3031  if (mod_props == NULL) {
3032  av_log(hwfc, AV_LOG_ERROR, "No DRM format modifier properties found for modifier 0x%016"PRIx64"\n",
3033  drm_mod.drmFormatModifier);
3034  av_free(modp.pDrmFormatModifierProperties);
3035  vulkan_frame_free(hwfc, f);
3036  return AVERROR_EXTERNAL;
3037  }
3038 
3039  fp->drm_format_modifier_properties[i] = *mod_props;
3040  av_free(modp.pDrmFormatModifierProperties);
3041  }
3042  }
3043 
3044  vulkan_frame_free(hwfc, f);
3045 
3046  /* If user did not specify a pool, hwfc->pool will be set to the internal one
3047  * in hwcontext.c just after this gets called */
3048  if (!hwfc->pool) {
3049  ffhwframesctx(hwfc)->pool_internal = av_buffer_pool_init2(sizeof(AVVkFrame),
3050  hwfc, vulkan_pool_alloc,
3051  NULL);
3052  if (!ffhwframesctx(hwfc)->pool_internal)
3053  return AVERROR(ENOMEM);
3054  }
3055 
3056  return 0;
3057 }
3058 
3059 static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
3060 {
3061  frame->buf[0] = av_buffer_pool_get(hwfc->pool);
3062  if (!frame->buf[0])
3063  return AVERROR(ENOMEM);
3064 
3065  frame->data[0] = frame->buf[0]->data;
3066  frame->format = AV_PIX_FMT_VULKAN;
3067  frame->width = hwfc->width;
3068  frame->height = hwfc->height;
3069 
3070  return 0;
3071 }
3072 
3073 static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc,
3074  enum AVHWFrameTransferDirection dir,
3075  enum AVPixelFormat **formats)
3076 {
3077  enum AVPixelFormat *fmts;
3078  int n = 2;
3079 
3080 #if CONFIG_CUDA
3081  n++;
3082 #endif
3083  fmts = av_malloc_array(n, sizeof(*fmts));
3084  if (!fmts)
3085  return AVERROR(ENOMEM);
3086 
3087  n = 0;
3088  fmts[n++] = hwfc->sw_format;
3089 #if CONFIG_CUDA
3090  fmts[n++] = AV_PIX_FMT_CUDA;
3091 #endif
3092  fmts[n++] = AV_PIX_FMT_NONE;
3093 
3094  *formats = fmts;
3095  return 0;
3096 }
3097 
3098 #if CONFIG_LIBDRM
3099 static void vulkan_unmap_from_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
3100 {
3101  vulkan_frame_free(hwfc, hwmap->priv);
3102 }
3103 
3104 static const struct {
3105  uint32_t drm_fourcc;
3106  VkFormat vk_format;
3107 } vulkan_drm_format_map[] = {
3108  { DRM_FORMAT_R8, VK_FORMAT_R8_UNORM },
3109  { DRM_FORMAT_R16, VK_FORMAT_R16_UNORM },
3110  { DRM_FORMAT_GR88, VK_FORMAT_R8G8_UNORM },
3111  { DRM_FORMAT_RG88, VK_FORMAT_R8G8_UNORM },
3112  { DRM_FORMAT_GR1616, VK_FORMAT_R16G16_UNORM },
3113  { DRM_FORMAT_RG1616, VK_FORMAT_R16G16_UNORM },
3114  { DRM_FORMAT_ARGB8888, VK_FORMAT_B8G8R8A8_UNORM },
3115  { DRM_FORMAT_XRGB8888, VK_FORMAT_B8G8R8A8_UNORM },
3116  { DRM_FORMAT_ABGR8888, VK_FORMAT_R8G8B8A8_UNORM },
3117  { DRM_FORMAT_XBGR8888, VK_FORMAT_R8G8B8A8_UNORM },
3118  { DRM_FORMAT_ARGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
3119  { DRM_FORMAT_ABGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
3120  { DRM_FORMAT_XRGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
3121  { DRM_FORMAT_XBGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
3122 
3123  // All these DRM_FORMATs were added in the same libdrm commit.
3124 #ifdef DRM_FORMAT_XYUV8888
3125  { DRM_FORMAT_XYUV8888, VK_FORMAT_R8G8B8A8_UNORM },
3126  { DRM_FORMAT_XVYU2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 } ,
3127  { DRM_FORMAT_XVYU12_16161616, VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16 } ,
3128  { DRM_FORMAT_XVYU16161616, VK_FORMAT_R16G16B16A16_UNORM } ,
3129 #endif
3130 };
3131 
3132 static inline VkFormat drm_to_vulkan_fmt(uint32_t drm_fourcc)
3133 {
3134  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
3135  if (vulkan_drm_format_map[i].drm_fourcc == drm_fourcc)
3136  return vulkan_drm_format_map[i].vk_format;
3137  return VK_FORMAT_UNDEFINED;
3138 }
3139 
3140 static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **frame,
3141  const AVFrame *src, int flags)
3142 {
3143  int err = 0;
3144  VkResult ret;
3145  AVVkFrame *f;
3146  int bind_counts = 0;
3147  AVHWDeviceContext *ctx = hwfc->device_ctx;
3148  VulkanDevicePriv *p = ctx->hwctx;
3149  AVVulkanDeviceContext *hwctx = &p->p;
3150  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3151  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3152  VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES];
3153  VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES];
3154 
3155  for (int i = 0; i < desc->nb_layers; i++) {
3156  if (drm_to_vulkan_fmt(desc->layers[i].format) == VK_FORMAT_UNDEFINED) {
3157  av_log(ctx, AV_LOG_ERROR, "Unsupported DMABUF layer format %#08x!\n",
3158  desc->layers[i].format);
3159  return AVERROR(EINVAL);
3160  }
3161  }
3162 
3163  if (!(f = av_vk_frame_alloc())) {
3164  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
3165  err = AVERROR(ENOMEM);
3166  goto fail;
3167  }
3168 
3169  f->tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
3170 
3171  for (int i = 0; i < desc->nb_layers; i++) {
3172  const int planes = desc->layers[i].nb_planes;
3173 
3174  /* Semaphore */
3175  VkSemaphoreTypeCreateInfo sem_type_info = {
3176  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3177  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
3178  .initialValue = 0,
3179  };
3180  VkSemaphoreCreateInfo sem_spawn = {
3181  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3182  .pNext = &sem_type_info,
3183  };
3184 
3185  /* Image creation */
3186  VkSubresourceLayout ext_img_layouts[AV_DRM_MAX_PLANES];
3187  VkImageDrmFormatModifierExplicitCreateInfoEXT ext_img_mod_spec = {
3188  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT,
3189  .drmFormatModifier = desc->objects[0].format_modifier,
3190  .drmFormatModifierPlaneCount = planes,
3191  .pPlaneLayouts = (const VkSubresourceLayout *)&ext_img_layouts,
3192  };
3193  VkExternalMemoryImageCreateInfo ext_img_spec = {
3194  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
3195  .pNext = &ext_img_mod_spec,
3196  .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3197  };
3198  VkImageCreateInfo create_info = {
3199  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
3200  .pNext = &ext_img_spec,
3201  .imageType = VK_IMAGE_TYPE_2D,
3202  .format = drm_to_vulkan_fmt(desc->layers[i].format),
3203  .extent.depth = 1,
3204  .mipLevels = 1,
3205  .arrayLayers = 1,
3206  .flags = 0x0,
3207  .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
3208  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, /* specs say so */
3209  .usage = 0x0, /* filled in below */
3210  .samples = VK_SAMPLE_COUNT_1_BIT,
3211  .pQueueFamilyIndices = p->img_qfs,
3212  .queueFamilyIndexCount = p->nb_img_qfs,
3213  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
3214  VK_SHARING_MODE_EXCLUSIVE,
3215  };
3216 
3217  /* Image format verification */
3218  VkExternalImageFormatProperties ext_props = {
3219  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
3220  };
3221  VkImageFormatProperties2 props_ret = {
3222  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
3223  .pNext = &ext_props,
3224  };
3225  VkPhysicalDeviceImageDrmFormatModifierInfoEXT props_drm_mod = {
3226  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
3227  .drmFormatModifier = ext_img_mod_spec.drmFormatModifier,
3228  .pQueueFamilyIndices = create_info.pQueueFamilyIndices,
3229  .queueFamilyIndexCount = create_info.queueFamilyIndexCount,
3230  .sharingMode = create_info.sharingMode,
3231  };
3232  VkPhysicalDeviceExternalImageFormatInfo props_ext = {
3233  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
3234  .pNext = &props_drm_mod,
3235  .handleType = ext_img_spec.handleTypes,
3236  };
3237  VkPhysicalDeviceImageFormatInfo2 fmt_props;
3238 
3239  if (flags & AV_HWFRAME_MAP_READ)
3240  create_info.usage |= VK_IMAGE_USAGE_SAMPLED_BIT |
3241  VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
3242  if (flags & AV_HWFRAME_MAP_WRITE)
3243  create_info.usage |= VK_IMAGE_USAGE_STORAGE_BIT |
3244  VK_IMAGE_USAGE_TRANSFER_DST_BIT;
3245 
3246  fmt_props = (VkPhysicalDeviceImageFormatInfo2) {
3247  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
3248  .pNext = &props_ext,
3249  .format = create_info.format,
3250  .type = create_info.imageType,
3251  .tiling = create_info.tiling,
3252  .usage = create_info.usage,
3253  .flags = create_info.flags,
3254  };
3255 
3256  /* Check if importing is possible for this combination of parameters */
3257  ret = vk->GetPhysicalDeviceImageFormatProperties2(hwctx->phys_dev,
3258  &fmt_props, &props_ret);
3259  if (ret != VK_SUCCESS) {
3260  av_log(ctx, AV_LOG_ERROR, "Cannot map DRM frame to Vulkan: %s\n",
3261  ff_vk_ret2str(ret));
3262  err = AVERROR_EXTERNAL;
3263  goto fail;
3264  }
3265 
3266  /* Set the image width/height */
3267  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
3268  hwfc->sw_format, src->width, src->height, i);
3269 
3270  /* Set the subresource layout based on the layer properties */
3271  for (int j = 0; j < planes; j++) {
3272  ext_img_layouts[j].offset = desc->layers[i].planes[j].offset;
3273  ext_img_layouts[j].rowPitch = desc->layers[i].planes[j].pitch;
3274  ext_img_layouts[j].size = 0; /* The specs say so for all 3 */
3275  ext_img_layouts[j].arrayPitch = 0;
3276  ext_img_layouts[j].depthPitch = 0;
3277  }
3278 
3279  /* Create image */
3280  ret = vk->CreateImage(hwctx->act_dev, &create_info,
3281  hwctx->alloc, &f->img[i]);
3282  if (ret != VK_SUCCESS) {
3283  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
3284  ff_vk_ret2str(ret));
3285  err = AVERROR(EINVAL);
3286  goto fail;
3287  }
3288 
3289  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3290  hwctx->alloc, &f->sem[i]);
3291  if (ret != VK_SUCCESS) {
3292  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3293  ff_vk_ret2str(ret));
3294  err = AVERROR_EXTERNAL;
3295  goto fail;
3296  }
3297 
3298  f->queue_family[i] = VK_QUEUE_FAMILY_EXTERNAL;
3299  f->layout[i] = create_info.initialLayout;
3300  f->access[i] = 0x0;
3301  f->sem_value[i] = 0;
3302  }
3303 
3304  for (int i = 0; i < desc->nb_layers; i++) {
3305  /* Memory requirements */
3306  VkImageMemoryRequirementsInfo2 req_desc = {
3307  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
3308  .image = f->img[i],
3309  };
3310  VkMemoryDedicatedRequirements ded_req = {
3311  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
3312  };
3313  VkMemoryRequirements2 req2 = {
3314  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
3315  .pNext = &ded_req,
3316  };
3317 
3318  /* Allocation/importing */
3319  VkMemoryFdPropertiesKHR fdmp = {
3320  .sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR,
3321  };
3322  /* This assumes that a layer will never be constructed from multiple
3323  * objects. If that was to happen in the real world, this code would
3324  * need to import each plane separately.
3325  */
3326  VkImportMemoryFdInfoKHR idesc = {
3327  .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
3328  .fd = dup(desc->objects[desc->layers[i].planes[0].object_index].fd),
3329  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3330  };
3331  VkMemoryDedicatedAllocateInfo ded_alloc = {
3332  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
3333  .pNext = &idesc,
3334  .image = req_desc.image,
3335  };
3336 
3337  /* Get object properties */
3338  ret = vk->GetMemoryFdPropertiesKHR(hwctx->act_dev,
3339  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3340  idesc.fd, &fdmp);
3341  if (ret != VK_SUCCESS) {
3342  av_log(hwfc, AV_LOG_ERROR, "Failed to get FD properties: %s\n",
3343  ff_vk_ret2str(ret));
3344  err = AVERROR_EXTERNAL;
3345  close(idesc.fd);
3346  goto fail;
3347  }
3348 
3349  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req2);
3350 
3351  /* Only a single bit must be set, not a range, and it must match */
3352  req2.memoryRequirements.memoryTypeBits = fdmp.memoryTypeBits;
3353 
3354  err = alloc_mem(ctx, &req2.memoryRequirements,
3355  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
3356  (ded_req.prefersDedicatedAllocation ||
3357  ded_req.requiresDedicatedAllocation) ?
3358  &ded_alloc : ded_alloc.pNext,
3359  &f->flags, &f->mem[i]);
3360  if (err) {
3361  close(idesc.fd);
3362  return err;
3363  }
3364 
3365  f->size[i] = req2.memoryRequirements.size;
3366  }
3367 
3368  for (int i = 0; i < desc->nb_layers; i++) {
3369  const int planes = desc->layers[i].nb_planes;
3370  for (int j = 0; j < planes; j++) {
3371  VkImageAspectFlagBits aspect = j == 0 ? VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT :
3372  j == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3373  VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
3374 
3375  plane_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO;
3376  plane_info[bind_counts].pNext = NULL;
3377  plane_info[bind_counts].planeAspect = aspect;
3378 
3379  bind_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
3380  bind_info[bind_counts].pNext = planes > 1 ? &plane_info[bind_counts] : NULL;
3381  bind_info[bind_counts].image = f->img[i];
3382  bind_info[bind_counts].memory = f->mem[i];
3383 
3384  /* Offset is already signalled via pPlaneLayouts above */
3385  bind_info[bind_counts].memoryOffset = 0;
3386 
3387  bind_counts++;
3388  }
3389  }
3390 
3391  /* Bind the allocated memory to the images */
3392  ret = vk->BindImageMemory2(hwctx->act_dev, bind_counts, bind_info);
3393  if (ret != VK_SUCCESS) {
3394  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
3395  ff_vk_ret2str(ret));
3396  err = AVERROR_EXTERNAL;
3397  goto fail;
3398  }
3399 
3400  *frame = f;
3401 
3402  return 0;
3403 
3404 fail:
3405  vulkan_frame_free(hwfc, f);
3406 
3407  return err;
3408 }
3409 
3410 static int vulkan_map_from_drm_frame_sync(AVHWFramesContext *hwfc, AVFrame *dst,
3411  const AVFrame *src, int flags)
3412 {
3413  int err;
3414  VkResult ret;
3415  AVHWDeviceContext *ctx = hwfc->device_ctx;
3416  VulkanDevicePriv *p = ctx->hwctx;
3417  VulkanFramesPriv *fp = hwfc->hwctx;
3418  AVVulkanDeviceContext *hwctx = &p->p;
3419  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3420 
3421  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3422 
3423 #ifdef DMA_BUF_IOCTL_EXPORT_SYNC_FILE
3424  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM) {
3425  VkCommandBuffer cmd_buf;
3426  FFVkExecContext *exec;
3427  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
3428  VkSemaphore drm_sync_sem[AV_DRM_MAX_PLANES] = { 0 };
3429  int nb_img_bar = 0;
3430 
3431  for (int i = 0; i < desc->nb_objects; i++) {
3432  VkSemaphoreTypeCreateInfo sem_type_info = {
3433  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3434  .semaphoreType = VK_SEMAPHORE_TYPE_BINARY,
3435  };
3436  VkSemaphoreCreateInfo sem_spawn = {
3437  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3438  .pNext = &sem_type_info,
3439  };
3440  VkImportSemaphoreFdInfoKHR import_info;
3441  struct dma_buf_export_sync_file implicit_fd_info = {
3442  .flags = DMA_BUF_SYNC_READ,
3443  .fd = -1,
3444  };
3445 
3446  if (ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE,
3447  &implicit_fd_info)) {
3448  err = AVERROR(errno);
3449  av_log(hwctx, AV_LOG_ERROR, "Failed to retrieve implicit DRM sync file: %s\n",
3450  av_err2str(err));
3451  for (; i >= 0; i--)
3452  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3453  return err;
3454  }
3455 
3456  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3457  hwctx->alloc, &drm_sync_sem[i]);
3458  if (ret != VK_SUCCESS) {
3459  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3460  ff_vk_ret2str(ret));
3461  err = AVERROR_EXTERNAL;
3462  for (; i >= 0; i--)
3463  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3464  return err;
3465  }
3466 
3467  import_info = (VkImportSemaphoreFdInfoKHR) {
3468  .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
3469  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3470  .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT,
3471  .semaphore = drm_sync_sem[i],
3472  .fd = implicit_fd_info.fd,
3473  };
3474 
3475  ret = vk->ImportSemaphoreFdKHR(hwctx->act_dev, &import_info);
3476  if (ret != VK_SUCCESS) {
3477  av_log(hwctx, AV_LOG_ERROR, "Failed to import semaphore: %s\n",
3478  ff_vk_ret2str(ret));
3479  err = AVERROR_EXTERNAL;
3480  for (; i >= 0; i--)
3481  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3482  return err;
3483  }
3484  }
3485 
3486  exec = ff_vk_exec_get(&p->vkctx, &fp->compute_exec);
3487  cmd_buf = exec->buf;
3488 
3489  ff_vk_exec_start(&p->vkctx, exec);
3490 
3491  /* Ownership of semaphores is passed */
3492  err = ff_vk_exec_add_dep_bool_sem(&p->vkctx, exec,
3493  drm_sync_sem, desc->nb_objects,
3494  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 1);
3495  if (err < 0)
3496  return err;
3497 
3498  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, dst,
3499  VK_PIPELINE_STAGE_2_NONE,
3500  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
3501  if (err < 0)
3502  return err;
3503 
3504  ff_vk_frame_barrier(&p->vkctx, exec, dst, img_bar, &nb_img_bar,
3505  VK_PIPELINE_STAGE_2_NONE,
3506  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
3507  ((flags & AV_HWFRAME_MAP_READ) ?
3508  VK_ACCESS_2_SHADER_SAMPLED_READ_BIT : 0x0) |
3509  ((flags & AV_HWFRAME_MAP_WRITE) ?
3510  VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0x0),
3511  VK_IMAGE_LAYOUT_GENERAL,
3512  p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0]);
3513 
3514  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
3515  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3516  .pImageMemoryBarriers = img_bar,
3517  .imageMemoryBarrierCount = nb_img_bar,
3518  });
3519 
3520  err = ff_vk_exec_submit(&p->vkctx, exec);
3521  if (err < 0)
3522  return err;
3523  } else
3524 #endif
3525  {
3526  AVVkFrame *f = (AVVkFrame *)dst->data[0];
3527  av_log(hwctx, AV_LOG_WARNING, "No support for synchronization when importing DMA-BUFs, "
3528  "image may be corrupted.\n");
3529  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3530  if (err)
3531  return err;
3532  }
3533 
3534  return 0;
3535 }
3536 
3537 static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3538  const AVFrame *src, int flags)
3539 {
3540  int err = 0;
3541  AVVkFrame *f;
3542 
3543  if ((err = vulkan_map_from_drm_frame_desc(hwfc, &f, src, flags)))
3544  return err;
3545 
3546  /* The unmapping function will free this */
3547  dst->data[0] = (uint8_t *)f;
3548  dst->width = src->width;
3549  dst->height = src->height;
3550 
3551  err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
3552  &vulkan_unmap_from_drm, f);
3553  if (err < 0)
3554  goto fail;
3555 
3556  err = vulkan_map_from_drm_frame_sync(hwfc, dst, src, flags);
3557  if (err < 0)
3558  return err;
3559 
3560  av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n");
3561 
3562  return 0;
3563 
3564 fail:
3565  vulkan_frame_free(hwfc->device_ctx->hwctx, f);
3566  dst->data[0] = NULL;
3567  return err;
3568 }
3569 
3570 #if CONFIG_VAAPI
3571 static int vulkan_map_from_vaapi(AVHWFramesContext *dst_fc,
3572  AVFrame *dst, const AVFrame *src,
3573  int flags)
3574 {
3575  int err;
3576  AVFrame *tmp = av_frame_alloc();
3577  AVHWFramesContext *vaapi_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3578  AVVAAPIDeviceContext *vaapi_ctx = vaapi_fc->device_ctx->hwctx;
3579  VASurfaceID surface_id = (VASurfaceID)(uintptr_t)src->data[3];
3580 
3581  if (!tmp)
3582  return AVERROR(ENOMEM);
3583 
3584  /* We have to sync since like the previous comment said, no semaphores */
3585  vaSyncSurface(vaapi_ctx->display, surface_id);
3586 
3587  tmp->format = AV_PIX_FMT_DRM_PRIME;
3588 
3589  err = av_hwframe_map(tmp, src, flags);
3590  if (err < 0)
3591  goto fail;
3592 
3593  err = vulkan_map_from_drm(dst_fc, dst, tmp, flags);
3594  if (err < 0)
3595  goto fail;
3596 
3597  err = ff_hwframe_map_replace(dst, src);
3598 
3599 fail:
3600  av_frame_free(&tmp);
3601  return err;
3602 }
3603 #endif
3604 #endif
3605 
3606 #if CONFIG_CUDA
3607 static int export_mem_to_cuda(AVHWDeviceContext *ctx,
3608  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3609  AVVkFrameInternal *dst_int, int idx,
3610  VkDeviceMemory mem, size_t size)
3611 {
3612  VkResult ret;
3613  VulkanDevicePriv *p = ctx->hwctx;
3614  AVVulkanDeviceContext *hwctx = &p->p;
3615  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3616 
3617 #ifdef _WIN32
3618  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3619  .type = IsWindows8OrGreater()
3620  ? CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32
3621  : CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT,
3622  .size = size,
3623  };
3624  VkMemoryGetWin32HandleInfoKHR export_info = {
3625  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR,
3626  .memory = mem,
3627  .handleType = IsWindows8OrGreater()
3628  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
3629  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3630  };
3631 
3632  ret = vk->GetMemoryWin32HandleKHR(hwctx->act_dev, &export_info,
3633  &ext_desc.handle.win32.handle);
3634  if (ret != VK_SUCCESS) {
3635  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a Win32 Handle: %s!\n",
3636  ff_vk_ret2str(ret));
3637  return AVERROR_EXTERNAL;
3638  }
3639  dst_int->ext_mem_handle[idx] = ext_desc.handle.win32.handle;
3640 #else
3641  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3642  .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD,
3643  .size = size,
3644  };
3645  VkMemoryGetFdInfoKHR export_info = {
3646  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
3647  .memory = mem,
3648  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR,
3649  };
3650 
3651  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
3652  &ext_desc.handle.fd);
3653  if (ret != VK_SUCCESS) {
3654  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a FD: %s!\n",
3655  ff_vk_ret2str(ret));
3656  return AVERROR_EXTERNAL;
3657  }
3658 #endif
3659 
3660  ret = CHECK_CU(cu->cuImportExternalMemory(&dst_int->ext_mem[idx], &ext_desc));
3661  if (ret < 0) {
3662 #ifndef _WIN32
3663  close(ext_desc.handle.fd);
3664 #endif
3665  return AVERROR_EXTERNAL;
3666  }
3667 
3668  return 0;
3669 }
3670 
3671 static int export_sem_to_cuda(AVHWDeviceContext *ctx,
3672  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3673  AVVkFrameInternal *dst_int, int idx,
3674  VkSemaphore sem)
3675 {
3676  VkResult ret;
3677  VulkanDevicePriv *p = ctx->hwctx;
3678  AVVulkanDeviceContext *hwctx = &p->p;
3679  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3680 
3681 #ifdef _WIN32
3682  VkSemaphoreGetWin32HandleInfoKHR sem_export = {
3683  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR,
3684  .semaphore = sem,
3685  .handleType = IsWindows8OrGreater()
3686  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
3687  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3688  };
3689  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3690  .type = 10 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32 */,
3691  };
3692 #else
3693  VkSemaphoreGetFdInfoKHR sem_export = {
3694  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
3695  .semaphore = sem,
3696  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
3697  };
3698  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3699  .type = 9 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD */,
3700  };
3701 #endif
3702 
3703 #ifdef _WIN32
3704  ret = vk->GetSemaphoreWin32HandleKHR(hwctx->act_dev, &sem_export,
3705  &ext_sem_desc.handle.win32.handle);
3706 #else
3707  ret = vk->GetSemaphoreFdKHR(hwctx->act_dev, &sem_export,
3708  &ext_sem_desc.handle.fd);
3709 #endif
3710  if (ret != VK_SUCCESS) {
3711  av_log(ctx, AV_LOG_ERROR, "Failed to export semaphore: %s\n",
3712  ff_vk_ret2str(ret));
3713  return AVERROR_EXTERNAL;
3714  }
3715 #ifdef _WIN32
3716  dst_int->ext_sem_handle[idx] = ext_sem_desc.handle.win32.handle;
3717 #endif
3718 
3719  ret = CHECK_CU(cu->cuImportExternalSemaphore(&dst_int->cu_sem[idx],
3720  &ext_sem_desc));
3721  if (ret < 0) {
3722 #ifndef _WIN32
3723  close(ext_sem_desc.handle.fd);
3724 #endif
3725  return AVERROR_EXTERNAL;
3726  }
3727 
3728  return 0;
3729 }
3730 
3731 static int vulkan_export_to_cuda(AVHWFramesContext *hwfc,
3732  AVBufferRef *cuda_hwfc,
3733  const AVFrame *frame)
3734 {
3735  int err;
3736  VkResult ret;
3737  AVVkFrame *dst_f;
3738  AVVkFrameInternal *dst_int;
3739  AVHWDeviceContext *ctx = hwfc->device_ctx;
3740  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3741  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3742  VulkanDevicePriv *p = ctx->hwctx;
3743  AVVulkanDeviceContext *hwctx = &p->p;
3744  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3745  int nb_images;
3746 
3747  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)cuda_hwfc->data;
3748  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3749  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3750  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3751  CudaFunctions *cu = cu_internal->cuda_dl;
3752  CUarray_format cufmt = desc->comp[0].depth > 8 ? CU_AD_FORMAT_UNSIGNED_INT16 :
3753  CU_AD_FORMAT_UNSIGNED_INT8;
3754 
3755  dst_f = (AVVkFrame *)frame->data[0];
3756  dst_int = dst_f->internal;
3757 
3758  if (!dst_int->cuda_fc_ref) {
3759  size_t offsets[3] = { 0 };
3760 
3761  dst_int->cuda_fc_ref = av_buffer_ref(cuda_hwfc);
3762  if (!dst_int->cuda_fc_ref)
3763  return AVERROR(ENOMEM);
3764 
3765  nb_images = ff_vk_count_images(dst_f);
3766  for (int i = 0; i < nb_images; i++) {
3767  err = export_mem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3768  dst_f->mem[i], dst_f->size[i]);
3769  if (err < 0)
3770  goto fail;
3771 
3772  err = export_sem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3773  dst_f->sem[i]);
3774  if (err < 0)
3775  goto fail;
3776  }
3777 
3778  if (nb_images != planes) {
3779  for (int i = 0; i < planes; i++) {
3780  VkImageSubresource subres = {
3781  .aspectMask = i == 2 ? VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT :
3782  i == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3783  VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT
3784  };
3785  VkSubresourceLayout layout = { 0 };
3786  vk->GetImageSubresourceLayout(hwctx->act_dev, dst_f->img[FFMIN(i, nb_images - 1)],
3787  &subres, &layout);
3788  offsets[i] = layout.offset;
3789  }
3790  }
3791 
3792  for (int i = 0; i < planes; i++) {
3793  CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC tex_desc = {
3794  .offset = offsets[i],
3795  .arrayDesc = {
3796  .Depth = 0,
3797  .Format = cufmt,
3798  .NumChannels = 1 + ((planes == 2) && i),
3799  .Flags = 0,
3800  },
3801  .numLevels = 1,
3802  };
3803  int p_w, p_h;
3804 
3805  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
3806  tex_desc.arrayDesc.Width = p_w;
3807  tex_desc.arrayDesc.Height = p_h;
3808 
3809  ret = CHECK_CU(cu->cuExternalMemoryGetMappedMipmappedArray(&dst_int->cu_mma[i],
3810  dst_int->ext_mem[FFMIN(i, nb_images - 1)],
3811  &tex_desc));
3812  if (ret < 0) {
3813  err = AVERROR_EXTERNAL;
3814  goto fail;
3815  }
3816 
3817  ret = CHECK_CU(cu->cuMipmappedArrayGetLevel(&dst_int->cu_array[i],
3818  dst_int->cu_mma[i], 0));
3819  if (ret < 0) {
3820  err = AVERROR_EXTERNAL;
3821  goto fail;
3822  }
3823 
3824  }
3825  }
3826 
3827  return 0;
3828 
3829 fail:
3830  vulkan_free_internal(dst_f);
3831  return err;
3832 }
3833 
3834 static int vulkan_transfer_data_from_cuda(AVHWFramesContext *hwfc,
3835  AVFrame *dst, const AVFrame *src)
3836 {
3837  int err;
3838  CUcontext dummy;
3839  AVVkFrame *dst_f;
3840  AVVkFrameInternal *dst_int;
3841  VulkanFramesPriv *fp = hwfc->hwctx;
3842  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3843  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3844 
3845  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3846  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3847  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3848  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3849  CudaFunctions *cu = cu_internal->cuda_dl;
3850  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
3851  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
3852 
3853  dst_f = (AVVkFrame *)dst->data[0];
3854 
3855  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
3856  if (err < 0)
3857  return err;
3858 
3859  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
3860  if (err < 0)
3861  return err;
3862 
3863  err = vulkan_export_to_cuda(hwfc, src->hw_frames_ctx, dst);
3864  if (err < 0) {
3865  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
3866  return err;
3867  }
3868 
3869  dst_int = dst_f->internal;
3870 
3871  for (int i = 0; i < planes; i++) {
3872  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
3873  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
3874  }
3875 
3876  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
3877  planes, cuda_dev->stream));
3878  if (err < 0)
3879  goto fail;
3880 
3881  for (int i = 0; i < planes; i++) {
3882  CUDA_MEMCPY2D cpy = {
3883  .srcMemoryType = CU_MEMORYTYPE_DEVICE,
3884  .srcDevice = (CUdeviceptr)src->data[i],
3885  .srcPitch = src->linesize[i],
3886  .srcY = 0,
3887 
3888  .dstMemoryType = CU_MEMORYTYPE_ARRAY,
3889  .dstArray = dst_int->cu_array[i],
3890  };
3891 
3892  int p_w, p_h;
3893  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
3894 
3895  cpy.WidthInBytes = p_w * desc->comp[i].step;
3896  cpy.Height = p_h;
3897 
3898  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
3899  if (err < 0)
3900  goto fail;
3901  }
3902 
3903  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
3904  planes, cuda_dev->stream));
3905  if (err < 0)
3906  goto fail;
3907 
3908  for (int i = 0; i < planes; i++)
3909  dst_f->sem_value[i]++;
3910 
3911  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
3912 
3913  av_log(hwfc, AV_LOG_VERBOSE, "Transferred CUDA image to Vulkan!\n");
3914 
3915  return err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
3916 
3917 fail:
3918  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
3919  vulkan_free_internal(dst_f);
3920  av_buffer_unref(&dst->buf[0]);
3921  return err;
3922 }
3923 #endif
3924 
3925 static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
3926  const AVFrame *src, int flags)
3927 {
3928  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3929 
3930  switch (src->format) {
3931 #if CONFIG_LIBDRM
3932 #if CONFIG_VAAPI
3933  case AV_PIX_FMT_VAAPI:
3934  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3935  return vulkan_map_from_vaapi(hwfc, dst, src, flags);
3936  else
3937  return AVERROR(ENOSYS);
3938 #endif
3939  case AV_PIX_FMT_DRM_PRIME:
3940  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3941  return vulkan_map_from_drm(hwfc, dst, src, flags);
3942  else
3943  return AVERROR(ENOSYS);
3944 #endif
3945  default:
3946  return AVERROR(ENOSYS);
3947  }
3948 }
3949 
3950 #if CONFIG_LIBDRM
3951 typedef struct VulkanDRMMapping {
3952  AVDRMFrameDescriptor drm_desc;
3953  AVVkFrame *source;
3954 } VulkanDRMMapping;
3955 
3956 static void vulkan_unmap_to_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
3957 {
3958  AVDRMFrameDescriptor *drm_desc = hwmap->priv;
3959 
3960  for (int i = 0; i < drm_desc->nb_objects; i++)
3961  close(drm_desc->objects[i].fd);
3962 
3963  av_free(drm_desc);
3964 }
3965 
3966 static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt)
3967 {
3968  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
3969  if (vulkan_drm_format_map[i].vk_format == vkfmt)
3970  return vulkan_drm_format_map[i].drm_fourcc;
3971  return DRM_FORMAT_INVALID;
3972 }
3973 
3974 #define MAX_MEMORY_PLANES 4
3975 static VkImageAspectFlags plane_index_to_aspect(int plane) {
3976  if (plane == 0) return VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
3977  if (plane == 1) return VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT;
3978  if (plane == 2) return VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
3979  if (plane == 3) return VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT;
3980 
3981  av_assert2 (0 && "Invalid plane index");
3982  return VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT;
3983 }
3984 
3985 static int vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3986  const AVFrame *src, int flags)
3987 {
3988  int err = 0;
3989  VkResult ret;
3990  AVVkFrame *f = (AVVkFrame *)src->data[0];
3991  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3992  AVVulkanDeviceContext *hwctx = &p->p;
3993  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3994  VulkanFramesPriv *fp = hwfc->hwctx;
3995  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3996  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
3997  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
3998  };
3999  VkSemaphoreWaitInfo wait_info = {
4000  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
4001  .flags = 0x0,
4002  .semaphoreCount = planes,
4003  };
4004 
4005  AVDRMFrameDescriptor *drm_desc = av_mallocz(sizeof(*drm_desc));
4006  if (!drm_desc)
4007  return AVERROR(ENOMEM);
4008 
4009  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_EXPORT);
4010  if (err < 0)
4011  goto end;
4012 
4013  /* Wait for the operation to finish so we can cleanly export it. */
4014  wait_info.pSemaphores = f->sem;
4015  wait_info.pValues = f->sem_value;
4016 
4017  vk->WaitSemaphores(hwctx->act_dev, &wait_info, UINT64_MAX);
4018 
4019  err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, &vulkan_unmap_to_drm, drm_desc);
4020  if (err < 0)
4021  goto end;
4022 
4023  ret = vk->GetImageDrmFormatModifierPropertiesEXT(hwctx->act_dev, f->img[0],
4024  &drm_mod);
4025  if (ret != VK_SUCCESS) {
4026  av_log(hwfc, AV_LOG_ERROR, "Failed to retrieve DRM format modifier!\n");
4027  err = AVERROR_EXTERNAL;
4028  goto end;
4029  }
4030 
4031  for (int i = 0; (i < planes) && (f->mem[i]); i++) {
4032  VkMemoryGetFdInfoKHR export_info = {
4033  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
4034  .memory = f->mem[i],
4035  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
4036  };
4037 
4038  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
4039  &drm_desc->objects[i].fd);
4040  if (ret != VK_SUCCESS) {
4041  av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a FD!\n");
4042  err = AVERROR_EXTERNAL;
4043  goto end;
4044  }
4045 
4046  drm_desc->nb_objects++;
4047  drm_desc->objects[i].size = f->size[i];
4048  drm_desc->objects[i].format_modifier = drm_mod.drmFormatModifier;
4049  }
4050 
4051  drm_desc->nb_layers = planes;
4052  for (int i = 0; i < drm_desc->nb_layers; i++) {
4053  VkFormat plane_vkfmt = av_vkfmt_from_pixfmt(hwfc->sw_format)[i];
4054 
4055  drm_desc->layers[i].format = vulkan_fmt_to_drm(plane_vkfmt);
4056  drm_desc->layers[i].nb_planes = fp->drm_format_modifier_properties[i].drmFormatModifierPlaneCount;
4057 
4058  if (drm_desc->layers[i].nb_planes > MAX_MEMORY_PLANES) {
4059  av_log(hwfc, AV_LOG_ERROR, "Too many memory planes for DRM format!\n");
4060  err = AVERROR_EXTERNAL;
4061  goto end;
4062  }
4063 
4064  for (int j = 0; j < drm_desc->layers[i].nb_planes; j++) {
4065  VkSubresourceLayout layout;
4066  VkImageSubresource sub = {
4067  .aspectMask = plane_index_to_aspect(j),
4068  };
4069 
4070  drm_desc->layers[i].planes[j].object_index = FFMIN(i, drm_desc->nb_objects - 1);
4071 
4072  vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, &layout);
4073  drm_desc->layers[i].planes[j].offset = layout.offset;
4074  drm_desc->layers[i].planes[j].pitch = layout.rowPitch;
4075  }
4076 
4077  if (drm_desc->layers[i].format == DRM_FORMAT_INVALID) {
4078  av_log(hwfc, AV_LOG_ERROR, "Cannot map to DRM layer, unsupported!\n");
4079  err = AVERROR_PATCHWELCOME;
4080  goto end;
4081  }
4082 
4083 
4084  if (f->tiling == VK_IMAGE_TILING_OPTIMAL)
4085  continue;
4086 
4087  }
4088 
4089  dst->width = src->width;
4090  dst->height = src->height;
4091  dst->data[0] = (uint8_t *)drm_desc;
4092 
4093  av_log(hwfc, AV_LOG_VERBOSE, "Mapped AVVkFrame to a DRM object!\n");
4094 
4095  return 0;
4096 
4097 end:
4098  av_free(drm_desc);
4099  return err;
4100 }
4101 
4102 #if CONFIG_VAAPI
4103 static int vulkan_map_to_vaapi(AVHWFramesContext *hwfc, AVFrame *dst,
4104  const AVFrame *src, int flags)
4105 {
4106  int err;
4107  AVFrame *tmp = av_frame_alloc();
4108  if (!tmp)
4109  return AVERROR(ENOMEM);
4110 
4111  tmp->format = AV_PIX_FMT_DRM_PRIME;
4112 
4113  err = vulkan_map_to_drm(hwfc, tmp, src, flags);
4114  if (err < 0)
4115  goto fail;
4116 
4117  err = av_hwframe_map(dst, tmp, flags);
4118  if (err < 0)
4119  goto fail;
4120 
4121  err = ff_hwframe_map_replace(dst, src);
4122 
4123 fail:
4124  av_frame_free(&tmp);
4125  return err;
4126 }
4127 #endif
4128 #endif
4129 
4130 static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
4131  const AVFrame *src, int flags)
4132 {
4133  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4134 
4135  switch (dst->format) {
4136 #if CONFIG_LIBDRM
4137  case AV_PIX_FMT_DRM_PRIME:
4138  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4139  return vulkan_map_to_drm(hwfc, dst, src, flags);
4140  else
4141  return AVERROR(ENOSYS);
4142 #if CONFIG_VAAPI
4143  case AV_PIX_FMT_VAAPI:
4144  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
4145  return vulkan_map_to_vaapi(hwfc, dst, src, flags);
4146  else
4147  return AVERROR(ENOSYS);
4148 #endif
4149 #endif
4150  default:
4151  break;
4152  }
4153  return AVERROR(ENOSYS);
4154 }
4155 
4156 static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf,
4157  AVFrame *swf, VkBufferImageCopy *region,
4158  int planes, int upload)
4159 {
4160  VkResult ret;
4161  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4162  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4163  AVVulkanDeviceContext *hwctx = &p->p;
4164 
4165  FFVkBuffer *vkbuf = (FFVkBuffer *)buf->data;
4166 
4167  const VkMappedMemoryRange flush_info = {
4168  .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
4169  .memory = vkbuf->mem,
4170  .size = VK_WHOLE_SIZE,
4171  };
4172 
4173  if (!upload && !(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
4174  ret = vk->InvalidateMappedMemoryRanges(hwctx->act_dev, 1,
4175  &flush_info);
4176  if (ret != VK_SUCCESS) {
4177  av_log(hwfc, AV_LOG_ERROR, "Failed to invalidate buffer data: %s\n",
4178  ff_vk_ret2str(ret));
4179  return AVERROR_EXTERNAL;
4180  }
4181  }
4182 
4183  if (upload) {
4184  for (int i = 0; i < planes; i++)
4185  av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset,
4186  region[i].bufferRowLength,
4187  swf->data[i],
4188  swf->linesize[i],
4189  swf->linesize[i],
4190  region[i].imageExtent.height);
4191  } else {
4192  for (int i = 0; i < planes; i++)
4193  av_image_copy_plane(swf->data[i],
4194  swf->linesize[i],
4195  vkbuf->mapped_mem + region[i].bufferOffset,
4196  region[i].bufferRowLength,
4197  swf->linesize[i],
4198  region[i].imageExtent.height);
4199  }
4200 
4201  if (upload && !(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
4202  ret = vk->FlushMappedMemoryRanges(hwctx->act_dev, 1,
4203  &flush_info);
4204  if (ret != VK_SUCCESS) {
4205  av_log(hwfc, AV_LOG_ERROR, "Failed to flush buffer data: %s\n",
4206  ff_vk_ret2str(ret));
4207  return AVERROR_EXTERNAL;
4208  }
4209  }
4210 
4211  return 0;
4212 }
4213 
4214 static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst,
4215  AVFrame *swf, VkBufferImageCopy *region, int upload)
4216 {
4217  int err;
4218  uint32_t p_w, p_h;
4219  VulkanFramesPriv *fp = hwfc->hwctx;
4220  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4221  const int planes = av_pix_fmt_count_planes(swf->format);
4222  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4223  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4224 
4225  size_t buf_offset = 0;
4226  for (int i = 0; i < planes; i++) {
4227  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4228 
4229  region[i] = (VkBufferImageCopy) {
4230  .bufferOffset = buf_offset,
4231  .bufferRowLength = FFALIGN(swf->linesize[i],
4232  p->props.properties.limits.optimalBufferCopyRowPitchAlignment),
4233  .bufferImageHeight = p_h,
4234  .imageSubresource.layerCount = 1,
4235  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4236  /* Rest of the fields adjusted/filled in later */
4237  };
4238 
4239  buf_offset += FFALIGN(p_h*region[i].bufferRowLength,
4240  p->props.properties.limits.optimalBufferCopyOffsetAlignment);
4241  }
4242 
4243  err = ff_vk_get_pooled_buffer(&p->vkctx, &fp->tmp, dst, buf_usage,
4244  NULL, buf_offset,
4245  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
4246  VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
4247  if (err < 0)
4248  return err;
4249 
4250  return 0;
4251 }
4252 
4253 static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs,
4254  AVFrame *swf, VkBufferImageCopy *region, int upload)
4255 {
4256  int err;
4257  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4258 
4259  int nb_src_bufs;
4260  const int planes = av_pix_fmt_count_planes(swf->format);
4261  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4262  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4263 
4264  /* We can't host map images with negative strides */
4265  for (int i = 0; i < planes; i++)
4266  if (swf->linesize[i] < 0)
4267  return AVERROR(EINVAL);
4268 
4269  /* Count the number of buffers in the software frame */
4270  nb_src_bufs = 0;
4271  while (swf->buf[nb_src_bufs])
4272  nb_src_bufs++;
4273 
4274  /* Single buffer contains all planes */
4275  if (nb_src_bufs == 1) {
4276  err = ff_vk_host_map_buffer(&p->vkctx, &dst[0],
4277  swf->data[0], swf->buf[0],
4278  buf_usage);
4279  if (err < 0)
4280  return err;
4281  (*nb_bufs)++;
4282 
4283  for (int i = 0; i < planes; i++)
4284  region[i].bufferOffset = ((FFVkBuffer *)dst[0]->data)->virtual_offset +
4285  swf->data[i] - swf->data[0];
4286  } else if (nb_src_bufs == planes) { /* One buffer per plane */
4287  for (int i = 0; i < planes; i++) {
4288  err = ff_vk_host_map_buffer(&p->vkctx, &dst[i],
4289  swf->data[i], swf->buf[i],
4290  buf_usage);
4291  if (err < 0)
4292  goto fail;
4293  (*nb_bufs)++;
4294 
4295  region[i].bufferOffset = ((FFVkBuffer *)dst[i]->data)->virtual_offset;
4296  }
4297  } else {
4298  /* Weird layout (3 planes, 2 buffers), patch welcome, fallback to copy */
4299  return AVERROR_PATCHWELCOME;
4300  }
4301 
4302  return 0;
4303 
4304 fail:
4305  for (int i = 0; i < (*nb_bufs); i++)
4306  av_buffer_unref(&dst[i]);
4307  return err;
4308 }
4309 
4310 static int vulkan_transfer_host(AVHWFramesContext *hwfc, AVFrame *hwf,
4311  AVFrame *swf, int upload)
4312 {
4313  VulkanFramesPriv *fp = hwfc->hwctx;
4314  AVVulkanFramesContext *hwfc_vk = &fp->p;
4315  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4316  AVVulkanDeviceContext *hwctx = &p->p;
4317  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4318 
4319  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4320  const int planes = av_pix_fmt_count_planes(swf->format);
4321  const int nb_images = ff_vk_count_images(hwf_vk);
4322 
4323  VkSemaphoreWaitInfo sem_wait;
4324  VkHostImageLayoutTransitionInfoEXT layout_ch_info[AV_NUM_DATA_POINTERS];
4325  int nb_layout_ch = 0;
4326 
4327  hwfc_vk->lock_frame(hwfc, hwf_vk);
4328 
4329  for (int i = 0; i < nb_images; i++) {
4330  int compat = 0;
4331  for (int j = 0; j < p->vkctx.host_image_props.copySrcLayoutCount; j++) {
4332  if (hwf_vk->layout[i] == p->vkctx.host_image_props.pCopySrcLayouts[j]) {
4333  compat = 1;
4334  break;
4335  }
4336  }
4337  if (compat)
4338  continue;
4339 
4340  layout_ch_info[nb_layout_ch] = (VkHostImageLayoutTransitionInfoEXT) {
4341  .sType = VK_STRUCTURE_TYPE_HOST_IMAGE_LAYOUT_TRANSITION_INFO_EXT,
4342  .image = hwf_vk->img[i],
4343  .oldLayout = hwf_vk->layout[i],
4344  .newLayout = VK_IMAGE_LAYOUT_GENERAL,
4345  .subresourceRange = {
4346  .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
4347  .levelCount = 1,
4348  .layerCount = 1,
4349  },
4350  };
4351 
4352  hwf_vk->layout[i] = layout_ch_info[nb_layout_ch].newLayout;
4353  nb_layout_ch++;
4354  }
4355 
4356  sem_wait = (VkSemaphoreWaitInfo) {
4357  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
4358  .pSemaphores = hwf_vk->sem,
4359  .pValues = hwf_vk->sem_value,
4360  .semaphoreCount = nb_images,
4361  };
4362 
4363  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
4364 
4365  if (nb_layout_ch)
4366  vk->TransitionImageLayoutEXT(hwctx->act_dev,
4367  nb_layout_ch, layout_ch_info);
4368 
4369  if (upload) {
4370  VkMemoryToImageCopyEXT region_info = {
4371  .sType = VK_STRUCTURE_TYPE_MEMORY_TO_IMAGE_COPY_EXT,
4372  .imageSubresource = {
4373  .layerCount = 1,
4374  },
4375  };
4376  VkCopyMemoryToImageInfoEXT copy_info = {
4377  .sType = VK_STRUCTURE_TYPE_COPY_MEMORY_TO_IMAGE_INFO_EXT,
4378  .flags = VK_HOST_IMAGE_COPY_MEMCPY_EXT,
4379  .regionCount = 1,
4380  .pRegions = &region_info,
4381  };
4382  for (int i = 0; i < planes; i++) {
4383  int img_idx = FFMIN(i, (nb_images - 1));
4384  uint32_t p_w, p_h;
4385  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4386 
4387  region_info.pHostPointer = swf->data[i];
4388  region_info.imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4389  region_info.imageExtent = (VkExtent3D){ p_w, p_h, 1 };
4390  copy_info.dstImage = hwf_vk->img[img_idx];
4391  copy_info.dstImageLayout = hwf_vk->layout[img_idx];
4392 
4393  vk->CopyMemoryToImageEXT(hwctx->act_dev, &copy_info);
4394  }
4395  } else {
4396  VkImageToMemoryCopyEXT region_info = {
4397  .sType = VK_STRUCTURE_TYPE_MEMORY_TO_IMAGE_COPY_EXT,
4398  .imageSubresource = {
4399  .layerCount = 1,
4400  },
4401  };
4402  VkCopyImageToMemoryInfoEXT copy_info = {
4403  .sType = VK_STRUCTURE_TYPE_COPY_IMAGE_TO_MEMORY_INFO_EXT,
4404  .flags = VK_HOST_IMAGE_COPY_MEMCPY_EXT,
4405  .regionCount = 1,
4406  .pRegions = &region_info,
4407  };
4408  for (int i = 0; i < planes; i++) {
4409  int img_idx = FFMIN(i, (nb_images - 1));
4410  uint32_t p_w, p_h;
4411  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4412 
4413  region_info.pHostPointer = swf->data[i];
4414  region_info.imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4415  region_info.imageExtent = (VkExtent3D){ p_w, p_h, 1 };
4416  copy_info.srcImage = hwf_vk->img[img_idx];
4417  copy_info.srcImageLayout = hwf_vk->layout[img_idx];
4418 
4419  vk->CopyImageToMemoryEXT(hwctx->act_dev, &copy_info);
4420  }
4421  }
4422 
4423  hwfc_vk->unlock_frame(hwfc, hwf_vk);
4424 
4425  return 0;
4426 }
4427 
4428 static int vulkan_transfer_frame(AVHWFramesContext *hwfc,
4429  AVFrame *swf, AVFrame *hwf,
4430  int upload)
4431 {
4432  int err;
4433  VulkanFramesPriv *fp = hwfc->hwctx;
4434  AVVulkanFramesContext *hwctx = &fp->p;
4435  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4436  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4437 
4438  int host_mapped = 0;
4439 
4440  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4441  VkBufferImageCopy region[AV_NUM_DATA_POINTERS]; // always one per plane
4442 
4443  const int planes = av_pix_fmt_count_planes(swf->format);
4444  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format);
4445  const int nb_images = ff_vk_count_images(hwf_vk);
4446 
4447  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
4448  int nb_img_bar = 0;
4449 
4450  AVBufferRef *bufs[AV_NUM_DATA_POINTERS];
4451  int nb_bufs = 0;
4452 
4453  VkCommandBuffer cmd_buf;
4454  FFVkExecContext *exec;
4455 
4456  /* Sanity checking */
4457  if ((swf->format != AV_PIX_FMT_NONE && !av_vkfmt_from_pixfmt(swf->format))) {
4458  av_log(hwfc, AV_LOG_ERROR, "Unsupported software frame pixel format!\n");
4459  return AVERROR(EINVAL);
4460  }
4461 
4462  if (swf->width > hwfc->width || swf->height > hwfc->height)
4463  return AVERROR(EINVAL);
4464 
4465  if (hwctx->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT)
4466  return vulkan_transfer_host(hwfc, hwf, swf, upload);
4467 
4468  for (int i = 0; i < av_pix_fmt_count_planes(swf->format); i++) {
4469  uint32_t p_w, p_h;
4470  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4471 
4472  /* Buffer region for this plane */
4473  region[i] = (VkBufferImageCopy) {
4474  .bufferOffset = 0,
4475  .bufferRowLength = swf->linesize[i],
4476  .bufferImageHeight = p_h,
4477  .imageSubresource.layerCount = 1,
4478  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4479  /* Rest of the fields adjusted/filled in later */
4480  };
4481  }
4482 
4483  /* Setup buffers first */
4484  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY && !p->avoid_host_import) {
4485  err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload);
4486  if (err >= 0)
4487  host_mapped = 1;
4488  }
4489 
4490  if (!host_mapped) {
4491  err = get_plane_buf(hwfc, &bufs[0], swf, region, upload);
4492  if (err < 0)
4493  goto end;
4494  nb_bufs = 1;
4495 
4496  if (upload) {
4497  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 1);
4498  if (err < 0)
4499  goto end;
4500  }
4501  }
4502 
4503  exec = ff_vk_exec_get(&p->vkctx, &fp->upload_exec);
4504  cmd_buf = exec->buf;
4505 
4506  ff_vk_exec_start(&p->vkctx, exec);
4507 
4508  /* Prep destination Vulkan frame */
4509  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, hwf,
4510  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4511  VK_PIPELINE_STAGE_2_TRANSFER_BIT);
4512  if (err < 0)
4513  goto end;
4514 
4515  /* No need to declare buf deps for synchronous transfers (downloads) */
4516  if (upload) {
4517  /* Add the software frame backing the buffers if we're host mapping */
4518  if (host_mapped) {
4519  err = ff_vk_exec_add_dep_sw_frame(&p->vkctx, exec, swf);
4520  if (err < 0) {
4521  ff_vk_exec_discard_deps(&p->vkctx, exec);
4522  goto end;
4523  }
4524  }
4525 
4526  /* Add the buffers as a dependency */
4527  err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, nb_bufs, 1);
4528  if (err < 0) {
4529  ff_vk_exec_discard_deps(&p->vkctx, exec);
4530  goto end;
4531  }
4532  }
4533 
4534  ff_vk_frame_barrier(&p->vkctx, exec, hwf, img_bar, &nb_img_bar,
4535  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4536  VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR,
4537  upload ? VK_ACCESS_TRANSFER_WRITE_BIT :
4538  VK_ACCESS_TRANSFER_READ_BIT,
4539  upload ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL :
4540  VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
4541  p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0]);
4542 
4543  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
4544  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
4545  .pImageMemoryBarriers = img_bar,
4546  .imageMemoryBarrierCount = nb_img_bar,
4547  });
4548 
4549  for (int i = 0; i < planes; i++) {
4550  int buf_idx = FFMIN(i, (nb_bufs - 1));
4551  int img_idx = FFMIN(i, (nb_images - 1));
4552  FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[buf_idx]->data;
4553 
4554  uint32_t orig_stride = region[i].bufferRowLength;
4555  region[i].bufferRowLength /= desc->comp[i].step;
4556  region[i].imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4557 
4558  if (upload)
4559  vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf,
4560  hwf_vk->img[img_idx],
4561  img_bar[img_idx].newLayout,
4562  1, &region[i]);
4563  else
4564  vk->CmdCopyImageToBuffer(cmd_buf, hwf_vk->img[img_idx],
4565  img_bar[img_idx].newLayout,
4566  vkbuf->buf,
4567  1, &region[i]);
4568 
4569  region[i].bufferRowLength = orig_stride;
4570  }
4571 
4572  err = ff_vk_exec_submit(&p->vkctx, exec);
4573  if (err < 0) {
4574  ff_vk_exec_discard_deps(&p->vkctx, exec);
4575  } else if (!upload) {
4576  ff_vk_exec_wait(&p->vkctx, exec);
4577  if (!host_mapped)
4578  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 0);
4579  }
4580 
4581 end:
4582  for (int i = 0; i < nb_bufs; i++)
4583  av_buffer_unref(&bufs[i]);
4584 
4585  return err;
4586 }
4587 
4588 static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst,
4589  const AVFrame *src)
4590 {
4591  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4592 
4593  switch (src->format) {
4594 #if CONFIG_CUDA
4595  case AV_PIX_FMT_CUDA:
4596 #ifdef _WIN32
4597  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4598  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4599 #else
4600  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4601  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4602 #endif
4603  return vulkan_transfer_data_from_cuda(hwfc, dst, src);
4604 #endif
4605  default:
4606  if (src->hw_frames_ctx)
4607  return AVERROR(ENOSYS);
4608  else
4609  return vulkan_transfer_frame(hwfc, (AVFrame *)src, dst, 1);
4610  }
4611 }
4612 
4613 #if CONFIG_CUDA
4614 static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst,
4615  const AVFrame *src)
4616 {
4617  int err;
4618  CUcontext dummy;
4619  AVVkFrame *dst_f;
4620  AVVkFrameInternal *dst_int;
4621  VulkanFramesPriv *fp = hwfc->hwctx;
4622  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
4623  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
4624  int nb_images;
4625 
4626  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)dst->hw_frames_ctx->data;
4627  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
4628  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
4629  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
4630  CudaFunctions *cu = cu_internal->cuda_dl;
4631  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
4632  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
4633 
4634  dst_f = (AVVkFrame *)src->data[0];
4635  nb_images = ff_vk_count_images(dst_f);
4636 
4637  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
4638  if (err < 0)
4639  return err;
4640 
4641  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
4642  if (err < 0)
4643  return err;
4644 
4645  err = vulkan_export_to_cuda(hwfc, dst->hw_frames_ctx, src);
4646  if (err < 0) {
4647  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4648  return err;
4649  }
4650 
4651  dst_int = dst_f->internal;
4652 
4653  for (int i = 0; i < planes; i++) {
4654  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
4655  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
4656  }
4657 
4658  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
4659  nb_images, cuda_dev->stream));
4660  if (err < 0)
4661  goto fail;
4662 
4663  for (int i = 0; i < planes; i++) {
4664  CUDA_MEMCPY2D cpy = {
4665  .dstMemoryType = CU_MEMORYTYPE_DEVICE,
4666  .dstDevice = (CUdeviceptr)dst->data[i],
4667  .dstPitch = dst->linesize[i],
4668  .dstY = 0,
4669 
4670  .srcMemoryType = CU_MEMORYTYPE_ARRAY,
4671  .srcArray = dst_int->cu_array[i],
4672  };
4673 
4674  int w, h;
4675  get_plane_wh(&w, &h, hwfc->sw_format, hwfc->width, hwfc->height, i);
4676 
4677  cpy.WidthInBytes = w * desc->comp[i].step;
4678  cpy.Height = h;
4679 
4680  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
4681  if (err < 0)
4682  goto fail;
4683  }
4684 
4685  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
4686  nb_images, cuda_dev->stream));
4687  if (err < 0)
4688  goto fail;
4689 
4690  for (int i = 0; i < planes; i++)
4691  dst_f->sem_value[i]++;
4692 
4693  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4694 
4695  av_log(hwfc, AV_LOG_VERBOSE, "Transferred Vulkan image to CUDA!\n");
4696 
4697  return prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
4698 
4699 fail:
4700  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4701  vulkan_free_internal(dst_f);
4702  av_buffer_unref(&dst->buf[0]);
4703  return err;
4704 }
4705 #endif
4706 
4707 static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst,
4708  const AVFrame *src)
4709 {
4710  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4711 
4712  switch (dst->format) {
4713 #if CONFIG_CUDA
4714  case AV_PIX_FMT_CUDA:
4715 #ifdef _WIN32
4716  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4717  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4718 #else
4719  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4720  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4721 #endif
4722  return vulkan_transfer_data_to_cuda(hwfc, dst, src);
4723 #endif
4724  default:
4725  if (dst->hw_frames_ctx)
4726  return AVERROR(ENOSYS);
4727  else
4728  return vulkan_transfer_frame(hwfc, dst, (AVFrame *)src, 0);
4729  }
4730 }
4731 
4732 static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc,
4733  AVHWFramesContext *src_fc, int flags)
4734 {
4735  return vulkan_frames_init(dst_fc);
4736 }
4737 
4738 AVVkFrame *av_vk_frame_alloc(void)
4739 {
4740  int err;
4741  AVVkFrame *f = av_mallocz(sizeof(AVVkFrame));
4742  if (!f)
4743  return NULL;
4744 
4745  f->internal = av_mallocz(sizeof(*f->internal));
4746  if (!f->internal) {
4747  av_free(f);
4748  return NULL;
4749  }
4750 
4751  err = pthread_mutex_init(&f->internal->update_mutex, NULL);
4752  if (err != 0) {
4753  av_free(f->internal);
4754  av_free(f);
4755  return NULL;
4756  }
4757 
4758  return f;
4759 }
4760 
4761 const HWContextType ff_hwcontext_type_vulkan = {
4762  .type = AV_HWDEVICE_TYPE_VULKAN,
4763  .name = "Vulkan",
4764 
4765  .device_hwctx_size = sizeof(VulkanDevicePriv),
4766  .frames_hwctx_size = sizeof(VulkanFramesPriv),
4767 
4768  .device_init = &vulkan_device_init,
4769  .device_uninit = &vulkan_device_uninit,
4770  .device_create = &vulkan_device_create,
4771  .device_derive = &vulkan_device_derive,
4772 
4773  .frames_get_constraints = &vulkan_frames_get_constraints,
4774  .frames_init = vulkan_frames_init,
4775  .frames_get_buffer = vulkan_get_buffer,
4776  .frames_uninit = vulkan_frames_uninit,
4777 
4778  .transfer_get_formats = vulkan_transfer_get_formats,
4779  .transfer_data_to = vulkan_transfer_data_to,
4780  .transfer_data_from = vulkan_transfer_data_from,
4781 
4782  .map_to = vulkan_map_to,
4783  .map_from = vulkan_map_from,
4784  .frames_derive_to = &vulkan_frames_derive_to,
4785 
4786  .pix_fmts = (const enum AVPixelFormat []) {
4787  AV_PIX_FMT_VULKAN,
4788  AV_PIX_FMT_NONE
4789  },
4790 };
flags
const SwsFlags flags[]
Definition: swscale.c:61
vulkan_loader.h
formats
formats
Definition: signature.h:47
AV_PIX_FMT_YUVA422P16
#define AV_PIX_FMT_YUVA422P16
Definition: pixfmt.h:596
load_libvulkan
static int load_libvulkan(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:585
pthread_mutex_t
_fmutex pthread_mutex_t
Definition: os2threads.h:53
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:88
FFHWFramesContext::pool_internal
AVBufferPool * pool_internal
Definition: hwcontext_internal.h:101
vulkan_device_init
static int vulkan_device_init(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1831
AV_PIX_FMT_GBRAP16
#define AV_PIX_FMT_GBRAP16
Definition: pixfmt.h:565
FF_ENABLE_DEPRECATION_WARNINGS
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:73
ff_vk_load_props
int ff_vk_load_props(FFVulkanContext *s)
Loads props/mprops/driver_props.
Definition: vulkan.c:142
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:216
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:79
AV_PIX_FMT_CUDA
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition: pixfmt.h:260
VulkanDeviceFeatures::vulkan_1_2
VkPhysicalDeviceVulkan12Features vulkan_1_2
Definition: hwcontext_vulkan.c:79
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
VulkanDevicePriv::libvulkan
void * libvulkan
Definition: hwcontext_vulkan.c:117
VulkanOptExtension::name
const char * name
Definition: hwcontext_vulkan.c:624
FFVkFormatEntry::nb_images
int nb_images
Definition: hwcontext_vulkan.c:356
host_map_frame
static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4253
AVCUDADeviceContextInternal
Definition: hwcontext_cuda_internal.h:31
AV_PIX_FMT_GRAY32
#define AV_PIX_FMT_GRAY32
Definition: pixfmt.h:523
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:202
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
hwcontext_cuda_internal.h
FFVulkanExtensions
uint64_t FFVulkanExtensions
Definition: vulkan_functions.h:29
AVBufferPool
The buffer pool.
Definition: buffer_internal.h:88
SET_OLD_QF
#define SET_OLD_QF(field, nb_field, type)
vulkan_transfer_data_to
static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4588
FF_VK_EXT_EXTERNAL_WIN32_MEMORY
#define FF_VK_EXT_EXTERNAL_WIN32_MEMORY
Definition: vulkan_functions.h:39
FF_VK_EXT_VIDEO_QUEUE
#define FF_VK_EXT_VIDEO_QUEUE
Definition: vulkan_functions.h:55
thread.h
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3447
CHECK_QUEUE
#define CHECK_QUEUE(type, required, fidx, ctx_qf, qc)
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
pthread_mutex_init
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, AVVulkanDeviceQueueFamily *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:356
FF_VK_EXT_PORTABILITY_SUBSET
#define FF_VK_EXT_PORTABILITY_SUBSET
Definition: vulkan_functions.h:70
vulkan_frames_get_constraints
static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx, const void *hwconfig, AVHWFramesConstraints *constraints)
Definition: hwcontext_vulkan.c:2192
FF_VK_EXT_VIDEO_MAINTENANCE_2
#define FF_VK_EXT_VIDEO_MAINTENANCE_2
Definition: vulkan_functions.h:57
AVVkFrameInternal::update_mutex
pthread_mutex_t update_mutex
Definition: hwcontext_vulkan.c:187
FF_VULKAN_DEBUG_PROFILE
@ FF_VULKAN_DEBUG_PROFILE
Definition: hwcontext_vulkan.c:745
av_unused
#define av_unused
Definition: attributes.h:131
vulkan_frames_derive_to
static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc, AVHWFramesContext *src_fc, int flags)
Definition: hwcontext_vulkan.c:4732
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:64
PICK_QF
#define PICK_QF(type, vid_op)
FF_VK_EXT_VIDEO_DECODE_H265
#define FF_VK_EXT_VIDEO_DECODE_H265
Definition: vulkan_functions.h:61
mode
Definition: swscale.c:56
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:427
pixdesc.h
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to a vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:69
optional_device_exts
static const VulkanOptExtension optional_device_exts[]
Definition: hwcontext_vulkan.c:635
AVFrame::width
int width
Definition: frame.h:499
AV_PIX_FMT_YUVA420P16
#define AV_PIX_FMT_YUVA420P16
Definition: pixfmt.h:595
w
uint8_t w
Definition: llviddspenc.c:38
AV_PIX_FMT_Y216
#define AV_PIX_FMT_Y216
Definition: pixfmt.h:608
create_frame
static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame, VkImageTiling tiling, VkImageUsageFlagBits usage, VkImageCreateFlags flags, int nb_layers, void *create_pnext)
Definition: hwcontext_vulkan.c:2557
AVDRMFrameDescriptor::nb_layers
int nb_layers
Number of layers in the frame.
Definition: hwcontext_drm.h:145
AV_PIX_FMT_DRM_PRIME
@ AV_PIX_FMT_DRM_PRIME
DRM-managed buffers exposed through PRIME buffer sharing.
Definition: pixfmt.h:351
AV_PIX_FMT_YUVA420P10
#define AV_PIX_FMT_YUVA420P10
Definition: pixfmt.h:590
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:238
ff_vk_find_struct
static const void * ff_vk_find_struct(const void *chain, VkStructureType stype)
Definition: vulkan.h:332
pthread_mutex_lock
static av_always_inline int pthread_mutex_lock(pthread_mutex_t *mutex)
Definition: os2threads.h:119
av_hwframe_map
int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
Map a hardware frame.
Definition: hwcontext.c:793
COPY_VAL
#define COPY_VAL(VAL)
nb_vk_formats_list
static const int nb_vk_formats_list
Definition: hwcontext_vulkan.c:469
data
const char data[16]
Definition: mxf.c:149
AVVulkanDeviceContext::queue_family_encode_index
attribute_deprecated int queue_family_encode_index
Queue family index for video encode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:152
AV_PIX_FMT_RGBA128
#define AV_PIX_FMT_RGBA128
Definition: pixfmt.h:630
try_export_flags
static void try_export_flags(AVHWFramesContext *hwfc, VkExternalMemoryHandleTypeFlags *comp_handle_types, VkExternalMemoryHandleTypeFlagBits *iexp, VkExternalMemoryHandleTypeFlagBits exp)
Definition: hwcontext_vulkan.c:2667
AV_PIX_FMT_YUV420P10
#define AV_PIX_FMT_YUV420P10
Definition: pixfmt.h:539
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:74
AV_PIX_FMT_XV30
#define AV_PIX_FMT_XV30
Definition: pixfmt.h:609
AVVulkanFramesContext::lock_frame
void(* lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Locks a frame, preventing other threads from changing frame properties.
Definition: hwcontext_vulkan.h:283
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:226
AVVAAPIDeviceContext::display
VADisplay display
The VADisplay handle, to be filled by the user.
Definition: hwcontext_vaapi.h:72
FF_VULKAN_DEBUG_PRACTICES
@ FF_VULKAN_DEBUG_PRACTICES
Definition: hwcontext_vulkan.c:743
vulkan_map_from
static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:4130
AV_PIX_FMT_BGR24
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:76
AV_PIX_FMT_BGRA
@ AV_PIX_FMT_BGRA
packed BGRA 8:8:8:8, 32bpp, BGRABGRA...
Definition: pixfmt.h:102
FFVkFormatEntry::vkf
VkFormat vkf
Definition: hwcontext_vulkan.c:352
ff_vk_exec_get
FFVkExecContext * ff_vk_exec_get(FFVulkanContext *s, FFVkExecPool *pool)
Retrieve an execution pool.
Definition: vulkan.c:547
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2964
AVDictionary
Definition: dict.c:32
ff_hwframe_map_create
int ff_hwframe_map_create(AVBufferRef *hwframe_ref, AVFrame *dst, const AVFrame *src, void(*unmap)(AVHWFramesContext *ctx, HWMapDescriptor *hwmap), void *priv)
Definition: hwcontext.c:741
FFMAX
#define FFMAX(a, b)
Definition: macros.h:47
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
HWMapDescriptor::priv
void * priv
Hardware-specific private data associated with the mapping.
Definition: hwcontext_internal.h:139
FF_VK_EXT_COOP_MATRIX
#define FF_VK_EXT_COOP_MATRIX
Definition: vulkan_functions.h:45
av_popcount
#define av_popcount
Definition: common.h:154
AVDRMFrameDescriptor
DRM frame descriptor.
Definition: hwcontext_drm.h:133
AVHWFramesConstraints::valid_hw_formats
enum AVPixelFormat * valid_hw_formats
A list of possible values for format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:449
AVERROR_UNKNOWN
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:73
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:220
AVFrame::buf
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:604
AV_PIX_FMT_YUVA422P10
#define AV_PIX_FMT_YUVA422P10
Definition: pixfmt.h:591
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
VulkanDeviceSelection::uuid
uint8_t uuid[VK_UUID_SIZE]
Definition: hwcontext_vulkan.c:1208
ff_vk_exec_add_dep_frame
int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage)
Definition: vulkan.c:779
FF_VULKAN_DEBUG_PRINTF
@ FF_VULKAN_DEBUG_PRINTF
Definition: hwcontext_vulkan.c:741
AV_PIX_FMT_P212
#define AV_PIX_FMT_P212
Definition: pixfmt.h:618
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:448
FFVkBuffer::buf
VkBuffer buf
Definition: vulkan.h:88
VulkanDeviceFeatures::host_image_copy
VkPhysicalDeviceHostImageCopyFeaturesEXT host_image_copy
Definition: hwcontext_vulkan.c:83
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:374
alloc_mem
static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req, VkMemoryPropertyFlagBits req_flags, const void *alloc_extension, VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
Definition: hwcontext_vulkan.c:2238
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
VulkanDevicePriv::compute_qf
AVVulkanDeviceQueueFamily * compute_qf
Definition: hwcontext_vulkan.c:120
AVHWFramesConstraints
This struct describes the constraints on hardware frames attached to a given device with a hardware-s...
Definition: hwcontext.h:444
FF_VK_EXT_HOST_IMAGE_COPY
#define FF_VK_EXT_HOST_IMAGE_COPY
Definition: vulkan_functions.h:52
AV_HWDEVICE_TYPE_CUDA
@ AV_HWDEVICE_TYPE_CUDA
Definition: hwcontext.h:30
FF_VK_EXT_EXPECT_ASSUME
#define FF_VK_EXT_EXPECT_ASSUME
Definition: vulkan_functions.h:50
AVDRMDeviceContext::fd
int fd
File descriptor of DRM device.
Definition: hwcontext_drm.h:166
PREP_MODE_DECODING_DPB
@ PREP_MODE_DECODING_DPB
Definition: hwcontext_vulkan.c:2446
FF_VK_EXT_EXTERNAL_FD_SEM
#define FF_VK_EXT_EXTERNAL_FD_SEM
Definition: vulkan_functions.h:35
VulkanDeviceFeatures::device
VkPhysicalDeviceFeatures2 device
Definition: hwcontext_vulkan.c:76
VulkanDeviceFeatures::video_maintenance_1
VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1
Definition: hwcontext_vulkan.c:89
close
static av_cold void close(AVCodecParserContext *s)
Definition: apv_parser.c:135
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3487
ASPECT_3PLANE
#define ASPECT_3PLANE
Definition: hwcontext_vulkan.c:349
VulkanDevicePriv::hprops
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops
Definition: hwcontext_vulkan.c:126
vulkan_device_derive
static int vulkan_device_derive(AVHWDeviceContext *ctx, AVHWDeviceContext *src_ctx, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:2089
VulkanOptExtension::flag
FFVulkanExtensions flag
Definition: hwcontext_vulkan.c:625
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AV_PIX_FMT_GBRP14
#define AV_PIX_FMT_GBRP14
Definition: pixfmt.h:560
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:302
AV_PIX_FMT_GBRAP
@ AV_PIX_FMT_GBRAP
planar GBRA 4:4:4:4 32bpp
Definition: pixfmt.h:212
VulkanDeviceSelection::drm_minor
uint32_t drm_minor
Definition: hwcontext_vulkan.c:1211
lock_frame
static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2804
fail
#define fail()
Definition: checkasm.h:201
AVDRMLayerDescriptor::nb_planes
int nb_planes
Number of planes in the layer.
Definition: hwcontext_drm.h:106
ff_vk_exec_add_dep_bool_sem
int ff_vk_exec_add_dep_bool_sem(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore *sem, int nb, VkPipelineStageFlagBits2 stage, int wait)
Definition: vulkan.c:713
AV_PIX_FMT_GBRP10
#define AV_PIX_FMT_GBRP10
Definition: pixfmt.h:558
AV_PIX_FMT_YUVA444P16
#define AV_PIX_FMT_YUVA444P16
Definition: pixfmt.h:597
AVVulkanFramesContext::flags
AVVkFrameFlags flags
A combination of AVVkFrameFlags.
Definition: hwcontext_vulkan.h:254
VulkanDevicePriv
Definition: hwcontext_vulkan.c:110
AVVulkanDeviceContext::nb_tx_queues
attribute_deprecated int nb_tx_queues
Definition: hwcontext_vulkan.h:135
AVDRMLayerDescriptor::planes
AVDRMPlaneDescriptor planes[AV_DRM_MAX_PLANES]
Array of planes in this layer.
Definition: hwcontext_drm.h:110
dummy
int dummy
Definition: motion.c:66
AVVkFrame::mem
VkDeviceMemory mem[AV_NUM_DATA_POINTERS]
Memory backing the images.
Definition: hwcontext_vulkan.h:315
device_features_copy_needed
static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
Definition: hwcontext_vulkan.c:264
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:208
av_buffer_pool_init2
AVBufferPool * av_buffer_pool_init2(size_t size, void *opaque, AVBufferRef *(*alloc)(void *opaque, size_t size), void(*pool_free)(void *opaque))
Allocate and initialize a buffer pool with a more complex allocator.
Definition: buffer.c:259
AVHWFramesConstraints::min_width
int min_width
The minimum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:462
lock_queue
static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1819
AVCUDADeviceContextInternal::cuda_device
CUdevice cuda_device
Definition: hwcontext_cuda_internal.h:34
VulkanDevicePriv::limit_queues
int limit_queues
Definition: hwcontext_vulkan.c:160
VulkanDevicePriv::vkctx
FFVulkanContext vkctx
Definition: hwcontext_vulkan.c:119
AV_PIX_FMT_XV48
#define AV_PIX_FMT_XV48
Definition: pixfmt.h:611
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
FF_VK_EXT_VIDEO_ENCODE_H265
#define FF_VK_EXT_VIDEO_ENCODE_H265
Definition: vulkan_functions.h:67
ff_vk_host_map_buffer
int ff_vk_host_map_buffer(FFVulkanContext *s, AVBufferRef **dst, uint8_t *src_data, const AVBufferRef *src_buf, VkBufferUsageFlags usage)
Maps a system RAM buffer into a Vulkan buffer.
Definition: vulkan.c:1361
ff_vk_ret2str
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:35
AV_PIX_FMT_GRAY16
#define AV_PIX_FMT_GRAY16
Definition: pixfmt.h:522
VulkanDeviceFeatures::vulkan_1_3
VkPhysicalDeviceVulkan13Features vulkan_1_3
Definition: hwcontext_vulkan.c:80
FF_VK_EXT_EXTERNAL_WIN32_SEM
#define FF_VK_EXT_EXTERNAL_WIN32_SEM
Definition: vulkan_functions.h:40
AVVulkanDeviceContext::queue_family_decode_index
attribute_deprecated int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:162
VulkanDevicePriv::props
VkPhysicalDeviceProperties2 props
Definition: hwcontext_vulkan.c:124
ff_vk_aspect_flag
VkImageAspectFlags ff_vk_aspect_flag(AVFrame *f, int p)
Get the aspect flag for a plane from an image.
Definition: vulkan.c:1512
VulkanFramesPriv::drm_format_modifier_properties
VkDrmFormatModifierPropertiesEXT drm_format_modifier_properties[5]
Definition: hwcontext_vulkan.c:183
vk_find_format_entry
static const struct FFVkFormatEntry * vk_find_format_entry(enum AVPixelFormat p)
Definition: hwcontext_vulkan.c:479
AVDRMPlaneDescriptor::offset
ptrdiff_t offset
Offset within that object of this plane.
Definition: hwcontext_drm.h:83
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:63
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:52
AV_PIX_FMT_YUV444P10
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:542
vk_dbg_callback
static VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT messageType, const VkDebugUtilsMessengerCallbackDataEXT *data, void *priv)
Definition: hwcontext_vulkan.c:681
AV_PIX_FMT_Y210
#define AV_PIX_FMT_Y210
Definition: pixfmt.h:606
AVVulkanDeviceQueueFamily::num
int num
Definition: hwcontext_vulkan.h:37
HWContextType::type
enum AVHWDeviceType type
Definition: hwcontext_internal.h:30
ffhwframesctx
static FFHWFramesContext * ffhwframesctx(AVHWFramesContext *ctx)
Definition: hwcontext_internal.h:115
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:210
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
ff_vk_link_struct
static void ff_vk_link_struct(void *chain, const void *in)
Definition: vulkan.h:345
check_layers
static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:919
AV_PIX_FMT_YUV422P16
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:551
AVVulkanDeviceContext::nb_encode_queues
attribute_deprecated int nb_encode_queues
Definition: hwcontext_vulkan.h:154
AVHWFramesContext::height
int height
Definition: hwcontext.h:220
AVHWFramesConstraints::valid_sw_formats
enum AVPixelFormat * valid_sw_formats
A list of possible values for sw_format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:456
vulkan_map_to
static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:3925
av_dict_get
AVDictionaryEntry * av_dict_get(const AVDictionary *m, const char *key, const AVDictionaryEntry *prev, int flags)
Get a dictionary entry with matching key.
Definition: dict.c:60
av_buffer_pool_get
AVBufferRef * av_buffer_pool_get(AVBufferPool *pool)
Allocate a new AVBuffer, reusing an old buffer from the pool when available.
Definition: buffer.c:390
AV_PIX_FMT_GBRAP10
#define AV_PIX_FMT_GBRAP10
Definition: pixfmt.h:562
AVHWFramesContext::pool
AVBufferPool * pool
A pool from which the frames are allocated by av_hwframe_get_buffer().
Definition: hwcontext.h:181
VulkanDeviceSelection::pci_device
uint32_t pci_device
Definition: hwcontext_vulkan.c:1214
s
#define s(width, name)
Definition: cbs_vp9.c:198
AV_PIX_FMT_GBRAP14
#define AV_PIX_FMT_GBRAP14
Definition: pixfmt.h:564
AV_PIX_FMT_GBRAP12
#define AV_PIX_FMT_GBRAP12
Definition: pixfmt.h:563
AV_PIX_FMT_YUVA420P
@ AV_PIX_FMT_YUVA420P
planar YUV 4:2:0, 20bpp, (1 Cr & Cb sample per 2x2 Y & A samples)
Definition: pixfmt.h:108
AV_PIX_FMT_RGB96
#define AV_PIX_FMT_RGB96
Definition: pixfmt.h:629
pthread_mutex_unlock
static av_always_inline int pthread_mutex_unlock(pthread_mutex_t *mutex)
Definition: os2threads.h:126
AV_PIX_FMT_YUV444P16
#define AV_PIX_FMT_YUV444P16
Definition: pixfmt.h:552
offsets
static const int offsets[]
Definition: hevc_pel.c:34
AV_CEIL_RSHIFT
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:60
ff_vk_load_functions
static int ff_vk_load_functions(AVHWDeviceContext *ctx, FFVulkanFunctions *vk, uint64_t extensions_mask, int has_inst, int has_dev)
Function loader.
Definition: vulkan_loader.h:119
VulkanDevicePriv::ext_sem_props_opaque
VkExternalSemaphoreProperties ext_sem_props_opaque
Definition: hwcontext_vulkan.c:130
prepare_frame
static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2450
vulkan_transfer_frame
static int vulkan_transfer_frame(AVHWFramesContext *hwfc, AVFrame *swf, AVFrame *hwf, int upload)
Definition: hwcontext_vulkan.c:4428
FF_VK_EXT_DEVICE_DRM
#define FF_VK_EXT_DEVICE_DRM
Definition: vulkan_functions.h:43
ff_vk_exec_wait
void ff_vk_exec_wait(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:552
av_strtok
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
Definition: avstring.c:179
ASPECT_2PLANE
#define ASPECT_2PLANE
Definition: hwcontext_vulkan.c:348
vulkan_device_uninit
static void vulkan_device_uninit(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1680
fc
#define fc(width, name, range_min, range_max)
Definition: cbs_av1.c:494
AVVulkanFramesContext::unlock_frame
void(* unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Similar to lock_frame(), unlocks a frame.
Definition: hwcontext_vulkan.h:288
AVVulkanFramesContext::img_flags
VkImageCreateFlags img_flags
Flags to set during image creation.
Definition: hwcontext_vulkan.h:260
AV_PIX_FMT_GBRAP32
#define AV_PIX_FMT_GBRAP32
Definition: pixfmt.h:566
AV_PIX_FMT_YUVA444P12
#define AV_PIX_FMT_YUVA444P12
Definition: pixfmt.h:594
vulkan_frames_uninit
static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2814
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:231
AV_PIX_FMT_YUV420P16
#define AV_PIX_FMT_YUV420P16
Definition: pixfmt.h:550
ctx
AVFormatContext * ctx
Definition: movenc.c:49
AVDRMObjectDescriptor::fd
int fd
DRM PRIME fd for the object.
Definition: hwcontext_drm.h:52
AV_PIX_FMT_GRAY14
#define AV_PIX_FMT_GRAY14
Definition: pixfmt.h:521
ff_vk_exec_add_dep_buf
int ff_vk_exec_add_dep_buf(FFVulkanContext *s, FFVkExecContext *e, AVBufferRef **deps, int nb_deps, int ref)
Execution dependency management.
Definition: vulkan.c:619
VulkanDeviceSelection::index
int index
Definition: hwcontext_vulkan.c:1216
AV_PIX_FMT_RGBF32
#define AV_PIX_FMT_RGBF32
Definition: pixfmt.h:626
VulkanFramesPriv::p
AVVulkanFramesContext p
The public AVVulkanFramesContext.
Definition: hwcontext_vulkan.c:167
vulkan_transfer_get_formats
static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats)
Definition: hwcontext_vulkan.c:3073
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:73
VulkanDevicePriv::avoid_host_import
int avoid_host_import
Definition: hwcontext_vulkan.c:157
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:287
FFVulkanDebugMode
FFVulkanDebugMode
Definition: hwcontext_vulkan.c:736
AV_PIX_FMT_GRAYF32
#define AV_PIX_FMT_GRAYF32
Definition: pixfmt.h:582
LIBAVUTIL_VERSION_MINOR
#define LIBAVUTIL_VERSION_MINOR
Definition: version.h:82
tmp
static uint8_t tmp[40]
Definition: aes_ctr.c:52
AV_PIX_FMT_RGBA
@ AV_PIX_FMT_RGBA
packed RGBA 8:8:8:8, 32bpp, RGBARGBA...
Definition: pixfmt.h:100
FFVkFormatEntry::nb_images_fallback
int nb_images_fallback
Definition: hwcontext_vulkan.c:357
vulkan_frame_free_cb
static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
Definition: hwcontext_vulkan.c:2366
AV_PIX_FMT_GRAY10
#define AV_PIX_FMT_GRAY10
Definition: pixfmt.h:519
if
if(ret)
Definition: filter_design.txt:179
av_vkfmt_from_pixfmt
const VkFormat * av_vkfmt_from_pixfmt(enum AVPixelFormat p)
Returns the optimal per-plane Vulkan format for a given sw_format, one for each plane.
Definition: hwcontext_vulkan.c:471
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:59
VulkanDevicePriv::qf_mutex
pthread_mutex_t ** qf_mutex
Definition: hwcontext_vulkan.c:136
AV_PIX_FMT_GBRP16
#define AV_PIX_FMT_GBRP16
Definition: pixfmt.h:561
opts
AVDictionary * opts
Definition: movenc.c:51
PREP_MODE_WRITE
@ PREP_MODE_WRITE
Definition: hwcontext_vulkan.c:2442
AV_PIX_FMT_RGBA64
#define AV_PIX_FMT_RGBA64
Definition: pixfmt.h:529
NULL
#define NULL
Definition: coverity.c:32
pick_video_queue_family
static int pick_video_queue_family(VkQueueFamilyProperties2 *qf, VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf, VkVideoCodecOperationFlagBitsKHR flags)
Definition: hwcontext_vulkan.c:1431
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:213
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
format
New swscale design to change SwsGraph is what coordinates multiple passes These can include cascaded scaling error diffusion and so on Or we could have separate passes for the vertical and horizontal scaling In between each SwsPass lies a fully allocated image buffer Graph passes may have different levels of e g we can have a single threaded error diffusion pass following a multi threaded scaling pass SwsGraph is internally recreated whenever the image format
Definition: swscale-v2.txt:14
FF_VK_EXT_DRM_MODIFIER_FLAGS
#define FF_VK_EXT_DRM_MODIFIER_FLAGS
Definition: vulkan_functions.h:33
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:113
FFVkFormatEntry
Definition: hwcontext_vulkan.c:351
AVPixFmtDescriptor::nb_components
uint8_t nb_components
The number of components each pixel has, (1-4)
Definition: pixdesc.h:71
FF_VK_EXT_SHADER_OBJECT
#define FF_VK_EXT_SHADER_OBJECT
Definition: vulkan_functions.h:47
AV_PIX_FMT_YUYV422
@ AV_PIX_FMT_YUYV422
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:74
AVVkFrameInternal
Definition: hwcontext_vulkan.c:186
FF_VK_EXT_VIDEO_DECODE_VP9
#define FF_VK_EXT_VIDEO_DECODE_VP9
Definition: vulkan_functions.h:62
FF_VK_EXT_SUBGROUP_ROTATE
#define FF_VK_EXT_SUBGROUP_ROTATE
Definition: vulkan_functions.h:51
FF_VK_EXT_VIDEO_ENCODE_QUEUE
#define FF_VK_EXT_VIDEO_ENCODE_QUEUE
Definition: vulkan_functions.h:65
VulkanDevicePriv::debug_ctx
VkDebugUtilsMessengerEXT debug_ctx
Definition: hwcontext_vulkan.c:142
FF_VULKAN_DEBUG_NONE
@ FF_VULKAN_DEBUG_NONE
Definition: hwcontext_vulkan.c:737
AVVulkanFramesContext::alloc_pnext
void * alloc_pnext[AV_NUM_DATA_POINTERS]
Extension data for memory allocation.
Definition: hwcontext_vulkan.h:247
setup_queue_families
static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
Definition: hwcontext_vulkan.c:1460
AVVulkanDeviceContext::unlock_queue
void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:178
LIBAVUTIL_VERSION_MAJOR
#define LIBAVUTIL_VERSION_MAJOR
Definition: version.h:81
AVVulkanDeviceContext::nb_decode_queues
attribute_deprecated int nb_decode_queues
Definition: hwcontext_vulkan.h:164
AV_PIX_FMT_P410
#define AV_PIX_FMT_P410
Definition: pixfmt.h:617
av_buffer_pool_uninit
void av_buffer_pool_uninit(AVBufferPool **ppool)
Mark the pool as being available for freeing.
Definition: buffer.c:328
AVVulkanDeviceContext::nb_qf
int nb_qf
Definition: hwcontext_vulkan.h:189
ff_hwcontext_type_vulkan
const HWContextType ff_hwcontext_type_vulkan
Definition: hwcontext_vulkan.c:4761
hwcontext_vulkan.h
AVVulkanDeviceContext::queue_family_tx_index
attribute_deprecated int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:133
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:101
vk_formats_list
static const struct FFVkFormatEntry vk_formats_list[]
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:268
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:227
AV_PIX_FMT_BGR0
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:265
AV_PIX_FMT_YUV422P10
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:540
alloc_bind_mem
static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f, void *alloc_pnext, size_t alloc_pnext_stride)
Definition: hwcontext_vulkan.c:2371
AVVulkanDeviceContext::qf
AVVulkanDeviceQueueFamily qf[64]
Queue families used.
Definition: hwcontext_vulkan.h:188
AV_PIX_FMT_GRAY8
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
Definition: pixfmt.h:81
FFVulkanContext
Definition: vulkan.h:274
exp
int8_t exp
Definition: eval.c:73
VulkanFramesPriv
Definition: hwcontext_vulkan.c:163
vulkan_frame_free
static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
Definition: hwcontext_vulkan.c:2332
index
int index
Definition: gxfenc.c:90
av_buffer_create
AVBufferRef * av_buffer_create(uint8_t *data, size_t size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:55
vkfmt_from_pixfmt2
static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p, VkImageTiling tiling, VkFormat fmts[AV_NUM_DATA_POINTERS], int *nb_images, VkImageAspectFlags *aspect, VkImageUsageFlags *supported_usage, int disable_multiplane, int need_storage)
Definition: hwcontext_vulkan.c:487
VulkanDeviceSelection
Definition: hwcontext_vulkan.c:1207
source
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a source
Definition: filter_design.txt:256
VulkanDevicePriv::nb_tot_qfs
uint32_t nb_tot_qfs
Definition: hwcontext_vulkan.c:137
VulkanDeviceFeatures
Definition: hwcontext_vulkan.c:75
AVDRMFrameDescriptor::layers
AVDRMLayerDescriptor layers[AV_DRM_MAX_PLANES]
Array of layers in the frame.
Definition: hwcontext_drm.h:149
usage
const char * usage
Definition: floatimg_cmp.c:62
PREP_MODE_DECODING_DST
@ PREP_MODE_DECODING_DST
Definition: hwcontext_vulkan.c:2445
AVVkFrame::size
size_t size[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:316
vulkan_pool_alloc
static AVBufferRef * vulkan_pool_alloc(void *opaque, size_t size)
Definition: hwcontext_vulkan.c:2729
VulkanDevicePriv::dprops
VkPhysicalDeviceDriverProperties dprops
Definition: hwcontext_vulkan.c:127
AV_PIX_FMT_X2BGR10
#define AV_PIX_FMT_X2BGR10
Definition: pixfmt.h:614
AV_HWFRAME_MAP_WRITE
@ AV_HWFRAME_MAP_WRITE
The mapping must be writeable.
Definition: hwcontext.h:519
PrepMode
PrepMode
Definition: hwcontext_vulkan.c:2440
FF_VK_EXT_VIDEO_MAINTENANCE_1
#define FF_VK_EXT_VIDEO_MAINTENANCE_1
Definition: vulkan_functions.h:56
VulkanDeviceSelection::has_drm
uint32_t has_drm
Definition: hwcontext_vulkan.c:1212
f
f
Definition: af_crystalizer.c:122
AVCUDADeviceContext::internal
AVCUDADeviceContextInternal * internal
Definition: hwcontext_cuda.h:45
AV_PIX_FMT_RGB24
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:75
VulkanDeviceFeatures::vulkan_1_1
VkPhysicalDeviceVulkan11Features vulkan_1_1
Definition: hwcontext_vulkan.c:78
sem_wait
#define sem_wait(psem)
Definition: semaphore.h:27
AV_PIX_FMT_P012
#define AV_PIX_FMT_P012
Definition: pixfmt.h:603
AV_PIX_FMT_FLAG_RGB
#define AV_PIX_FMT_FLAG_RGB
The pixel format contains RGB-like data (as opposed to YUV/grayscale).
Definition: pixdesc.h:136
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition: dsp.h:87
AV_HWFRAME_MAP_READ
@ AV_HWFRAME_MAP_READ
The mapping must be readable.
Definition: hwcontext.h:515
AVVkFrame
Definition: hwcontext_vulkan.h:297
av_vk_frame_alloc
AVVkFrame * av_vk_frame_alloc(void)
Allocates a single AVVkFrame and initializes everything as 0.
Definition: hwcontext_vulkan.c:4738
FF_VULKAN_DEBUG_VALIDATE
@ FF_VULKAN_DEBUG_VALIDATE
Definition: hwcontext_vulkan.c:739
vulkan.h
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:122
FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
#define FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
Definition: vulkan_functions.h:32
vulkan_device_free
static void vulkan_device_free(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1657
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:424
FF_VK_EXT_NO_FLAG
#define FF_VK_EXT_NO_FLAG
Definition: vulkan_functions.h:71
AV_PIX_FMT_GBRPF32
#define AV_PIX_FMT_GBRPF32
Definition: pixfmt.h:578
AV_PIX_FMT_YUV422P12
#define AV_PIX_FMT_YUV422P12
Definition: pixfmt.h:544
VulkanDeviceFeatures::cooperative_matrix
VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix
Definition: hwcontext_vulkan.c:101
AV_PIX_FMT_RGB48
#define AV_PIX_FMT_RGB48
Definition: pixfmt.h:525
size
int size
Definition: twinvq_data.h:10344
ff_vk_exec_add_dep_sw_frame
int ff_vk_exec_add_dep_sw_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f)
Definition: vulkan.c:646
vulkan_transfer_data_from
static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4707
AV_NUM_DATA_POINTERS
#define AV_NUM_DATA_POINTERS
Definition: frame.h:428
FF_VK_EXT_PUSH_DESCRIPTOR
#define FF_VK_EXT_PUSH_DESCRIPTOR
Definition: vulkan_functions.h:48
VulkanDevicePriv::use_linear_images
int use_linear_images
Definition: hwcontext_vulkan.c:145
AV_PIX_FMT_YUV444P12
#define AV_PIX_FMT_YUV444P12
Definition: pixfmt.h:546
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:514
AV_PIX_FMT_NV16
@ AV_PIX_FMT_NV16
interleaved chroma YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:198
VulkanDeviceSelection::vendor_id
uint32_t vendor_id
Definition: hwcontext_vulkan.c:1215
PREP_MODE_GENERAL
@ PREP_MODE_GENERAL
Definition: hwcontext_vulkan.c:2441
AV_PIX_FMT_Y212
#define AV_PIX_FMT_Y212
Definition: pixfmt.h:607
AVVulkanDeviceContext::queue_family_index
attribute_deprecated int queue_family_index
Queue family index for graphics operations, and the number of queues enabled for it.
Definition: hwcontext_vulkan.h:124
AVDRMObjectDescriptor::size
size_t size
Total size of the object.
Definition: hwcontext_drm.h:58
AV_PIX_FMT_YUVA444P
@ AV_PIX_FMT_YUVA444P
planar YUV 4:4:4 32bpp, (1 Cr & Cb sample per 1x1 Y & A samples)
Definition: pixfmt.h:174
VulkanFramesPriv::upload_exec
FFVkExecPool upload_exec
Definition: hwcontext_vulkan.c:173
AVVulkanDeviceQueueFamily::idx
int idx
Definition: hwcontext_vulkan.h:35
AV_PIX_FMT_YUVA444P10
#define AV_PIX_FMT_YUVA444P10
Definition: pixfmt.h:592
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
AVHWFramesConstraints::max_width
int max_width
The maximum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:469
FFVkExecContext
Definition: vulkan.h:111
VulkanOptExtension
Definition: hwcontext_vulkan.c:623
AV_PIX_FMT_RGB0
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:263
AV_PIX_FMT_P216
#define AV_PIX_FMT_P216
Definition: pixfmt.h:620
CHECK_CU
#define CHECK_CU(x)
Definition: cuviddec.c:117
AV_PIX_FMT_P210
#define AV_PIX_FMT_P210
Definition: pixfmt.h:616
AV_PIX_FMT_VAAPI
@ AV_PIX_FMT_VAAPI
Hardware acceleration through VA-API, data[3] contains a VASurfaceID.
Definition: pixfmt.h:126
VulkanDeviceFeatures::subgroup_rotate
VkPhysicalDeviceShaderSubgroupRotateFeaturesKHR subgroup_rotate
Definition: hwcontext_vulkan.c:82
FF_VK_EXT_VIDEO_DECODE_QUEUE
#define FF_VK_EXT_VIDEO_DECODE_QUEUE
Definition: vulkan_functions.h:59
AVVulkanDeviceContext::lock_queue
void(* lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Locks a queue, preventing other threads from submitting any command buffers to this queue.
Definition: hwcontext_vulkan.h:173
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:221
vulkan_free_internal
static void vulkan_free_internal(AVVkFrame *f)
Definition: hwcontext_vulkan.c:2296
VulkanDeviceFeatures::timeline_semaphore
VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore
Definition: hwcontext_vulkan.c:81
AV_HWDEVICE_TYPE_VAAPI
@ AV_HWDEVICE_TYPE_VAAPI
Definition: hwcontext.h:31
pthread_mutex_destroy
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
FF_VK_EXT_EXTERNAL_HOST_MEMORY
#define FF_VK_EXT_EXTERNAL_HOST_MEMORY
Definition: vulkan_functions.h:36
planes
static const struct @527 planes[]
AV_PIX_FMT_UYVA
@ AV_PIX_FMT_UYVA
packed UYVA 4:4:4:4, 32bpp (1 Cr & Cb sample per 1x1 Y & A samples), UYVAUYVA...
Definition: pixfmt.h:444
AVDRMFrameDescriptor::objects
AVDRMObjectDescriptor objects[AV_DRM_MAX_PLANES]
Array of objects making up the frame.
Definition: hwcontext_drm.h:141
AVCUDADeviceContextInternal::cuda_dl
CudaFunctions * cuda_dl
Definition: hwcontext_cuda_internal.h:32
av_assert2
#define av_assert2(cond)
assert() equivalent, that does lie in speed critical code.
Definition: avassert.h:68
ff_vk_exec_start
int ff_vk_exec_start(FFVulkanContext *s, FFVkExecContext *e)
Start/submit/wait an execution.
Definition: vulkan.c:559
FF_VK_EXT_RELAXED_EXTENDED_INSTR
#define FF_VK_EXT_RELAXED_EXTENDED_INSTR
Definition: vulkan_functions.h:49
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
FF_VK_EXT_VIDEO_DECODE_H264
#define FF_VK_EXT_VIDEO_DECODE_H264
Definition: vulkan_functions.h:60
VulkanFramesPriv::compute_exec
FFVkExecPool compute_exec
Definition: hwcontext_vulkan.c:170
AVVulkanDeviceContext::queue_family_comp_index
attribute_deprecated int queue_family_comp_index
Queue family index for compute operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:142
VulkanDevicePriv::disable_host_transfer
int disable_host_transfer
Definition: hwcontext_vulkan.c:154
AVDRMObjectDescriptor::format_modifier
uint64_t format_modifier
Format modifier applied to the object (DRM_FORMAT_MOD_*).
Definition: hwcontext_drm.h:65
VkFormat
enum VkFormat VkFormat
Definition: hwcontext_stub.c:25
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:32
AV_PIX_FMT_GBRP12
#define AV_PIX_FMT_GBRP12
Definition: pixfmt.h:559
weights
static const int weights[]
Definition: hevc_pel.c:32
AV_PIX_FMT_NV24
@ AV_PIX_FMT_NV24
planar YUV 4:4:4, 24bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:371
plane_info
Definition: vf_edgedetect.c:53
VulkanDevicePriv::nb_img_qfs
uint32_t nb_img_qfs
Definition: hwcontext_vulkan.c:139
ff_vk_frame_barrier
void ff_vk_frame_barrier(FFVulkanContext *s, FFVkExecContext *e, AVFrame *pic, VkImageMemoryBarrier2 *bar, int *nb_bar, VkPipelineStageFlags src_stage, VkPipelineStageFlags dst_stage, VkAccessFlagBits new_access, VkImageLayout new_layout, uint32_t new_qf)
Definition: vulkan.c:2013
vulkan_frames_init
static int vulkan_frames_init(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2832
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
FFVkBuffer::mem
VkDeviceMemory mem
Definition: vulkan.h:89
AV_PIX_FMT_X2RGB10
#define AV_PIX_FMT_X2RGB10
Definition: pixfmt.h:613
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:256
VulkanDeviceSelection::has_uuid
int has_uuid
Definition: hwcontext_vulkan.c:1209
hwcontext_drm.h
AVDRMPlaneDescriptor::object_index
int object_index
Index of the object containing this plane in the objects array of the enclosing frame descriptor.
Definition: hwcontext_drm.h:79
AV_PIX_FMT_BGR565
#define AV_PIX_FMT_BGR565
Definition: pixfmt.h:531
ff_hwframe_map_replace
int ff_hwframe_map_replace(AVFrame *dst, const AVFrame *src)
Replace the current hwmap of dst with the one from src, used for indirect mappings like VAAPI->(DRM)-...
Definition: hwcontext.c:948
VulkanDevicePriv::img_qfs
uint32_t img_qfs[64]
Definition: hwcontext_vulkan.c:138
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:264
mod
static int mod(int a, int b)
Modulo operation with only positive remainders.
Definition: vf_v360.c:755
AV_PIX_FMT_P016
#define AV_PIX_FMT_P016
Definition: pixfmt.h:604
AV_PIX_FMT_RGB565
#define AV_PIX_FMT_RGB565
Definition: pixfmt.h:526
AVHWFrameTransferDirection
AVHWFrameTransferDirection
Definition: hwcontext.h:406
AVVkFrame::sem
VkSemaphore sem[AV_NUM_DATA_POINTERS]
Synchronization timeline semaphores, one for each VkImage.
Definition: hwcontext_vulkan.h:335
create_instance
static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:1070
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:118
FF_VK_EXT_EXTERNAL_FD_MEMORY
#define FF_VK_EXT_EXTERNAL_FD_MEMORY
Definition: vulkan_functions.h:34
AVCUDADeviceContext
This struct is allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_cuda.h:42
hwcontext_vaapi.h
AVDRMLayerDescriptor::format
uint32_t format
Format of the layer (DRM_FORMAT_*).
Definition: hwcontext_drm.h:100
VulkanDevicePriv::transfer_qf
AVVulkanDeviceQueueFamily * transfer_qf
Definition: hwcontext_vulkan.c:121
ret
ret
Definition: filter_design.txt:187
pixfmt
enum AVPixelFormat pixfmt
Definition: kmsgrab.c:367
AVHWDeviceContext::type
enum AVHWDeviceType type
This field identifies the underlying API used for hardware access.
Definition: hwcontext.h:75
AV_PIX_FMT_NV12
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:96
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:265
AVHWFramesContext::device_ctx
AVHWDeviceContext * device_ctx
The parent AVHWDeviceContext.
Definition: hwcontext.h:137
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:278
cuda_check.h
AVHWFramesContext::hwctx
void * hwctx
The format-specific data, allocated and freed automatically along with this context.
Definition: hwcontext.h:153
VulkanFramesPriv::tmp
AVBufferPool * tmp
Definition: hwcontext_vulkan.c:177
vulkan_get_buffer
static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
Definition: hwcontext_vulkan.c:3059
pick_queue_family
static int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf, VkQueueFlagBits flags)
Definition: hwcontext_vulkan.c:1402
FFVkExecPool
Definition: vulkan.h:252
vulkan_transfer_host
static int vulkan_transfer_host(AVHWFramesContext *hwfc, AVFrame *hwf, AVFrame *swf, int upload)
Definition: hwcontext_vulkan.c:4310
unlock_queue
static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1825
AVHWFramesConstraints::max_height
int max_height
Definition: hwcontext.h:470
AVVkFrame::internal
struct AVVkFrameInternal * internal
Internal data.
Definition: hwcontext_vulkan.h:348
ff_vk_qf_find
AVVulkanDeviceQueueFamily * ff_vk_qf_find(FFVulkanContext *s, VkQueueFlagBits dev_family, VkVideoCodecOperationFlagBitsKHR vid_ops)
Chooses an appropriate QF.
Definition: vulkan.c:274
AV_PIX_FMT_YUV420P12
#define AV_PIX_FMT_YUV420P12
Definition: pixfmt.h:543
FF_VK_EXT_DESCRIPTOR_BUFFER
#define FF_VK_EXT_DESCRIPTOR_BUFFER
Definition: vulkan_functions.h:42
AV_PIX_FMT_UYVY422
@ AV_PIX_FMT_UYVY422
packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1
Definition: pixfmt.h:88
check_extensions
static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode debug_mode)
Definition: hwcontext_vulkan.c:750
FFVkExecContext::buf
VkCommandBuffer buf
Definition: vulkan.h:122
SIZE_SPECIFIER
#define SIZE_SPECIFIER
Definition: internal.h:129
vulkan_device_create
static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:2073
AVFrame::height
int height
Definition: frame.h:499
PREP_MODE_ENCODING_DPB
@ PREP_MODE_ENCODING_DPB
Definition: hwcontext_vulkan.c:2447
FFVkFormatEntry::pixfmt
enum AVPixelFormat pixfmt
Definition: hwcontext_vulkan.c:353
AVHWFramesConstraints::min_height
int min_height
Definition: hwcontext.h:463
RELEASE_PROPS
#define RELEASE_PROPS(props, count)
Definition: hwcontext_vulkan.c:729
mode
mode
Definition: ebur128.h:83
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
AV_PIX_FMT_YUVA422P12
#define AV_PIX_FMT_YUVA422P12
Definition: pixfmt.h:593
AV_PIX_FMT_GBRAPF32
#define AV_PIX_FMT_GBRAPF32
Definition: pixfmt.h:579
VulkanDevicePriv::feats
VulkanDeviceFeatures feats
Definition: hwcontext_vulkan.c:133
LIBAVUTIL_VERSION_MICRO
#define LIBAVUTIL_VERSION_MICRO
Definition: version.h:83
find_device
static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
Definition: hwcontext_vulkan.c:1231
AVVulkanDeviceContext::nb_graphics_queues
attribute_deprecated int nb_graphics_queues
Definition: hwcontext_vulkan.h:126
FF_VK_STRUCT_EXT
#define FF_VK_STRUCT_EXT(CTX, BASE, STRUCT_P, EXT_FLAG, TYPE)
Definition: vulkan.h:354
optional_instance_exts
static const VulkanOptExtension optional_instance_exts[]
Definition: hwcontext_vulkan.c:628
AV_PIX_FMT_FLAG_PLANAR
#define AV_PIX_FMT_FLAG_PLANAR
At least one pixel component is not in the first data plane.
Definition: pixdesc.h:132
ff_vk_map_feats_to_usage
VkImageUsageFlags ff_vk_map_feats_to_usage(VkFormatFeatureFlagBits2 feats)
Map between usage and features.
FF_VK_EXT_ATOMIC_FLOAT
#define FF_VK_EXT_ATOMIC_FLOAT
Definition: vulkan_functions.h:44
FFVkFormatEntry::fallback
const VkFormat fallback[5]
Definition: hwcontext_vulkan.c:358
Windows::Graphics::DirectX::Direct3D11::p
IDirect3DDxgiInterfaceAccess _COM_Outptr_ void ** p
Definition: vsrc_gfxcapture_winrt.hpp:53
PREP_MODE_EXTERNAL_IMPORT
@ PREP_MODE_EXTERNAL_IMPORT
Definition: hwcontext_vulkan.c:2444
AV_PIX_FMT_RGBAF32
#define AV_PIX_FMT_RGBAF32
Definition: pixfmt.h:627
FF_VK_EXT_VIDEO_DECODE_AV1
#define FF_VK_EXT_VIDEO_DECODE_AV1
Definition: vulkan_functions.h:63
AV_PIX_FMT_YUV444P
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:78
AVVulkanFramesContext::tiling
VkImageTiling tiling
Controls the tiling of allocated frames.
Definition: hwcontext_vulkan.h:217
VulkanDeviceSelection::drm_major
uint32_t drm_major
Definition: hwcontext_vulkan.c:1210
get_plane_buf
static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4214
AV_PIX_FMT_P010
#define AV_PIX_FMT_P010
Definition: pixfmt.h:602
unlock_frame
static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2809
AV_DRM_MAX_PLANES
@ AV_DRM_MAX_PLANES
The maximum number of layers/planes in a DRM frame.
Definition: hwcontext_drm.h:39
FF_DISABLE_DEPRECATION_WARNINGS
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:72
AVVkFrame::sem_value
uint64_t sem_value[AV_NUM_DATA_POINTERS]
Up to date semaphore value at which each image becomes accessible.
Definition: hwcontext_vulkan.h:343
AV_PIX_FMT_GBRP
@ AV_PIX_FMT_GBRP
planar GBR 4:4:4 24bpp
Definition: pixfmt.h:165
av_strdup
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:272
desc
const char * desc
Definition: libsvtav1.c:79
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:112
AVVulkanFramesContext::nb_layers
int nb_layers
Number of layers each image will have.
Definition: hwcontext_vulkan.h:273
AV_PIX_FMT_YUV422P
@ AV_PIX_FMT_YUV422P
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:77
VulkanFramesPriv::download_exec
FFVkExecPool download_exec
Definition: hwcontext_vulkan.c:174
mem.h
AVVkFrame::layout
VkImageLayout layout[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:327
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
hwcontext_internal.h
FF_VK_EXT_VIDEO_ENCODE_H264
#define FF_VK_EXT_VIDEO_ENCODE_H264
Definition: vulkan_functions.h:66
ADD_QUEUE
#define ADD_QUEUE(ctx_qf, qc, flag)
AVVulkanDeviceContext::nb_enabled_inst_extensions
int nb_enabled_inst_extensions
Definition: hwcontext_vulkan.h:102
av_free
#define av_free(p)
Definition: tableprint_vlc.h:34
AVDictionaryEntry
Definition: dict.h:90
get_plane_wh
static void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format, int frame_w, int frame_h, int plane)
Definition: hwcontext_vulkan.c:2540
ff_vk_count_images
static int ff_vk_count_images(AVVkFrame *f)
Definition: vulkan.h:323
ff_vk_exec_discard_deps
void ff_vk_exec_discard_deps(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:591
PREP_MODE_EXTERNAL_EXPORT
@ PREP_MODE_EXTERNAL_EXPORT
Definition: hwcontext_vulkan.c:2443
FFALIGN
#define FFALIGN(x, a)
Definition: macros.h:78
AV_PIX_FMT_P416
#define AV_PIX_FMT_P416
Definition: pixfmt.h:621
VulkanDevicePriv::mprops
VkPhysicalDeviceMemoryProperties mprops
Definition: hwcontext_vulkan.c:125
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
FF_VULKAN_DEBUG_NB
@ FF_VULKAN_DEBUG_NB
Definition: hwcontext_vulkan.c:747
FFVkBuffer
Definition: vulkan.h:87
vk_dev_type
static const char * vk_dev_type(enum VkPhysicalDeviceType type)
Definition: hwcontext_vulkan.c:1219
VulkanDevicePriv::disable_multiplane
int disable_multiplane
Definition: hwcontext_vulkan.c:151
imgutils.h
ff_vk_exec_submit
int ff_vk_exec_submit(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:904
AV_PIX_FMT_XV36
#define AV_PIX_FMT_XV36
Definition: pixfmt.h:610
hwcontext.h
AVDRMPlaneDescriptor::pitch
ptrdiff_t pitch
Pitch (linesize) of this plane.
Definition: hwcontext_drm.h:87
AVFrame::linesize
int linesize[AV_NUM_DATA_POINTERS]
For video, a positive or negative value, which is typically indicating the size in bytes of each pict...
Definition: frame.h:472
VulkanDeviceFeatures::descriptor_buffer
VkPhysicalDeviceDescriptorBufferFeaturesEXT descriptor_buffer
Definition: hwcontext_vulkan.c:102
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
FFVkFormatEntry::vk_planes
int vk_planes
Definition: hwcontext_vulkan.c:355
AVVulkanDeviceQueueFamily
Definition: hwcontext_vulkan.h:33
HWContextType
Definition: hwcontext_internal.h:29
FF_VK_EXT_VIDEO_ENCODE_AV1
#define FF_VK_EXT_VIDEO_ENCODE_AV1
Definition: vulkan_functions.h:68
AV_PIX_FMT_P412
#define AV_PIX_FMT_P412
Definition: pixfmt.h:619
device_features_init
static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
Definition: hwcontext_vulkan.c:205
VulkanDevicePriv::contiguous_planes
int contiguous_planes
Definition: hwcontext_vulkan.c:148
AVVAAPIDeviceContext
VAAPI connection details.
Definition: hwcontext_vaapi.h:68
h
h
Definition: vp9dsp_template.c:2070
AVVulkanDeviceContext::device_features
VkPhysicalDeviceFeatures2 device_features
This structure should be set to the set of features that present and enabled during device creation.
Definition: hwcontext_vulkan.h:92
AVDictionaryEntry::value
char * value
Definition: dict.h:92
avstring.h
AVDRMDeviceContext
DRM device.
Definition: hwcontext_drm.h:157
AV_PIX_FMT_GRAY12
#define AV_PIX_FMT_GRAY12
Definition: pixfmt.h:520
copy_buffer_data
static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf, AVFrame *swf, VkBufferImageCopy *region, int planes, int upload)
Definition: hwcontext_vulkan.c:4156
VulkanDeviceFeatures::atomic_float
VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float
Definition: hwcontext_vulkan.c:103
ADD_VAL_TO_LIST
#define ADD_VAL_TO_LIST(list, count, val)
Definition: hwcontext_vulkan.c:715
AV_PIX_FMT_BAYER_RGGB16
#define AV_PIX_FMT_BAYER_RGGB16
Definition: pixfmt.h:572
AVDRMFrameDescriptor::nb_objects
int nb_objects
Number of DRM objects making up this frame.
Definition: hwcontext_drm.h:137
VulkanDeviceFeatures::shader_object
VkPhysicalDeviceShaderObjectFeaturesEXT shader_object
Definition: hwcontext_vulkan.c:100
HWMapDescriptor
Definition: hwcontext_internal.h:120
AVVulkanDeviceQueueFamily::flags
VkQueueFlagBits flags
Definition: hwcontext_vulkan.h:41
AVVulkanDeviceQueueFamily::video_caps
VkVideoCodecOperationFlagBitsKHR video_caps
Definition: hwcontext_vulkan.h:44
FFVulkanFunctions
Definition: vulkan_functions.h:276
VulkanDevicePriv::p
AVVulkanDeviceContext p
The public AVVulkanDeviceContext.
Definition: hwcontext_vulkan.c:114
VulkanFramesPriv::modifier_info
VkImageDrmFormatModifierListCreateInfoEXT * modifier_info
Definition: hwcontext_vulkan.c:180
FFVkFormatEntry::aspect
VkImageAspectFlags aspect
Definition: hwcontext_vulkan.c:354
ff_vk_get_pooled_buffer
int ff_vk_get_pooled_buffer(FFVulkanContext *ctx, AVBufferPool **buf_pool, AVBufferRef **buf, VkBufferUsageFlags usage, void *create_pNext, size_t size, VkMemoryPropertyFlagBits mem_props)
Initialize a pool and create AVBufferRefs containing FFVkBuffer.
Definition: vulkan.c:1254
VulkanDeviceSelection::name
const char * name
Definition: hwcontext_vulkan.c:1213
src
#define src
Definition: vp8dsp.c:248
AV_HWDEVICE_TYPE_DRM
@ AV_HWDEVICE_TYPE_DRM
Definition: hwcontext.h:36
AVVulkanDeviceContext::nb_comp_queues
attribute_deprecated int nb_comp_queues
Definition: hwcontext_vulkan.h:144
AV_PIX_FMT_YUVA422P
@ AV_PIX_FMT_YUVA422P
planar YUV 4:2:2 24bpp, (1 Cr & Cb sample per 2x1 Y & A samples)
Definition: pixfmt.h:173
vulkan_device_create_internal
static int vulkan_device_create_internal(AVHWDeviceContext *ctx, VulkanDeviceSelection *dev_select, int disable_multiplane, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:1712
w32dlfcn.h
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:3367
vulkan_device_has_rebar
static int vulkan_device_has_rebar(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1693
Generated on Tue Sep 23 2025 19:23:03 for FFmpeg by   doxygen 1.8.17