FFmpeg
•All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Modules Pages
hwcontext_vulkan.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) Lynne
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #define VK_NO_PROTOTYPES
22 #define VK_ENABLE_BETA_EXTENSIONS
23 
24 #ifdef _WIN32
25 #include <windows.h> /* Included to prevent conflicts with CreateSemaphore */
26 #include <versionhelpers.h>
27 #include "compat/w32dlfcn.h"
28 #else
29 #include <dlfcn.h>
30 #include <unistd.h>
31 #endif
32 
33 #include "thread.h"
34 
35 #include "config.h"
36 #include "pixdesc.h"
37 #include "avstring.h"
38 #include "imgutils.h"
39 #include "hwcontext.h"
40 #include "hwcontext_internal.h"
41 #include "hwcontext_vulkan.h"
42 #include "mem.h"
43 
44 #include "vulkan.h"
45 #include "vulkan_loader.h"
46 
47 #if CONFIG_VAAPI
48 #include "hwcontext_vaapi.h"
49 #endif
50 
51 #if CONFIG_LIBDRM
52 #if CONFIG_VAAPI
53 #include <va/va_drmcommon.h>
54 #endif
55 #ifdef __linux__
56 #include <sys/sysmacros.h>
57 #endif
58 #include <sys/stat.h>
59 #include <xf86drm.h>
60 #include <drm_fourcc.h>
61 #include "hwcontext_drm.h"
62 #endif
63 
64 #if HAVE_LINUX_DMA_BUF_H
65 #include <sys/ioctl.h>
66 #include <linux/dma-buf.h>
67 #endif
68 
69 #if CONFIG_CUDA
70 #include "hwcontext_cuda_internal.h"
71 #include "cuda_check.h"
72 #define CHECK_CU(x) FF_CUDA_CHECK_DL(cuda_cu, cu, x)
73 #endif
74 
75 typedef struct VulkanDeviceFeatures {
76  VkPhysicalDeviceFeatures2 device;
77 
78  VkPhysicalDeviceVulkan11Features vulkan_1_1;
79  VkPhysicalDeviceVulkan12Features vulkan_1_2;
80  VkPhysicalDeviceVulkan13Features vulkan_1_3;
81  VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore;
82 
83  VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1;
84 #ifdef VK_KHR_video_maintenance2
85  VkPhysicalDeviceVideoMaintenance2FeaturesKHR video_maintenance_2;
86 #endif
87 
88  VkPhysicalDeviceShaderObjectFeaturesEXT shader_object;
89  VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix;
90  VkPhysicalDeviceDescriptorBufferFeaturesEXT descriptor_buffer;
91  VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float;
92 
93  VkPhysicalDeviceOpticalFlowFeaturesNV optical_flow;
94 
95 #ifdef VK_KHR_shader_relaxed_extended_instruction
96  VkPhysicalDeviceShaderRelaxedExtendedInstructionFeaturesKHR relaxed_extended_instruction;
97 #endif
98 } VulkanDeviceFeatures;
99 
100 typedef struct VulkanDevicePriv {
101  /**
102  * The public AVVulkanDeviceContext. See hwcontext_vulkan.h for it.
103  */
104  AVVulkanDeviceContext p;
105 
106  /* Vulkan library and loader functions */
107  void *libvulkan;
108 
109  FFVulkanContext vkctx;
110  AVVulkanDeviceQueueFamily *compute_qf;
111  AVVulkanDeviceQueueFamily *transfer_qf;
112 
113  /* Properties */
114  VkPhysicalDeviceProperties2 props;
115  VkPhysicalDeviceMemoryProperties mprops;
116  VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops;
117 
118  /* Opaque FD external semaphore properties */
119  VkExternalSemaphoreProperties ext_sem_props_opaque;
120 
121  /* Enabled features */
122  VulkanDeviceFeatures feats;
123 
124  /* Queues */
125  pthread_mutex_t **qf_mutex;
126  uint32_t nb_tot_qfs;
127  uint32_t img_qfs[64];
128  uint32_t nb_img_qfs;
129 
130  /* Debug callback */
131  VkDebugUtilsMessengerEXT debug_ctx;
132 
133  /* Settings */
134  int use_linear_images;
135 
136  /* Option to allocate all image planes in a single allocation */
137  int contiguous_planes;
138 
139  /* Disable multiplane images */
140  int disable_multiplane;
141 
142  /* Nvidia */
143  int dev_is_nvidia;
144 } VulkanDevicePriv;
145 
146 typedef struct VulkanFramesPriv {
147  /**
148  * The public AVVulkanFramesContext. See hwcontext_vulkan.h for it.
149  */
150  AVVulkanFramesContext p;
151 
152  /* Image conversions */
153  FFVkExecPool compute_exec;
154 
155  /* Image transfers */
156  FFVkExecPool upload_exec;
157  FFVkExecPool download_exec;
158 
159  /* Temporary buffer pools */
160  AVBufferPool *tmp;
161 
162  /* Modifier info list to free at uninit */
163  VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
164 } VulkanFramesPriv;
165 
166 typedef struct AVVkFrameInternal {
167  pthread_mutex_t update_mutex;
168 
169 #if CONFIG_CUDA
170  /* Importing external memory into cuda is really expensive so we keep the
171  * memory imported all the time */
172  AVBufferRef *cuda_fc_ref; /* Need to keep it around for uninit */
173  CUexternalMemory ext_mem[AV_NUM_DATA_POINTERS];
174  CUmipmappedArray cu_mma[AV_NUM_DATA_POINTERS];
175  CUarray cu_array[AV_NUM_DATA_POINTERS];
176  CUexternalSemaphore cu_sem[AV_NUM_DATA_POINTERS];
177 #ifdef _WIN32
178  HANDLE ext_mem_handle[AV_NUM_DATA_POINTERS];
179  HANDLE ext_sem_handle[AV_NUM_DATA_POINTERS];
180 #endif
181 #endif
182 } AVVkFrameInternal;
183 
184 /* Initialize all structs in VulkanDeviceFeatures */
185 static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
186 {
187  VulkanDevicePriv *p = ctx->hwctx;
188 
189 #define OPT_CHAIN(STRUCT_P, EXT_FLAG, TYPE) \
190  do { \
191  if ((EXT_FLAG == FF_VK_EXT_NO_FLAG) || \
192  (p->vkctx.extensions & EXT_FLAG)) { \
193  (STRUCT_P)->sType = TYPE; \
194  ff_vk_link_struct(&feats->device, STRUCT_P); \
195  } \
196  } while (0)
197 
198  feats->device = (VkPhysicalDeviceFeatures2) {
199  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
200  };
201 
202  OPT_CHAIN(&feats->vulkan_1_1, FF_VK_EXT_NO_FLAG,
203  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES);
204  OPT_CHAIN(&feats->vulkan_1_2, FF_VK_EXT_NO_FLAG,
205  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES);
206  OPT_CHAIN(&feats->vulkan_1_3, FF_VK_EXT_NO_FLAG,
207  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES);
208 
209  OPT_CHAIN(&feats->timeline_semaphore, FF_VK_EXT_PORTABILITY_SUBSET,
210  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES);
211 
212  OPT_CHAIN(&feats->video_maintenance_1, FF_VK_EXT_VIDEO_MAINTENANCE_1,
213  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_1_FEATURES_KHR);
214 #ifdef VK_KHR_video_maintenance2
215  OPT_CHAIN(&feats->video_maintenance_2, FF_VK_EXT_VIDEO_MAINTENANCE_2,
216  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_2_FEATURES_KHR);
217 #endif
218 
219  OPT_CHAIN(&feats->shader_object, FF_VK_EXT_SHADER_OBJECT,
220  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OBJECT_FEATURES_EXT);
221  OPT_CHAIN(&feats->cooperative_matrix, FF_VK_EXT_COOP_MATRIX,
222  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR);
223  OPT_CHAIN(&feats->descriptor_buffer, FF_VK_EXT_DESCRIPTOR_BUFFER,
224  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_BUFFER_FEATURES_EXT);
225  OPT_CHAIN(&feats->atomic_float, FF_VK_EXT_ATOMIC_FLOAT,
226  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT);
227 
228 #ifdef VK_KHR_shader_relaxed_extended_instruction
229  OPT_CHAIN(&feats->relaxed_extended_instruction, FF_VK_EXT_RELAXED_EXTENDED_INSTR,
230  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_RELAXED_EXTENDED_INSTRUCTION_FEATURES_KHR);
231 #endif
232 
233  OPT_CHAIN(&feats->optical_flow, FF_VK_EXT_OPTICAL_FLOW,
234  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_OPTICAL_FLOW_FEATURES_NV);
235 #undef OPT_CHAIN
236 }
237 
238 /* Copy all needed device features */
239 static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
240 {
241 #define COPY_VAL(VAL) \
242  do { \
243  dst->VAL = src->VAL; \
244  } while (0) \
245 
246  COPY_VAL(device.features.shaderImageGatherExtended);
247  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
248  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
249  COPY_VAL(device.features.fragmentStoresAndAtomics);
250  COPY_VAL(device.features.vertexPipelineStoresAndAtomics);
251  COPY_VAL(device.features.shaderInt64);
252  COPY_VAL(device.features.shaderInt16);
253  COPY_VAL(device.features.shaderFloat64);
254  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
255  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
256 
257  COPY_VAL(vulkan_1_1.samplerYcbcrConversion);
258  COPY_VAL(vulkan_1_1.storagePushConstant16);
259  COPY_VAL(vulkan_1_1.storageBuffer16BitAccess);
260  COPY_VAL(vulkan_1_1.uniformAndStorageBuffer16BitAccess);
261 
262  COPY_VAL(vulkan_1_2.timelineSemaphore);
263  COPY_VAL(vulkan_1_2.scalarBlockLayout);
264  COPY_VAL(vulkan_1_2.bufferDeviceAddress);
265  COPY_VAL(vulkan_1_2.hostQueryReset);
266  COPY_VAL(vulkan_1_2.storagePushConstant8);
267  COPY_VAL(vulkan_1_2.shaderInt8);
268  COPY_VAL(vulkan_1_2.storageBuffer8BitAccess);
269  COPY_VAL(vulkan_1_2.uniformAndStorageBuffer8BitAccess);
270  COPY_VAL(vulkan_1_2.shaderFloat16);
271  COPY_VAL(vulkan_1_2.shaderBufferInt64Atomics);
272  COPY_VAL(vulkan_1_2.shaderSharedInt64Atomics);
273  COPY_VAL(vulkan_1_2.vulkanMemoryModel);
274  COPY_VAL(vulkan_1_2.vulkanMemoryModelDeviceScope);
275 
276  COPY_VAL(vulkan_1_3.dynamicRendering);
277  COPY_VAL(vulkan_1_3.maintenance4);
278  COPY_VAL(vulkan_1_3.synchronization2);
279  COPY_VAL(vulkan_1_3.computeFullSubgroups);
280  COPY_VAL(vulkan_1_3.shaderZeroInitializeWorkgroupMemory);
281  COPY_VAL(vulkan_1_3.dynamicRendering);
282 
283  COPY_VAL(timeline_semaphore.timelineSemaphore);
284 
285  COPY_VAL(video_maintenance_1.videoMaintenance1);
286 #ifdef VK_KHR_video_maintenance2
287  COPY_VAL(video_maintenance_2.videoMaintenance2);
288 #endif
289 
290  COPY_VAL(shader_object.shaderObject);
291 
292  COPY_VAL(cooperative_matrix.cooperativeMatrix);
293 
294  COPY_VAL(descriptor_buffer.descriptorBuffer);
295  COPY_VAL(descriptor_buffer.descriptorBufferPushDescriptors);
296 
297  COPY_VAL(atomic_float.shaderBufferFloat32Atomics);
298  COPY_VAL(atomic_float.shaderBufferFloat32AtomicAdd);
299 
300 #ifdef VK_KHR_shader_relaxed_extended_instruction
301  COPY_VAL(relaxed_extended_instruction.shaderRelaxedExtendedInstruction);
302 #endif
303 
304  COPY_VAL(optical_flow.opticalFlow);
305 #undef COPY_VAL
306 }
307 
308 #define ASPECT_2PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT)
309 #define ASPECT_3PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT | VK_IMAGE_ASPECT_PLANE_2_BIT)
310 
311 static const struct FFVkFormatEntry {
312  VkFormat vkf;
313  enum AVPixelFormat pixfmt;
314  VkImageAspectFlags aspect;
315  int vk_planes;
316  int nb_images;
317  int nb_images_fallback;
318  const VkFormat fallback[5];
319 } vk_formats_list[] = {
320  /* Gray formats */
321  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GRAY8, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8_UNORM } },
322  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
323  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY12, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
324  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY14, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
325  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
326  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GRAY32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_UINT } },
327  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GRAYF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_SFLOAT } },
328 
329  /* RGB formats */
330  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGRA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
331  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGBA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
332  { VK_FORMAT_R8G8B8_UNORM, AV_PIX_FMT_RGB24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8_UNORM } },
333  { VK_FORMAT_B8G8R8_UNORM, AV_PIX_FMT_BGR24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8_UNORM } },
334  { VK_FORMAT_R16G16B16_UNORM, AV_PIX_FMT_RGB48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16_UNORM } },
335  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_RGBA64, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
336  { VK_FORMAT_R5G6B5_UNORM_PACK16, AV_PIX_FMT_RGB565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R5G6B5_UNORM_PACK16 } },
337  { VK_FORMAT_B5G6R5_UNORM_PACK16, AV_PIX_FMT_BGR565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B5G6R5_UNORM_PACK16 } },
338  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGR0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
339  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGB0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
340  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_X2RGB10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2R10G10B10_UNORM_PACK32 } },
341  { VK_FORMAT_A2B10G10R10_UNORM_PACK32, AV_PIX_FMT_X2BGR10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2B10G10R10_UNORM_PACK32 } },
342  { VK_FORMAT_R32G32B32_SFLOAT, AV_PIX_FMT_RGBF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_SFLOAT } },
343  { VK_FORMAT_R32G32B32A32_SFLOAT, AV_PIX_FMT_RGBAF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_SFLOAT } },
344  { VK_FORMAT_R32G32B32_UINT, AV_PIX_FMT_RGB96, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_UINT } },
345  { VK_FORMAT_R32G32B32A32_UINT, AV_PIX_FMT_RGBA128, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_UINT } },
346 
347  /* Planar RGB */
348  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP10, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
349  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP12, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
350  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP14, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
351  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP16, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
352  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRPF32, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
353 
354  /* Planar RGB + Alpha */
355  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRAP, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
356  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
357  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
358  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP14, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
359  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
360  { VK_FORMAT_R32_UINT, AV_PIX_FMT_GBRAP32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT } },
361  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRAPF32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
362 
363  /* Two-plane 420 YUV at 8, 10, 12 and 16 bits */
364  { VK_FORMAT_G8_B8R8_2PLANE_420_UNORM, AV_PIX_FMT_NV12, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
365  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P010, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
366  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P012, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
367  { VK_FORMAT_G16_B16R16_2PLANE_420_UNORM, AV_PIX_FMT_P016, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
368 
369  /* Two-plane 422 YUV at 8, 10 and 16 bits */
370  { VK_FORMAT_G8_B8R8_2PLANE_422_UNORM, AV_PIX_FMT_NV16, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
371  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P210, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
372  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P212, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
373  { VK_FORMAT_G16_B16R16_2PLANE_422_UNORM, AV_PIX_FMT_P216, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
374 
375  /* Two-plane 444 YUV at 8, 10 and 16 bits */
376  { VK_FORMAT_G8_B8R8_2PLANE_444_UNORM, AV_PIX_FMT_NV24, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
377  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P410, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
378  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P412, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
379  { VK_FORMAT_G16_B16R16_2PLANE_444_UNORM, AV_PIX_FMT_P416, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
380 
381  /* Three-plane 420, 422, 444 at 8, 10, 12 and 16 bits */
382  { VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
383  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
384  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
385  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
386  { VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
387  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
388  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
389  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
390  { VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
391  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
392  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
393  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
394 
395  /* Single plane 422 at 8, 10, 12 and 16 bits */
396  { VK_FORMAT_G8B8G8R8_422_UNORM, AV_PIX_FMT_YUYV422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
397  { VK_FORMAT_B8G8R8G8_422_UNORM, AV_PIX_FMT_UYVY422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
398  { VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16, AV_PIX_FMT_Y210, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
399  { VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16, AV_PIX_FMT_Y212, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
400  { VK_FORMAT_G16B16G16R16_422_UNORM, AV_PIX_FMT_Y216, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
401 
402  /* Single plane 444 at 8, 10, 12 and 16 bits */
403  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_UYVA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
404  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_XV30, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
405  { VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16, AV_PIX_FMT_XV36, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
406  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_XV48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
407 };
408 static const int nb_vk_formats_list = FF_ARRAY_ELEMS(vk_formats_list);
409 
410 const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p)
411 {
412  for (int i = 0; i < nb_vk_formats_list; i++)
413  if (vk_formats_list[i].pixfmt == p)
414  return vk_formats_list[i].fallback;
415  return NULL;
416 }
417 
418 static const struct FFVkFormatEntry *vk_find_format_entry(enum AVPixelFormat p)
419 {
420  for (int i = 0; i < nb_vk_formats_list; i++)
421  if (vk_formats_list[i].pixfmt == p)
422  return &vk_formats_list[i];
423  return NULL;
424 }
425 
426 /* Malitia pura, Khronos */
427 #define FN_MAP_TO(dst_t, dst_name, src_t, src_name) \
428  static av_unused dst_t map_ ##src_name## _to_ ##dst_name(src_t src) \
429  { \
430  dst_t dst = 0x0; \
431  MAP_TO(VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT, \
432  VK_IMAGE_USAGE_SAMPLED_BIT); \
433  MAP_TO(VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT, \
434  VK_IMAGE_USAGE_TRANSFER_SRC_BIT); \
435  MAP_TO(VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT, \
436  VK_IMAGE_USAGE_TRANSFER_DST_BIT); \
437  MAP_TO(VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT, \
438  VK_IMAGE_USAGE_STORAGE_BIT); \
439  MAP_TO(VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT, \
440  VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT); \
441  MAP_TO(VK_FORMAT_FEATURE_2_VIDEO_DECODE_OUTPUT_BIT_KHR, \
442  VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR); \
443  MAP_TO(VK_FORMAT_FEATURE_2_VIDEO_DECODE_DPB_BIT_KHR, \
444  VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR); \
445  MAP_TO(VK_FORMAT_FEATURE_2_VIDEO_ENCODE_DPB_BIT_KHR, \
446  VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR); \
447  MAP_TO(VK_FORMAT_FEATURE_2_VIDEO_ENCODE_INPUT_BIT_KHR, \
448  VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR); \
449  return dst; \
450  }
451 
452 #define MAP_TO(flag1, flag2) if (src & flag2) dst |= flag1;
453 FN_MAP_TO(VkFormatFeatureFlagBits2, feats, VkImageUsageFlags, usage)
454 #undef MAP_TO
455 #define MAP_TO(flag1, flag2) if (src & flag1) dst |= flag2;
456 FN_MAP_TO(VkImageUsageFlags, usage, VkFormatFeatureFlagBits2, feats)
457 #undef MAP_TO
458 #undef FN_MAP_TO
459 
460 static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
461  VkImageTiling tiling,
462  VkFormat fmts[AV_NUM_DATA_POINTERS], /* Output format list */
463  int *nb_images, /* Output number of images */
464  VkImageAspectFlags *aspect, /* Output aspect */
465  VkImageUsageFlags *supported_usage, /* Output supported usage */
466  int disable_multiplane, int need_storage)
467 {
468  VulkanDevicePriv *priv = dev_ctx->hwctx;
469  AVVulkanDeviceContext *hwctx = &priv->p;
470  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
471 
472  const VkFormatFeatureFlagBits2 basic_flags = VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT |
473  VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
474  VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
475 
476  for (int i = 0; i < nb_vk_formats_list; i++) {
477  if (vk_formats_list[i].pixfmt == p) {
478  VkFormatProperties3 fprops = {
479  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3,
480  };
481  VkFormatProperties2 prop = {
482  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
483  .pNext = &fprops,
484  };
485  VkFormatFeatureFlagBits2 feats_primary, feats_secondary;
486  int basics_primary = 0, basics_secondary = 0;
487  int storage_primary = 0, storage_secondary = 0;
488 
489  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
490  vk_formats_list[i].vkf,
491  &prop);
492 
493  feats_primary = tiling == VK_IMAGE_TILING_LINEAR ?
494  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
495  basics_primary = (feats_primary & basic_flags) == basic_flags;
496  storage_primary = !!(feats_primary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
497 
498  if (vk_formats_list[i].vkf != vk_formats_list[i].fallback[0]) {
499  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
500  vk_formats_list[i].fallback[0],
501  &prop);
502  feats_secondary = tiling == VK_IMAGE_TILING_LINEAR ?
503  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
504  basics_secondary = (feats_secondary & basic_flags) == basic_flags;
505  storage_secondary = !!(feats_secondary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
506  } else {
507  basics_secondary = basics_primary;
508  storage_secondary = storage_primary;
509  }
510 
511  if (basics_primary &&
512  !(disable_multiplane && vk_formats_list[i].vk_planes > 1) &&
513  (!need_storage || (need_storage && (storage_primary | storage_secondary)))) {
514  if (fmts) {
515  if (vk_formats_list[i].nb_images > 1) {
516  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
517  fmts[j] = vk_formats_list[i].fallback[j];
518  } else {
519  fmts[0] = vk_formats_list[i].vkf;
520  }
521  }
522  if (nb_images)
523  *nb_images = 1;
524  if (aspect)
525  *aspect = vk_formats_list[i].aspect;
526  if (supported_usage)
527  *supported_usage = map_feats_to_usage(feats_primary) |
528  ((need_storage && (storage_primary | storage_secondary)) ?
529  VK_IMAGE_USAGE_STORAGE_BIT : 0);
530  return 0;
531  } else if (basics_secondary &&
532  (!need_storage || (need_storage && storage_secondary))) {
533  if (fmts) {
534  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
535  fmts[j] = vk_formats_list[i].fallback[j];
536  }
537  if (nb_images)
538  *nb_images = vk_formats_list[i].nb_images_fallback;
539  if (aspect)
540  *aspect = vk_formats_list[i].aspect;
541  if (supported_usage)
542  *supported_usage = map_feats_to_usage(feats_secondary);
543  return 0;
544  } else {
545  return AVERROR(ENOTSUP);
546  }
547  }
548  }
549 
550  return AVERROR(EINVAL);
551 }
552 
553 #if CONFIG_VULKAN_STATIC
554 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,
555  const char *pName);
556 #endif
557 
558 static int load_libvulkan(AVHWDeviceContext *ctx)
559 {
560  VulkanDevicePriv *p = ctx->hwctx;
561  AVVulkanDeviceContext *hwctx = &p->p;
562 
563 #if CONFIG_VULKAN_STATIC
564  hwctx->get_proc_addr = vkGetInstanceProcAddr;
565 #else
566  static const char *lib_names[] = {
567 #if defined(_WIN32)
568  "vulkan-1.dll",
569 #elif defined(__APPLE__)
570  "libvulkan.dylib",
571  "libvulkan.1.dylib",
572  "libMoltenVK.dylib",
573 #else
574  "libvulkan.so.1",
575  "libvulkan.so",
576 #endif
577  };
578 
579  for (int i = 0; i < FF_ARRAY_ELEMS(lib_names); i++) {
580  p->libvulkan = dlopen(lib_names[i], RTLD_NOW | RTLD_LOCAL);
581  if (p->libvulkan)
582  break;
583  }
584 
585  if (!p->libvulkan) {
586  av_log(ctx, AV_LOG_ERROR, "Unable to open the libvulkan library!\n");
587  return AVERROR_UNKNOWN;
588  }
589 
590  hwctx->get_proc_addr = (PFN_vkGetInstanceProcAddr)dlsym(p->libvulkan, "vkGetInstanceProcAddr");
591 #endif /* CONFIG_VULKAN_STATIC */
592 
593  return 0;
594 }
595 
596 typedef struct VulkanOptExtension {
597  const char *name;
598  FFVulkanExtensions flag;
599 } VulkanOptExtension;
600 
601 static const VulkanOptExtension optional_instance_exts[] = {
602  { VK_EXT_LAYER_SETTINGS_EXTENSION_NAME, FF_VK_EXT_NO_FLAG },
603 #ifdef __APPLE__
604  { VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME, FF_VK_EXT_NO_FLAG },
605 #endif
606 };
607 
608 static const VulkanOptExtension optional_device_exts[] = {
609  /* Misc or required by other extensions */
610  { VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME, FF_VK_EXT_PORTABILITY_SUBSET },
611  { VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, FF_VK_EXT_PUSH_DESCRIPTOR },
612  { VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME, FF_VK_EXT_DESCRIPTOR_BUFFER },
613  { VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME, FF_VK_EXT_DEVICE_DRM },
614  { VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, FF_VK_EXT_ATOMIC_FLOAT },
615  { VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME, FF_VK_EXT_COOP_MATRIX },
616  { VK_NV_OPTICAL_FLOW_EXTENSION_NAME, FF_VK_EXT_OPTICAL_FLOW },
617  { VK_EXT_SHADER_OBJECT_EXTENSION_NAME, FF_VK_EXT_SHADER_OBJECT },
618  { VK_KHR_VIDEO_MAINTENANCE_1_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_1 },
619 #ifdef VK_KHR_video_maintenance2
620  { VK_KHR_VIDEO_MAINTENANCE_2_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_2 },
621 #endif
622 
623  /* Imports/exports */
624  { VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_MEMORY },
625  { VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_DMABUF_MEMORY },
626  { VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, FF_VK_EXT_DRM_MODIFIER_FLAGS },
627  { VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_SEM },
628  { VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_HOST_MEMORY },
629 #ifdef _WIN32
630  { VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_MEMORY },
631  { VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_SEM },
632 #endif
633 
634  /* Video encoding/decoding */
635  { VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_QUEUE },
636  { VK_KHR_VIDEO_ENCODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_QUEUE },
637  { VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_QUEUE },
638  { VK_KHR_VIDEO_ENCODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H264 },
639  { VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H264 },
640  { VK_KHR_VIDEO_ENCODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H265 },
641  { VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H265 },
642  { VK_KHR_VIDEO_DECODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_AV1 },
643 };
644 
645 static VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
646  VkDebugUtilsMessageTypeFlagsEXT messageType,
647  const VkDebugUtilsMessengerCallbackDataEXT *data,
648  void *priv)
649 {
650  int l;
651  AVHWDeviceContext *ctx = priv;
652 
653  /* Ignore false positives */
654  switch (data->messageIdNumber) {
655  case 0x086974c1: /* BestPractices-vkCreateCommandPool-command-buffer-reset */
656  case 0xfd92477a: /* BestPractices-vkAllocateMemory-small-allocation */
657  case 0x618ab1e7: /* VUID-VkImageViewCreateInfo-usage-02275 */
658  case 0x30f4ac70: /* VUID-VkImageCreateInfo-pNext-06811 */
659  case 0xa05b236e: /* UNASSIGNED-Threading-MultipleThreads-Write */
660  return VK_FALSE;
661  default:
662  break;
663  }
664 
665  switch (severity) {
666  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT: l = AV_LOG_VERBOSE; break;
667  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT: l = AV_LOG_INFO; break;
668  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT: l = AV_LOG_WARNING; break;
669  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT: l = AV_LOG_ERROR; break;
670  default: l = AV_LOG_DEBUG; break;
671  }
672 
673  av_log(ctx, l, "%s\n", data->pMessage);
674  for (int i = 0; i < data->cmdBufLabelCount; i++)
675  av_log(ctx, l, "\t%i: %s\n", i, data->pCmdBufLabels[i].pLabelName);
676 
677  return VK_FALSE;
678 }
679 
680 #define ADD_VAL_TO_LIST(list, count, val) \
681  do { \
682  list = av_realloc_array(list, sizeof(*list), ++count); \
683  if (!list) { \
684  err = AVERROR(ENOMEM); \
685  goto fail; \
686  } \
687  list[count - 1] = av_strdup(val); \
688  if (!list[count - 1]) { \
689  err = AVERROR(ENOMEM); \
690  goto fail; \
691  } \
692  } while(0)
693 
694 #define RELEASE_PROPS(props, count) \
695  if (props) { \
696  for (int i = 0; i < count; i++) \
697  av_free((void *)((props)[i])); \
698  av_free((void *)props); \
699  }
700 
701 enum FFVulkanDebugMode {
702  FF_VULKAN_DEBUG_NONE = 0,
703  /* Standard GPU-assisted validation */
704  FF_VULKAN_DEBUG_VALIDATE = 1,
705  /* Passes printfs in shaders to the debug callback */
706  FF_VULKAN_DEBUG_PRINTF = 2,
707  /* Enables extra printouts */
708  FF_VULKAN_DEBUG_PRACTICES = 3,
709  /* Disables validation but keeps shader debug info and optimizations */
710  FF_VULKAN_DEBUG_PROFILE = 4,
711 
712  FF_VULKAN_DEBUG_NB,
713 };
714 
715 static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts,
716  const char * const **dst, uint32_t *num,
717  enum FFVulkanDebugMode debug_mode)
718 {
719  const char *tstr;
720  const char **extension_names = NULL;
721  VulkanDevicePriv *p = ctx->hwctx;
722  AVVulkanDeviceContext *hwctx = &p->p;
723  FFVulkanFunctions *vk = &p->vkctx.vkfn;
724  int err = 0, found, extensions_found = 0;
725 
726  const char *mod;
727  int optional_exts_num;
728  uint32_t sup_ext_count;
729  char *user_exts_str = NULL;
730  AVDictionaryEntry *user_exts;
731  VkExtensionProperties *sup_ext;
732  const VulkanOptExtension *optional_exts;
733 
734  if (!dev) {
735  mod = "instance";
736  optional_exts = optional_instance_exts;
737  optional_exts_num = FF_ARRAY_ELEMS(optional_instance_exts);
738  user_exts = av_dict_get(opts, "instance_extensions", NULL, 0);
739  if (user_exts) {
740  user_exts_str = av_strdup(user_exts->value);
741  if (!user_exts_str) {
742  err = AVERROR(ENOMEM);
743  goto fail;
744  }
745  }
746  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, NULL);
747  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
748  if (!sup_ext)
749  return AVERROR(ENOMEM);
750  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, sup_ext);
751  } else {
752  mod = "device";
753  optional_exts = optional_device_exts;
754  optional_exts_num = FF_ARRAY_ELEMS(optional_device_exts);
755  user_exts = av_dict_get(opts, "device_extensions", NULL, 0);
756  if (user_exts) {
757  user_exts_str = av_strdup(user_exts->value);
758  if (!user_exts_str) {
759  err = AVERROR(ENOMEM);
760  goto fail;
761  }
762  }
763  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
764  &sup_ext_count, NULL);
765  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
766  if (!sup_ext)
767  return AVERROR(ENOMEM);
768  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
769  &sup_ext_count, sup_ext);
770  }
771 
772  for (int i = 0; i < optional_exts_num; i++) {
773  tstr = optional_exts[i].name;
774  found = 0;
775 
776  if (dev &&
777  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
778  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
779  (debug_mode == FF_VULKAN_DEBUG_PRACTICES)) &&
780  !strcmp(tstr, VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME)) {
781  continue;
782  }
783 
784  for (int j = 0; j < sup_ext_count; j++) {
785  if (!strcmp(tstr, sup_ext[j].extensionName)) {
786  found = 1;
787  break;
788  }
789  }
790  if (!found)
791  continue;
792 
793  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
794  p->vkctx.extensions |= optional_exts[i].flag;
795  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
796  }
797 
798  if (!dev &&
799  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
800  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
801  (debug_mode == FF_VULKAN_DEBUG_PRACTICES))) {
802  tstr = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
803  found = 0;
804  for (int j = 0; j < sup_ext_count; j++) {
805  if (!strcmp(tstr, sup_ext[j].extensionName)) {
806  found = 1;
807  break;
808  }
809  }
810  if (found) {
811  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
812  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
813  } else {
814  av_log(ctx, AV_LOG_ERROR, "Debug extension \"%s\" not found!\n",
815  tstr);
816  err = AVERROR(EINVAL);
817  goto fail;
818  }
819  }
820 
821 #ifdef VK_KHR_shader_relaxed_extended_instruction
822  if (((debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
823  (debug_mode == FF_VULKAN_DEBUG_PROFILE)) && dev) {
824  tstr = VK_KHR_SHADER_RELAXED_EXTENDED_INSTRUCTION_EXTENSION_NAME;
825  found = 0;
826  for (int j = 0; j < sup_ext_count; j++) {
827  if (!strcmp(tstr, sup_ext[j].extensionName)) {
828  found = 1;
829  break;
830  }
831  }
832  if (found) {
833  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
834  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
835  } else {
836  av_log(ctx, AV_LOG_ERROR, "Debug_printf/profile enabled, but extension \"%s\" not found!\n",
837  tstr);
838  err = AVERROR(EINVAL);
839  goto fail;
840  }
841  }
842 #endif
843 
844  if (user_exts_str) {
845  char *save, *token = av_strtok(user_exts_str, "+", &save);
846  while (token) {
847  found = 0;
848  for (int j = 0; j < sup_ext_count; j++) {
849  if (!strcmp(token, sup_ext[j].extensionName)) {
850  found = 1;
851  break;
852  }
853  }
854  if (found) {
855  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension \"%s\"\n", mod, token);
856  ADD_VAL_TO_LIST(extension_names, extensions_found, token);
857  } else {
858  av_log(ctx, AV_LOG_WARNING, "%s extension \"%s\" not found, excluding.\n",
859  mod, token);
860  }
861  token = av_strtok(NULL, "+", &save);
862  }
863  }
864 
865  *dst = extension_names;
866  *num = extensions_found;
867 
868  av_free(user_exts_str);
869  av_free(sup_ext);
870  return 0;
871 
872 fail:
873  RELEASE_PROPS(extension_names, extensions_found);
874  av_free(user_exts_str);
875  av_free(sup_ext);
876  return err;
877 }
878 
879 static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts,
880  const char * const **dst, uint32_t *num,
881  enum FFVulkanDebugMode *debug_mode)
882 {
883  int err = 0;
884  VulkanDevicePriv *priv = ctx->hwctx;
885  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
886 
887  static const char layer_standard_validation[] = { "VK_LAYER_KHRONOS_validation" };
888  int layer_standard_validation_found = 0;
889 
890  uint32_t sup_layer_count;
891  VkLayerProperties *sup_layers;
892 
893  AVDictionaryEntry *user_layers = av_dict_get(opts, "layers", NULL, 0);
894  char *user_layers_str = NULL;
895  char *save, *token;
896 
897  const char **enabled_layers = NULL;
898  uint32_t enabled_layers_count = 0;
899 
900  AVDictionaryEntry *debug_opt = av_dict_get(opts, "debug", NULL, 0);
901  enum FFVulkanDebugMode mode;
902 
903  *debug_mode = mode = FF_VULKAN_DEBUG_NONE;
904 
905  /* Get a list of all layers */
906  vk->EnumerateInstanceLayerProperties(&sup_layer_count, NULL);
907  sup_layers = av_malloc_array(sup_layer_count, sizeof(VkLayerProperties));
908  if (!sup_layers)
909  return AVERROR(ENOMEM);
910  vk->EnumerateInstanceLayerProperties(&sup_layer_count, sup_layers);
911 
912  av_log(ctx, AV_LOG_VERBOSE, "Supported layers:\n");
913  for (int i = 0; i < sup_layer_count; i++)
914  av_log(ctx, AV_LOG_VERBOSE, "\t%s\n", sup_layers[i].layerName);
915 
916  /* If no user layers or debug layers are given, return */
917  if (!debug_opt && !user_layers)
918  goto end;
919 
920  /* Check for any properly supported validation layer */
921  if (debug_opt) {
922  if (!strcmp(debug_opt->value, "profile")) {
923  mode = FF_VULKAN_DEBUG_PROFILE;
924  } else if (!strcmp(debug_opt->value, "printf")) {
925  mode = FF_VULKAN_DEBUG_PRINTF;
926  } else if (!strcmp(debug_opt->value, "validate")) {
927  mode = FF_VULKAN_DEBUG_VALIDATE;
928  } else if (!strcmp(debug_opt->value, "practices")) {
929  mode = FF_VULKAN_DEBUG_PRACTICES;
930  } else {
931  char *end_ptr = NULL;
932  int idx = strtol(debug_opt->value, &end_ptr, 10);
933  if (end_ptr == debug_opt->value || end_ptr[0] != '\0' ||
934  idx < 0 || idx >= FF_VULKAN_DEBUG_NB) {
935  av_log(ctx, AV_LOG_ERROR, "Invalid debugging mode \"%s\"\n",
936  debug_opt->value);
937  err = AVERROR(EINVAL);
938  goto end;
939  }
940  mode = idx;
941  }
942  }
943 
944  /* If mode is VALIDATE or PRINTF, try to find the standard validation layer extension */
945  if ((mode == FF_VULKAN_DEBUG_VALIDATE) ||
946  (mode == FF_VULKAN_DEBUG_PRINTF) ||
947  (mode == FF_VULKAN_DEBUG_PRACTICES)) {
948  for (int i = 0; i < sup_layer_count; i++) {
949  if (!strcmp(layer_standard_validation, sup_layers[i].layerName)) {
950  av_log(ctx, AV_LOG_VERBOSE, "Standard validation layer %s is enabled\n",
951  layer_standard_validation);
952  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, layer_standard_validation);
953  *debug_mode = mode;
954  layer_standard_validation_found = 1;
955  break;
956  }
957  }
958  if (!layer_standard_validation_found) {
959  av_log(ctx, AV_LOG_ERROR,
960  "Validation Layer \"%s\" not supported\n", layer_standard_validation);
961  err = AVERROR(ENOTSUP);
962  goto end;
963  }
964  } else if (mode == FF_VULKAN_DEBUG_PROFILE) {
965  *debug_mode = mode;
966  }
967 
968  /* Process any custom layers enabled */
969  if (user_layers) {
970  int found;
971 
972  user_layers_str = av_strdup(user_layers->value);
973  if (!user_layers_str) {
974  err = AVERROR(ENOMEM);
975  goto fail;
976  }
977 
978  token = av_strtok(user_layers_str, "+", &save);
979  while (token) {
980  found = 0;
981 
982  /* If debug=1/2 was specified as an option, skip this layer */
983  if (!strcmp(layer_standard_validation, token) && layer_standard_validation_found) {
984  token = av_strtok(NULL, "+", &save);
985  break;
986  }
987 
988  /* Try to find the layer in the list of supported layers */
989  for (int j = 0; j < sup_layer_count; j++) {
990  if (!strcmp(token, sup_layers[j].layerName)) {
991  found = 1;
992  break;
993  }
994  }
995 
996  if (found) {
997  av_log(ctx, AV_LOG_VERBOSE, "Using layer: %s\n", token);
998  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, token);
999 
1000  /* If debug was not set as an option, force it */
1001  if (!strcmp(layer_standard_validation, token))
1002  *debug_mode = FF_VULKAN_DEBUG_VALIDATE;
1003  } else {
1004  av_log(ctx, AV_LOG_ERROR,
1005  "Layer \"%s\" not supported\n", token);
1006  err = AVERROR(EINVAL);
1007  goto end;
1008  }
1009 
1010  token = av_strtok(NULL, "+", &save);
1011  }
1012  }
1013 
1014 fail:
1015 end:
1016  av_free(sup_layers);
1017  av_free(user_layers_str);
1018 
1019  if (err < 0) {
1020  RELEASE_PROPS(enabled_layers, enabled_layers_count);
1021  } else {
1022  *dst = enabled_layers;
1023  *num = enabled_layers_count;
1024  }
1025 
1026  return err;
1027 }
1028 
1029 /* Creates a VkInstance */
1030 static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts,
1031  enum FFVulkanDebugMode *debug_mode)
1032 {
1033  int err = 0;
1034  VkResult ret;
1035  VulkanDevicePriv *p = ctx->hwctx;
1036  AVVulkanDeviceContext *hwctx = &p->p;
1037  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1038  VkApplicationInfo application_info = {
1039  .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
1040  .pApplicationName = "ffmpeg",
1041  .applicationVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1042  LIBAVUTIL_VERSION_MINOR,
1043  LIBAVUTIL_VERSION_MICRO),
1044  .pEngineName = "libavutil",
1045  .apiVersion = VK_API_VERSION_1_3,
1046  .engineVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1047  LIBAVUTIL_VERSION_MINOR,
1048  LIBAVUTIL_VERSION_MICRO),
1049  };
1050  VkValidationFeaturesEXT validation_features = {
1051  .sType = VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT,
1052  };
1053  VkInstanceCreateInfo inst_props = {
1054  .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
1055  .pApplicationInfo = &application_info,
1056  };
1057 
1058  if (!hwctx->get_proc_addr) {
1059  err = load_libvulkan(ctx);
1060  if (err < 0)
1061  return err;
1062  }
1063 
1064  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 0, 0);
1065  if (err < 0) {
1066  av_log(ctx, AV_LOG_ERROR, "Unable to load instance enumeration functions!\n");
1067  return err;
1068  }
1069 
1070  err = check_layers(ctx, opts, &inst_props.ppEnabledLayerNames,
1071  &inst_props.enabledLayerCount, debug_mode);
1072  if (err)
1073  goto fail;
1074 
1075  /* Check for present/missing extensions */
1076  err = check_extensions(ctx, 0, opts, &inst_props.ppEnabledExtensionNames,
1077  &inst_props.enabledExtensionCount, *debug_mode);
1078  hwctx->enabled_inst_extensions = inst_props.ppEnabledExtensionNames;
1079  hwctx->nb_enabled_inst_extensions = inst_props.enabledExtensionCount;
1080  if (err < 0)
1081  goto fail;
1082 
1083  /* Enable debug features if needed */
1084  if (*debug_mode == FF_VULKAN_DEBUG_VALIDATE) {
1085  static const VkValidationFeatureEnableEXT feat_list_validate[] = {
1086  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1087  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1088  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_EXT,
1089  };
1090  validation_features.pEnabledValidationFeatures = feat_list_validate;
1091  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_validate);
1092  inst_props.pNext = &validation_features;
1093  } else if (*debug_mode == FF_VULKAN_DEBUG_PRINTF) {
1094  static const VkValidationFeatureEnableEXT feat_list_debug[] = {
1095  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1096  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1097  VK_VALIDATION_FEATURE_ENABLE_DEBUG_PRINTF_EXT,
1098  };
1099  validation_features.pEnabledValidationFeatures = feat_list_debug;
1100  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_debug);
1101  inst_props.pNext = &validation_features;
1102  } else if (*debug_mode == FF_VULKAN_DEBUG_PRACTICES) {
1103  static const VkValidationFeatureEnableEXT feat_list_practices[] = {
1104  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1105  VK_VALIDATION_FEATURE_ENABLE_BEST_PRACTICES_EXT,
1106  };
1107  validation_features.pEnabledValidationFeatures = feat_list_practices;
1108  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_practices);
1109  inst_props.pNext = &validation_features;
1110  }
1111 
1112 #ifdef __APPLE__
1113  for (int i = 0; i < inst_props.enabledExtensionCount; i++) {
1114  if (!strcmp(VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME,
1115  inst_props.ppEnabledExtensionNames[i])) {
1116  inst_props.flags |= VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR;
1117  break;
1118  }
1119  }
1120 #endif
1121 
1122  /* Try to create the instance */
1123  ret = vk->CreateInstance(&inst_props, hwctx->alloc, &hwctx->inst);
1124 
1125  /* Check for errors */
1126  if (ret != VK_SUCCESS) {
1127  av_log(ctx, AV_LOG_ERROR, "Instance creation failure: %s\n",
1128  ff_vk_ret2str(ret));
1129  err = AVERROR_EXTERNAL;
1130  goto fail;
1131  }
1132 
1133  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 0);
1134  if (err < 0) {
1135  av_log(ctx, AV_LOG_ERROR, "Unable to load instance functions!\n");
1136  goto fail;
1137  }
1138 
1139  /* Setup debugging callback if needed */
1140  if ((*debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
1141  (*debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
1142  (*debug_mode == FF_VULKAN_DEBUG_PRACTICES)) {
1143  VkDebugUtilsMessengerCreateInfoEXT dbg = {
1144  .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
1145  .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT |
1146  VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
1147  VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
1148  VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
1149  .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
1150  VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
1151  VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
1152  .pfnUserCallback = vk_dbg_callback,
1153  .pUserData = ctx,
1154  };
1155 
1156  vk->CreateDebugUtilsMessengerEXT(hwctx->inst, &dbg,
1157  hwctx->alloc, &p->debug_ctx);
1158  }
1159 
1160  err = 0;
1161 
1162 fail:
1163  RELEASE_PROPS(inst_props.ppEnabledLayerNames, inst_props.enabledLayerCount);
1164  return err;
1165 }
1166 
1167 typedef struct VulkanDeviceSelection {
1168  uint8_t uuid[VK_UUID_SIZE]; /* Will use this first unless !has_uuid */
1169  int has_uuid;
1170  uint32_t drm_major; /* Will use this second unless !has_drm */
1171  uint32_t drm_minor; /* Will use this second unless !has_drm */
1172  uint32_t has_drm; /* has drm node info */
1173  const char *name; /* Will use this third unless NULL */
1174  uint32_t pci_device; /* Will use this fourth unless 0x0 */
1175  uint32_t vendor_id; /* Last resort to find something deterministic */
1176  int index; /* Finally fall back to index */
1177 } VulkanDeviceSelection;
1178 
1179 static const char *vk_dev_type(enum VkPhysicalDeviceType type)
1180 {
1181  switch (type) {
1182  case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "integrated";
1183  case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "discrete";
1184  case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "virtual";
1185  case VK_PHYSICAL_DEVICE_TYPE_CPU: return "software";
1186  default: return "unknown";
1187  }
1188 }
1189 
1190 /* Finds a device */
1191 static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
1192 {
1193  int err = 0, choice = -1;
1194  uint32_t num;
1195  VkResult ret;
1196  VulkanDevicePriv *p = ctx->hwctx;
1197  AVVulkanDeviceContext *hwctx = &p->p;
1198  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1199  VkPhysicalDevice *devices = NULL;
1200  VkPhysicalDeviceIDProperties *idp = NULL;
1201  VkPhysicalDeviceProperties2 *prop = NULL;
1202  VkPhysicalDeviceDrmPropertiesEXT *drm_prop = NULL;
1203 
1204  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, NULL);
1205  if (ret != VK_SUCCESS || !num) {
1206  av_log(ctx, AV_LOG_ERROR, "No devices found: %s!\n", ff_vk_ret2str(ret));
1207  return AVERROR(ENODEV);
1208  }
1209 
1210  devices = av_malloc_array(num, sizeof(VkPhysicalDevice));
1211  if (!devices)
1212  return AVERROR(ENOMEM);
1213 
1214  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, devices);
1215  if (ret != VK_SUCCESS) {
1216  av_log(ctx, AV_LOG_ERROR, "Failed enumerating devices: %s\n",
1217  ff_vk_ret2str(ret));
1218  err = AVERROR(ENODEV);
1219  goto end;
1220  }
1221 
1222  prop = av_calloc(num, sizeof(*prop));
1223  if (!prop) {
1224  err = AVERROR(ENOMEM);
1225  goto end;
1226  }
1227 
1228  idp = av_calloc(num, sizeof(*idp));
1229  if (!idp) {
1230  err = AVERROR(ENOMEM);
1231  goto end;
1232  }
1233 
1234  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1235  drm_prop = av_calloc(num, sizeof(*drm_prop));
1236  if (!drm_prop) {
1237  err = AVERROR(ENOMEM);
1238  goto end;
1239  }
1240  }
1241 
1242  av_log(ctx, AV_LOG_VERBOSE, "GPU listing:\n");
1243  for (int i = 0; i < num; i++) {
1244  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1245  drm_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT;
1246  idp[i].pNext = &drm_prop[i];
1247  }
1248  idp[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES;
1249  prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1250  prop[i].pNext = &idp[i];
1251 
1252  vk->GetPhysicalDeviceProperties2(devices[i], &prop[i]);
1253  av_log(ctx, AV_LOG_VERBOSE, " %d: %s (%s) (0x%x)\n", i,
1254  prop[i].properties.deviceName,
1255  vk_dev_type(prop[i].properties.deviceType),
1256  prop[i].properties.deviceID);
1257  }
1258 
1259  if (select->has_uuid) {
1260  for (int i = 0; i < num; i++) {
1261  if (!strncmp(idp[i].deviceUUID, select->uuid, VK_UUID_SIZE)) {
1262  choice = i;
1263  goto end;
1264  }
1265  }
1266  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given UUID!\n");
1267  err = AVERROR(ENODEV);
1268  goto end;
1269  } else if ((p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) && select->has_drm) {
1270  for (int i = 0; i < num; i++) {
1271  if ((select->drm_major == drm_prop[i].primaryMajor &&
1272  select->drm_minor == drm_prop[i].primaryMinor) ||
1273  (select->drm_major == drm_prop[i].renderMajor &&
1274  select->drm_minor == drm_prop[i].renderMinor)) {
1275  choice = i;
1276  goto end;
1277  }
1278  }
1279  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given DRM node numbers %i:%i!\n",
1280  select->drm_major, select->drm_minor);
1281  err = AVERROR(ENODEV);
1282  goto end;
1283  } else if (select->name) {
1284  av_log(ctx, AV_LOG_VERBOSE, "Requested device: %s\n", select->name);
1285  for (int i = 0; i < num; i++) {
1286  if (strstr(prop[i].properties.deviceName, select->name)) {
1287  choice = i;
1288  goto end;
1289  }
1290  }
1291  av_log(ctx, AV_LOG_ERROR, "Unable to find device \"%s\"!\n",
1292  select->name);
1293  err = AVERROR(ENODEV);
1294  goto end;
1295  } else if (select->pci_device) {
1296  av_log(ctx, AV_LOG_VERBOSE, "Requested device: 0x%x\n", select->pci_device);
1297  for (int i = 0; i < num; i++) {
1298  if (select->pci_device == prop[i].properties.deviceID) {
1299  choice = i;
1300  goto end;
1301  }
1302  }
1303  av_log(ctx, AV_LOG_ERROR, "Unable to find device with PCI ID 0x%x!\n",
1304  select->pci_device);
1305  err = AVERROR(EINVAL);
1306  goto end;
1307  } else if (select->vendor_id) {
1308  av_log(ctx, AV_LOG_VERBOSE, "Requested vendor: 0x%x\n", select->vendor_id);
1309  for (int i = 0; i < num; i++) {
1310  if (select->vendor_id == prop[i].properties.vendorID) {
1311  choice = i;
1312  goto end;
1313  }
1314  }
1315  av_log(ctx, AV_LOG_ERROR, "Unable to find device with Vendor ID 0x%x!\n",
1316  select->vendor_id);
1317  err = AVERROR(ENODEV);
1318  goto end;
1319  } else {
1320  if (select->index < num) {
1321  choice = select->index;
1322  goto end;
1323  }
1324  av_log(ctx, AV_LOG_ERROR, "Unable to find device with index %i!\n",
1325  select->index);
1326  err = AVERROR(ENODEV);
1327  goto end;
1328  }
1329 
1330 end:
1331  if (choice > -1) {
1332  av_log(ctx, AV_LOG_VERBOSE, "Device %d selected: %s (%s) (0x%x)\n",
1333  choice, prop[choice].properties.deviceName,
1334  vk_dev_type(prop[choice].properties.deviceType),
1335  prop[choice].properties.deviceID);
1336  hwctx->phys_dev = devices[choice];
1337  }
1338 
1339  av_free(devices);
1340  av_free(prop);
1341  av_free(idp);
1342  av_free(drm_prop);
1343 
1344  return err;
1345 }
1346 
1347 /* Picks the least used qf with the fewest unneeded flags, or -1 if none found */
1348 static inline int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf,
1349  VkQueueFlagBits flags)
1350 {
1351  int index = -1;
1352  uint32_t min_score = UINT32_MAX;
1353 
1354  for (int i = 0; i < num_qf; i++) {
1355  VkQueueFlagBits qflags = qf[i].queueFamilyProperties.queueFlags;
1356 
1357  /* Per the spec, reporting transfer caps is optional for these 2 types */
1358  if ((flags & VK_QUEUE_TRANSFER_BIT) &&
1359  (qflags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT)))
1360  qflags |= VK_QUEUE_TRANSFER_BIT;
1361 
1362  if (qflags & flags) {
1363  uint32_t score = av_popcount(qflags) + qf[i].queueFamilyProperties.timestampValidBits;
1364  if (score < min_score) {
1365  index = i;
1366  min_score = score;
1367  }
1368  }
1369  }
1370 
1371  if (index > -1)
1372  qf[index].queueFamilyProperties.timestampValidBits++;
1373 
1374  return index;
1375 }
1376 
1377 static inline int pick_video_queue_family(VkQueueFamilyProperties2 *qf,
1378  VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf,
1379  VkVideoCodecOperationFlagBitsKHR flags)
1380 {
1381  int index = -1;
1382  uint32_t min_score = UINT32_MAX;
1383 
1384  for (int i = 0; i < num_qf; i++) {
1385  const VkQueueFlagBits qflags = qf[i].queueFamilyProperties.queueFlags;
1386  const VkQueueFlagBits vflags = qf_vid[i].videoCodecOperations;
1387 
1388  if (!(qflags & (VK_QUEUE_VIDEO_ENCODE_BIT_KHR | VK_QUEUE_VIDEO_DECODE_BIT_KHR)))
1389  continue;
1390 
1391  if (vflags & flags) {
1392  uint32_t score = av_popcount(vflags) + qf[i].queueFamilyProperties.timestampValidBits;
1393  if (score < min_score) {
1394  index = i;
1395  min_score = score;
1396  }
1397  }
1398  }
1399 
1400  if (index > -1)
1401  qf[index].queueFamilyProperties.timestampValidBits++;
1402 
1403  return index;
1404 }
1405 
1406 static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
1407 {
1408  uint32_t num;
1409  VulkanDevicePriv *p = ctx->hwctx;
1410  AVVulkanDeviceContext *hwctx = &p->p;
1411  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1412 
1413  VkQueueFamilyProperties2 *qf = NULL;
1414  VkQueueFamilyVideoPropertiesKHR *qf_vid = NULL;
1415 
1416  /* First get the number of queue families */
1417  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &num, NULL);
1418  if (!num) {
1419  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1420  return AVERROR_EXTERNAL;
1421  }
1422 
1423  /* Then allocate memory */
1424  qf = av_malloc_array(num, sizeof(VkQueueFamilyProperties2));
1425  if (!qf)
1426  return AVERROR(ENOMEM);
1427 
1428  qf_vid = av_malloc_array(num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1429  if (!qf_vid)
1430  return AVERROR(ENOMEM);
1431 
1432  for (uint32_t i = 0; i < num; i++) {
1433  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1434  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1435  };
1436  qf[i] = (VkQueueFamilyProperties2) {
1437  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1438  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1439  };
1440  }
1441 
1442  /* Finally retrieve the queue families */
1443  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &num, qf);
1444 
1445  av_log(ctx, AV_LOG_VERBOSE, "Queue families:\n");
1446  for (int i = 0; i < num; i++) {
1447  av_log(ctx, AV_LOG_VERBOSE, " %i:%s%s%s%s%s%s%s%s (queues: %i)\n", i,
1448  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_GRAPHICS_BIT) ? " graphics" : "",
1449  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_COMPUTE_BIT) ? " compute" : "",
1450  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_TRANSFER_BIT) ? " transfer" : "",
1451  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_ENCODE_BIT_KHR) ? " encode" : "",
1452  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_DECODE_BIT_KHR) ? " decode" : "",
1453  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_SPARSE_BINDING_BIT) ? " sparse" : "",
1454  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_OPTICAL_FLOW_BIT_NV) ? " optical_flow" : "",
1455  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_PROTECTED_BIT) ? " protected" : "",
1456  qf[i].queueFamilyProperties.queueCount);
1457 
1458  /* We use this field to keep a score of how many times we've used that
1459  * queue family in order to make better choices. */
1460  qf[i].queueFamilyProperties.timestampValidBits = 0;
1461  }
1462 
1463  hwctx->nb_qf = 0;
1464 
1465  /* Pick each queue family to use */
1466 #define PICK_QF(type, vid_op) \
1467  do { \
1468  uint32_t i; \
1469  uint32_t idx; \
1470  \
1471  if (vid_op) \
1472  idx = pick_video_queue_family(qf, qf_vid, num, vid_op); \
1473  else \
1474  idx = pick_queue_family(qf, num, type); \
1475  \
1476  if (idx == -1) \
1477  continue; \
1478  \
1479  for (i = 0; i < hwctx->nb_qf; i++) { \
1480  if (hwctx->qf[i].idx == idx) { \
1481  hwctx->qf[i].flags |= type; \
1482  hwctx->qf[i].video_caps |= vid_op; \
1483  break; \
1484  } \
1485  } \
1486  if (i == hwctx->nb_qf) { \
1487  hwctx->qf[i].idx = idx; \
1488  hwctx->qf[i].num = qf[idx].queueFamilyProperties.queueCount; \
1489  hwctx->qf[i].flags = type; \
1490  hwctx->qf[i].video_caps = vid_op; \
1491  hwctx->nb_qf++; \
1492  } \
1493  } while (0)
1494 
1495  PICK_QF(VK_QUEUE_GRAPHICS_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1496  PICK_QF(VK_QUEUE_COMPUTE_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1497  PICK_QF(VK_QUEUE_TRANSFER_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1498  PICK_QF(VK_QUEUE_OPTICAL_FLOW_BIT_NV, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1499 
1500  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR);
1501  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR);
1502 
1503  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR);
1504  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR);
1505 
1506  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR);
1507 
1508  av_free(qf);
1509  av_free(qf_vid);
1510 
1511 #undef PICK_QF
1512 
1513  cd->pQueueCreateInfos = av_malloc_array(hwctx->nb_qf,
1514  sizeof(VkDeviceQueueCreateInfo));
1515  if (!cd->pQueueCreateInfos)
1516  return AVERROR(ENOMEM);
1517 
1518  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1519  int dup = 0;
1520  float *weights = NULL;
1521  VkDeviceQueueCreateInfo *pc;
1522  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++) {
1523  if (hwctx->qf[i].idx == cd->pQueueCreateInfos[j].queueFamilyIndex) {
1524  dup = 1;
1525  break;
1526  }
1527  }
1528  if (dup)
1529  continue;
1530 
1531  weights = av_malloc_array(hwctx->qf[i].num, sizeof(float));
1532  if (!weights) {
1533  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++)
1534  av_free((void *)cd->pQueueCreateInfos[i].pQueuePriorities);
1535  av_free((void *)cd->pQueueCreateInfos);
1536  return AVERROR(ENOMEM);
1537  }
1538 
1539  for (uint32_t j = 0; j < hwctx->qf[i].num; j++)
1540  weights[j] = 1.0;
1541 
1542  pc = (VkDeviceQueueCreateInfo *)cd->pQueueCreateInfos;
1543  pc[cd->queueCreateInfoCount++] = (VkDeviceQueueCreateInfo) {
1544  .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
1545  .queueFamilyIndex = hwctx->qf[i].idx,
1546  .queueCount = hwctx->qf[i].num,
1547  .pQueuePriorities = weights,
1548  };
1549  }
1550 
1551 #if FF_API_VULKAN_FIXED_QUEUES
1552 FF_DISABLE_DEPRECATION_WARNINGS
1553  /* Setup deprecated fields */
1554  hwctx->queue_family_index = -1;
1555  hwctx->queue_family_comp_index = -1;
1556  hwctx->queue_family_tx_index = -1;
1557  hwctx->queue_family_encode_index = -1;
1558  hwctx->queue_family_decode_index = -1;
1559 
1560 #define SET_OLD_QF(field, nb_field, type) \
1561  do { \
1562  if (field < 0 && hwctx->qf[i].flags & type) { \
1563  field = hwctx->qf[i].idx; \
1564  nb_field = hwctx->qf[i].num; \
1565  } \
1566  } while (0)
1567 
1568  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1569  SET_OLD_QF(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
1570  SET_OLD_QF(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
1571  SET_OLD_QF(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
1572  SET_OLD_QF(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
1573  SET_OLD_QF(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
1574  }
1575 
1576 #undef SET_OLD_QF
1577 FF_ENABLE_DEPRECATION_WARNINGS
1578 #endif
1579 
1580  return 0;
1581 }
1582 
1583 /* Only resources created by vulkan_device_create should be released here,
1584  * resources created by vulkan_device_init should be released by
1585  * vulkan_device_uninit, to make sure we don't free user provided resources,
1586  * and there is no leak.
1587  */
1588 static void vulkan_device_free(AVHWDeviceContext *ctx)
1589 {
1590  VulkanDevicePriv *p = ctx->hwctx;
1591  AVVulkanDeviceContext *hwctx = &p->p;
1592  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1593 
1594  if (hwctx->act_dev)
1595  vk->DestroyDevice(hwctx->act_dev, hwctx->alloc);
1596 
1597  if (p->debug_ctx)
1598  vk->DestroyDebugUtilsMessengerEXT(hwctx->inst, p->debug_ctx,
1599  hwctx->alloc);
1600 
1601  if (hwctx->inst)
1602  vk->DestroyInstance(hwctx->inst, hwctx->alloc);
1603 
1604  if (p->libvulkan)
1605  dlclose(p->libvulkan);
1606 
1607  RELEASE_PROPS(hwctx->enabled_inst_extensions, hwctx->nb_enabled_inst_extensions);
1608  RELEASE_PROPS(hwctx->enabled_dev_extensions, hwctx->nb_enabled_dev_extensions);
1609 }
1610 
1611 static void vulkan_device_uninit(AVHWDeviceContext *ctx)
1612 {
1613  VulkanDevicePriv *p = ctx->hwctx;
1614 
1615  for (uint32_t i = 0; i < p->nb_tot_qfs; i++) {
1616  pthread_mutex_destroy(p->qf_mutex[i]);
1617  av_freep(&p->qf_mutex[i]);
1618  }
1619  av_freep(&p->qf_mutex);
1620 
1621  ff_vk_uninit(&p->vkctx);
1622 }
1623 
1624 static int vulkan_device_create_internal(AVHWDeviceContext *ctx,
1625  VulkanDeviceSelection *dev_select,
1626  int disable_multiplane,
1627  AVDictionary *opts, int flags)
1628 {
1629  int err = 0;
1630  VkResult ret;
1631  AVDictionaryEntry *opt_d;
1632  VulkanDevicePriv *p = ctx->hwctx;
1633  AVVulkanDeviceContext *hwctx = &p->p;
1634  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1635  enum FFVulkanDebugMode debug_mode = FF_VULKAN_DEBUG_NONE;
1636  VulkanDeviceFeatures supported_feats = { 0 };
1637  VkDeviceCreateInfo dev_info = {
1638  .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
1639  };
1640 
1641  /* Create an instance if not given one */
1642  if ((err = create_instance(ctx, opts, &debug_mode)))
1643  goto end;
1644 
1645  /* Find a physical device (if not given one) */
1646  if ((err = find_device(ctx, dev_select)))
1647  goto end;
1648 
1649  /* Find and enable extensions for the physical device */
1650  if ((err = check_extensions(ctx, 1, opts, &dev_info.ppEnabledExtensionNames,
1651  &dev_info.enabledExtensionCount, debug_mode))) {
1652  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1653  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1654  av_free((void *)dev_info.pQueueCreateInfos);
1655  goto end;
1656  }
1657 
1658  /* Get all supported features for the physical device */
1659  device_features_init(ctx, &supported_feats);
1660  vk->GetPhysicalDeviceFeatures2(hwctx->phys_dev, &supported_feats.device);
1661 
1662  /* Copy all needed features from those supported and activate them */
1663  device_features_init(ctx, &p->feats);
1664  device_features_copy_needed(&p->feats, &supported_feats);
1665  dev_info.pNext = p->feats.device.pNext;
1666  dev_info.pEnabledFeatures = &p->feats.device.features;
1667 
1668  /* Setup enabled queue families */
1669  if ((err = setup_queue_families(ctx, &dev_info)))
1670  goto end;
1671 
1672  /* Finally create the device */
1673  ret = vk->CreateDevice(hwctx->phys_dev, &dev_info, hwctx->alloc,
1674  &hwctx->act_dev);
1675 
1676  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1677  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1678  av_free((void *)dev_info.pQueueCreateInfos);
1679 
1680  if (ret != VK_SUCCESS) {
1681  av_log(ctx, AV_LOG_ERROR, "Device creation failure: %s\n",
1682  ff_vk_ret2str(ret));
1683  for (int i = 0; i < dev_info.enabledExtensionCount; i++)
1684  av_free((void *)dev_info.ppEnabledExtensionNames[i]);
1685  av_free((void *)dev_info.ppEnabledExtensionNames);
1686  err = AVERROR_EXTERNAL;
1687  goto end;
1688  }
1689 
1690  /* Tiled images setting, use them by default */
1691  opt_d = av_dict_get(opts, "linear_images", NULL, 0);
1692  if (opt_d)
1693  p->use_linear_images = strtol(opt_d->value, NULL, 10);
1694 
1695  /* The disable_multiplane argument takes precedent over the option */
1696  p->disable_multiplane = disable_multiplane;
1697  if (!p->disable_multiplane) {
1698  opt_d = av_dict_get(opts, "disable_multiplane", NULL, 0);
1699  if (opt_d)
1700  p->disable_multiplane = strtol(opt_d->value, NULL, 10);
1701  }
1702 
1703  /* Set the public device feature struct and its pNext chain */
1704  hwctx->device_features = p->feats.device;
1705 
1706  /* Set the list of all active extensions */
1707  hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames;
1708  hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount;
1709 
1710  /* The extension lists need to be freed */
1711  ctx->free = vulkan_device_free;
1712 
1713 end:
1714  return err;
1715 }
1716 
1717 static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1718 {
1719  VulkanDevicePriv *p = ctx->hwctx;
1720  pthread_mutex_lock(&p->qf_mutex[queue_family][index]);
1721 }
1722 
1723 static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1724 {
1725  VulkanDevicePriv *p = ctx->hwctx;
1726  pthread_mutex_unlock(&p->qf_mutex[queue_family][index]);
1727 }
1728 
1729 static int vulkan_device_init(AVHWDeviceContext *ctx)
1730 {
1731  int err = 0;
1732  uint32_t qf_num;
1733  VulkanDevicePriv *p = ctx->hwctx;
1734  AVVulkanDeviceContext *hwctx = &p->p;
1735  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1736  VkQueueFamilyProperties2 *qf;
1737  VkQueueFamilyVideoPropertiesKHR *qf_vid;
1738  VkPhysicalDeviceExternalSemaphoreInfo ext_sem_props_info;
1739  int graph_index, comp_index, tx_index, enc_index, dec_index;
1740 
1741  /* Set device extension flags */
1742  for (int i = 0; i < hwctx->nb_enabled_dev_extensions; i++) {
1743  for (int j = 0; j < FF_ARRAY_ELEMS(optional_device_exts); j++) {
1744  if (!strcmp(hwctx->enabled_dev_extensions[i],
1745  optional_device_exts[j].name)) {
1746  p->vkctx.extensions |= optional_device_exts[j].flag;
1747  break;
1748  }
1749  }
1750  }
1751 
1752  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 1);
1753  if (err < 0) {
1754  av_log(ctx, AV_LOG_ERROR, "Unable to load functions!\n");
1755  return err;
1756  }
1757 
1758  p->props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1759  p->props.pNext = &p->hprops;
1760  p->hprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT;
1761 
1762  vk->GetPhysicalDeviceProperties2(hwctx->phys_dev, &p->props);
1763  av_log(ctx, AV_LOG_VERBOSE, "Using device: %s\n",
1764  p->props.properties.deviceName);
1765  av_log(ctx, AV_LOG_VERBOSE, "Alignments:\n");
1766  av_log(ctx, AV_LOG_VERBOSE, " optimalBufferCopyRowPitchAlignment: %"PRIu64"\n",
1767  p->props.properties.limits.optimalBufferCopyRowPitchAlignment);
1768  av_log(ctx, AV_LOG_VERBOSE, " minMemoryMapAlignment: %"SIZE_SPECIFIER"\n",
1769  p->props.properties.limits.minMemoryMapAlignment);
1770  av_log(ctx, AV_LOG_VERBOSE, " nonCoherentAtomSize: %"PRIu64"\n",
1771  p->props.properties.limits.nonCoherentAtomSize);
1772  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY)
1773  av_log(ctx, AV_LOG_VERBOSE, " minImportedHostPointerAlignment: %"PRIu64"\n",
1774  p->hprops.minImportedHostPointerAlignment);
1775 
1776  p->dev_is_nvidia = (p->props.properties.vendorID == 0x10de);
1777 
1778  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &qf_num, NULL);
1779  if (!qf_num) {
1780  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1781  return AVERROR_EXTERNAL;
1782  }
1783 
1784  ext_sem_props_info = (VkPhysicalDeviceExternalSemaphoreInfo) {
1785  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO,
1786  };
1787 
1788  /* Opaque FD semaphore properties */
1789  ext_sem_props_info.handleType =
1790 #ifdef _WIN32
1791  IsWindows8OrGreater()
1792  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
1793  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT;
1794 #else
1795  VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
1796 #endif
1797  p->ext_sem_props_opaque.sType = VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES;
1798  vk->GetPhysicalDeviceExternalSemaphoreProperties(hwctx->phys_dev,
1799  &ext_sem_props_info,
1800  &p->ext_sem_props_opaque);
1801 
1802  qf = av_malloc_array(qf_num, sizeof(VkQueueFamilyProperties2));
1803  if (!qf)
1804  return AVERROR(ENOMEM);
1805 
1806  qf_vid = av_malloc_array(qf_num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1807  if (!qf_vid) {
1808  av_free(qf);
1809  return AVERROR(ENOMEM);
1810  }
1811 
1812  for (uint32_t i = 0; i < qf_num; i++) {
1813  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1814  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1815  };
1816  qf[i] = (VkQueueFamilyProperties2) {
1817  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1818  .pNext = p->vkctx.extensions & FF_VK_EXT_VIDEO_QUEUE ? &qf_vid[i] : NULL,
1819  };
1820  }
1821 
1822  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &qf_num, qf);
1823 
1824  p->qf_mutex = av_calloc(qf_num, sizeof(*p->qf_mutex));
1825  if (!p->qf_mutex) {
1826  err = AVERROR(ENOMEM);
1827  goto end;
1828  }
1829  p->nb_tot_qfs = qf_num;
1830 
1831  for (uint32_t i = 0; i < qf_num; i++) {
1832  p->qf_mutex[i] = av_calloc(qf[i].queueFamilyProperties.queueCount,
1833  sizeof(**p->qf_mutex));
1834  if (!p->qf_mutex[i]) {
1835  err = AVERROR(ENOMEM);
1836  goto end;
1837  }
1838  for (uint32_t j = 0; j < qf[i].queueFamilyProperties.queueCount; j++) {
1839  err = pthread_mutex_init(&p->qf_mutex[i][j], NULL);
1840  if (err != 0) {
1841  av_log(ctx, AV_LOG_ERROR, "pthread_mutex_init failed : %s\n",
1842  av_err2str(err));
1843  err = AVERROR(err);
1844  goto end;
1845  }
1846  }
1847  }
1848 
1849 #if FF_API_VULKAN_FIXED_QUEUES
1850 FF_DISABLE_DEPRECATION_WARNINGS
1851  graph_index = hwctx->nb_graphics_queues ? hwctx->queue_family_index : -1;
1852  comp_index = hwctx->nb_comp_queues ? hwctx->queue_family_comp_index : -1;
1853  tx_index = hwctx->nb_tx_queues ? hwctx->queue_family_tx_index : -1;
1854  dec_index = hwctx->nb_decode_queues ? hwctx->queue_family_decode_index : -1;
1855  enc_index = hwctx->nb_encode_queues ? hwctx->queue_family_encode_index : -1;
1856 
1857 #define CHECK_QUEUE(type, required, fidx, ctx_qf, qc) \
1858  do { \
1859  if (ctx_qf < 0 && required) { \
1860  av_log(ctx, AV_LOG_ERROR, "%s queue family is required, but marked as missing" \
1861  " in the context!\n", type); \
1862  err = AVERROR(EINVAL); \
1863  goto end; \
1864  } else if (fidx < 0 || ctx_qf < 0) { \
1865  break; \
1866  } else if (ctx_qf >= qf_num) { \
1867  av_log(ctx, AV_LOG_ERROR, "Invalid %s family index %i (device has %i families)!\n", \
1868  type, ctx_qf, qf_num); \
1869  err = AVERROR(EINVAL); \
1870  goto end; \
1871  } \
1872  \
1873  av_log(ctx, AV_LOG_VERBOSE, "Using queue family %i (queues: %i)" \
1874  " for%s%s%s%s%s\n", \
1875  ctx_qf, qc, \
1876  ctx_qf == graph_index ? " graphics" : "", \
1877  ctx_qf == comp_index ? " compute" : "", \
1878  ctx_qf == tx_index ? " transfers" : "", \
1879  ctx_qf == enc_index ? " encode" : "", \
1880  ctx_qf == dec_index ? " decode" : ""); \
1881  graph_index = (ctx_qf == graph_index) ? -1 : graph_index; \
1882  comp_index = (ctx_qf == comp_index) ? -1 : comp_index; \
1883  tx_index = (ctx_qf == tx_index) ? -1 : tx_index; \
1884  enc_index = (ctx_qf == enc_index) ? -1 : enc_index; \
1885  dec_index = (ctx_qf == dec_index) ? -1 : dec_index; \
1886  } while (0)
1887 
1888  CHECK_QUEUE("graphics", 0, graph_index, hwctx->queue_family_index, hwctx->nb_graphics_queues);
1889  CHECK_QUEUE("compute", 1, comp_index, hwctx->queue_family_comp_index, hwctx->nb_comp_queues);
1890  CHECK_QUEUE("upload", 1, tx_index, hwctx->queue_family_tx_index, hwctx->nb_tx_queues);
1891  CHECK_QUEUE("decode", 0, dec_index, hwctx->queue_family_decode_index, hwctx->nb_decode_queues);
1892  CHECK_QUEUE("encode", 0, enc_index, hwctx->queue_family_encode_index, hwctx->nb_encode_queues);
1893 
1894 #undef CHECK_QUEUE
1895 
1896  /* Update the new queue family fields. If non-zero already,
1897  * it means API users have set it. */
1898  if (!hwctx->nb_qf) {
1899 #define ADD_QUEUE(ctx_qf, qc, flag) \
1900  do { \
1901  if (ctx_qf != -1) { \
1902  hwctx->qf[hwctx->nb_qf++] = (AVVulkanDeviceQueueFamily) { \
1903  .idx = ctx_qf, \
1904  .num = qc, \
1905  .flags = flag, \
1906  }; \
1907  } \
1908  } while (0)
1909 
1910  ADD_QUEUE(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
1911  ADD_QUEUE(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
1912  ADD_QUEUE(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
1913  ADD_QUEUE(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
1914  ADD_QUEUE(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
1915 #undef ADD_QUEUE
1916  }
1917 FF_ENABLE_DEPRECATION_WARNINGS
1918 #endif
1919 
1920  for (int i = 0; i < hwctx->nb_qf; i++) {
1921  if (!hwctx->qf[i].video_caps &&
1922  hwctx->qf[i].flags & (VK_QUEUE_VIDEO_DECODE_BIT_KHR |
1923  VK_QUEUE_VIDEO_ENCODE_BIT_KHR)) {
1924  hwctx->qf[i].video_caps = qf_vid[hwctx->qf[i].idx].videoCodecOperations;
1925  }
1926  }
1927 
1928  /* Setup array for pQueueFamilyIndices with used queue families */
1929  p->nb_img_qfs = 0;
1930  for (int i = 0; i < hwctx->nb_qf; i++) {
1931  int seen = 0;
1932  /* Make sure each entry is unique
1933  * (VUID-VkBufferCreateInfo-sharingMode-01419) */
1934  for (int j = (i - 1); j >= 0; j--) {
1935  if (hwctx->qf[i].idx == hwctx->qf[j].idx) {
1936  seen = 1;
1937  break;
1938  }
1939  }
1940  if (!seen)
1941  p->img_qfs[p->nb_img_qfs++] = hwctx->qf[i].idx;
1942  }
1943 
1944  if (!hwctx->lock_queue)
1945  hwctx->lock_queue = lock_queue;
1946  if (!hwctx->unlock_queue)
1947  hwctx->unlock_queue = unlock_queue;
1948 
1949  /* Get device capabilities */
1950  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
1951 
1952  p->vkctx.device = ctx;
1953  p->vkctx.hwctx = hwctx;
1954 
1955  ff_vk_load_props(&p->vkctx);
1956  p->compute_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_COMPUTE_BIT, 0);
1957  p->transfer_qf = ff_vk_qf_find(&p->vkctx, VK_QUEUE_TRANSFER_BIT, 0);
1958 
1959 end:
1960  av_free(qf_vid);
1961  av_free(qf);
1962  return err;
1963 }
1964 
1965 static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device,
1966  AVDictionary *opts, int flags)
1967 {
1968  VulkanDeviceSelection dev_select = { 0 };
1969  if (device && device[0]) {
1970  char *end = NULL;
1971  dev_select.index = strtol(device, &end, 10);
1972  if (end == device) {
1973  dev_select.index = 0;
1974  dev_select.name = device;
1975  }
1976  }
1977 
1978  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
1979 }
1980 
1981 static int vulkan_device_derive(AVHWDeviceContext *ctx,
1982  AVHWDeviceContext *src_ctx,
1983  AVDictionary *opts, int flags)
1984 {
1985  av_unused VulkanDeviceSelection dev_select = { 0 };
1986 
1987  /* If there's only one device on the system, then even if its not covered
1988  * by the following checks (e.g. non-PCIe ARM GPU), having an empty
1989  * dev_select will mean it'll get picked. */
1990  switch(src_ctx->type) {
1991 #if CONFIG_VAAPI
1992  case AV_HWDEVICE_TYPE_VAAPI: {
1993  AVVAAPIDeviceContext *src_hwctx = src_ctx->hwctx;
1994  VADisplay dpy = src_hwctx->display;
1995 #if VA_CHECK_VERSION(1, 15, 0)
1996  VAStatus vas;
1997  VADisplayAttribute attr = {
1998  .type = VADisplayPCIID,
1999  };
2000 #endif
2001  const char *vendor;
2002 
2003 #if VA_CHECK_VERSION(1, 15, 0)
2004  vas = vaGetDisplayAttributes(dpy, &attr, 1);
2005  if (vas == VA_STATUS_SUCCESS && attr.flags != VA_DISPLAY_ATTRIB_NOT_SUPPORTED)
2006  dev_select.pci_device = (attr.value & 0xFFFF);
2007 #endif
2008 
2009  if (!dev_select.pci_device) {
2010  vendor = vaQueryVendorString(dpy);
2011  if (!vendor) {
2012  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from VAAPI!\n");
2013  return AVERROR_EXTERNAL;
2014  }
2015 
2016  if (strstr(vendor, "AMD"))
2017  dev_select.vendor_id = 0x1002;
2018  }
2019 
2020  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2021  }
2022 #endif
2023 #if CONFIG_LIBDRM
2024  case AV_HWDEVICE_TYPE_DRM: {
2025  int err;
2026  struct stat drm_node_info;
2027  drmDevice *drm_dev_info;
2028  AVDRMDeviceContext *src_hwctx = src_ctx->hwctx;
2029 
2030  err = fstat(src_hwctx->fd, &drm_node_info);
2031  if (err) {
2032  av_log(ctx, AV_LOG_ERROR, "Unable to get node info from DRM fd: %s!\n",
2033  av_err2str(AVERROR(errno)));
2034  return AVERROR_EXTERNAL;
2035  }
2036 
2037  dev_select.drm_major = major(drm_node_info.st_dev);
2038  dev_select.drm_minor = minor(drm_node_info.st_dev);
2039  dev_select.has_drm = 1;
2040 
2041  err = drmGetDevice(src_hwctx->fd, &drm_dev_info);
2042  if (err) {
2043  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from DRM fd: %s!\n",
2044  av_err2str(AVERROR(errno)));
2045  return AVERROR_EXTERNAL;
2046  }
2047 
2048  if (drm_dev_info->bustype == DRM_BUS_PCI)
2049  dev_select.pci_device = drm_dev_info->deviceinfo.pci->device_id;
2050 
2051  drmFreeDevice(&drm_dev_info);
2052 
2053  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2054  }
2055 #endif
2056 #if CONFIG_CUDA
2057  case AV_HWDEVICE_TYPE_CUDA: {
2058  AVHWDeviceContext *cuda_cu = src_ctx;
2059  AVCUDADeviceContext *src_hwctx = src_ctx->hwctx;
2060  AVCUDADeviceContextInternal *cu_internal = src_hwctx->internal;
2061  CudaFunctions *cu = cu_internal->cuda_dl;
2062 
2063  int ret = CHECK_CU(cu->cuDeviceGetUuid((CUuuid *)&dev_select.uuid,
2064  cu_internal->cuda_device));
2065  if (ret < 0) {
2066  av_log(ctx, AV_LOG_ERROR, "Unable to get UUID from CUDA!\n");
2067  return AVERROR_EXTERNAL;
2068  }
2069 
2070  dev_select.has_uuid = 1;
2071 
2072  /*
2073  * CUDA is not able to import multiplane images, so always derive a
2074  * Vulkan device with multiplane disabled.
2075  */
2076  return vulkan_device_create_internal(ctx, &dev_select, 1, opts, flags);
2077  }
2078 #endif
2079  default:
2080  return AVERROR(ENOSYS);
2081  }
2082 }
2083 
2084 static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx,
2085  const void *hwconfig,
2086  AVHWFramesConstraints *constraints)
2087 {
2088  int count = 0;
2089  VulkanDevicePriv *p = ctx->hwctx;
2090 
2091  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2092  count += vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2093  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2094  VK_IMAGE_TILING_OPTIMAL,
2095  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0;
2096  }
2097 
2098  constraints->valid_sw_formats = av_malloc_array(count + 1,
2099  sizeof(enum AVPixelFormat));
2100  if (!constraints->valid_sw_formats)
2101  return AVERROR(ENOMEM);
2102 
2103  count = 0;
2104  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2105  if (vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2106  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2107  VK_IMAGE_TILING_OPTIMAL,
2108  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0) {
2109  constraints->valid_sw_formats[count++] = vk_formats_list[i].pixfmt;
2110  }
2111  }
2112 
2113  constraints->valid_sw_formats[count++] = AV_PIX_FMT_NONE;
2114 
2115  constraints->min_width = 1;
2116  constraints->min_height = 1;
2117  constraints->max_width = p->props.properties.limits.maxImageDimension2D;
2118  constraints->max_height = p->props.properties.limits.maxImageDimension2D;
2119 
2120  constraints->valid_hw_formats = av_malloc_array(2, sizeof(enum AVPixelFormat));
2121  if (!constraints->valid_hw_formats)
2122  return AVERROR(ENOMEM);
2123 
2124  constraints->valid_hw_formats[0] = AV_PIX_FMT_VULKAN;
2125  constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
2126 
2127  return 0;
2128 }
2129 
2130 static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req,
2131  VkMemoryPropertyFlagBits req_flags, const void *alloc_extension,
2132  VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
2133 {
2134  VkResult ret;
2135  int index = -1;
2136  VulkanDevicePriv *p = ctx->hwctx;
2137  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2138  AVVulkanDeviceContext *dev_hwctx = &p->p;
2139  VkMemoryAllocateInfo alloc_info = {
2140  .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
2141  .pNext = alloc_extension,
2142  .allocationSize = req->size,
2143  };
2144 
2145  /* The vulkan spec requires memory types to be sorted in the "optimal"
2146  * order, so the first matching type we find will be the best/fastest one */
2147  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
2148  const VkMemoryType *type = &p->mprops.memoryTypes[i];
2149 
2150  /* The memory type must be supported by the requirements (bitfield) */
2151  if (!(req->memoryTypeBits & (1 << i)))
2152  continue;
2153 
2154  /* The memory type flags must include our properties */
2155  if ((type->propertyFlags & req_flags) != req_flags)
2156  continue;
2157 
2158  /* The memory type must be large enough */
2159  if (req->size > p->mprops.memoryHeaps[type->heapIndex].size)
2160  continue;
2161 
2162  /* Found a suitable memory type */
2163  index = i;
2164  break;
2165  }
2166 
2167  if (index < 0) {
2168  av_log(ctx, AV_LOG_ERROR, "No memory type found for flags 0x%x\n",
2169  req_flags);
2170  return AVERROR(EINVAL);
2171  }
2172 
2173  alloc_info.memoryTypeIndex = index;
2174 
2175  ret = vk->AllocateMemory(dev_hwctx->act_dev, &alloc_info,
2176  dev_hwctx->alloc, mem);
2177  if (ret != VK_SUCCESS) {
2178  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory: %s\n",
2179  ff_vk_ret2str(ret));
2180  return AVERROR(ENOMEM);
2181  }
2182 
2183  *mem_flags |= p->mprops.memoryTypes[index].propertyFlags;
2184 
2185  return 0;
2186 }
2187 
2188 static void vulkan_free_internal(AVVkFrame *f)
2189 {
2190  av_unused AVVkFrameInternal *internal = f->internal;
2191 
2192 #if CONFIG_CUDA
2193  if (internal->cuda_fc_ref) {
2194  AVHWFramesContext *cuda_fc = (AVHWFramesContext *)internal->cuda_fc_ref->data;
2195  int planes = av_pix_fmt_count_planes(cuda_fc->sw_format);
2196  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
2197  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
2198  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
2199  CudaFunctions *cu = cu_internal->cuda_dl;
2200 
2201  for (int i = 0; i < planes; i++) {
2202  if (internal->cu_sem[i])
2203  CHECK_CU(cu->cuDestroyExternalSemaphore(internal->cu_sem[i]));
2204  if (internal->cu_mma[i])
2205  CHECK_CU(cu->cuMipmappedArrayDestroy(internal->cu_mma[i]));
2206  if (internal->ext_mem[i])
2207  CHECK_CU(cu->cuDestroyExternalMemory(internal->ext_mem[i]));
2208 #ifdef _WIN32
2209  if (internal->ext_sem_handle[i])
2210  CloseHandle(internal->ext_sem_handle[i]);
2211  if (internal->ext_mem_handle[i])
2212  CloseHandle(internal->ext_mem_handle[i]);
2213 #endif
2214  }
2215 
2216  av_buffer_unref(&internal->cuda_fc_ref);
2217  }
2218 #endif
2219 
2220  pthread_mutex_destroy(&internal->update_mutex);
2221  av_freep(&f->internal);
2222 }
2223 
2224 static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
2225 {
2226  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2227  AVVulkanDeviceContext *hwctx = &p->p;
2228  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2229  int nb_images = ff_vk_count_images(f);
2230  int nb_sems = 0;
2231 
2232  while (nb_sems < FF_ARRAY_ELEMS(f->sem) && f->sem[nb_sems])
2233  nb_sems++;
2234 
2235  if (nb_sems) {
2236  VkSemaphoreWaitInfo sem_wait = {
2237  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
2238  .flags = 0x0,
2239  .pSemaphores = f->sem,
2240  .pValues = f->sem_value,
2241  .semaphoreCount = nb_sems,
2242  };
2243 
2244  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
2245  }
2246 
2247  vulkan_free_internal(f);
2248 
2249  for (int i = 0; i < nb_images; i++) {
2250  vk->DestroyImage(hwctx->act_dev, f->img[i], hwctx->alloc);
2251  vk->FreeMemory(hwctx->act_dev, f->mem[i], hwctx->alloc);
2252  vk->DestroySemaphore(hwctx->act_dev, f->sem[i], hwctx->alloc);
2253  }
2254 
2255  av_free(f);
2256 }
2257 
2258 static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
2259 {
2260  vulkan_frame_free(opaque, (AVVkFrame*)data);
2261 }
2262 
2263 static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f,
2264  void *alloc_pnext, size_t alloc_pnext_stride)
2265 {
2266  int img_cnt = 0, err;
2267  VkResult ret;
2268  AVHWDeviceContext *ctx = hwfc->device_ctx;
2269  VulkanDevicePriv *p = ctx->hwctx;
2270  AVVulkanDeviceContext *hwctx = &p->p;
2271  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2272  VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } };
2273 
2274  while (f->img[img_cnt]) {
2275  int use_ded_mem;
2276  VkImageMemoryRequirementsInfo2 req_desc = {
2277  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
2278  .image = f->img[img_cnt],
2279  };
2280  VkMemoryDedicatedAllocateInfo ded_alloc = {
2281  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
2282  .pNext = (void *)(((uint8_t *)alloc_pnext) + img_cnt*alloc_pnext_stride),
2283  };
2284  VkMemoryDedicatedRequirements ded_req = {
2285  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
2286  };
2287  VkMemoryRequirements2 req = {
2288  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
2289  .pNext = &ded_req,
2290  };
2291 
2292  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req);
2293 
2294  if (f->tiling == VK_IMAGE_TILING_LINEAR)
2295  req.memoryRequirements.size = FFALIGN(req.memoryRequirements.size,
2296  p->props.properties.limits.minMemoryMapAlignment);
2297 
2298  /* In case the implementation prefers/requires dedicated allocation */
2299  use_ded_mem = ded_req.prefersDedicatedAllocation |
2300  ded_req.requiresDedicatedAllocation;
2301  if (use_ded_mem)
2302  ded_alloc.image = f->img[img_cnt];
2303 
2304  /* Allocate memory */
2305  if ((err = alloc_mem(ctx, &req.memoryRequirements,
2306  f->tiling == VK_IMAGE_TILING_LINEAR ?
2307  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT :
2308  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
2309  use_ded_mem ? &ded_alloc : (void *)ded_alloc.pNext,
2310  &f->flags, &f->mem[img_cnt])))
2311  return err;
2312 
2313  f->size[img_cnt] = req.memoryRequirements.size;
2314  bind_info[img_cnt].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
2315  bind_info[img_cnt].image = f->img[img_cnt];
2316  bind_info[img_cnt].memory = f->mem[img_cnt];
2317 
2318  img_cnt++;
2319  }
2320 
2321  /* Bind the allocated memory to the images */
2322  ret = vk->BindImageMemory2(hwctx->act_dev, img_cnt, bind_info);
2323  if (ret != VK_SUCCESS) {
2324  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
2325  ff_vk_ret2str(ret));
2326  return AVERROR_EXTERNAL;
2327  }
2328 
2329  return 0;
2330 }
2331 
2332 enum PrepMode {
2333  PREP_MODE_GENERAL,
2334  PREP_MODE_WRITE,
2335  PREP_MODE_EXTERNAL_EXPORT,
2336  PREP_MODE_EXTERNAL_IMPORT,
2337  PREP_MODE_DECODING_DST,
2338  PREP_MODE_DECODING_DPB,
2339  PREP_MODE_ENCODING_DPB,
2340 };
2341 
2342 static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2343  AVVkFrame *frame, enum PrepMode pmode)
2344 {
2345  int err;
2346  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2347  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2348  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
2349  int nb_img_bar = 0;
2350 
2351  uint32_t dst_qf = VK_QUEUE_FAMILY_IGNORED;
2352  VkImageLayout new_layout;
2353  VkAccessFlags2 new_access;
2354  VkPipelineStageFlagBits2 src_stage = VK_PIPELINE_STAGE_2_NONE;
2355 
2356  /* This is dirty - but it works. The vulkan.c dependency system doesn't
2357  * free non-refcounted frames, and non-refcounted hardware frames cannot
2358  * happen anywhere outside of here. */
2359  AVBufferRef tmp_ref = {
2360  .data = (uint8_t *)hwfc,
2361  };
2362  AVFrame tmp_frame = {
2363  .data[0] = (uint8_t *)frame,
2364  .hw_frames_ctx = &tmp_ref,
2365  };
2366 
2367  VkCommandBuffer cmd_buf;
2368  FFVkExecContext *exec = ff_vk_exec_get(&p->vkctx, ectx);
2369  cmd_buf = exec->buf;
2370  ff_vk_exec_start(&p->vkctx, exec);
2371 
2372  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, &tmp_frame,
2373  VK_PIPELINE_STAGE_2_NONE,
2374  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
2375  if (err < 0)
2376  return err;
2377 
2378  switch (pmode) {
2379  case PREP_MODE_GENERAL:
2380  new_layout = VK_IMAGE_LAYOUT_GENERAL;
2381  new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2382  break;
2383  case PREP_MODE_WRITE:
2384  new_layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
2385  new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2386  break;
2387  case PREP_MODE_EXTERNAL_IMPORT:
2388  new_layout = VK_IMAGE_LAYOUT_GENERAL;
2389  new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2390  break;
2391  case PREP_MODE_EXTERNAL_EXPORT:
2392  new_layout = VK_IMAGE_LAYOUT_GENERAL;
2393  new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2394  dst_qf = VK_QUEUE_FAMILY_EXTERNAL_KHR;
2395  src_stage = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
2396  break;
2397  case PREP_MODE_DECODING_DST:
2398  new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR;
2399  new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2400  break;
2401  case PREP_MODE_DECODING_DPB:
2402  new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
2403  new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2404  break;
2405  case PREP_MODE_ENCODING_DPB:
2406  new_layout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
2407  new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2408  break;
2409  }
2410 
2411  ff_vk_frame_barrier(&p->vkctx, exec, &tmp_frame, img_bar, &nb_img_bar,
2412  src_stage,
2413  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
2414  new_access, new_layout, dst_qf);
2415 
2416  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
2417  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
2418  .pImageMemoryBarriers = img_bar,
2419  .imageMemoryBarrierCount = nb_img_bar,
2420  });
2421 
2422  err = ff_vk_exec_submit(&p->vkctx, exec);
2423  if (err < 0)
2424  return err;
2425 
2426  /* We can do this because there are no real dependencies */
2427  ff_vk_exec_discard_deps(&p->vkctx, exec);
2428 
2429  return 0;
2430 }
2431 
2432 static inline void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format,
2433  int frame_w, int frame_h, int plane)
2434 {
2435  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(format);
2436 
2437  /* Currently always true unless gray + alpha support is added */
2438  if (!plane || (plane == 3) || desc->flags & AV_PIX_FMT_FLAG_RGB ||
2439  !(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) {
2440  *w = frame_w;
2441  *h = frame_h;
2442  return;
2443  }
2444 
2445  *w = AV_CEIL_RSHIFT(frame_w, desc->log2_chroma_w);
2446  *h = AV_CEIL_RSHIFT(frame_h, desc->log2_chroma_h);
2447 }
2448 
2449 static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame,
2450  VkImageTiling tiling, VkImageUsageFlagBits usage,
2451  VkImageCreateFlags flags, int nb_layers,
2452  void *create_pnext)
2453 {
2454  int err;
2455  VkResult ret;
2456  AVVulkanFramesContext *hwfc_vk = hwfc->hwctx;
2457  AVHWDeviceContext *ctx = hwfc->device_ctx;
2458  VulkanDevicePriv *p = ctx->hwctx;
2459  AVVulkanDeviceContext *hwctx = &p->p;
2460  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2461  AVVkFrame *f;
2462 
2463  VkSemaphoreTypeCreateInfo sem_type_info = {
2464  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
2465  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
2466  .initialValue = 0,
2467  };
2468  VkSemaphoreCreateInfo sem_spawn = {
2469  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
2470  .pNext = &sem_type_info,
2471  };
2472 
2473  VkExportSemaphoreCreateInfo ext_sem_info_opaque = {
2474  .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
2475 #ifdef _WIN32
2476  .handleTypes = IsWindows8OrGreater()
2477  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
2478  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
2479 #else
2480  .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
2481 #endif
2482  };
2483 
2484  /* Check if exporting is supported before chaining any structs */
2485  if (p->ext_sem_props_opaque.externalSemaphoreFeatures & VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT) {
2486  if (p->vkctx.extensions & (FF_VK_EXT_EXTERNAL_WIN32_SEM | FF_VK_EXT_EXTERNAL_FD_SEM))
2487  ff_vk_link_struct(&sem_type_info, &ext_sem_info_opaque);
2488  }
2489 
2490  f = av_vk_frame_alloc();
2491  if (!f) {
2492  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
2493  return AVERROR(ENOMEM);
2494  }
2495 
2496  // TODO: check witdh and height for alignment in case of multiplanar (must be mod-2 if subsampled)
2497 
2498  /* Create the images */
2499  for (int i = 0; (hwfc_vk->format[i] != VK_FORMAT_UNDEFINED); i++) {
2500  VkImageCreateInfo create_info = {
2501  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
2502  .pNext = create_pnext,
2503  .imageType = VK_IMAGE_TYPE_2D,
2504  .format = hwfc_vk->format[i],
2505  .extent.depth = 1,
2506  .mipLevels = 1,
2507  .arrayLayers = nb_layers,
2508  .flags = flags,
2509  .tiling = tiling,
2510  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
2511  .usage = usage,
2512  .samples = VK_SAMPLE_COUNT_1_BIT,
2513  .pQueueFamilyIndices = p->img_qfs,
2514  .queueFamilyIndexCount = p->nb_img_qfs,
2515  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2516  VK_SHARING_MODE_EXCLUSIVE,
2517  };
2518 
2519  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
2520  hwfc->sw_format, hwfc->width, hwfc->height, i);
2521 
2522  ret = vk->CreateImage(hwctx->act_dev, &create_info,
2523  hwctx->alloc, &f->img[i]);
2524  if (ret != VK_SUCCESS) {
2525  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
2526  ff_vk_ret2str(ret));
2527  err = AVERROR(EINVAL);
2528  goto fail;
2529  }
2530 
2531  /* Create semaphore */
2532  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
2533  hwctx->alloc, &f->sem[i]);
2534  if (ret != VK_SUCCESS) {
2535  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
2536  ff_vk_ret2str(ret));
2537  err = AVERROR_EXTERNAL;
2538  goto fail;
2539  }
2540 
2541  f->queue_family[i] = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2542  f->layout[i] = create_info.initialLayout;
2543  f->access[i] = 0x0;
2544  f->sem_value[i] = 0;
2545  }
2546 
2547  f->flags = 0x0;
2548  f->tiling = tiling;
2549 
2550  *frame = f;
2551  return 0;
2552 
2553 fail:
2554  vulkan_frame_free(hwfc, f);
2555  return err;
2556 }
2557 
2558 /* Checks if an export flag is enabled, and if it is ORs it with *iexp */
2559 static void try_export_flags(AVHWFramesContext *hwfc,
2560  VkExternalMemoryHandleTypeFlags *comp_handle_types,
2561  VkExternalMemoryHandleTypeFlagBits *iexp,
2562  VkExternalMemoryHandleTypeFlagBits exp)
2563 {
2564  VkResult ret;
2565  AVVulkanFramesContext *hwctx = hwfc->hwctx;
2566  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2567  AVVulkanDeviceContext *dev_hwctx = &p->p;
2568  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2569 
2570  const VkImageDrmFormatModifierListCreateInfoEXT *drm_mod_info =
2571  ff_vk_find_struct(hwctx->create_pnext,
2572  VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
2573  int has_mods = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT && drm_mod_info;
2574  int nb_mods;
2575 
2576  VkExternalImageFormatProperties eprops = {
2577  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
2578  };
2579  VkImageFormatProperties2 props = {
2580  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
2581  .pNext = &eprops,
2582  };
2583  VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
2584  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
2585  .pNext = NULL,
2586  .pQueueFamilyIndices = p->img_qfs,
2587  .queueFamilyIndexCount = p->nb_img_qfs,
2588  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2589  VK_SHARING_MODE_EXCLUSIVE,
2590  };
2591  VkPhysicalDeviceExternalImageFormatInfo enext = {
2592  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
2593  .handleType = exp,
2594  .pNext = has_mods ? &phy_dev_mod_info : NULL,
2595  };
2596  VkPhysicalDeviceImageFormatInfo2 pinfo = {
2597  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
2598  .pNext = !exp ? NULL : &enext,
2599  .format = av_vkfmt_from_pixfmt(hwfc->sw_format)[0],
2600  .type = VK_IMAGE_TYPE_2D,
2601  .tiling = hwctx->tiling,
2602  .usage = hwctx->usage,
2603  .flags = VK_IMAGE_CREATE_ALIAS_BIT,
2604  };
2605 
2606  nb_mods = has_mods ? drm_mod_info->drmFormatModifierCount : 1;
2607  for (int i = 0; i < nb_mods; i++) {
2608  if (has_mods)
2609  phy_dev_mod_info.drmFormatModifier = drm_mod_info->pDrmFormatModifiers[i];
2610 
2611  ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
2612  &pinfo, &props);
2613 
2614  if (ret == VK_SUCCESS) {
2615  *iexp |= exp;
2616  *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
2617  }
2618  }
2619 }
2620 
2621 static AVBufferRef *vulkan_pool_alloc(void *opaque, size_t size)
2622 {
2623  int err;
2624  AVVkFrame *f;
2625  AVBufferRef *avbuf = NULL;
2626  AVHWFramesContext *hwfc = opaque;
2627  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2628  VulkanFramesPriv *fp = hwfc->hwctx;
2629  AVVulkanFramesContext *hwctx = &fp->p;
2630  VkExternalMemoryHandleTypeFlags e = 0x0;
2631  VkExportMemoryAllocateInfo eminfo[AV_NUM_DATA_POINTERS];
2632 
2633  VkExternalMemoryImageCreateInfo eiinfo = {
2634  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
2635  .pNext = hwctx->create_pnext,
2636  };
2637 
2638 #ifdef _WIN32
2639  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY)
2640  try_export_flags(hwfc, &eiinfo.handleTypes, &e, IsWindows8OrGreater()
2641  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
2642  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT);
2643 #else
2644  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY)
2645  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2646  VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT);
2647 #endif
2648 
2649  for (int i = 0; i < av_pix_fmt_count_planes(hwfc->sw_format); i++) {
2650  eminfo[i].sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
2651  eminfo[i].pNext = hwctx->alloc_pnext[i];
2652  eminfo[i].handleTypes = e;
2653  }
2654 
2655  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
2656  hwctx->nb_layers,
2657  eiinfo.handleTypes ? &eiinfo : hwctx->create_pnext);
2658  if (err)
2659  return NULL;
2660 
2661  err = alloc_bind_mem(hwfc, f, eminfo, sizeof(*eminfo));
2662  if (err)
2663  goto fail;
2664 
2665  if ( (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2666  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR))
2667  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DPB);
2668  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)
2669  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DST);
2670  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR)
2671  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_ENCODING_DPB);
2672  else if (hwctx->usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
2673  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_WRITE);
2674  else
2675  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_GENERAL);
2676  if (err)
2677  goto fail;
2678 
2679  avbuf = av_buffer_create((uint8_t *)f, sizeof(AVVkFrame),
2680  vulkan_frame_free_cb, hwfc, 0);
2681  if (!avbuf)
2682  goto fail;
2683 
2684  return avbuf;
2685 
2686 fail:
2687  vulkan_frame_free(hwfc, f);
2688  return NULL;
2689 }
2690 
2691 static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2692 {
2693  pthread_mutex_lock(&vkf->internal->update_mutex);
2694 }
2695 
2696 static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2697 {
2698  pthread_mutex_unlock(&vkf->internal->update_mutex);
2699 }
2700 
2701 static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
2702 {
2703  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2704  VulkanFramesPriv *fp = hwfc->hwctx;
2705 
2706  if (fp->modifier_info) {
2707  if (fp->modifier_info->pDrmFormatModifiers)
2708  av_freep(&fp->modifier_info->pDrmFormatModifiers);
2709  av_freep(&fp->modifier_info);
2710  }
2711 
2712  ff_vk_exec_pool_free(&p->vkctx, &fp->compute_exec);
2713  ff_vk_exec_pool_free(&p->vkctx, &fp->upload_exec);
2714  ff_vk_exec_pool_free(&p->vkctx, &fp->download_exec);
2715 
2716  av_buffer_pool_uninit(&fp->tmp);
2717 }
2718 
2719 static int vulkan_frames_init(AVHWFramesContext *hwfc)
2720 {
2721  int err;
2722  AVVkFrame *f;
2723  VulkanFramesPriv *fp = hwfc->hwctx;
2724  AVVulkanFramesContext *hwctx = &fp->p;
2725  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2726  VkImageUsageFlagBits supported_usage;
2727  const struct FFVkFormatEntry *fmt;
2728  int disable_multiplane = p->disable_multiplane ||
2729  (hwctx->flags & AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE);
2730 
2731  /* Defaults */
2732  if (!hwctx->nb_layers)
2733  hwctx->nb_layers = 1;
2734 
2735  /* VK_IMAGE_TILING_OPTIMAL == 0, can't check for it really */
2736  if (p->use_linear_images &&
2737  (hwctx->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT))
2738  hwctx->tiling = VK_IMAGE_TILING_LINEAR;
2739 
2740 
2741  fmt = vk_find_format_entry(hwfc->sw_format);
2742  if (!fmt) {
2743  av_log(hwfc, AV_LOG_ERROR, "Unsupported pixel format: %s!\n",
2744  av_get_pix_fmt_name(hwfc->sw_format));
2745  return AVERROR(EINVAL);
2746  }
2747 
2748  if (hwctx->format[0] != VK_FORMAT_UNDEFINED) {
2749  if (hwctx->format[0] != fmt->vkf) {
2750  for (int i = 0; i < fmt->nb_images_fallback; i++) {
2751  if (hwctx->format[i] != fmt->fallback[i]) {
2752  av_log(hwfc, AV_LOG_ERROR, "Incompatible Vulkan format given "
2753  "for the current sw_format %s!\n",
2754  av_get_pix_fmt_name(hwfc->sw_format));
2755  return AVERROR(EINVAL);
2756  }
2757  }
2758  }
2759 
2760  /* Check if the sw_format itself is supported */
2761  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
2762  hwctx->tiling, NULL,
2763  NULL, NULL, &supported_usage, 0,
2764  !hwctx->usage ||
2765  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
2766  if (err < 0) {
2767  av_log(hwfc, AV_LOG_ERROR, "Unsupported sw format: %s!\n",
2768  av_get_pix_fmt_name(hwfc->sw_format));
2769  return AVERROR(EINVAL);
2770  }
2771  } else {
2772  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
2773  hwctx->tiling, hwctx->format, NULL,
2774  NULL, &supported_usage,
2775  disable_multiplane,
2776  !hwctx->usage ||
2777  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
2778  if (err < 0)
2779  return err;
2780  }
2781 
2782  /* Image usage flags */
2783  if (!hwctx->usage) {
2784  hwctx->usage = supported_usage & (VK_BUFFER_USAGE_TRANSFER_DST_BIT |
2785  VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
2786  VK_IMAGE_USAGE_STORAGE_BIT |
2787  VK_IMAGE_USAGE_SAMPLED_BIT);
2788 
2789  /* Enables encoding of images, if supported by format and extensions */
2790  if ((supported_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
2791  (p->vkctx.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
2792  FF_VK_EXT_VIDEO_MAINTENANCE_1)))
2793  hwctx->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
2794  }
2795 
2796  /* Image creation flags.
2797  * Only fill them in automatically if the image is not going to be used as
2798  * a DPB-only image, and we have SAMPLED/STORAGE bits set. */
2799  if (!hwctx->img_flags) {
2800  int is_lone_dpb = ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) ||
2801  ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2802  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)));
2803  int sampleable = hwctx->usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
2804  VK_IMAGE_USAGE_STORAGE_BIT);
2805  hwctx->img_flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
2806  if (sampleable && !is_lone_dpb) {
2807  hwctx->img_flags |= VK_IMAGE_CREATE_ALIAS_BIT;
2808  if ((fmt->vk_planes > 1) && (hwctx->format[0] == fmt->vkf))
2809  hwctx->img_flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
2810  }
2811  }
2812 
2813  /* If the image has an ENCODE_SRC usage, and the maintenance1
2814  * extension is supported, check if it has a profile list.
2815  * If there's no profile list, or it has no encode operations,
2816  * then allow creating the image with no specific profile. */
2817  if ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
2818  (p->vkctx.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
2819  FF_VK_EXT_VIDEO_MAINTENANCE_1))) {
2820  const VkVideoProfileListInfoKHR *pl;
2821  pl = ff_vk_find_struct(hwctx->create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
2822  if (!pl) {
2823  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
2824  } else {
2825  uint32_t i;
2826  for (i = 0; i < pl->profileCount; i++) {
2827  /* Video ops start at exactly 0x00010000 */
2828  if (pl->pProfiles[i].videoCodecOperation & 0xFFFF0000)
2829  break;
2830  }
2831  if (i == pl->profileCount)
2832  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
2833  }
2834  }
2835 
2836  if (!hwctx->lock_frame)
2837  hwctx->lock_frame = lock_frame;
2838 
2839  if (!hwctx->unlock_frame)
2840  hwctx->unlock_frame = unlock_frame;
2841 
2842  err = ff_vk_exec_pool_init(&p->vkctx, p->compute_qf, &fp->compute_exec,
2843  p->compute_qf->num, 0, 0, 0, NULL);
2844  if (err)
2845  return err;
2846 
2847  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->upload_exec,
2848  p->transfer_qf->num*2, 0, 0, 0, NULL);
2849  if (err)
2850  return err;
2851 
2852  err = ff_vk_exec_pool_init(&p->vkctx, p->transfer_qf, &fp->download_exec,
2853  p->transfer_qf->num, 0, 0, 0, NULL);
2854  if (err)
2855  return err;
2856 
2857  /* Test to see if allocation will fail */
2858  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
2859  hwctx->nb_layers, hwctx->create_pnext);
2860  if (err)
2861  return err;
2862 
2863  vulkan_frame_free(hwfc, f);
2864 
2865  /* If user did not specify a pool, hwfc->pool will be set to the internal one
2866  * in hwcontext.c just after this gets called */
2867  if (!hwfc->pool) {
2868  ffhwframesctx(hwfc)->pool_internal = av_buffer_pool_init2(sizeof(AVVkFrame),
2869  hwfc, vulkan_pool_alloc,
2870  NULL);
2871  if (!ffhwframesctx(hwfc)->pool_internal)
2872  return AVERROR(ENOMEM);
2873  }
2874 
2875  return 0;
2876 }
2877 
2878 static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
2879 {
2880  frame->buf[0] = av_buffer_pool_get(hwfc->pool);
2881  if (!frame->buf[0])
2882  return AVERROR(ENOMEM);
2883 
2884  frame->data[0] = frame->buf[0]->data;
2885  frame->format = AV_PIX_FMT_VULKAN;
2886  frame->width = hwfc->width;
2887  frame->height = hwfc->height;
2888 
2889  return 0;
2890 }
2891 
2892 static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc,
2893  enum AVHWFrameTransferDirection dir,
2894  enum AVPixelFormat **formats)
2895 {
2896  enum AVPixelFormat *fmts;
2897  int n = 2;
2898 
2899 #if CONFIG_CUDA
2900  n++;
2901 #endif
2902  fmts = av_malloc_array(n, sizeof(*fmts));
2903  if (!fmts)
2904  return AVERROR(ENOMEM);
2905 
2906  n = 0;
2907  fmts[n++] = hwfc->sw_format;
2908 #if CONFIG_CUDA
2909  fmts[n++] = AV_PIX_FMT_CUDA;
2910 #endif
2911  fmts[n++] = AV_PIX_FMT_NONE;
2912 
2913  *formats = fmts;
2914  return 0;
2915 }
2916 
2917 #if CONFIG_LIBDRM
2918 static void vulkan_unmap_from_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
2919 {
2920  vulkan_frame_free(hwfc, hwmap->priv);
2921 }
2922 
2923 static const struct {
2924  uint32_t drm_fourcc;
2925  VkFormat vk_format;
2926 } vulkan_drm_format_map[] = {
2927  { DRM_FORMAT_R8, VK_FORMAT_R8_UNORM },
2928  { DRM_FORMAT_R16, VK_FORMAT_R16_UNORM },
2929  { DRM_FORMAT_GR88, VK_FORMAT_R8G8_UNORM },
2930  { DRM_FORMAT_RG88, VK_FORMAT_R8G8_UNORM },
2931  { DRM_FORMAT_GR1616, VK_FORMAT_R16G16_UNORM },
2932  { DRM_FORMAT_RG1616, VK_FORMAT_R16G16_UNORM },
2933  { DRM_FORMAT_ARGB8888, VK_FORMAT_B8G8R8A8_UNORM },
2934  { DRM_FORMAT_XRGB8888, VK_FORMAT_B8G8R8A8_UNORM },
2935  { DRM_FORMAT_ABGR8888, VK_FORMAT_R8G8B8A8_UNORM },
2936  { DRM_FORMAT_XBGR8888, VK_FORMAT_R8G8B8A8_UNORM },
2937  { DRM_FORMAT_ARGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
2938  { DRM_FORMAT_ABGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
2939  { DRM_FORMAT_XRGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
2940  { DRM_FORMAT_XBGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
2941 
2942  // All these DRM_FORMATs were added in the same libdrm commit.
2943 #ifdef DRM_FORMAT_XYUV8888
2944  { DRM_FORMAT_XYUV8888, VK_FORMAT_R8G8B8A8_UNORM },
2945  { DRM_FORMAT_XVYU2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 } ,
2946  { DRM_FORMAT_XVYU12_16161616, VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16 } ,
2947  { DRM_FORMAT_XVYU16161616, VK_FORMAT_R16G16B16A16_UNORM } ,
2948 #endif
2949 };
2950 
2951 static inline VkFormat drm_to_vulkan_fmt(uint32_t drm_fourcc)
2952 {
2953  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
2954  if (vulkan_drm_format_map[i].drm_fourcc == drm_fourcc)
2955  return vulkan_drm_format_map[i].vk_format;
2956  return VK_FORMAT_UNDEFINED;
2957 }
2958 
2959 static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **frame,
2960  const AVFrame *src, int flags)
2961 {
2962  int err = 0;
2963  VkResult ret;
2964  AVVkFrame *f;
2965  int bind_counts = 0;
2966  AVHWDeviceContext *ctx = hwfc->device_ctx;
2967  VulkanDevicePriv *p = ctx->hwctx;
2968  AVVulkanDeviceContext *hwctx = &p->p;
2969  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2970  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
2971  VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES];
2972  VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES];
2973 
2974  for (int i = 0; i < desc->nb_layers; i++) {
2975  if (drm_to_vulkan_fmt(desc->layers[i].format) == VK_FORMAT_UNDEFINED) {
2976  av_log(ctx, AV_LOG_ERROR, "Unsupported DMABUF layer format %#08x!\n",
2977  desc->layers[i].format);
2978  return AVERROR(EINVAL);
2979  }
2980  }
2981 
2982  if (!(f = av_vk_frame_alloc())) {
2983  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
2984  err = AVERROR(ENOMEM);
2985  goto fail;
2986  }
2987 
2988  f->tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
2989 
2990  for (int i = 0; i < desc->nb_layers; i++) {
2991  const int planes = desc->layers[i].nb_planes;
2992 
2993  /* Semaphore */
2994  VkSemaphoreTypeCreateInfo sem_type_info = {
2995  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
2996  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
2997  .initialValue = 0,
2998  };
2999  VkSemaphoreCreateInfo sem_spawn = {
3000  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3001  .pNext = &sem_type_info,
3002  };
3003 
3004  /* Image creation */
3005  VkSubresourceLayout ext_img_layouts[AV_DRM_MAX_PLANES];
3006  VkImageDrmFormatModifierExplicitCreateInfoEXT ext_img_mod_spec = {
3007  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT,
3008  .drmFormatModifier = desc->objects[0].format_modifier,
3009  .drmFormatModifierPlaneCount = planes,
3010  .pPlaneLayouts = (const VkSubresourceLayout *)&ext_img_layouts,
3011  };
3012  VkExternalMemoryImageCreateInfo ext_img_spec = {
3013  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
3014  .pNext = &ext_img_mod_spec,
3015  .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3016  };
3017  VkImageCreateInfo create_info = {
3018  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
3019  .pNext = &ext_img_spec,
3020  .imageType = VK_IMAGE_TYPE_2D,
3021  .format = drm_to_vulkan_fmt(desc->layers[i].format),
3022  .extent.depth = 1,
3023  .mipLevels = 1,
3024  .arrayLayers = 1,
3025  .flags = 0x0,
3026  .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
3027  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, /* specs say so */
3028  .usage = 0x0, /* filled in below */
3029  .samples = VK_SAMPLE_COUNT_1_BIT,
3030  .pQueueFamilyIndices = p->img_qfs,
3031  .queueFamilyIndexCount = p->nb_img_qfs,
3032  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
3033  VK_SHARING_MODE_EXCLUSIVE,
3034  };
3035 
3036  /* Image format verification */
3037  VkExternalImageFormatProperties ext_props = {
3038  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
3039  };
3040  VkImageFormatProperties2 props_ret = {
3041  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
3042  .pNext = &ext_props,
3043  };
3044  VkPhysicalDeviceImageDrmFormatModifierInfoEXT props_drm_mod = {
3045  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
3046  .drmFormatModifier = ext_img_mod_spec.drmFormatModifier,
3047  .pQueueFamilyIndices = create_info.pQueueFamilyIndices,
3048  .queueFamilyIndexCount = create_info.queueFamilyIndexCount,
3049  .sharingMode = create_info.sharingMode,
3050  };
3051  VkPhysicalDeviceExternalImageFormatInfo props_ext = {
3052  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
3053  .pNext = &props_drm_mod,
3054  .handleType = ext_img_spec.handleTypes,
3055  };
3056  VkPhysicalDeviceImageFormatInfo2 fmt_props;
3057 
3058  if (flags & AV_HWFRAME_MAP_READ)
3059  create_info.usage |= VK_IMAGE_USAGE_SAMPLED_BIT |
3060  VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
3061  if (flags & AV_HWFRAME_MAP_WRITE)
3062  create_info.usage |= VK_IMAGE_USAGE_STORAGE_BIT |
3063  VK_IMAGE_USAGE_TRANSFER_DST_BIT;
3064 
3065  fmt_props = (VkPhysicalDeviceImageFormatInfo2) {
3066  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
3067  .pNext = &props_ext,
3068  .format = create_info.format,
3069  .type = create_info.imageType,
3070  .tiling = create_info.tiling,
3071  .usage = create_info.usage,
3072  .flags = create_info.flags,
3073  };
3074 
3075  /* Check if importing is possible for this combination of parameters */
3076  ret = vk->GetPhysicalDeviceImageFormatProperties2(hwctx->phys_dev,
3077  &fmt_props, &props_ret);
3078  if (ret != VK_SUCCESS) {
3079  av_log(ctx, AV_LOG_ERROR, "Cannot map DRM frame to Vulkan: %s\n",
3080  ff_vk_ret2str(ret));
3081  err = AVERROR_EXTERNAL;
3082  goto fail;
3083  }
3084 
3085  /* Set the image width/height */
3086  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
3087  hwfc->sw_format, src->width, src->height, i);
3088 
3089  /* Set the subresource layout based on the layer properties */
3090  for (int j = 0; j < planes; j++) {
3091  ext_img_layouts[j].offset = desc->layers[i].planes[j].offset;
3092  ext_img_layouts[j].rowPitch = desc->layers[i].planes[j].pitch;
3093  ext_img_layouts[j].size = 0; /* The specs say so for all 3 */
3094  ext_img_layouts[j].arrayPitch = 0;
3095  ext_img_layouts[j].depthPitch = 0;
3096  }
3097 
3098  /* Create image */
3099  ret = vk->CreateImage(hwctx->act_dev, &create_info,
3100  hwctx->alloc, &f->img[i]);
3101  if (ret != VK_SUCCESS) {
3102  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
3103  ff_vk_ret2str(ret));
3104  err = AVERROR(EINVAL);
3105  goto fail;
3106  }
3107 
3108  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3109  hwctx->alloc, &f->sem[i]);
3110  if (ret != VK_SUCCESS) {
3111  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3112  ff_vk_ret2str(ret));
3113  err = AVERROR_EXTERNAL;
3114  goto fail;
3115  }
3116 
3117  f->queue_family[i] = VK_QUEUE_FAMILY_EXTERNAL;
3118  f->layout[i] = create_info.initialLayout;
3119  f->access[i] = 0x0;
3120  f->sem_value[i] = 0;
3121  }
3122 
3123  for (int i = 0; i < desc->nb_layers; i++) {
3124  /* Memory requirements */
3125  VkImageMemoryRequirementsInfo2 req_desc = {
3126  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
3127  .image = f->img[i],
3128  };
3129  VkMemoryDedicatedRequirements ded_req = {
3130  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
3131  };
3132  VkMemoryRequirements2 req2 = {
3133  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
3134  .pNext = &ded_req,
3135  };
3136 
3137  /* Allocation/importing */
3138  VkMemoryFdPropertiesKHR fdmp = {
3139  .sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR,
3140  };
3141  /* This assumes that a layer will never be constructed from multiple
3142  * objects. If that was to happen in the real world, this code would
3143  * need to import each plane separately.
3144  */
3145  VkImportMemoryFdInfoKHR idesc = {
3146  .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
3147  .fd = dup(desc->objects[desc->layers[i].planes[0].object_index].fd),
3148  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3149  };
3150  VkMemoryDedicatedAllocateInfo ded_alloc = {
3151  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
3152  .pNext = &idesc,
3153  .image = req_desc.image,
3154  };
3155 
3156  /* Get object properties */
3157  ret = vk->GetMemoryFdPropertiesKHR(hwctx->act_dev,
3158  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3159  idesc.fd, &fdmp);
3160  if (ret != VK_SUCCESS) {
3161  av_log(hwfc, AV_LOG_ERROR, "Failed to get FD properties: %s\n",
3162  ff_vk_ret2str(ret));
3163  err = AVERROR_EXTERNAL;
3164  close(idesc.fd);
3165  goto fail;
3166  }
3167 
3168  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req2);
3169 
3170  /* Only a single bit must be set, not a range, and it must match */
3171  req2.memoryRequirements.memoryTypeBits = fdmp.memoryTypeBits;
3172 
3173  err = alloc_mem(ctx, &req2.memoryRequirements,
3174  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
3175  (ded_req.prefersDedicatedAllocation ||
3176  ded_req.requiresDedicatedAllocation) ?
3177  &ded_alloc : ded_alloc.pNext,
3178  &f->flags, &f->mem[i]);
3179  if (err) {
3180  close(idesc.fd);
3181  return err;
3182  }
3183 
3184  f->size[i] = req2.memoryRequirements.size;
3185  }
3186 
3187  for (int i = 0; i < desc->nb_layers; i++) {
3188  const int planes = desc->layers[i].nb_planes;
3189  for (int j = 0; j < planes; j++) {
3190  VkImageAspectFlagBits aspect = j == 0 ? VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT :
3191  j == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3192  VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
3193 
3194  plane_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO;
3195  plane_info[bind_counts].pNext = NULL;
3196  plane_info[bind_counts].planeAspect = aspect;
3197 
3198  bind_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
3199  bind_info[bind_counts].pNext = planes > 1 ? &plane_info[bind_counts] : NULL;
3200  bind_info[bind_counts].image = f->img[i];
3201  bind_info[bind_counts].memory = f->mem[i];
3202 
3203  /* Offset is already signalled via pPlaneLayouts above */
3204  bind_info[bind_counts].memoryOffset = 0;
3205 
3206  bind_counts++;
3207  }
3208  }
3209 
3210  /* Bind the allocated memory to the images */
3211  ret = vk->BindImageMemory2(hwctx->act_dev, bind_counts, bind_info);
3212  if (ret != VK_SUCCESS) {
3213  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
3214  ff_vk_ret2str(ret));
3215  err = AVERROR_EXTERNAL;
3216  goto fail;
3217  }
3218 
3219  *frame = f;
3220 
3221  return 0;
3222 
3223 fail:
3224  vulkan_frame_free(hwfc, f);
3225 
3226  return err;
3227 }
3228 
3229 static int vulkan_map_from_drm_frame_sync(AVHWFramesContext *hwfc, AVFrame *dst,
3230  const AVFrame *src, int flags)
3231 {
3232  int err;
3233  VkResult ret;
3234  AVHWDeviceContext *ctx = hwfc->device_ctx;
3235  VulkanDevicePriv *p = ctx->hwctx;
3236  VulkanFramesPriv *fp = hwfc->hwctx;
3237  AVVulkanDeviceContext *hwctx = &p->p;
3238  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3239 
3240  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3241 
3242 #ifdef DMA_BUF_IOCTL_EXPORT_SYNC_FILE
3243  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM) {
3244  VkCommandBuffer cmd_buf;
3245  FFVkExecContext *exec;
3246  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
3247  VkSemaphore drm_sync_sem[AV_DRM_MAX_PLANES] = { 0 };
3248  int nb_img_bar = 0;
3249 
3250  for (int i = 0; i < desc->nb_objects; i++) {
3251  VkSemaphoreTypeCreateInfo sem_type_info = {
3252  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3253  .semaphoreType = VK_SEMAPHORE_TYPE_BINARY,
3254  };
3255  VkSemaphoreCreateInfo sem_spawn = {
3256  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3257  .pNext = &sem_type_info,
3258  };
3259  VkImportSemaphoreFdInfoKHR import_info;
3260  struct dma_buf_export_sync_file implicit_fd_info = {
3261  .flags = DMA_BUF_SYNC_READ,
3262  .fd = -1,
3263  };
3264 
3265  if (ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE,
3266  &implicit_fd_info)) {
3267  err = AVERROR(errno);
3268  av_log(hwctx, AV_LOG_ERROR, "Failed to retrieve implicit DRM sync file: %s\n",
3269  av_err2str(err));
3270  for (; i >= 0; i--)
3271  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3272  return err;
3273  }
3274 
3275  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3276  hwctx->alloc, &drm_sync_sem[i]);
3277  if (ret != VK_SUCCESS) {
3278  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3279  ff_vk_ret2str(ret));
3280  err = AVERROR_EXTERNAL;
3281  for (; i >= 0; i--)
3282  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3283  return err;
3284  }
3285 
3286  import_info = (VkImportSemaphoreFdInfoKHR) {
3287  .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
3288  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3289  .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT,
3290  .semaphore = drm_sync_sem[i],
3291  .fd = implicit_fd_info.fd,
3292  };
3293 
3294  ret = vk->ImportSemaphoreFdKHR(hwctx->act_dev, &import_info);
3295  if (ret != VK_SUCCESS) {
3296  av_log(hwctx, AV_LOG_ERROR, "Failed to import semaphore: %s\n",
3297  ff_vk_ret2str(ret));
3298  err = AVERROR_EXTERNAL;
3299  for (; i >= 0; i--)
3300  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3301  return err;
3302  }
3303  }
3304 
3305  exec = ff_vk_exec_get(&p->vkctx, &fp->compute_exec);
3306  cmd_buf = exec->buf;
3307 
3308  ff_vk_exec_start(&p->vkctx, exec);
3309 
3310  /* Ownership of semaphores is passed */
3311  err = ff_vk_exec_add_dep_bool_sem(&p->vkctx, exec,
3312  drm_sync_sem, desc->nb_objects,
3313  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 1);
3314  if (err < 0)
3315  return err;
3316 
3317  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, dst,
3318  VK_PIPELINE_STAGE_2_NONE,
3319  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
3320  if (err < 0)
3321  return err;
3322 
3323  ff_vk_frame_barrier(&p->vkctx, exec, dst, img_bar, &nb_img_bar,
3324  VK_PIPELINE_STAGE_2_NONE,
3325  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
3326  ((flags & AV_HWFRAME_MAP_READ) ?
3327  VK_ACCESS_2_SHADER_SAMPLED_READ_BIT : 0x0) |
3328  ((flags & AV_HWFRAME_MAP_WRITE) ?
3329  VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0x0),
3330  VK_IMAGE_LAYOUT_GENERAL,
3331  VK_QUEUE_FAMILY_IGNORED);
3332 
3333  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
3334  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3335  .pImageMemoryBarriers = img_bar,
3336  .imageMemoryBarrierCount = nb_img_bar,
3337  });
3338 
3339  err = ff_vk_exec_submit(&p->vkctx, exec);
3340  if (err < 0)
3341  return err;
3342  } else
3343 #endif
3344  {
3345  AVVkFrame *f = (AVVkFrame *)dst->data[0];
3346  av_log(hwctx, AV_LOG_WARNING, "No support for synchronization when importing DMA-BUFs, "
3347  "image may be corrupted.\n");
3348  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3349  if (err)
3350  return err;
3351  }
3352 
3353  return 0;
3354 }
3355 
3356 static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3357  const AVFrame *src, int flags)
3358 {
3359  int err = 0;
3360  AVVkFrame *f;
3361 
3362  if ((err = vulkan_map_from_drm_frame_desc(hwfc, &f, src, flags)))
3363  return err;
3364 
3365  /* The unmapping function will free this */
3366  dst->data[0] = (uint8_t *)f;
3367  dst->width = src->width;
3368  dst->height = src->height;
3369 
3370  err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
3371  &vulkan_unmap_from_drm, f);
3372  if (err < 0)
3373  goto fail;
3374 
3375  err = vulkan_map_from_drm_frame_sync(hwfc, dst, src, flags);
3376  if (err < 0)
3377  return err;
3378 
3379  av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n");
3380 
3381  return 0;
3382 
3383 fail:
3384  vulkan_frame_free(hwfc->device_ctx->hwctx, f);
3385  dst->data[0] = NULL;
3386  return err;
3387 }
3388 
3389 #if CONFIG_VAAPI
3390 static int vulkan_map_from_vaapi(AVHWFramesContext *dst_fc,
3391  AVFrame *dst, const AVFrame *src,
3392  int flags)
3393 {
3394  int err;
3395  AVFrame *tmp = av_frame_alloc();
3396  AVHWFramesContext *vaapi_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3397  AVVAAPIDeviceContext *vaapi_ctx = vaapi_fc->device_ctx->hwctx;
3398  VASurfaceID surface_id = (VASurfaceID)(uintptr_t)src->data[3];
3399 
3400  if (!tmp)
3401  return AVERROR(ENOMEM);
3402 
3403  /* We have to sync since like the previous comment said, no semaphores */
3404  vaSyncSurface(vaapi_ctx->display, surface_id);
3405 
3406  tmp->format = AV_PIX_FMT_DRM_PRIME;
3407 
3408  err = av_hwframe_map(tmp, src, flags);
3409  if (err < 0)
3410  goto fail;
3411 
3412  err = vulkan_map_from_drm(dst_fc, dst, tmp, flags);
3413  if (err < 0)
3414  goto fail;
3415 
3416  err = ff_hwframe_map_replace(dst, src);
3417 
3418 fail:
3419  av_frame_free(&tmp);
3420  return err;
3421 }
3422 #endif
3423 #endif
3424 
3425 #if CONFIG_CUDA
3426 static int export_mem_to_cuda(AVHWDeviceContext *ctx,
3427  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3428  AVVkFrameInternal *dst_int, int idx,
3429  VkDeviceMemory mem, size_t size)
3430 {
3431  VkResult ret;
3432  VulkanDevicePriv *p = ctx->hwctx;
3433  AVVulkanDeviceContext *hwctx = &p->p;
3434  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3435 
3436 #ifdef _WIN32
3437  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3438  .type = IsWindows8OrGreater()
3439  ? CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32
3440  : CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT,
3441  .size = size,
3442  };
3443  VkMemoryGetWin32HandleInfoKHR export_info = {
3444  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR,
3445  .memory = mem,
3446  .handleType = IsWindows8OrGreater()
3447  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
3448  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3449  };
3450 
3451  ret = vk->GetMemoryWin32HandleKHR(hwctx->act_dev, &export_info,
3452  &ext_desc.handle.win32.handle);
3453  if (ret != VK_SUCCESS) {
3454  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a Win32 Handle: %s!\n",
3455  ff_vk_ret2str(ret));
3456  return AVERROR_EXTERNAL;
3457  }
3458  dst_int->ext_mem_handle[idx] = ext_desc.handle.win32.handle;
3459 #else
3460  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3461  .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD,
3462  .size = size,
3463  };
3464  VkMemoryGetFdInfoKHR export_info = {
3465  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
3466  .memory = mem,
3467  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR,
3468  };
3469 
3470  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
3471  &ext_desc.handle.fd);
3472  if (ret != VK_SUCCESS) {
3473  av_log(ctx, AV_LOG_ERROR, "Unable to export the image as a FD: %s!\n",
3474  ff_vk_ret2str(ret));
3475  return AVERROR_EXTERNAL;
3476  }
3477 #endif
3478 
3479  ret = CHECK_CU(cu->cuImportExternalMemory(&dst_int->ext_mem[idx], &ext_desc));
3480  if (ret < 0) {
3481 #ifndef _WIN32
3482  close(ext_desc.handle.fd);
3483 #endif
3484  return AVERROR_EXTERNAL;
3485  }
3486 
3487  return 0;
3488 }
3489 
3490 static int export_sem_to_cuda(AVHWDeviceContext *ctx,
3491  AVHWDeviceContext *cuda_cu, CudaFunctions *cu,
3492  AVVkFrameInternal *dst_int, int idx,
3493  VkSemaphore sem)
3494 {
3495  VkResult ret;
3496  VulkanDevicePriv *p = ctx->hwctx;
3497  AVVulkanDeviceContext *hwctx = &p->p;
3498  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3499 
3500 #ifdef _WIN32
3501  VkSemaphoreGetWin32HandleInfoKHR sem_export = {
3502  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR,
3503  .semaphore = sem,
3504  .handleType = IsWindows8OrGreater()
3505  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
3506  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3507  };
3508  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3509  .type = 10 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32 */,
3510  };
3511 #else
3512  VkSemaphoreGetFdInfoKHR sem_export = {
3513  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
3514  .semaphore = sem,
3515  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
3516  };
3517  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3518  .type = 9 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD */,
3519  };
3520 #endif
3521 
3522 #ifdef _WIN32
3523  ret = vk->GetSemaphoreWin32HandleKHR(hwctx->act_dev, &sem_export,
3524  &ext_sem_desc.handle.win32.handle);
3525 #else
3526  ret = vk->GetSemaphoreFdKHR(hwctx->act_dev, &sem_export,
3527  &ext_sem_desc.handle.fd);
3528 #endif
3529  if (ret != VK_SUCCESS) {
3530  av_log(ctx, AV_LOG_ERROR, "Failed to export semaphore: %s\n",
3531  ff_vk_ret2str(ret));
3532  return AVERROR_EXTERNAL;
3533  }
3534 #ifdef _WIN32
3535  dst_int->ext_sem_handle[idx] = ext_sem_desc.handle.win32.handle;
3536 #endif
3537 
3538  ret = CHECK_CU(cu->cuImportExternalSemaphore(&dst_int->cu_sem[idx],
3539  &ext_sem_desc));
3540  if (ret < 0) {
3541 #ifndef _WIN32
3542  close(ext_sem_desc.handle.fd);
3543 #endif
3544  return AVERROR_EXTERNAL;
3545  }
3546 
3547  return 0;
3548 }
3549 
3550 static int vulkan_export_to_cuda(AVHWFramesContext *hwfc,
3551  AVBufferRef *cuda_hwfc,
3552  const AVFrame *frame)
3553 {
3554  int err;
3555  VkResult ret;
3556  AVVkFrame *dst_f;
3557  AVVkFrameInternal *dst_int;
3558  AVHWDeviceContext *ctx = hwfc->device_ctx;
3559  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3560  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3561  VulkanDevicePriv *p = ctx->hwctx;
3562  AVVulkanDeviceContext *hwctx = &p->p;
3563  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3564  int nb_images;
3565 
3566  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)cuda_hwfc->data;
3567  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3568  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3569  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3570  CudaFunctions *cu = cu_internal->cuda_dl;
3571  CUarray_format cufmt = desc->comp[0].depth > 8 ? CU_AD_FORMAT_UNSIGNED_INT16 :
3572  CU_AD_FORMAT_UNSIGNED_INT8;
3573 
3574  dst_f = (AVVkFrame *)frame->data[0];
3575  dst_int = dst_f->internal;
3576 
3577  if (!dst_int->cuda_fc_ref) {
3578  size_t offsets[3] = { 0 };
3579 
3580  dst_int->cuda_fc_ref = av_buffer_ref(cuda_hwfc);
3581  if (!dst_int->cuda_fc_ref)
3582  return AVERROR(ENOMEM);
3583 
3584  nb_images = ff_vk_count_images(dst_f);
3585  for (int i = 0; i < nb_images; i++) {
3586  err = export_mem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3587  dst_f->mem[i], dst_f->size[i]);
3588  if (err < 0)
3589  goto fail;
3590 
3591  err = export_sem_to_cuda(ctx, cuda_cu, cu, dst_int, i,
3592  dst_f->sem[i]);
3593  if (err < 0)
3594  goto fail;
3595  }
3596 
3597  if (nb_images != planes) {
3598  for (int i = 0; i < planes; i++) {
3599  VkImageSubresource subres = {
3600  .aspectMask = i == 2 ? VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT :
3601  i == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3602  VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT
3603  };
3604  VkSubresourceLayout layout = { 0 };
3605  vk->GetImageSubresourceLayout(hwctx->act_dev, dst_f->img[FFMIN(i, nb_images - 1)],
3606  &subres, &layout);
3607  offsets[i] = layout.offset;
3608  }
3609  }
3610 
3611  for (int i = 0; i < planes; i++) {
3612  CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC tex_desc = {
3613  .offset = offsets[i],
3614  .arrayDesc = {
3615  .Depth = 0,
3616  .Format = cufmt,
3617  .NumChannels = 1 + ((planes == 2) && i),
3618  .Flags = 0,
3619  },
3620  .numLevels = 1,
3621  };
3622  int p_w, p_h;
3623 
3624  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
3625  tex_desc.arrayDesc.Width = p_w;
3626  tex_desc.arrayDesc.Height = p_h;
3627 
3628  ret = CHECK_CU(cu->cuExternalMemoryGetMappedMipmappedArray(&dst_int->cu_mma[i],
3629  dst_int->ext_mem[FFMIN(i, nb_images - 1)],
3630  &tex_desc));
3631  if (ret < 0) {
3632  err = AVERROR_EXTERNAL;
3633  goto fail;
3634  }
3635 
3636  ret = CHECK_CU(cu->cuMipmappedArrayGetLevel(&dst_int->cu_array[i],
3637  dst_int->cu_mma[i], 0));
3638  if (ret < 0) {
3639  err = AVERROR_EXTERNAL;
3640  goto fail;
3641  }
3642 
3643  }
3644  }
3645 
3646  return 0;
3647 
3648 fail:
3649  vulkan_free_internal(dst_f);
3650  return err;
3651 }
3652 
3653 static int vulkan_transfer_data_from_cuda(AVHWFramesContext *hwfc,
3654  AVFrame *dst, const AVFrame *src)
3655 {
3656  int err;
3657  CUcontext dummy;
3658  AVVkFrame *dst_f;
3659  AVVkFrameInternal *dst_int;
3660  VulkanFramesPriv *fp = hwfc->hwctx;
3661  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3662  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3663 
3664  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3665  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3666  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3667  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3668  CudaFunctions *cu = cu_internal->cuda_dl;
3669  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
3670  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
3671 
3672  dst_f = (AVVkFrame *)dst->data[0];
3673 
3674  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
3675  if (err < 0)
3676  return err;
3677 
3678  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
3679  if (err < 0)
3680  return err;
3681 
3682  err = vulkan_export_to_cuda(hwfc, src->hw_frames_ctx, dst);
3683  if (err < 0) {
3684  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
3685  return err;
3686  }
3687 
3688  dst_int = dst_f->internal;
3689 
3690  for (int i = 0; i < planes; i++) {
3691  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
3692  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
3693  }
3694 
3695  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
3696  planes, cuda_dev->stream));
3697  if (err < 0)
3698  goto fail;
3699 
3700  for (int i = 0; i < planes; i++) {
3701  CUDA_MEMCPY2D cpy = {
3702  .srcMemoryType = CU_MEMORYTYPE_DEVICE,
3703  .srcDevice = (CUdeviceptr)src->data[i],
3704  .srcPitch = src->linesize[i],
3705  .srcY = 0,
3706 
3707  .dstMemoryType = CU_MEMORYTYPE_ARRAY,
3708  .dstArray = dst_int->cu_array[i],
3709  };
3710 
3711  int p_w, p_h;
3712  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
3713 
3714  cpy.WidthInBytes = p_w * desc->comp[i].step;
3715  cpy.Height = p_h;
3716 
3717  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
3718  if (err < 0)
3719  goto fail;
3720  }
3721 
3722  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
3723  planes, cuda_dev->stream));
3724  if (err < 0)
3725  goto fail;
3726 
3727  for (int i = 0; i < planes; i++)
3728  dst_f->sem_value[i]++;
3729 
3730  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
3731 
3732  av_log(hwfc, AV_LOG_VERBOSE, "Transferred CUDA image to Vulkan!\n");
3733 
3734  return err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
3735 
3736 fail:
3737  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
3738  vulkan_free_internal(dst_f);
3739  av_buffer_unref(&dst->buf[0]);
3740  return err;
3741 }
3742 #endif
3743 
3744 static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
3745  const AVFrame *src, int flags)
3746 {
3747  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3748 
3749  switch (src->format) {
3750 #if CONFIG_LIBDRM
3751 #if CONFIG_VAAPI
3752  case AV_PIX_FMT_VAAPI:
3753  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3754  return vulkan_map_from_vaapi(hwfc, dst, src, flags);
3755  else
3756  return AVERROR(ENOSYS);
3757 #endif
3758  case AV_PIX_FMT_DRM_PRIME:
3759  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3760  return vulkan_map_from_drm(hwfc, dst, src, flags);
3761  else
3762  return AVERROR(ENOSYS);
3763 #endif
3764  default:
3765  return AVERROR(ENOSYS);
3766  }
3767 }
3768 
3769 #if CONFIG_LIBDRM
3770 typedef struct VulkanDRMMapping {
3771  AVDRMFrameDescriptor drm_desc;
3772  AVVkFrame *source;
3773 } VulkanDRMMapping;
3774 
3775 static void vulkan_unmap_to_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
3776 {
3777  AVDRMFrameDescriptor *drm_desc = hwmap->priv;
3778 
3779  for (int i = 0; i < drm_desc->nb_objects; i++)
3780  close(drm_desc->objects[i].fd);
3781 
3782  av_free(drm_desc);
3783 }
3784 
3785 static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt)
3786 {
3787  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
3788  if (vulkan_drm_format_map[i].vk_format == vkfmt)
3789  return vulkan_drm_format_map[i].drm_fourcc;
3790  return DRM_FORMAT_INVALID;
3791 }
3792 
3793 static int vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3794  const AVFrame *src, int flags)
3795 {
3796  int err = 0;
3797  VkResult ret;
3798  AVVkFrame *f = (AVVkFrame *)src->data[0];
3799  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3800  AVVulkanDeviceContext *hwctx = &p->p;
3801  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3802  VulkanFramesPriv *fp = hwfc->hwctx;
3803  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3804  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
3805  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
3806  };
3807  VkSemaphoreWaitInfo wait_info = {
3808  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
3809  .flags = 0x0,
3810  .semaphoreCount = planes,
3811  };
3812 
3813  AVDRMFrameDescriptor *drm_desc = av_mallocz(sizeof(*drm_desc));
3814  if (!drm_desc)
3815  return AVERROR(ENOMEM);
3816 
3817  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_EXPORT);
3818  if (err < 0)
3819  goto end;
3820 
3821  /* Wait for the operation to finish so we can cleanly export it. */
3822  wait_info.pSemaphores = f->sem;
3823  wait_info.pValues = f->sem_value;
3824 
3825  vk->WaitSemaphores(hwctx->act_dev, &wait_info, UINT64_MAX);
3826 
3827  err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, &vulkan_unmap_to_drm, drm_desc);
3828  if (err < 0)
3829  goto end;
3830 
3831  ret = vk->GetImageDrmFormatModifierPropertiesEXT(hwctx->act_dev, f->img[0],
3832  &drm_mod);
3833  if (ret != VK_SUCCESS) {
3834  av_log(hwfc, AV_LOG_ERROR, "Failed to retrieve DRM format modifier!\n");
3835  err = AVERROR_EXTERNAL;
3836  goto end;
3837  }
3838 
3839  for (int i = 0; (i < planes) && (f->mem[i]); i++) {
3840  VkMemoryGetFdInfoKHR export_info = {
3841  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
3842  .memory = f->mem[i],
3843  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3844  };
3845 
3846  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
3847  &drm_desc->objects[i].fd);
3848  if (ret != VK_SUCCESS) {
3849  av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a FD!\n");
3850  err = AVERROR_EXTERNAL;
3851  goto end;
3852  }
3853 
3854  drm_desc->nb_objects++;
3855  drm_desc->objects[i].size = f->size[i];
3856  drm_desc->objects[i].format_modifier = drm_mod.drmFormatModifier;
3857  }
3858 
3859  drm_desc->nb_layers = planes;
3860  for (int i = 0; i < drm_desc->nb_layers; i++) {
3861  VkSubresourceLayout layout;
3862  VkImageSubresource sub = {
3863  .aspectMask = VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT,
3864  };
3865  VkFormat plane_vkfmt = av_vkfmt_from_pixfmt(hwfc->sw_format)[i];
3866 
3867  drm_desc->layers[i].format = vulkan_fmt_to_drm(plane_vkfmt);
3868  drm_desc->layers[i].nb_planes = 1;
3869 
3870  if (drm_desc->layers[i].format == DRM_FORMAT_INVALID) {
3871  av_log(hwfc, AV_LOG_ERROR, "Cannot map to DRM layer, unsupported!\n");
3872  err = AVERROR_PATCHWELCOME;
3873  goto end;
3874  }
3875 
3876  drm_desc->layers[i].planes[0].object_index = FFMIN(i, drm_desc->nb_objects - 1);
3877 
3878  if (f->tiling == VK_IMAGE_TILING_OPTIMAL)
3879  continue;
3880 
3881  vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, &layout);
3882  drm_desc->layers[i].planes[0].offset = layout.offset;
3883  drm_desc->layers[i].planes[0].pitch = layout.rowPitch;
3884  }
3885 
3886  dst->width = src->width;
3887  dst->height = src->height;
3888  dst->data[0] = (uint8_t *)drm_desc;
3889 
3890  av_log(hwfc, AV_LOG_VERBOSE, "Mapped AVVkFrame to a DRM object!\n");
3891 
3892  return 0;
3893 
3894 end:
3895  av_free(drm_desc);
3896  return err;
3897 }
3898 
3899 #if CONFIG_VAAPI
3900 static int vulkan_map_to_vaapi(AVHWFramesContext *hwfc, AVFrame *dst,
3901  const AVFrame *src, int flags)
3902 {
3903  int err;
3904  AVFrame *tmp = av_frame_alloc();
3905  if (!tmp)
3906  return AVERROR(ENOMEM);
3907 
3908  tmp->format = AV_PIX_FMT_DRM_PRIME;
3909 
3910  err = vulkan_map_to_drm(hwfc, tmp, src, flags);
3911  if (err < 0)
3912  goto fail;
3913 
3914  err = av_hwframe_map(dst, tmp, flags);
3915  if (err < 0)
3916  goto fail;
3917 
3918  err = ff_hwframe_map_replace(dst, src);
3919 
3920 fail:
3921  av_frame_free(&tmp);
3922  return err;
3923 }
3924 #endif
3925 #endif
3926 
3927 static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
3928  const AVFrame *src, int flags)
3929 {
3930  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3931 
3932  switch (dst->format) {
3933 #if CONFIG_LIBDRM
3934  case AV_PIX_FMT_DRM_PRIME:
3935  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3936  return vulkan_map_to_drm(hwfc, dst, src, flags);
3937  else
3938  return AVERROR(ENOSYS);
3939 #if CONFIG_VAAPI
3940  case AV_PIX_FMT_VAAPI:
3941  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3942  return vulkan_map_to_vaapi(hwfc, dst, src, flags);
3943  else
3944  return AVERROR(ENOSYS);
3945 #endif
3946 #endif
3947  default:
3948  break;
3949  }
3950  return AVERROR(ENOSYS);
3951 }
3952 
3953 static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf,
3954  AVFrame *swf, VkBufferImageCopy *region,
3955  int planes, int upload)
3956 {
3957  VkResult ret;
3958  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3959  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3960  AVVulkanDeviceContext *hwctx = &p->p;
3961 
3962  FFVkBuffer *vkbuf = (FFVkBuffer *)buf->data;
3963 
3964  const VkMappedMemoryRange flush_info = {
3965  .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
3966  .memory = vkbuf->mem,
3967  .size = VK_WHOLE_SIZE,
3968  };
3969 
3970  if (!upload && !(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
3971  ret = vk->InvalidateMappedMemoryRanges(hwctx->act_dev, 1,
3972  &flush_info);
3973  if (ret != VK_SUCCESS) {
3974  av_log(hwfc, AV_LOG_ERROR, "Failed to invalidate buffer data: %s\n",
3975  ff_vk_ret2str(ret));
3976  return AVERROR_EXTERNAL;
3977  }
3978  }
3979 
3980  if (upload) {
3981  for (int i = 0; i < planes; i++)
3982  av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset,
3983  region[i].bufferRowLength,
3984  swf->data[i],
3985  swf->linesize[i],
3986  swf->linesize[i],
3987  region[i].imageExtent.height);
3988  } else {
3989  for (int i = 0; i < planes; i++)
3990  av_image_copy_plane(swf->data[i],
3991  swf->linesize[i],
3992  vkbuf->mapped_mem + region[i].bufferOffset,
3993  region[i].bufferRowLength,
3994  swf->linesize[i],
3995  region[i].imageExtent.height);
3996  }
3997 
3998  if (upload && !(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
3999  ret = vk->FlushMappedMemoryRanges(hwctx->act_dev, 1,
4000  &flush_info);
4001  if (ret != VK_SUCCESS) {
4002  av_log(hwfc, AV_LOG_ERROR, "Failed to flush buffer data: %s\n",
4003  ff_vk_ret2str(ret));
4004  return AVERROR_EXTERNAL;
4005  }
4006  }
4007 
4008  return 0;
4009 }
4010 
4011 static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst,
4012  AVFrame *swf, VkBufferImageCopy *region, int upload)
4013 {
4014  int err;
4015  uint32_t p_w, p_h;
4016  VulkanFramesPriv *fp = hwfc->hwctx;
4017  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4018  const int planes = av_pix_fmt_count_planes(swf->format);
4019  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4020  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4021 
4022  size_t buf_offset = 0;
4023  for (int i = 0; i < planes; i++) {
4024  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4025 
4026  region[i] = (VkBufferImageCopy) {
4027  .bufferOffset = buf_offset,
4028  .bufferRowLength = FFALIGN(swf->linesize[i],
4029  p->props.properties.limits.optimalBufferCopyRowPitchAlignment),
4030  .bufferImageHeight = p_h,
4031  .imageSubresource.layerCount = 1,
4032  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4033  /* Rest of the fields adjusted/filled in later */
4034  };
4035 
4036  buf_offset += FFALIGN(p_h*region[i].bufferRowLength,
4037  p->props.properties.limits.optimalBufferCopyOffsetAlignment);
4038  }
4039 
4040  err = ff_vk_get_pooled_buffer(&p->vkctx, &fp->tmp, dst, buf_usage,
4041  NULL, buf_offset,
4042  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
4043  VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
4044  if (err < 0)
4045  return err;
4046 
4047  return 0;
4048 }
4049 
4050 static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs,
4051  AVFrame *swf, VkBufferImageCopy *region, int upload)
4052 {
4053  int err;
4054  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4055 
4056  int nb_src_bufs;
4057  const int planes = av_pix_fmt_count_planes(swf->format);
4058  VkBufferUsageFlags buf_usage = upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4059  VK_BUFFER_USAGE_TRANSFER_DST_BIT;
4060 
4061  /* We can't host map images with negative strides */
4062  for (int i = 0; i < planes; i++)
4063  if (swf->linesize[i] < 0)
4064  return AVERROR(EINVAL);
4065 
4066  /* Count the number of buffers in the software frame */
4067  nb_src_bufs = 0;
4068  while (swf->buf[nb_src_bufs])
4069  nb_src_bufs++;
4070 
4071  /* Single buffer contains all planes */
4072  if (nb_src_bufs == 1) {
4073  err = ff_vk_host_map_buffer(&p->vkctx, &dst[0],
4074  swf->data[0], swf->buf[0],
4075  buf_usage);
4076  if (err < 0)
4077  return err;
4078  (*nb_bufs)++;
4079 
4080  for (int i = 0; i < planes; i++)
4081  region[i].bufferOffset = ((FFVkBuffer *)dst[0]->data)->virtual_offset +
4082  swf->data[i] - swf->data[0];
4083  } else if (nb_src_bufs == planes) { /* One buffer per plane */
4084  for (int i = 0; i < planes; i++) {
4085  err = ff_vk_host_map_buffer(&p->vkctx, &dst[i],
4086  swf->data[i], swf->buf[i],
4087  buf_usage);
4088  if (err < 0)
4089  goto fail;
4090  (*nb_bufs)++;
4091 
4092  region[i].bufferOffset = ((FFVkBuffer *)dst[i]->data)->virtual_offset;
4093  }
4094  } else {
4095  /* Weird layout (3 planes, 2 buffers), patch welcome, fallback to copy */
4096  return AVERROR_PATCHWELCOME;
4097  }
4098 
4099  return 0;
4100 
4101 fail:
4102  for (int i = 0; i < (*nb_bufs); i++)
4103  av_buffer_unref(&dst[i]);
4104  return err;
4105 }
4106 
4107 static int vulkan_transfer_frame(AVHWFramesContext *hwfc,
4108  AVFrame *swf, AVFrame *hwf,
4109  int upload)
4110 {
4111  int err;
4112  VulkanFramesPriv *fp = hwfc->hwctx;
4113  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4114  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4115 
4116  int host_mapped = 0;
4117 
4118  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4119  VkBufferImageCopy region[AV_NUM_DATA_POINTERS]; // always one per plane
4120 
4121  const int planes = av_pix_fmt_count_planes(swf->format);
4122  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format);
4123  const int nb_images = ff_vk_count_images(hwf_vk);
4124 
4125  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
4126  int nb_img_bar = 0;
4127 
4128  AVBufferRef *bufs[AV_NUM_DATA_POINTERS];
4129  int nb_bufs = 0;
4130 
4131  VkCommandBuffer cmd_buf;
4132  FFVkExecContext *exec;
4133 
4134  /* Sanity checking */
4135  if ((swf->format != AV_PIX_FMT_NONE && !av_vkfmt_from_pixfmt(swf->format))) {
4136  av_log(hwfc, AV_LOG_ERROR, "Unsupported software frame pixel format!\n");
4137  return AVERROR(EINVAL);
4138  }
4139 
4140  if (swf->width > hwfc->width || swf->height > hwfc->height)
4141  return AVERROR(EINVAL);
4142 
4143  for (int i = 0; i < av_pix_fmt_count_planes(swf->format); i++) {
4144  uint32_t p_w, p_h;
4145  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4146 
4147  /* Buffer region for this plane */
4148  region[i] = (VkBufferImageCopy) {
4149  .bufferOffset = 0,
4150  .bufferRowLength = swf->linesize[i],
4151  .bufferImageHeight = p_h,
4152  .imageSubresource.layerCount = 1,
4153  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4154  /* Rest of the fields adjusted/filled in later */
4155  };
4156  }
4157 
4158  /* Setup buffers first */
4159  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY) {
4160  err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload);
4161  if (err >= 0)
4162  host_mapped = 1;
4163  }
4164 
4165  if (!host_mapped) {
4166  err = get_plane_buf(hwfc, &bufs[0], swf, region, upload);
4167  if (err < 0)
4168  goto end;
4169  nb_bufs = 1;
4170 
4171  if (upload) {
4172  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 1);
4173  if (err < 0)
4174  goto end;
4175  }
4176  }
4177 
4178  exec = ff_vk_exec_get(&p->vkctx, &fp->upload_exec);
4179  cmd_buf = exec->buf;
4180 
4181  ff_vk_exec_start(&p->vkctx, exec);
4182 
4183  /* Prep destination Vulkan frame */
4184  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, hwf,
4185  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4186  VK_PIPELINE_STAGE_2_TRANSFER_BIT);
4187  if (err < 0)
4188  goto end;
4189 
4190  /* No need to declare buf deps for synchronous transfers (downloads) */
4191  if (upload) {
4192  /* Add the software frame backing the buffers if we're host mapping */
4193  if (host_mapped) {
4194  err = ff_vk_exec_add_dep_sw_frame(&p->vkctx, exec, swf);
4195  if (err < 0) {
4196  ff_vk_exec_discard_deps(&p->vkctx, exec);
4197  goto end;
4198  }
4199  }
4200 
4201  /* Add the buffers as a dependency */
4202  err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, nb_bufs, 1);
4203  if (err < 0) {
4204  ff_vk_exec_discard_deps(&p->vkctx, exec);
4205  goto end;
4206  }
4207  }
4208 
4209  ff_vk_frame_barrier(&p->vkctx, exec, hwf, img_bar, &nb_img_bar,
4210  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4211  VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR,
4212  upload ? VK_ACCESS_TRANSFER_WRITE_BIT :
4213  VK_ACCESS_TRANSFER_READ_BIT,
4214  upload ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL :
4215  VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
4216  VK_QUEUE_FAMILY_IGNORED);
4217 
4218  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
4219  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
4220  .pImageMemoryBarriers = img_bar,
4221  .imageMemoryBarrierCount = nb_img_bar,
4222  });
4223 
4224  for (int i = 0; i < planes; i++) {
4225  int buf_idx = FFMIN(i, (nb_bufs - 1));
4226  int img_idx = FFMIN(i, (nb_images - 1));
4227  FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[buf_idx]->data;
4228 
4229  uint32_t orig_stride = region[i].bufferRowLength;
4230  region[i].bufferRowLength /= desc->comp[i].step;
4231  region[i].imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4232 
4233  if (upload)
4234  vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf,
4235  hwf_vk->img[img_idx],
4236  img_bar[img_idx].newLayout,
4237  1, &region[i]);
4238  else
4239  vk->CmdCopyImageToBuffer(cmd_buf, hwf_vk->img[img_idx],
4240  img_bar[img_idx].newLayout,
4241  vkbuf->buf,
4242  1, &region[i]);
4243 
4244  region[i].bufferRowLength = orig_stride;
4245  }
4246 
4247  err = ff_vk_exec_submit(&p->vkctx, exec);
4248  if (err < 0) {
4249  ff_vk_exec_discard_deps(&p->vkctx, exec);
4250  } else if (!upload) {
4251  ff_vk_exec_wait(&p->vkctx, exec);
4252  if (!host_mapped)
4253  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 0);
4254  }
4255 
4256 end:
4257  for (int i = 0; i < nb_bufs; i++)
4258  av_buffer_unref(&bufs[i]);
4259 
4260  return err;
4261 }
4262 
4263 static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst,
4264  const AVFrame *src)
4265 {
4266  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4267 
4268  switch (src->format) {
4269 #if CONFIG_CUDA
4270  case AV_PIX_FMT_CUDA:
4271 #ifdef _WIN32
4272  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4273  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4274 #else
4275  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4276  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4277 #endif
4278  return vulkan_transfer_data_from_cuda(hwfc, dst, src);
4279 #endif
4280  default:
4281  if (src->hw_frames_ctx)
4282  return AVERROR(ENOSYS);
4283  else
4284  return vulkan_transfer_frame(hwfc, (AVFrame *)src, dst, 1);
4285  }
4286 }
4287 
4288 #if CONFIG_CUDA
4289 static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst,
4290  const AVFrame *src)
4291 {
4292  int err;
4293  CUcontext dummy;
4294  AVVkFrame *dst_f;
4295  AVVkFrameInternal *dst_int;
4296  VulkanFramesPriv *fp = hwfc->hwctx;
4297  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
4298  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
4299  int nb_images;
4300 
4301  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)dst->hw_frames_ctx->data;
4302  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
4303  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
4304  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
4305  CudaFunctions *cu = cu_internal->cuda_dl;
4306  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
4307  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
4308 
4309  dst_f = (AVVkFrame *)src->data[0];
4310  nb_images = ff_vk_count_images(dst_f);
4311 
4312  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
4313  if (err < 0)
4314  return err;
4315 
4316  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
4317  if (err < 0)
4318  return err;
4319 
4320  err = vulkan_export_to_cuda(hwfc, dst->hw_frames_ctx, src);
4321  if (err < 0) {
4322  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4323  return err;
4324  }
4325 
4326  dst_int = dst_f->internal;
4327 
4328  for (int i = 0; i < planes; i++) {
4329  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
4330  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
4331  }
4332 
4333  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
4334  nb_images, cuda_dev->stream));
4335  if (err < 0)
4336  goto fail;
4337 
4338  for (int i = 0; i < planes; i++) {
4339  CUDA_MEMCPY2D cpy = {
4340  .dstMemoryType = CU_MEMORYTYPE_DEVICE,
4341  .dstDevice = (CUdeviceptr)dst->data[i],
4342  .dstPitch = dst->linesize[i],
4343  .dstY = 0,
4344 
4345  .srcMemoryType = CU_MEMORYTYPE_ARRAY,
4346  .srcArray = dst_int->cu_array[i],
4347  };
4348 
4349  int w, h;
4350  get_plane_wh(&w, &h, hwfc->sw_format, hwfc->width, hwfc->height, i);
4351 
4352  cpy.WidthInBytes = w * desc->comp[i].step;
4353  cpy.Height = h;
4354 
4355  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
4356  if (err < 0)
4357  goto fail;
4358  }
4359 
4360  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
4361  nb_images, cuda_dev->stream));
4362  if (err < 0)
4363  goto fail;
4364 
4365  for (int i = 0; i < planes; i++)
4366  dst_f->sem_value[i]++;
4367 
4368  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4369 
4370  av_log(hwfc, AV_LOG_VERBOSE, "Transferred Vulkan image to CUDA!\n");
4371 
4372  return prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
4373 
4374 fail:
4375  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4376  vulkan_free_internal(dst_f);
4377  av_buffer_unref(&dst->buf[0]);
4378  return err;
4379 }
4380 #endif
4381 
4382 static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst,
4383  const AVFrame *src)
4384 {
4385  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4386 
4387  switch (dst->format) {
4388 #if CONFIG_CUDA
4389  case AV_PIX_FMT_CUDA:
4390 #ifdef _WIN32
4391  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4392  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4393 #else
4394  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4395  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4396 #endif
4397  return vulkan_transfer_data_to_cuda(hwfc, dst, src);
4398 #endif
4399  default:
4400  if (dst->hw_frames_ctx)
4401  return AVERROR(ENOSYS);
4402  else
4403  return vulkan_transfer_frame(hwfc, dst, (AVFrame *)src, 0);
4404  }
4405 }
4406 
4407 static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc,
4408  AVHWFramesContext *src_fc, int flags)
4409 {
4410  return vulkan_frames_init(dst_fc);
4411 }
4412 
4413 AVVkFrame *av_vk_frame_alloc(void)
4414 {
4415  int err;
4416  AVVkFrame *f = av_mallocz(sizeof(AVVkFrame));
4417  if (!f)
4418  return NULL;
4419 
4420  f->internal = av_mallocz(sizeof(*f->internal));
4421  if (!f->internal) {
4422  av_free(f);
4423  return NULL;
4424  }
4425 
4426  err = pthread_mutex_init(&f->internal->update_mutex, NULL);
4427  if (err != 0) {
4428  av_free(f->internal);
4429  av_free(f);
4430  return NULL;
4431  }
4432 
4433  return f;
4434 }
4435 
4436 const HWContextType ff_hwcontext_type_vulkan = {
4437  .type = AV_HWDEVICE_TYPE_VULKAN,
4438  .name = "Vulkan",
4439 
4440  .device_hwctx_size = sizeof(VulkanDevicePriv),
4441  .frames_hwctx_size = sizeof(VulkanFramesPriv),
4442 
4443  .device_init = &vulkan_device_init,
4444  .device_uninit = &vulkan_device_uninit,
4445  .device_create = &vulkan_device_create,
4446  .device_derive = &vulkan_device_derive,
4447 
4448  .frames_get_constraints = &vulkan_frames_get_constraints,
4449  .frames_init = vulkan_frames_init,
4450  .frames_get_buffer = vulkan_get_buffer,
4451  .frames_uninit = vulkan_frames_uninit,
4452 
4453  .transfer_get_formats = vulkan_transfer_get_formats,
4454  .transfer_data_to = vulkan_transfer_data_to,
4455  .transfer_data_from = vulkan_transfer_data_from,
4456 
4457  .map_to = vulkan_map_to,
4458  .map_from = vulkan_map_from,
4459  .frames_derive_to = &vulkan_frames_derive_to,
4460 
4461  .pix_fmts = (const enum AVPixelFormat []) {
4462  AV_PIX_FMT_VULKAN,
4463  AV_PIX_FMT_NONE
4464  },
4465 };
flags
const SwsFlags flags[]
Definition: swscale.c:61
vulkan_loader.h
formats
formats
Definition: signature.h:47
load_libvulkan
static int load_libvulkan(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:558
pthread_mutex_t
_fmutex pthread_mutex_t
Definition: os2threads.h:53
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:86
FFHWFramesContext::pool_internal
AVBufferPool * pool_internal
Definition: hwcontext_internal.h:101
vulkan_device_init
static int vulkan_device_init(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1729
AV_PIX_FMT_GBRAP16
#define AV_PIX_FMT_GBRAP16
Definition: pixfmt.h:551
FF_ENABLE_DEPRECATION_WARNINGS
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:73
ff_vk_load_props
int ff_vk_load_props(FFVulkanContext *s)
Loads props/mprops/driver_props.
Definition: vulkan.c:106
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:215
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:79
AV_PIX_FMT_CUDA
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition: pixfmt.h:260
VulkanDeviceFeatures::vulkan_1_2
VkPhysicalDeviceVulkan12Features vulkan_1_2
Definition: hwcontext_vulkan.c:79
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
VulkanDevicePriv::libvulkan
void * libvulkan
Definition: hwcontext_vulkan.c:107
VulkanOptExtension::name
const char * name
Definition: hwcontext_vulkan.c:597
FFVkFormatEntry::nb_images
int nb_images
Definition: hwcontext_vulkan.c:316
host_map_frame
static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4050
AVCUDADeviceContextInternal
Definition: hwcontext_cuda_internal.h:31
AV_PIX_FMT_GRAY32
#define AV_PIX_FMT_GRAY32
Definition: pixfmt.h:512
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:202
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
hwcontext_cuda_internal.h
FFVulkanExtensions
uint64_t FFVulkanExtensions
Definition: vulkan_functions.h:29
AVBufferPool
The buffer pool.
Definition: buffer_internal.h:88
SET_OLD_QF
#define SET_OLD_QF(field, nb_field, type)
vulkan_transfer_data_to
static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4263
FF_VK_EXT_EXTERNAL_WIN32_MEMORY
#define FF_VK_EXT_EXTERNAL_WIN32_MEMORY
Definition: vulkan_functions.h:39
FF_VK_EXT_VIDEO_QUEUE
#define FF_VK_EXT_VIDEO_QUEUE
Definition: vulkan_functions.h:52
thread.h
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3341
CHECK_QUEUE
#define CHECK_QUEUE(type, required, fidx, ctx_qf, qc)
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
pthread_mutex_init
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, AVVulkanDeviceQueueFamily *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:296
FF_VK_EXT_PORTABILITY_SUBSET
#define FF_VK_EXT_PORTABILITY_SUBSET
Definition: vulkan_functions.h:65
vulkan_frames_get_constraints
static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx, const void *hwconfig, AVHWFramesConstraints *constraints)
Definition: hwcontext_vulkan.c:2084
FF_VK_EXT_VIDEO_MAINTENANCE_2
#define FF_VK_EXT_VIDEO_MAINTENANCE_2
Definition: vulkan_functions.h:54
AV_HWFRAME_MAP_READ
@ AV_HWFRAME_MAP_READ
The mapping must be readable.
Definition: hwcontext.h:513
AVVkFrameInternal::update_mutex
pthread_mutex_t update_mutex
Definition: hwcontext_vulkan.c:167
FF_VULKAN_DEBUG_PROFILE
@ FF_VULKAN_DEBUG_PROFILE
Definition: hwcontext_vulkan.c:710
av_unused
#define av_unused
Definition: attributes.h:131
vulkan_frames_derive_to
static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc, AVHWFramesContext *src_fc, int flags)
Definition: hwcontext_vulkan.c:4407
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:63
PICK_QF
#define PICK_QF(type, vid_op)
FF_VK_EXT_VIDEO_DECODE_H265
#define FF_VK_EXT_VIDEO_DECODE_H265
Definition: vulkan_functions.h:58
mode
Definition: swscale.c:56
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:410
tmp
static uint8_t tmp[11]
Definition: aes_ctr.c:28
pixdesc.h
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to a vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:69
optional_device_exts
static const VulkanOptExtension optional_device_exts[]
Definition: hwcontext_vulkan.c:608
AVFrame::width
int width
Definition: frame.h:482
w
uint8_t w
Definition: llviddspenc.c:38
AV_PIX_FMT_Y216
#define AV_PIX_FMT_Y216
Definition: pixfmt.h:591
create_frame
static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame, VkImageTiling tiling, VkImageUsageFlagBits usage, VkImageCreateFlags flags, int nb_layers, void *create_pnext)
Definition: hwcontext_vulkan.c:2449
AVDRMFrameDescriptor::nb_layers
int nb_layers
Number of layers in the frame.
Definition: hwcontext_drm.h:145
AV_PIX_FMT_DRM_PRIME
@ AV_PIX_FMT_DRM_PRIME
DRM-managed buffers exposed through PRIME buffer sharing.
Definition: pixfmt.h:351
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:238
ff_vk_find_struct
static const void * ff_vk_find_struct(const void *chain, VkStructureType stype)
Definition: vulkan.h:331
pthread_mutex_lock
static av_always_inline int pthread_mutex_lock(pthread_mutex_t *mutex)
Definition: os2threads.h:119
av_hwframe_map
int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
Map a hardware frame.
Definition: hwcontext.c:782
COPY_VAL
#define COPY_VAL(VAL)
nb_vk_formats_list
static const int nb_vk_formats_list
Definition: hwcontext_vulkan.c:408
data
const char data[16]
Definition: mxf.c:149
AVVulkanDeviceContext::queue_family_encode_index
attribute_deprecated int queue_family_encode_index
Queue family index for video encode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:152
AV_PIX_FMT_RGBA128
#define AV_PIX_FMT_RGBA128
Definition: pixfmt.h:613
try_export_flags
static void try_export_flags(AVHWFramesContext *hwfc, VkExternalMemoryHandleTypeFlags *comp_handle_types, VkExternalMemoryHandleTypeFlagBits *iexp, VkExternalMemoryHandleTypeFlagBits exp)
Definition: hwcontext_vulkan.c:2559
AV_PIX_FMT_YUV420P10
#define AV_PIX_FMT_YUV420P10
Definition: pixfmt.h:528
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:74
AV_PIX_FMT_XV30
#define AV_PIX_FMT_XV30
Definition: pixfmt.h:592
AVVulkanFramesContext::lock_frame
void(* lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Locks a frame, preventing other threads from changing frame properties.
Definition: hwcontext_vulkan.h:283
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:225
AVVAAPIDeviceContext::display
VADisplay display
The VADisplay handle, to be filled by the user.
Definition: hwcontext_vaapi.h:72
FF_VULKAN_DEBUG_PRACTICES
@ FF_VULKAN_DEBUG_PRACTICES
Definition: hwcontext_vulkan.c:708
vulkan_map_from
static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:3927
AV_PIX_FMT_BGR24
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:76
AV_PIX_FMT_BGRA
@ AV_PIX_FMT_BGRA
packed BGRA 8:8:8:8, 32bpp, BGRABGRA...
Definition: pixfmt.h:102
FFVkFormatEntry::vkf
VkFormat vkf
Definition: hwcontext_vulkan.c:312
ff_vk_exec_get
FFVkExecContext * ff_vk_exec_get(FFVulkanContext *s, FFVkExecPool *pool)
Retrieve an execution pool.
Definition: vulkan.c:477
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2893
AVDictionary
Definition: dict.c:32
ff_hwframe_map_create
int ff_hwframe_map_create(AVBufferRef *hwframe_ref, AVFrame *dst, const AVFrame *src, void(*unmap)(AVHWFramesContext *ctx, HWMapDescriptor *hwmap), void *priv)
Definition: hwcontext.c:730
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
HWMapDescriptor::priv
void * priv
Hardware-specific private data associated with the mapping.
Definition: hwcontext_internal.h:139
FF_VK_EXT_COOP_MATRIX
#define FF_VK_EXT_COOP_MATRIX
Definition: vulkan_functions.h:45
av_popcount
#define av_popcount
Definition: common.h:154
AVDRMFrameDescriptor
DRM frame descriptor.
Definition: hwcontext_drm.h:133
AVHWFramesConstraints::valid_hw_formats
enum AVPixelFormat * valid_hw_formats
A list of possible values for format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:447
AVERROR_UNKNOWN
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:73
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:218
AVFrame::buf
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:587
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
VulkanDeviceSelection::uuid
uint8_t uuid[VK_UUID_SIZE]
Definition: hwcontext_vulkan.c:1168
ff_vk_exec_add_dep_frame
int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage)
Definition: vulkan.c:709
FF_VULKAN_DEBUG_PRINTF
@ FF_VULKAN_DEBUG_PRINTF
Definition: hwcontext_vulkan.c:706
AV_PIX_FMT_P212
#define AV_PIX_FMT_P212
Definition: pixfmt.h:601
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:431
FFVkBuffer::buf
VkBuffer buf
Definition: vulkan.h:89
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:374
alloc_mem
static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req, VkMemoryPropertyFlagBits req_flags, const void *alloc_extension, VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
Definition: hwcontext_vulkan.c:2130
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
VulkanDevicePriv::compute_qf
AVVulkanDeviceQueueFamily * compute_qf
Definition: hwcontext_vulkan.c:110
AVHWFramesConstraints
This struct describes the constraints on hardware frames attached to a given device with a hardware-s...
Definition: hwcontext.h:442
AV_HWDEVICE_TYPE_CUDA
@ AV_HWDEVICE_TYPE_CUDA
Definition: hwcontext.h:30
AVDRMDeviceContext::fd
int fd
File descriptor of DRM device.
Definition: hwcontext_drm.h:166
PREP_MODE_DECODING_DPB
@ PREP_MODE_DECODING_DPB
Definition: hwcontext_vulkan.c:2338
FF_VK_EXT_EXTERNAL_FD_SEM
#define FF_VK_EXT_EXTERNAL_FD_SEM
Definition: vulkan_functions.h:35
VulkanDeviceFeatures::device
VkPhysicalDeviceFeatures2 device
Definition: hwcontext_vulkan.c:76
VulkanDeviceFeatures::video_maintenance_1
VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1
Definition: hwcontext_vulkan.c:83
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3381
ASPECT_3PLANE
#define ASPECT_3PLANE
Definition: hwcontext_vulkan.c:309
VulkanDevicePriv::hprops
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops
Definition: hwcontext_vulkan.c:116
vulkan_device_derive
static int vulkan_device_derive(AVHWDeviceContext *ctx, AVHWDeviceContext *src_ctx, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:1981
VulkanOptExtension::flag
FFVulkanExtensions flag
Definition: hwcontext_vulkan.c:598
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AV_PIX_FMT_GBRP14
#define AV_PIX_FMT_GBRP14
Definition: pixfmt.h:546
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:302
AV_PIX_FMT_GBRAP
@ AV_PIX_FMT_GBRAP
planar GBRA 4:4:4:4 32bpp
Definition: pixfmt.h:212
VulkanDeviceSelection::drm_minor
uint32_t drm_minor
Definition: hwcontext_vulkan.c:1171
lock_frame
static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2691
fail
#define fail()
Definition: checkasm.h:193
AVDRMLayerDescriptor::nb_planes
int nb_planes
Number of planes in the layer.
Definition: hwcontext_drm.h:106
ff_vk_exec_add_dep_bool_sem
int ff_vk_exec_add_dep_bool_sem(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore *sem, int nb, VkPipelineStageFlagBits2 stage, int wait)
Definition: vulkan.c:643
AV_PIX_FMT_GBRP10
#define AV_PIX_FMT_GBRP10
Definition: pixfmt.h:544
AVVulkanFramesContext::flags
AVVkFrameFlags flags
A combination of AVVkFrameFlags.
Definition: hwcontext_vulkan.h:254
VulkanDevicePriv
Definition: hwcontext_vulkan.c:100
AVVulkanDeviceContext::nb_tx_queues
attribute_deprecated int nb_tx_queues
Definition: hwcontext_vulkan.h:135
AVDRMLayerDescriptor::planes
AVDRMPlaneDescriptor planes[AV_DRM_MAX_PLANES]
Array of planes in this layer.
Definition: hwcontext_drm.h:110
dummy
int dummy
Definition: motion.c:66
AVVkFrame::mem
VkDeviceMemory mem[AV_NUM_DATA_POINTERS]
Memory backing the images.
Definition: hwcontext_vulkan.h:315
device_features_copy_needed
static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
Definition: hwcontext_vulkan.c:239
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:208
av_buffer_pool_init2
AVBufferPool * av_buffer_pool_init2(size_t size, void *opaque, AVBufferRef *(*alloc)(void *opaque, size_t size), void(*pool_free)(void *opaque))
Allocate and initialize a buffer pool with a more complex allocator.
Definition: buffer.c:259
AVHWFramesConstraints::min_width
int min_width
The minimum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:460
lock_queue
static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1717
AVCUDADeviceContextInternal::cuda_device
CUdevice cuda_device
Definition: hwcontext_cuda_internal.h:34
VulkanDevicePriv::vkctx
FFVulkanContext vkctx
Definition: hwcontext_vulkan.c:109
AV_PIX_FMT_XV48
#define AV_PIX_FMT_XV48
Definition: pixfmt.h:594
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
FF_VK_EXT_VIDEO_ENCODE_H265
#define FF_VK_EXT_VIDEO_ENCODE_H265
Definition: vulkan_functions.h:63
ff_vk_host_map_buffer
int ff_vk_host_map_buffer(FFVulkanContext *s, AVBufferRef **dst, uint8_t *src_data, const AVBufferRef *src_buf, VkBufferUsageFlags usage)
Maps a system RAM buffer into a Vulkan buffer.
Definition: vulkan.c:1281
ff_vk_ret2str
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:35
AV_PIX_FMT_GRAY16
#define AV_PIX_FMT_GRAY16
Definition: pixfmt.h:511
VulkanDeviceFeatures::vulkan_1_3
VkPhysicalDeviceVulkan13Features vulkan_1_3
Definition: hwcontext_vulkan.c:80
FF_VK_EXT_EXTERNAL_WIN32_SEM
#define FF_VK_EXT_EXTERNAL_WIN32_SEM
Definition: vulkan_functions.h:40
AVVulkanDeviceContext::queue_family_decode_index
attribute_deprecated int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:162
VulkanDevicePriv::props
VkPhysicalDeviceProperties2 props
Definition: hwcontext_vulkan.c:114
ff_vk_aspect_flag
VkImageAspectFlags ff_vk_aspect_flag(AVFrame *f, int p)
Get the aspect flag for a plane from an image.
Definition: vulkan.c:1431
vk_find_format_entry
static const struct FFVkFormatEntry * vk_find_format_entry(enum AVPixelFormat p)
Definition: hwcontext_vulkan.c:418
AVDRMPlaneDescriptor::offset
ptrdiff_t offset
Offset within that object of this plane.
Definition: hwcontext_drm.h:83
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:61
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:51
AV_PIX_FMT_YUV444P10
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:531
vk_dbg_callback
static VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT messageType, const VkDebugUtilsMessengerCallbackDataEXT *data, void *priv)
Definition: hwcontext_vulkan.c:645
AV_PIX_FMT_Y210
#define AV_PIX_FMT_Y210
Definition: pixfmt.h:589
AVVulkanDeviceQueueFamily::num
int num
Definition: hwcontext_vulkan.h:37
HWContextType::type
enum AVHWDeviceType type
Definition: hwcontext_internal.h:30
ffhwframesctx
static FFHWFramesContext * ffhwframesctx(AVHWFramesContext *ctx)
Definition: hwcontext_internal.h:115
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
ff_vk_link_struct
static void ff_vk_link_struct(void *chain, const void *in)
Definition: vulkan.h:344
check_layers
static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:879
AV_PIX_FMT_YUV422P16
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:540
AVVulkanDeviceContext::nb_encode_queues
attribute_deprecated int nb_encode_queues
Definition: hwcontext_vulkan.h:154
AVHWFramesContext::height
int height
Definition: hwcontext.h:218
AVHWFramesConstraints::valid_sw_formats
enum AVPixelFormat * valid_sw_formats
A list of possible values for sw_format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:454
vulkan_map_to
static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:3744
av_dict_get
AVDictionaryEntry * av_dict_get(const AVDictionary *m, const char *key, const AVDictionaryEntry *prev, int flags)
Get a dictionary entry with matching key.
Definition: dict.c:60
av_buffer_pool_get
AVBufferRef * av_buffer_pool_get(AVBufferPool *pool)
Allocate a new AVBuffer, reusing an old buffer from the pool when available.
Definition: buffer.c:390
AV_PIX_FMT_GBRAP10
#define AV_PIX_FMT_GBRAP10
Definition: pixfmt.h:548
AVHWFramesContext::pool
AVBufferPool * pool
A pool from which the frames are allocated by av_hwframe_get_buffer().
Definition: hwcontext.h:179
VulkanDeviceSelection::pci_device
uint32_t pci_device
Definition: hwcontext_vulkan.c:1174
AV_PIX_FMT_GBRAP14
#define AV_PIX_FMT_GBRAP14
Definition: pixfmt.h:550
AV_PIX_FMT_GBRAP12
#define AV_PIX_FMT_GBRAP12
Definition: pixfmt.h:549
AV_PIX_FMT_RGB96
#define AV_PIX_FMT_RGB96
Definition: pixfmt.h:612
pthread_mutex_unlock
static av_always_inline int pthread_mutex_unlock(pthread_mutex_t *mutex)
Definition: os2threads.h:126
AV_PIX_FMT_YUV444P16
#define AV_PIX_FMT_YUV444P16
Definition: pixfmt.h:541
offsets
static const int offsets[]
Definition: hevc_pel.c:34
AV_CEIL_RSHIFT
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:60
format
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample format(the sample packing is implied by the sample format) and sample rate. The lists are not just lists
ff_vk_load_functions
static int ff_vk_load_functions(AVHWDeviceContext *ctx, FFVulkanFunctions *vk, uint64_t extensions_mask, int has_inst, int has_dev)
Function loader.
Definition: vulkan_loader.h:108
VulkanDevicePriv::ext_sem_props_opaque
VkExternalSemaphoreProperties ext_sem_props_opaque
Definition: hwcontext_vulkan.c:119
prepare_frame
static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2342
vulkan_transfer_frame
static int vulkan_transfer_frame(AVHWFramesContext *hwfc, AVFrame *swf, AVFrame *hwf, int upload)
Definition: hwcontext_vulkan.c:4107
FF_VK_EXT_DEVICE_DRM
#define FF_VK_EXT_DEVICE_DRM
Definition: vulkan_functions.h:43
ff_vk_exec_wait
void ff_vk_exec_wait(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:482
av_strtok
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
Definition: avstring.c:178
ASPECT_2PLANE
#define ASPECT_2PLANE
Definition: hwcontext_vulkan.c:308
vulkan_device_uninit
static void vulkan_device_uninit(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1611
fc
#define fc(width, name, range_min, range_max)
Definition: cbs_av1.c:493
AVVulkanFramesContext::unlock_frame
void(* unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Similar to lock_frame(), unlocks a frame.
Definition: hwcontext_vulkan.h:288
AVVulkanFramesContext::img_flags
VkImageCreateFlags img_flags
Flags to set during image creation.
Definition: hwcontext_vulkan.h:260
AV_PIX_FMT_GBRAP32
#define AV_PIX_FMT_GBRAP32
Definition: pixfmt.h:552
VulkanDeviceFeatures::optical_flow
VkPhysicalDeviceOpticalFlowFeaturesNV optical_flow
Definition: hwcontext_vulkan.c:93
vulkan_frames_uninit
static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2701
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:230
AV_PIX_FMT_YUV420P16
#define AV_PIX_FMT_YUV420P16
Definition: pixfmt.h:539
ctx
AVFormatContext * ctx
Definition: movenc.c:49
AVDRMObjectDescriptor::fd
int fd
DRM PRIME fd for the object.
Definition: hwcontext_drm.h:52
AV_PIX_FMT_GRAY14
#define AV_PIX_FMT_GRAY14
Definition: pixfmt.h:510
ff_vk_exec_add_dep_buf
int ff_vk_exec_add_dep_buf(FFVulkanContext *s, FFVkExecContext *e, AVBufferRef **deps, int nb_deps, int ref)
Execution dependency management.
Definition: vulkan.c:549
VulkanDeviceSelection::index
int index
Definition: hwcontext_vulkan.c:1176
AV_PIX_FMT_RGBF32
#define AV_PIX_FMT_RGBF32
Definition: pixfmt.h:609
VulkanFramesPriv::p
AVVulkanFramesContext p
The public AVVulkanFramesContext.
Definition: hwcontext_vulkan.c:150
vulkan_transfer_get_formats
static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats)
Definition: hwcontext_vulkan.c:2892
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:73
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:233
FFVulkanDebugMode
FFVulkanDebugMode
Definition: hwcontext_vulkan.c:701
AV_PIX_FMT_GRAYF32
#define AV_PIX_FMT_GRAYF32
Definition: pixfmt.h:565
LIBAVUTIL_VERSION_MINOR
#define LIBAVUTIL_VERSION_MINOR
Definition: version.h:82
AV_PIX_FMT_RGBA
@ AV_PIX_FMT_RGBA
packed RGBA 8:8:8:8, 32bpp, RGBARGBA...
Definition: pixfmt.h:100
FFVkFormatEntry::nb_images_fallback
int nb_images_fallback
Definition: hwcontext_vulkan.c:317
vulkan_frame_free_cb
static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
Definition: hwcontext_vulkan.c:2258
AV_PIX_FMT_GRAY10
#define AV_PIX_FMT_GRAY10
Definition: pixfmt.h:508
if
if(ret)
Definition: filter_design.txt:179
av_vkfmt_from_pixfmt
const VkFormat * av_vkfmt_from_pixfmt(enum AVPixelFormat p)
Returns the optimal per-plane Vulkan format for a given sw_format, one for each plane.
Definition: hwcontext_vulkan.c:410
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:59
VulkanDevicePriv::qf_mutex
pthread_mutex_t ** qf_mutex
Definition: hwcontext_vulkan.c:125
AV_PIX_FMT_GBRP16
#define AV_PIX_FMT_GBRP16
Definition: pixfmt.h:547
opts
AVDictionary * opts
Definition: movenc.c:51
PREP_MODE_WRITE
@ PREP_MODE_WRITE
Definition: hwcontext_vulkan.c:2334
AV_PIX_FMT_RGBA64
#define AV_PIX_FMT_RGBA64
Definition: pixfmt.h:518
NULL
#define NULL
Definition: coverity.c:32
pick_video_queue_family
static int pick_video_queue_family(VkQueueFamilyProperties2 *qf, VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf, VkVideoCodecOperationFlagBitsKHR flags)
Definition: hwcontext_vulkan.c:1377
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:211
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
FF_VK_EXT_DRM_MODIFIER_FLAGS
#define FF_VK_EXT_DRM_MODIFIER_FLAGS
Definition: vulkan_functions.h:33
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:113
FFVkFormatEntry
Definition: hwcontext_vulkan.c:311
FF_VK_EXT_SHADER_OBJECT
#define FF_VK_EXT_SHADER_OBJECT
Definition: vulkan_functions.h:47
AV_PIX_FMT_YUYV422
@ AV_PIX_FMT_YUYV422
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:74
AVVkFrameInternal
Definition: hwcontext_vulkan.c:166
FF_VK_EXT_VIDEO_ENCODE_QUEUE
#define FF_VK_EXT_VIDEO_ENCODE_QUEUE
Definition: vulkan_functions.h:61
VulkanDevicePriv::debug_ctx
VkDebugUtilsMessengerEXT debug_ctx
Definition: hwcontext_vulkan.c:131
FF_VULKAN_DEBUG_NONE
@ FF_VULKAN_DEBUG_NONE
Definition: hwcontext_vulkan.c:702
AVVulkanFramesContext::alloc_pnext
void * alloc_pnext[AV_NUM_DATA_POINTERS]
Extension data for memory allocation.
Definition: hwcontext_vulkan.h:247
setup_queue_families
static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
Definition: hwcontext_vulkan.c:1406
AVVulkanDeviceContext::unlock_queue
void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:178
LIBAVUTIL_VERSION_MAJOR
#define LIBAVUTIL_VERSION_MAJOR
Definition: version.h:81
AVVulkanDeviceContext::nb_decode_queues
attribute_deprecated int nb_decode_queues
Definition: hwcontext_vulkan.h:164
AV_PIX_FMT_P410
#define AV_PIX_FMT_P410
Definition: pixfmt.h:600
av_buffer_pool_uninit
void av_buffer_pool_uninit(AVBufferPool **ppool)
Mark the pool as being available for freeing.
Definition: buffer.c:328
AVVulkanDeviceContext::nb_qf
int nb_qf
Definition: hwcontext_vulkan.h:189
ff_hwcontext_type_vulkan
const HWContextType ff_hwcontext_type_vulkan
Definition: hwcontext_vulkan.c:4436
hwcontext_vulkan.h
AVVulkanDeviceContext::queue_family_tx_index
attribute_deprecated int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:133
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:101
vk_formats_list
static const struct FFVkFormatEntry vk_formats_list[]
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:268
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:227
AV_PIX_FMT_BGR0
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:265
AV_PIX_FMT_YUV422P10
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:529
alloc_bind_mem
static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f, void *alloc_pnext, size_t alloc_pnext_stride)
Definition: hwcontext_vulkan.c:2263
AVVulkanDeviceContext::qf
AVVulkanDeviceQueueFamily qf[64]
Queue families used.
Definition: hwcontext_vulkan.h:188
AV_PIX_FMT_GRAY8
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
Definition: pixfmt.h:81
FFVulkanContext
Definition: vulkan.h:276
exp
int8_t exp
Definition: eval.c:73
close
av_cold void CBS_FUNC() close(CodedBitstreamContext **ctx_ptr)
Close a context and free all internal state.
Definition: cbs.c:140
VulkanFramesPriv
Definition: hwcontext_vulkan.c:146
vulkan_frame_free
static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
Definition: hwcontext_vulkan.c:2224
index
int index
Definition: gxfenc.c:90
av_buffer_create
AVBufferRef * av_buffer_create(uint8_t *data, size_t size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:55
vkfmt_from_pixfmt2
static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p, VkImageTiling tiling, VkFormat fmts[AV_NUM_DATA_POINTERS], int *nb_images, VkImageAspectFlags *aspect, VkImageUsageFlags *supported_usage, int disable_multiplane, int need_storage)
Definition: hwcontext_vulkan.c:460
VulkanDeviceSelection
Definition: hwcontext_vulkan.c:1167
source
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a source
Definition: filter_design.txt:255
VulkanDevicePriv::nb_tot_qfs
uint32_t nb_tot_qfs
Definition: hwcontext_vulkan.c:126
FFVulkanContext::device
AVHWDeviceContext * device
Definition: vulkan.h:304
VulkanDeviceFeatures
Definition: hwcontext_vulkan.c:75
AVDRMFrameDescriptor::layers
AVDRMLayerDescriptor layers[AV_DRM_MAX_PLANES]
Array of layers in the frame.
Definition: hwcontext_drm.h:149
usage
const char * usage
Definition: floatimg_cmp.c:60
PREP_MODE_DECODING_DST
@ PREP_MODE_DECODING_DST
Definition: hwcontext_vulkan.c:2337
AVVkFrame::size
size_t size[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:316
vulkan_pool_alloc
static AVBufferRef * vulkan_pool_alloc(void *opaque, size_t size)
Definition: hwcontext_vulkan.c:2621
AV_PIX_FMT_X2BGR10
#define AV_PIX_FMT_X2BGR10
Definition: pixfmt.h:597
PrepMode
PrepMode
Definition: hwcontext_vulkan.c:2332
FF_VK_EXT_VIDEO_MAINTENANCE_1
#define FF_VK_EXT_VIDEO_MAINTENANCE_1
Definition: vulkan_functions.h:53
VulkanDeviceSelection::has_drm
uint32_t has_drm
Definition: hwcontext_vulkan.c:1172
f
f
Definition: af_crystalizer.c:122
AVCUDADeviceContext::internal
AVCUDADeviceContextInternal * internal
Definition: hwcontext_cuda.h:45
AV_PIX_FMT_RGB24
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:75
VulkanDeviceFeatures::vulkan_1_1
VkPhysicalDeviceVulkan11Features vulkan_1_1
Definition: hwcontext_vulkan.c:78
sem_wait
#define sem_wait(psem)
Definition: semaphore.h:27
AV_PIX_FMT_P012
#define AV_PIX_FMT_P012
Definition: pixfmt.h:586
AV_PIX_FMT_FLAG_RGB
#define AV_PIX_FMT_FLAG_RGB
The pixel format contains RGB-like data (as opposed to YUV/grayscale).
Definition: pixdesc.h:136
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition: dsp.h:83
AVVkFrame
Definition: hwcontext_vulkan.h:297
av_vk_frame_alloc
AVVkFrame * av_vk_frame_alloc(void)
Allocates a single AVVkFrame and initializes everything as 0.
Definition: hwcontext_vulkan.c:4413
FF_VULKAN_DEBUG_VALIDATE
@ FF_VULKAN_DEBUG_VALIDATE
Definition: hwcontext_vulkan.c:704
vulkan.h
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:122
FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
#define FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
Definition: vulkan_functions.h:32
vulkan_device_free
static void vulkan_device_free(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1588
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:424
FF_VK_EXT_NO_FLAG
#define FF_VK_EXT_NO_FLAG
Definition: vulkan_functions.h:66
AV_PIX_FMT_GBRPF32
#define AV_PIX_FMT_GBRPF32
Definition: pixfmt.h:561
AV_PIX_FMT_YUV422P12
#define AV_PIX_FMT_YUV422P12
Definition: pixfmt.h:533
VulkanDeviceFeatures::cooperative_matrix
VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix
Definition: hwcontext_vulkan.c:89
AV_PIX_FMT_RGB48
#define AV_PIX_FMT_RGB48
Definition: pixfmt.h:514
size
int size
Definition: twinvq_data.h:10344
ff_vk_exec_add_dep_sw_frame
int ff_vk_exec_add_dep_sw_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f)
Definition: vulkan.c:576
vulkan_transfer_data_from
static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4382
AV_NUM_DATA_POINTERS
#define AV_NUM_DATA_POINTERS
Definition: frame.h:411
FF_VK_EXT_PUSH_DESCRIPTOR
#define FF_VK_EXT_PUSH_DESCRIPTOR
Definition: vulkan_functions.h:48
VulkanDevicePriv::use_linear_images
int use_linear_images
Definition: hwcontext_vulkan.c:134
planes
static const struct @489 planes[]
AV_PIX_FMT_YUV444P12
#define AV_PIX_FMT_YUV444P12
Definition: pixfmt.h:535
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:497
AV_HWFRAME_MAP_WRITE
@ AV_HWFRAME_MAP_WRITE
The mapping must be writeable.
Definition: hwcontext.h:517
AV_PIX_FMT_NV16
@ AV_PIX_FMT_NV16
interleaved chroma YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:198
VulkanDeviceSelection::vendor_id
uint32_t vendor_id
Definition: hwcontext_vulkan.c:1175
PREP_MODE_GENERAL
@ PREP_MODE_GENERAL
Definition: hwcontext_vulkan.c:2333
AV_PIX_FMT_Y212
#define AV_PIX_FMT_Y212
Definition: pixfmt.h:590
AVVulkanDeviceContext::queue_family_index
attribute_deprecated int queue_family_index
Queue family index for graphics operations, and the number of queues enabled for it.
Definition: hwcontext_vulkan.h:124
AVDRMObjectDescriptor::size
size_t size
Total size of the object.
Definition: hwcontext_drm.h:58
VulkanFramesPriv::upload_exec
FFVkExecPool upload_exec
Definition: hwcontext_vulkan.c:156
AVVulkanDeviceQueueFamily::idx
int idx
Definition: hwcontext_vulkan.h:35
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
AVHWFramesConstraints::max_width
int max_width
The maximum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:467
FFVkExecContext
Definition: vulkan.h:112
VulkanOptExtension
Definition: hwcontext_vulkan.c:596
AV_PIX_FMT_RGB0
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition: pixfmt.h:263
AV_PIX_FMT_P216
#define AV_PIX_FMT_P216
Definition: pixfmt.h:603
CHECK_CU
#define CHECK_CU(x)
Definition: cuviddec.c:117
AV_PIX_FMT_P210
#define AV_PIX_FMT_P210
Definition: pixfmt.h:599
AV_PIX_FMT_VAAPI
@ AV_PIX_FMT_VAAPI
Hardware acceleration through VA-API, data[3] contains a VASurfaceID.
Definition: pixfmt.h:126
FF_VK_EXT_VIDEO_DECODE_QUEUE
#define FF_VK_EXT_VIDEO_DECODE_QUEUE
Definition: vulkan_functions.h:56
VulkanDevicePriv::dev_is_nvidia
int dev_is_nvidia
Definition: hwcontext_vulkan.c:143
AVVulkanDeviceContext::lock_queue
void(* lock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Locks a queue, preventing other threads from submitting any command buffers to this queue.
Definition: hwcontext_vulkan.h:173
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:220
vulkan_free_internal
static void vulkan_free_internal(AVVkFrame *f)
Definition: hwcontext_vulkan.c:2188
VulkanDeviceFeatures::timeline_semaphore
VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore
Definition: hwcontext_vulkan.c:81
AV_HWDEVICE_TYPE_VAAPI
@ AV_HWDEVICE_TYPE_VAAPI
Definition: hwcontext.h:31
pthread_mutex_destroy
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
FF_VK_EXT_EXTERNAL_HOST_MEMORY
#define FF_VK_EXT_EXTERNAL_HOST_MEMORY
Definition: vulkan_functions.h:36
AV_PIX_FMT_UYVA
@ AV_PIX_FMT_UYVA
packed UYVA 4:4:4:4, 32bpp (1 Cr & Cb sample per 1x1 Y & A samples), UYVAUYVA...
Definition: pixfmt.h:444
AVDRMFrameDescriptor::objects
AVDRMObjectDescriptor objects[AV_DRM_MAX_PLANES]
Array of objects making up the frame.
Definition: hwcontext_drm.h:141
AVCUDADeviceContextInternal::cuda_dl
CudaFunctions * cuda_dl
Definition: hwcontext_cuda_internal.h:32
ff_vk_exec_start
int ff_vk_exec_start(FFVulkanContext *s, FFVkExecContext *e)
Start/submit/wait an execution.
Definition: vulkan.c:489
FF_VK_EXT_RELAXED_EXTENDED_INSTR
#define FF_VK_EXT_RELAXED_EXTENDED_INSTR
Definition: vulkan_functions.h:49
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
FF_VK_EXT_VIDEO_DECODE_H264
#define FF_VK_EXT_VIDEO_DECODE_H264
Definition: vulkan_functions.h:57
VulkanFramesPriv::compute_exec
FFVkExecPool compute_exec
Definition: hwcontext_vulkan.c:153
AVVulkanDeviceContext::queue_family_comp_index
attribute_deprecated int queue_family_comp_index
Queue family index for compute operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:142
AVDRMObjectDescriptor::format_modifier
uint64_t format_modifier
Format modifier applied to the object (DRM_FORMAT_MOD_*).
Definition: hwcontext_drm.h:65
VkFormat
enum VkFormat VkFormat
Definition: hwcontext_stub.c:25
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:32
AV_PIX_FMT_GBRP12
#define AV_PIX_FMT_GBRP12
Definition: pixfmt.h:545
weights
static const int weights[]
Definition: hevc_pel.c:32
AV_PIX_FMT_NV24
@ AV_PIX_FMT_NV24
planar YUV 4:4:4, 24bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:371
plane_info
Definition: vf_edgedetect.c:53
VulkanDevicePriv::nb_img_qfs
uint32_t nb_img_qfs
Definition: hwcontext_vulkan.c:128
ff_vk_frame_barrier
void ff_vk_frame_barrier(FFVulkanContext *s, FFVkExecContext *e, AVFrame *pic, VkImageMemoryBarrier2 *bar, int *nb_bar, VkPipelineStageFlags src_stage, VkPipelineStageFlags dst_stage, VkAccessFlagBits new_access, VkImageLayout new_layout, uint32_t new_qf)
Definition: vulkan.c:1952
vulkan_frames_init
static int vulkan_frames_init(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2719
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
FFVkBuffer::mem
VkDeviceMemory mem
Definition: vulkan.h:90
AV_PIX_FMT_X2RGB10
#define AV_PIX_FMT_X2RGB10
Definition: pixfmt.h:596
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:256
VulkanDeviceSelection::has_uuid
int has_uuid
Definition: hwcontext_vulkan.c:1169
hwcontext_drm.h
AVDRMPlaneDescriptor::object_index
int object_index
Index of the object containing this plane in the objects array of the enclosing frame descriptor.
Definition: hwcontext_drm.h:79
AV_PIX_FMT_BGR565
#define AV_PIX_FMT_BGR565
Definition: pixfmt.h:520
FFVulkanContext::extensions
FFVulkanExtensions extensions
Definition: vulkan.h:281
ff_hwframe_map_replace
int ff_hwframe_map_replace(AVFrame *dst, const AVFrame *src)
Replace the current hwmap of dst with the one from src, used for indirect mappings like VAAPI->(DRM)-...
Definition: hwcontext.c:937
VulkanDevicePriv::img_qfs
uint32_t img_qfs[64]
Definition: hwcontext_vulkan.c:127
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:264
mod
static int mod(int a, int b)
Modulo operation with only positive remainders.
Definition: vf_v360.c:755
AV_PIX_FMT_P016
#define AV_PIX_FMT_P016
Definition: pixfmt.h:587
AV_PIX_FMT_RGB565
#define AV_PIX_FMT_RGB565
Definition: pixfmt.h:515
AVHWFrameTransferDirection
AVHWFrameTransferDirection
Definition: hwcontext.h:404
AVVkFrame::sem
VkSemaphore sem[AV_NUM_DATA_POINTERS]
Synchronization timeline semaphores, one for each VkImage.
Definition: hwcontext_vulkan.h:335
create_instance
static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:1030
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:116
FF_VK_EXT_EXTERNAL_FD_MEMORY
#define FF_VK_EXT_EXTERNAL_FD_MEMORY
Definition: vulkan_functions.h:34
AVCUDADeviceContext
This struct is allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_cuda.h:42
hwcontext_vaapi.h
AVDRMLayerDescriptor::format
uint32_t format
Format of the layer (DRM_FORMAT_*).
Definition: hwcontext_drm.h:100
VulkanDevicePriv::transfer_qf
AVVulkanDeviceQueueFamily * transfer_qf
Definition: hwcontext_vulkan.c:111
ret
ret
Definition: filter_design.txt:187
pixfmt
enum AVPixelFormat pixfmt
Definition: kmsgrab.c:367
AVHWDeviceContext::type
enum AVHWDeviceType type
This field identifies the underlying API used for hardware access.
Definition: hwcontext.h:73
AV_DRM_MAX_PLANES
@ AV_DRM_MAX_PLANES
The maximum number of layers/planes in a DRM frame.
Definition: hwcontext_drm.h:39
AV_PIX_FMT_NV12
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:96
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
AVHWFramesContext::device_ctx
AVHWDeviceContext * device_ctx
The parent AVHWDeviceContext.
Definition: hwcontext.h:135
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:280
cuda_check.h
AVHWFramesContext::hwctx
void * hwctx
The format-specific data, allocated and freed automatically along with this context.
Definition: hwcontext.h:151
VulkanFramesPriv::tmp
AVBufferPool * tmp
Definition: hwcontext_vulkan.c:160
vulkan_get_buffer
static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
Definition: hwcontext_vulkan.c:2878
pick_queue_family
static int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf, VkQueueFlagBits flags)
Definition: hwcontext_vulkan.c:1348
FFVkExecPool
Definition: vulkan.h:254
unlock_queue
static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1723
AVHWFramesConstraints::max_height
int max_height
Definition: hwcontext.h:468
AVVkFrame::internal
struct AVVkFrameInternal * internal
Internal data.
Definition: hwcontext_vulkan.h:348
ff_vk_qf_find
AVVulkanDeviceQueueFamily * ff_vk_qf_find(FFVulkanContext *s, VkQueueFlagBits dev_family, VkVideoCodecOperationFlagBitsKHR vid_ops)
Chooses an appropriate QF.
Definition: vulkan.c:220
AV_PIX_FMT_YUV420P12
#define AV_PIX_FMT_YUV420P12
Definition: pixfmt.h:532
FF_VK_EXT_DESCRIPTOR_BUFFER
#define FF_VK_EXT_DESCRIPTOR_BUFFER
Definition: vulkan_functions.h:42
AV_PIX_FMT_UYVY422
@ AV_PIX_FMT_UYVY422
packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1
Definition: pixfmt.h:88
check_extensions
static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode debug_mode)
Definition: hwcontext_vulkan.c:715
FFVkExecContext::buf
VkCommandBuffer buf
Definition: vulkan.h:124
SIZE_SPECIFIER
#define SIZE_SPECIFIER
Definition: internal.h:129
vulkan_device_create
static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:1965
AVFrame::height
int height
Definition: frame.h:482
PREP_MODE_ENCODING_DPB
@ PREP_MODE_ENCODING_DPB
Definition: hwcontext_vulkan.c:2339
FFVkFormatEntry::pixfmt
enum AVPixelFormat pixfmt
Definition: hwcontext_vulkan.c:313
AVHWFramesConstraints::min_height
int min_height
Definition: hwcontext.h:461
RELEASE_PROPS
#define RELEASE_PROPS(props, count)
Definition: hwcontext_vulkan.c:694
mode
mode
Definition: ebur128.h:83
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
AV_PIX_FMT_GBRAPF32
#define AV_PIX_FMT_GBRAPF32
Definition: pixfmt.h:562
VulkanDevicePriv::feats
VulkanDeviceFeatures feats
Definition: hwcontext_vulkan.c:122
LIBAVUTIL_VERSION_MICRO
#define LIBAVUTIL_VERSION_MICRO
Definition: version.h:83
find_device
static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
Definition: hwcontext_vulkan.c:1191
AVVulkanDeviceContext::nb_graphics_queues
attribute_deprecated int nb_graphics_queues
Definition: hwcontext_vulkan.h:126
optional_instance_exts
static const VulkanOptExtension optional_instance_exts[]
Definition: hwcontext_vulkan.c:601
AV_PIX_FMT_FLAG_PLANAR
#define AV_PIX_FMT_FLAG_PLANAR
At least one pixel component is not in the first data plane.
Definition: pixdesc.h:132
FF_VK_EXT_ATOMIC_FLOAT
#define FF_VK_EXT_ATOMIC_FLOAT
Definition: vulkan_functions.h:44
FFVkFormatEntry::fallback
const VkFormat fallback[5]
Definition: hwcontext_vulkan.c:318
PREP_MODE_EXTERNAL_IMPORT
@ PREP_MODE_EXTERNAL_IMPORT
Definition: hwcontext_vulkan.c:2336
AV_PIX_FMT_RGBAF32
#define AV_PIX_FMT_RGBAF32
Definition: pixfmt.h:610
FF_VK_EXT_VIDEO_DECODE_AV1
#define FF_VK_EXT_VIDEO_DECODE_AV1
Definition: vulkan_functions.h:59
AV_PIX_FMT_YUV444P
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:78
AVVulkanFramesContext::tiling
VkImageTiling tiling
Controls the tiling of allocated frames.
Definition: hwcontext_vulkan.h:217
VulkanDeviceSelection::drm_major
uint32_t drm_major
Definition: hwcontext_vulkan.c:1170
get_plane_buf
static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:4011
AV_PIX_FMT_P010
#define AV_PIX_FMT_P010
Definition: pixfmt.h:585
unlock_frame
static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2696
FF_DISABLE_DEPRECATION_WARNINGS
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:72
AVVkFrame::sem_value
uint64_t sem_value[AV_NUM_DATA_POINTERS]
Up to date semaphore value at which each image becomes accessible.
Definition: hwcontext_vulkan.h:343
av_strdup
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:272
desc
const char * desc
Definition: libsvtav1.c:79
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:112
AVVulkanFramesContext::nb_layers
int nb_layers
Number of layers each image will have.
Definition: hwcontext_vulkan.h:273
FFVulkanContext::hwctx
AVVulkanDeviceContext * hwctx
Definition: vulkan.h:305
AV_PIX_FMT_YUV422P
@ AV_PIX_FMT_YUV422P
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:77
VulkanFramesPriv::download_exec
FFVkExecPool download_exec
Definition: hwcontext_vulkan.c:157
mem.h
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
hwcontext_internal.h
FF_VK_EXT_VIDEO_ENCODE_H264
#define FF_VK_EXT_VIDEO_ENCODE_H264
Definition: vulkan_functions.h:62
ADD_QUEUE
#define ADD_QUEUE(ctx_qf, qc, flag)
AVVulkanDeviceContext::nb_enabled_inst_extensions
int nb_enabled_inst_extensions
Definition: hwcontext_vulkan.h:102
av_free
#define av_free(p)
Definition: tableprint_vlc.h:34
AVDictionaryEntry
Definition: dict.h:89
get_plane_wh
static void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format, int frame_w, int frame_h, int plane)
Definition: hwcontext_vulkan.c:2432
ff_vk_count_images
static int ff_vk_count_images(AVVkFrame *f)
Definition: vulkan.h:322
ff_vk_exec_discard_deps
void ff_vk_exec_discard_deps(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:521
PREP_MODE_EXTERNAL_EXPORT
@ PREP_MODE_EXTERNAL_EXPORT
Definition: hwcontext_vulkan.c:2335
FFALIGN
#define FFALIGN(x, a)
Definition: macros.h:78
AV_PIX_FMT_P416
#define AV_PIX_FMT_P416
Definition: pixfmt.h:604
VulkanDevicePriv::mprops
VkPhysicalDeviceMemoryProperties mprops
Definition: hwcontext_vulkan.c:115
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
FF_VULKAN_DEBUG_NB
@ FF_VULKAN_DEBUG_NB
Definition: hwcontext_vulkan.c:712
FFVkBuffer
Definition: vulkan.h:88
vk_dev_type
static const char * vk_dev_type(enum VkPhysicalDeviceType type)
Definition: hwcontext_vulkan.c:1179
VulkanDevicePriv::disable_multiplane
int disable_multiplane
Definition: hwcontext_vulkan.c:140
imgutils.h
ff_vk_exec_submit
int ff_vk_exec_submit(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:834
FF_VK_EXT_OPTICAL_FLOW
#define FF_VK_EXT_OPTICAL_FLOW
Definition: vulkan_functions.h:46
AV_PIX_FMT_XV36
#define AV_PIX_FMT_XV36
Definition: pixfmt.h:593
hwcontext.h
AVDRMPlaneDescriptor::pitch
ptrdiff_t pitch
Pitch (linesize) of this plane.
Definition: hwcontext_drm.h:87
AVFrame::linesize
int linesize[AV_NUM_DATA_POINTERS]
For video, a positive or negative value, which is typically indicating the size in bytes of each pict...
Definition: frame.h:455
VulkanDeviceFeatures::descriptor_buffer
VkPhysicalDeviceDescriptorBufferFeaturesEXT descriptor_buffer
Definition: hwcontext_vulkan.c:90
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
FFVkFormatEntry::vk_planes
int vk_planes
Definition: hwcontext_vulkan.c:315
AVVulkanDeviceQueueFamily
Definition: hwcontext_vulkan.h:33
HWContextType
Definition: hwcontext_internal.h:29
AV_PIX_FMT_P412
#define AV_PIX_FMT_P412
Definition: pixfmt.h:602
device_features_init
static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
Definition: hwcontext_vulkan.c:185
VulkanDevicePriv::contiguous_planes
int contiguous_planes
Definition: hwcontext_vulkan.c:137
FN_MAP_TO
#define FN_MAP_TO(dst_t, dst_name, src_t, src_name)
Definition: hwcontext_vulkan.c:427
AVVAAPIDeviceContext
VAAPI connection details.
Definition: hwcontext_vaapi.h:68
h
h
Definition: vp9dsp_template.c:2070
AVVulkanDeviceContext::device_features
VkPhysicalDeviceFeatures2 device_features
This structure should be set to the set of features that present and enabled during device creation.
Definition: hwcontext_vulkan.h:92
AVDictionaryEntry::value
char * value
Definition: dict.h:91
avstring.h
AVDRMDeviceContext
DRM device.
Definition: hwcontext_drm.h:157
AV_PIX_FMT_GRAY12
#define AV_PIX_FMT_GRAY12
Definition: pixfmt.h:509
OPT_CHAIN
#define OPT_CHAIN(STRUCT_P, EXT_FLAG, TYPE)
copy_buffer_data
static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf, AVFrame *swf, VkBufferImageCopy *region, int planes, int upload)
Definition: hwcontext_vulkan.c:3953
VulkanDeviceFeatures::atomic_float
VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float
Definition: hwcontext_vulkan.c:91
ADD_VAL_TO_LIST
#define ADD_VAL_TO_LIST(list, count, val)
Definition: hwcontext_vulkan.c:680
AVDRMFrameDescriptor::nb_objects
int nb_objects
Number of DRM objects making up this frame.
Definition: hwcontext_drm.h:137
VulkanDeviceFeatures::shader_object
VkPhysicalDeviceShaderObjectFeaturesEXT shader_object
Definition: hwcontext_vulkan.c:88
HWMapDescriptor
Definition: hwcontext_internal.h:120
AVVulkanDeviceQueueFamily::flags
VkQueueFlagBits flags
Definition: hwcontext_vulkan.h:41
AVVulkanDeviceQueueFamily::video_caps
VkVideoCodecOperationFlagBitsKHR video_caps
Definition: hwcontext_vulkan.h:44
FFVulkanFunctions
Definition: vulkan_functions.h:265
VulkanDevicePriv::p
AVVulkanDeviceContext p
The public AVVulkanDeviceContext.
Definition: hwcontext_vulkan.c:104
VulkanFramesPriv::modifier_info
VkImageDrmFormatModifierListCreateInfoEXT * modifier_info
Definition: hwcontext_vulkan.c:163
FFVkFormatEntry::aspect
VkImageAspectFlags aspect
Definition: hwcontext_vulkan.c:314
ff_vk_get_pooled_buffer
int ff_vk_get_pooled_buffer(FFVulkanContext *ctx, AVBufferPool **buf_pool, AVBufferRef **buf, VkBufferUsageFlags usage, void *create_pNext, size_t size, VkMemoryPropertyFlagBits mem_props)
Initialize a pool and create AVBufferRefs containing FFVkBuffer.
Definition: vulkan.c:1174
VulkanDeviceSelection::name
const char * name
Definition: hwcontext_vulkan.c:1173
src
#define src
Definition: vp8dsp.c:248
AV_HWDEVICE_TYPE_DRM
@ AV_HWDEVICE_TYPE_DRM
Definition: hwcontext.h:36
AVVulkanDeviceContext::nb_comp_queues
attribute_deprecated int nb_comp_queues
Definition: hwcontext_vulkan.h:144
vulkan_device_create_internal
static int vulkan_device_create_internal(AVHWDeviceContext *ctx, VulkanDeviceSelection *dev_select, int disable_multiplane, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:1624
w32dlfcn.h
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:3261
Generated on Fri Apr 4 2025 19:23:00 for FFmpeg by   doxygen 1.8.17