Halide 17.0.2
Halide compiler and libraries
Loading...
Searching...
No Matches
vulkan_resources.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_VULKAN_RESOURCES_H
2#define HALIDE_RUNTIME_VULKAN_RESOURCES_H
3
4#include "vulkan_internal.h"
5#include "vulkan_memory.h"
6
7// --------------------------------------------------------------------------
8
9namespace Halide {
10namespace Runtime {
11namespace Internal {
12namespace Vulkan {
13
14// Defines the specialization constants used for dynamically overiding the dispatch size
16 uint32_t constant_id[3] = {0}; // zero if unused
17};
18
19// Data used to override specialization constants for dynamic dispatching
26
27// Specialization constant binding information
33
34// Shared memory allocation variable information
36 uint32_t constant_id = 0; // specialization constant to override allocation array size (or zero if unused)
39 const char *variable_name = nullptr;
40};
41
42// Entry point metadata for shader modules
58
59// Compilation cache for compiled shader modules
67
69
70// --------------------------------------------------------------------------
71
72namespace { // internalize
73
74// --------------------------------------------------------------------------
75
76int vk_create_command_pool(void *user_context, VulkanMemoryAllocator *allocator, uint32_t queue_index, VkCommandPool *command_pool) {
77#ifdef DEBUG_RUNTIME
78 debug(user_context)
79 << " vk_create_command_pool (user_context: " << user_context << ", "
80 << "allocator: " << (void *)allocator << ", "
81 << "queue_index: " << queue_index << ")\n";
82#endif
83
84 if (allocator == nullptr) {
85 error(user_context) << "Vulkan: Failed to create command pool ... invalid allocator pointer!\n";
87 }
88
90 {
92 nullptr, // pointer to struct extending this
93 VK_COMMAND_POOL_CREATE_TRANSIENT_BIT, // flags. Assume transient short-lived single-use command buffers
94 queue_index // queue family index corresponding to the compute command queue
95 };
96
97 VkResult result = vkCreateCommandPool(allocator->current_device(), &command_pool_info, allocator->callbacks(), command_pool);
98 if (result != VK_SUCCESS) {
99 error(user_context) << "Vulkan: Failed to create command pool!\n";
101 }
103}
104
105int vk_destroy_command_pool(void *user_context, VulkanMemoryAllocator *allocator, VkCommandPool command_pool) {
106#ifdef DEBUG_RUNTIME
107 debug(user_context)
108 << " vk_destroy_command_pool (user_context: " << user_context << ", "
109 << "allocator: " << (void *)allocator << ", "
110 << "command_pool: " << (void *)command_pool << ")\n";
111#endif
112 if (allocator == nullptr) {
113 error(user_context) << "Vulkan: Failed to destroy command pool ... invalid allocator pointer!\n";
115 }
116
117 vkDestroyCommandPool(allocator->current_device(), command_pool, allocator->callbacks());
119}
120
121// --
122
123int vk_create_command_buffer(void *user_context, VulkanMemoryAllocator *allocator, VkCommandPool command_pool, VkCommandBuffer *command_buffer) {
124#ifdef DEBUG_RUNTIME
125 debug(user_context)
126 << " vk_create_command_buffer (user_context: " << user_context << ", "
127 << "allocator: " << (void *)allocator << ", "
128 << "command_pool: " << (void *)command_pool << ")\n";
129#endif
130 if (allocator == nullptr) {
131 error(user_context) << "Vulkan: Failed to create command buffer ... invalid allocator pointer!\n";
133 }
134
136 {
138 nullptr, // pointer to struct extending this
139 command_pool, // command pool for allocation
140 VK_COMMAND_BUFFER_LEVEL_PRIMARY, // command buffer level
141 1 // number to allocate
142 };
143
145 if (result != VK_SUCCESS) {
146 error(user_context) << "Vulkan: Failed to allocate command buffers!\n";
148 }
150}
151
152int vk_destroy_command_buffer(void *user_context, VulkanMemoryAllocator *allocator, VkCommandPool command_pool, VkCommandBuffer command_buffer) {
153#ifdef DEBUG_RUNTIME
154 debug(user_context)
155 << " vk_destroy_command_buffer (user_context: " << user_context << ", "
156 << "allocator: " << (void *)allocator << ", "
157 << "command_pool: " << (void *)command_pool << ", "
158 << "command_buffer: " << (void *)command_buffer << ")\n";
159#endif
160 if (allocator == nullptr) {
161 error(user_context) << "Vulkan: Failed to destroy command buffer ... invalid allocator pointer!\n";
163 }
164
165 vkFreeCommandBuffers(allocator->current_device(), command_pool, 1, &command_buffer);
167}
168
169int vk_fill_command_buffer_with_dispatch_call(void *user_context,
170 VkDevice device,
172 VkPipeline compute_pipeline,
173 VkPipelineLayout pipeline_layout,
174 VkDescriptorSet descriptor_set,
176 int blocksX, int blocksY, int blocksZ) {
177
178#ifdef DEBUG_RUNTIME
179 debug(user_context)
180 << " vk_fill_command_buffer_with_dispatch_call (user_context: " << user_context << ", "
181 << "device: " << (void *)device << ", "
182 << "command_buffer: " << (void *)command_buffer << ", "
183 << "pipeline_layout: " << (void *)pipeline_layout << ", "
184 << "descriptor_set: " << (void *)descriptor_set << ", "
185 << "descriptor_set_index: " << descriptor_set_index << ", "
186 << "blocks: " << blocksX << ", " << blocksY << ", " << blocksZ << ")\n";
187#endif
188
191 nullptr, // pointer to struct extending this
193 nullptr // pointer to parent command buffer
194 };
195
197 if (result != VK_SUCCESS) {
198 error(user_context) << "vkBeginCommandBuffer returned " << vk_get_error_name(result) << "\n";
200 }
201
204 descriptor_set_index, 1, &descriptor_set, 0, nullptr);
206
208 if (result != VK_SUCCESS) {
209 error(user_context) << "vkEndCommandBuffer returned " << vk_get_error_name(result) << "\n";
211 }
212
214}
215
216int vk_submit_command_buffer(void *user_context, VkQueue queue, VkCommandBuffer command_buffer) {
217#ifdef DEBUG_RUNTIME
218 debug(user_context)
219 << " vk_submit_command_buffer (user_context: " << user_context << ", "
220 << "queue: " << (void *)queue << ", "
221 << "command_buffer: " << (void *)command_buffer << ")\n";
222#endif
223
225 {
226 VK_STRUCTURE_TYPE_SUBMIT_INFO, // struct type
227 nullptr, // pointer to struct extending this
228 0, // wait semaphore count
229 nullptr, // semaphores
230 nullptr, // pipeline stages where semaphore waits occur
231 1, // how many command buffers to execute
232 &command_buffer, // the command buffers
233 0, // number of semaphores to signal
234 nullptr // the semaphores to signal
235 };
236
237 VkResult result = vkQueueSubmit(queue, 1, &submit_info, 0);
238 if (result != VK_SUCCESS) {
239 error(user_context) << "Vulkan: vkQueueSubmit returned " << vk_get_error_name(result) << "\n";
241 }
243}
244
245// --
246
247bool vk_needs_scalar_uniform_buffer(void *user_context,
248 size_t arg_sizes[],
249 void *args[],
251 int i = 0;
252 while (arg_sizes[i] > 0) {
253 if (!arg_is_buffer[i]) {
254 return true;
255 }
256 i++;
257 }
258 return false;
259}
260
262 size_t arg_sizes[],
263 void *args[],
265
266 // first binding is for passing scalar parameters in a buffer (if necessary)
267 uint32_t bindings_count = vk_needs_scalar_uniform_buffer(user_context, arg_sizes, args, arg_is_buffer);
268
269 int i = 0;
270 while (arg_sizes[i] > 0) {
271 if (arg_is_buffer[i]) {
272 bindings_count++;
273 }
274 i++;
275 }
276 return bindings_count;
277}
278
279// --
280
281int vk_create_descriptor_pool(void *user_context,
282 VulkanMemoryAllocator *allocator,
283 uint32_t uniform_buffer_count,
284 uint32_t storage_buffer_count,
285 VkDescriptorPool *descriptor_pool) {
286#ifdef DEBUG_RUNTIME
287 debug(user_context)
288 << " vk_create_descriptor_pool (user_context: " << user_context << ", "
289 << "allocator: " << (void *)allocator << ", "
290 << "uniform_buffer_count: " << (uint32_t)uniform_buffer_count << ", "
291 << "storage_buffer_count: " << (uint32_t)storage_buffer_count << ")\n";
292#endif
293 if (allocator == nullptr) {
294 error(user_context) << "Vulkan: Failed to create descriptor pool ... invalid allocator pointer!\n";
296 }
297
299 pool_config.entry_size = sizeof(VkDescriptorPoolSize);
300 pool_config.minimum_capacity = (uniform_buffer_count ? 1 : 0) + (storage_buffer_count ? 1 : 0);
301 BlockStorage pool_sizes(user_context, pool_config);
302
303 // First binding is reserved for passing scalar parameters as a uniform buffer
304 if (uniform_buffer_count > 0) {
306 VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, // descriptor type
307 uniform_buffer_count // all kernel args are packed into uniform buffers
308 };
309 pool_sizes.append(user_context, &uniform_buffer_size);
310 }
311
312 if (storage_buffer_count > 0) {
314 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, // descriptor type
315 storage_buffer_count // all halide buffers are passed as storage buffers
316 };
317 pool_sizes.append(user_context, &storage_buffer_size);
318 }
319
322 nullptr, // point to struct extending this
323 0, // flags
324 1, // this pool will only be used for creating one descriptor set!
325 (uint32_t)pool_sizes.size(), // pool size count
326 (const VkDescriptorPoolSize *)pool_sizes.data() // ptr to descriptr pool sizes
327 };
328
329 VkResult result = vkCreateDescriptorPool(allocator->current_device(), &descriptor_pool_info, allocator->callbacks(), descriptor_pool);
330 if (result != VK_SUCCESS) {
331 error(user_context) << "Vulkan: Failed to create descriptor pool! vkCreateDescriptorPool returned " << vk_get_error_name(result) << "\n";
333 }
335}
336
337int vk_destroy_descriptor_pool(void *user_context,
338 VulkanMemoryAllocator *allocator,
339 VkDescriptorPool descriptor_pool) {
340#ifdef DEBUG_RUNTIME
341 debug(user_context)
342 << " vk_destroy_descriptor_pool (user_context: " << user_context << ", "
343 << "allocator: " << (void *)allocator << ", "
344 << "descriptor_pool: " << (void *)descriptor_pool << ")\n";
345#endif
346 if (allocator == nullptr) {
347 error(user_context) << "Vulkan: Failed to destroy descriptor pool ... invalid allocator pointer!\n";
349 }
350 vkDestroyDescriptorPool(allocator->current_device(), descriptor_pool, allocator->callbacks());
352}
353
354// --
355
356int vk_create_descriptor_set_layout(void *user_context,
357 VulkanMemoryAllocator *allocator,
358 uint32_t uniform_buffer_count,
359 uint32_t storage_buffer_count,
360 VkDescriptorSetLayout *layout) {
361
362#ifdef DEBUG_RUNTIME
363 debug(user_context)
364 << " vk_create_descriptor_set_layout (user_context: " << user_context << ", "
365 << "allocator: " << (void *)allocator << ", "
366 << "uniform_buffer_count: " << uniform_buffer_count << ", "
367 << "storage_buffer_count: " << storage_buffer_count << ", "
368 << "layout: " << (void *)layout << ")\n";
369#endif
370 if (allocator == nullptr) {
371 error(user_context) << "Vulkan: Failed to create descriptor set layout ... invalid allocator pointer!\n";
373 }
374
376 layout_config.entry_size = sizeof(VkDescriptorSetLayoutBinding);
377 layout_config.minimum_capacity = uniform_buffer_count + storage_buffer_count;
379
380 // add all uniform buffers first
381 for (uint32_t n = 0; n < uniform_buffer_count; ++n) {
383 (uint32_t)layout_bindings.size(), // binding index
384 VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, // descriptor type
385 1, // descriptor count
386 VK_SHADER_STAGE_COMPUTE_BIT, // stage flags
387 nullptr // immutable samplers
388 };
389
390#ifdef DEBUG_RUNTIME
391 debug(user_context)
392 << " [" << (uint32_t)layout_bindings.size() << "] : UNIFORM_BUFFER\n";
393#endif
394
395 layout_bindings.append(user_context, &uniform_buffer_layout);
396 }
397
398 // Add all other storage buffers
399 for (uint32_t n = 0; n < storage_buffer_count; ++n) {
400
401 // halide buffers will be passed as STORAGE_BUFFERS
403 (uint32_t)layout_bindings.size(), // binding index
404 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, // descriptor type
405 1, // descriptor count
406 VK_SHADER_STAGE_COMPUTE_BIT, // stage flags
407 nullptr // immutable samplers
408 };
409#ifdef DEBUG_RUNTIME
410 debug(user_context)
411 << " [" << (uint32_t)layout_bindings.size() << "] : STORAGE_BUFFER\n";
412#endif
413
414 layout_bindings.append(user_context, &storage_buffer_layout);
415 }
416
417 // Create the LayoutInfo struct
420 nullptr, // pointer to a struct extending this info
421 0, // flags
422 (uint32_t)layout_bindings.size(), // binding count
423 (VkDescriptorSetLayoutBinding *)layout_bindings.data() // pointer to layout bindings array
424 };
425
426 // Create the descriptor set layout
427 VkResult result = vkCreateDescriptorSetLayout(allocator->current_device(), &layout_info, allocator->callbacks(), layout);
428 if (result != VK_SUCCESS) {
429 error(user_context) << "vkCreateDescriptorSetLayout returned " << vk_get_error_name(result) << "\n";
431 }
432
434}
435
436int vk_destroy_descriptor_set_layout(void *user_context,
437 VulkanMemoryAllocator *allocator,
439
440#ifdef DEBUG_RUNTIME
441 debug(user_context)
442 << " vk_destroy_descriptor_set_layout (user_context: " << user_context << ", "
443 << "allocator: " << (void *)allocator << ", "
444 << "layout: " << (void *)descriptor_set_layout << ")\n";
445#endif
446 if (allocator == nullptr) {
447 error(user_context) << "Vulkan: Failed to destroy descriptor set layout ... invalid allocator pointer!\n";
449 }
452}
453
454// --
455
456int vk_create_descriptor_set(void *user_context,
457 VulkanMemoryAllocator *allocator,
459 VkDescriptorPool descriptor_pool,
460 VkDescriptorSet *descriptor_set) {
461#ifdef DEBUG_RUNTIME
462 debug(user_context)
463 << " vk_create_descriptor_set (user_context: " << user_context << ", "
464 << "allocator: " << (void *)allocator << ", "
465 << "descriptor_set_layout: " << (void *)descriptor_set_layout << ", "
466 << "descriptor_pool: " << (void *)descriptor_pool << ")\n";
467#endif
468 if (allocator == nullptr) {
469 error(user_context) << "Vulkan: Failed to create descriptor set ... invalid allocator pointer!\n";
471 }
472
474 {
476 nullptr, // pointer to struct extending this
477 descriptor_pool, // pool from which to allocate sets
478 1, // number of descriptor sets
479 &descriptor_set_layout // pointer to array of descriptor set layouts
480 };
481
482 VkResult result = vkAllocateDescriptorSets(allocator->current_device(), &descriptor_set_info, descriptor_set);
483 if (result != VK_SUCCESS) {
484 error(user_context) << "Vulkan: vkAllocateDescriptorSets returned " << vk_get_error_name(result) << "\n";
486 }
487
489}
490
491int vk_update_descriptor_set(void *user_context,
492 VulkanMemoryAllocator *allocator,
494 size_t uniform_buffer_count,
495 size_t storage_buffer_count,
496 size_t arg_sizes[],
497 void *args[],
499 VkDescriptorSet descriptor_set) {
500#ifdef DEBUG_RUNTIME
501 debug(user_context)
502 << " vk_update_descriptor_set (user_context: " << user_context << ", "
503 << "allocator: " << (void *)allocator << ", "
504 << "scalar_args_buffer: " << (void *)scalar_args_buffer << ", "
505 << "uniform_buffer_count: " << (uint32_t)uniform_buffer_count << ", "
506 << "storage_buffer_count: " << (uint32_t)storage_buffer_count << ", "
507 << "descriptor_set: " << (void *)descriptor_set << ")\n";
508#endif
509 if (allocator == nullptr) {
510 error(user_context) << "Vulkan: Failed to create descriptor set ... invalid allocator pointer!\n";
512 }
513
515 dbi_config.minimum_capacity = storage_buffer_count + uniform_buffer_count;
516 dbi_config.entry_size = sizeof(VkDescriptorBufferInfo);
518
520 wds_config.minimum_capacity = storage_buffer_count + uniform_buffer_count;
521 wds_config.entry_size = sizeof(VkWriteDescriptorSet);
523
524 // First binding will be the scalar args buffer (if needed) passed as a UNIFORM BUFFER
526 if (scalar_args_buffer != nullptr) {
528 *scalar_args_buffer, // the buffer
529 0, // offset
530 VK_WHOLE_SIZE // range
531 };
534
535#ifdef DEBUG_RUNTIME
536 debug(user_context) << " [" << (uint32_t)write_descriptor_set.size() << "] UNIFORM_BUFFER : "
537 << "buffer=" << (void *)scalar_args_buffer << " "
538 << "offset=" << (uint32_t)(0) << " "
539 << "size=VK_WHOLE_SIZE\n";
540#endif
543 nullptr, // pointer to struct extending this
544 descriptor_set, // descriptor set to update
545 0, // binding slot
546 0, // array elem
547 1, // num to update
548 VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, // descriptor type
549 nullptr, // for images
550 scalar_args_entry, // info for buffer
551 nullptr // for texel buffers
552 };
554 }
555
556 // Add all the other device buffers as STORAGE BUFFERs
557 for (size_t i = 0; arg_sizes[i] > 0; i++) {
558 if (arg_is_buffer[i]) {
559
560 // get the allocated region for the buffer
561 MemoryRegion *device_region = reinterpret_cast<MemoryRegion *>(((halide_buffer_t *)args[i])->device);
562 MemoryRegion *owner = allocator->owner_of(user_context, device_region);
563
564 // retrieve the buffer from the region
565 VkBuffer *device_buffer = reinterpret_cast<VkBuffer *>(owner->handle);
566 if (device_buffer == nullptr) {
567 error(user_context) << "Vulkan: Failed to retrieve buffer for device memory!\n";
569 }
570
571 VkDeviceSize range_offset = device_region->range.head_offset;
572 VkDeviceSize range_size = device_region->size - device_region->range.head_offset - device_region->range.tail_offset;
573 halide_abort_if_false(user_context, (device_region->size - device_region->range.head_offset - device_region->range.tail_offset) > 0);
575 *device_buffer, // the buffer
576 range_offset, // range offset
577 range_size // range size
578 };
579 descriptor_buffer_info.append(user_context, &device_buffer_info);
581
582#ifdef DEBUG_RUNTIME
583 debug(user_context) << " [" << (uint32_t)write_descriptor_set.size() << "] STORAGE_BUFFER : "
584 << "region=" << (void *)device_region << " "
585 << "buffer=" << (void *)device_buffer << " "
586 << "offset=" << (uint32_t)(range_offset) << " "
587 << "size=" << (uint32_t)(range_size) << "\n";
588#endif
589
592 nullptr, // pointer to struct extending this
593 descriptor_set, // descriptor set to update
594 (uint32_t)write_descriptor_set.size(), // binding slot
595 0, // array elem
596 1, // num to update
597 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, // descriptor type
598 nullptr, // for images
599 device_buffer_entry, // info for buffer
600 nullptr // for texel buffers
601 };
603 }
604 }
605
606 // issue the update call to populate the descriptor set
609}
610
611// --
612
613size_t vk_estimate_scalar_uniform_buffer_size(void *user_context,
614 size_t arg_sizes[],
615 void *args[],
617 int i = 0;
619 while (arg_sizes[i] > 0) {
620 if (!arg_is_buffer[i]) {
622 }
623 i++;
624 }
626}
627
629 VulkanMemoryAllocator *allocator,
630 size_t scalar_buffer_size) {
631
632#ifdef DEBUG_RUNTIME
633 debug(user_context)
634 << " vk_create_scalar_uniform_buffer (user_context: " << user_context << ", "
635 << "allocator: " << (void *)allocator << ", "
636 << "scalar_buffer_size: " << (uint32_t)scalar_buffer_size << ")\n";
637#endif
638
639 if (allocator == nullptr) {
640 error(user_context) << "Vulkan: Failed to create scalar uniform buffer ... invalid allocator pointer!\n";
641 return nullptr;
642 }
643
646 request.properties.usage = MemoryUsage::UniformStorage;
647 request.properties.caching = MemoryCaching::UncachedCoherent;
648 request.properties.visibility = MemoryVisibility::HostToDevice;
649
650 // allocate a new region
651 MemoryRegion *region = allocator->reserve(user_context, request);
652 if ((region == nullptr) || (region->handle == nullptr)) {
653 error(user_context) << "Vulkan: Failed to create scalar uniform buffer ... unable to allocate device memory!\n";
654 return nullptr;
655 }
656
657 // return the allocated region for the uniform buffer
658 return region;
659}
660
661int vk_update_scalar_uniform_buffer(void *user_context,
662 VulkanMemoryAllocator *allocator,
663 MemoryRegion *region,
664 size_t arg_sizes[],
665 void *args[],
667
668#ifdef DEBUG_RUNTIME
669 debug(user_context)
670 << " vk_update_scalar_uniform_buffer (user_context: " << user_context << ", "
671 << "region: " << (void *)region << ")\n";
672#endif
673
674 if (allocator == nullptr) {
675 error(user_context) << "Vulkan: Failed to update scalar uniform buffer ... invalid allocator pointer!\n";
677 }
678
679 if ((region == nullptr) || (region->handle == nullptr)) {
680 error(user_context) << "Vulkan: Failed to update scalar uniform buffer ... invalid memory region!\n";
682 }
683
684 // map the region to a host ptr
685 uint8_t *host_ptr = (uint8_t *)allocator->map(user_context, region);
686 if (host_ptr == nullptr) {
687 error(user_context) << "Vulkan: Failed to update scalar uniform buffer ... unable to map host pointer to device memory!\n";
689 }
690
691 // copy to the (host-visible/coherent) scalar uniform buffer
692 size_t arg_offset = 0;
693 for (size_t i = 0; arg_sizes[i] > 0; i++) {
694 if (!arg_is_buffer[i]) {
697 }
698 }
699
700 // unmap the pointer to the buffer for the region
701 allocator->unmap(user_context, region);
703}
704
705int vk_destroy_scalar_uniform_buffer(void *user_context, VulkanMemoryAllocator *allocator,
707
708#ifdef DEBUG_RUNTIME
709 debug(user_context)
710 << " vk_destroy_scalar_uniform_buffer (user_context: " << user_context << ", "
711 << "allocator: " << (void *)allocator << ", "
712 << "scalar_args_region: " << (void *)scalar_args_region << ")\n";
713#endif
714 if (allocator == nullptr) {
715 error(user_context) << "Vulkan: Failed to destroy scalar uniform buffer ... invalid allocator pointer!\n";
717 }
718
719 if (!scalar_args_region) {
721 }
722
723 int error_code = halide_error_code_success;
724 if (halide_can_reuse_device_allocations(user_context)) {
725 error_code = allocator->release(user_context, scalar_args_region);
726 } else {
727 error_code = allocator->reclaim(user_context, scalar_args_region);
728 }
729 return error_code;
730}
731
732// --
733
734int vk_create_pipeline_layout(void *user_context,
735 VulkanMemoryAllocator *allocator,
737 VkDescriptorSetLayout *descriptor_set_layouts,
738 VkPipelineLayout *pipeline_layout) {
739
740#ifdef DEBUG_RUNTIME
741 debug(user_context)
742 << " vk_create_pipeline_layout (user_context: " << user_context << ", "
743 << "allocator: " << (void *)allocator << ", "
744 << "descriptor_set_count: " << descriptor_set_count << ", "
745 << "descriptor_set_layouts: " << (void *)descriptor_set_layouts << ", "
746 << "pipeline_layout: " << (void *)pipeline_layout << ")\n";
747#endif
748 if (allocator == nullptr) {
749 error(user_context) << "Vulkan: Failed to create pipeline layout ... invalid allocator pointer!\n";
751 }
752
755 nullptr, // pointer to a structure extending this
756 0, // flags
757 descriptor_set_count, // number of descriptor sets
758 descriptor_set_layouts, // pointer to the descriptor sets
759 0, // number of push constant ranges
760 nullptr // pointer to push constant range structs
761 };
762
763 VkResult result = vkCreatePipelineLayout(allocator->current_device(), &pipeline_layout_info, allocator->callbacks(), pipeline_layout);
764 if (result != VK_SUCCESS) {
765 error(user_context) << "Vulkan: vkCreatePipelineLayout returned " << vk_get_error_name(result) << "\n";
767 }
769}
770
771int vk_destroy_pipeline_layout(void *user_context,
772 VulkanMemoryAllocator *allocator,
773 VkPipelineLayout pipeline_layout) {
774
775#ifdef DEBUG_RUNTIME
776 debug(user_context)
777 << " vk_destroy_pipeline_layout (user_context: " << user_context << ", "
778 << "allocator: " << (void *)allocator << ", "
779 << "pipeline_layout: " << (void *)pipeline_layout << ")\n";
780#endif
781
782 if (allocator == nullptr) {
783 error(user_context) << "Vulkan: Failed to destroy pipeline layout ... invalid allocator pointer!\n";
785 }
786
787 vkDestroyPipelineLayout(allocator->current_device(), pipeline_layout, allocator->callbacks());
789}
790
791// --
792
793int vk_create_compute_pipeline(void *user_context,
794 VulkanMemoryAllocator *allocator,
795 const char *pipeline_name,
796 VkShaderModule shader_module,
797 VkPipelineLayout pipeline_layout,
799 VkPipeline *compute_pipeline) {
800
801#ifdef DEBUG_RUNTIME
802 debug(user_context)
803 << " vk_create_compute_pipeline (user_context: " << user_context << ", "
804 << "allocator: " << (void *)allocator << ", "
805 << "shader_module: " << (void *)shader_module << ", "
806 << "pipeline_layout: " << (void *)pipeline_layout << ")\n";
807#endif
808 if (allocator == nullptr) {
809 error(user_context) << "Vulkan: Failed to create compute pipeline ... invalid allocator pointer!\n";
811 }
812
814 {
816 nullptr, // pointer to a structure extending this
817 0, // flags
818 // VkPipelineShaderStageCreatInfo
819 {
821 nullptr, // pointer to a structure extending this
822 0, // flags
823 VK_SHADER_STAGE_COMPUTE_BIT, // compute stage shader
824 shader_module, // shader module
825 pipeline_name, // entry point name
826 specialization_info, // pointer to VkSpecializationInfo struct
827 },
828 pipeline_layout, // pipeline layout
829 0, // base pipeline handle for derived pipeline
830 0 // base pipeline index for derived pipeline
831 };
832
833 VkResult result = vkCreateComputePipelines(allocator->current_device(), 0, 1, &compute_pipeline_info, allocator->callbacks(), compute_pipeline);
834 if (result != VK_SUCCESS) {
835 error(user_context) << "Vulkan: Failed to create compute pipeline! vkCreateComputePipelines returned " << vk_get_error_name(result) << "\n";
837 }
838
840}
841
842int vk_setup_compute_pipeline(void *user_context,
843 VulkanMemoryAllocator *allocator,
844 VulkanShaderBinding *shader_bindings,
845 VulkanDispatchData *dispatch_data,
846 VkShaderModule shader_module,
847 VkPipelineLayout pipeline_layout,
848 VkPipeline *compute_pipeline) {
849
850#ifdef DEBUG_RUNTIME
851 debug(user_context)
852 << " vk_setup_compute_pipeline (user_context: " << user_context << ", "
853 << "entry_point_name: '" << shader_bindings->entry_point_name << "', "
854 << "allocator: " << (void *)allocator << ", "
855 << "shader_bindings: " << (void *)shader_bindings << ", "
856 << "dispatch_data: " << (void *)dispatch_data << ", "
857 << "shader_module: " << (void *)shader_module << ", "
858 << "pipeline_layout: " << (void *)pipeline_layout << ")\n";
859#endif
860
861 if (allocator == nullptr) {
862 error(user_context) << "Vulkan: Failed to setup compute pipeline ... invalid allocator pointer!\n";
864 }
865
866 if (shader_bindings == nullptr) {
867 error(user_context) << "Vulkan: Failed to setup compute pipeline ... invalid shader bindings!\n";
869 }
870
871 if (shader_bindings == nullptr) {
872 error(user_context) << "Vulkan: Failed to setup compute pipeline ... invalid dispatch data!\n";
874 }
875
876 VkResult result = VK_SUCCESS;
877 const char *entry_point_name = shader_bindings->entry_point_name;
878 if (entry_point_name == nullptr) {
879 error(user_context) << "Vulkan: Failed to setup compute pipeline ... missing entry point name!\n";
881 }
882
884 uint32_t dispatch_constant_ids[4] = {0, 0, 0, 0};
885 uint32_t dispatch_constant_values[4] = {0, 0, 0, 0};
886
887 // locate the mapping for overriding any dynamic shared memory allocation sizes
888 if (shader_bindings->shared_memory_allocations_count && dispatch_data->shared_mem_bytes) {
889
893
894 for (uint32_t sm = 0; sm < shader_bindings->shared_memory_allocations_count; sm++) {
896 if (allocation->constant_id == 0) {
897 // static fixed-size allocation
898 static_shared_mem_bytes += allocation->type_size * allocation->array_size;
899 } else {
900 // dynamic allocation
901 if (shared_mem_constant_id > 0) {
902 error(user_context) << "Vulkan: Multiple dynamic shared memory allocations found! Only one is suported!!\n";
904 break;
905 }
906 shared_mem_constant_id = allocation->constant_id;
907 shared_mem_type_size = allocation->type_size;
908 }
909 }
911 debug(user_context) << " pipeline uses " << static_shared_mem_bytes << " bytes of static shared memory\n";
912 debug(user_context) << " dispatch requests " << dispatch_data->shared_mem_bytes << " bytes of shared memory\n";
913 debug(user_context) << " dynamic shared memory " << shared_mem_bytes_avail << " bytes available\n";
914
915 // setup the dynamic array size
918 debug(user_context) << " setting shared memory to " << (uint32_t)dynamic_array_size << " elements "
919 << "(or " << (uint32_t)shared_mem_bytes_avail << " bytes)\n";
920
921 // save the shared mem specialization constant in the first slot
925 }
926 }
927
928 // locate the mapping for overriding any dynamic workgroup local sizes
929 if (shader_bindings->dispatch_data.local_size_binding.constant_id[0] != 0) {
930 for (uint32_t dim = 0; dim < 3; dim++) {
934 }
935 }
936
937 // verify the specialization constants actually exist
938 for (uint32_t dc = 0; dc < dispatch_constant_index; dc++) {
939 const uint32_t invalid_index = uint32_t(-1);
941 for (uint32_t sc = 0; sc < shader_bindings->specialization_constants_count; sc++) {
943 debug(user_context) << " binding specialization constant [" << dispatch_constant_ids[dc] << "] "
944 << "'" << shader_bindings->specialization_constants[sc].constant_name << "' "
945 << " => " << dispatch_constant_values[dc] << "\n";
946 found_index = sc;
947 break;
948 }
949 }
950 if (found_index == invalid_index) {
951 error(user_context) << "Vulkan: Failed to locate dispatch constant index for shader binding!\n";
953 }
954 }
955
956 // don't even attempt to create the pipeline layout if we encountered errors in the shader binding
957 if (result != VK_SUCCESS) {
958 error(user_context) << "Vulkan: Failed to decode shader bindings! " << vk_get_error_name(result) << "\n";
960 }
961
962 // Prepare specialization mapping for all dispatch constants
966 for (uint32_t dc = 0; dc < dispatch_constant_index && dc < 4; dc++) {
968 specialization_map_entries[dc].size = sizeof(uint32_t);
969 specialization_map_entries[dc].offset = dc * sizeof(uint32_t);
971 }
972
973 if (dispatch_constant_count > 0) {
974
975 // Prepare specialization info block for the shader stage
981
982 // Recreate the pipeline with the requested shared memory allocation
983 if (shader_bindings->compute_pipeline) {
984 int error_code = vk_destroy_compute_pipeline(user_context, allocator, shader_bindings->compute_pipeline);
985 if (error_code != halide_error_code_success) {
986 error(user_context) << "Vulkan: Failed to destroy compute pipeline!\n";
988 }
989 shader_bindings->compute_pipeline = {0};
990 }
991
992 int error_code = vk_create_compute_pipeline(user_context, allocator, entry_point_name, shader_module, pipeline_layout, &specialization_info, &(shader_bindings->compute_pipeline));
993 if (error_code != halide_error_code_success) {
994 error(user_context) << "Vulkan: Failed to create compute pipeline!\n";
995 return error_code;
996 }
997
998 } else {
999
1000 // Construct and re-use the fixed pipeline
1001 if (shader_bindings->compute_pipeline == 0) {
1002 int error_code = vk_create_compute_pipeline(user_context, allocator, entry_point_name, shader_module, pipeline_layout, nullptr, &(shader_bindings->compute_pipeline));
1003 if (error_code != halide_error_code_success) {
1004 error(user_context) << "Vulkan: Failed to create compute pipeline!\n";
1005 return error_code;
1006 }
1007 }
1008 }
1009
1011}
1012
1013int vk_destroy_compute_pipeline(void *user_context,
1014 VulkanMemoryAllocator *allocator,
1015 VkPipeline compute_pipeline) {
1016#ifdef DEBUG_RUNTIME
1017 debug(user_context)
1018 << " vk_destroy_compute_pipeline (user_context: " << user_context << ", "
1019 << "allocator: " << (void *)allocator << ", "
1020 << "device: " << (void *)allocator->current_device() << ", "
1021 << "compute_pipeline: " << (void *)compute_pipeline << ")\n";
1022#endif
1023 if (allocator == nullptr) {
1024 error(user_context) << "Vulkan: Failed to destroy compute pipeline ... invalid allocator pointer!\n";
1026 }
1027
1028 vkDestroyPipeline(allocator->current_device(), compute_pipeline, allocator->callbacks());
1030}
1031
1032// --------------------------------------------------------------------------
1033
1035#ifdef DEBUG_RUNTIME
1036 debug(user_context)
1037 << " vk_decode_shader_bindings (user_context: " << user_context << ", "
1038 << "allocator: " << (void *)allocator << ", "
1039 << "module_ptr: " << (void *)module_ptr << ", "
1040 << "module_size: " << module_size << ")\n";
1041
1043#endif
1044
1045 if (allocator == nullptr) {
1046 error(user_context) << "Vulkan: Failed to decode shader bindings ... invalid allocator pointer!\n";
1047 return nullptr;
1048 }
1049
1050 if ((module_ptr == nullptr) || (module_size < (2 * sizeof(uint32_t)))) {
1051 error(user_context) << "Vulkan: Failed to decode shader bindings ... invalid module buffer!\n";
1052 return nullptr;
1053 }
1054
1055 // Decode the sidecar for the module that lists the descriptor sets
1056 // corresponding to each entry point contained in the module.
1057 //
1058 // Construct a shader binding for each entry point that defines all
1059 // the buffers, constants, shared memory, and workgroup sizes
1060 // that are required for execution.
1061 //
1062 // Like the SPIR-V code module, each entry is one word (1x uint32_t).
1063 // Variable length sections are prefixed with their length (ie number of entries).
1064 //
1065 // [0] Header word count (total length of header)
1066 // [1] Number of descriptor sets
1067 // ... For each descriptor set ...
1068 // ... [0] Length of entry point name (padded to nearest word size)
1069 // ....... [*] Entry point string data (padded with null chars)
1070 // ... [1] Number of uniform buffers for this descriptor set
1071 // ... [2] Number of storage buffers for this descriptor set
1072 // ... [3] Number of specialization constants for this descriptor set
1073 // ....... For each specialization constant ...
1074 // ....... [0] Length of constant name string (padded to nearest word size)
1075 // ........... [*] Constant name string data (padded with null chars)
1076 // ....... [1] Constant id (as used in VkSpecializationMapEntry for binding)
1077 // ....... [2] Size of data type (in bytes)
1078 // ... [4] Number of shared memory allocations for this descriptor set
1079 // ....... For each allocation ...
1080 // ....... [0] Length of variable name string (padded to nearest word size)
1081 // ........... [*] Variable name string data (padded with null chars)
1082 // ....... [1] Constant id to use for overriding array size (zero if it is not bound to a specialization constant)
1083 // ....... [2] Size of data type (in bytes)
1084 // ....... [3] Size of array (ie element count)
1085 // ... [4] Dynamic workgroup dimensions bound to specialization constants
1086 // ....... [0] Constant id to use for local_size_x (zero if it was statically declared and not bound to a specialization constant)
1087 // ....... [1] Constant id to use for local_size_y
1088 // ....... [2] Constant id ot use for local_size_z
1089 //
1090 // NOTE: See CodeGen_Vulkan_Dev::SPIRV_Emitter::encode_header() for the encoding
1091 //
1092 // Both vk_decode_shader_bindings() and vk_compile_shader_module() will
1093 // need to be updated if the header encoding ever changes!
1094 //
1096 uint32_t idx = 1; // skip past the header_word_count
1097 uint32_t shader_count = module_ptr[idx++];
1098 if (shader_count < 1) {
1099 error(user_context) << "Vulkan: Failed to decode shader bindings ... no descriptors found!\n";
1100 return nullptr; // no descriptors
1101 }
1102
1103 // allocate an array of shader bindings (one for each entry point in the module)
1104 VkSystemAllocationScope alloc_scope = VkSystemAllocationScope::VK_SYSTEM_ALLOCATION_SCOPE_OBJECT;
1105 size_t shader_bindings_size = shader_count * sizeof(VulkanShaderBinding);
1106 VulkanShaderBinding *shader_bindings = (VulkanShaderBinding *)vk_host_malloc(user_context, shader_bindings_size, 0, alloc_scope, allocator->callbacks());
1107 if (shader_bindings == nullptr) {
1108 error(user_context) << "Vulkan: Failed to allocate shader_bindings! Out of memory!\n";
1109 return nullptr;
1110 }
1111 memset(shader_bindings, 0, shader_bindings_size);
1112
1113 // decode and fill in the shader binding for each entry point
1114 for (uint32_t n = 0; (n < shader_count) && (idx < module_entries); n++) {
1115 halide_debug_assert(user_context, (idx + 8) < module_entries); // should be at least 8 entries
1116
1117 // [0] Length of entry point name (padded to nearest word size)
1119
1120 // [*] Entry point string data (padded with null chars)
1121 const char *entry_point_name = (const char *)(module_ptr + idx); // NOTE: module owns string data
1122 idx += entry_point_name_length; // skip past string data
1123
1124 // [1] Number of uniform buffers for this descriptor set
1125 uint32_t uniform_buffer_count = module_ptr[idx++];
1126
1127 // [2] Number of storage buffers for this descriptor set
1128 uint32_t storage_buffer_count = module_ptr[idx++];
1129
1130 // [3] Number of specialization constants for this descriptor set
1131 uint32_t specialization_constants_count = module_ptr[idx++];
1132
1133 // Decode all specialization constants
1134 VulkanSpecializationConstant *specialization_constants = nullptr;
1135 if (specialization_constants_count > 0) {
1136
1137 // Allocate an array to store the decoded specialization constant data
1138 size_t specialization_constants_size = specialization_constants_count * sizeof(VulkanSpecializationConstant);
1139 specialization_constants = (VulkanSpecializationConstant *)vk_host_malloc(user_context, specialization_constants_size, 0, alloc_scope, allocator->callbacks());
1140 if (specialization_constants == nullptr) {
1141 error(user_context) << "Vulkan: Failed to allocate specialization_constants! Out of memory!\n";
1142 return nullptr;
1143 }
1144 memset(specialization_constants, 0, specialization_constants_size);
1145
1146 // For each specialization constant ...
1147 for (uint32_t sc = 0; sc < specialization_constants_count; sc++) {
1148 halide_debug_assert(user_context, (idx + 4) < module_entries); // should be at least 4 entries
1149
1150 // [0] Length of constant name string (padded to nearest word size)
1152
1153 // [*] Constant name string data (padded with null chars)
1154 const char *constant_name = (const char *)(module_ptr + idx);
1155 specialization_constants[sc].constant_name = constant_name; // NOTE: module owns string data
1156 idx += constant_name_length; // skip past string data
1157
1158 // [1] Constant id (as used in VkSpecializationMapEntry for binding)
1159 specialization_constants[sc].constant_id = module_ptr[idx++];
1160
1161 // [2] Size of data type (in bytes)
1162 specialization_constants[sc].type_size = module_ptr[idx++];
1163 }
1164 }
1165
1166 // [4] Number of shared memory allocations for this descriptor set
1167 uint32_t shared_memory_allocations_count = module_ptr[idx++]; // [3]
1168
1169 // Decode all shared memory allocations ...
1170 VulkanSharedMemoryAllocation *shared_memory_allocations = nullptr;
1171 if (shared_memory_allocations_count > 0) {
1172
1173 // Allocate an array to store the decoded shared memory allocation data
1174 size_t shared_memory_allocations_size = shared_memory_allocations_count * sizeof(VulkanSharedMemoryAllocation);
1175 shared_memory_allocations = (VulkanSharedMemoryAllocation *)vk_host_malloc(user_context, shared_memory_allocations_size, 0, alloc_scope, allocator->callbacks());
1176 if (shared_memory_allocations == nullptr) {
1177 error(user_context) << "Vulkan: Failed to allocate shared_memory_allocations! Out of memory!\n";
1178 return nullptr;
1179 }
1180 memset(shared_memory_allocations, 0, shared_memory_allocations_size);
1181
1182 // For each shared memory allocation ...
1183 for (uint32_t sm = 0; sm < shared_memory_allocations_count && (idx < module_entries); sm++) {
1184 halide_debug_assert(user_context, (idx + 4) < module_entries); // should be at least 4 entries
1185
1186 // [0] Length of variable name string (padded to nearest word size)
1188
1189 // [*] Variable name string data (padded with null chars)
1190 const char *variable_name = (const char *)(module_ptr + idx);
1191 shared_memory_allocations[sm].variable_name = variable_name; // NOTE: module owns string data
1192 idx += variable_name_length; // skip past string data
1193
1194 // [1] Constant id to use for overriding array size
1195 shared_memory_allocations[sm].constant_id = module_ptr[idx++];
1196
1197 // [2] Size of data type (in bytes)
1198 shared_memory_allocations[sm].type_size = module_ptr[idx++];
1199
1200 // [3] Size of array (ie element count)
1201 shared_memory_allocations[sm].array_size = module_ptr[idx++];
1202 }
1203 }
1204
1205 // [4] Dynamic workgroup dimensions bound to specialization constants
1206 halide_debug_assert(user_context, (idx + 3) < module_entries); // should be at least 3 entries
1207 for (uint32_t dim = 0; dim < 3 && (idx < module_entries); dim++) {
1208 shader_bindings[n].dispatch_data.local_size_binding.constant_id[dim] = module_ptr[idx++];
1209 }
1210
1211#ifdef DEBUG_RUNTIME
1212
1213 debug(user_context) << " [" << n << "] '" << (const char *)entry_point_name << "'\n";
1214
1215 debug(user_context) << " uniform_buffer_count=" << uniform_buffer_count << "\n"
1216 << " storage_buffer_count=" << storage_buffer_count << "\n";
1217
1218 debug(user_context) << " specialization_constants_count=" << specialization_constants_count << "\n";
1219 for (uint32_t sc = 0; sc < specialization_constants_count; sc++) {
1220 debug(user_context) << " [" << sc << "] "
1221 << "constant_name='" << (const char *)specialization_constants[sc].constant_name << "' "
1222 << "constant_id=" << specialization_constants[sc].constant_id << " "
1223 << "type_size=" << specialization_constants[sc].type_size << "\n";
1224 }
1225
1226 debug(user_context) << " shared_memory_allocations_count=" << shared_memory_allocations_count << "\n";
1227 for (uint32_t sm = 0; sm < shared_memory_allocations_count; sm++) {
1228 debug(user_context) << " [" << sm << "] "
1229 << "variable_name='" << (const char *)shared_memory_allocations[sm].variable_name << "' "
1230 << "constant_id=" << shared_memory_allocations[sm].constant_id << " "
1231 << "type_size=" << shared_memory_allocations[sm].type_size << " "
1232 << "array_size=" << shared_memory_allocations[sm].array_size << "\n";
1233 }
1234 debug(user_context) << " local_size_binding=[";
1235 for (uint32_t dim = 0; dim < 3 && (idx < module_entries); dim++) {
1236 debug(user_context) << shader_bindings[n].dispatch_data.local_size_binding.constant_id[dim] << " ";
1237 }
1238 debug(user_context) << "]\n";
1239#endif
1240 shader_bindings[n].entry_point_name = entry_point_name; // NOTE: module owns string data
1241 shader_bindings[n].uniform_buffer_count = uniform_buffer_count;
1242 shader_bindings[n].storage_buffer_count = storage_buffer_count;
1243 shader_bindings[n].specialization_constants_count = specialization_constants_count;
1244 shader_bindings[n].specialization_constants = specialization_constants;
1245 shader_bindings[n].shared_memory_allocations_count = shared_memory_allocations_count;
1246 shader_bindings[n].shared_memory_allocations = shared_memory_allocations;
1247 }
1248
1249#ifdef DEBUG_RUNTIME
1250 uint64_t t_after = halide_current_time_ns(user_context);
1251 debug(user_context) << " Time: " << (t_after - t_before) / 1.0e6 << " ms\n";
1252#endif
1253
1254 return shader_bindings;
1255}
1256
1258 const char *ptr, int size) {
1259#ifdef DEBUG_RUNTIME
1260 debug(user_context)
1261 << " vk_compile_shader_module (user_context: " << user_context << ", "
1262 << "allocator: " << (void *)allocator << ", "
1263 << "device: " << (void *)allocator->current_device() << ", "
1264 << "module: " << (void *)ptr << ", "
1265 << "size: " << size << ")\n";
1266
1268#endif
1269
1270 if (allocator == nullptr) {
1271 error(user_context) << "Vulkan: Failed to compile shader modules ... invalid allocator pointer!\n";
1272 return nullptr;
1273 }
1274
1275 if ((ptr == nullptr) || (size <= 0)) {
1276 error(user_context) << "Vulkan: Failed to compile shader modules ... invalid program source buffer!\n";
1277 return nullptr;
1278 }
1279
1280 const uint32_t *module_ptr = (const uint32_t *)ptr;
1281 const uint32_t module_size = (const uint32_t)size;
1282
1283 halide_debug_assert(user_context, module_ptr != nullptr);
1284 halide_debug_assert(user_context, module_size >= (2 * sizeof(uint32_t)));
1285
1287 uint32_t shader_count = module_ptr[1];
1289
1290 // skip past the preamble header to the start of the SPIR-V binary
1292 size_t binary_size = (size - header_size);
1293
1294#ifdef DEBUG_RUNTIME
1295 debug(user_context) << "Vulkan: Decoding module ("
1296 << "module_ptr: " << (void *)module_ptr << ", "
1297 << "header_word_count: " << header_word_count << ", "
1298 << "header_size: " << header_size << ", "
1299 << "binar_ptr: " << (void *)binary_ptr << ", "
1300 << "binary_size: " << (uint32_t)binary_size << ")\n";
1301#endif
1302
1305 nullptr, // pointer to structure extending this
1306 0, // flags (curently unused)
1307 (size_t)binary_size, // code size in bytes
1308 (const uint32_t *)binary_ptr // source
1309 };
1310
1311 VkSystemAllocationScope alloc_scope = VkSystemAllocationScope::VK_SYSTEM_ALLOCATION_SCOPE_OBJECT;
1313 if (cache_entry == nullptr) {
1314 error(user_context) << "Vulkan: Failed to allocate compilation cache entry! Out of memory!\n";
1315 return nullptr;
1316 }
1318
1319 // decode the entry point data and extract the shader bindings
1321 if (decoded_bindings == nullptr) {
1322 error(user_context) << "Vulkan: Failed to decode shader bindings!\n";
1323 return nullptr;
1324 }
1325
1326 // save the shader bindings in the cache entry
1327 cache_entry->shader_bindings = decoded_bindings;
1328 cache_entry->shader_count = shader_count;
1329
1330 VkResult result = vkCreateShaderModule(allocator->current_device(), &shader_info, allocator->callbacks(), &cache_entry->shader_module);
1331 if ((result != VK_SUCCESS)) {
1332 error(user_context) << "Vulkan: vkCreateShaderModule Failed! Error returned: " << vk_get_error_name(result) << "\n";
1333 vk_host_free(user_context, cache_entry->shader_bindings, allocator->callbacks());
1334 vk_host_free(user_context, cache_entry, allocator->callbacks());
1335 return nullptr;
1336 }
1337
1338 // allocate an array for storing the descriptor set layouts
1339 if (cache_entry->shader_count) {
1340 cache_entry->descriptor_set_layouts = (VkDescriptorSetLayout *)vk_host_malloc(user_context, cache_entry->shader_count * sizeof(VkDescriptorSetLayout), 0, alloc_scope, allocator->callbacks());
1341 if (cache_entry->descriptor_set_layouts == nullptr) {
1342 error(user_context) << "Vulkan: Failed to allocate descriptor set layouts for cache entry! Out of memory!\n";
1343 return nullptr;
1344 }
1345 memset(cache_entry->descriptor_set_layouts, 0, cache_entry->shader_count * sizeof(VkDescriptorSetLayout));
1346 }
1347
1348#ifdef DEBUG_RUNTIME
1349 uint64_t t_after = halide_current_time_ns(user_context);
1350 debug(user_context) << " Time: " << (t_after - t_before) / 1.0e6 << " ms\n";
1351#endif
1352
1353 return cache_entry;
1354}
1355
1356int vk_destroy_shader_modules(void *user_context, VulkanMemoryAllocator *allocator) {
1357
1358#ifdef DEBUG_RUNTIME
1359 debug(user_context)
1360 << " vk_destroy_shader_modules (user_context: " << user_context << ", "
1361 << "allocator: " << (void *)allocator << ", "
1362 << "device: " << (void *)allocator->current_device() << ")\n";
1363
1365#endif
1366
1367 if (allocator == nullptr) {
1368 error(user_context) << "Vulkan: Failed to destroy shader modules ... invalid allocator pointer!\n";
1370 }
1371
1372 // Functor to match compilation cache destruction call with scoped params
1373 struct DestroyShaderModule {
1374 void *user_context = nullptr;
1375 VulkanMemoryAllocator *allocator = nullptr;
1376
1377 DestroyShaderModule(void *ctx, VulkanMemoryAllocator *allocator)
1378 : user_context(ctx), allocator(allocator) {
1379 }
1380
1381 void operator()(VulkanCompilationCacheEntry *cache_entry) {
1382 if (cache_entry != nullptr) {
1383 if (cache_entry->descriptor_set_layouts) {
1384 for (uint32_t n = 0; n < cache_entry->shader_count; n++) {
1385 debug(user_context) << " destroying descriptor set layout [" << n << "] " << cache_entry->shader_bindings[n].entry_point_name << "\n";
1386 vk_destroy_descriptor_set_layout(user_context, allocator, cache_entry->descriptor_set_layouts[n]);
1387 cache_entry->descriptor_set_layouts[n] = {0};
1388 }
1389 vk_host_free(user_context, cache_entry->descriptor_set_layouts, allocator->callbacks());
1390 cache_entry->descriptor_set_layouts = nullptr;
1391 }
1392 if (cache_entry->pipeline_layout) {
1393 debug(user_context) << " destroying pipeline layout " << (void *)cache_entry->pipeline_layout << "\n";
1394 vk_destroy_pipeline_layout(user_context, allocator, cache_entry->pipeline_layout);
1395 cache_entry->pipeline_layout = {0};
1396 }
1397 if (cache_entry->shader_bindings) {
1398 for (uint32_t n = 0; n < cache_entry->shader_count; n++) {
1399 if (cache_entry->shader_bindings[n].args_region) {
1400 vk_destroy_scalar_uniform_buffer(user_context, allocator, cache_entry->shader_bindings[n].args_region);
1401 cache_entry->shader_bindings[n].args_region = nullptr;
1402 }
1403 if (cache_entry->shader_bindings[n].descriptor_pool) {
1404 vk_destroy_descriptor_pool(user_context, allocator, cache_entry->shader_bindings[n].descriptor_pool);
1405 cache_entry->shader_bindings[n].descriptor_pool = {0};
1406 }
1407 if (cache_entry->shader_bindings[n].specialization_constants) {
1408 vk_host_free(user_context, cache_entry->shader_bindings[n].specialization_constants, allocator->callbacks());
1409 cache_entry->shader_bindings[n].specialization_constants = nullptr;
1410 }
1411 if (cache_entry->shader_bindings[n].shared_memory_allocations) {
1412 vk_host_free(user_context, cache_entry->shader_bindings[n].shared_memory_allocations, allocator->callbacks());
1413 cache_entry->shader_bindings[n].shared_memory_allocations = nullptr;
1414 }
1415 if (cache_entry->shader_bindings[n].compute_pipeline) {
1416 vk_destroy_compute_pipeline(user_context, allocator, cache_entry->shader_bindings[n].compute_pipeline);
1417 cache_entry->shader_bindings[n].compute_pipeline = {0};
1418 }
1419 }
1420
1421 vk_host_free(user_context, cache_entry->shader_bindings, allocator->callbacks());
1422 cache_entry->shader_bindings = nullptr;
1423 }
1424 if (cache_entry->shader_module) {
1425 debug(user_context) << " . destroying shader module " << (void *)cache_entry->shader_module << "\n";
1426 vkDestroyShaderModule(allocator->current_device(), cache_entry->shader_module, allocator->callbacks());
1427 cache_entry->shader_module = {0};
1428 }
1429 cache_entry->shader_count = 0;
1430 vk_host_free(user_context, cache_entry, allocator->callbacks());
1431 cache_entry = nullptr;
1432 }
1433 }
1434 };
1435
1436 DestroyShaderModule module_destructor(user_context, allocator);
1437 compilation_cache.delete_context(user_context, allocator->current_device(), module_destructor);
1438
1439#ifdef DEBUG_RUNTIME
1440 uint64_t t_after = halide_current_time_ns(user_context);
1441 debug(user_context) << " Time: " << (t_after - t_before) / 1.0e6 << " ms\n";
1442#endif
1444}
1445
1446// --------------------------------------------------------------------------
1447
1450 int d, bool from_host, bool to_host) {
1451 if (d == 0) {
1452
1453 if ((!from_host && to_host) ||
1454 (from_host && !to_host) ||
1455 (!from_host && !to_host)) {
1456
1457 VkBufferCopy buffer_copy = {
1458 c.src_begin + src_offset, // srcOffset
1459 dst_offset, // dstOffset
1460 c.chunk_size // size
1461 };
1462
1463 VkBuffer *src_buffer = reinterpret_cast<VkBuffer *>(c.src);
1464 VkBuffer *dst_buffer = reinterpret_cast<VkBuffer *>(c.dst);
1465 if (!src_buffer || !dst_buffer) {
1466 error(user_context) << "Vulkan: Failed to retrieve buffer for device memory!\n";
1468 }
1469
1471
1472 } else if ((c.dst + dst_offset) != (c.src + src_offset)) {
1473 // Could reach here if a user called directly into the
1474 // Vulkan API for a device->host copy on a source buffer
1475 // with device_dirty = false.
1476 memcpy((void *)(c.dst + dst_offset), (void *)(c.src + src_offset), c.chunk_size);
1477 }
1478 } else {
1479 // TODO: deal with negative strides. Currently the code in
1480 // device_buffer_utils.h does not do so either.
1481 uint64_t src_off = 0, dst_off = 0;
1482 for (uint64_t i = 0; i < c.extent[d - 1]; i++) {
1483 int err = vk_do_multidimensional_copy(user_context, command_buffer, c,
1486 d - 1, from_host, to_host);
1487 dst_off += c.dst_stride_bytes[d - 1];
1488 src_off += c.src_stride_bytes[d - 1];
1489 if (err) {
1490 return err;
1491 }
1492 }
1493 }
1495}
1496
1497int vk_device_crop_from_offset(void *user_context,
1498 const struct halide_buffer_t *src,
1499 int64_t offset,
1500 struct halide_buffer_t *dst) {
1501
1502 VulkanContext ctx(user_context);
1503 if (ctx.error != halide_error_code_success) {
1504 error(user_context) << "Vulkan: Failed to acquire context!\n";
1505 return ctx.error;
1506 }
1507
1508#ifdef DEBUG_RUNTIME
1510#endif
1511
1512 if (offset < 0) {
1513 error(user_context) << "Vulkan: Invalid offset for device crop!\n";
1515 }
1516
1517 // get the allocated region for the device
1518 MemoryRegion *device_region = reinterpret_cast<MemoryRegion *>(src->device);
1519 if (device_region == nullptr) {
1520 error(user_context) << "Vulkan: Failed to crop region! Invalide device region!\n";
1522 }
1523
1524 // create the croppeg region from the allocated region
1525 MemoryRegion *cropped_region = ctx.allocator->create_crop(user_context, device_region, (uint64_t)offset);
1526 if ((cropped_region == nullptr) || (cropped_region->handle == nullptr)) {
1527 error(user_context) << "Vulkan: Failed to crop region! Unable to create memory region!\n";
1529 }
1530
1531 // update the destination to the cropped region
1532 dst->device = (uint64_t)cropped_region;
1533 dst->device_interface = src->device_interface;
1534
1535#ifdef DEBUG_RUNTIME
1536 uint64_t t_after = halide_current_time_ns(user_context);
1537 debug(user_context) << " Time: " << (t_after - t_before) / 1.0e6 << " ms\n";
1538#endif
1539
1541}
1542
1543// --------------------------------------------------------------------------
1544
1545} // namespace
1546} // namespace Vulkan
1547} // namespace Internal
1548} // namespace Runtime
1549} // namespace Halide
1550
1551#endif // HALIDE_RUNTIME_VULKAN_RESOURCES_H
bool halide_can_reuse_device_allocations(void *user_context)
Determines whether on device_free the memory is returned immediately to the device API,...
@ halide_error_code_internal_error
There is a bug in the Halide compiler.
@ halide_error_code_generic_error
An uncategorized error occurred.
@ halide_error_code_device_crop_failed
Cropping/slicing a buffer failed for some other reason.
@ halide_error_code_success
There was no error.
Vulkan Memory Allocator class interface for managing large memory requests stored as contiguous block...
int reclaim(void *user_context, MemoryRegion *region)
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
int release(void *user_context, MemoryRegion *region)
int unmap(void *user_context, MemoryRegion *region)
MemoryRegion * owner_of(void *user_context, MemoryRegion *region)
const VkAllocationCallbacks * callbacks() const
void * map(void *user_context, MemoryRegion *region)
VKAPI_ATTR void VKAPI_CALL vkCmdDispatch(VkCommandBuffer commandBuffer, uint32_t groupCountX, uint32_t groupCountY, uint32_t groupCountZ)
@ VK_COMMAND_BUFFER_LEVEL_PRIMARY
VKAPI_ATTR void VKAPI_CALL vkDestroyShaderModule(VkDevice device, VkShaderModule shaderModule, const VkAllocationCallbacks *pAllocator)
@ VK_SHADER_STAGE_COMPUTE_BIT
VKAPI_ATTR void VKAPI_CALL vkFreeCommandBuffers(VkDevice device, VkCommandPool commandPool, uint32_t commandBufferCount, const VkCommandBuffer *pCommandBuffers)
VkSystemAllocationScope
VKAPI_ATTR void VKAPI_CALL vkCmdBindPipeline(VkCommandBuffer commandBuffer, VkPipelineBindPoint pipelineBindPoint, VkPipeline pipeline)
VKAPI_ATTR VkResult VKAPI_CALL vkCreateDescriptorPool(VkDevice device, const VkDescriptorPoolCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkDescriptorPool *pDescriptorPool)
VKAPI_ATTR VkResult VKAPI_CALL vkAllocateDescriptorSets(VkDevice device, const VkDescriptorSetAllocateInfo *pAllocateInfo, VkDescriptorSet *pDescriptorSets)
uint64_t VkDeviceSize
Definition mini_vulkan.h:71
VKAPI_ATTR VkResult VKAPI_CALL vkCreateDescriptorSetLayout(VkDevice device, const VkDescriptorSetLayoutCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkDescriptorSetLayout *pSetLayout)
@ VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT
@ VK_PIPELINE_BIND_POINT_COMPUTE
VKAPI_ATTR void VKAPI_CALL vkCmdCopyBuffer(VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkBuffer dstBuffer, uint32_t regionCount, const VkBufferCopy *pRegions)
VKAPI_ATTR void VKAPI_CALL vkDestroyCommandPool(VkDevice device, VkCommandPool commandPool, const VkAllocationCallbacks *pAllocator)
VKAPI_ATTR VkResult VKAPI_CALL vkCreateCommandPool(VkDevice device, const VkCommandPoolCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkCommandPool *pCommandPool)
VKAPI_ATTR void VKAPI_CALL vkDestroyPipeline(VkDevice device, VkPipeline pipeline, const VkAllocationCallbacks *pAllocator)
VKAPI_ATTR VkResult VKAPI_CALL vkCreateComputePipelines(VkDevice device, VkPipelineCache pipelineCache, uint32_t createInfoCount, const VkComputePipelineCreateInfo *pCreateInfos, const VkAllocationCallbacks *pAllocator, VkPipeline *pPipelines)
VKAPI_ATTR VkResult VKAPI_CALL vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo *pSubmits, VkFence fence)
VKAPI_ATTR void VKAPI_CALL vkCmdBindDescriptorSets(VkCommandBuffer commandBuffer, VkPipelineBindPoint pipelineBindPoint, VkPipelineLayout layout, uint32_t firstSet, uint32_t descriptorSetCount, const VkDescriptorSet *pDescriptorSets, uint32_t dynamicOffsetCount, const uint32_t *pDynamicOffsets)
#define VK_WHOLE_SIZE
VKAPI_ATTR VkResult VKAPI_CALL vkAllocateCommandBuffers(VkDevice device, const VkCommandBufferAllocateInfo *pAllocateInfo, VkCommandBuffer *pCommandBuffers)
VkResult
@ VK_SUCCESS
@ VK_ERROR_INITIALIZATION_FAILED
@ VK_ERROR_TOO_MANY_OBJECTS
@ VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER
@ VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
VKAPI_ATTR void VKAPI_CALL vkDestroyPipelineLayout(VkDevice device, VkPipelineLayout pipelineLayout, const VkAllocationCallbacks *pAllocator)
VKAPI_ATTR VkResult VKAPI_CALL vkCreateShaderModule(VkDevice device, const VkShaderModuleCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkShaderModule *pShaderModule)
VKAPI_ATTR void VKAPI_CALL vkUpdateDescriptorSets(VkDevice device, uint32_t descriptorWriteCount, const VkWriteDescriptorSet *pDescriptorWrites, uint32_t descriptorCopyCount, const VkCopyDescriptorSet *pDescriptorCopies)
@ VK_COMMAND_POOL_CREATE_TRANSIENT_BIT
VKAPI_ATTR VkResult VKAPI_CALL vkEndCommandBuffer(VkCommandBuffer commandBuffer)
VKAPI_ATTR void VKAPI_CALL vkDestroyDescriptorPool(VkDevice device, VkDescriptorPool descriptorPool, const VkAllocationCallbacks *pAllocator)
VKAPI_ATTR void VKAPI_CALL vkDestroyDescriptorSetLayout(VkDevice device, VkDescriptorSetLayout descriptorSetLayout, const VkAllocationCallbacks *pAllocator)
VKAPI_ATTR VkResult VKAPI_CALL vkCreatePipelineLayout(VkDevice device, const VkPipelineLayoutCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkPipelineLayout *pPipelineLayout)
@ VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO
@ VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO
@ VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO
@ VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO
@ VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET
@ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO
@ VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO
@ VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO
@ VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO
@ VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO
@ VK_STRUCTURE_TYPE_SUBMIT_INFO
@ VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO
VKAPI_ATTR VkResult VKAPI_CALL vkBeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBeginInfo *pBeginInfo)
WEAK Halide::Internal::GPUCompilationCache< VkDevice, VulkanCompilationCacheEntry * > compilation_cache
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Expr cast(Expr a)
Cast an expression to the halide type corresponding to the C++ type T.
Definition IROperator.h:364
unsigned __INT64_TYPE__ uint64_t
signed __INT64_TYPE__ int64_t
#define halide_debug_assert(user_context, cond)
halide_debug_assert() is like halide_assert(), but only expands into a check when DEBUG_RUNTIME is de...
unsigned __INT8_TYPE__ uint8_t
void * memcpy(void *s1, const void *s2, size_t n)
__SIZE_TYPE__ size_t
void * memset(void *s, int val, size_t n)
unsigned __INT32_TYPE__ uint32_t
#define halide_abort_if_false(user_context, cond)
WEAK int64_t halide_current_time_ns(void *user_context)
signed __INT8_TYPE__ int8_t
#define WEAK
VkDescriptorSetLayout * descriptor_set_layouts
uint32_t shader_count
VulkanShaderBinding * shader_bindings
VkPipelineLayout pipeline_layout
VkShaderModule shader_module
The raw representation of an image passed around by generated Halide code.
uint64_t device
A device-handle for e.g.
const struct halide_device_interface_t * device_interface
The interface used to interpret the above handle.