Halide 17.0.2
Halide compiler and libraries
Loading...
Searching...
No Matches
vulkan_memory.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_VULKAN_MEMORY_H
2#define HALIDE_RUNTIME_VULKAN_MEMORY_H
3
6#include "vulkan_internal.h"
7
8// Uncomment to enable verbose memory allocation debugging
9// #define HL_VK_DEBUG_MEM 1
10
11namespace Halide {
12namespace Runtime {
13namespace Internal {
14namespace Vulkan {
15
16// --------------------------------------------------------------------------
17
18// Enable external client to override Vulkan allocation callbacks (if they so desire)
20WEAK const VkAllocationCallbacks *custom_allocation_callbacks = nullptr; // nullptr => use Vulkan runtime implementation
21
22// --------------------------------------------------------------------------
23
24// Runtime configuration parameters to adjust the behaviour of the block allocator
26 size_t maximum_pool_size = 0; //< Maximum number of bytes to allocate for the entire pool (including all blocks). Specified in bytes. Zero means no constraint
27 size_t minimum_block_size = 32 * 1024 * 1024; //< Default block size is 32MB
28 size_t maximum_block_size = 0; //< Specified in bytes. Zero means no constraint
29 size_t maximum_block_count = 0; //< Maximum number of blocks to allocate. Zero means no constraint
30 size_t nearest_multiple = 32; //< Always round up the requested region sizes to the given integer value. Zero means no constraint
31};
33
34// --------------------------------------------------------------------------
35
36/** Vulkan Memory Allocator class interface for managing large
37 * memory requests stored as contiguous blocks of memory, which
38 * are then sub-allocated into smaller regions of
39 * memory to avoid the excessive cost of vkAllocate and the limited
40 * number of available allocation calls through the API.
41 */
43public:
44 // disable copy constructors and assignment
47
48 // disable non-factory constrction
51
52 // Factory methods for creation / destruction
53 static VulkanMemoryAllocator *create(void *user_context, const VulkanMemoryConfig &config,
56 const VkAllocationCallbacks *alloc_callbacks = nullptr);
57
58 static int destroy(void *user_context, VulkanMemoryAllocator *allocator);
59
60 // Public interface methods
61 MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
62 int conform(void *user_context, MemoryRequest *request); //< conforms the given memory request into one that can be allocated
63 int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
64 int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
65 int retain(void *user_context, MemoryRegion *region); //< retain the region and increase its use count
66 bool collect(void *user_context); //< returns true if any blocks were removed
67 int release(void *user_context);
68 int destroy(void *user_context);
69
70 void *map(void *user_context, MemoryRegion *region);
71 int unmap(void *user_context, MemoryRegion *region);
72 MemoryRegion *create_crop(void *user_context, MemoryRegion *region, uint64_t offset);
73 int destroy_crop(void *user_context, MemoryRegion *region);
74 MemoryRegion *owner_of(void *user_context, MemoryRegion *region);
75
77 return this->device;
78 }
80 return this->physical_device;
81 }
83 return this->alloc_callbacks;
84 }
85
86 static const VulkanMemoryConfig &default_config();
87
88 static int allocate_block(void *instance_ptr, MemoryBlock *block);
89 static int deallocate_block(void *instance_ptr, MemoryBlock *block);
91
92 static int allocate_region(void *instance_ptr, MemoryRegion *region);
93 static int deallocate_region(void *instance_ptr, MemoryRegion *region);
95
96 size_t bytes_allocated_for_blocks() const;
97 size_t blocks_allocated() const;
98
99 size_t bytes_allocated_for_regions() const;
100 size_t regions_allocated() const;
101
102private:
103 static constexpr uint32_t invalid_usage_flags = uint32_t(-1);
104 static constexpr uint32_t invalid_memory_type = uint32_t(VK_MAX_MEMORY_TYPES);
105
106 // Initializes a new instance
107 int initialize(void *user_context, const VulkanMemoryConfig &config,
110 const VkAllocationCallbacks *alloc_callbacks = nullptr);
111
112 uint32_t select_memory_usage(void *user_context, MemoryProperties properties) const;
113
114 uint32_t select_memory_type(void *user_context,
115 VkPhysicalDevice physical_device,
116 MemoryProperties properties,
118
119 int lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements);
120
121 size_t block_byte_count = 0;
122 size_t block_count = 0;
123 size_t region_byte_count = 0;
124 size_t region_count = 0;
125 void *owner_context = nullptr;
126 VulkanMemoryConfig config;
127 VkDevice device = nullptr;
128 VkPhysicalDevice physical_device = nullptr;
129 VkPhysicalDeviceLimits physical_device_limits = {};
130 const VkAllocationCallbacks *alloc_callbacks = nullptr;
131 BlockAllocator *block_allocator = nullptr;
132};
133
137 const VkAllocationCallbacks *alloc_callbacks) {
138
139 if (system_allocator.allocate == nullptr) {
140 error(user_context) << "VulkanBlockAllocator: Unable to create instance! Missing system allocator interface!\n";
141 return nullptr;
142 }
143
144 VulkanMemoryAllocator *result = reinterpret_cast<VulkanMemoryAllocator *>(
145 system_allocator.allocate(user_context, sizeof(VulkanMemoryAllocator)));
146
147 if (result == nullptr) {
148 error(user_context) << "VulkanMemoryAllocator: Failed to create instance! Out of memory!\n";
149 return nullptr; // caller must handle error case for out-of-memory
150 }
151
152 result->initialize(user_context, cfg, dev, phys_dev, system_allocator, alloc_callbacks);
153 return result;
154}
155
156int VulkanMemoryAllocator::destroy(void *user_context, VulkanMemoryAllocator *instance) {
157 if (instance == nullptr) {
158 error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Invalide instance pointer!\n";
160 }
161 const BlockAllocator::MemoryAllocators &allocators = instance->block_allocator->current_allocators();
162 instance->destroy(user_context);
163 BlockAllocator::destroy(user_context, instance->block_allocator);
164 if (allocators.system.deallocate == nullptr) {
165 error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Missing system allocator interface!\n";
167 }
168 allocators.system.deallocate(user_context, instance);
170}
171
172int VulkanMemoryAllocator::initialize(void *user_context,
175 const VkAllocationCallbacks *callbacks) {
176
177 owner_context = user_context;
178 config = cfg;
179 device = dev;
180 physical_device = phys_dev;
181 alloc_callbacks = callbacks;
182 region_count = 0;
183 region_byte_count = 0;
184 block_count = 0;
185 block_byte_count = 0;
187 allocators.system = system_allocator;
191 block_allocator_config.maximum_pool_size = cfg.maximum_pool_size;
192 block_allocator_config.maximum_block_count = cfg.maximum_block_count;
193 block_allocator_config.maximum_block_size = cfg.maximum_block_size;
194 block_allocator_config.minimum_block_size = cfg.minimum_block_size;
195 block_allocator_config.nearest_multiple = cfg.nearest_multiple;
196 block_allocator = BlockAllocator::create(user_context, block_allocator_config, allocators);
197 if (block_allocator == nullptr) {
198 error(user_context) << "VulkanMemoryAllocator: Failed to create BlockAllocator! Out of memory?!\n";
200 }
201
202 // get the physical device properties to determine limits and allocation requirements
204 memset(&physical_device_limits, 0, sizeof(VkPhysicalDeviceLimits));
206 memcpy(&physical_device_limits, &(physical_device_properties.limits), sizeof(VkPhysicalDeviceLimits));
208}
209
211#if defined(HL_VK_DEBUG_MEM)
212 debug(nullptr) << "VulkanMemoryAllocator: Reserving memory ("
213 << "user_context=" << user_context << " "
214 << "block_allocator=" << (void *)(block_allocator) << " "
215 << "request_size=" << (uint32_t)(request.size) << " "
216 << "device=" << (void *)(device) << " "
217 << "physical_device=" << (void *)(physical_device) << ") ...\n";
218#endif
219
220 if ((device == nullptr) || (physical_device == nullptr)) {
221 error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid device handle!\n";
222 return nullptr;
223 }
224
225 if (block_allocator == nullptr) {
226 error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid block allocator!\n";
227 return nullptr;
228 }
229
230 return block_allocator->reserve(this, request);
231}
232
233void *VulkanMemoryAllocator::map(void *user_context, MemoryRegion *region) {
234#if defined(HL_VK_DEBUG_MEM)
235 debug(nullptr) << "VulkanMemoryAllocator: Mapping region ("
236 << "user_context=" << user_context << " "
237 << "device=" << (void *)(device) << " "
238 << "physical_device=" << (void *)(physical_device) << " "
239 << "region=" << (void *)(region) << " "
240 << "region_size=" << (uint32_t)region->size << " "
241 << "region_offset=" << (uint32_t)region->offset << " "
242 << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
243#endif
244 if ((device == nullptr) || (physical_device == nullptr)) {
245 error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid device handle!\n";
246 return nullptr;
247 }
248
249 if (block_allocator == nullptr) {
250 error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid block allocator!\n";
251 return nullptr;
252 }
253
254 MemoryRegion *owner = owner_of(user_context, region);
256 if (region_allocator == nullptr) {
257 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid region allocator handle!\n";
258 return nullptr; // NOTE: caller must handle nullptr
259 }
260
261 BlockResource *block_resource = region_allocator->block_resource();
262 if (block_resource == nullptr) {
263 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid block resource handle!\n";
264 return nullptr; // NOTE: caller must handle nullptr
265 }
266
267 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
268 if (device_memory == nullptr) {
269 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid device memory handle!\n";
270 return nullptr; // NOTE: caller must handle nullptr
271 }
272
273 void *mapped_ptr = nullptr;
275 VkDeviceSize memory_size = region->size - region->range.tail_offset - region->range.head_offset;
276 if (((double)region->size - (double)region->range.tail_offset - (double)region->range.head_offset) <= 0.0) {
277 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid memory range !\n";
278 return nullptr;
279 }
280#if defined(HL_VK_DEBUG_MEM)
281 debug(nullptr) << "VulkanMemoryAllocator: MapMemory ("
282 << "user_context=" << user_context << "\n"
283 << " region_size=" << (uint32_t)region->size << "\n"
284 << " region_offset=" << (uint32_t)region->offset << "\n"
285 << " region_range.head_offset=" << (uint32_t)region->range.head_offset << "\n"
286 << " region_range.tail_offset=" << (uint32_t)region->range.tail_offset << "\n"
287 << " memory_offset=" << (uint32_t)memory_offset << "\n"
288 << " memory_size=" << (uint32_t)memory_size << "\n)\n";
289#endif
290 VkResult result = vkMapMemory(device, *device_memory, memory_offset, memory_size, 0, (void **)(&mapped_ptr));
291 if (result != VK_SUCCESS) {
292 error(user_context) << "VulkanMemoryAllocator: Mapping region failed! vkMapMemory returned error code: " << vk_get_error_name(result) << "\n";
293 return nullptr;
294 }
295
296 return mapped_ptr;
297}
298
299int VulkanMemoryAllocator::unmap(void *user_context, MemoryRegion *region) {
300#if defined(HL_VK_DEBUG_MEM)
301 debug(nullptr) << "VulkanMemoryAllocator: Unmapping region ("
302 << "user_context=" << user_context << " "
303 << "device=" << (void *)(device) << " "
304 << "physical_device=" << (void *)(physical_device) << " "
305 << "region=" << (void *)(region) << " "
306 << "region_size=" << (uint32_t)region->size << " "
307 << "region_offset=" << (uint32_t)region->offset << " "
308 << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
309#endif
310 if ((device == nullptr) || (physical_device == nullptr)) {
311 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device handle!\n";
313 }
314
315 MemoryRegion *owner = owner_of(user_context, region);
317 if (region_allocator == nullptr) {
318 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
320 }
321
322 BlockResource *block_resource = region_allocator->block_resource();
323 if (block_resource == nullptr) {
324 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid block resource handle!\n";
326 }
327
328 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
329 if (device_memory == nullptr) {
330 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device memory handle!\n";
332 }
333
336}
337
339#if defined(HL_VK_DEBUG_MEM)
340 debug(nullptr) << "VulkanMemoryAllocator: Cropping region ("
341 << "user_context=" << user_context << " "
342 << "device=" << (void *)(device) << " "
343 << "physical_device=" << (void *)(physical_device) << " "
344 << "region=" << (void *)(region) << " "
345 << "region_size=" << (uint32_t)region->size << " "
346 << "region_offset=" << (uint32_t)region->offset << " "
347 << "crop_offset=" << (int64_t)offset << ") ...\n";
348#endif
349 if ((device == nullptr) || (physical_device == nullptr)) {
350 error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Invalid device handle!\n";
351 return nullptr;
352 }
353
354 MemoryRegion *owner = owner_of(user_context, region);
356 if (region_allocator == nullptr) {
357 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
358 return nullptr; // NOTE: caller must handle nullptr
359 }
360
361 // increment usage count
362 int error_code = region_allocator->retain(this, owner);
363 if (error_code != halide_error_code_success) {
364 error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Failed to retain memory region!\n";
365 return nullptr; // NOTE: caller must handle nullptr
366 }
367
368 // create a new region to return, and copy all the other region's properties
369 const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
370 if (allocators.system.allocate == nullptr) {
371 error(user_context) << "VulkanMemoryAllocator: Unable to create crop! Missing system allocator interface!\n";
372 return nullptr;
373 }
374
375 MemoryRegion *memory_region = reinterpret_cast<MemoryRegion *>(
376 allocators.system.allocate(user_context, sizeof(MemoryRegion)));
377
378 if (memory_region == nullptr) {
379 error(user_context) << "VulkanMemoryAllocator: Failed to allocate memory region! Out of memory!\n";
380 return nullptr;
381 }
383
384 // point the handle to the owner of the allocated region, and update the head offset
385 memory_region->is_owner = false;
386 memory_region->handle = (void *)owner;
387 memory_region->range.head_offset = owner->range.head_offset + offset;
388 return memory_region;
389}
390
391int VulkanMemoryAllocator::destroy_crop(void *user_context, MemoryRegion *region) {
392 if (region == nullptr) {
393 error(user_context) << "VulkanMemoryAllocator: Failed to destroy crop! Invalid memory region!\n";
395 }
396
397 MemoryRegion *owner = owner_of(user_context, region);
399 if (region_allocator == nullptr) {
400 error(user_context) << "VulkanMemoryAllocator: Unable to destroy crop region! Invalid region allocator handle!\n";
402 }
403
404 // decrement usage count
405 int error_code = region_allocator->release(this, owner);
406 if (error_code != halide_error_code_success) {
407 error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Region allocator failed to release memory region!\n";
408 return error_code;
409 }
410
411 // discard the copied region struct
412 const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
413 if (allocators.system.deallocate == nullptr) {
414 error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Missing system allocator interface!\n";
416 }
417 allocators.system.deallocate(user_context, region);
419}
420
422 if (region->is_owner) {
423 return region;
424 } else {
425 // If this is a cropped region, use the handle to retrieve the owner of the allocation
426 return reinterpret_cast<MemoryRegion *>(region->handle);
427 }
428}
429
430int VulkanMemoryAllocator::release(void *user_context, MemoryRegion *region) {
431#if defined(HL_VK_DEBUG_MEM)
432 debug(nullptr) << "VulkanMemoryAllocator: Releasing region ("
433 << "user_context=" << user_context << " "
434 << "region=" << (void *)(region) << " "
435 << "size=" << (uint32_t)region->size << " "
436 << "offset=" << (uint32_t)region->offset << ") ...\n";
437#endif
438 if ((device == nullptr) || (physical_device == nullptr)) {
439 error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid device handle!\n";
441 }
442 if (block_allocator == nullptr) {
443 error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid block allocator!\n";
445 }
446 return block_allocator->release(this, region);
447}
448
449int VulkanMemoryAllocator::reclaim(void *user_context, MemoryRegion *region) {
450#if defined(HL_VK_DEBUG_MEM)
451 debug(nullptr) << "VulkanMemoryAllocator: Reclaiming region ("
452 << "user_context=" << user_context << " "
453 << "region=" << (void *)(region) << " "
454 << "size=" << (uint32_t)region->size << " "
455 << "offset=" << (uint32_t)region->offset << ") ...\n";
456#endif
457 if ((device == nullptr) || (physical_device == nullptr)) {
458 error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid device handle!\n";
460 }
461 if (block_allocator == nullptr) {
462 error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid block allocator!\n";
464 }
465 return block_allocator->reclaim(this, region);
466}
467
468int VulkanMemoryAllocator::retain(void *user_context, MemoryRegion *region) {
469#if defined(HL_VK_DEBUG_MEM)
470 debug(nullptr) << "VulkanMemoryAllocator: Retaining region ("
471 << "user_context=" << user_context << " "
472 << "region=" << (void *)(region) << " "
473 << "size=" << (uint32_t)region->size << " "
474 << "offset=" << (uint32_t)region->offset << ") ...\n";
475#endif
476 if ((device == nullptr) || (physical_device == nullptr)) {
477 error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid device handle!\n";
479 }
480 if (block_allocator == nullptr) {
481 error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid block allocator!\n";
483 }
484 return block_allocator->retain(this, region);
485}
486
487bool VulkanMemoryAllocator::collect(void *user_context) {
488#if defined(HL_VK_DEBUG_MEM)
489 debug(nullptr) << "VulkanMemoryAllocator: Collecting unused memory ("
490 << "user_context=" << user_context << ") ... \n";
491#endif
492 if ((device == nullptr) || (physical_device == nullptr) || (block_allocator == nullptr)) {
493 return false;
494 }
495 return block_allocator->collect(this);
496}
497
498int VulkanMemoryAllocator::release(void *user_context) {
499#if defined(HL_VK_DEBUG_MEM)
500 debug(nullptr) << "VulkanMemoryAllocator: Releasing block allocator ("
501 << "user_context=" << user_context << ") ... \n";
502#endif
503 if ((device == nullptr) || (physical_device == nullptr)) {
504 error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid device handle!\n";
506 }
507 if (block_allocator == nullptr) {
508 error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid block allocator!\n";
510 }
511
512 return block_allocator->release(this);
513}
514
515int VulkanMemoryAllocator::destroy(void *user_context) {
516#if defined(HL_VK_DEBUG_MEM)
517 debug(nullptr) << "VulkanMemoryAllocator: Destroying allocator ("
518 << "user_context=" << user_context << ") ... \n";
519#endif
520 if (block_allocator != nullptr) {
521 block_allocator->destroy(this);
522 }
523 region_count = 0;
524 region_byte_count = 0;
525 block_count = 0;
526 block_byte_count = 0;
528}
529
530const VulkanMemoryConfig &
532 static VulkanMemoryConfig result;
533 return result;
534}
535
536// --
537int VulkanMemoryAllocator::lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements) {
538#if defined(HL_VK_DEBUG_MEM)
539 debug(nullptr) << "VulkanMemoryAllocator: Looking up requirements ("
540 << "user_context=" << user_context << " "
541 << "size=" << (uint32_t)block->size << ", "
542 << "usage_flags=" << usage_flags << ") ... \n";
543#endif
546 nullptr, // struct extending this
547 0, // create flags
548 size, // buffer size (in bytes)
549 usage_flags, // buffer usage flags
550 VK_SHARING_MODE_EXCLUSIVE, // sharing mode
551 0, nullptr};
552
553 // Create a buffer to determine alignment requirements
554 VkBuffer buffer = {0};
555 VkResult result = vkCreateBuffer(this->device, &create_info, this->alloc_callbacks, &buffer);
556 if (result != VK_SUCCESS) {
557#if defined(HL_VK_DEBUG_MEM)
558 debug(nullptr) << "VulkanMemoryAllocator: Failed to create buffer to find requirements!\n\t"
559 << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
560#endif
562 }
563
565 vkDestroyBuffer(this->device, buffer, this->alloc_callbacks);
567}
568
570
571 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
572 if (instance == nullptr) {
574 }
575
576 void *user_context = instance->owner_context;
577#if defined(HL_VK_DEBUG_MEM)
578 debug(nullptr) << "VulkanMemoryAllocator: Conforming block request ("
579 << "user_context=" << user_context << " "
580 << "request=" << (void *)(request) << ") ... \n";
581#endif
582
583 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
584 error(user_context) << "VulkanRegionAllocator: Unable to conform block request! Invalid device handle!\n";
586 }
587
589 uint32_t usage_flags = instance->select_memory_usage(user_context, request->properties);
590 int error_code = instance->lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
591 if (error_code != halide_error_code_success) {
592 error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
593 return error_code;
594 }
595
596#if defined(HL_VK_DEBUG_MEM)
597 debug(nullptr) << "VulkanMemoryAllocator: Block allocated ("
598 << "size=" << (uint32_t)request->size << ", "
599 << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
600 << "required_size=" << (uint32_t)memory_requirements.size << ", "
601 << "uniform_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minUniformBufferOffsetAlignment << ", "
602 << "storage_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minStorageBufferOffsetAlignment << ", "
603 << "dedicated=" << (request->dedicated ? "true" : "false") << ")\n";
604#endif
605
606 request->size = memory_requirements.size;
607 request->properties.alignment = memory_requirements.alignment;
609}
610
612 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
613 if (instance == nullptr) {
615 }
616
617 void *user_context = instance->owner_context;
618 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
619 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
621 }
622
623 if (block == nullptr) {
624 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
626 }
627
628#if defined(HL_VK_DEBUG_MEM)
629 debug(nullptr) << "VulkanMemoryAllocator: Allocating block ("
630 << "user_context=" << user_context << " "
631 << "block=" << (void *)(block) << " "
632 << "size=" << (uint64_t)block->size << ", "
633 << "dedicated=" << (block->dedicated ? "true" : "false") << " "
634 << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
635 << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
636 << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
637#endif
638
639 // Find an appropriate memory type given the flags
640 uint32_t memory_type = instance->select_memory_type(user_context, instance->physical_device, block->properties, 0);
641 if (memory_type == invalid_memory_type) {
642 error(user_context) << "VulkanMemoryAllocator: Unable to find appropriate memory type for device!\n";
644 }
645
646 // Allocate memory
649 nullptr, // struct extending this
650 block->size, // size of allocation in bytes
651 memory_type // memory type index from physical device
652 };
653
655 if (device_memory == nullptr) {
656 debug(nullptr) << "VulkanBlockAllocator: Unable to allocate block! Failed to allocate device memory handle!\n";
658 }
659
660 VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
661 if (result != VK_SUCCESS) {
662 debug(nullptr) << "VulkanMemoryAllocator: Allocation failed! vkAllocateMemory returned: " << vk_get_error_name(result) << "\n";
664 }
665#ifdef DEBUG_RUNTIME
666 debug(nullptr) << "vkAllocateMemory: Allocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
667#endif
668
669 block->handle = (void *)device_memory;
670 instance->block_byte_count += block->size;
671 instance->block_count++;
673}
674
676 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
677 if (instance == nullptr) {
679 }
680
681 void *user_context = instance->owner_context;
682#if defined(HL_VK_DEBUG_MEM)
683 debug(nullptr) << "VulkanMemoryAllocator: Deallocating block ("
684 << "user_context=" << user_context << " "
685 << "block=" << (void *)(block) << ") ... \n";
686#endif
687
688 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
689 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
691 }
692
693 if (block == nullptr) {
694 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
696 }
697
698#if defined(HL_VK_DEBUG_MEM)
699 debug(nullptr) << "VulkanBlockAllocator: deallocating block ("
700 << "size=" << (uint32_t)block->size << ", "
701 << "dedicated=" << (block->dedicated ? "true" : "false") << " "
702 << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
703 << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
704 << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
705#endif
706
707 if (block->handle == nullptr) {
708 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid handle!\n";
710 }
711
712 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block->handle);
713 if (device_memory == nullptr) {
714 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device memory handle!\n";
716 }
717
718 vkFreeMemory(instance->device, *device_memory, instance->alloc_callbacks);
719#ifdef DEBUG_RUNTIME
720 debug(nullptr) << "vkFreeMemory: Deallocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
721#endif
722
723 if (instance->block_count > 0) {
724 instance->block_count--;
725 } else {
726 error(nullptr) << "VulkanRegionAllocator: Block counter invalid ... reseting to zero!\n";
727 instance->block_count = 0;
728 }
729
730 if (int64_t(instance->block_byte_count) - int64_t(block->size) >= 0) {
731 instance->block_byte_count -= block->size;
732 } else {
733 error(nullptr) << "VulkanRegionAllocator: Block byte counter invalid ... reseting to zero!\n";
734 instance->block_byte_count = 0;
735 }
736
737 block->handle = nullptr;
738 vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
739 device_memory = nullptr;
741}
742
744 return block_count;
745}
746
748 return block_byte_count;
749}
750
751uint32_t VulkanMemoryAllocator::select_memory_type(void *user_context,
752 VkPhysicalDevice physical_device,
753 MemoryProperties properties,
754 uint32_t required_flags) const {
755
756 uint32_t want_flags = 0; //< preferred memory flags for requested access type
757 uint32_t need_flags = 0; //< must have in order to enable requested access
758 switch (properties.visibility) {
761 break;
764 break;
768 break;
771 break;
774 default:
775 error(nullptr) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
776 << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
777 return invalid_memory_type;
778 };
779
780 switch (properties.caching) {
784 }
785 break;
789 }
790 break;
794 }
795 break;
798 break;
800 default:
801 error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory caching request!\n\t"
802 << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
803 return invalid_memory_type;
804 };
805
808
809 uint32_t result = invalid_memory_type;
810 for (uint32_t i = 0; i < device_memory_properties.memoryTypeCount; ++i) {
811
812 // if required flags are given, see if the memory type matches the requirement
813 if (required_flags) {
814 if (((required_flags >> i) & 1) == 0) {
815 continue;
816 }
817 }
818
819 const VkMemoryPropertyFlags properties = device_memory_properties.memoryTypes[i].propertyFlags;
820 if (need_flags) {
821 if ((properties & need_flags) != need_flags) {
822 continue;
823 }
824 }
825
826 if (want_flags) {
827 if ((properties & want_flags) != want_flags) {
828 continue;
829 }
830 }
831
832 result = i;
833 break;
834 }
835
836 if (result == invalid_memory_type) {
837 error(user_context) << "VulkanBlockAllocator: Failed to find appropriate memory type for given properties:\n\t"
838 << "usage=" << halide_memory_usage_name(properties.usage) << " "
839 << "caching=" << halide_memory_caching_name(properties.caching) << " "
840 << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
841 return invalid_memory_type;
842 }
843
844 return result;
845}
846
847// --
848
850
851 // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
852 // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
853 // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
854 // whose size may be required to larger than our requested size (even though we will only ever touch the
855 // size of the region we're managing as within our block)
856
858 uint32_t usage_flags = select_memory_usage(user_context, request->properties);
859 int error_code = lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
860 if (error_code != halide_error_code_success) {
861 error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
862 return error_code;
863 }
864
865#if defined(HL_VK_DEBUG_MEM)
866 debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
867 << "requested_size=" << (uint32_t)region->size << ", "
868 << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
869 << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
870#endif
871
872 // Enforce any alignment constraints reported by the device limits for each usage type
874 if ((request->alignment % this->physical_device_limits.minStorageBufferOffsetAlignment) != 0) {
875 request->alignment = this->physical_device_limits.minStorageBufferOffsetAlignment;
876 }
878 if ((request->alignment % this->physical_device_limits.minUniformBufferOffsetAlignment) != 0) {
879 request->alignment = this->physical_device_limits.minUniformBufferOffsetAlignment;
880 }
881 }
882
883 // Ensure the request ends on an aligned address
884 if (request->alignment > config.nearest_multiple) {
885 request->properties.nearest_multiple = request->alignment;
886 }
887
888 size_t actual_alignment = conform_alignment(request->alignment, memory_requirements.alignment);
890 size_t actual_size = conform_size(actual_offset, memory_requirements.size, actual_alignment, request->properties.nearest_multiple);
891
892#if defined(HL_VK_DEBUG_MEM)
893 if ((request->size != actual_size) || (request->alignment != actual_alignment) || (request->offset != actual_offset)) {
894 debug(nullptr) << "VulkanMemoryAllocator: Adjusting request to match requirements (\n"
895 << " size = " << (uint64_t)request->size << " => " << (uint64_t)actual_size << ",\n"
896 << " alignment = " << (uint64_t)request->alignment << " => " << (uint64_t)actual_alignment << ",\n"
897 << " offset = " << (uint64_t)request->offset << " => " << (uint64_t)actual_offset << ",\n"
898 << " required.size = " << (uint64_t)memory_requirements.size << ",\n"
899 << " required.alignment = " << (uint64_t)memory_requirements.alignment << "\n)\n";
900 }
901#endif
902 request->size = actual_size;
903 request->alignment = actual_alignment;
904 request->offset = actual_offset;
905
907}
908
910
911 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
912 if (instance == nullptr) {
914 }
915
916 void *user_context = instance->owner_context;
917#if defined(HL_VK_DEBUG_MEM)
918 debug(nullptr) << "VulkanMemoryAllocator: Conforming region request ("
919 << "user_context=" << user_context << " "
920 << "request=" << (void *)(region) << ") ... \n";
921#endif
922
923 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
924 error(user_context) << "VulkanRegionAllocator: Unable to conform region request! Invalid device handle!\n";
926 }
927
928#if defined(HL_VK_DEBUG_MEM)
929 debug(nullptr) << "VulkanRegionAllocator: Conforming region request ("
930 << "size=" << (uint32_t)request->size << ", "
931 << "offset=" << (uint32_t)request->offset << ", "
932 << "dedicated=" << (request->dedicated ? "true" : "false") << " "
933 << "usage=" << halide_memory_usage_name(request->properties.usage) << " "
934 << "caching=" << halide_memory_caching_name(request->properties.caching) << " "
935 << "visibility=" << halide_memory_visibility_name(request->properties.visibility) << ")\n";
936#endif
937
938 return instance->conform(user_context, request);
939}
940
942
943 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
944 if (instance == nullptr) {
946 }
947
948 void *user_context = instance->owner_context;
949#if defined(HL_VK_DEBUG_MEM)
950 debug(nullptr) << "VulkanMemoryAllocator: Allocating region ("
951 << "user_context=" << user_context << " "
952 << "region=" << (void *)(region) << ") ... \n";
953#endif
954
955 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
956 error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid device handle!\n";
958 }
959
960 if (region == nullptr) {
961 error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid pointer!\n";
963 }
964
965#if defined(HL_VK_DEBUG_MEM)
966 debug(nullptr) << "VulkanRegionAllocator: Allocating region ("
967 << "size=" << (uint32_t)region->size << ", "
968 << "offset=" << (uint32_t)region->offset << ", "
969 << "dedicated=" << (region->dedicated ? "true" : "false") << " "
970 << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
971 << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
972 << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
973#endif
974
975 uint32_t usage_flags = instance->select_memory_usage(user_context, region->properties);
976
979 nullptr, // struct extending this
980 0, // create flags
981 region->size, // buffer size (in bytes)
982 usage_flags, // buffer usage flags
983 VK_SHARING_MODE_EXCLUSIVE, // sharing mode
984 0, nullptr};
985
986 VkBuffer *buffer = (VkBuffer *)vk_host_malloc(nullptr, sizeof(VkBuffer), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
987 if (buffer == nullptr) {
988 error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Failed to allocate buffer handle!\n";
990 }
991
992 VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
993 if (result != VK_SUCCESS) {
994 error(user_context) << "VulkanRegionAllocator: Failed to create buffer!\n\t"
995 << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
997 }
998
999 // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
1000 // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
1001 // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
1002 // whose size may be required to larger than our requested size (even though we will only ever touch the
1003 // size of the region we're managing as within our block)
1005 vkGetBufferMemoryRequirements(instance->device, *buffer, &memory_requirements);
1006
1007#if defined(HL_VK_DEBUG_MEM)
1008 debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
1009 << "requested_size=" << (uint32_t)region->size << ", "
1010 << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
1011 << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
1012#endif
1013
1014 if (memory_requirements.size > region->size) {
1015 vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1016#ifdef DEBUG_RUNTIME
1017 debug(nullptr) << "VulkanMemoryAllocator: Reallocating buffer to match required size ("
1018 << (uint64_t)region->size << " => " << (uint64_t)memory_requirements.size << " bytes) ...\n";
1019#endif
1020 create_info.size = memory_requirements.size;
1021 VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1022 if (result != VK_SUCCESS) {
1023 error(user_context) << "VulkanRegionAllocator: Failed to recreate buffer!\n\t"
1024 << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1026 }
1027 }
1028
1029#ifdef DEBUG_RUNTIME
1030 debug(nullptr) << "vkCreateBuffer: Created buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1031#endif
1032
1034 if (region_allocator == nullptr) {
1035 error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid region allocator!\n";
1037 }
1038
1039 BlockResource *block_resource = region_allocator->block_resource();
1040 if (block_resource == nullptr) {
1041 error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid block resource handle!\n";
1043 }
1044
1045 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
1046 if (device_memory == nullptr) {
1047 error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid device memory handle!\n";
1049 }
1050
1051 // Finally, bind buffer to the device memory
1052 result = vkBindBufferMemory(instance->device, *buffer, *device_memory, region->offset);
1053 if (result != VK_SUCCESS) {
1054 error(user_context) << "VulkanRegionAllocator: Failed to bind buffer!\n\t"
1055 << "vkBindBufferMemory returned: " << vk_get_error_name(result) << "\n";
1057 }
1058
1059 region->handle = (void *)buffer;
1060 region->is_owner = true;
1061 instance->region_byte_count += region->size;
1062 instance->region_count++;
1064}
1065
1067 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1068 if (instance == nullptr) {
1070 }
1071
1072 void *user_context = instance->owner_context;
1073#if defined(HL_VK_DEBUG_MEM)
1074 debug(nullptr) << "VulkanMemoryAllocator: Deallocating region ("
1075 << "user_context=" << user_context << " "
1076 << "region=" << (void *)(region) << ") ... \n";
1077#endif
1078
1079 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1080 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid device handle!\n";
1082 }
1083
1084 if (region == nullptr) {
1085 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid pointer!\n";
1087 }
1088
1089#if defined(HL_VK_DEBUG_MEM)
1090 debug(nullptr) << "VulkanRegionAllocator: Deallocating region ("
1091 << "size=" << (uint32_t)region->size << ", "
1092 << "offset=" << (uint32_t)region->offset << ", "
1093 << "dedicated=" << (region->dedicated ? "true" : "false") << " "
1094 << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
1095 << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
1096 << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
1097#endif
1098
1099 if (region->handle == nullptr) {
1100 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid handle!\n";
1102 }
1103
1104 VkBuffer *buffer = reinterpret_cast<VkBuffer *>(region->handle);
1105 if (buffer == nullptr) {
1106 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid buffer handle!\n";
1108 }
1109
1110 vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1111#ifdef DEBUG_RUNTIME
1112 debug(nullptr) << "vkDestroyBuffer: Destroyed buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1113#endif
1114 region->handle = nullptr;
1115 if (instance->region_count > 0) {
1116 instance->region_count--;
1117 } else {
1118 error(nullptr) << "VulkanRegionAllocator: Region counter invalid ... reseting to zero!\n";
1119 instance->region_count = 0;
1121 }
1122
1123 if (int64_t(instance->region_byte_count) - int64_t(region->size) >= 0) {
1124 instance->region_byte_count -= region->size;
1125 } else {
1126 error(nullptr) << "VulkanRegionAllocator: Region byte counter invalid ... reseting to zero!\n";
1127 instance->region_byte_count = 0;
1129 }
1130 vk_host_free(nullptr, buffer, instance->alloc_callbacks);
1131 buffer = nullptr;
1133}
1134
1136 return region_count;
1137}
1138
1140 return region_byte_count;
1141}
1142
1143uint32_t VulkanMemoryAllocator::select_memory_usage(void *user_context, MemoryProperties properties) const {
1144 uint32_t result = 0;
1145 switch (properties.usage) {
1148 break;
1152 break;
1155 break;
1158 break;
1161 break;
1164 default:
1165 error(user_context) << "VulkanRegionAllocator: Unable to convert type! Invalid memory usage request!\n\t"
1166 << "usage=" << halide_memory_usage_name(properties.usage) << "\n";
1167 return invalid_usage_flags;
1168 };
1169
1170 if (result == invalid_usage_flags) {
1171 error(user_context) << "VulkanRegionAllocator: Failed to find appropriate memory usage for given properties:\n\t"
1172 << "usage=" << halide_memory_usage_name(properties.usage) << " "
1173 << "caching=" << halide_memory_caching_name(properties.caching) << " "
1174 << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
1175 return invalid_usage_flags;
1176 }
1177
1178 return result;
1179}
1180
1181// --------------------------------------------------------------------------
1182
1183namespace {
1184
1185// --------------------------------------------------------------------------
1186// Halide System allocator for host allocations
1187void *vk_system_malloc(void *user_context, size_t size) {
1188 return malloc(size);
1189}
1190
1191void vk_system_free(void *user_context, void *ptr) {
1192 free(ptr);
1193}
1194
1195// Vulkan host-side allocation
1196void *vk_host_malloc(void *user_context, size_t size, size_t alignment, VkSystemAllocationScope scope, const VkAllocationCallbacks *callbacks) {
1197 if (callbacks) {
1198 return callbacks->pfnAllocation(user_context, size, alignment, scope);
1199 } else {
1200 return vk_system_malloc(user_context, size);
1201 }
1202}
1203
1204void vk_host_free(void *user_context, void *ptr, const VkAllocationCallbacks *callbacks) {
1205 if (callbacks) {
1206 return callbacks->pfnFree(user_context, ptr);
1207 } else {
1208 return vk_system_free(user_context, ptr);
1209 }
1210}
1211
1212VulkanMemoryAllocator *vk_create_memory_allocator(void *user_context,
1213 VkDevice device,
1214 VkPhysicalDevice physical_device,
1215 const VkAllocationCallbacks *alloc_callbacks) {
1216
1218 VulkanMemoryConfig config = memory_allocator_config;
1219
1220 // Parse the allocation config string (if specified).
1221 //
1222 // `HL_VK_ALLOC_CONFIG=N:N:N` will tell Halide to configure the Vulkan memory
1223 // allocator use the given constraints specified as three integer values
1224 // separated by a `:` or `;`. These values correspond to `minimum_block_size`,
1225 // `maximum_block_size` and `maximum_block_count`.
1226 //
1227 const char *alloc_config = vk_get_alloc_config_internal(user_context);
1230 alloc_config_values.parse(user_context, alloc_config, HL_VK_ENV_DELIM);
1231 if (alloc_config_values.size() > 0) {
1232 config.maximum_pool_size = atoi(alloc_config_values[0]) * 1024 * 1024;
1233 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_pool_size << " for maximum pool size (in bytes)\n";
1234 }
1235 if (alloc_config_values.size() > 1) {
1236 config.minimum_block_size = atoi(alloc_config_values[1]) * 1024 * 1024;
1237 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.minimum_block_size << " for minimum block size (in bytes)\n";
1238 }
1239 if (alloc_config_values.size() > 2) {
1240 config.maximum_block_size = atoi(alloc_config_values[2]) * 1024 * 1024;
1241 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_size << " for maximum block size (in bytes)\n";
1242 }
1243 if (alloc_config_values.size() > 3) {
1244 config.maximum_block_count = atoi(alloc_config_values[3]);
1245 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_count << " for maximum block count\n";
1246 }
1247 if (alloc_config_values.size() > 4) {
1248 config.nearest_multiple = atoi(alloc_config_values[4]);
1249 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.nearest_multiple << " for nearest multiple\n";
1250 }
1251 }
1252
1253 return VulkanMemoryAllocator::create(user_context,
1254 config, device, physical_device,
1255 system_allocator, alloc_callbacks);
1256}
1257
1258int vk_destroy_memory_allocator(void *user_context, VulkanMemoryAllocator *allocator) {
1259 if (allocator != nullptr) {
1260 VulkanMemoryAllocator::destroy(user_context, allocator);
1261 allocator = nullptr;
1262 }
1264}
1265
1266// --------------------------------------------------------------------------
1267
1268int vk_clear_device_buffer(void *user_context,
1269 VulkanMemoryAllocator *allocator,
1270 VkCommandPool command_pool,
1273
1274#ifdef DEBUG_RUNTIME
1275 debug(user_context)
1276 << " vk_clear_device_buffer (user_context: " << user_context << ", "
1277 << "allocator: " << (void *)allocator << ", "
1278 << "command_pool: " << (void *)command_pool << ", "
1279 << "command_queue: " << (void *)command_queue << ", "
1280 << "device_buffer: " << (void *)device_buffer << ")\n";
1281#endif
1282
1283 // create a command buffer
1285 int error_code = vk_create_command_buffer(user_context, allocator, command_pool, &command_buffer);
1286 if (error_code != halide_error_code_success) {
1287 error(user_context) << "Vulkan: Failed to create command buffer!\n";
1288 return error_code;
1289 }
1290
1291 // begin the command buffer
1293 {
1295 nullptr, // pointer to struct extending this
1297 nullptr // pointer to parent command buffer
1298 };
1299
1301 if (result != VK_SUCCESS) {
1302 error(user_context) << "Vulkan: vkBeginCommandBuffer returned " << vk_get_error_name(result) << "\n";
1304 }
1305
1306 // fill buffer with zero values up to the size of the buffer
1308
1309 // end the command buffer
1311 if (result != VK_SUCCESS) {
1312 error(user_context) << "Vulkan: vkEndCommandBuffer returned " << vk_get_error_name(result) << "\n";
1314 }
1315
1316 // submit the command buffer
1318 {
1319 VK_STRUCTURE_TYPE_SUBMIT_INFO, // struct type
1320 nullptr, // pointer to struct extending this
1321 0, // wait semaphore count
1322 nullptr, // semaphores
1323 nullptr, // pipeline stages where semaphore waits occur
1324 1, // how many command buffers to execute
1325 &command_buffer, // the command buffers
1326 0, // number of semaphores to signal
1327 nullptr // the semaphores to signal
1328 };
1329
1330 result = vkQueueSubmit(command_queue, 1, &submit_info, 0);
1331 if (result != VK_SUCCESS) {
1332 error(user_context) << "Vulkan: vkQueueSubmit returned " << vk_get_error_name(result) << "\n";
1334 }
1335
1336 // wait for memset to finish
1338 if (result != VK_SUCCESS) {
1339 error(user_context) << "Vulkan: vkQueueWaitIdle returned " << vk_get_error_name(result) << "\n";
1341 }
1342
1343 error_code = vk_destroy_command_buffer(user_context, allocator, command_pool, command_buffer);
1344 if (error_code != halide_error_code_success) {
1345 error(user_context) << "Vulkan: Failed to destroy command buffer!\n";
1346 return error_code;
1347 }
1348
1350}
1351
1352// --------------------------------------------------------------------------
1353
1354} // namespace
1355} // namespace Vulkan
1356} // namespace Internal
1357} // namespace Runtime
1358} // namespace Halide
1359
1360// --------------------------------------------------------------------------
1361
1362extern "C" {
1363
1364// --------------------------------------------------------------------------
1365
1367 using namespace Halide::Runtime::Internal::Vulkan;
1368 ScopedSpinLock lock(&custom_allocation_callbacks_lock);
1369 custom_allocation_callbacks = callbacks;
1370}
1371
1373 using namespace Halide::Runtime::Internal::Vulkan;
1374 ScopedSpinLock lock(&custom_allocation_callbacks_lock);
1375 return custom_allocation_callbacks;
1376}
1377
1378// --------------------------------------------------------------------------
1379
1380} // extern "C"
1381
1382#endif // HALIDE_RUNTIME_VULKAN_MEMORY_H
@ halide_error_code_internal_error
There is a bug in the Halide compiler.
@ halide_error_code_generic_error
An uncategorized error occurred.
@ halide_error_code_success
There was no error.
@ halide_error_code_device_malloc_failed
The Halide runtime encountered an error while trying to allocate memory on device.
@ halide_error_code_out_of_memory
A call to halide_malloc returned NULL.
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
int release(void *user_context, MemoryRegion *region)
int retain(void *user_context, MemoryRegion *region)
static void destroy(void *user_context, BlockAllocator *block_allocator)
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
int reclaim(void *user_context, MemoryRegion *region)
const MemoryAllocators & current_allocators() const
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
Vulkan Memory Allocator class interface for managing large memory requests stored as contiguous block...
int reclaim(void *user_context, MemoryRegion *region)
static const VulkanMemoryConfig & default_config()
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
VulkanMemoryAllocator(const VulkanMemoryAllocator &)=delete
int release(void *user_context, MemoryRegion *region)
static int allocate_block(void *instance_ptr, MemoryBlock *block)
int conform(void *user_context, MemoryRequest *request)
static int deallocate_region(void *instance_ptr, MemoryRegion *region)
VulkanMemoryAllocator & operator=(const VulkanMemoryAllocator &)=delete
static int destroy(void *user_context, VulkanMemoryAllocator *allocator)
int unmap(void *user_context, MemoryRegion *region)
static int conform_region_request(void *instance_ptr, MemoryRequest *request)
MemoryRegion * create_crop(void *user_context, MemoryRegion *region, uint64_t offset)
int destroy_crop(void *user_context, MemoryRegion *region)
static int allocate_region(void *instance_ptr, MemoryRegion *region)
static int conform_block_request(void *instance_ptr, MemoryRequest *request)
MemoryRegion * owner_of(void *user_context, MemoryRegion *region)
const VkAllocationCallbacks * callbacks() const
static int deallocate_block(void *instance_ptr, MemoryBlock *block)
void * map(void *user_context, MemoryRegion *region)
int retain(void *user_context, MemoryRegion *region)
static VulkanMemoryAllocator * create(void *user_context, const VulkanMemoryConfig &config, VkDevice dev, VkPhysicalDevice phys_dev, const SystemMemoryAllocatorFns &system_allocator, const VkAllocationCallbacks *alloc_callbacks=nullptr)
WEAK const char * halide_memory_caching_name(MemoryCaching value)
WEAK const char * halide_memory_usage_name(MemoryUsage value)
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
VKAPI_ATTR VkResult VKAPI_CALL vkBindBufferMemory(VkDevice device, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize memoryOffset)
VkFlags VkMemoryPropertyFlags
VKAPI_ATTR void VKAPI_CALL vkCmdFillBuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer, VkDeviceSize dstOffset, VkDeviceSize size, uint32_t data)
VKAPI_ATTR VkResult VKAPI_CALL vkAllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pAllocateInfo, const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMemory)
#define VK_MAX_MEMORY_TYPES
@ VK_SHARING_MODE_EXCLUSIVE
@ VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
@ VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT
@ VK_MEMORY_PROPERTY_HOST_CACHED_BIT
@ VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT
VKAPI_ATTR void VKAPI_CALL vkGetBufferMemoryRequirements(VkDevice device, VkBuffer buffer, VkMemoryRequirements *pMemoryRequirements)
VkSystemAllocationScope
@ VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
VKAPI_ATTR VkResult VKAPI_CALL vkQueueWaitIdle(VkQueue queue)
uint64_t VkDeviceSize
Definition mini_vulkan.h:71
VKAPI_ATTR void VKAPI_CALL vkFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks *pAllocator)
@ VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceProperties *pProperties)
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceMemoryProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceMemoryProperties *pMemoryProperties)
VKAPI_ATTR VkResult VKAPI_CALL vkCreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkBuffer *pBuffer)
VKAPI_ATTR VkResult VKAPI_CALL vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo *pSubmits, VkFence fence)
VKAPI_ATTR VkResult VKAPI_CALL vkMapMemory(VkDevice device, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags, void **ppData)
#define VK_WHOLE_SIZE
@ VK_BUFFER_USAGE_TRANSFER_DST_BIT
@ VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT
@ VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
@ VK_BUFFER_USAGE_TRANSFER_SRC_BIT
VkResult
@ VK_SUCCESS
VKAPI_ATTR void VKAPI_CALL vkUnmapMemory(VkDevice device, VkDeviceMemory memory)
VKAPI_ATTR VkResult VKAPI_CALL vkEndCommandBuffer(VkCommandBuffer commandBuffer)
VKAPI_ATTR void VKAPI_CALL vkDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks *pAllocator)
@ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO
@ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO
@ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO
@ VK_STRUCTURE_TYPE_SUBMIT_INFO
VKAPI_ATTR VkResult VKAPI_CALL vkBeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBeginInfo *pBeginInfo)
WEAK VulkanMemoryConfig memory_allocator_config
WEAK const VkAllocationCallbacks * custom_allocation_callbacks
WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Expr cast(Expr a)
Cast an expression to the halide type corresponding to the C++ type T.
Definition IROperator.h:364
Expr print(const std::vector< Expr > &values)
Create an Expr that prints out its value whenever it is evaluated.
unsigned __INT64_TYPE__ uint64_t
signed __INT64_TYPE__ int64_t
int atoi(const char *)
void * malloc(size_t)
void * memcpy(void *s1, const void *s2, size_t n)
void * memset(void *s, int val, size_t n)
unsigned __INT32_TYPE__ uint32_t
#define WEAK
void free(void *)
static bool is_empty(const char *str)
PFN_vkAllocationFunction pfnAllocation
PFN_vkFreeFunction pfnFree
VkDeviceSize minStorageBufferOffsetAlignment
VkDeviceSize minUniformBufferOffsetAlignment
#define HL_VK_ENV_DELIM
WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks)
WEAK const VkAllocationCallbacks * halide_vulkan_get_allocation_callbacks(void *user_context)