1 #include <msp/core/algorithm.h>
2 #include <msp/core/maputils.h>
3 #include <msp/graphics/vulkancontext_platform.h>
4 #include <msp/stringcodec/utf8.h>
5 #include <msp/strings/format.h>
8 #include "memoryallocator.h"
16 MemoryAllocator::MemoryAllocator(Device &d):
18 phys_device(handle_cast<VkPhysicalDevice>(device.get_context().get_private().physical_device))
20 const VulkanFunctions &vk = device.get_functions();
22 VkPhysicalDeviceMemoryProperties mem_props;
23 vk.GetPhysicalDeviceMemoryProperties(mem_props);
25 for(unsigned i=0; i<mem_props.memoryHeapCount; ++i)
26 if(mem_props.memoryHeaps[i].flags&VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
27 total_device_memory += mem_props.memoryHeaps[i].size;
29 default_region_size = total_device_memory/256;
30 default_region_size -= default_region_size%min_alignment;
31 direct_alloc_threshold = default_region_size/4;
33 const VkMemoryPropertyFlags host_flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
34 pools.resize(mem_props.memoryTypeCount);
35 for(unsigned i=0; i<mem_props.memoryTypeCount; ++i)
37 VkMemoryPropertyFlags flags = mem_props.memoryTypes[i].propertyFlags;
38 if(flags&VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
40 if((flags&host_flags)==host_flags)
41 pools[i].type = STREAMING_MEMORY;
43 pools[i].type = DEVICE_MEMORY;
45 else if((flags&host_flags)==host_flags)
46 pools[i].type = STAGING_MEMORY;
50 MemoryAllocator::~MemoryAllocator()
52 const VulkanFunctions &vk = device.get_functions();
54 for(Region &r: regions)
56 vk.FreeMemory(r.memory);
59 unsigned MemoryAllocator::find_memory_pool(unsigned mask, MemoryType type) const
61 for(unsigned i=0; i<pools.size(); ++i)
62 if((mask&(1<<i)) && pools[i].type==type)
64 if(type==DEVICE_MEMORY || type==STAGING_MEMORY)
65 return find_memory_pool(mask, STREAMING_MEMORY);
66 throw runtime_error("Unable to find suitable memory type");
69 unsigned MemoryAllocator::create_region(unsigned pool_index, size_t size, bool direct)
71 const VulkanFunctions &vk = device.get_functions();
73 VkMemoryAllocateInfo alloc_info = { };
74 alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
75 alloc_info.allocationSize = size;
76 alloc_info.memoryTypeIndex = pool_index;
79 vk.AllocateMemory(alloc_info, region.memory);
81 region.pool = pool_index;
82 region.direct = direct;
84 regions.push_back(region);
86 return regions.size()-1;
89 vector<unsigned>::iterator MemoryAllocator::lower_bound_by_size(vector<unsigned> &indices, size_t size) const
91 return lower_bound(indices, size, [this](unsigned j, unsigned s){ return blocks[j].size<s; });
94 unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits, MemoryType type)
96 unsigned pool_index = find_memory_pool(type_bits, type);
97 Pool &pool = pools[pool_index];
99 if(size>=direct_alloc_threshold)
102 block.region = create_region(pool_index, size, true);
104 block.allocated = true;
106 blocks.push_back(block);
107 return blocks.size()-1;
110 if(pool.can_consolidate && blocks[pool.free_blocks.back()].size<size+align)
111 consolidate(pool_index);
113 auto i = lower_bound_by_size(pool.free_blocks, size);
114 for(; i!=pool.free_blocks.end(); ++i)
116 Block &block = blocks[*i];
117 size_t offset = align-1-(block.offset+align-1)%align;
118 if(offset+size<=block.size)
122 unsigned block_index;
123 if(i!=pool.free_blocks.end())
126 pool.free_blocks.erase(i);
127 if(pool.free_blocks.empty())
128 pool.can_consolidate = false;
133 block.region = create_region(pool_index, default_region_size, false);
134 block.size = regions[block.region].size;
136 blocks.push_back(block);
137 block_index = blocks.size()-1;
140 size_t offset = align-1-(blocks[block_index].offset+align-1)%align;
143 unsigned head_index = block_index;
144 block_index = split_block(block_index, offset);
145 pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, blocks[head_index].size), head_index);
148 size += min_alignment-1;
149 size -= size%min_alignment;
150 if(blocks[block_index].size>=size+min_alignment)
152 unsigned tail_index = split_block(block_index, size);
153 pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, blocks[tail_index].size), tail_index);
156 blocks[block_index].allocated = true;
161 unsigned MemoryAllocator::split_block(unsigned index, size_t head_size)
163 blocks.emplace_back();
164 Block &block = blocks[index];
165 Block &tail = blocks.back();
166 unsigned tail_index = blocks.size()-1;
168 tail.region = block.region;
169 tail.offset = block.offset+head_size;
170 tail.size = block.size-head_size;
172 tail.next = block.next;
174 block.size = head_size;
175 block.next = tail_index;
180 void MemoryAllocator::consolidate(unsigned pool_index)
182 Pool &pool = pools[pool_index];
184 vector<unsigned> merged_blocks;
186 for(unsigned j=0; j<pool.free_blocks.size(); ++j)
188 unsigned block_index = pool.free_blocks[j];
189 Block &block = blocks[block_index];
192 if(block.prev<0 || blocks[block.prev].allocated)
194 if(block.next>=0 && !blocks[block.next].allocated)
196 merge_block_with_next(block_index);
198 while(block.next>=0 && !blocks[block.next].allocated)
199 merge_block_with_next(block_index);
201 merged_blocks.insert(lower_bound_by_size(merged_blocks, block.size), block_index);
209 pool.free_blocks[i] = block_index;
213 pool.free_blocks.resize(i+merged_blocks.size());
215 if(!merged_blocks.empty())
217 unsigned j = merged_blocks.size();
218 for(unsigned k=pool.free_blocks.size()-1; j; --k)
220 if(!i || blocks[merged_blocks[j-1]].size>blocks[pool.free_blocks[i-1]].size)
221 pool.free_blocks[k] = merged_blocks[--j];
223 pool.free_blocks[k] = pool.free_blocks[--i];
228 void MemoryAllocator::merge_block_with_next(unsigned index)
230 Block &block = blocks[index];
232 Block &next = blocks[block.next];
233 block.size += next.size;
234 block.next = next.next;
236 blocks[block.next].prev = index;
241 unsigned MemoryAllocator::allocate(VkBuffer buffer, MemoryType type)
243 const VulkanFunctions &vk = device.get_functions();
245 VkMemoryRequirements requirements;
246 vk.GetBufferMemoryRequirements(buffer, requirements);
248 unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
250 Block &block = blocks[block_index];
251 vk.BindBufferMemory(buffer, regions[block.region].memory, block.offset);
253 return block_index+1;
256 unsigned MemoryAllocator::allocate(VkImage image, MemoryType type)
258 const VulkanFunctions &vk = device.get_functions();
260 VkMemoryRequirements requirements;
261 vk.GetImageMemoryRequirements(image, requirements);
263 unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
265 Block &block = blocks[block_index];
266 vk.BindImageMemory(image, regions[block.region].memory, block.offset);
268 return block_index+1;
271 void MemoryAllocator::release(unsigned id)
273 if(!id || id>blocks.size() || !blocks[id-1].allocated)
276 unsigned block_index = id-1;
277 Block &block = blocks[block_index];
279 block.allocated = false;
281 Region ®ion = regions[block.region];
284 const VulkanFunctions &vk = device.get_functions();
286 vk.FreeMemory(region.memory);
292 Pool &pool = pools[region.pool];
293 pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, block.size), block_index);
294 if((block.prev>=0 && !blocks[block.prev].allocated) || (block.next>=0 && !blocks[block.next].allocated))
295 pool.can_consolidate = true;
298 void *MemoryAllocator::map(unsigned id)
300 if(!id || id>blocks.size() || !blocks[id-1].allocated)
303 Block &block = blocks[id-1];
304 Region ®ion = regions[block.region];
305 if(!region.mapped_address)
307 const VulkanFunctions &vk = device.get_functions();
308 vk.MapMemory(region.memory, 0, region.size, 0, ®ion.mapped_address);
313 return static_cast<char *>(region.mapped_address)+block.offset;
316 void MemoryAllocator::unmap(unsigned id)
318 if(!id || id>blocks.size() || !blocks[id-1].allocated)
321 Block &block = blocks[id-1];
322 Region ®ion = regions[block.region];
324 if(!regions[block.region].mapped_address)
325 throw invalid_operation("MemoryAllocator::unmap");
326 else if(!--region.map_count)
328 const VulkanFunctions &vk = device.get_functions();
329 vk.UnmapMemory(region.memory);
330 region.mapped_address = 0;
334 string MemoryAllocator::get_debug() const
336 static const StringCodec::unichar bar_chars[] = { 0xB7, 0x2596, 0x258C, 0x2597, 0x2584, 0x2599, 0x2590, 0x259F, 0x2588 }; // ·▖▌▗▄▙▐▟█
339 for(unsigned i=0; i<pools.size(); ++i)
341 const Pool &pool = pools[i];
344 size_t total_heap = 0;
345 size_t total_used = 0;
346 for(unsigned j=0; j<regions.size(); ++j)
347 if(regions[j].pool==static_cast<int>(i))
349 total_heap += regions[j].size;
350 pool_debug += format(" Region %d: %d kB", j, (regions[j].size+512)/1024);
351 if(regions[j].direct)
352 pool_debug += ", direct";
355 int block_index = -1;
356 for(unsigned k=0; (block_index<0 && k<blocks.size()); ++k)
357 if(blocks[k].region==static_cast<int>(j) && blocks[k].offset==0)
360 unsigned slice_index = 0;
361 unsigned slice_data = 0;
365 StringCodec::Utf8::Encoder bar_enc;
366 while(block_index>=0)
368 const Block &block = blocks[block_index];
370 total_used += block.size;
371 const char *state_str = (block.allocated ? "allocated" : "free");
372 region_debug += format(" Block %d: %d bytes at %d, %s\n", block_index, block.size, block.offset, state_str);
373 block_index = block.next;
375 size_t block_end = block.offset+block.size;
378 size_t slice_end = regions[j].size*(slice_index+1)/140;
379 slice_data |= 1<<(block.allocated+slice_index%2*2);
380 if(slice_end>block_end)
385 slice_data = 5+((slice_data>>1)&5)-(slice_data&5);
386 bar_enc.encode_char(bar_chars[(slice_data&3)+3*((slice_data>>2)&3)], bar);
393 if(!regions[j].direct)
395 pool_debug += region_debug;
398 if(!pool_debug.empty())
400 MemoryType t = pool.type;
401 const char *type_str = (t==DEVICE_MEMORY ? "device" : t==STAGING_MEMORY ? "staging" :
402 t==STREAMING_MEMORY ? "streaming" : "unknown");
403 debug += format("Pool %d: %s, %d/%d kB used\n", i, type_str, (total_used+512)/1024, (total_heap+512)/1024);
407 if(!pool.free_blocks.empty())
409 debug += " Free blocks:\n";
410 for(unsigned j: pool.free_blocks)
412 const char *type = (blocks[j].type==BUFFER ? "buffer" : blocks[j].type==IMAGE ? "image" : "undecided");
413 debug += format(" Block %d: %d bytes, %s\n", j, blocks[j].size, type);