source/backends/vulkan/memoryallocator.cpp

   1 #include <msp/core/algorithm.h>
   2 #include <msp/core/maputils.h>
   3 #include <msp/graphics/vulkancontext_platform.h>
   4 #include <msp/stringcodec/utf8.h>
   5 #include <msp/strings/format.h>
   6 #include "device.h"
   7 #include "error.h"
   8 #include "memoryallocator.h"
   9 #include "vulkan.h"
  10
  11 using namespace std;
  12
  13 namespace Msp {
  14 namespace GL {
  15
  16 MemoryAllocator::MemoryAllocator(Device &d):
  17         device(d),
  18         phys_device(handle_cast<VkPhysicalDevice>(device.get_context().get_private().physical_device))
  19 {
  20         const VulkanFunctions &vk = device.get_functions();
  21
  22         VkPhysicalDeviceMemoryProperties mem_props;
  23         vk.GetPhysicalDeviceMemoryProperties(mem_props);
  24
  25         for(unsigned i=0; i<mem_props.memoryHeapCount; ++i)
  26                 if(mem_props.memoryHeaps[i].flags&VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
  27                         total_device_memory += mem_props.memoryHeaps[i].size;
  28
  29         default_region_size = total_device_memory/256;
  30         default_region_size -= default_region_size%min_alignment;
  31         direct_alloc_threshold = default_region_size/4;
  32
  33         const VkMemoryPropertyFlags host_flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
  34         pools.resize(mem_props.memoryTypeCount);
  35         for(unsigned i=0; i<mem_props.memoryTypeCount; ++i)
  36         {
  37                 VkMemoryPropertyFlags flags = mem_props.memoryTypes[i].propertyFlags;
  38                 if(flags&VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
  39                 {
  40                         if((flags&host_flags)==host_flags)
  41                                 pools[i].type = STREAMING_MEMORY;
  42                         else
  43                                 pools[i].type = DEVICE_MEMORY;
  44                 }
  45                 else if((flags&host_flags)==host_flags)
  46                         pools[i].type = STAGING_MEMORY;
  47         }
  48 }
  49
  50 MemoryAllocator::~MemoryAllocator()
  51 {
  52         const VulkanFunctions &vk = device.get_functions();
  53
  54         for(Region &r: regions)
  55                 if(r.memory)
  56                         vk.FreeMemory(r.memory);
  57 }
  58
  59 unsigned MemoryAllocator::find_memory_pool(unsigned mask, MemoryType type) const
  60 {
  61         for(unsigned i=0; i<pools.size(); ++i)
  62                 if((mask&(1<<i)) && pools[i].type==type)
  63                         return i;
  64         if(type==DEVICE_MEMORY || type==STAGING_MEMORY)
  65                 return find_memory_pool(mask, STREAMING_MEMORY);
  66         throw runtime_error("Unable to find suitable memory type");
  67 }
  68
  69 unsigned MemoryAllocator::create_region(unsigned pool_index, size_t size, bool direct)
  70 {
  71         const VulkanFunctions &vk = device.get_functions();
  72
  73         VkMemoryAllocateInfo alloc_info = { };
  74         alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
  75         alloc_info.allocationSize = size;
  76         alloc_info.memoryTypeIndex = pool_index;
  77
  78         Region region;
  79         vk.AllocateMemory(alloc_info, region.memory);
  80
  81         region.pool = pool_index;
  82         region.direct = direct;
  83         region.size = size;
  84         regions.push_back(region);
  85
  86         return regions.size()-1;
  87 }
  88
  89 vector<unsigned>::iterator MemoryAllocator::lower_bound_by_size(vector<unsigned> &indices, size_t size) const
  90 {
  91         return lower_bound(indices, size, [this](unsigned j, unsigned s){ return blocks[j].size<s; });
  92 }
  93
  94 unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits, MemoryType type)
  95 {
  96         unsigned pool_index = find_memory_pool(type_bits, type);
  97         Pool &pool = pools[pool_index];
  98
  99         if(size>=direct_alloc_threshold)
 100         {
 101                 Block block;
 102                 block.region = create_region(pool_index, size, true);
 103                 block.size = size;
 104                 block.allocated = true;
 105
 106                 blocks.push_back(block);
 107                 return blocks.size()-1;
 108         }
 109
 110         if(pool.can_consolidate && blocks[pool.free_blocks.back()].size<size+align)
 111                 consolidate(pool_index);
 112
 113         auto i = lower_bound_by_size(pool.free_blocks, size);
 114         for(; i!=pool.free_blocks.end(); ++i)
 115         {
 116                 Block &block = blocks[*i];
 117                 size_t offset = align-1-(block.offset+align-1)%align;
 118                 if(offset+size<=block.size)
 119                         break;
 120         }
 121
 122         unsigned block_index;
 123         if(i!=pool.free_blocks.end())
 124         {
 125                 block_index = *i;
 126                 pool.free_blocks.erase(i);
 127                 if(pool.free_blocks.empty())
 128                         pool.can_consolidate = false;
 129         }
 130         else
 131         {
 132                 Block block;
 133                 block.region = create_region(pool_index, default_region_size, false);
 134                 block.size = regions[block.region].size;
 135
 136                 blocks.push_back(block);
 137                 block_index = blocks.size()-1;
 138         }
 139
 140         size_t offset = align-1-(blocks[block_index].offset+align-1)%align;
 141         if(offset)
 142         {
 143                 unsigned head_index = block_index;
 144                 block_index = split_block(block_index, offset);
 145                 pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, blocks[head_index].size), head_index);
 146         }
 147
 148         size += min_alignment-1;
 149         size -= size%min_alignment;
 150         if(blocks[block_index].size>=size+min_alignment)
 151         {
 152                 unsigned tail_index = split_block(block_index, size);
 153                 pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, blocks[tail_index].size), tail_index);
 154         }
 155
 156         blocks[block_index].allocated = true;
 157
 158         return block_index;
 159 }
 160
 161 unsigned MemoryAllocator::split_block(unsigned index, size_t head_size)
 162 {
 163         blocks.emplace_back();
 164         Block &block = blocks[index];
 165         Block &tail = blocks.back();
 166         unsigned tail_index = blocks.size()-1;
 167
 168         tail.region = block.region;
 169         tail.offset = block.offset+head_size;
 170         tail.size = block.size-head_size;
 171         tail.prev = index;
 172         tail.next = block.next;
 173
 174         block.size = head_size;
 175         block.next = tail_index;
 176
 177         return tail_index;
 178 }
 179
 180 void MemoryAllocator::consolidate(unsigned pool_index)
 181 {
 182         Pool &pool = pools[pool_index];
 183
 184         vector<unsigned> merged_blocks;
 185         unsigned i = 0;
 186         for(unsigned j=0; j<pool.free_blocks.size(); ++j)
 187         {
 188                 unsigned block_index = pool.free_blocks[j];
 189                 Block &block = blocks[block_index];
 190                 if(!block.allocated)
 191                 {
 192                         if(block.prev<0 || blocks[block.prev].allocated)
 193                         {
 194                                 if(block.next>=0 && !blocks[block.next].allocated)
 195                                 {
 196                                         merge_block_with_next(block_index);
 197
 198                                         while(block.next>=0 && !blocks[block.next].allocated)
 199                                                 merge_block_with_next(block_index);
 200
 201                                         merged_blocks.insert(lower_bound_by_size(merged_blocks, block.size), block_index);
 202                                 }
 203                         }
 204                         else
 205                                 continue;
 206                 }
 207
 208                 if(j!=i)
 209                         pool.free_blocks[i] = block_index;
 210                 ++i;
 211         }
 212
 213         pool.free_blocks.resize(i+merged_blocks.size());
 214
 215         if(!merged_blocks.empty())
 216         {
 217                 unsigned j = merged_blocks.size();
 218                 for(unsigned k=pool.free_blocks.size()-1; j; --k)
 219                 {
 220                         if(!i || blocks[merged_blocks[j-1]].size>blocks[pool.free_blocks[i-1]].size)
 221                                 pool.free_blocks[k] = merged_blocks[--j];
 222                         else
 223                                 pool.free_blocks[k] = pool.free_blocks[--i];
 224                 }
 225         }
 226 }
 227
 228 void MemoryAllocator::merge_block_with_next(unsigned index)
 229 {
 230         Block &block = blocks[index];
 231
 232         Block &next = blocks[block.next];
 233         block.size += next.size;
 234         block.next = next.next;
 235         if(block.next>=0)
 236                 blocks[block.next].prev = index;
 237
 238         next = Block();
 239 }
 240
 241 unsigned MemoryAllocator::allocate(VkBuffer buffer, MemoryType type)
 242 {
 243         const VulkanFunctions &vk = device.get_functions();
 244
 245         VkMemoryRequirements requirements;
 246         vk.GetBufferMemoryRequirements(buffer, requirements);
 247
 248         unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
 249
 250         Block &block = blocks[block_index];
 251         vk.BindBufferMemory(buffer, regions[block.region].memory, block.offset);
 252
 253         return block_index+1;
 254 }
 255
 256 unsigned MemoryAllocator::allocate(VkImage image, MemoryType type)
 257 {
 258         const VulkanFunctions &vk = device.get_functions();
 259
 260         VkMemoryRequirements requirements;
 261         vk.GetImageMemoryRequirements(image, requirements);
 262
 263         unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
 264
 265         Block &block = blocks[block_index];
 266         vk.BindImageMemory(image, regions[block.region].memory, block.offset);
 267
 268         return block_index+1;
 269 }
 270
 271 void MemoryAllocator::release(unsigned id)
 272 {
 273         if(!id || id>blocks.size() || !blocks[id-1].allocated)
 274                 throw key_error(id);
 275
 276         unsigned block_index = id-1;
 277         Block &block = blocks[block_index];
 278
 279         block.allocated = false;
 280
 281         Region &region = regions[block.region];
 282         if(region.direct)
 283         {
 284                 const VulkanFunctions &vk = device.get_functions();
 285
 286                 vk.FreeMemory(region.memory);
 287                 region = Region();
 288                 block = Block();
 289                 return;
 290         }
 291
 292         Pool &pool = pools[region.pool];
 293         pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, block.size), block_index);
 294         if((block.prev>=0 && !blocks[block.prev].allocated) || (block.next>=0 && !blocks[block.next].allocated))
 295                 pool.can_consolidate = true;
 296 }
 297
 298 void *MemoryAllocator::map(unsigned id)
 299 {
 300         if(!id || id>blocks.size() || !blocks[id-1].allocated)
 301                 throw key_error(id);
 302
 303         Block &block = blocks[id-1];
 304         Region &region = regions[block.region];
 305         if(!region.mapped_address)
 306         {
 307                 const VulkanFunctions &vk = device.get_functions();
 308                 vk.MapMemory(region.memory, 0, region.size, 0, &region.mapped_address);
 309         }
 310
 311         ++region.map_count;
 312
 313         return static_cast<char *>(region.mapped_address)+block.offset;
 314 }
 315
 316 void MemoryAllocator::unmap(unsigned id)
 317 {
 318         if(!id || id>blocks.size() || !blocks[id-1].allocated)
 319                 throw key_error(id);
 320
 321         Block &block = blocks[id-1];
 322         Region &region = regions[block.region];
 323
 324         if(!regions[block.region].mapped_address)
 325                 throw invalid_operation("MemoryAllocator::unmap");
 326         else if(!--region.map_count)
 327         {
 328                 const VulkanFunctions &vk = device.get_functions();
 329                 vk.UnmapMemory(region.memory);
 330                 region.mapped_address = 0;
 331         }
 332 }
 333
 334 string MemoryAllocator::get_debug() const
 335 {
 336         static const StringCodec::unichar bar_chars[] = { 0xB7, 0x2596, 0x258C, 0x2597, 0x2584, 0x2599, 0x2590, 0x259F, 0x2588 };  // ·▖▌▗▄▙▐▟█
 337
 338         string debug;
 339         for(unsigned i=0; i<pools.size(); ++i)
 340         {
 341                 const Pool &pool = pools[i];
 342
 343                 string pool_debug;
 344                 size_t total_heap = 0;
 345                 size_t total_used = 0;
 346                 for(unsigned j=0; j<regions.size(); ++j)
 347                         if(regions[j].pool==static_cast<int>(i))
 348                         {
 349                                 total_heap += regions[j].size;
 350                                 pool_debug += format("  Region %d: %d kB", j, (regions[j].size+512)/1024);
 351                                 if(regions[j].direct)
 352                                         pool_debug += ", direct";
 353                                 pool_debug += '\n';
 354
 355                                 int block_index = -1;
 356                                 for(unsigned k=0; (block_index<0 && k<blocks.size()); ++k)
 357                                         if(blocks[k].region==static_cast<int>(j) && blocks[k].offset==0)
 358                                                 block_index = k;
 359
 360                                 unsigned slice_index = 0;
 361                                 unsigned slice_data = 0;
 362
 363                                 string bar = "    [";
 364                                 string region_debug;
 365                                 StringCodec::Utf8::Encoder bar_enc;
 366                                 while(block_index>=0)
 367                                 {
 368                                         const Block &block = blocks[block_index];
 369                                         if(block.allocated)
 370                                                 total_used += block.size;
 371                                         const char *state_str = (block.allocated ? "allocated" : "free");
 372                                         region_debug += format("    Block %d: %d bytes at %d, %s\n", block_index, block.size, block.offset, state_str);
 373                                         block_index = block.next;
 374
 375                                         size_t block_end = block.offset+block.size;
 376                                         while(1)
 377                                         {
 378                                                 size_t slice_end = regions[j].size*(slice_index+1)/140;
 379                                                 slice_data |= 1<<(block.allocated+slice_index%2*2);
 380                                                 if(slice_end>block_end)
 381                                                         break;
 382                                                 ++slice_index;
 383                                                 if(slice_index%2==0)
 384                                                 {
 385                                                         slice_data = 5+((slice_data>>1)&5)-(slice_data&5);
 386                                                         bar_enc.encode_char(bar_chars[(slice_data&3)+3*((slice_data>>2)&3)], bar);
 387                                                         slice_data = 0;
 388                                                 }
 389                                         }
 390                                 }
 391
 392                                 bar += "]\n";
 393                                 if(!regions[j].direct)
 394                                         pool_debug += bar;
 395                                 pool_debug += region_debug;
 396                         }
 397
 398                 if(!pool_debug.empty())
 399                 {
 400                         MemoryType t = pool.type;
 401                         const char *type_str = (t==DEVICE_MEMORY ? "device" : t==STAGING_MEMORY ? "staging" :
 402                                 t==STREAMING_MEMORY ? "streaming" : "unknown");
 403                         debug += format("Pool %d: %s, %d/%d kB used\n", i, type_str, (total_used+512)/1024, (total_heap+512)/1024);
 404                         debug += pool_debug;
 405                 }
 406
 407                 if(!pool.free_blocks.empty())
 408                 {
 409                         debug += "  Free blocks:\n";
 410                         for(unsigned j: pool.free_blocks)
 411                         {
 412                                 const char *type = (blocks[j].type==BUFFER ? "buffer" : blocks[j].type==IMAGE ? "image" : "undecided");
 413                                 debug += format("    Block %d: %d bytes, %s\n", j, blocks[j].size, type);
 414                         }
 415                 }
 416         }
 417
 418         return debug;
 419 }
 420
 421 } // namespace GL
 422 } // namespace Msp