source/backends/vulkan/memoryallocator.cpp

   1 #include <msp/core/algorithm.h>
   2 #include <msp/core/maputils.h>
   3 #include <msp/graphics/vulkancontext_platform.h>
   4 #include "device.h"
   5 #include "error.h"
   6 #include "memoryallocator.h"
   7 #include "vulkan.h"
   8
   9 using namespace std;
  10
  11 namespace Msp {
  12 namespace GL {
  13
  14 MemoryAllocator::MemoryAllocator(Device &d):
  15         device(d),
  16         phys_device(handle_cast<VkPhysicalDevice>(device.get_context().get_private().physical_device))
  17 {
  18         const VulkanFunctions &vk = device.get_functions();
  19
  20         VkPhysicalDeviceMemoryProperties mem_props;
  21         vk.GetPhysicalDeviceMemoryProperties(mem_props);
  22
  23         for(unsigned i=0; i<mem_props.memoryHeapCount; ++i)
  24                 if(mem_props.memoryHeaps[i].flags&VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
  25                         total_device_memory += mem_props.memoryHeaps[i].size;
  26
  27         default_region_size = total_device_memory/256;
  28         default_region_size -= default_region_size%min_alignment;
  29         direct_alloc_threshold = default_region_size/4;
  30
  31         const VkMemoryPropertyFlags host_flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
  32         pools.resize(mem_props.memoryTypeCount);
  33         for(unsigned i=0; i<mem_props.memoryTypeCount; ++i)
  34         {
  35                 VkMemoryPropertyFlags flags = mem_props.memoryTypes[i].propertyFlags;
  36                 if(flags&VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
  37                 {
  38                         if((flags&host_flags)==host_flags)
  39                                 pools[i].type = STREAMING_MEMORY;
  40                         else
  41                                 pools[i].type = DEVICE_MEMORY;
  42                 }
  43                 else if((flags&host_flags)==host_flags)
  44                         pools[i].type = STAGING_MEMORY;
  45         }
  46 }
  47
  48 MemoryAllocator::~MemoryAllocator()
  49 {
  50         const VulkanFunctions &vk = device.get_functions();
  51
  52         for(Region &r: regions)
  53                 if(r.memory)
  54                         vk.FreeMemory(r.memory);
  55 }
  56
  57 unsigned MemoryAllocator::find_memory_pool(unsigned mask, MemoryType type)
  58 {
  59         for(unsigned i=0; i<pools.size(); ++i)
  60                 if((mask&(1<<i)) && pools[i].type==type)
  61                         return i;
  62         if(type==DEVICE_MEMORY || type==STAGING_MEMORY)
  63                 return find_memory_pool(mask, STREAMING_MEMORY);
  64         throw runtime_error("Unable to find suitable memory type");
  65 }
  66
  67 unsigned MemoryAllocator::create_region(unsigned pool_index, size_t size, bool direct)
  68 {
  69         const VulkanFunctions &vk = device.get_functions();
  70
  71         VkMemoryAllocateInfo alloc_info = { };
  72         alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
  73         alloc_info.allocationSize = size;
  74         alloc_info.memoryTypeIndex = pool_index;
  75
  76         Region region;
  77         vk.AllocateMemory(alloc_info, region.memory);
  78
  79         region.pool = pool_index;
  80         region.direct = direct;
  81         region.size = size;
  82         regions.push_back(region);
  83
  84         return regions.size()-1;
  85 }
  86
  87 vector<unsigned>::iterator MemoryAllocator::lower_bound_by_size(vector<unsigned> &indices, size_t size)
  88 {
  89         return lower_bound(indices, size, [this](unsigned j, unsigned s){ return blocks[j].size<s; });
  90 }
  91
  92 unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits, MemoryType type)
  93 {
  94         unsigned pool_index = find_memory_pool(type_bits, type);
  95         Pool &pool = pools[pool_index];
  96
  97         if(size>=direct_alloc_threshold)
  98         {
  99                 Block block;
 100                 block.region = create_region(pool_index, size, true);
 101                 block.size = size;
 102                 block.allocated = true;
 103
 104                 blocks.push_back(block);
 105                 return blocks.size()-1;
 106         }
 107
 108         if(pool.can_consolidate && blocks[pool.free_blocks.back()].size<size+align)
 109                 consolidate(pool_index);
 110
 111         auto i = lower_bound_by_size(pool.free_blocks, size);
 112         for(; i!=pool.free_blocks.end(); ++i)
 113         {
 114                 Block &block = blocks[*i];
 115                 size_t offset = align-1-(block.offset+align-1)%align;
 116                 if(offset+size<=block.size)
 117                         break;
 118         }
 119
 120         unsigned block_index;
 121         if(i!=pool.free_blocks.end())
 122         {
 123                 block_index = *i;
 124                 pool.free_blocks.erase(i);
 125                 if(pool.free_blocks.empty())
 126                         pool.can_consolidate = false;
 127         }
 128         else
 129         {
 130                 Block block;
 131                 block.region = create_region(pool_index, default_region_size, false);
 132                 block.size = default_region_size;
 133
 134                 blocks.push_back(block);
 135                 block_index = blocks.size()-1;
 136         }
 137
 138         size_t offset = align-1-(blocks[block_index].offset+align-1)%align;
 139         if(offset)
 140         {
 141                 unsigned head_index = block_index;
 142                 block_index = split_block(block_index, offset);
 143                 pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, blocks[head_index].size), head_index);
 144         }
 145
 146         size += min_alignment-1;
 147         size -= size%min_alignment;
 148         if(blocks[block_index].size>=size+min_alignment)
 149         {
 150                 unsigned tail_index = split_block(block_index, size);
 151                 pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, blocks[tail_index].size), tail_index);
 152         }
 153
 154         blocks[block_index].allocated = true;
 155
 156         return block_index;
 157 }
 158
 159 unsigned MemoryAllocator::split_block(unsigned index, size_t head_size)
 160 {
 161         blocks.emplace_back();
 162         Block &block = blocks[index];
 163         Block &tail = blocks.back();
 164         unsigned tail_index = blocks.size()-1;
 165
 166         tail.region = block.region;
 167         tail.offset = block.offset+head_size;
 168         tail.size = block.size-head_size;
 169         tail.prev = index;
 170         tail.next = block.next;
 171
 172         block.size = head_size;
 173         block.next = tail_index;
 174
 175         return tail_index;
 176 }
 177
 178 void MemoryAllocator::consolidate(unsigned pool_index)
 179 {
 180         Pool &pool = pools[pool_index];
 181
 182         vector<unsigned> merged_blocks;
 183         unsigned i = 0;
 184         for(unsigned j=0; j<pool.free_blocks.size(); ++j)
 185         {
 186                 unsigned block_index = pool.free_blocks[j];
 187                 Block &block = blocks[block_index];
 188                 if(!block.allocated)
 189                 {
 190                         if(block.prev<0 || blocks[block.prev].allocated)
 191                         {
 192                                 if(block.next>=0 && !blocks[block.next].allocated)
 193                                 {
 194                                         merge_block_with_next(block_index);
 195
 196                                         while(block.next>=0 && !blocks[block.next].allocated)
 197                                                 merge_block_with_next(block_index);
 198
 199                                         merged_blocks.insert(lower_bound_by_size(merged_blocks, block.size), block_index);
 200                                 }
 201                         }
 202                         else
 203                                 continue;
 204                 }
 205
 206                 if(j!=i)
 207                         pool.free_blocks[i] = block_index;
 208                 ++i;
 209         }
 210
 211         pool.free_blocks.resize(i+merged_blocks.size());
 212
 213         if(!merged_blocks.empty())
 214         {
 215                 unsigned j = merged_blocks.size();
 216                 for(unsigned k=pool.free_blocks.size()-1; j; --k)
 217                 {
 218                         if(!i || blocks[merged_blocks[j-1]].size>blocks[pool.free_blocks[i-1]].size)
 219                                 pool.free_blocks[k] = merged_blocks[--j];
 220                         else
 221                                 pool.free_blocks[k] = pool.free_blocks[--i];
 222                 }
 223         }
 224 }
 225
 226 void MemoryAllocator::merge_block_with_next(unsigned index)
 227 {
 228         Block &block = blocks[index];
 229
 230         Block &next = blocks[block.next];
 231         block.size += next.size;
 232         block.next = next.next;
 233         if(block.next>=0)
 234                 blocks[block.next].prev = index;
 235
 236         next = Block();
 237 }
 238
 239 unsigned MemoryAllocator::allocate(VkBuffer buffer, MemoryType type)
 240 {
 241         const VulkanFunctions &vk = device.get_functions();
 242
 243         VkMemoryRequirements requirements;
 244         vk.GetBufferMemoryRequirements(buffer, requirements);
 245
 246         unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
 247
 248         Block &block = blocks[block_index];
 249         vk.BindBufferMemory(buffer, regions[block.region].memory, block.offset);
 250
 251         return block_index+1;
 252 }
 253
 254 unsigned MemoryAllocator::allocate(VkImage image, MemoryType type)
 255 {
 256         const VulkanFunctions &vk = device.get_functions();
 257
 258         VkMemoryRequirements requirements;
 259         vk.GetImageMemoryRequirements(image, requirements);
 260
 261         unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
 262
 263         Block &block = blocks[block_index];
 264         vk.BindImageMemory(image, regions[block.region].memory, block.offset);
 265
 266         return block_index+1;
 267 }
 268
 269 void MemoryAllocator::release(unsigned id)
 270 {
 271         if(!id || id>blocks.size() || !blocks[id-1].allocated)
 272                 throw key_error(id);
 273
 274         unsigned block_index = id-1;
 275         Block &block = blocks[block_index];
 276
 277         block.allocated = false;
 278
 279         Region &region = regions[block.region];
 280         if(region.direct)
 281         {
 282                 const VulkanFunctions &vk = device.get_functions();
 283
 284                 vk.FreeMemory(region.memory);
 285                 region = Region();
 286                 block = Block();
 287                 return;
 288         }
 289
 290         Pool &pool = pools[region.pool];
 291         pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, block.size), block_index);
 292         if((block.prev>=0 && !blocks[block.prev].allocated) || (block.next>=0 && !blocks[block.next].allocated))
 293                 pool.can_consolidate = true;
 294 }
 295
 296 void *MemoryAllocator::map(unsigned id)
 297 {
 298         if(!id || id>blocks.size() || !blocks[id-1].allocated)
 299                 throw key_error(id);
 300
 301         Block &block = blocks[id-1];
 302         Region &region = regions[block.region];
 303         if(!region.mapped_address)
 304         {
 305                 const VulkanFunctions &vk = device.get_functions();
 306                 vk.MapMemory(region.memory, 0, region.size, 0, &region.mapped_address);
 307         }
 308
 309         ++region.map_count;
 310
 311         return static_cast<char *>(region.mapped_address)+block.offset;
 312 }
 313
 314 void MemoryAllocator::unmap(unsigned id)
 315 {
 316         if(!id || id>blocks.size() || !blocks[id-1].allocated)
 317                 throw key_error(id);
 318
 319         Block &block = blocks[id-1];
 320         Region &region = regions[block.region];
 321
 322         if(!regions[block.region].mapped_address)
 323                 throw invalid_operation("MemoryAllocator::unmap");
 324         else if(!--region.map_count)
 325         {
 326                 const VulkanFunctions &vk = device.get_functions();
 327                 vk.UnmapMemory(region.memory);
 328                 region.mapped_address = 0;
 329         }
 330 }
 331
 332 } // namespace GL
 333 } // namespace Msp