]> git.tdb.fi Git - libs/gl.git/blob - source/backends/vulkan/memoryallocator.cpp
Align staging memory with an alignment suitable for image transfers
[libs/gl.git] / source / backends / vulkan / memoryallocator.cpp
1 #include <msp/core/algorithm.h>
2 #include <msp/core/maputils.h>
3 #include <msp/graphics/vulkancontext_platform.h>
4 #include "device.h"
5 #include "error.h"
6 #include "memoryallocator.h"
7 #include "vulkan.h"
8
9 using namespace std;
10
11 namespace Msp {
12 namespace GL {
13
14 MemoryAllocator::MemoryAllocator(Device &d):
15         device(d),
16         phys_device(handle_cast<VkPhysicalDevice>(device.get_context().get_private().physical_device))
17 {
18         const VulkanFunctions &vk = device.get_functions();
19
20         VkPhysicalDeviceMemoryProperties mem_props;
21         vk.GetPhysicalDeviceMemoryProperties(mem_props);
22
23         for(unsigned i=0; i<mem_props.memoryHeapCount; ++i)
24                 if(mem_props.memoryHeaps[i].flags&VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)
25                         total_device_memory += mem_props.memoryHeaps[i].size;
26
27         default_region_size = total_device_memory/256;
28         default_region_size -= default_region_size%min_alignment;
29         direct_alloc_threshold = default_region_size/4;
30
31         const VkMemoryPropertyFlags host_flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
32         pools.resize(mem_props.memoryTypeCount);
33         for(unsigned i=0; i<mem_props.memoryTypeCount; ++i)
34         {
35                 VkMemoryPropertyFlags flags = mem_props.memoryTypes[i].propertyFlags;
36                 if(flags&VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
37                 {
38                         if((flags&host_flags)==host_flags)
39                                 pools[i].type = STREAMING_MEMORY;
40                         else
41                                 pools[i].type = DEVICE_MEMORY;
42                 }
43                 else if((flags&host_flags)==host_flags)
44                         pools[i].type = STAGING_MEMORY;
45         }
46 }
47
48 MemoryAllocator::~MemoryAllocator()
49 {
50         const VulkanFunctions &vk = device.get_functions();
51
52         for(Region &r: regions)
53                 if(r.memory)
54                         vk.FreeMemory(r.memory);
55 }
56
57 unsigned MemoryAllocator::find_memory_pool(unsigned mask, MemoryType type)
58 {
59         for(unsigned i=0; i<pools.size(); ++i)
60                 if((mask&(1<<i)) && pools[i].type==type)
61                         return i;
62         if(type==DEVICE_MEMORY || type==STAGING_MEMORY)
63                 return find_memory_pool(mask, STREAMING_MEMORY);
64         throw runtime_error("Unable to find suitable memory type");
65 }
66
67 unsigned MemoryAllocator::create_region(unsigned pool_index, size_t size, bool direct)
68 {
69         const VulkanFunctions &vk = device.get_functions();
70
71         VkMemoryAllocateInfo alloc_info = { };
72         alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
73         alloc_info.allocationSize = size;
74         alloc_info.memoryTypeIndex = pool_index;
75
76         Region region;
77         vk.AllocateMemory(alloc_info, region.memory);
78
79         region.pool = pool_index;
80         region.direct = direct;
81         region.size = size;
82         regions.push_back(region);
83
84         return regions.size()-1;
85 }
86
87 vector<unsigned>::iterator MemoryAllocator::lower_bound_by_size(vector<unsigned> &indices, size_t size)
88 {
89         return lower_bound(indices, size, [this](unsigned j, unsigned s){ return blocks[j].size<s; });
90 }
91
92 unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits, MemoryType type)
93 {
94         unsigned pool_index = find_memory_pool(type_bits, type);
95         Pool &pool = pools[pool_index];
96
97         if(size>=direct_alloc_threshold)
98         {
99                 Block block;
100                 block.region = create_region(pool_index, size, true);
101                 block.size = size;
102                 block.allocated = true;
103
104                 blocks.push_back(block);
105                 return blocks.size()-1;
106         }
107
108         if(pool.can_consolidate && blocks[pool.free_blocks.back()].size<size+align)
109                 consolidate(pool_index);
110
111         auto i = lower_bound_by_size(pool.free_blocks, size);
112         for(; i!=pool.free_blocks.end(); ++i)
113         {
114                 Block &block = blocks[*i];
115                 size_t offset = align-1-(block.offset+align-1)%align;
116                 if(offset+size<=block.size)
117                         break;
118         }
119
120         unsigned block_index;
121         if(i!=pool.free_blocks.end())
122         {
123                 block_index = *i;
124                 pool.free_blocks.erase(i);
125                 if(pool.free_blocks.empty())
126                         pool.can_consolidate = false;
127         }
128         else
129         {
130                 Block block;
131                 block.region = create_region(pool_index, default_region_size, false);
132                 block.size = default_region_size;
133
134                 blocks.push_back(block);
135                 block_index = blocks.size()-1;
136         }
137
138         size_t offset = align-1-(blocks[block_index].offset+align-1)%align;
139         if(offset)
140         {
141                 unsigned head_index = block_index;
142                 block_index = split_block(block_index, offset);
143                 pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, blocks[head_index].size), head_index);
144         }
145
146         size += min_alignment-1;
147         size -= size%min_alignment;
148         if(blocks[block_index].size>=size+min_alignment)
149         {
150                 unsigned tail_index = split_block(block_index, size);
151                 pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, blocks[tail_index].size), tail_index);
152         }
153
154         blocks[block_index].allocated = true;
155
156         return block_index;
157 }
158
159 unsigned MemoryAllocator::split_block(unsigned index, size_t head_size)
160 {
161         blocks.emplace_back();
162         Block &block = blocks[index];
163         Block &tail = blocks.back();
164         unsigned tail_index = blocks.size()-1;
165
166         tail.region = block.region;
167         tail.offset = block.offset+head_size;
168         tail.size = block.size-head_size;
169         tail.prev = index;
170         tail.next = block.next;
171
172         block.size = head_size;
173         block.next = tail_index;
174
175         return tail_index;
176 }
177
178 void MemoryAllocator::consolidate(unsigned pool_index)
179 {
180         Pool &pool = pools[pool_index];
181
182         vector<unsigned> merged_blocks;
183         unsigned i = 0;
184         for(unsigned j=0; j<pool.free_blocks.size(); ++j)
185         {
186                 unsigned block_index = pool.free_blocks[j];
187                 Block &block = blocks[block_index];
188                 if(!block.allocated)
189                 {
190                         if(block.prev<0 || blocks[block.prev].allocated)
191                         {
192                                 if(block.next>=0 && !blocks[block.next].allocated)
193                                 {
194                                         merge_block_with_next(block_index);
195
196                                         while(block.next>=0 && !blocks[block.next].allocated)
197                                                 merge_block_with_next(block_index);
198
199                                         merged_blocks.insert(lower_bound_by_size(merged_blocks, block.size), block_index);
200                                 }
201                         }
202                         else
203                                 continue;
204                 }
205
206                 if(j!=i)
207                         pool.free_blocks[i] = block_index;
208                 ++i;
209         }
210
211         pool.free_blocks.resize(i+merged_blocks.size());
212
213         if(!merged_blocks.empty())
214         {
215                 unsigned j = merged_blocks.size();
216                 for(unsigned k=pool.free_blocks.size()-1; j; --k)
217                 {
218                         if(!i || blocks[merged_blocks[j-1]].size>blocks[pool.free_blocks[i-1]].size)
219                                 pool.free_blocks[k] = merged_blocks[--j];
220                         else
221                                 pool.free_blocks[k] = pool.free_blocks[--i];
222                 }
223         }
224 }
225
226 void MemoryAllocator::merge_block_with_next(unsigned index)
227 {
228         Block &block = blocks[index];
229
230         Block &next = blocks[block.next];
231         block.size += next.size;
232         block.next = next.next;
233         if(block.next>=0)
234                 blocks[block.next].prev = index;
235
236         next = Block();
237 }
238
239 unsigned MemoryAllocator::allocate(VkBuffer buffer, MemoryType type)
240 {
241         const VulkanFunctions &vk = device.get_functions();
242
243         VkMemoryRequirements requirements;
244         vk.GetBufferMemoryRequirements(buffer, requirements);
245
246         unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
247
248         Block &block = blocks[block_index];
249         vk.BindBufferMemory(buffer, regions[block.region].memory, block.offset);
250
251         return block_index+1;
252 }
253
254 unsigned MemoryAllocator::allocate(VkImage image, MemoryType type)
255 {
256         const VulkanFunctions &vk = device.get_functions();
257
258         VkMemoryRequirements requirements;
259         vk.GetImageMemoryRequirements(image, requirements);
260
261         unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
262
263         Block &block = blocks[block_index];
264         vk.BindImageMemory(image, regions[block.region].memory, block.offset);
265
266         return block_index+1;
267 }
268
269 void MemoryAllocator::release(unsigned id)
270 {
271         if(!id || id>blocks.size() || !blocks[id-1].allocated)
272                 throw key_error(id);
273
274         unsigned block_index = id-1;
275         Block &block = blocks[block_index];
276
277         block.allocated = false;
278
279         Region &region = regions[block.region];
280         if(region.direct)
281         {
282                 const VulkanFunctions &vk = device.get_functions();
283
284                 vk.FreeMemory(region.memory);
285                 region = Region();
286                 block = Block();
287                 return;
288         }
289
290         Pool &pool = pools[region.pool];
291         pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, block.size), block_index);
292         if((block.prev>=0 && !blocks[block.prev].allocated) || (block.next>=0 && !blocks[block.next].allocated))
293                 pool.can_consolidate = true;
294 }
295
296 void *MemoryAllocator::map(unsigned id)
297 {
298         if(!id || id>blocks.size() || !blocks[id-1].allocated)
299                 throw key_error(id);
300
301         Block &block = blocks[id-1];
302         Region &region = regions[block.region];
303         if(!region.mapped_address)
304         {
305                 const VulkanFunctions &vk = device.get_functions();
306                 vk.MapMemory(region.memory, 0, region.size, 0, &region.mapped_address);
307         }
308
309         ++region.map_count;
310
311         return static_cast<char *>(region.mapped_address)+block.offset;
312 }
313
314 void MemoryAllocator::unmap(unsigned id)
315 {
316         if(!id || id>blocks.size() || !blocks[id-1].allocated)
317                 throw key_error(id);
318
319         Block &block = blocks[id-1];
320         Region &region = regions[block.region];
321
322         if(!regions[block.region].mapped_address)
323                 throw invalid_operation("MemoryAllocator::unmap");
324         else if(!--region.map_count)
325         {
326                 const VulkanFunctions &vk = device.get_functions();
327                 vk.UnmapMemory(region.memory);
328                 region.mapped_address = 0;
329         }
330 }
331
332 } // namespace GL
333 } // namespace Msp