Avoid allocating buffers and images too close together

Vulkan requires that both types of resources do not reside on the same
"page" in memory.

diff --git a/source/backends/vulkan/memoryallocator.cpp b/source/backends/vulkan/memoryallocator.cpp
index 82155faf41624ef03829cd994ebbbfc2eab246ab..8dbe94eb5858b66d79f5723ecb314f02859b7b79 100644 (file)
--- a/source/backends/vulkan/memoryallocator.cpp
+++ b/source/backends/vulkan/memoryallocator.cpp
@@ -91,9 +91,43 @@ vector<unsigned>::iterator MemoryAllocator::lower_bound_by_size(vector<unsigned>
        return lower_bound(indices, size, [this](unsigned j, unsigned s){ return blocks[j].size<s; });
 }
 
-unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits, MemoryType type)
+size_t MemoryAllocator::get_alloc_offset(const Block &block, size_t size, size_t align, BlockType type) const
 {
-       unsigned pool_index = find_memory_pool(type_bits, type);
+       size_t offset = block.offset;
+       if(type!=block.type && block.prev>=0 && type!=blocks[block.prev].type)
+       {
+               offset += buffer_image_granularity-1;
+               offset -= offset%buffer_image_granularity;
+       }
+
+       offset += align-1;
+       offset -= offset%align;
+
+       if(type==BUFFER)
+       {
+               size_t offset2 = block.offset+block.size-size;
+               offset2 -= offset2%align;
+               offset = max(offset, offset2);
+       }
+
+       return offset-block.offset;
+}
+
+void MemoryAllocator::update_largest_free(Pool &pool)
+{
+       for(auto i=pool.free_blocks.end(); ((pool.largest_free_buffer<0 || pool.largest_free_image<0) && i!=pool.free_blocks.begin()); )
+       {
+               --i;
+               if(pool.largest_free_buffer<0 && (blocks[*i].type==BUFFER || blocks[*i].type==UNDECIDED))
+                       pool.largest_free_buffer = *i;
+               if(pool.largest_free_image<0 && (blocks[*i].type==IMAGE || blocks[*i].type==UNDECIDED))
+                       pool.largest_free_image = *i;
+       }
+}
+
+unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits, MemoryType mem_type, BlockType block_type)
+{
+       unsigned pool_index = find_memory_pool(type_bits, mem_type);
        Pool &pool = pools[pool_index];
 
        if(size>=direct_alloc_threshold)
@@ -102,21 +136,26 @@ unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits
                block.region = create_region(pool_index, size, true);
                block.size = size;
                block.allocated = true;
+               block.type = block_type;
 
                blocks.push_back(block);
                return blocks.size()-1;
        }
 
-       if(pool.can_consolidate && blocks[pool.free_blocks.back()].size<size+align)
+       int largest_free = (block_type==BUFFER ? pool.largest_free_buffer : pool.largest_free_image);
+       if(pool.can_consolidate && blocks[largest_free].size<size+align)
                consolidate(pool_index);
 
        auto i = lower_bound_by_size(pool.free_blocks, size);
        for(; i!=pool.free_blocks.end(); ++i)
        {
                Block &block = blocks[*i];
-               size_t offset = align-1-(block.offset+align-1)%align;
-               if(offset+size<=block.size)
-                       break;
+               if(block.type==UNDECIDED || block.type==block_type)
+               {
+                       size_t offset = get_alloc_offset(block, size, align, block_type);
+                       if(offset+size<=block.size)
+                               break;
+               }
        }
 
        unsigned block_index;
@@ -126,6 +165,10 @@ unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits
                pool.free_blocks.erase(i);
                if(pool.free_blocks.empty())
                        pool.can_consolidate = false;
+               if(static_cast<int>(block_index)==pool.largest_free_buffer)
+                       pool.largest_free_buffer = -1;
+               if(static_cast<int>(block_index)==pool.largest_free_image)
+                       pool.largest_free_image = -1;
        }
        else
        {
@@ -137,7 +180,7 @@ unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits
                block_index = blocks.size()-1;
        }
 
-       size_t offset = align-1-(blocks[block_index].offset+align-1)%align;
+       size_t offset = get_alloc_offset(blocks[block_index], size, align, block_type);
        if(offset)
        {
                unsigned head_index = block_index;
@@ -154,6 +197,9 @@ unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits
        }
 
        blocks[block_index].allocated = true;
+       blocks[block_index].type = block_type;
+
+       update_largest_free(pool);
 
        return block_index;
 }
@@ -223,6 +269,10 @@ void MemoryAllocator::consolidate(unsigned pool_index)
                                pool.free_blocks[k] = pool.free_blocks[--i];
                }
        }
+
+       pool.largest_free_buffer = -1;
+       pool.largest_free_image = -1;
+       update_largest_free(pool);
 }
 
 void MemoryAllocator::merge_block_with_next(unsigned index)
@@ -245,7 +295,7 @@ unsigned MemoryAllocator::allocate(VkBuffer buffer, MemoryType type)
        VkMemoryRequirements requirements;
        vk.GetBufferMemoryRequirements(buffer, requirements);
 
-       unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
+       unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type, BUFFER);
 
        Block &block = blocks[block_index];
        vk.BindBufferMemory(buffer, regions[block.region].memory, block.offset);
@@ -260,7 +310,7 @@ unsigned MemoryAllocator::allocate(VkImage image, MemoryType type)
        VkMemoryRequirements requirements;
        vk.GetImageMemoryRequirements(image, requirements);
 
-       unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
+       unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type, IMAGE);
 
        Block &block = blocks[block_index];
        vk.BindImageMemory(image, regions[block.region].memory, block.offset);
@@ -293,6 +343,17 @@ void MemoryAllocator::release(unsigned id)
        pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, block.size), block_index);
        if((block.prev>=0 && !blocks[block.prev].allocated) || (block.next>=0 && !blocks[block.next].allocated))
                pool.can_consolidate = true;
+
+       if(block.type==BUFFER)
+       {
+               if(pool.largest_free_buffer<0 || blocks[pool.largest_free_buffer].size<block.size)
+                       pool.largest_free_buffer = block_index;
+       }
+       else if(block.type==IMAGE)
+       {
+               if(pool.largest_free_image<0 || blocks[pool.largest_free_image].size<block.size)
+                       pool.largest_free_image = block_index;
+       }
 }
 
 void *MemoryAllocator::map(unsigned id)
@@ -369,7 +430,8 @@ string MemoryAllocator::get_debug() const
                                        if(block.allocated)
                                                total_used += block.size;
                                        const char *state_str = (block.allocated ? "allocated" : "free");
-                                       region_debug += format("    Block %d: %d bytes at %d, %s\n", block_index, block.size, block.offset, state_str);
+                                       const char *type_str = (block.type==BUFFER ? "buffer" : block.type==IMAGE ? "image" : "undecided");
+                                       region_debug += format("    Block %d: %d bytes at %d, %s %s\n", block_index, block.size, block.offset, state_str, type_str);
                                        block_index = block.next;
 
                                        size_t block_end = block.offset+block.size;
@@ -410,7 +472,20 @@ string MemoryAllocator::get_debug() const
                        for(unsigned j: pool.free_blocks)
                        {
                                const char *type = (blocks[j].type==BUFFER ? "buffer" : blocks[j].type==IMAGE ? "image" : "undecided");
-                               debug += format("    Block %d: %d bytes, %s\n", j, blocks[j].size, type);
+                               debug += format("    Block %d: %d bytes, %s", j, blocks[j].size, type);
+                               unsigned largest_flags = (static_cast<int>(j)==pool.largest_free_buffer)+(static_cast<int>(j)==pool.largest_free_image)*2;
+                               if(largest_flags)
+                               {
+                                       debug += " (largest free ";
+                                       if(largest_flags&1)
+                                               debug += "buffer";
+                                       if(largest_flags==3)
+                                               debug += ", ";
+                                       if(largest_flags&2)
+                                               debug += "image";
+                                       debug += ')';
+                               }
+                               debug += '\n';
                        }
                }
        }

diff --git a/source/backends/vulkan/memoryallocator.h b/source/backends/vulkan/memoryallocator.h
index 0b185585e89e14bab48536449c7c68c8d1bbcfd3..70cda51643137e89e5af061fccd4b7f2e02b6460 100644 (file)
--- a/source/backends/vulkan/memoryallocator.h
+++ b/source/backends/vulkan/memoryallocator.h
@@ -22,10 +22,19 @@ enum MemoryType
 class MemoryAllocator: public NonCopyable
 {
 private:
+       enum BlockType
+       {
+               UNDECIDED,
+               BUFFER,
+               IMAGE
+       };
+
        struct Pool
        {
                MemoryType type = UNKNOWN_MEMORY;
                std::vector<unsigned> free_blocks;
+               int largest_free_buffer = -1;
+               int largest_free_image = -1;
                bool can_consolidate = false;
        };
 
@@ -43,6 +52,7 @@ private:
        {
                int region = -1;
                bool allocated = false;
+               BlockType type = UNDECIDED;
                std::size_t offset = 0;
                std::size_t size = 0;
                int prev = -1;
@@ -55,6 +65,7 @@ private:
        std::size_t default_region_size = 0;
        std::size_t direct_alloc_threshold = 0;
        std::size_t min_alignment = 256;
+       std::size_t buffer_image_granularity = 131072;
        std::vector<Pool> pools;
        std::vector<Region> regions;
        std::vector<Block> blocks;
@@ -67,7 +78,9 @@ private:
        unsigned find_memory_pool(unsigned, MemoryType) const;
        unsigned create_region(unsigned, size_t, bool);
        std::vector<unsigned>::iterator lower_bound_by_size(std::vector<unsigned> &, std::size_t) const;
-       unsigned allocate(std::size_t, std::size_t, unsigned, MemoryType);
+       std::size_t get_alloc_offset(const Block &, std::size_t, std::size_t, BlockType) const;
+       void update_largest_free(Pool &);
+       unsigned allocate(std::size_t, std::size_t, unsigned, MemoryType, BlockType);
        unsigned split_block(unsigned, std::size_t);
        void consolidate(unsigned);
        void merge_block_with_next(unsigned);