From: Mikko Rasa <tdb@tdb.fi>
Date: Tue, 8 Mar 2022 14:00:22 +0000 (+0200)
Subject: Avoid allocating buffers and images too close together
X-Git-Url: http://git.tdb.fi/?a=commitdiff_plain;h=ef891fe4bf86704c37b4269d1aca190405455c12;p=libs%2Fgl.git

Avoid allocating buffers and images too close together

Vulkan requires that both types of resources do not reside on the same
"page" in memory.
---

diff --git a/source/backends/vulkan/memoryallocator.cpp b/source/backends/vulkan/memoryallocator.cpp
index 82155faf..8dbe94eb 100644
--- a/source/backends/vulkan/memoryallocator.cpp
+++ b/source/backends/vulkan/memoryallocator.cpp
@@ -91,9 +91,43 @@ vector<unsigned>::iterator MemoryAllocator::lower_bound_by_size(vector<unsigned>
 	return lower_bound(indices, size, [this](unsigned j, unsigned s){ return blocks[j].size<s; });
 }
 
-unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits, MemoryType type)
+size_t MemoryAllocator::get_alloc_offset(const Block &block, size_t size, size_t align, BlockType type) const
 {
-	unsigned pool_index = find_memory_pool(type_bits, type);
+	size_t offset = block.offset;
+	if(type!=block.type && block.prev>=0 && type!=blocks[block.prev].type)
+	{
+		offset += buffer_image_granularity-1;
+		offset -= offset%buffer_image_granularity;
+	}
+
+	offset += align-1;
+	offset -= offset%align;
+
+	if(type==BUFFER)
+	{
+		size_t offset2 = block.offset+block.size-size;
+		offset2 -= offset2%align;
+		offset = max(offset, offset2);
+	}
+
+	return offset-block.offset;
+}
+
+void MemoryAllocator::update_largest_free(Pool &pool)
+{
+	for(auto i=pool.free_blocks.end(); ((pool.largest_free_buffer<0 || pool.largest_free_image<0) && i!=pool.free_blocks.begin()); )
+	{
+		--i;
+		if(pool.largest_free_buffer<0 && (blocks[*i].type==BUFFER || blocks[*i].type==UNDECIDED))
+			pool.largest_free_buffer = *i;
+		if(pool.largest_free_image<0 && (blocks[*i].type==IMAGE || blocks[*i].type==UNDECIDED))
+			pool.largest_free_image = *i;
+	}
+}
+
+unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits, MemoryType mem_type, BlockType block_type)
+{
+	unsigned pool_index = find_memory_pool(type_bits, mem_type);
 	Pool &pool = pools[pool_index];
 
 	if(size>=direct_alloc_threshold)
@@ -102,21 +136,26 @@ unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits
 		block.region = create_region(pool_index, size, true);
 		block.size = size;
 		block.allocated = true;
+		block.type = block_type;
 
 		blocks.push_back(block);
 		return blocks.size()-1;
 	}
 
-	if(pool.can_consolidate && blocks[pool.free_blocks.back()].size<size+align)
+	int largest_free = (block_type==BUFFER ? pool.largest_free_buffer : pool.largest_free_image);
+	if(pool.can_consolidate && blocks[largest_free].size<size+align)
 		consolidate(pool_index);
 
 	auto i = lower_bound_by_size(pool.free_blocks, size);
 	for(; i!=pool.free_blocks.end(); ++i)
 	{
 		Block &block = blocks[*i];
-		size_t offset = align-1-(block.offset+align-1)%align;
-		if(offset+size<=block.size)
-			break;
+		if(block.type==UNDECIDED || block.type==block_type)
+		{
+			size_t offset = get_alloc_offset(block, size, align, block_type);
+			if(offset+size<=block.size)
+				break;
+		}
 	}
 
 	unsigned block_index;
@@ -126,6 +165,10 @@ unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits
 		pool.free_blocks.erase(i);
 		if(pool.free_blocks.empty())
 			pool.can_consolidate = false;
+		if(static_cast<int>(block_index)==pool.largest_free_buffer)
+			pool.largest_free_buffer = -1;
+		if(static_cast<int>(block_index)==pool.largest_free_image)
+			pool.largest_free_image = -1;
 	}
 	else
 	{
@@ -137,7 +180,7 @@ unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits
 		block_index = blocks.size()-1;
 	}
 
-	size_t offset = align-1-(blocks[block_index].offset+align-1)%align;
+	size_t offset = get_alloc_offset(blocks[block_index], size, align, block_type);
 	if(offset)
 	{
 		unsigned head_index = block_index;
@@ -154,6 +197,9 @@ unsigned MemoryAllocator::allocate(size_t size, size_t align, unsigned type_bits
 	}
 
 	blocks[block_index].allocated = true;
+	blocks[block_index].type = block_type;
+
+	update_largest_free(pool);
 
 	return block_index;
 }
@@ -223,6 +269,10 @@ void MemoryAllocator::consolidate(unsigned pool_index)
 				pool.free_blocks[k] = pool.free_blocks[--i];
 		}
 	}
+
+	pool.largest_free_buffer = -1;
+	pool.largest_free_image = -1;
+	update_largest_free(pool);
 }
 
 void MemoryAllocator::merge_block_with_next(unsigned index)
@@ -245,7 +295,7 @@ unsigned MemoryAllocator::allocate(VkBuffer buffer, MemoryType type)
 	VkMemoryRequirements requirements;
 	vk.GetBufferMemoryRequirements(buffer, requirements);
 
-	unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
+	unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type, BUFFER);
 
 	Block &block = blocks[block_index];
 	vk.BindBufferMemory(buffer, regions[block.region].memory, block.offset);
@@ -260,7 +310,7 @@ unsigned MemoryAllocator::allocate(VkImage image, MemoryType type)
 	VkMemoryRequirements requirements;
 	vk.GetImageMemoryRequirements(image, requirements);
 
-	unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type);
+	unsigned block_index = allocate(requirements.size, requirements.alignment, requirements.memoryTypeBits, type, IMAGE);
 
 	Block &block = blocks[block_index];
 	vk.BindImageMemory(image, regions[block.region].memory, block.offset);
@@ -293,6 +343,17 @@ void MemoryAllocator::release(unsigned id)
 	pool.free_blocks.insert(lower_bound_by_size(pool.free_blocks, block.size), block_index);
 	if((block.prev>=0 && !blocks[block.prev].allocated) || (block.next>=0 && !blocks[block.next].allocated))
 		pool.can_consolidate = true;
+
+	if(block.type==BUFFER)
+	{
+		if(pool.largest_free_buffer<0 || blocks[pool.largest_free_buffer].size<block.size)
+			pool.largest_free_buffer = block_index;
+	}
+	else if(block.type==IMAGE)
+	{
+		if(pool.largest_free_image<0 || blocks[pool.largest_free_image].size<block.size)
+			pool.largest_free_image = block_index;
+	}
 }
 
 void *MemoryAllocator::map(unsigned id)
@@ -369,7 +430,8 @@ string MemoryAllocator::get_debug() const
 					if(block.allocated)
 						total_used += block.size;
 					const char *state_str = (block.allocated ? "allocated" : "free");
-					region_debug += format("    Block %d: %d bytes at %d, %s\n", block_index, block.size, block.offset, state_str);
+					const char *type_str = (block.type==BUFFER ? "buffer" : block.type==IMAGE ? "image" : "undecided");
+					region_debug += format("    Block %d: %d bytes at %d, %s %s\n", block_index, block.size, block.offset, state_str, type_str);
 					block_index = block.next;
 
 					size_t block_end = block.offset+block.size;
@@ -410,7 +472,20 @@ string MemoryAllocator::get_debug() const
 			for(unsigned j: pool.free_blocks)
 			{
 				const char *type = (blocks[j].type==BUFFER ? "buffer" : blocks[j].type==IMAGE ? "image" : "undecided");
-				debug += format("    Block %d: %d bytes, %s\n", j, blocks[j].size, type);
+				debug += format("    Block %d: %d bytes, %s", j, blocks[j].size, type);
+				unsigned largest_flags = (static_cast<int>(j)==pool.largest_free_buffer)+(static_cast<int>(j)==pool.largest_free_image)*2;
+				if(largest_flags)
+				{
+					debug += " (largest free ";
+					if(largest_flags&1)
+						debug += "buffer";
+					if(largest_flags==3)
+						debug += ", ";
+					if(largest_flags&2)
+						debug += "image";
+					debug += ')';
+				}
+				debug += '\n';
 			}
 		}
 	}
diff --git a/source/backends/vulkan/memoryallocator.h b/source/backends/vulkan/memoryallocator.h
index 0b185585..70cda516 100644
--- a/source/backends/vulkan/memoryallocator.h
+++ b/source/backends/vulkan/memoryallocator.h
@@ -22,10 +22,19 @@ enum MemoryType
 class MemoryAllocator: public NonCopyable
 {
 private:
+	enum BlockType
+	{
+		UNDECIDED,
+		BUFFER,
+		IMAGE
+	};
+
 	struct Pool
 	{
 		MemoryType type = UNKNOWN_MEMORY;
 		std::vector<unsigned> free_blocks;
+		int largest_free_buffer = -1;
+		int largest_free_image = -1;
 		bool can_consolidate = false;
 	};
 
@@ -43,6 +52,7 @@ private:
 	{
 		int region = -1;
 		bool allocated = false;
+		BlockType type = UNDECIDED;
 		std::size_t offset = 0;
 		std::size_t size = 0;
 		int prev = -1;
@@ -55,6 +65,7 @@ private:
 	std::size_t default_region_size = 0;
 	std::size_t direct_alloc_threshold = 0;
 	std::size_t min_alignment = 256;
+	std::size_t buffer_image_granularity = 131072;
 	std::vector<Pool> pools;
 	std::vector<Region> regions;
 	std::vector<Block> blocks;
@@ -67,7 +78,9 @@ private:
 	unsigned find_memory_pool(unsigned, MemoryType) const;
 	unsigned create_region(unsigned, size_t, bool);
 	std::vector<unsigned>::iterator lower_bound_by_size(std::vector<unsigned> &, std::size_t) const;
-	unsigned allocate(std::size_t, std::size_t, unsigned, MemoryType);
+	std::size_t get_alloc_offset(const Block &, std::size_t, std::size_t, BlockType) const;
+	void update_largest_free(Pool &);
+	unsigned allocate(std::size_t, std::size_t, unsigned, MemoryType, BlockType);
 	unsigned split_block(unsigned, std::size_t);
 	void consolidate(unsigned);
 	void merge_block_with_next(unsigned);