From: Mikko Rasa <tdb@tdb.fi>
Date: Tue, 14 Dec 2021 12:37:57 +0000 (+0200)
Subject: Use secondary command buffers to record render pass contents
X-Git-Url: http://git.tdb.fi/?a=commitdiff_plain;h=ce9cd2d34cbcbd772c9fa753b48b8e6a4d80f1e3;p=libs%2Fgl.git

Use secondary command buffers to record render pass contents

This allows greater flexibility for ordering transfers and barriers
relative to render passes.
---

diff --git a/source/backends/vulkan/commands_backend.cpp b/source/backends/vulkan/commands_backend.cpp
index 9c074c87..137a817d 100644
--- a/source/backends/vulkan/commands_backend.cpp
+++ b/source/backends/vulkan/commands_backend.cpp
@@ -9,6 +9,7 @@
 #include "pipelinestate.h"
 #include "rect.h"
 #include "semaphore.h"
+#include "structurebuilder.h"
 #include "swapchaintexture.h"
 #include "vulkan.h"
 
@@ -28,7 +29,7 @@ VulkanCommands::~VulkanCommands()
 	vk.QueueWaitIdle();
 }
 
-void VulkanCommands::begin_buffer()
+void VulkanCommands::begin_buffer(VkRenderPass render_pass)
 {
 	if(!current_pool)
 		throw invalid_operation("VulkanCommands::begin_buffer");
@@ -41,26 +42,41 @@ void VulkanCommands::begin_buffer()
 		current_pool->in_use = true;
 	}
 
-	if(current_pool->next_buffer>=current_pool->buffers.size())
+	CommandBuffers &buffers = (render_pass ? current_pool->secondary : current_pool->primary);
+
+	if(buffers.next_buffer>=buffers.buffers.size())
 	{
 		VkCommandBufferAllocateInfo alloc_info = { };
 		alloc_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
 		alloc_info.commandPool = handle_cast<::VkCommandPool>(current_pool->pool);
-		alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
+		alloc_info.level = (render_pass ? VK_COMMAND_BUFFER_LEVEL_SECONDARY : VK_COMMAND_BUFFER_LEVEL_PRIMARY);
 		alloc_info.commandBufferCount = 1;
 
 		VkCommandBuffer buffer;
 		vk.AllocateCommandBuffers(alloc_info, &buffer);
-		current_pool->buffers.push_back(buffer);
+		buffers.buffers.push_back(buffer);
 	}
 
-	current_buffer = current_pool->buffers[current_pool->next_buffer++];
+	VkCommandBuffer buffer = buffers.buffers[buffers.next_buffer++];
+	(render_pass ? pass_buffer : primary_buffer) = buffer;
 
 	VkCommandBufferBeginInfo begin_info = { };
 	begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
 	begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
+	if(render_pass)
+		begin_info.flags |= VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT;
+
+	VkCommandBufferInheritanceInfo inherit_info = { };
+	if(render_pass)
+	{
+		inherit_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO;
+		inherit_info.renderPass = handle_cast<::VkRenderPass>(render_pass);
+		inherit_info.subpass = 0;
 
-	vk.BeginCommandBuffer(current_buffer, begin_info);
+		begin_info.pInheritanceInfo = &inherit_info;
+	}
+
+	vk.BeginCommandBuffer(buffer, begin_info);
 }
 
 void VulkanCommands::begin_render_pass(bool clear, const ClearValue *clear_values)
@@ -71,12 +87,8 @@ void VulkanCommands::begin_render_pass(bool clear, const ClearValue *clear_value
 
 	viewport = pipeline_state->get_viewport();
 
-	const VulkanFunctions &vk = device.get_functions();
-
-	if(!current_buffer)
-		begin_buffer();
-
-	device.get_transfer_queue().dispatch_transfers(current_buffer);
+	if(!primary_buffer)
+		begin_buffer(0);
 
 	Synchronizer &sync = device.get_synchronizer();
 	sync.reset();
@@ -92,27 +104,31 @@ void VulkanCommands::begin_render_pass(bool clear, const ClearValue *clear_value
 
 	framebuffer->refresh();
 
-	sync.barrier(current_buffer);
+	sync.barrier(primary_buffer);
+
+	begin_buffer(render_pass);
+
+	StructureBuilder sb(pass_begin_info, 2);
+	VkRenderPassBeginInfo *&begin_info = sb.add<VkRenderPassBeginInfo>(1);
+	VkClearValue *&vk_clear_values = sb.add<VkClearValue>(FrameFormat::MAX_ATTACHMENTS);
 
-	VkRenderPassBeginInfo begin_info = { };
-	begin_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
-	begin_info.renderPass = handle_cast<::VkRenderPass>(render_pass);
-	begin_info.framebuffer = handle_cast<::VkFramebuffer>(framebuffer->handle);
+	begin_info->sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
+	begin_info->renderPass = handle_cast<::VkRenderPass>(render_pass);
+	begin_info->framebuffer = handle_cast<::VkFramebuffer>(framebuffer->handle);
 
 	if(viewport)
 	{
-		begin_info.renderArea.offset.x = viewport->left;
-		begin_info.renderArea.offset.y = viewport->bottom;
-		begin_info.renderArea.extent.width = viewport->width;
-		begin_info.renderArea.extent.height = viewport->height;
+		begin_info->renderArea.offset.x = viewport->left;
+		begin_info->renderArea.offset.y = viewport->bottom;
+		begin_info->renderArea.extent.width = viewport->width;
+		begin_info->renderArea.extent.height = viewport->height;
 	}
 	else
 	{
-		begin_info.renderArea.extent.width = framebuffer->get_width();
-		begin_info.renderArea.extent.height = framebuffer->get_height();
+		begin_info->renderArea.extent.width = framebuffer->get_width();
+		begin_info->renderArea.extent.height = framebuffer->get_height();
 	}
 
-	VkClearValue vk_clear_values[7];
 	if(clear_values)
 	{
 		unsigned i = 0;
@@ -132,20 +148,25 @@ void VulkanCommands::begin_render_pass(bool clear, const ClearValue *clear_value
 			++i;
 		}
 
-		begin_info.clearValueCount = framebuffer->get_format().size();
-		begin_info.pClearValues = vk_clear_values;
+		begin_info->clearValueCount = framebuffer->get_format().size();
+		begin_info->pClearValues = vk_clear_values;
 	}
-
-	vk.CmdBeginRenderPass(current_buffer, begin_info, VK_SUBPASS_CONTENTS_INLINE);
 }
 
 void VulkanCommands::end_render_pass()
 {
 	const VulkanFunctions &vk = device.get_functions();
 
-	vk.CmdEndRenderPass(current_buffer);
+	vk.EndCommandBuffer(pass_buffer);
+
+	const VkRenderPassBeginInfo &begin_info = *reinterpret_cast<const VkRenderPassBeginInfo *>(pass_begin_info.data());
+	vk.CmdBeginRenderPass(primary_buffer, begin_info, VK_SUBPASS_CONTENTS_SECONDARY_COMMAND_BUFFERS);
+	vk.CmdExecuteCommands(primary_buffer, 1, &pass_buffer);
+	vk.CmdEndRenderPass(primary_buffer);
+
 	framebuffer = 0;
 	viewport = 0;
+	pass_buffer = 0;
 }
 
 void VulkanCommands::begin_frame(unsigned index)
@@ -166,13 +187,14 @@ void VulkanCommands::begin_frame(unsigned index)
 		current_pool->fence.wait();
 		vk.ResetCommandPool(current_pool->pool, 0);
 		current_pool->in_use = false;
-		current_pool->next_buffer = 0;
+		current_pool->primary.next_buffer = 0;
+		current_pool->secondary.next_buffer = 0;
 	}
 }
 
 void VulkanCommands::submit_frame(Semaphore *wait_sem, Semaphore *signal_sem)
 {
-	if(!current_buffer)
+	if(!primary_buffer)
 		return;
 
 	const VulkanFunctions &vk = device.get_functions();
@@ -183,7 +205,7 @@ void VulkanCommands::submit_frame(Semaphore *wait_sem, Semaphore *signal_sem)
 	if(framebuffer)
 		end_render_pass();
 
-	vk.EndCommandBuffer(current_buffer);
+	vk.EndCommandBuffer(primary_buffer);
 
 	VkSubmitInfo submit_info = { };
 	submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
@@ -191,13 +213,13 @@ void VulkanCommands::submit_frame(Semaphore *wait_sem, Semaphore *signal_sem)
 	submit_info.pWaitSemaphores = &vk_wait_sem;
 	submit_info.pWaitDstStageMask = &wait_stages;
 	submit_info.commandBufferCount = 1;
-	submit_info.pCommandBuffers = handle_cast<::VkCommandBuffer *>(&current_buffer);
+	submit_info.pCommandBuffers = handle_cast<::VkCommandBuffer *>(&primary_buffer);
 	submit_info.signalSemaphoreCount = (signal_sem ? 1 : 0);
 	submit_info.pSignalSemaphores = &vk_signal_sem;
 
 	vk.QueueSubmit(1, &submit_info, current_pool->fence.handle);
 
-	current_buffer = 0;
+	primary_buffer = 0;
 }
 
 void VulkanCommands::use_pipeline(const PipelineState *ps)
@@ -233,9 +255,9 @@ void VulkanCommands::draw_instanced(const Batch &batch, unsigned count)
 		 begin_render_pass(false, 0);
 
 	pipeline_state->refresh();
-	pipeline_state->apply(current_buffer, fb_is_swapchain);
+	pipeline_state->apply(pass_buffer, fb_is_swapchain);
 	unsigned first_index = batch.get_offset()/batch.get_index_size();
-	vk.CmdDrawIndexed(current_buffer, batch.size(), count, first_index, 0, 0);
+	vk.CmdDrawIndexed(pass_buffer, batch.size(), count, first_index, 0, 0);
 }
 
 void VulkanCommands::resolve_multisample(Framebuffer &)
diff --git a/source/backends/vulkan/commands_backend.h b/source/backends/vulkan/commands_backend.h
index 994acca2..f96a441c 100644
--- a/source/backends/vulkan/commands_backend.h
+++ b/source/backends/vulkan/commands_backend.h
@@ -21,12 +21,18 @@ class SwapChain;
 class VulkanCommands
 {
 protected:
+	struct CommandBuffers
+	{
+		std::vector<VkCommandBuffer> buffers;
+		unsigned next_buffer = 0;
+	};
+
 	struct CommandPool
 	{
 		Device &device;
 		VkCommandPool pool = 0;
-		std::vector<VkCommandBuffer> buffers;
-		unsigned next_buffer = 0;
+		CommandBuffers primary;
+		CommandBuffers secondary;
 		Fence fence;
 		bool in_use = false;
 		
@@ -38,16 +44,18 @@ protected:
 	Device &device;
 	std::vector<CommandPool> command_pools;
 	CommandPool *current_pool = 0;
-	VkCommandBuffer current_buffer = 0;
+	VkCommandBuffer primary_buffer = 0;
+	VkCommandBuffer pass_buffer = 0;
 	const PipelineState *pipeline_state = 0;
 	const Framebuffer *framebuffer = 0;
 	const Rect *viewport = 0;
 	bool fb_is_swapchain = false;
+	std::vector<char> pass_begin_info;
 
 	VulkanCommands();
 	~VulkanCommands();
 
-	void begin_buffer();
+	void begin_buffer(VkRenderPass);
 	void begin_render_pass(bool, const ClearValue *);
 	void end_render_pass();
 
diff --git a/source/backends/vulkan/vulkan.cpp b/source/backends/vulkan/vulkan.cpp
index 1f697f66..c8054695 100644
--- a/source/backends/vulkan/vulkan.cpp
+++ b/source/backends/vulkan/vulkan.cpp
@@ -19,6 +19,7 @@ VulkanFunctions::VulkanFunctions(const Graphics::VulkanContext &c):
 	vkBeginCommandBuffer(context.get_function<PFN_vkBeginCommandBuffer>("vkBeginCommandBuffer")),
 	vkEndCommandBuffer(context.get_function<PFN_vkEndCommandBuffer>("vkEndCommandBuffer")),
 	vkQueueSubmit(context.get_function<PFN_vkQueueSubmit>("vkQueueSubmit")),
+	vkCmdExecuteCommands(context.get_function<PFN_vkCmdExecuteCommands>("vkCmdExecuteCommands")),
 	// 7
 	vkCreateFence(context.get_function<PFN_vkCreateFence>("vkCreateFence")),
 	vkDestroyFence(context.get_function<PFN_vkDestroyFence>("vkDestroyFence")),
diff --git a/source/backends/vulkan/vulkan.h b/source/backends/vulkan/vulkan.h
index 60de01d1..a862acfe 100644
--- a/source/backends/vulkan/vulkan.h
+++ b/source/backends/vulkan/vulkan.h
@@ -106,6 +106,7 @@ private:
 	PFN_vkBeginCommandBuffer vkBeginCommandBuffer = 0;  // 6.4
 	PFN_vkEndCommandBuffer vkEndCommandBuffer = 0;  // 6.4
 	PFN_vkQueueSubmit vkQueueSubmit = 0;  // 6.5
+	PFN_vkCmdExecuteCommands vkCmdExecuteCommands = 0;  // 6.7
 	PFN_vkCreateFence vkCreateFence = 0;  // 7.3
 	PFN_vkDestroyFence vkDestroyFence = 0;  // 7.3
 	PFN_vkGetFenceStatus vkGetFenceStatus = 0;  // 7.3
@@ -201,6 +202,9 @@ public:
 	Result QueueSubmit(std::uint32_t submitCount, const VkSubmitInfo *pSubmits, VkFence fence) const
 	{ return { vkQueueSubmit(graphicsQueue, submitCount, pSubmits, handle_cast<::VkFence>(fence)), "vkQueueSubmit" }; }
 
+	void CmdExecuteCommands(VkCommandBuffer commandBuffer, std::uint32_t commandBufferCount, const VkCommandBuffer *pCommandBuffers) const
+	{ vkCmdExecuteCommands(handle_cast<::VkCommandBuffer>(commandBuffer), commandBufferCount, handle_cast<const ::VkCommandBuffer *>(pCommandBuffers)); }
+
 	// Chapter 7: Synchronization and Cache Control
 	Result CreateFence(const VkFenceCreateInfo &rCreateInfo, VkFence &rFence) const
 	{ return { vkCreateFence(device, &rCreateInfo, 0, handle_cast<::VkFence *>(&rFence)), "vkCreateFence" }; }