Vulkan: Add explicit synchronization on frame boundaries (#1290)

src/Cafe/HW/Latte/Renderer/Vulkan/SwapchainInfoVk.cpp

@@ -146,8 +146,17 @@ void SwapchainInfoVk::Create()
 			UnrecoverableError("Failed to create semaphore for swapchain acquire");
 	}
 
+	VkFenceCreateInfo fenceInfo = {};
+	fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+	fenceInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT;
+	result = vkCreateFence(m_logicalDevice, &fenceInfo, nullptr, &m_imageAvailableFence);
+	if (result != VK_SUCCESS)
+		UnrecoverableError("Failed to create fence for swapchain");
+
 	m_acquireIndex = 0;
 	hasDefinedSwapchainImage = false;
+
+	m_queueDepth = 0;
 }
 
 void SwapchainInfoVk::Cleanup()
@@ -177,6 +186,12 @@ void SwapchainInfoVk::Cleanup()
 	m_swapchainFramebuffers.clear();
 
 
+	if (m_imageAvailableFence)
+	{
+		WaitAvailableFence();
+		vkDestroyFence(m_logicalDevice, m_imageAvailableFence, nullptr);
+		m_imageAvailableFence = nullptr;
+	}
 	if (m_swapchain)
 	{
 		vkDestroySwapchainKHR(m_logicalDevice, m_swapchain, nullptr);
@@ -189,6 +204,18 @@ bool SwapchainInfoVk::IsValid() const
 	return m_swapchain && !m_acquireSemaphores.empty();
 }
 
+void SwapchainInfoVk::WaitAvailableFence()
+{
+	if(m_awaitableFence != VK_NULL_HANDLE)
+		vkWaitForFences(m_logicalDevice, 1, &m_awaitableFence, VK_TRUE, UINT64_MAX);
+	m_awaitableFence = VK_NULL_HANDLE;
+}
+
+void SwapchainInfoVk::ResetAvailableFence() const
+{
+	vkResetFences(m_logicalDevice, 1, &m_imageAvailableFence);
+}
+
 VkSemaphore SwapchainInfoVk::ConsumeAcquireSemaphore()
 {
 	VkSemaphore ret = m_currentSemaphore;
@@ -198,8 +225,10 @@ VkSemaphore SwapchainInfoVk::ConsumeAcquireSemaphore()
 
 bool SwapchainInfoVk::AcquireImage()
 {
+	ResetAvailableFence();
+
 	VkSemaphore acquireSemaphore = m_acquireSemaphores[m_acquireIndex];
-	VkResult result = vkAcquireNextImageKHR(m_logicalDevice, m_swapchain, 1'000'000'000, acquireSemaphore, nullptr, &swapchainImageIndex);
+	VkResult result = vkAcquireNextImageKHR(m_logicalDevice, m_swapchain, 1'000'000'000, acquireSemaphore, m_imageAvailableFence, &swapchainImageIndex);
 	if (result == VK_ERROR_OUT_OF_DATE_KHR || result == VK_SUBOPTIMAL_KHR)
 		m_shouldRecreate = true;
 	if (result == VK_TIMEOUT)
@@ -216,6 +245,7 @@ bool SwapchainInfoVk::AcquireImage()
 		return false;
 	}
 	m_currentSemaphore = acquireSemaphore;
+	m_awaitableFence = m_imageAvailableFence;
 	m_acquireIndex = (m_acquireIndex + 1) % m_swapchainImages.size();
 
 	return true;
@@ -319,6 +349,7 @@ VkExtent2D SwapchainInfoVk::ChooseSwapExtent(const VkSurfaceCapabilitiesKHR& cap
 
 VkPresentModeKHR SwapchainInfoVk::ChoosePresentMode(const std::vector<VkPresentModeKHR>& modes)
 {
+	m_maxQueued = 0;
 	const auto vsyncState = (VSync)GetConfig().vsync.GetValue();
 	if (vsyncState == VSync::MAILBOX)
 	{
@@ -345,6 +376,7 @@ VkPresentModeKHR SwapchainInfoVk::ChoosePresentMode(const std::vector<VkPresentM
 		return VK_PRESENT_MODE_FIFO_KHR;
 	}
 
+	m_maxQueued = 1;
 	return VK_PRESENT_MODE_FIFO_KHR;
 }
 

src/Cafe/HW/Latte/Renderer/Vulkan/SwapchainInfoVk.h

@@ -26,6 +26,9 @@ struct SwapchainInfoVk
 
 	bool IsValid() const;
 
+	void WaitAvailableFence();
+	void ResetAvailableFence() const;
+
 	bool AcquireImage();
 	// retrieve semaphore of last acquire for submitting a wait operation
 	// only one wait operation must be submitted per acquire (which submits a single signal operation)
@@ -68,6 +71,9 @@ struct SwapchainInfoVk
 	VkSwapchainKHR m_swapchain{};
 	Vector2i m_desiredExtent{};
 	uint32 swapchainImageIndex = (uint32)-1;
+	uint64 m_presentId = 1;
+	uint64 m_queueDepth = 0; // number of frames with pending presentation requests
+	uint64 m_maxQueued = 0; // the maximum number of frames with presentation requests.
 
 
 	// swapchain image ringbuffer (indexed by swapchainImageIndex)
@@ -81,6 +87,8 @@ struct SwapchainInfoVk
 private:
 	uint32 m_acquireIndex = 0;
 	std::vector<VkSemaphore> m_acquireSemaphores; // indexed by m_acquireIndex
+	VkFence m_imageAvailableFence{};
+	VkFence m_awaitableFence = VK_NULL_HANDLE;
 	VkSemaphore m_currentSemaphore = VK_NULL_HANDLE;
 
 	std::array<uint32, 2> m_swapchainQueueFamilyIndices;

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanAPI.h

@@ -188,6 +188,9 @@ VKFUNC_DEVICE(vkCmdPipelineBarrier2KHR);
 VKFUNC_DEVICE(vkCmdBeginRenderingKHR);
 VKFUNC_DEVICE(vkCmdEndRenderingKHR);
 
+// khr_present_wait
+VKFUNC_DEVICE(vkWaitForPresentKHR);
+
 // transform feedback extension
 VKFUNC_DEVICE(vkCmdBindTransformFeedbackBuffersEXT);
 VKFUNC_DEVICE(vkCmdBeginTransformFeedbackEXT);

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanRenderer.cpp

@@ -47,7 +47,9 @@ const  std::vector<const char*> kOptionalDeviceExtensions =
 	VK_EXT_FILTER_CUBIC_EXTENSION_NAME, // not supported by any device yet
 	VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME,
 	VK_KHR_SYNCHRONIZATION_2_EXTENSION_NAME,
-	VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME
+	VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME,
+	VK_KHR_PRESENT_WAIT_EXTENSION_NAME,
+	VK_KHR_PRESENT_ID_EXTENSION_NAME
 };
 
 const std::vector<const char*> kRequiredDeviceExtensions =
@@ -252,12 +254,24 @@ void VulkanRenderer::GetDeviceFeatures()
 	pcc.pNext = prevStruct;
 	prevStruct = &pcc;
 
+	VkPhysicalDevicePresentIdFeaturesKHR pidf{};
+	pidf.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRESENT_ID_FEATURES_KHR;
+	pidf.pNext = prevStruct;
+	prevStruct = &pidf;
+
+	VkPhysicalDevicePresentWaitFeaturesKHR pwf{};
+	pwf.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRESENT_WAIT_FEATURES_KHR;
+	pwf.pNext = prevStruct;
+	prevStruct = &pwf;
+
 	VkPhysicalDeviceFeatures2 physicalDeviceFeatures2{};
 	physicalDeviceFeatures2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
 	physicalDeviceFeatures2.pNext = prevStruct;
 
 	vkGetPhysicalDeviceFeatures2(m_physicalDevice, &physicalDeviceFeatures2);
 
+	cemuLog_log(LogType::Force, "Vulkan: present_wait extension: {}", (pwf.presentWait && pidf.presentId) ? "supported" : "unsupported");
+
 	/* Get Vulkan device properties and limits */
 	VkPhysicalDeviceFloatControlsPropertiesKHR pfcp{};
 	prevStruct = nullptr;
@@ -490,6 +504,24 @@ VulkanRenderer::VulkanRenderer()
 		customBorderColorFeature.customBorderColors = VK_TRUE;
 		customBorderColorFeature.customBorderColorWithoutFormat = VK_TRUE;
 	}
+	// enable VK_KHR_present_id
+	VkPhysicalDevicePresentIdFeaturesKHR presentIdFeature{};
+	if(m_featureControl.deviceExtensions.present_wait)
+	{
+		presentIdFeature.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRESENT_ID_FEATURES_KHR;
+		presentIdFeature.pNext = deviceExtensionFeatures;
+		deviceExtensionFeatures = &presentIdFeature;
+		presentIdFeature.presentId = VK_TRUE;
+	}
+	// enable VK_KHR_present_wait
+	VkPhysicalDevicePresentWaitFeaturesKHR presentWaitFeature{};
+	if(m_featureControl.deviceExtensions.present_wait)
+	{
+		presentWaitFeature.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRESENT_WAIT_FEATURES_KHR;
+		presentWaitFeature.pNext = deviceExtensionFeatures;
+		deviceExtensionFeatures = &presentWaitFeature;
+		presentWaitFeature.presentWait = VK_TRUE;
+	}
 
 	std::vector<const char*> used_extensions;
 	VkDeviceCreateInfo createInfo = CreateDeviceCreateInfo(queueCreateInfos, deviceFeatures, deviceExtensionFeatures, used_extensions);
@@ -1047,6 +1079,10 @@ VkDeviceCreateInfo VulkanRenderer::CreateDeviceCreateInfo(const std::vector<VkDe
 		used_extensions.emplace_back(VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME);
 	if (m_featureControl.deviceExtensions.shader_float_controls)
 		used_extensions.emplace_back(VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME);
+	if (m_featureControl.deviceExtensions.present_wait)
+		used_extensions.emplace_back(VK_KHR_PRESENT_ID_EXTENSION_NAME);
+	if (m_featureControl.deviceExtensions.present_wait)
+		used_extensions.emplace_back(VK_KHR_PRESENT_WAIT_EXTENSION_NAME);
 
 	VkDeviceCreateInfo createInfo{};
 	createInfo.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
@@ -1144,6 +1180,7 @@ bool VulkanRenderer::CheckDeviceExtensionSupport(const VkPhysicalDevice device,
 	info.deviceExtensions.shader_float_controls = isExtensionAvailable(VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME);
 	info.deviceExtensions.dynamic_rendering = false; // isExtensionAvailable(VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME);
 	// dynamic rendering doesn't provide any benefits for us right now. Driver implementations are very unoptimized as of Feb 2022
+	info.deviceExtensions.present_wait = isExtensionAvailable(VK_KHR_PRESENT_WAIT_EXTENSION_NAME) && isExtensionAvailable(VK_KHR_PRESENT_ID_EXTENSION_NAME);
 
 	// check for framedebuggers
 	info.debugMarkersSupported = false;
@@ -2695,11 +2732,21 @@ void VulkanRenderer::SwapBuffer(bool mainWindow)
 		ClearColorImageRaw(chainInfo.m_swapchainImages[chainInfo.swapchainImageIndex], 0, 0, clearColor, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
 	}
 
+	const size_t currentFrameCmdBufferID = GetCurrentCommandBufferId();
+
 	VkSemaphore presentSemaphore = chainInfo.m_presentSemaphores[chainInfo.swapchainImageIndex];
 	SubmitCommandBuffer(presentSemaphore); // submit all command and signal semaphore
 
 	cemu_assert_debug(m_numSubmittedCmdBuffers > 0);
 
+	// wait for the previous frame to finish rendering
+	WaitCommandBufferFinished(m_commandBufferIDOfPrevFrame);
+	m_commandBufferIDOfPrevFrame = currentFrameCmdBufferID;
+
+	chainInfo.WaitAvailableFence();
+
+	VkPresentIdKHR presentId = {};
+
 	VkPresentInfoKHR presentInfo = {};
 	presentInfo.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
 	presentInfo.swapchainCount = 1;
@@ -2709,6 +2756,24 @@ void VulkanRenderer::SwapBuffer(bool mainWindow)
 	presentInfo.waitSemaphoreCount = 1;
 	presentInfo.pWaitSemaphores = &presentSemaphore;
 
+	// if present_wait is available and enabled, add frame markers to present requests
+	// and limit the number of queued present operations
+	if (m_featureControl.deviceExtensions.present_wait && chainInfo.m_maxQueued > 0)
+	{
+		presentId.sType = VK_STRUCTURE_TYPE_PRESENT_ID_KHR;
+		presentId.swapchainCount = 1;
+		presentId.pPresentIds = &chainInfo.m_presentId;
+
+		presentInfo.pNext = &presentId;
+
+		if(chainInfo.m_queueDepth >= chainInfo.m_maxQueued)
+		{
+			uint64 waitFrameId = chainInfo.m_presentId - chainInfo.m_queueDepth;
+			vkWaitForPresentKHR(m_logicalDevice, chainInfo.m_swapchain, waitFrameId, 40'000'000);
+			chainInfo.m_queueDepth--;
+		}
+	}
+
 	VkResult result = vkQueuePresentKHR(m_presentQueue, &presentInfo);
 	if (result < 0 && result != VK_ERROR_OUT_OF_DATE_KHR)
 	{
@@ -2717,6 +2782,12 @@ void VulkanRenderer::SwapBuffer(bool mainWindow)
 	if(result == VK_ERROR_OUT_OF_DATE_KHR || result == VK_SUBOPTIMAL_KHR)
 		chainInfo.m_shouldRecreate = true;
 
+	if(result >= 0)
+	{
+		chainInfo.m_queueDepth++;
+		chainInfo.m_presentId++;
+	}
+
 	chainInfo.hasDefinedSwapchainImage = false;
 
 	chainInfo.swapchainImageIndex = -1;

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanRenderer.h

@@ -450,14 +450,15 @@ class VulkanRenderer : public Renderer
 			bool synchronization2 = false; // VK_KHR_synchronization2
 			bool dynamic_rendering = false; // VK_KHR_dynamic_rendering
 			bool shader_float_controls = false; // VK_KHR_shader_float_controls
+			bool present_wait = false; // VK_KHR_present_wait
 		}deviceExtensions;
 
 		struct
 		{
 			bool shaderRoundingModeRTEFloat32{ false };
 		}shaderFloatControls; // from VK_KHR_shader_float_controls
 
-		struct  
+		struct
 		{
 			bool debug_utils = false; // VK_EXT_DEBUG_UTILS
 		}instanceExtensions;
@@ -635,6 +636,7 @@ class VulkanRenderer : public Renderer
 
 	size_t m_commandBufferIndex = 0; // current buffer being filled
 	size_t m_commandBufferSyncIndex = 0; // latest buffer that finished execution (updated on submit)
+	size_t m_commandBufferIDOfPrevFrame = 0;
 	std::array<VkFence, kCommandBufferPoolSize> m_cmd_buffer_fences;
 	std::array<VkCommandBuffer, kCommandBufferPoolSize> m_commandBuffers;
 	std::array<VkSemaphore, kCommandBufferPoolSize> m_commandBufferSemaphores;