From d38f53a4feab06bcd2915247700156ecc07ef282 Mon Sep 17 00:00:00 2001 From: Simon Tippe Date: Thu, 13 Jul 2023 17:13:44 +0200 Subject: [PATCH] Submission now are being done at end of the frame --- src/engine/graphics/CommandList.cpp | 44 ++++++- src/engine/graphics/CommandList.h | 16 ++- src/engine/graphics/GraphicsDevice.cpp | 174 +++++++++++++++++-------- src/engine/graphics/GraphicsDevice.h | 26 ++-- src/engine/graphics/Queue.h | 17 ++- 5 files changed, 206 insertions(+), 71 deletions(-) diff --git a/src/engine/graphics/CommandList.cpp b/src/engine/graphics/CommandList.cpp index 12b92d990..4ab764fa1 100644 --- a/src/engine/graphics/CommandList.cpp +++ b/src/engine/graphics/CommandList.cpp @@ -8,8 +8,8 @@ namespace Atlas { namespace Graphics { CommandList::CommandList(GraphicsDevice* device, QueueType queueType, uint32_t queueFamilyIndex, - bool frameIndependent) : memoryManager(device->memoryManager), device(device->device), - frameIndependent(frameIndependent), queueType(queueType), queueFamilyIndex(queueFamilyIndex) { + const std::vector& queues, bool frameIndependent) : memoryManager(device->memoryManager), + device(device->device), frameIndependent(frameIndependent), queueType(queueType), queueFamilyIndex(queueFamilyIndex) { VkCommandPoolCreateInfo poolCreateInfo = Initializers::InitCommandPoolCreateInfo(queueFamilyIndex); VK_CHECK(vkCreateCommandPool(device->device, &poolCreateInfo, nullptr, &commandPool)) @@ -17,12 +17,20 @@ namespace Atlas { VkCommandBufferAllocateInfo bufferAllocateInfo = Initializers::InitCommandBufferAllocateInfo(commandPool, 1); VK_CHECK(vkAllocateCommandBuffers(device->device, &bufferAllocateInfo, &commandBuffer)) - VkSemaphoreCreateInfo semaphoreInfo = Initializers::InitSemaphoreCreateInfo(); - VK_CHECK(vkCreateSemaphore(device->device, &semaphoreInfo, nullptr, &semaphore)) - VkFenceCreateInfo fenceInfo = Initializers::InitFenceCreateInfo(); VK_CHECK(vkCreateFence(device->device, &fenceInfo, nullptr, &fence)) + for (auto queue : queues) { + Semaphore semaphore{ + .queue = queue + }; + + VkSemaphoreCreateInfo semaphoreInfo = Initializers::InitSemaphoreCreateInfo(); + VK_CHECK(vkCreateSemaphore(device->device, &semaphoreInfo, nullptr, &semaphore.semaphore)) + + semaphores.push_back(semaphore); + } + descriptorPool = new DescriptorPool(device); isComplete = true; @@ -33,7 +41,9 @@ namespace Atlas { delete descriptorPool; - vkDestroySemaphore(device, semaphore, nullptr); + for (auto& semaphore : semaphores) + vkDestroySemaphore(device, semaphore.semaphore, nullptr); + vkDestroyFence(device, fence, nullptr); vkDestroyCommandPool(device, commandPool, nullptr); @@ -951,6 +961,28 @@ namespace Atlas { } + const VkSemaphore CommandList::GetSemaphore(VkQueue queue) { + + for (auto& semaphore : semaphores) { + if (semaphore.queue == queue) + return semaphore.semaphore; + } + + assert(0 && "Queue not found in available semaphores"); + + } + + const std::vector CommandList::GetSemaphores() const { + + std::vector resultSemaphores; + + for (auto& semaphore : semaphores) + resultSemaphores.push_back(semaphore.semaphore); + + return resultSemaphores; + + } + } } \ No newline at end of file diff --git a/src/engine/graphics/CommandList.h b/src/engine/graphics/CommandList.h index 20579b5fc..57d30ef5f 100644 --- a/src/engine/graphics/CommandList.h +++ b/src/engine/graphics/CommandList.h @@ -37,7 +37,7 @@ namespace Atlas { public: CommandList(GraphicsDevice* device, QueueType queueType, uint32_t queueFamilyIndex, - bool frameIndependent = false); + const std::vector& queues, bool frameIndependent = false); CommandList(const CommandList& that) = delete; @@ -148,7 +148,6 @@ namespace Atlas { VkCommandPool commandPool; VkCommandBuffer commandBuffer; - VkSemaphore semaphore; VkFence fence; uint32_t queueFamilyIndex; @@ -165,6 +164,8 @@ namespace Atlas { std::vector dependencies; ExecutionOrder executionOrder = ExecutionOrder::Sequential; + int32_t id = 0; + private: struct DescriptorBindingData { Buffer* buffers[DESCRIPTOR_SET_COUNT][BINDINGS_PER_DESCRIPTOR_SET]; @@ -208,12 +209,21 @@ namespace Atlas { } }descriptorBindingData; + struct Semaphore { + VkSemaphore semaphore; + VkQueue queue; + }; + void BindDescriptorSets(); void ResetDescriptors(); const VkExtent2D GetRenderPassExtent() const; + const VkSemaphore GetSemaphore(VkQueue queue); + + const std::vector GetSemaphores() const; + VkDevice device; MemoryManager* memoryManager = nullptr; DescriptorPool* descriptorPool = nullptr; @@ -221,6 +231,8 @@ namespace Atlas { std::atomic_bool isLocked = true; std::atomic_bool isSubmitted = true; + std::vector semaphores; + }; } diff --git a/src/engine/graphics/GraphicsDevice.cpp b/src/engine/graphics/GraphicsDevice.cpp index d0ab8c7aa..6a7c127e3 100644 --- a/src/engine/graphics/GraphicsDevice.cpp +++ b/src/engine/graphics/GraphicsDevice.cpp @@ -338,47 +338,19 @@ namespace Atlas { assert(swapChain->isComplete && "Swap chain should be complete." && " The swap chain might have an invalid size due to a window resize"); - // After the submission of a command list, we don't unlock it anymore - // for further use in this frame. Instead, we will unlock it again - // when we get back to this frames data and start a new frame with it. auto frame = GetFrameData(); cmd->executionOrder = order; - std::vector waitStages = { waitStage }; - - std::vector waitSemaphores; - std::vector submitSemaphores; - // Leave out any dependencies if the swap chain isn't complete - if (swapChain->isComplete) { - waitSemaphores = { frame->semaphore }; - submitSemaphores = { cmd->semaphore }; - if (frame->submittedCommandLists.size() > 0 && order == ExecutionOrder::Sequential) { - waitSemaphores[0] = frame->submittedCommandLists.back()->semaphore; - } - else if (order == ExecutionOrder::Parallel) { - - } - } - - VkSubmitInfo submit = {}; - submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - submit.pNext = nullptr; - submit.pWaitDstStageMask = waitStages.data(); - submit.waitSemaphoreCount = uint32_t(waitSemaphores.size()); - submit.pWaitSemaphores = waitSemaphores.data(); - submit.signalSemaphoreCount = uint32_t(submitSemaphores.size()); - submit.pSignalSemaphores = submitSemaphores.data(); - submit.commandBufferCount = 1; - submit.pCommandBuffers = &cmd->commandBuffer; - - auto queue = FindAndLockQueue(cmd->queueType); - VK_CHECK(vkQueueSubmit(queue->queue, 1, &submit, cmd->fence)) - queue->mutex.unlock(); - // Make sure only one command list at a time can be added std::lock_guard lock(frame->submissionMutex); + CommandListSubmission submission = { + .cmd = cmd, + .waitStage = waitStage + }; + + frame->submissions.push_back(submission); frame->submittedCommandLists.push_back(cmd); cmd->isSubmitted = true; @@ -429,13 +401,16 @@ namespace Atlas { assert(allListSubmitted && "Not all command list were submitted before frame completion." && "Consider using a frame independent command lists for longer executions."); + auto presenterQueue = SubmitAllCommandLists(); + if (frame->submittedCommandLists.size() && swapChain->isComplete) { + std::vector semaphores; // For now, we will only use sequential execution of queue submits, // which means only the latest submit can signal its semaphore here //for (auto cmd : frameData->submittedCommandLists) // semaphores.push_back(cmd->semaphore); - semaphores.push_back(frame->submittedCommandLists.back()->semaphore); + semaphores.push_back(frame->submittedCommandLists.back()->GetSemaphore(presenterQueue->queue)); VkPresentInfoKHR presentInfo = {}; presentInfo.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR; @@ -446,7 +421,6 @@ namespace Atlas { presentInfo.waitSemaphoreCount = uint32_t(semaphores.size()); presentInfo.pImageIndices = &swapChain->aquiredImageIndex; - auto presenterQueue = FindAndLockQueue(QueueType::PresentationQueue); auto result = vkQueuePresentKHR(presenterQueue->queue, &presentInfo); presenterQueue->mutex.unlock(); @@ -515,6 +489,85 @@ namespace Atlas { } + Ref GraphicsDevice::SubmitAllCommandLists() { + + auto frame = GetFrameData(); + + // Assume all submission are in order + Ref nextQueue = nullptr; + Ref queue = nullptr; + VkSemaphore previousSemaphore = frame->semaphore; + for (size_t i = 0; i < frame->submissions.size(); i++) { + auto submission = &frame->submissions[i]; + auto nextSubmission = i + 1 < frame->submissions.size() ? &frame->submissions[i + 1] : nullptr; + + auto queueType = submission->cmd->queueType; + + if (!queue) { + queue = FindAndLockQueue(queueType); + } + + if (!queue->IsTypeSupported(queueType)) { + queue = FindAndLockQueue(queueType); + } + + if (nextSubmission != nullptr && !queue->IsTypeSupported(nextSubmission->cmd->queueType)) { + nextQueue = FindAndLockQueue(nextSubmission->cmd->queueType); + } + else { + nextQueue = queue; + } + + if (nextSubmission == nullptr) { + nextQueue = FindAndLockQueue(QueueType::PresentationQueue); + } + + SubmitCommandList(submission, previousSemaphore, queue, nextQueue); + previousSemaphore = submission->cmd->GetSemaphore(nextQueue->queue); + + if (nextQueue != queue) { + queue->mutex.unlock(); + } + + queue = nextQueue; + } + + return queue; + } + + void GraphicsDevice::SubmitCommandList(CommandListSubmission* submission, VkSemaphore previousSemaphore, + const Ref& queue, const Ref& nextQueue) { + + // After the submission of a command list, we don't unlock it anymore + // for further use in this frame. Instead, we will unlock it again + // when we get back to this frames data and start a new frame with it. + auto frame = GetFrameData(); + + auto cmd = submission->cmd; + std::vector waitStages = { submission->waitStage }; + + std::vector waitSemaphores; + std::vector submitSemaphores = { cmd->GetSemaphore(nextQueue->queue) }; + + // Leave out any dependencies if the swap chain isn't complete + if (swapChain->isComplete) { + waitSemaphores = { previousSemaphore }; + } + + VkSubmitInfo submit = {}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.pNext = nullptr; + submit.pWaitDstStageMask = waitStages.data(); + submit.waitSemaphoreCount = uint32_t(waitSemaphores.size()); + submit.pWaitSemaphores = waitSemaphores.data(); + submit.signalSemaphoreCount = uint32_t(submitSemaphores.size()); + submit.pSignalSemaphores = submitSemaphores.data(); + submit.commandBufferCount = 1; + submit.pCommandBuffers = &cmd->commandBuffer; + + VK_CHECK(vkQueueSubmit(queue->queue, 1, &submit, cmd->fence)) + } + bool GraphicsDevice::SelectPhysicalDevice(VkInstance instance, VkSurfaceKHR surface, const std::vector& requiredExtensions) { @@ -623,11 +676,23 @@ namespace Atlas { VkBool32 presentSupport = false; vkGetPhysicalDeviceSurfaceSupportKHR(device, counter, surface, &presentSupport); + bool supportsGraphics = true; + bool supportsTransfer = true; + if (!(queueFamily.queueFlags & VK_QUEUE_GRAPHICS_BIT)) + supportsGraphics = false; + + if (!(queueFamily.queueFlags & VK_QUEUE_COMPUTE_BIT)) + supportsGraphics = false; + + if (!(queueFamily.queueFlags & VK_QUEUE_TRANSFER_BIT)) + supportsTransfer = false; + queueFamilyIndices.families[counter].queues.resize(size_t(queueFamily.queueCount)); queueFamilyIndices.families[counter].queuePriorities.resize(size_t(queueFamily.queueCount)); queueFamilyIndices.families[counter].index = counter; - queueFamilyIndices.families[counter].flags = queueFamily.queueFlags; + queueFamilyIndices.families[counter].supportsGraphics = supportsGraphics; + queueFamilyIndices.families[counter].supportsTransfer = supportsTransfer; queueFamilyIndices.families[counter].supportsPresentation = presentSupport; for (uint32_t i = 0; i < queueFamily.queueCount; i++) { @@ -636,6 +701,10 @@ namespace Atlas { queue->familyIndex = counter; queue->index = i; + queue->supportsGraphics = supportsGraphics; + queue->supportsTransfer = supportsTransfer; + queue->supportsPresentation = presentSupport; + queueFamilyIndices.families[counter].queues[i] = queue; queueFamilyIndices.families[counter].queuePriorities[i] = 1.0f; } @@ -645,22 +714,12 @@ namespace Atlas { counter = 0; for (auto& queueFamily : queueFamilyIndices.families) { - bool isGraphicsFamilyValid = true; - bool isTransferFamilyValid = true; - - if (!(queueFamily.flags & VK_QUEUE_GRAPHICS_BIT)) - isGraphicsFamilyValid = false; - - if (!(queueFamily.flags & VK_QUEUE_COMPUTE_BIT)) - isGraphicsFamilyValid = false; - - if (!(queueFamily.flags & VK_QUEUE_TRANSFER_BIT)) - isTransferFamilyValid = false; + - if (isGraphicsFamilyValid) { + if (queueFamily.supportsGraphics) { queueFamilyIndices.queueFamilies[QueueType::GraphicsQueue] = counter; } - if (isTransferFamilyValid) { + if (queueFamily.supportsTransfer) { queueFamilyIndices.queueFamilies[QueueType::TransferQueue] = counter; } if (queueFamily.supportsPresentation) { @@ -683,7 +742,7 @@ namespace Atlas { continue; } - if (queueFamily.flags & VK_QUEUE_TRANSFER_BIT) { + if (queueFamily.supportsTransfer) { queueFamilyIndices.queueFamilies[QueueType::TransferQueue] = counter; break; } @@ -936,8 +995,16 @@ namespace Atlas { if (it == cmdLists.end()) { auto queueFamilyIndex = queueFamilyIndices.queueFamilies[queueType]; + + std::vector queues; + for (auto& queueFamily : queueFamilyIndices.families) { + for (auto& queue : queueFamily.queues) { + queues.push_back(queue->queue); + } + } + CommandList *cmd = new CommandList(this, queueType, - queueFamilyIndex.value(), frameIndependent); + queueFamilyIndex.value(), queues, frameIndependent); cmdLists.push_back(cmd); return cmd; } @@ -968,7 +1035,8 @@ namespace Atlas { // Backup plan, search for all queues Ref lastSupportedQueue = nullptr; for (auto& queueFamily : queueFamilyIndices.families) { - if (queueType != PresentationQueue && (!(queueFamily.flags & neededFlags)) || + if ((queueType == GraphicsQueue && !queueFamily.supportsGraphics) || + (queueType == TransferQueue && !queueFamily.supportsTransfer) || (queueType == PresentationQueue && !queueFamily.supportsPresentation)) continue; diff --git a/src/engine/graphics/GraphicsDevice.h b/src/engine/graphics/GraphicsDevice.h index ca5bb6462..a373db1ef 100644 --- a/src/engine/graphics/GraphicsDevice.h +++ b/src/engine/graphics/GraphicsDevice.h @@ -29,6 +29,12 @@ namespace Atlas { class Instance; class ImguiWrapper; + struct CommandListSubmission { + CommandList* cmd; + + VkPipelineStageFlags waitStage; + }; + class FrameData { public: VkSemaphore semaphore = VK_NULL_HANDLE; @@ -40,6 +46,8 @@ namespace Atlas { std::mutex submissionMutex; std::vector submittedCommandLists; + std::vector submissions; + void WaitAndReset(VkDevice device) { if (submittedCommandLists.size() > 0) { std::vector fences; @@ -48,13 +56,6 @@ namespace Atlas { } VK_CHECK(vkWaitForFences(device, uint32_t(fences.size()), fences.data(), true, 1000000000)) VK_CHECK(vkResetFences(device, uint32_t(fences.size()), fences.data())) - - for (auto commandList : submittedCommandLists) { - vkDestroySemaphore(device, commandList->semaphore, nullptr); - - VkSemaphoreCreateInfo semaphoreInfo = Initializers::InitSemaphoreCreateInfo(); - VK_CHECK(vkCreateSemaphore(device, &semaphoreInfo, nullptr, &commandList->semaphore)) - } } for (auto commandList : submittedCommandLists) { @@ -62,6 +63,7 @@ namespace Atlas { commandList->isLocked = false; } submittedCommandLists.clear(); + submissions.clear(); } void RecreateSemaphore(VkDevice device) { @@ -113,7 +115,7 @@ namespace Atlas { bool frameIndependentList = false); void SubmitCommandList(CommandList* cmd, VkPipelineStageFlags waitStage = - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, ExecutionOrder order = ExecutionOrder::Sequential); + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, ExecutionOrder order = ExecutionOrder::Sequential); void FlushCommandList(CommandList* cmd); @@ -151,7 +153,8 @@ namespace Atlas { std::vector> queues; std::vector queuePriorities; - VkQueueFlags flags; + bool supportsGraphics; + bool supportsTransfer; bool supportsPresentation; }; @@ -166,6 +169,11 @@ namespace Atlas { } }; + Ref SubmitAllCommandLists(); + + void SubmitCommandList(CommandListSubmission* submission, VkSemaphore previousSemaphore, + const Ref& queue, const Ref& nextQueue); + bool SelectPhysicalDevice(VkInstance instance, VkSurfaceKHR surface, const std::vector& requiredExtensions); diff --git a/src/engine/graphics/Queue.h b/src/engine/graphics/Queue.h index 7c56febd6..5c3ad9b50 100644 --- a/src/engine/graphics/Queue.h +++ b/src/engine/graphics/Queue.h @@ -13,12 +13,27 @@ namespace Atlas { TransferQueue }; - struct Queue { + class Queue { + + public: + bool IsTypeSupported(QueueType queueType) const { + switch (queueType) { + case GraphicsQueue: return supportsGraphics; + case PresentationQueue: return supportsPresentation; + case TransferQueue: return supportsTransfer; + default: return false; + } + } + VkQueue queue; uint32_t familyIndex; uint32_t index; + bool supportsGraphics; + bool supportsTransfer; + bool supportsPresentation; + std::mutex mutex; };