Skip to content

Commit

Permalink
Submission now are being done at end of the frame
Browse files Browse the repository at this point in the history
  • Loading branch information
@tippesi
tippesi committed Jul 13, 2023
1 parent aa0c843 commit d38f53a
Show file tree
Hide file tree
Showing 5 changed files with 206 additions and 71 deletions.
44 changes: 38 additions & 6 deletions src/engine/graphics/CommandList.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,29 @@ namespace Atlas {
namespace Graphics {

CommandList::CommandList(GraphicsDevice* device, QueueType queueType, uint32_t queueFamilyIndex,
bool frameIndependent) : memoryManager(device->memoryManager), device(device->device),
frameIndependent(frameIndependent), queueType(queueType), queueFamilyIndex(queueFamilyIndex) {
const std::vector<VkQueue>& queues, bool frameIndependent) : memoryManager(device->memoryManager),
device(device->device), frameIndependent(frameIndependent), queueType(queueType), queueFamilyIndex(queueFamilyIndex) {

VkCommandPoolCreateInfo poolCreateInfo = Initializers::InitCommandPoolCreateInfo(queueFamilyIndex);
VK_CHECK(vkCreateCommandPool(device->device, &poolCreateInfo, nullptr, &commandPool))

VkCommandBufferAllocateInfo bufferAllocateInfo = Initializers::InitCommandBufferAllocateInfo(commandPool, 1);
VK_CHECK(vkAllocateCommandBuffers(device->device, &bufferAllocateInfo, &commandBuffer))

VkSemaphoreCreateInfo semaphoreInfo = Initializers::InitSemaphoreCreateInfo();
VK_CHECK(vkCreateSemaphore(device->device, &semaphoreInfo, nullptr, &semaphore))

VkFenceCreateInfo fenceInfo = Initializers::InitFenceCreateInfo();
VK_CHECK(vkCreateFence(device->device, &fenceInfo, nullptr, &fence))

for (auto queue : queues) {
Semaphore semaphore{
.queue = queue
};

VkSemaphoreCreateInfo semaphoreInfo = Initializers::InitSemaphoreCreateInfo();
VK_CHECK(vkCreateSemaphore(device->device, &semaphoreInfo, nullptr, &semaphore.semaphore))

semaphores.push_back(semaphore);
}

descriptorPool = new DescriptorPool(device);

isComplete = true;
Expand All @@ -33,7 +41,9 @@ namespace Atlas {

delete descriptorPool;

vkDestroySemaphore(device, semaphore, nullptr);
for (auto& semaphore : semaphores)
vkDestroySemaphore(device, semaphore.semaphore, nullptr);

vkDestroyFence(device, fence, nullptr);
vkDestroyCommandPool(device, commandPool, nullptr);

Expand Down Expand Up @@ -951,6 +961,28 @@ namespace Atlas {

}

const VkSemaphore CommandList::GetSemaphore(VkQueue queue) {

for (auto& semaphore : semaphores) {
if (semaphore.queue == queue)
return semaphore.semaphore;
}

assert(0 && "Queue not found in available semaphores");

}

const std::vector<VkSemaphore> CommandList::GetSemaphores() const {

std::vector<VkSemaphore> resultSemaphores;

for (auto& semaphore : semaphores)
resultSemaphores.push_back(semaphore.semaphore);

return resultSemaphores;

}

}

}
16 changes: 14 additions & 2 deletions src/engine/graphics/CommandList.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ namespace Atlas {

public:
CommandList(GraphicsDevice* device, QueueType queueType, uint32_t queueFamilyIndex,
bool frameIndependent = false);
const std::vector<VkQueue>& queues, bool frameIndependent = false);

CommandList(const CommandList& that) = delete;

Expand Down Expand Up @@ -148,7 +148,6 @@ namespace Atlas {

VkCommandPool commandPool;
VkCommandBuffer commandBuffer;
VkSemaphore semaphore;
VkFence fence;
uint32_t queueFamilyIndex;

Expand All @@ -165,6 +164,8 @@ namespace Atlas {
std::vector<CommandList*> dependencies;
ExecutionOrder executionOrder = ExecutionOrder::Sequential;

int32_t id = 0;

private:
struct DescriptorBindingData {
Buffer* buffers[DESCRIPTOR_SET_COUNT][BINDINGS_PER_DESCRIPTOR_SET];
Expand Down Expand Up @@ -208,19 +209,30 @@ namespace Atlas {
}
}descriptorBindingData;

struct Semaphore {
VkSemaphore semaphore;
VkQueue queue;
};

void BindDescriptorSets();

void ResetDescriptors();

const VkExtent2D GetRenderPassExtent() const;

const VkSemaphore GetSemaphore(VkQueue queue);

const std::vector<VkSemaphore> GetSemaphores() const;

VkDevice device;
MemoryManager* memoryManager = nullptr;
DescriptorPool* descriptorPool = nullptr;

std::atomic_bool isLocked = true;
std::atomic_bool isSubmitted = true;

std::vector<Semaphore> semaphores;

};

}
Expand Down
174 changes: 121 additions & 53 deletions src/engine/graphics/GraphicsDevice.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,47 +338,19 @@ namespace Atlas {
assert(swapChain->isComplete && "Swap chain should be complete."
&& " The swap chain might have an invalid size due to a window resize");

// After the submission of a command list, we don't unlock it anymore
// for further use in this frame. Instead, we will unlock it again
// when we get back to this frames data and start a new frame with it.
auto frame = GetFrameData();

cmd->executionOrder = order;

std::vector<VkPipelineStageFlags> waitStages = { waitStage };

std::vector<VkSemaphore> waitSemaphores;
std::vector<VkSemaphore> submitSemaphores;
// Leave out any dependencies if the swap chain isn't complete
if (swapChain->isComplete) {
waitSemaphores = { frame->semaphore };
submitSemaphores = { cmd->semaphore };
if (frame->submittedCommandLists.size() > 0 && order == ExecutionOrder::Sequential) {
waitSemaphores[0] = frame->submittedCommandLists.back()->semaphore;
}
else if (order == ExecutionOrder::Parallel) {

}
}

VkSubmitInfo submit = {};
submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit.pNext = nullptr;
submit.pWaitDstStageMask = waitStages.data();
submit.waitSemaphoreCount = uint32_t(waitSemaphores.size());
submit.pWaitSemaphores = waitSemaphores.data();
submit.signalSemaphoreCount = uint32_t(submitSemaphores.size());
submit.pSignalSemaphores = submitSemaphores.data();
submit.commandBufferCount = 1;
submit.pCommandBuffers = &cmd->commandBuffer;

auto queue = FindAndLockQueue(cmd->queueType);
VK_CHECK(vkQueueSubmit(queue->queue, 1, &submit, cmd->fence))
queue->mutex.unlock();

// Make sure only one command list at a time can be added
std::lock_guard<std::mutex> lock(frame->submissionMutex);

CommandListSubmission submission = {
.cmd = cmd,
.waitStage = waitStage
};

frame->submissions.push_back(submission);
frame->submittedCommandLists.push_back(cmd);
cmd->isSubmitted = true;

Expand Down Expand Up @@ -429,13 +401,16 @@ namespace Atlas {
assert(allListSubmitted && "Not all command list were submitted before frame completion." &&
"Consider using a frame independent command lists for longer executions.");

auto presenterQueue = SubmitAllCommandLists();

if (frame->submittedCommandLists.size() && swapChain->isComplete) {

std::vector<VkSemaphore> semaphores;
// For now, we will only use sequential execution of queue submits,
// which means only the latest submit can signal its semaphore here
//for (auto cmd : frameData->submittedCommandLists)
// semaphores.push_back(cmd->semaphore);
semaphores.push_back(frame->submittedCommandLists.back()->semaphore);
semaphores.push_back(frame->submittedCommandLists.back()->GetSemaphore(presenterQueue->queue));

VkPresentInfoKHR presentInfo = {};
presentInfo.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
Expand All @@ -446,7 +421,6 @@ namespace Atlas {
presentInfo.waitSemaphoreCount = uint32_t(semaphores.size());
presentInfo.pImageIndices = &swapChain->aquiredImageIndex;

auto presenterQueue = FindAndLockQueue(QueueType::PresentationQueue);
auto result = vkQueuePresentKHR(presenterQueue->queue, &presentInfo);
presenterQueue->mutex.unlock();

Expand Down Expand Up @@ -515,6 +489,85 @@ namespace Atlas {

}

Ref<Queue> GraphicsDevice::SubmitAllCommandLists() {

auto frame = GetFrameData();

// Assume all submission are in order
Ref<Queue> nextQueue = nullptr;
Ref<Queue> queue = nullptr;
VkSemaphore previousSemaphore = frame->semaphore;
for (size_t i = 0; i < frame->submissions.size(); i++) {
auto submission = &frame->submissions[i];
auto nextSubmission = i + 1 < frame->submissions.size() ? &frame->submissions[i + 1] : nullptr;

auto queueType = submission->cmd->queueType;

if (!queue) {
queue = FindAndLockQueue(queueType);
}

if (!queue->IsTypeSupported(queueType)) {
queue = FindAndLockQueue(queueType);
}

if (nextSubmission != nullptr && !queue->IsTypeSupported(nextSubmission->cmd->queueType)) {
nextQueue = FindAndLockQueue(nextSubmission->cmd->queueType);
}
else {
nextQueue = queue;
}

if (nextSubmission == nullptr) {
nextQueue = FindAndLockQueue(QueueType::PresentationQueue);
}

SubmitCommandList(submission, previousSemaphore, queue, nextQueue);
previousSemaphore = submission->cmd->GetSemaphore(nextQueue->queue);

if (nextQueue != queue) {
queue->mutex.unlock();
}

queue = nextQueue;
}

return queue;
}

void GraphicsDevice::SubmitCommandList(CommandListSubmission* submission, VkSemaphore previousSemaphore,
const Ref<Queue>& queue, const Ref<Queue>& nextQueue) {

// After the submission of a command list, we don't unlock it anymore
// for further use in this frame. Instead, we will unlock it again
// when we get back to this frames data and start a new frame with it.
auto frame = GetFrameData();

auto cmd = submission->cmd;
std::vector<VkPipelineStageFlags> waitStages = { submission->waitStage };

std::vector<VkSemaphore> waitSemaphores;
std::vector<VkSemaphore> submitSemaphores = { cmd->GetSemaphore(nextQueue->queue) };

// Leave out any dependencies if the swap chain isn't complete
if (swapChain->isComplete) {
waitSemaphores = { previousSemaphore };
}

VkSubmitInfo submit = {};
submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit.pNext = nullptr;
submit.pWaitDstStageMask = waitStages.data();
submit.waitSemaphoreCount = uint32_t(waitSemaphores.size());
submit.pWaitSemaphores = waitSemaphores.data();
submit.signalSemaphoreCount = uint32_t(submitSemaphores.size());
submit.pSignalSemaphores = submitSemaphores.data();
submit.commandBufferCount = 1;
submit.pCommandBuffers = &cmd->commandBuffer;

VK_CHECK(vkQueueSubmit(queue->queue, 1, &submit, cmd->fence))
}

bool GraphicsDevice::SelectPhysicalDevice(VkInstance instance, VkSurfaceKHR surface,
const std::vector<const char*>& requiredExtensions) {

Expand Down Expand Up @@ -623,11 +676,23 @@ namespace Atlas {
VkBool32 presentSupport = false;
vkGetPhysicalDeviceSurfaceSupportKHR(device, counter, surface, &presentSupport);

bool supportsGraphics = true;
bool supportsTransfer = true;
if (!(queueFamily.queueFlags & VK_QUEUE_GRAPHICS_BIT))
supportsGraphics = false;

if (!(queueFamily.queueFlags & VK_QUEUE_COMPUTE_BIT))
supportsGraphics = false;

if (!(queueFamily.queueFlags & VK_QUEUE_TRANSFER_BIT))
supportsTransfer = false;

queueFamilyIndices.families[counter].queues.resize(size_t(queueFamily.queueCount));
queueFamilyIndices.families[counter].queuePriorities.resize(size_t(queueFamily.queueCount));

queueFamilyIndices.families[counter].index = counter;
queueFamilyIndices.families[counter].flags = queueFamily.queueFlags;
queueFamilyIndices.families[counter].supportsGraphics = supportsGraphics;
queueFamilyIndices.families[counter].supportsTransfer = supportsTransfer;
queueFamilyIndices.families[counter].supportsPresentation = presentSupport;

for (uint32_t i = 0; i < queueFamily.queueCount; i++) {
Expand All @@ -636,6 +701,10 @@ namespace Atlas {
queue->familyIndex = counter;
queue->index = i;

queue->supportsGraphics = supportsGraphics;
queue->supportsTransfer = supportsTransfer;
queue->supportsPresentation = presentSupport;

queueFamilyIndices.families[counter].queues[i] = queue;
queueFamilyIndices.families[counter].queuePriorities[i] = 1.0f;
}
Expand All @@ -645,22 +714,12 @@ namespace Atlas {

counter = 0;
for (auto& queueFamily : queueFamilyIndices.families) {
bool isGraphicsFamilyValid = true;
bool isTransferFamilyValid = true;

if (!(queueFamily.flags & VK_QUEUE_GRAPHICS_BIT))
isGraphicsFamilyValid = false;

if (!(queueFamily.flags & VK_QUEUE_COMPUTE_BIT))
isGraphicsFamilyValid = false;

if (!(queueFamily.flags & VK_QUEUE_TRANSFER_BIT))
isTransferFamilyValid = false;


if (isGraphicsFamilyValid) {
if (queueFamily.supportsGraphics) {
queueFamilyIndices.queueFamilies[QueueType::GraphicsQueue] = counter;
}
if (isTransferFamilyValid) {
if (queueFamily.supportsTransfer) {
queueFamilyIndices.queueFamilies[QueueType::TransferQueue] = counter;
}
if (queueFamily.supportsPresentation) {
Expand All @@ -683,7 +742,7 @@ namespace Atlas {
continue;
}

if (queueFamily.flags & VK_QUEUE_TRANSFER_BIT) {
if (queueFamily.supportsTransfer) {
queueFamilyIndices.queueFamilies[QueueType::TransferQueue] = counter;
break;
}
Expand Down Expand Up @@ -936,8 +995,16 @@ namespace Atlas {

if (it == cmdLists.end()) {
auto queueFamilyIndex = queueFamilyIndices.queueFamilies[queueType];

std::vector<VkQueue> queues;
for (auto& queueFamily : queueFamilyIndices.families) {
for (auto& queue : queueFamily.queues) {
queues.push_back(queue->queue);
}
}

CommandList *cmd = new CommandList(this, queueType,
queueFamilyIndex.value(), frameIndependent);
queueFamilyIndex.value(), queues, frameIndependent);
cmdLists.push_back(cmd);
return cmd;
}
Expand Down Expand Up @@ -968,7 +1035,8 @@ namespace Atlas {
// Backup plan, search for all queues
Ref<Queue> lastSupportedQueue = nullptr;
for (auto& queueFamily : queueFamilyIndices.families) {
if (queueType != PresentationQueue && (!(queueFamily.flags & neededFlags)) ||
if ((queueType == GraphicsQueue && !queueFamily.supportsGraphics) ||
(queueType == TransferQueue && !queueFamily.supportsTransfer) ||
(queueType == PresentationQueue && !queueFamily.supportsPresentation))
continue;

Expand Down
Loading

0 comments on commit d38f53a

Please sign in to comment.