Add Queue::submit() for command buffer submission

MarijnS95 · claude · MarijnS95 · commit 7caa51c655e4 · 2026-04-07T12:07:23.000+02:00
Move command buffer submission logic from each backend's Device into Queue::submit(), which takes ownership of the command buffers. For now it blocks internally until completion; a TODO marks that it will return a Fence once the Fence abstraction from PR #1007 is available. - Metal: commit() + waitUntilCompleted() - Vulkan: vkEndCommandBuffer() + vkQueueSubmit() with temporary fence + vkWaitForFences() - DX12: CmdList::Close() + ExecuteCommandLists() + Queue::Signal()/Fence::SetEventOnCompletion() wait VulkanQueue now stores a VkDevice handle (with a TODO for lifetime management) so it can create/destroy fences independently. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
diff --git a/include/API/Device.h b/include/API/Device.h
@@ -62,6 +62,19 @@ class Queue {
 public:
   virtual ~Queue() = 0;
 
+  /// Submit command buffers for execution and block until completion.
+  // TODO: Return a Fence instead of blocking, once the Fence abstraction
+  // from PR #1007 is available.
+  virtual llvm::Error
+  submit(llvm::SmallVector<std::unique_ptr<CommandBuffer>> CBs) = 0;
+
+  /// Convenience overload for submitting a single command buffer.
+  llvm::Error submit(std::unique_ptr<CommandBuffer> CB) {
+    llvm::SmallVector<std::unique_ptr<CommandBuffer>> CBs;
+    CBs.push_back(std::move(CB));
+    return submit(std::move(CBs));
+  }
+
 protected:
   Queue() = default;
 };
diff --git a/lib/API/DX/Device.cpp b/lib/API/DX/Device.cpp
@@ -295,22 +295,28 @@ class DXBuffer : public offloadtest::Buffer {
 
 class DXQueue : public offloadtest::Queue {
 public:
+  using Queue::submit;
+
   ComPtr<ID3D12CommandQueue> Queue;
 
   DXQueue(ComPtr<ID3D12CommandQueue> Queue) : Queue(Queue) {}
-  virtual ~DXQueue() {}
+  ~DXQueue() override {}
 
   static llvm::Expected<DXQueue>
   createGraphicsQueue(ComPtr<ID3D12Device> Device) {
     const D3D12_COMMAND_QUEUE_DESC Desc = {D3D12_COMMAND_LIST_TYPE_DIRECT, 0,
                                            D3D12_COMMAND_QUEUE_FLAG_NONE, 0};
-    ComPtr<ID3D12CommandQueue> Queue;
-    if (auto Err =
-            HR::toError(Device->CreateCommandQueue(&Desc, IID_PPV_ARGS(&Queue)),
-                        "Failed to create command queue."))
+    ComPtr<ID3D12CommandQueue> CmdQueue;
+    if (auto Err = HR::toError(
+            Device->CreateCommandQueue(&Desc, IID_PPV_ARGS(&CmdQueue)),
+            "Failed to create command queue."))
       return Err;
-    return DXQueue(Queue);
+    return DXQueue(CmdQueue);
   }
+
+  llvm::Error
+  submit(llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs)
+      override;
 };
 
 class DXCommandBuffer : public offloadtest::CommandBuffer {
@@ -373,6 +379,52 @@ class DXCommandBuffer : public offloadtest::CommandBuffer {
   DXCommandBuffer() : CommandBuffer(GPUAPI::DirectX) {}
 };
 
+llvm::Error DXQueue::submit(
+    llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs) {
+  // This is a hack but it works since this is all single threaded code.
+  static uint64_t FenceCounter = 0;
+
+  for (auto &CB : CBs) {
+    auto &DCB = CB->as<DXCommandBuffer>();
+    if (auto Err =
+            HR::toError(DCB.CmdList->Close(), "Failed to close command list."))
+      return Err;
+
+    ID3D12CommandList *const CmdLists[] = {DCB.CmdList.Get()};
+    Queue->ExecuteCommandLists(1, CmdLists);
+
+    const uint64_t CurrentCounter = FenceCounter + 1;
+    if (auto Err = HR::toError(Queue->Signal(DCB.Fence.Get(), CurrentCounter),
+                               "Failed to add signal."))
+      return Err;
+
+    if (DCB.Fence->GetCompletedValue() < CurrentCounter) {
+#ifdef _WIN32
+      HANDLE Event = DCB.Event;
+#else // WSL
+      HANDLE Event = reinterpret_cast<HANDLE>(DCB.Event);
+#endif
+      if (auto Err = HR::toError(
+              DCB.Fence->SetEventOnCompletion(CurrentCounter, Event),
+              "Failed to register end event."))
+        return Err;
+
+#ifdef _WIN32
+      WaitForSingleObject(DCB.Event, INFINITE);
+#else // WSL
+      pollfd PollEvent;
+      PollEvent.fd = DCB.Event;
+      PollEvent.events = POLLIN;
+      PollEvent.revents = 0;
+      if (poll(&PollEvent, 1, -1) == -1)
+        return llvm::createStringError(
+            std::error_code(errno, std::system_category()), strerror(errno));
+#endif
+    }
+    FenceCounter = CurrentCounter;
+  }
+  return llvm::Error::success();
+}
 class DXDevice : public offloadtest::Device {
 private:
   ComPtr<IDXCoreAdapter> Adapter;
@@ -1179,8 +1231,10 @@ class DXDevice : public offloadtest::Device {
     IS.CB->CmdList->ResourceBarrier(1, &Barrier);
   }
 
+  // waitForSignal is used for tile mapping synchronization, not command buffer
+  // submission. TODO: Replace with a proper fence abstraction.
   llvm::Error waitForSignal(InvocationState &IS) {
-    // This is a hack but it works since this is all single threaded code.
+    // Reuse the command buffer's fence for a quick queue-level signal/wait.
     static uint64_t FenceCounter = 0;
     const uint64_t CurrentCounter = FenceCounter + 1;
 
@@ -1217,14 +1271,7 @@ class DXDevice : public offloadtest::Device {
   }
 
   llvm::Error executeCommandList(InvocationState &IS) {
-    if (auto Err = HR::toError(IS.CB->CmdList->Close(),
-                               "Failed to close command list."))
-      return Err;
-
-    ID3D12CommandList *const CmdLists[] = {IS.CB->CmdList.Get()};
-    GraphicsQueue.Queue->ExecuteCommandLists(1, CmdLists);
-
-    return waitForSignal(IS);
+    return GraphicsQueue.submit(std::move(IS.CB));
   }
 
   llvm::Error createComputeCommands(Pipeline &P, InvocationState &IS) {
diff --git a/lib/API/MTL/MTLDevice.cpp b/lib/API/MTL/MTLDevice.cpp
@@ -75,12 +75,18 @@ static MTL::VertexFormat getMTLVertexFormat(DataFormat Format, int Channels) {
 namespace {
 class MTLQueue : public offloadtest::Queue {
 public:
+  using Queue::submit;
+
   MTL::CommandQueue *Queue;
   MTLQueue(MTL::CommandQueue *Queue) : Queue(Queue) {}
-  ~MTLQueue() {
+  ~MTLQueue() override {
     if (Queue)
       Queue->release();
   }
+
+  llvm::Error
+  submit(llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs)
+      override;
 };
 
 class MTLBuffer : public offloadtest::Buffer {
@@ -122,6 +128,19 @@ class MTLCommandBuffer : public offloadtest::CommandBuffer {
   MTLCommandBuffer() : CommandBuffer(GPUAPI::Metal) {}
 };
 
+llvm::Error MTLQueue::submit(
+    llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs) {
+  for (auto &CB : CBs) {
+    auto &MCB = CB->as<MTLCommandBuffer>();
+    MCB.CmdBuffer->commit();
+    MCB.CmdBuffer->waitUntilCompleted();
+
+    NS::Error *Err = MCB.CmdBuffer->error();
+    if (Err)
+      return toError(Err);
+  }
+  return llvm::Error::success();
+}
 class MTLDevice : public offloadtest::Device {
   Capabilities Caps;
   MTL::Device *Device;
@@ -506,15 +525,7 @@ class MTLDevice : public offloadtest::Device {
   }
 
   llvm::Error executeCommands(InvocationState &IS) {
-    IS.CB->CmdBuffer->commit();
-    IS.CB->CmdBuffer->waitUntilCompleted();
-
-    // Check and surface any errors that occurred during execution.
-    NS::Error *CBErr = IS.CB->CmdBuffer->error();
-    if (CBErr)
-      return toError(CBErr);
-
-    return llvm::Error::success();
+    return GraphicsQueue.submit(std::move(IS.CB));
   }
 
   llvm::Error copyBack(Pipeline &P, InvocationState &IS) {
diff --git a/lib/API/VK/Device.cpp b/lib/API/VK/Device.cpp
@@ -393,10 +393,18 @@ class VulkanBuffer : public offloadtest::Buffer {
 
 class VulkanQueue : public offloadtest::Queue {
 public:
+  using Queue::submit;
+
   VkQueue Queue = VK_NULL_HANDLE;
   uint32_t QueueFamilyIdx = 0;
-  VulkanQueue(VkQueue Q, uint32_t QueueFamilyIdx)
-      : Queue(Q), QueueFamilyIdx(QueueFamilyIdx) {}
+  // TODO: Ensure device lifetime is managed (e.g. via shared_ptr).
+  VkDevice Device = VK_NULL_HANDLE;
+  VulkanQueue(VkQueue Q, uint32_t QueueFamilyIdx, VkDevice Device)
+      : Queue(Q), QueueFamilyIdx(QueueFamilyIdx), Device(Device) {}
+
+  llvm::Error
+  submit(llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs)
+      override;
 };
 
 class VulkanCommandBuffer : public offloadtest::CommandBuffer {
@@ -449,6 +457,37 @@ class VulkanCommandBuffer : public offloadtest::CommandBuffer {
   VulkanCommandBuffer() : CommandBuffer(GPUAPI::Vulkan) {}
 };
 
+llvm::Error VulkanQueue::submit(
+    llvm::SmallVector<std::unique_ptr<offloadtest::CommandBuffer>> CBs) {
+  for (auto &CB : CBs) {
+    auto &VCB = CB->as<VulkanCommandBuffer>();
+    if (vkEndCommandBuffer(VCB.CmdBuffer))
+      return llvm::createStringError(std::errc::device_or_resource_busy,
+                                     "Could not end command buffer.");
+
+    VkSubmitInfo SubmitInfo = {};
+    SubmitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+    SubmitInfo.commandBufferCount = 1;
+    SubmitInfo.pCommandBuffers = &VCB.CmdBuffer;
+
+    VkFenceCreateInfo FenceInfo = {};
+    FenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+    VkFence Fence;
+    if (vkCreateFence(Device, &FenceInfo, nullptr, &Fence))
+      return llvm::createStringError(std::errc::device_or_resource_busy,
+                                     "Could not create fence.");
+
+    if (vkQueueSubmit(Queue, 1, &SubmitInfo, Fence))
+      return llvm::createStringError(std::errc::device_or_resource_busy,
+                                     "Failed to submit to queue.");
+    if (vkWaitForFences(Device, 1, &Fence, VK_TRUE, UINT64_MAX))
+      return llvm::createStringError(std::errc::device_or_resource_busy,
+                                     "Failed waiting for fence.");
+
+    vkDestroyFence(Device, Fence, nullptr);
+  }
+  return llvm::Error::success();
+}
 class VulkanDevice : public offloadtest::Device {
 private:
   std::shared_ptr<VulkanInstance> Instance;
@@ -645,7 +684,8 @@ class VulkanDevice : public offloadtest::Device {
     VkQueue DeviceQueue = VK_NULL_HANDLE;
     vkGetDeviceQueue(Device, QueueFamilyIdx, 0, &DeviceQueue);
 
-    const VulkanQueue GraphicsQueue = VulkanQueue(DeviceQueue, QueueFamilyIdx);
+    const VulkanQueue GraphicsQueue =
+        VulkanQueue(DeviceQueue, QueueFamilyIdx, Device);
 
     return std::make_unique<VulkanDevice>(Instance, PhysicalDevice, Props,
                                           Device, std::move(GraphicsQueue),
@@ -1179,34 +1219,8 @@ class VulkanDevice : public offloadtest::Device {
     return llvm::Error::success();
   }
 
-  llvm::Error executeCommandBuffer(InvocationState &IS,
-                                   VkPipelineStageFlags WaitMask = 0) {
-    if (vkEndCommandBuffer(IS.CB->CmdBuffer))
-      return llvm::createStringError(std::errc::device_or_resource_busy,
-                                     "Could not end command buffer.");
-
-    VkSubmitInfo SubmitInfo = {};
-    SubmitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
-    SubmitInfo.commandBufferCount = 1;
-    SubmitInfo.pCommandBuffers = &IS.CB->CmdBuffer;
-    SubmitInfo.pWaitDstStageMask = &WaitMask;
-    VkFenceCreateInfo FenceInfo = {};
-    FenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-    VkFence Fence;
-    if (vkCreateFence(Device, &FenceInfo, nullptr, &Fence))
-      return llvm::createStringError(std::errc::device_or_resource_busy,
-                                     "Could not create fence.");
-
-    // Submit to the queue
-    if (vkQueueSubmit(GraphicsQueue.Queue, 1, &SubmitInfo, Fence))
-      return llvm::createStringError(std::errc::device_or_resource_busy,
-                                     "Failed to submit to queue.");
-    if (vkWaitForFences(Device, 1, &Fence, VK_TRUE, UINT64_MAX))
-      return llvm::createStringError(std::errc::device_or_resource_busy,
-                                     "Failed waiting for fence.");
-
-    vkDestroyFence(Device, Fence, nullptr);
-    return llvm::Error::success();
+  llvm::Error executeCommandBuffer(InvocationState &IS) {
+    return GraphicsQueue.submit(std::move(IS.CB));
   }
 
   llvm::Error createDescriptorPool(Pipeline &P, InvocationState &IS) {
@@ -2392,7 +2406,7 @@ class VulkanDevice : public offloadtest::Device {
     if (auto Err = createCommands(P, State))
       return Err;
     llvm::outs() << "Commands created.\n";
-    if (auto Err = executeCommandBuffer(State, VK_PIPELINE_STAGE_TRANSFER_BIT))
+    if (auto Err = executeCommandBuffer(State))
       return Err;
     llvm::outs() << "Executed compute command buffer.\n";
     if (auto Err = readBackData(P, State))
