vxdna: Add contiguous heap VA and error handling for QEMU vxdna

Add expandable device heap support in vxdna (QEMU virtio-accel backend)
with contiguous virtual address management and robust error handling.

Heap VA management:
- Reserve a contiguous HEAP_MAX_SIZE (512MB) VA range with PROT_NONE
  on first DEV_HEAP BO creation. Use MAP_FIXED to place each expansion
  chunk at m_heap_base + m_heap_cur_offset, ensuring the guest kernel
  sees a contiguous UVA space matching its expectations.
- Track heap state with m_heap_destroyed flag: once any DEV_HEAP BO is
  destroyed, the contiguous VA invariant is broken so all further
  DEV/DEV_HEAP allocations are rejected with -EINVAL.
- Add bounds check: reject expansion if m_heap_cur_offset + size
  exceeds HEAP_MAX_SIZE with -ENOSPC.
- Move m_heap_cur_offset update after add_bo() and response write to
  avoid permanently wasting VA budget on failed operations.

Error handling:
- vxdna_ccmd_error_wrap: Remove exception re-throwing after catch.
  Re-throwing caused QEMU to send 0x1200 error responses and skip
  fence signaling, hanging the guest indefinitely on sync_wait.
  Now errors are logged and written to the response buffer only.
- write_err_rsp: Log and return instead of throwing when resp resource
  is missing, preventing exceptions from escaping the error wrapper.
- vxdna_bo constructor: Add mmap_target parameter to support MAP_FIXED
  placement. Add get_type() accessor for BO type queries.

Cleanup:
- Clean up reserved VA range in vxdna_context destructor.
- Update Doxygen for vxdna_ccmd_error_wrap to reflect non-re-throwing
  behavior.

Signed-off-by: Wendy Liang <wendy.liang@amd.com>

Author: wendyliang25

src/vxdna/src/vaccel_amdxdna.cpp

@@ -71,7 +71,8 @@ vxdna_bo(int ctx_fd_in, const struct amdxdna_ccmd_create_bo_req *req)
     vxdna_dbg("Create bo: ctx_fd=%d, type=%d, size=%lu", m_ctx_fd, m_bo_type, m_size);
     ret = ioctl(m_ctx_fd, DRM_IOCTL_AMDXDNA_CREATE_BO, &args);
     if (ret)
-        VACCEL_THROW_MSG(-errno, "Create bo failed ret %d", ret);
+        VACCEL_THROW_MSG(-errno, "Create bo failed ret %d, errno %d, %s",
+                         ret, errno, strerror(errno));
 
     m_bo_handle = args.handle;
     bo_info.handle = m_bo_handle;
@@ -90,7 +91,8 @@ vxdna_bo(int ctx_fd_in, const struct amdxdna_ccmd_create_bo_req *req)
 
 vxdna_bo::
 vxdna_bo(const std::shared_ptr<vaccel_resource> &res, int ctx_fd_in,
-         const struct amdxdna_ccmd_create_bo_req *req)
+         const struct amdxdna_ccmd_create_bo_req *req,
+         void *mmap_target)
          : m_opaque_handle(res->get_opaque_handle())
 {
     struct amdxdna_drm_get_bo_info bo_info = {};
@@ -171,18 +173,18 @@ vxdna_bo(const std::shared_ptr<vaccel_resource> &res, int ctx_fd_in,
 
     vxdna_dbg("mmap is required for handle: res_id=%u, handle=%u, opaque_handle=%d, vaddr=%lx, xdna_addr=%lx",
               res->get_res_id(), m_bo_handle, res->get_opaque_handle(), m_vaddr, m_xdna_addr);
-    // mmap is required for non-dev BOs
     uint64_t resv_vaddr = 0, resv_size = 0, va_to_map = 0;
     void *resv_va = nullptr;
     int flags = MAP_SHARED | MAP_LOCKED;
-    if (req->map_align) {
+    if (mmap_target) {
+        va_to_map = reinterpret_cast<uint64_t>(mmap_target);
+        flags |= MAP_FIXED;
+    } else if (req->map_align) {
         resv_va = ::mmap(0, m_map_size + req->map_align, PROT_READ | PROT_WRITE,
                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
         if (resv_va == MAP_FAILED) {
-            // Clean up if we created the BO
-            if (has_created_bo) {
+            if (has_created_bo)
                 close_gem_handle(m_ctx_fd, m_bo_handle);
-            }
             VACCEL_THROW_MSG(-ENOMEM, "Reserve vaddr range failed, map_align=%zu", req->map_align);
         }
 
@@ -198,22 +200,21 @@ vxdna_bo(const std::shared_ptr<vaccel_resource> &res, int ctx_fd_in,
         int saved_errno = errno;
         if (resv_va && resv_va != MAP_FAILED)
             ::munmap(resv_va, resv_size);
-        // Clean up if we created the BO
-        if (has_created_bo) {
+        if (has_created_bo)
             close_gem_handle(m_ctx_fd, m_bo_handle);
-        }
         VACCEL_THROW_MSG(-saved_errno,
                          "Map bo failed, errno %d, %s, to map startaddr 0x%lx, map_offset 0x%lx, map_size 0x%lx",
                          saved_errno, strerror(saved_errno), va_to_map, m_map_offset, m_map_size);
     }
     m_vaddr = reinterpret_cast<uint64_t>(va);
 
-    if (req->map_align && m_vaddr > resv_vaddr)
+    if (!mmap_target && req->map_align && m_vaddr > resv_vaddr)
         ::munmap(resv_va, static_cast<size_t>(m_vaddr - resv_vaddr));
-    if (resv_vaddr + resv_size > m_vaddr + m_map_size)
+    if (!mmap_target && resv_vaddr + resv_size > m_vaddr + m_map_size)
         munmap(reinterpret_cast<void *>(m_vaddr + m_map_size),
                static_cast<size_t>(resv_vaddr + resv_size - m_vaddr - m_map_size));
-    vxdna_dbg("Created BO with resource: type=%u, res_id=%u", req->bo_type, req->res_id);
+    vxdna_dbg("Created BO with resource: type=%u, res_id=%u, vaddr=0x%lx",
+              req->bo_type, req->res_id, m_vaddr);
 }
 
 vxdna_bo::
@@ -420,12 +421,39 @@ void
 vxdna_context::
 create_bo(const struct amdxdna_ccmd_create_bo_req *req)
 {
+    if (m_heap_destroyed &&
+        (req->bo_type == AMDXDNA_BO_DEV || req->bo_type == AMDXDNA_BO_DEV_HEAP))
+        VACCEL_THROW_MSG(-EINVAL, "Heap destroyed, cannot allocate type %u", req->bo_type);
+
     std::shared_ptr<vxdna_bo> xdna_bo;
     if (req->bo_type != AMDXDNA_BO_DEV) {
         auto res = get_device().get_resource(req->res_id);
         if (!res)
             VACCEL_THROW_MSG(-EINVAL, "Res: %u not found", req->res_id);
-        xdna_bo = std::make_shared<vxdna_bo>(res, get_fd(), req);
+
+        void *mmap_target = nullptr;
+        if (req->bo_type == AMDXDNA_BO_DEV_HEAP) {
+            if (!m_heap_base) {
+                m_heap_base = ::mmap(0, HEAP_MAX_SIZE, PROT_NONE,
+                                     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+                if (m_heap_base == MAP_FAILED) {
+                    m_heap_base = nullptr;
+                    VACCEL_THROW_MSG(-ENOMEM,
+                                     "Failed to reserve heap VA range (%zu bytes)",
+                                     HEAP_MAX_SIZE);
+                }
+                vxdna_dbg("Reserved heap VA range: base=%p, size=0x%zx",
+                          m_heap_base, HEAP_MAX_SIZE);
+            }
+            if (m_heap_cur_offset + req->size > HEAP_MAX_SIZE)
+                VACCEL_THROW_MSG(-ENOSPC,
+                                 "Heap expansion exceeds max size (%zu + %lu > %zu)",
+                                 m_heap_cur_offset, (unsigned long)req->size,
+                                 HEAP_MAX_SIZE);
+            mmap_target = static_cast<char *>(m_heap_base) + m_heap_cur_offset;
+        }
+
+        xdna_bo = std::make_shared<vxdna_bo>(res, get_fd(), req, mmap_target);
     } else {
         xdna_bo = std::make_shared<vxdna_bo>(get_fd(), req);
     }
@@ -440,6 +468,9 @@ create_bo(const struct amdxdna_ccmd_create_bo_req *req)
         VACCEL_THROW_MSG(-EINVAL, "Resp resource not found for context %u", get_id());
     (void)resp_res->write(req->hdr.rsp_off, &rsp, sizeof(rsp));
     add_bo(std::move(xdna_bo));
+
+    if (req->bo_type == AMDXDNA_BO_DEV_HEAP)
+        m_heap_cur_offset += req->size;
     vxdna_dbg("Created bo: handle=%u, xdna_addr=%lu", rsp.handle, rsp.xdna_addr);
 }
 
@@ -454,6 +485,9 @@ void
 vxdna_context::
 remove_bo(uint32_t handle)
 {
+    auto bo = m_bo_table.lookup(handle);
+    if (bo && bo->get_type() == AMDXDNA_BO_DEV_HEAP)
+        m_heap_destroyed = true;
     vxdna_dbg("Removing bo: handle=%u", handle);
     m_bo_table.erase(handle);
 }
@@ -674,8 +708,11 @@ vxdna_context::
 write_err_rsp(int err)
 {
     auto resp_res = get_resp_res();
-    if (!resp_res)
-        VACCEL_THROW_MSG(-EINVAL, "Resp resource not found for context %u", get_id());
+    if (!resp_res) {
+        vxdna_err("write_err_rsp: no resp resource for ctx %u, err %d dropped",
+                  get_id(), err);
+        return;
+    }
     struct amdxdna_ccmd_rsp rsp = {};
     rsp.ret = err;
     rsp.base.len = sizeof(rsp);

src/vxdna/src/vaccel_amdxdna.h

@@ -16,13 +16,15 @@
 #include <stddef.h>
 #include <stdexcept>
 #include <memory>
+#include <sys/mman.h>
 
 #include "drm_hw.h" // from xdna shim virtio
 
 #include "drm_local/amdxdna_accel.h"
 #include "amdxdna_proto.h"
 #include "vaccel.h"
 #include "vaccel_internal.h"
+#include "../util/vxdna_debug.h"
 
 /**
  * @brief AMDXDNA Buffer Object wrapper
@@ -50,7 +52,9 @@ class vxdna_bo {
      * @param req BO creation request with type, size, and alignment
      * @throws vaccel_error on DRM ioctl failure
      */
-    vxdna_bo(const std::shared_ptr<vaccel_resource> &res, int ctx_fd_in, const struct amdxdna_ccmd_create_bo_req *req);
+    vxdna_bo(const std::shared_ptr<vaccel_resource> &res, int ctx_fd_in,
+             const struct amdxdna_ccmd_create_bo_req *req,
+             void *mmap_target = nullptr);
 
     /**
      * @brief Construct device-local BO
@@ -100,6 +104,15 @@ class vxdna_bo {
         return m_bo_handle;
     }
 
+    /**
+     * @brief Get the BO type
+     * @return BO type (AMDXDNA_BO_DEV, AMDXDNA_BO_DEV_HEAP, etc.)
+     */
+    uint32_t get_type() const noexcept
+    {
+        return m_bo_type;
+    }
+
 private:
     uint64_t m_size = 0;          /**< Buffer size in bytes */
     uint64_t m_vaddr = 0;         /**< Virtual address (after mmap) */
@@ -156,6 +169,9 @@ class vxdna_context : public vaccel_context<vxdna_context, vxdna> {
      */
     ~vxdna_context() {
         vxdna_dbg("Context destroying: ctx_id=%u, fd=%d", get_id(), get_fd());
+        m_bo_table.clear();
+        if (m_heap_base)
+            ::munmap(m_heap_base, HEAP_MAX_SIZE);
         close(get_fd());
     }
 
@@ -493,6 +509,11 @@ class vxdna_context : public vaccel_context<vxdna_context, vxdna> {
     std::shared_ptr<vaccel_resource> m_resp_res;
     vaccel_map<uint32_t, std::shared_ptr<vxdna_bo>> m_bo_table;
     vaccel_map<uint32_t, std::shared_ptr<vxdna_hwctx>> m_hwctx_table;
+
+    static constexpr size_t HEAP_MAX_SIZE = 512UL << 20;
+    void *m_heap_base = nullptr;
+    size_t m_heap_cur_offset = 0;
+    bool m_heap_destroyed = false;
 };
 
 /**
@@ -711,41 +732,36 @@ int vxdna_device_process_ccmd(void *cookie, const void *cmd_buf, size_t buf_size
  * @brief Exception wrapper for ccmd handlers
  *
  * Wraps a ccmd handler function to catch exceptions and write
- * appropriate error responses to the context's response buffer
- * before re-throwing.
- *
- * Usage:
- * @code
- * vxdna_ccmd_error_wrap(ctx, [&]() {
- *     ctx->create_bo(req);
- * });
- * @endcode
+ * appropriate error responses to the context's response buffer.
+ * Exceptions are NOT re-thrown so that QEMU receives a zero return
+ * from vaccel_submit_ccmd and signals the fence normally. Re-throwing
+ * would cause QEMU to send 0x1200 error responses which cascades
+ * into resource lookup failures for subsequent commands.
  *
  * Error handling:
- * - vaccel_error: Writes e.code() to response, re-throws
- * - std::exception: Writes -EIO to response, re-throws
- * - Unknown: Writes -EIO to response, throws std::runtime_error
+ * - vaccel_error: Logs error, writes e.code() to response
+ * - std::exception: Logs error, writes -EIO to response
+ * - Unknown: Logs error, writes -EIO to response
  *
  * @tparam ContextType Context type (must have write_err_rsp method)
  * @tparam F Callable type (lambda, function, etc.)
  * @param ctx Context to write error response to
  * @param f Handler function to execute
- * @throws Re-throws caught exceptions after writing response
  */
 template<typename ContextType, typename F> void
 vxdna_ccmd_error_wrap(const std::shared_ptr<ContextType> &ctx, F &&f)
 {
     try {
         f();
     } catch (const vaccel_error& e) {
+        vxdna_err("ccmd failed: %s", e.what());
         ctx->write_err_rsp(e.code());
-        throw e;
     } catch (const std::exception& e) {
+        vxdna_err("ccmd failed (unexpected): %s", e.what());
         ctx->write_err_rsp(-EIO);
-        throw e;
     } catch (...) {
+        vxdna_err("ccmd failed (unknown exception)");
         ctx->write_err_rsp(-EIO);
-        throw std::runtime_error("Unknown exception");
     }
 }