fgviewer: visualize intermediate buffers

* Intermediate Buffer Visualization: Enabled live monitoring of internal
  `FrameGraph` render targets directly in the `fgviewer` web UI.
* HTTP Polling Architecture: Switched from WebSocket binary pushes to native
  `<img>` polling (`/api/image`)
* Robust Resource Tracking: Replaced string-based lookups with
  `(ViewId, ResourceId)` composite keys to prevent cross-view collisions and
  ensure accurate reads.
* Format Post-Processing: Extracted readback conversions (HDR tonemapping,
  depth normalization, MSAA downsampling, single-channel expansion) into
  `DebugServer`.
* UI Polish: Added a live-updating full-screen image modal and explicitly
  filtered internal debug passes from the Graphviz/JSON exports to prevent
  DOM thrashing.
* WIP: Currently Unsupported:
  * GL backend: failed with:
      OpenGL framebuffer error 0x8cd6 (GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT) in "readTexture" at line 4198
  * Mipmaps & Subresources: Reading specific mip levels or array layers is
    explicitly skipped.
  * Shadowmaps: Variance Shadow Maps (VSM) will physically evaluate to `0.0`
    and appear completely empty/black in scenes without active shadow
    casters (due to inverted-Z).
  * Stencil: Resolving and reading stencil buffer data is not supported by
    the backend.
* WIP: Untested outside of MacOS+metal/vk

Author: poweifeng

filament/CMakeLists.txt

@@ -155,6 +155,7 @@ set(SRCS
         src/ds/StructureDescriptorSet.cpp
         src/fg/Blackboard.cpp
         src/fg/DependencyGraph.cpp
+        src/fg/FgviewerManager.cpp
         src/fg/FrameGraph.cpp
         src/fg/FrameGraphPass.cpp
         src/fg/FrameGraphResources.cpp

filament/backend/src/DataReshaper.h

@@ -284,6 +284,7 @@ class DataReshaper {
                 switch (srcType) {
                     case UBYTE: reshaper = reshapeImageImpl<float, uint8_t>; break;
                     case FLOAT: reshaper = reshapeImageImpl<float, float>; break;
+                    case HALF: reshaper = reshapeImageImpl<float, math::half>; break;
                     case INT: reshaper = reshapeImageImpl<float, int32_t>; break;
                     case UINT: reshaper = reshapeImageImpl<float, uint32_t>; break;
                     case UINT_10F_11F_11F_REV:
@@ -300,6 +301,7 @@ class DataReshaper {
                 switch (srcType) {
                     case UBYTE: reshaper = reshapeImageImpl<int32_t, uint8_t>; break;
                     case FLOAT: reshaper = reshapeImageImpl<int32_t, float>; break;
+                    case HALF: reshaper = reshapeImageImpl<int32_t, math::half>; break;
                     case INT: reshaper = reshapeImageImpl<int32_t, int32_t>; break;
                     case UINT: reshaper = reshapeImageImpl<int32_t, uint32_t>; break;
                     case UINT_10F_11F_11F_REV:
@@ -316,6 +318,7 @@ class DataReshaper {
                 switch (srcType) {
                     case UBYTE: reshaper = reshapeImageImpl<uint32_t, uint8_t>; break;
                     case FLOAT: reshaper = reshapeImageImpl<uint32_t, float>; break;
+                    case HALF: reshaper = reshapeImageImpl<uint32_t, math::half>; break;
                     case INT: reshaper = reshapeImageImpl<uint32_t, int32_t>; break;
                     case UINT: reshaper = reshapeImageImpl<uint32_t, uint32_t>; break;
                     case UINT_10F_11F_11F_REV:

filament/backend/src/vulkan/VulkanReadPixels.cpp

@@ -223,16 +223,19 @@ void VulkanReadPixels::run(fvkmemory::resource_ptr<VulkanTexture> srcTexture, ui
         bpp *= componentCount;
     }
 
+    uint32_t const samples = srcTexture->samples > 1 ? srcTexture->samples : 1;
+
     // Use a VkBuffer as the staging area for readback.
     // Using a buffer instead of a linearly tiled VkImage unifies the readback path and avoids
     // driver/validation layer issues since some implementations strictly prohibit linear tiling
     // for certain formats (such as depth/stencil).
     VkBuffer stagingBuffer = VK_NULL_HANDLE;
     VkMemoryRequirements memReqs;
 
+    uint32_t const stagingSize = width * height * bpp * samples;
     VkBufferCreateInfo bufferInfo = {
         .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
-        .size = width * height * bpp,
+        .size = stagingSize,
         .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT,
     };
     // TODO: we could use a staging buffer pool, but this is on another thread.  We'd need to
@@ -371,6 +374,9 @@ void VulkanReadPixels::run(fvkmemory::resource_ptr<VulkanTexture> srcTexture, ui
         uint8_t const* srcPixels;
         vkMapMemory(device, stagingMemory, 0, VK_WHOLE_SIZE, 0, (void**) &srcPixels);
 
+        // If MSAA, MoltenVK returns samples in planar layout (Sample 0 is the first width * height
+        // pixels). So we can simply ask DataReshaper to read width * height elements with standard
+        // row pitch!
         int const rowPitch = width * bpp;
         if (!DataReshaper::reshapeImage(&p, componentType, componentCount, srcPixels, rowPitch,
                     static_cast<int>(width), static_cast<int>(height), swizzle)) {

filament/backend/src/vulkan/utils/Conversion.cpp

@@ -770,6 +770,7 @@ PixelDataType getComponentType(VkFormat format) {
         case VK_FORMAT_D16_UNORM: return PixelDataType::USHORT;
         case VK_FORMAT_D32_SFLOAT: return PixelDataType::FLOAT;
         case VK_FORMAT_X8_D24_UNORM_PACK32: return PixelDataType::UINT;
+        case VK_FORMAT_B10G11R11_UFLOAT_PACK32: return PixelDataType::UINT_10F_11F_11F_REV;
         // For combined depth/stencil formats, we return the primary (depth) aspect type.
         // Stencil aspect overrides are handled dynamically during readback.
         case VK_FORMAT_D24_UNORM_S8_UINT: return PixelDataType::UINT;
@@ -882,6 +883,9 @@ uint32_t getComponentCount(VkFormat format) {
         case VK_FORMAT_R32G32B32A32_SINT:
         case VK_FORMAT_R32G32B32A32_SFLOAT:
             return 4;
+        case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
+        case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32:
+            return 3;
         default: assert_invariant(false && "Unknown data type, conversion is not supported.");
     }
     return {};

filament/src/PostProcessManager.cpp

@@ -3676,7 +3676,7 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::blitDepth(FrameGraph& fg,
 
 FrameGraphId<FrameGraphTexture> PostProcessManager::resolve(FrameGraph& fg,
         utils::StaticString outputBufferName, FrameGraphId<FrameGraphTexture> const input,
-        FrameGraphTexture::Descriptor outDesc) noexcept {
+        FrameGraphTexture::Descriptor outDesc, utils::CString customPassName) noexcept {
 
     // Don't do anything if we're not a MSAA buffer
     auto const& inDesc = fg.getDescriptor(input);
@@ -3690,7 +3690,8 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::resolve(FrameGraph& fg,
     //     through shaders or some other manipulation.
     if ((isDepthFormat(inDesc.format) || isStencilFormat(inDesc.format)) &&
             (!mDepthStencilResolveSupported)) {
-        return resolveDepthWithShader(fg, outputBufferName, input, outDesc);
+        return resolveDepthWithShader(fg, outputBufferName, input, outDesc,
+                std::move(customPassName));
     }
 
     outDesc.width = inDesc.width;
@@ -3703,7 +3704,8 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::resolve(FrameGraph& fg,
         FrameGraphId<FrameGraphTexture> output;
     };
 
-    auto const& ppResolve = fg.addPass<ResolveData>("resolve",
+    auto const& ppResolve = fg.addPass<ResolveData>(
+            customPassName.empty() ? "resolve" : customPassName.c_str(),
             [&](FrameGraph::Builder& builder, auto& data) {
                 data.input = builder.read(input, FrameGraphTexture::Usage::BLIT_SRC);
                 data.output = builder.createTexture(outputBufferName, outDesc);
@@ -3731,7 +3733,7 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::resolve(FrameGraph& fg,
 
 FrameGraphId<FrameGraphTexture> PostProcessManager::resolveDepthWithShader(FrameGraph& fg,
         utils::StaticString outputBufferName, FrameGraphId<FrameGraphTexture> const input,
-        FrameGraphTexture::Descriptor outDesc) noexcept {
+        FrameGraphTexture::Descriptor outDesc, utils::CString customPassName) noexcept {
 
     // Don't do anything if we're not a MSAA buffer
     auto const& inDesc = fg.getDescriptor(input);
@@ -3754,7 +3756,8 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::resolveDepthWithShader(Frame
         FrameGraphId<FrameGraphTexture> output;
     };
 
-    auto const& ppResolve = fg.addPass<ResolveData>("resolveDepthWithShader",
+    auto const& ppResolve = fg.addPass<ResolveData>(
+            customPassName.empty() ? "resolveDepthWithShader" : customPassName.c_str(),
             [&](FrameGraph::Builder& builder, auto& data) {
                 data.input = builder.sample(input);
                 data.output = builder.createTexture(outputBufferName, outDesc);

filament/src/PostProcessManager.h

@@ -301,16 +301,19 @@ class PostProcessManager {
 
     // Resolves base level of input and outputs a texture from outDesc.
     // outDesc with, height, format and samples will be overridden.
-    FrameGraphId<FrameGraphTexture> resolve(FrameGraph& fg,
-            utils::StaticString outputBufferName, FrameGraphId<FrameGraphTexture> input,
-            FrameGraphTexture::Descriptor outDesc) noexcept;
+    // customPassName is an optional parameter to override the default "resolve" pass name.
+    FrameGraphId<FrameGraphTexture> resolve(FrameGraph& fg, utils::StaticString outputBufferName,
+            FrameGraphId<FrameGraphTexture> input, FrameGraphTexture::Descriptor outDesc,
+            utils::CString customPassName = {}) noexcept;
 
     // Resolves base level of input and outputs a texture from outDesc using a shader instead of
     // driver-implemented API.
     // outDesc with, height, format and samples will be overridden.
+    // customPassName is an optional parameter to override the default "resolveDepthWithShader" pass
+    // name.
     FrameGraphId<FrameGraphTexture> resolveDepthWithShader(FrameGraph& fg,
             utils::StaticString outputBufferName, FrameGraphId<FrameGraphTexture> input,
-            FrameGraphTexture::Descriptor outDesc) noexcept;
+            FrameGraphTexture::Descriptor outDesc, utils::CString customPassName = {}) noexcept;
 
     // VSM shadow mipmap pass
     FrameGraphId<FrameGraphTexture> vsmMipmapPass(FrameGraph& fg,

filament/src/details/Engine.cpp

@@ -919,8 +919,15 @@ int FEngine::loop() {
 #endif
     if (fgviewerPortString != nullptr) {
         const int fgviewerPort = atoi(fgviewerPortString);
-        debug.fgviewerServer = new fgviewer::DebugServer(fgviewerPort);
-
+        debug.fgviewerServer = new fgviewer::DebugServer(fgviewerPort,
+                fgviewer::DebugServer::ReadbackRequest(
+                        [this](fgviewer::ViewHandle viewId, uint32_t id, utils::CString const& name,
+                                std::function<void(fgviewer::DebugServer::PixelBuffer, uint32_t,
+                                        uint32_t, fgviewer::DebugServer::PixelDataFormat,
+                                        fgviewer::DebugServer::FormatInfo)>
+                                        callback) {
+                            requestTextureReadback(viewId, id, name, std::move(callback));
+                        }));
         // Sometimes the server can fail to spin up (e.g. if the above port is already in use).
         // When this occurs, carry onward, developers can look at civetweb.txt for details.
         if (!debug.fgviewerServer->isReady()) {
@@ -1775,4 +1782,15 @@ Engine::Config Engine::BuilderDetails::validateConfig(Config config) noexcept {
     return config;
 }
 
+#if FILAMENT_ENABLE_FGVIEWER
+void FEngine::requestTextureReadback(fgviewer::ViewHandle viewId, uint32_t id,
+        const utils::CString& name,
+        std::function<void(fgviewer::DebugServer::PixelBuffer, uint32_t, uint32_t,
+                fgviewer::DebugServer::PixelDataFormat, fgviewer::DebugServer::FormatInfo)>&&
+                callback) {
+    std::unique_lock<utils::Mutex> lock(mReadbackRequestsMutex);
+    mReadbackRequests.emplace_back(ReadbackRequest{ viewId, id, name, std::move(callback) });
+}
+#endif
+
 } // namespace filament

filament/src/details/Engine.h

@@ -569,6 +569,26 @@ class FEngine : public Engine, public utils::FeatureFlagManager {
 
     backend::Driver& getDriver() const noexcept { return *mDriver; }
 
+#if FILAMENT_ENABLE_FGVIEWER
+    struct ReadbackRequest {
+        using Callback = std::function<void(fgviewer::DebugServer::PixelBuffer, uint32_t, uint32_t,
+                fgviewer::DebugServer::PixelDataFormat, fgviewer::DebugServer::FormatInfo)>;
+        fgviewer::ViewHandle viewId;
+        uint32_t id;
+        utils::CString name;
+        Callback callback;
+    };
+
+    void requestTextureReadback(fgviewer::ViewHandle viewId, uint32_t id,
+            utils::CString const& name,
+            std::function<void(fgviewer::DebugServer::PixelBuffer, uint32_t, uint32_t,
+                    fgviewer::DebugServer::PixelDataFormat, fgviewer::DebugServer::FormatInfo)>&&
+                    callback);
+
+    utils::Mutex mReadbackRequestsMutex;
+    std::vector<ReadbackRequest> mReadbackRequests;
+#endif
+
     utils::Slice<const Engine::FeatureFlag> getFeatureFlags() const noexcept;
     bool setFeatureFlag(char const* name, bool value) noexcept;
     std::optional<bool> getFeatureFlag(char const* name) const noexcept;

filament/src/details/Renderer.cpp

@@ -34,6 +34,7 @@
 
 #include "ds/StructureDescriptorSet.h"
 
+#include "fg/FgviewerManager.h"
 #include "fg/FrameGraph.h"
 #include "fg/FrameGraphId.h"
 #include "fg/FrameGraphResources.h"
@@ -1549,25 +1550,23 @@ void FRenderer::renderJob(DriverApi& driver, RootArenaScope& rootArenaScope, FVi
 //    auto debug = structure
 //    fg.forwardResource(fgViewRenderTarget, debug ? debug : input);
 
-    fg.forwardResource(fgViewRenderTarget, input);
+    FgviewerManager::handleFgReadbacks(engine, fg, viewRenderTarget, view.getViewHandle());
 
+    fg.forwardResource(fgViewRenderTarget, input);
     fg.present(fgViewRenderTarget);
 
     fg.compile();
 
-#if FILAMENT_ENABLE_FGVIEWER
-    fgviewer::DebugServer* fgviewerServer = engine.debug.fgviewerServer;
-    if (UTILS_LIKELY(fgviewerServer)) {
-        fgviewerServer->update(view.getViewHandle(), fg.getFrameGraphInfo(view.getName()));
-    }
-#endif
+    FgviewerManager::update(engine, fg, view);
 
     //utils::io::sstream graphviz;
     //fg.export_graphviz(graphviz, view.getName());
     //DLOG(INFO) << graphviz.c_str();
 
     fg.execute(driver);
 
+    FgviewerManager::tick(engine);
+
     // save the current history entry and destroy the oldest entry
     view.commitFrameHistory(engine);
 

filament/src/fg/DependencyGraph.cpp

@@ -23,6 +23,10 @@
 #include <utils/CString.h>
 #include <utils/ostream.h>
 
+#if FILAMENT_ENABLE_FGVIEWER
+#include <fgviewer/FrameGraphInfo.h>
+#endif
+
 #include <iterator>
 #include <cstdint>
 
@@ -155,14 +159,32 @@ void DependencyGraph::export_graphviz(utils::io::ostream& out, char const* name)
 
     auto const& nodes = mNodes;
 
-    for (Node const* node : nodes) {
+    auto skipNode = [](char const* name) {
+#if FILAMENT_ENABLE_FGVIEWER
+        if (!name) {
+            return false;
+        }
+        std::string_view view{ name };
+        return view == std::string_view(fgviewer::READBACK_PASS_NAME) || view == std::string_view(fgviewer::RESOLVED_MONITOR_PASS_NAME);
+#else
+        return false;
+#endif
+    };
+
+    for (Node const* node: nodes) {
+        if (skipNode(node->getName())) {
+            continue;
+        }
         uint32_t id = node->getId();
         utils::CString s = node->graphvizify();
         out << "\"N" << id << "\" " << s.c_str() << "\n";
     }
 
     out << "\n";
-    for (Node const* node : nodes) {
+    for (Node const* node: nodes) {
+        if (skipNode(node->getName())) {
+            continue;
+        }
         uint32_t id = node->getId();
 
         auto edges = getOutgoingEdges(node);
@@ -174,22 +196,48 @@ void DependencyGraph::export_graphviz(utils::io::ostream& out, char const* name)
 
         // render the valid edges
         if (first != pos) {
-            out << "N" << id << " -> { ";
-            while (first != pos) {
-                Node const* ref = getNode((*first++)->to);
-                out << "N" << ref->getId() << " ";
+            bool hasValidEdges = false;
+            for (auto it = first; it != pos; ++it) {
+                Node const* ref = getNode((*it)->to);
+                if (skipNode(ref->getName())) {
+                    continue;
+                }
+                hasValidEdges = true;
+            }
+            if (hasValidEdges) {
+                out << "N" << id << " -> { ";
+                while (first != pos) {
+                    Node const* ref = getNode((*first++)->to);
+                    if (skipNode(ref->getName())) {
+                        continue;
+                    }
+                    out << "N" << ref->getId() << " ";
+                }
+                out << "} [color=" << s.c_str() << "2]\n";
             }
-            out << "} [color=" << s.c_str() << "2]\n";
         }
 
         // render the invalid edges
         if (first != edges.end()) {
-            out << "N" << id << " -> { ";
-            while (first != edges.end()) {
-                Node const* ref = getNode((*first++)->to);
-                out << "N" << ref->getId() << " ";
+            bool hasInvalidEdges = false;
+            for (auto it = first; it != edges.end(); ++it) {
+                Node const* ref = getNode((*it)->to);
+                if (skipNode(ref->getName())) {
+                    continue;
+                }
+                hasInvalidEdges = true;
+            }
+            if (hasInvalidEdges) {
+                out << "N" << id << " -> { ";
+                while (first != edges.end()) {
+                    Node const* ref = getNode((*first++)->to);
+                    if (skipNode(ref->getName())) {
+                        continue;
+                    }
+                    out << "N" << ref->getId() << " ";
+                }
+                out << "} [color=" << s.c_str() << "4 style=dashed]\n";
             }
-            out << "} [color=" << s.c_str() << "4 style=dashed]\n";
         }
     }