Reduce delays in webRTC remote display.

cpp/open3d/t/io/file_format/FileASSIMP.cpp

@@ -16,6 +16,7 @@
 #include <unordered_map>
 #include <vector>
 
+#include "open3d/core/Dtype.h"
 #include "open3d/core/ParallelFor.h"
 #include "open3d/core/TensorFunction.h"
 #include "open3d/t/io/ImageIO.h"
@@ -177,11 +178,31 @@ bool ReadTriangleMeshUsingASSIMP(
     return true;
 }
 
-static void SetTextureMaterialProperty(aiMaterial* mat,
-                                       aiScene* scene,
-                                       int texture_idx,
-                                       aiTextureType tt,
-                                       t::geometry::Image& img) {
+namespace {
+
+// Checks whether a texture map is present, not empty and uint8 or uint16
+// datatype.
+bool HasValidTexture(const visualization::rendering::Material& material,
+                     const std::string& key) {
+    if (!material.HasTextureMap(key) || material.GetTextureMap(key).IsEmpty()) {
+        return false;
+    }
+    const auto& dt = material.GetTextureMap(key).GetDtype();
+    if (dt != core::UInt8 && dt != core::UInt16) {
+        utility::LogWarning(
+                "Skipping texture map '{}' with unsupported data type '{}'. "
+                "Only uint8 and uint16 are supported.",
+                key, dt.ToString());
+        return false;
+    }
+    return true;
+}
+
+void SetTextureMaterialProperty(aiMaterial* mat,
+                                aiScene* scene,
+                                int texture_idx,
+                                aiTextureType tt,
+                                t::geometry::Image& img) {
     // Encode image as PNG
     std::vector<uint8_t> img_buffer;
     WriteImageToPNGInMemory(img_buffer, img, 6);
@@ -208,7 +229,6 @@ static void SetTextureMaterialProperty(aiMaterial* mat,
     mat->AddProperty(&mode, 1, AI_MATKEY_MAPPINGMODE_V(tt, 0));
 }
 
-namespace {
 // Add hash function for tuple key
 struct TupleHash {
     size_t operator()(const std::tuple<int64_t, float, float>& t) const {
@@ -551,18 +571,19 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename,
         // model has one we just export it, otherwise if both roughness and
         // metal maps are available we combine them, otherwise if only one or
         // the other is available we just export the one map.
+        const auto& material = w_mesh.GetMaterial();
         int n_textures = 0;
-        if (w_mesh.GetMaterial().HasAlbedoMap()) ++n_textures;
-        if (w_mesh.GetMaterial().HasNormalMap()) ++n_textures;
-        if (w_mesh.GetMaterial().HasAOMap()) ++n_textures;
-        if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) {
+        if (HasValidTexture(material, "albedo")) ++n_textures;
+        if (HasValidTexture(material, "normal")) ++n_textures;
+        if (HasValidTexture(material, "ambient_occlusion")) ++n_textures;
+        if (HasValidTexture(material, "ao_rough_metal")) {
             ++n_textures;
-        } else if (w_mesh.GetMaterial().HasRoughnessMap() &&
-                   w_mesh.GetMaterial().HasMetallicMap()) {
+        } else if (HasValidTexture(material, "roughness") &&
+                   HasValidTexture(material, "metallic")) {
             ++n_textures;
         } else {
-            if (w_mesh.GetMaterial().HasRoughnessMap()) ++n_textures;
-            if (w_mesh.GetMaterial().HasMetallicMap()) ++n_textures;
+            if (HasValidTexture(material, "roughness")) ++n_textures;
+            if (HasValidTexture(material, "metallic")) ++n_textures;
         }
         if (n_textures > 0) {
             ai_scene->mTextures = new aiTexture*[n_textures];
@@ -574,27 +595,27 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename,
 
         // Now embed the textures that are available...
         int current_idx = 0;
-        if (w_mesh.GetMaterial().HasAlbedoMap()) {
-            auto img = w_mesh.GetMaterial().GetAlbedoMap();
+        if (HasValidTexture(material, "albedo")) {
+            auto img = material.GetAlbedoMap();
             SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
                                        aiTextureType_DIFFUSE, img);
             SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
                                        aiTextureType_BASE_COLOR, img);
             ++current_idx;
         }
-        if (w_mesh.GetMaterial().HasAOMap()) {
+        if (HasValidTexture(w_mesh.GetMaterial(), "ambient_occlusion")) {
             auto img = w_mesh.GetMaterial().GetAOMap();
             SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
                                        aiTextureType_LIGHTMAP, img);
             ++current_idx;
         }
-        if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) {
+        if (HasValidTexture(w_mesh.GetMaterial(), "ao_rough_metal")) {
             auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap();
             SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
                                        aiTextureType_UNKNOWN, img);
             ++current_idx;
-        } else if (w_mesh.GetMaterial().HasRoughnessMap() &&
-                   w_mesh.GetMaterial().HasMetallicMap()) {
+        } else if (HasValidTexture(w_mesh.GetMaterial(), "roughness") &&
+                   HasValidTexture(w_mesh.GetMaterial(), "metallic")) {
             auto rough = w_mesh.GetMaterial().GetRoughnessMap().AsTensor();
             auto metal = w_mesh.GetMaterial().GetMetallicMap().AsTensor();
             if (rough.GetShape() != metal.GetShape()) {
@@ -617,21 +638,21 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename,
                                        aiTextureType_UNKNOWN, rough_metal_img);
             ++current_idx;
         } else {
-            if (w_mesh.GetMaterial().HasRoughnessMap()) {
-                auto img = w_mesh.GetMaterial().GetRoughnessMap();
+            if (HasValidTexture(material, "roughness")) {
+                auto img = material.GetRoughnessMap();
                 SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
                                            aiTextureType_UNKNOWN, img);
                 ++current_idx;
             }
-            if (w_mesh.GetMaterial().HasMetallicMap()) {
-                auto img = w_mesh.GetMaterial().GetMetallicMap();
+            if (HasValidTexture(material, "metallic")) {
+                auto img = material.GetMetallicMap();
                 SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
                                            aiTextureType_UNKNOWN, img);
                 ++current_idx;
             }
         }
-        if (w_mesh.GetMaterial().HasNormalMap()) {
-            auto img = w_mesh.GetMaterial().GetNormalMap();
+        if (HasValidTexture(material, "normal")) {
+            auto img = material.GetNormalMap();
             SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
                                        aiTextureType_NORMALS, img);
             ++current_idx;

cpp/open3d/visualization/gui/BitmapWindowSystem.cpp

@@ -11,6 +11,7 @@
 #include <mutex>
 #include <queue>
 #include <thread>
+#include <unordered_set>
 
 #include "open3d/geometry/Image.h"
 #include "open3d/utility/Logging.h"
@@ -44,10 +45,21 @@ struct BitmapEvent {
     virtual void Execute() = 0;
 };
 
+// Forward declaration: BitmapDrawEvent needs a pointer to BitmapEventQueue
+// to clear the per-window pending-draw flag before calling OnDraw().
+struct BitmapEventQueue;
+
 struct BitmapDrawEvent : public BitmapEvent {
-    BitmapDrawEvent(BitmapWindow *target) : BitmapEvent(target) {}
+    // queue_ is used to clear the pending-draw flag before OnDraw() so that
+    // a PostRedraw() called *inside* OnDraw() is not suppressed.
+    BitmapEventQueue *queue_;
+
+    BitmapDrawEvent(BitmapWindow *target, BitmapEventQueue *queue)
+        : BitmapEvent(target), queue_(queue) {}
 
-    void Execute() override { event_target->o3d_window->OnDraw(); }
+    // Execute() is defined after BitmapEventQueue (below) because it calls
+    // clear_pending_draw(), which requires the full BitmapEventQueue type.
+    void Execute() override;
 };
 
 struct BitmapResizeEvent : public BitmapEvent {
@@ -93,15 +105,23 @@ struct BitmapTextInputEvent : public BitmapEvent {
     }
 };
 
-/// Thread safe event queue (multiple producers and consumers). pop_front() and
-/// push() are protected by a mutex. push() may fail if the mutex cannot be
-/// acquired immediately. empty() is not protected and is not reliable.
+/// Thread safe event queue (multiple producers and consumers).
+/// pop_front() and push() are protected by a mutex.
+/// push() may fail if the mutex cannot be acquired immediately.
+/// empty() is not protected and is not reliable.
+///
+/// Extended to support:
+/// - Draw coalescing: at most one pending draw per window (push_draw).
+/// - Input coalescing: MOVE/DRAG replace latest, WHEEL accumulates
+///   dx/dy (replace_or_merge_mouse). Old mouse positions are stale;
+///   processing them forces stale frames to be encoded and sent.
 struct BitmapEventQueue : public std::queue<std::shared_ptr<BitmapEvent>> {
     using value_t = std::shared_ptr<BitmapEvent>;
     using super = std::queue<value_t>;
 
     using super::empty;  // not reliable
     using super::super;
+
     // pop + front needs to be atomic for thread safety. This is exception safe
     // since shared_ptr copy ctor is noexcept, when it is returned by value.
     value_t pop_front() {
@@ -110,17 +130,95 @@ struct BitmapEventQueue : public std::queue<std::shared_ptr<BitmapEvent>> {
         super::pop();
         return evt;
     }
+
     void push(const value_t &event) {
         if (evt_q_mutex_.try_lock()) {
             super::push(event);
             evt_q_mutex_.unlock();
         }
     }
 
+    // Push a draw event only if no draw is already pending for this window.
+    // Returns true if pushed. The caller must pass the shared_ptr to the draw
+    // event; this function inserts the window into pending_draw_windows_ so
+    // that BitmapDrawEvent::Execute() can clear it on completion.
+    bool push_draw(BitmapWindow *window, const value_t &event) {
+        if (evt_q_mutex_.try_lock()) {
+            bool pushed = false;
+            if (pending_draw_windows_.find(window) ==
+                pending_draw_windows_.end()) {
+                pending_draw_windows_.insert(window);
+                super::push(event);
+                pushed = true;
+            }
+            evt_q_mutex_.unlock();
+            return pushed;
+        }
+        return false;
+    }
+
+    // Called by BitmapDrawEvent::Execute() just before OnDraw() so that a
+    // redraw posted *during* drawing is not suppressed.
+    void clear_pending_draw(BitmapWindow *window) {
+        std::lock_guard<std::mutex> lock(evt_q_mutex_);
+        pending_draw_windows_.erase(window);
+    }
+
+    // Remove all pending state for a window that is being destroyed.
+    void remove_window(BitmapWindow *window) {
+        std::lock_guard<std::mutex> lock(evt_q_mutex_);
+        pending_draw_windows_.erase(window);
+    }
+
+    // For MOVE/DRAG: replace the last queued event of the same (target, type)
+    // with the new event (latest absolute position wins; camera controllers
+    // derive delta from absolute coords so intermediate positions are useless).
+    // For WHEEL: accumulate wheel.dx/dy into the last queued event of the same
+    // (target, type) so that the total scroll amount is preserved even when
+    // multiple notches fire faster than the render loop.
+    // Falls back to a normal push when no matching event is at the back.
+    void replace_or_merge_mouse(const value_t &event) {
+        std::lock_guard<std::mutex> lock(evt_q_mutex_);
+        auto *new_evt = static_cast<BitmapMouseEvent *>(event.get());
+        if (!super::c.empty()) {
+            auto *back_mouse =
+                    dynamic_cast<BitmapMouseEvent *>(super::c.back().get());
+            if (back_mouse &&
+                back_mouse->event_target == new_evt->event_target &&
+                back_mouse->event.type == new_evt->event.type) {
+                if (new_evt->event.type == MouseEvent::WHEEL) {
+                    // Accumulate scroll deltas; update cursor position and
+                    // other fields to the latest event values.
+                    back_mouse->event.wheel.dx += new_evt->event.wheel.dx;
+                    back_mouse->event.wheel.dy += new_evt->event.wheel.dy;
+                    back_mouse->event.x = new_evt->event.x;
+                    back_mouse->event.y = new_evt->event.y;
+                    back_mouse->event.modifiers = new_evt->event.modifiers;
+                    back_mouse->event.wheel.isTrackpad =
+                            new_evt->event.wheel.isTrackpad;
+                } else {
+                    // Replace: only the latest absolute position matters.
+                    back_mouse->event = new_evt->event;
+                }
+                return;
+            }
+        }
+        super::push(event);
+    }
+
 private:
     std::mutex evt_q_mutex_;
+    // Windows with a draw event currently in the queue (not yet executed).
+    std::unordered_set<BitmapWindow *> pending_draw_windows_;
 };
 
+// Out-of-class definition: BitmapEventQueue is now fully defined so
+// clear_pending_draw() can be called.
+void BitmapDrawEvent::Execute() {
+    queue_->clear_pending_draw(event_target);
+    event_target->o3d_window->OnDraw();
+}
+
 }  // namespace
 
 struct BitmapWindowSystem::Impl {
@@ -187,18 +285,32 @@ void BitmapWindowSystem::DestroyWindow(OSWindow w) {
     while (!filtered_reversed.empty()) {
         impl_->event_queue_.push(filtered_reversed.pop_front());
     }
+    // Clear any pending-draw entry for this window so the coalescing set
+    // does not hold a dangling pointer.
+    impl_->event_queue_.remove_window(the_deceased);
     // Requiem aeternam dona ei. Requiscat in pace.
     delete (BitmapWindow *)w;
 }
 
 void BitmapWindowSystem::PostRedrawEvent(OSWindow w) {
     auto hw = (BitmapWindow *)w;
-    impl_->event_queue_.push(std::make_shared<BitmapDrawEvent>(hw));
+    // push_draw is a no-op when a draw event is already queued for this
+    // window, preventing redundant renders from piling up.
+    impl_->event_queue_.push_draw(
+            hw, std::make_shared<BitmapDrawEvent>(hw, &impl_->event_queue_));
 }
 
 void BitmapWindowSystem::PostMouseEvent(OSWindow w, const MouseEvent &e) {
     auto hw = (BitmapWindow *)w;
-    impl_->event_queue_.push(std::make_shared<BitmapMouseEvent>(hw, e));
+    if (e.type == MouseEvent::MOVE || e.type == MouseEvent::DRAG ||
+        e.type == MouseEvent::WHEEL) {
+        // Coalesce: MOVE/DRAG replace latest (absolute position); WHEEL
+        // accumulates dx/dy. Only the most recent state matters for rendering.
+        impl_->event_queue_.replace_or_merge_mouse(
+                std::make_shared<BitmapMouseEvent>(hw, e));
+    } else {
+        impl_->event_queue_.push(std::make_shared<BitmapMouseEvent>(hw, e));
+    }
 }
 
 void BitmapWindowSystem::PostKeyEvent(OSWindow w, const KeyEvent &e) {