From e913b9e7c09a378e43c73ec5c4fc1dfbd61a79a8 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 20 May 2026 00:31:46 -0700 Subject: [PATCH 01/11] [WebRTC] Reduce remote rendering lag: input/draw coalescing, async encode, VP9 preference - Remove duplicate redraw on input (WebRTCWindowSystem.cpp) - Draw coalescing (BitmapWindowSystem.cpp) - Input (mouse event) coalescing (BitmapWindowSystem.cpp) - JS requestAnimationFrame coalescing / throttling (webrtcstreamer.js) - Async encoder thread (PeerConnectionManager.cpp/.h) - Data channel low-latency mode (webrtcstreamer.js) - Reduced startup delay (500ms -> 250ms) (WebRTCWindowSystem.cpp) - VP9 codec preference over VP8 (webrtcstreamer.js) --- .../visualization/gui/BitmapWindowSystem.cpp | 126 +++++++++++++++- .../webrtc_server/PeerConnectionManager.cpp | 59 ++++++-- .../webrtc_server/PeerConnectionManager.h | 14 ++ .../webrtc_server/WebRTCWindowSystem.cpp | 8 +- .../webrtc_server/html/webrtcstreamer.js | 136 +++++++++++++++--- 5 files changed, 302 insertions(+), 41 deletions(-) diff --git a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp index 3db9d0cefef..b73d680b381 100644 --- a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp +++ b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "open3d/geometry/Image.h" #include "open3d/utility/Logging.h" @@ -44,10 +45,21 @@ struct BitmapEvent { virtual void Execute() = 0; }; +// Forward declaration: BitmapDrawEvent needs a pointer to BitmapEventQueue +// to clear the per-window pending-draw flag before calling OnDraw(). +struct BitmapEventQueue; + struct BitmapDrawEvent : public BitmapEvent { - BitmapDrawEvent(BitmapWindow *target) : BitmapEvent(target) {} + // queue_ is used to clear the pending-draw flag before OnDraw() so that + // a PostRedraw() called *inside* OnDraw() is not suppressed. + BitmapEventQueue *queue_; + + BitmapDrawEvent(BitmapWindow *target, BitmapEventQueue *queue) + : BitmapEvent(target), queue_(queue) {} - void Execute() override { event_target->o3d_window->OnDraw(); } + // Execute() is defined after BitmapEventQueue (below) because it calls + // clear_pending_draw(), which requires the full BitmapEventQueue type. + void Execute() override; }; struct BitmapResizeEvent : public BitmapEvent { @@ -93,15 +105,23 @@ struct BitmapTextInputEvent : public BitmapEvent { } }; -/// Thread safe event queue (multiple producers and consumers). pop_front() and -/// push() are protected by a mutex. push() may fail if the mutex cannot be -/// acquired immediately. empty() is not protected and is not reliable. +/// Thread safe event queue (multiple producers and consumers). +/// pop_front() and push() are protected by a mutex. +/// push() may fail if the mutex cannot be acquired immediately. +/// empty() is not protected and is not reliable. +/// +/// Extended to support: +/// - Draw coalescing: at most one pending draw per window (push_draw). +/// - Input coalescing: MOVE/DRAG replace latest, WHEEL accumulates +/// dx/dy (replace_or_merge_mouse). Old mouse positions are stale; +/// processing them forces stale frames to be encoded and sent. struct BitmapEventQueue : public std::queue> { using value_t = std::shared_ptr; using super = std::queue; using super::empty; // not reliable using super::super; + // pop + front needs to be atomic for thread safety. This is exception safe // since shared_ptr copy ctor is noexcept, when it is returned by value. value_t pop_front() { @@ -110,6 +130,7 @@ struct BitmapEventQueue : public std::queue> { super::pop(); return evt; } + void push(const value_t &event) { if (evt_q_mutex_.try_lock()) { super::push(event); @@ -117,10 +138,87 @@ struct BitmapEventQueue : public std::queue> { } } + // Push a draw event only if no draw is already pending for this window. + // Returns true if pushed. The caller must pass the shared_ptr to the draw + // event; this function inserts the window into pending_draw_windows_ so + // that BitmapDrawEvent::Execute() can clear it on completion. + bool push_draw(BitmapWindow *window, const value_t &event) { + if (evt_q_mutex_.try_lock()) { + bool pushed = false; + if (pending_draw_windows_.find(window) == + pending_draw_windows_.end()) { + pending_draw_windows_.insert(window); + super::push(event); + pushed = true; + } + evt_q_mutex_.unlock(); + return pushed; + } + return false; + } + + // Called by BitmapDrawEvent::Execute() just before OnDraw() so that a + // redraw posted *during* drawing is not suppressed. + void clear_pending_draw(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // Remove all pending state for a window that is being destroyed. + void remove_window(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // For MOVE/DRAG: replace the last queued event of the same (target, type) + // with the new event (latest absolute position wins; camera controllers + // derive delta from absolute coords so intermediate positions are useless). + // For WHEEL: accumulate wheel.dx/dy into the last queued event of the same + // (target, type) so that the total scroll amount is preserved even when + // multiple notches fire faster than the render loop. + // Falls back to a normal push when no matching event is at the back. + void replace_or_merge_mouse(const value_t &event) { + std::lock_guard lock(evt_q_mutex_); + auto *new_evt = static_cast(event.get()); + if (!super::c.empty()) { + auto *back_mouse = + dynamic_cast(super::c.back().get()); + if (back_mouse && + back_mouse->event_target == new_evt->event_target && + back_mouse->event.type == new_evt->event.type) { + if (new_evt->event.type == MouseEvent::WHEEL) { + // Accumulate scroll deltas; update cursor position and + // other fields to the latest event values. + back_mouse->event.wheel.dx += new_evt->event.wheel.dx; + back_mouse->event.wheel.dy += new_evt->event.wheel.dy; + back_mouse->event.x = new_evt->event.x; + back_mouse->event.y = new_evt->event.y; + back_mouse->event.modifiers = new_evt->event.modifiers; + back_mouse->event.wheel.isTrackpad = + new_evt->event.wheel.isTrackpad; + } else { + // Replace: only the latest absolute position matters. + back_mouse->event = new_evt->event; + } + return; + } + } + super::push(event); + } + private: std::mutex evt_q_mutex_; + // Windows with a draw event currently in the queue (not yet executed). + std::unordered_set pending_draw_windows_; }; +// Out-of-class definition: BitmapEventQueue is now fully defined so +// clear_pending_draw() can be called. +void BitmapDrawEvent::Execute() { + queue_->clear_pending_draw(event_target); + event_target->o3d_window->OnDraw(); +} + } // namespace struct BitmapWindowSystem::Impl { @@ -187,18 +285,32 @@ void BitmapWindowSystem::DestroyWindow(OSWindow w) { while (!filtered_reversed.empty()) { impl_->event_queue_.push(filtered_reversed.pop_front()); } + // Clear any pending-draw entry for this window so the coalescing set + // does not hold a dangling pointer. + impl_->event_queue_.remove_window(the_deceased); // Requiem aeternam dona ei. Requiscat in pace. delete (BitmapWindow *)w; } void BitmapWindowSystem::PostRedrawEvent(OSWindow w) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw)); + // push_draw is a no-op when a draw event is already queued for this + // window, preventing redundant renders from piling up. + impl_->event_queue_.push_draw( + hw, std::make_shared(hw, &impl_->event_queue_)); } void BitmapWindowSystem::PostMouseEvent(OSWindow w, const MouseEvent &e) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw, e)); + if (e.type == MouseEvent::MOVE || e.type == MouseEvent::DRAG || + e.type == MouseEvent::WHEEL) { + // Coalesce: MOVE/DRAG replace latest (absolute position); WHEEL + // accumulates dx/dy. Only the most recent state matters for rendering. + impl_->event_queue_.replace_or_merge_mouse( + std::make_shared(hw, e)); + } else { + impl_->event_queue_.push(std::make_shared(hw, e)); + } } void BitmapWindowSystem::PostKeyEvent(OSWindow w, const KeyEvent &e) { diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index c29696c39e2..a3f3f5cbc32 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -195,9 +195,21 @@ PeerConnectionManager::PeerConnectionManager( } return this->HangUp(peerid); }; + + // Start async encoder thread. + encoder_running_ = true; + encoder_thread_ = std::thread(&PeerConnectionManager::EncoderThreadLoop, + this); } -PeerConnectionManager::~PeerConnectionManager() {} +PeerConnectionManager::~PeerConnectionManager() { + // Stop async encoder thread before WebRTC resources are torn down. + encoder_running_ = false; + pending_frames_cv_.notify_all(); + if (encoder_thread_.joinable()) { + encoder_thread_.join(); + } +} // Return deviceList as JSON vector. const Json::Value PeerConnectionManager::GetMediaList() { @@ -729,19 +741,44 @@ void PeerConnectionManager::CloseWindowConnections( } } +// Encoder thread: wakes on each new frame, drains the per-window latest-frame +// map, and calls video_track_source->OnFrame() (libyuv + WebRTC encode) +// off the render thread so frame delivery never blocks GUI redraws. +void PeerConnectionManager::EncoderThreadLoop() { + while (encoder_running_) { + std::unordered_map> snapshot; + { + std::unique_lock lock(pending_frames_mutex_); + pending_frames_cv_.wait(lock, [this] { + return !pending_frames_.empty() || !encoder_running_; + }); + if (!encoder_running_) break; + // Drain: take all pending frames in one batch; late-arriving frames + // for the same window have already overwritten earlier ones, so we + // encode only the latest per window (implicit frame coalescing). + snapshot = std::move(pending_frames_); + } + for (const auto &kv : snapshot) { + auto video_track_source = GetVideoTrackSource(kv.first); + if (video_track_source && kv.second) { + video_track_source->OnFrame(kv.second); + } + } + } +} + void PeerConnectionManager::OnFrame(const std::string &window_uid, const std::shared_ptr &im) { - // Get the WebRTC stream that corresponds to the window_uid. - // video_track_source is nullptr if the server is running but no client is - // connected. - rtc::scoped_refptr video_track_source = - GetVideoTrackSource(window_uid); - if (video_track_source) { - // TODO: this OnFrame(im); is a blocking call. Do we need to handle - // OnFrame in a separate thread? e.g. attach to a queue of frames, even - // if the queue size is just 1. - video_track_source->OnFrame(im); + // Skip if no peer is connected for this window. + if (!GetVideoTrackSource(window_uid)) return; + // Post the latest frame; overwrites any pending unencoded frame for this + // window (frame coalescing) so the encoder thread always sees the freshest + // content without blocking the render thread. + { + std::lock_guard lock(pending_frames_mutex_); + pending_frames_[window_uid] = im; } + pending_frames_cv_.notify_one(); } } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h index 0120e20b4fe..72a005e96c4 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h @@ -21,6 +21,8 @@ #include #include +#include +#include #include #include #include @@ -451,6 +453,18 @@ class PeerConnectionManager { const std::regex publish_filter_; std::map func_; std::string webrtc_port_range_; + + // Async encoder thread: decouples the render thread from the blocking + // libyuv + WebRTC encode path. OnFrame() posts the latest frame per window; + // the thread drains the map and calls video_track_source->OnFrame(). + std::unordered_map> + pending_frames_; + std::mutex pending_frames_mutex_; + std::condition_variable pending_frames_cv_; + std::atomic encoder_running_{false}; + std::thread encoder_thread_; + + void EncoderThreadLoop(); }; } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index 667a7c88be4..ab238201e7d 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -364,8 +364,10 @@ std::string WebRTCWindowSystem::OnDataChannelMessage( if (impl_->data_channel_message_callbacks_.count(class_name) != 0) { reply = impl_->data_channel_message_callbacks_.at(class_name)( message); - const auto os_window = GetOSWindowByUID(window_uid); - if (os_window) PostRedrawEvent(os_window); + // Do not call PostRedrawEvent here. Mouse/keyboard callbacks + // already schedule a redraw via Window::OnMouseEvent → PostRedraw. + // An extra PostRedrawEvent here creates a duplicate draw event + // before the input event is even processed, causing backlog. return reply; } else { reply = fmt::format( @@ -405,7 +407,7 @@ void WebRTCWindowSystem::OnFrame(const std::string &window_uid, void WebRTCWindowSystem::SendInitFrames(const std::string &window_uid) { utility::LogInfo("Sending init frames to {}.", window_uid); static const int s_max_initial_frames = 5; - static const int s_sleep_between_frames_ms = 100; + static const int s_sleep_between_frames_ms = 50; const auto os_window = GetOSWindowByUID(window_uid); if (!os_window) return; for (int i = 0; os_window != nullptr && i < s_max_initial_frames; ++i) { diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index d7bcd8f6c13..ec4e7e703b0 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -62,6 +62,13 @@ let WebRtcStreamer = (function() { // Open3D-specific functions. this.onClose = onClose; this.commsFetch = commsFetch; + + // Pending coalesced pointer/wheel events. A single requestAnimationFrame + // flushes both at most once per browser frame (~60 Hz), preventing a + // backlog of stale events from queuing up on the server. + this.pendingPointerEvent = null; // MOVE or DRAG (latest wins) + this.pendingWheelEvent = null; // WHEEL (dx/dy accumulated) + this.rafPending = false; } const logAndReturn = function(value) { @@ -182,6 +189,26 @@ let WebRtcStreamer = (function() { } }; + // Schedule a requestAnimationFrame flush of pending coalesced events. + // Only one rAF is scheduled at a time; calling again while one is already + // pending is a no-op. When the frame fires, the latest pointer event and + // the accumulated wheel event (if any) are sent and cleared. + WebRtcStreamer.prototype._scheduleRafFlush = function() { + if (this.rafPending) return; + this.rafPending = true; + requestAnimationFrame(() => { + this.rafPending = false; + if (this.pendingPointerEvent !== null) { + this.sendJsonData(this.pendingPointerEvent); + this.pendingPointerEvent = null; + } + if (this.pendingWheelEvent !== null) { + this.sendJsonData(this.pendingWheelEvent); + this.pendingWheelEvent = null; + } + }); + }; + WebRtcStreamer.prototype.addEventListeners = function(windowUID) { if (this.videoElt) { var parentDivElt = this.videoElt.parentElement; @@ -333,7 +360,9 @@ let WebRtcStreamer = (function() { // - Open3D: L=1, M=2, R=4 // - JavaScript: L=1, R=2, M=4 event.preventDefault(); - var open3dMouseEvent = { + // Throttle to one event per animation frame. Only the latest + // absolute position matters; intermediate positions are stale. + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: event.buttons === 0 ? 'MOVE' : 'DRAG', @@ -344,7 +373,7 @@ let WebRtcStreamer = (function() { buttons: event.buttons, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('touchmove', (event) => { // TODO: Known differences. Currently only left-key drag works. @@ -352,7 +381,8 @@ let WebRtcStreamer = (function() { // - JavaScript: L=1, R=2, M=4 event.preventDefault(); var rect = event.target.getBoundingClientRect(); - var open3dMouseEvent = { + // Throttle to one event per animation frame (latest wins). + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: 'DRAG', @@ -363,7 +393,7 @@ let WebRtcStreamer = (function() { buttons: 1, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('mouseleave', (event) => { var open3dMouseEvent = { @@ -397,20 +427,36 @@ let WebRtcStreamer = (function() { dx = dx === 0 ? dx : (-dx / Math.abs(dx)) * 1; dy = dy === 0 ? dy : (-dy / Math.abs(dy)) * 1; - var open3dMouseEvent = { - window_uid: windowUID, - class_name: 'MouseEvent', - type: 'WHEEL', - x: event.offsetX, - y: event.offsetY, - modifiers: WebRtcStreamer._getModifiers(event), - wheel: { - dx: dx, - dy: dy, - isTrackpad: isTrackpad ? 1 : 0, - }, - }; - this.sendJsonData(open3dMouseEvent); + if (this.pendingWheelEvent === null) { + // First wheel event in this frame: create a new pending + // event and schedule a flush. + this.pendingWheelEvent = { + window_uid: windowUID, + class_name: 'MouseEvent', + type: 'WHEEL', + x: event.offsetX, + y: event.offsetY, + modifiers: WebRtcStreamer._getModifiers(event), + wheel: { + dx: dx, + dy: dy, + isTrackpad: isTrackpad ? 1 : 0, + }, + }; + this._scheduleRafFlush(); + } else { + // Subsequent wheel events in the same frame: accumulate + // dx/dy so the total scroll amount is preserved, and + // update cursor position to the latest coordinates. + this.pendingWheelEvent.wheel.dx += dx; + this.pendingWheelEvent.wheel.dy += dy; + this.pendingWheelEvent.x = event.offsetX; + this.pendingWheelEvent.y = event.offsetY; + this.pendingWheelEvent.modifiers = + WebRtcStreamer._getModifiers(event); + this.pendingWheelEvent.wheel.isTrackpad = + isTrackpad ? 1 : 0; + } }, {passive: false}); } }; @@ -464,6 +510,48 @@ let WebRtcStreamer = (function() { this.pc.addStream(stream); } + // Prefer VP9 via the standard setCodecPreferences() API. + // Must be called on the transceiver BEFORE createOffer(). + // For receive-only mode (no local stream), we create the video + // transceiver explicitly — offerToReceiveVideo creates it lazily + // inside createOffer(), which is too late to set preferences. + // Falls back silently to VP8 if the browser does not support the + // API or if VP9 is not in the browser's capabilities. + var pc = this.pc; + if (typeof RTCRtpReceiver !== 'undefined' && + RTCRtpReceiver.getCapabilities && pc.addTransceiver) { + var videoCaps = RTCRtpReceiver.getCapabilities('video'); + if (videoCaps) { + var vp9Codecs = videoCaps.codecs.filter(function(c) { + return c.mimeType === 'video/VP9'; + }); + if (vp9Codecs.length) { + var preferredCodecs = vp9Codecs.concat( + videoCaps.codecs.filter(function(c) { + return c.mimeType !== 'video/VP9'; + })); + if (stream) { + // Local video being sent: find the transceiver + // addStream() created and apply preferences. + pc.getTransceivers().forEach(function(t) { + if (t.sender && t.sender.track && + t.sender.track.kind === 'video' && + t.setCodecPreferences) { + t.setCodecPreferences(preferredCodecs); + } + }); + } else { + // Receive-only: create transceiver explicitly. + var vt = pc.addTransceiver( + 'video', {direction: 'recvonly'}); + if (vt.setCodecPreferences) { + vt.setCodecPreferences(preferredCodecs); + } + } + } + } + } + // clear early candidates this.earlyCandidates.length = 0; @@ -599,9 +687,17 @@ let WebRtcStreamer = (function() { } }; - // Local datachannel sends data + // Local datachannel sends data. + // Use unordered, unreliable delivery for mouse/input events. Ordered + // reliable delivery causes head-of-line blocking: a lost packet stalls + // all subsequent events until it is retransmitted. For interactive + // mouse events the latest position is all that matters, so a dropped + // message is better than a delayed one. maxRetransmits: 0 means the + // browser sends once and moves on; ordered: false removes sequencing. try { - this.dataChannel = pc.createDataChannel('ClientDataChannel'); + this.dataChannel = pc.createDataChannel( + 'ClientDataChannel', + {ordered: false, maxRetransmits: 0}); var dataChannel = this.dataChannel; dataChannel.onopen = function() { console.log('local datachannel open'); From a14390e3f831448f9008d55c1ff415b3dc4436c1 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 20 May 2026 13:39:00 -0700 Subject: [PATCH 02/11] Explicitly warn and skip writing float textures (unsupported) in gltf. --- cpp/open3d/t/io/file_format/FileASSIMP.cpp | 75 ++++++++++++------- .../webrtc_server/PeerConnectionManager.cpp | 8 +- 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/cpp/open3d/t/io/file_format/FileASSIMP.cpp b/cpp/open3d/t/io/file_format/FileASSIMP.cpp index 1ce61332bd9..aff2fa3e6b6 100644 --- a/cpp/open3d/t/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/t/io/file_format/FileASSIMP.cpp @@ -16,6 +16,7 @@ #include #include +#include "open3d/core/Dtype.h" #include "open3d/core/ParallelFor.h" #include "open3d/core/TensorFunction.h" #include "open3d/t/io/ImageIO.h" @@ -177,11 +178,31 @@ bool ReadTriangleMeshUsingASSIMP( return true; } -static void SetTextureMaterialProperty(aiMaterial* mat, - aiScene* scene, - int texture_idx, - aiTextureType tt, - t::geometry::Image& img) { +namespace { + +// Checks whether a texture map is present, not empty and uint8 or uint16 +// datatype. +bool HasValidTexture(const visualization::rendering::Material& material, + const std::string& key) { + if (!material.HasTextureMap(key) || material.GetTextureMap(key).IsEmpty()) { + return false; + } + const auto& dt = material.GetTextureMap(key).GetDtype(); + if (dt != core::UInt8 && dt != core::UInt16) { + utility::LogWarning( + "Skipping texture map '{}' with unsupported data type '{}'. " + "Only uint8 and uint16 are supported.", + key, dt.ToString()); + return false; + } + return true; +} + +void SetTextureMaterialProperty(aiMaterial* mat, + aiScene* scene, + int texture_idx, + aiTextureType tt, + t::geometry::Image& img) { // Encode image as PNG std::vector img_buffer; WriteImageToPNGInMemory(img_buffer, img, 6); @@ -208,7 +229,6 @@ static void SetTextureMaterialProperty(aiMaterial* mat, mat->AddProperty(&mode, 1, AI_MATKEY_MAPPINGMODE_V(tt, 0)); } -namespace { // Add hash function for tuple key struct TupleHash { size_t operator()(const std::tuple& t) const { @@ -551,17 +571,18 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, // model has one we just export it, otherwise if both roughness and // metal maps are available we combine them, otherwise if only one or // the other is available we just export the one map. + const auto& material = w_mesh.GetMaterial(); int n_textures = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) ++n_textures; - if (w_mesh.GetMaterial().HasNormalMap()) ++n_textures; - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { + if (HasValidTexture(material, "albedo")) ++n_textures; + if (HasValidTexture(material, "normal")) ++n_textures; + if (HasValidTexture(material, "ao_rough_metal")) { ++n_textures; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { + } else if (HasValidTexture(material, "roughness") && + HasValidTexture(material, "metallic")) { ++n_textures; } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) ++n_textures; - if (w_mesh.GetMaterial().HasMetallicMap()) ++n_textures; + if (HasValidTexture(material, "roughness")) ++n_textures; + if (HasValidTexture(material, "metallic")) ++n_textures; } if (n_textures > 0) { ai_scene->mTextures = new aiTexture*[n_textures]; @@ -573,23 +594,23 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, // Now embed the textures that are available... int current_idx = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) { - auto img = w_mesh.GetMaterial().GetAlbedoMap(); + if (HasValidTexture(material, "albedo")) { + auto img = material.GetAlbedoMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_DIFFUSE, img); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_BASE_COLOR, img); ++current_idx; } - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { - auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap(); + if (HasValidTexture(material, "ao_rough_metal")) { + auto img = material.GetAORoughnessMetalMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_UNKNOWN, img); ++current_idx; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { - auto rough = w_mesh.GetMaterial().GetRoughnessMap(); - auto metal = w_mesh.GetMaterial().GetMetallicMap(); + } else if (HasValidTexture(material, "roughness") && + HasValidTexture(material, "metallic")) { + auto rough = material.GetRoughnessMap(); + auto metal = material.GetMetallicMap(); auto rows = rough.GetRows(); auto cols = rough.GetCols(); auto rough_metal = @@ -620,21 +641,21 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, aiTextureType_UNKNOWN, rough_metal); ++current_idx; } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) { - auto img = w_mesh.GetMaterial().GetRoughnessMap(); + if (HasValidTexture(material, "roughness")) { + auto img = material.GetRoughnessMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_UNKNOWN, img); ++current_idx; } - if (w_mesh.GetMaterial().HasMetallicMap()) { - auto img = w_mesh.GetMaterial().GetMetallicMap(); + if (HasValidTexture(material, "metallic")) { + auto img = material.GetMetallicMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_UNKNOWN, img); ++current_idx; } } - if (w_mesh.GetMaterial().HasNormalMap()) { - auto img = w_mesh.GetMaterial().GetNormalMap(); + if (HasValidTexture(material, "normal")) { + auto img = material.GetNormalMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_NORMALS, img); ++current_idx; diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index a3f3f5cbc32..9d5865d71ff 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -198,17 +198,15 @@ PeerConnectionManager::PeerConnectionManager( // Start async encoder thread. encoder_running_ = true; - encoder_thread_ = std::thread(&PeerConnectionManager::EncoderThreadLoop, - this); + encoder_thread_ = + std::thread(&PeerConnectionManager::EncoderThreadLoop, this); } PeerConnectionManager::~PeerConnectionManager() { // Stop async encoder thread before WebRTC resources are torn down. encoder_running_ = false; pending_frames_cv_.notify_all(); - if (encoder_thread_.joinable()) { - encoder_thread_.join(); - } + encoder_thread_.join(); } // Return deviceList as JSON vector. From 9469dce69204231b49377c5c314bf66dac4d7f66 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 21 May 2026 13:49:11 -0700 Subject: [PATCH 03/11] Pin license-webpack-plugin - windows wheel build error. --- python/js/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/js/package.json b/python/js/package.json index d364052ec22..5556ea5df4a 100644 --- a/python/js/package.json +++ b/python/js/package.json @@ -37,7 +37,8 @@ }, "resolutions": { "glob": "7.2.3", - "abab": "^2.0.6" + "abab": "^2.0.6", + "license-webpack-plugin": "^4.0.0" }, "dependencies": { "@jupyter-widgets/base": "^2 || ^3 || ^4 || ^5 || ^6", From 99b96d2bfaf0d4508f9efe4b9c2b507556f1df96 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 21 May 2026 16:11:14 -0700 Subject: [PATCH 04/11] [WebRTC] Optimize latency by adjusting pacing and jitter buffer settings sends zero playout delay in every RTP packet's header extension when adaptation is necessary, drops resolution rather than framerate prevents WebRTC's autonomous resolution scaling; Replaced playoutDelayHint with jitterBufferTarget = 0 --- .../webrtc_server/PeerConnectionManager.cpp | 23 +++++++++++++++++++ .../webrtc_server/html/webrtcstreamer.js | 21 +++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index 9d5865d71ff..68ff8256e29 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,18 @@ static IceServer GetIceServerFromUrl(const std::string &url) { static webrtc::PeerConnectionFactoryDependencies CreatePeerConnectionFactoryDependencies() { + // Disable WebRTC's pacing delay and allow the pacer to drain its queue + // immediately. This reduces the added latency from packet smoothing, which + // is unnecessary for a local interactive streaming use case. + // Force playout delay to zero in the RTP header extension so the receiver + // renders frames immediately rather than buffering for jitter smoothing. + // Disable automatic resolution reduction so quality is only degraded if the + // encoder is genuinely overloaded (content hint kFluid still allows it). + webrtc::field_trial::InitFieldTrialsFromString( + "WebRTC-Pacer-DrainQueue/Enabled/" + "WebRTC-ForceSendPlayoutDelay/min_ms:0,max_ms:0/" + "WebRTC-Video-DisableAutomaticResize/Enabled/"); + webrtc::PeerConnectionFactoryDependencies dependencies; dependencies.network_thread = nullptr; dependencies.worker_thread = rtc::Thread::Current(); @@ -531,6 +544,10 @@ bool PeerConnectionManager::InitializePeerConnection() { PeerConnectionManager::PeerConnectionObserver * PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { webrtc::PeerConnectionInterface::RTCConfiguration config; + // Max bundle multiplexes all media and data channels on a single transport, + // eliminating separate ICE/DTLS handshakes per track and reducing latency. + config.bundle_policy = + webrtc::PeerConnectionInterface::kBundlePolicyMaxBundle; for (auto ice_server : ice_server_list_) { webrtc::PeerConnectionInterface::IceServer server; IceServer srv = GetIceServerFromUrl(ice_server); @@ -664,6 +681,12 @@ bool PeerConnectionManager::AddStreams( opts); video_track = peer_connection_factory_->CreateVideoTrack( window_uid + "_video", videoScaled); + // Prefer framerate over resolution when the encoder + // is under pressure (bandwidth or CPU constrained). + // For interactive 3D rendering, motion smoothness + // matters more than pixel-perfect resolution. + video_track->set_content_hint( + webrtc::VideoTrackInterface::ContentHint::kFluid); } if ((video_track) && (!stream->AddTrack(video_track))) { diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index ec4e7e703b0..45dea6bf245 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -677,11 +677,22 @@ let WebRtcStreamer = (function() { const recvs = pc.getReceivers(); recvs.forEach((recv) => { - if (recv.track && recv.track.kind === 'video' && - typeof recv.getParameters != 'undefined') { - console.log( - 'codecs:' + - JSON.stringify(recv.getParameters().codecs)); + if (recv.track && recv.track.kind === 'video') { + // Minimize browser jitter buffer to reduce playout + // latency. The server already sends RTP playout-delay + // header extensions with min=max=0 via the + // WebRTC-ForceSendPlayoutDelay field trial, but set + // jitterBufferTarget as well for browsers that honour + // the JS API over the in-band RTP extension. + if (typeof recv.jitterBufferTarget !== 'undefined') { + recv.jitterBufferTarget = 0; + } + if (typeof recv.getParameters != 'undefined') { + console.log( + 'codecs:' + + JSON.stringify( + recv.getParameters().codecs)); + } } }); } From 1a46e95ad7b273a8a6933230366bc01edebf99b8 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> Date: Thu, 21 May 2026 23:49:26 -0700 Subject: [PATCH 05/11] undo non-working windows jupyterlab fix. --- python/js/package.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/js/package.json b/python/js/package.json index 5556ea5df4a..0e2f7d78069 100644 --- a/python/js/package.json +++ b/python/js/package.json @@ -37,8 +37,7 @@ }, "resolutions": { "glob": "7.2.3", - "abab": "^2.0.6", - "license-webpack-plugin": "^4.0.0" + "abab": "^2.0.6" }, "dependencies": { "@jupyter-widgets/base": "^2 || ^3 || ^4 || ^5 || ^6", @@ -55,4 +54,4 @@ } } } -} \ No newline at end of file +} From 84f58be2feed823364588b2fd29472e2362f64c1 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Sat, 23 May 2026 15:29:26 -0700 Subject: [PATCH 06/11] [WebRTC] Refactor texture handling in ASSIMP export and improve data channel reliability --- cpp/open3d/t/io/file_format/FileASSIMP.cpp | 148 +++++++++--------- .../webrtc_server/WebRTCWindowSystem.cpp | 6 +- .../webrtc_server/html/webrtcstreamer.js | 17 +- 3 files changed, 84 insertions(+), 87 deletions(-) diff --git a/cpp/open3d/t/io/file_format/FileASSIMP.cpp b/cpp/open3d/t/io/file_format/FileASSIMP.cpp index 768ac51eca8..9cae51c804f 100644 --- a/cpp/open3d/t/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/t/io/file_format/FileASSIMP.cpp @@ -566,97 +566,97 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, ai_mat->AddProperty(&ac, 1, AI_MATKEY_COLOR_EMISSIVE); } - // Count texture maps... + // Build a list of texture-embed actions in a single pass. Each action + // is a lambda that writes one texture slot into the assimp scene; the + // slot index is passed at call time. . + // // NOTE: GLTF2 expects a single combined roughness/metal map. If the // model has one we just export it, otherwise if both roughness and // metal maps are available we combine them, otherwise if only one or // the other is available we just export the one map. const auto& material = w_mesh.GetMaterial(); - int n_textures = 0; - if (HasValidTexture(material, "albedo")) ++n_textures; - if (HasValidTexture(material, "normal")) ++n_textures; - if (HasValidTexture(material, "ambient_occlusion")) ++n_textures; + using TextureAction = std::function; + std::vector texture_actions; + + if (HasValidTexture(material, "albedo")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAlbedoMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_DIFFUSE, img); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_BASE_COLOR, img); + }); + } + if (HasValidTexture(material, "ambient_occlusion")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAOMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_LIGHTMAP, img); + }); + } if (HasValidTexture(material, "ao_rough_metal")) { - ++n_textures; + texture_actions.push_back([&](int idx) { + auto img = material.GetAORoughnessMetalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); } else if (HasValidTexture(material, "roughness") && HasValidTexture(material, "metallic")) { - ++n_textures; + texture_actions.push_back([&](int idx) { + auto rough = material.GetRoughnessMap().AsTensor(); + auto metal = material.GetMetallicMap().AsTensor(); + if (rough.GetShape() != metal.GetShape()) { + utility::LogError( + "RoughnessMap (shape={}) and MetallicMap " + "(shape={}) must have the same shape.", + rough.GetShape(), metal.GetShape()); + } + auto rows = rough.GetShape(0); + auto cols = rough.GetShape(1); + auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, + core::Dtype::UInt8); + rough_metal.Slice(2, 2, 3) = + metal.Slice(2, 0, 1); // blue channel is metal + rough_metal.Slice(2, 1, 2) = + rough.Slice(2, 0, 1); // green channel is roughness + geometry::Image rough_metal_img(rough_metal); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, + rough_metal_img); + }); } else { - if (HasValidTexture(material, "roughness")) ++n_textures; - if (HasValidTexture(material, "metallic")) ++n_textures; + if (HasValidTexture(material, "roughness")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetRoughnessMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } + if (HasValidTexture(material, "metallic")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetMetallicMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } } + if (HasValidTexture(material, "normal")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetNormalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_NORMALS, img); + }); + } + + int n_textures = static_cast(texture_actions.size()); if (n_textures > 0) { ai_scene->mTextures = new aiTexture*[n_textures]; for (int i = 0; i < n_textures; ++i) { ai_scene->mTextures[i] = new aiTexture(); + texture_actions[i](i); } ai_scene->mNumTextures = n_textures; } - - // Now embed the textures that are available... - int current_idx = 0; - if (HasValidTexture(material, "albedo")) { - auto img = material.GetAlbedoMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_DIFFUSE, img); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_BASE_COLOR, img); - ++current_idx; - } - if (HasValidTexture(w_mesh.GetMaterial(), "ambient_occlusion")) { - auto img = w_mesh.GetMaterial().GetAOMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_LIGHTMAP, img); - ++current_idx; - } - if (HasValidTexture(w_mesh.GetMaterial(), "ao_rough_metal")) { - auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } else if (HasValidTexture(w_mesh.GetMaterial(), "roughness") && - HasValidTexture(w_mesh.GetMaterial(), "metallic")) { - auto rough = w_mesh.GetMaterial().GetRoughnessMap().AsTensor(); - auto metal = w_mesh.GetMaterial().GetMetallicMap().AsTensor(); - if (rough.GetShape() != metal.GetShape()) { - utility::LogError( - "RoughnessMap (shape={}) and MetallicMap (shape={}) " - "must have the same shape.", - rough.GetShape(), metal.GetShape()); - } - auto rows = rough.GetShape(0); - auto cols = rough.GetShape(1); - auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, - core::Dtype::UInt8); - rough_metal.Slice(2, 2, 3) = - metal.Slice(2, 0, 1); // blue channel is metal - rough_metal.Slice(2, 1, 2) = - rough.Slice(2, 0, 1); // green channel is roughness - - geometry::Image rough_metal_img(rough_metal); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, rough_metal_img); - ++current_idx; - } else { - if (HasValidTexture(material, "roughness")) { - auto img = material.GetRoughnessMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - if (HasValidTexture(material, "metallic")) { - auto img = material.GetMetallicMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - } - if (HasValidTexture(material, "normal")) { - auto img = material.GetNormalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_NORMALS, img); - ++current_idx; - } } ai_scene->mMaterials[0] = ai_mat; diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index ab238201e7d..c703ac5f66a 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -364,10 +364,8 @@ std::string WebRTCWindowSystem::OnDataChannelMessage( if (impl_->data_channel_message_callbacks_.count(class_name) != 0) { reply = impl_->data_channel_message_callbacks_.at(class_name)( message); - // Do not call PostRedrawEvent here. Mouse/keyboard callbacks - // already schedule a redraw via Window::OnMouseEvent → PostRedraw. - // An extra PostRedrawEvent here creates a duplicate draw event - // before the input event is even processed, causing backlog. + // Custom callbacks that mutate GUI state (e.g. add/remove geometry) + // must call window->PostRedraw() or post_redraw() themselves. return reply; } else { reply = fmt::format( diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index 45dea6bf245..e3c385d6fb4 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -699,16 +699,15 @@ let WebRtcStreamer = (function() { }; // Local datachannel sends data. - // Use unordered, unreliable delivery for mouse/input events. Ordered - // reliable delivery causes head-of-line blocking: a lost packet stalls - // all subsequent events until it is retransmitted. For interactive - // mouse events the latest position is all that matters, so a dropped - // message is better than a delayed one. maxRetransmits: 0 means the - // browser sends once and moves on; ordered: false removes sequencing. + // Use reliable ordered delivery (the default). While unordered + + // unreliable would reduce head-of-line blocking for mouse MOVE/DRAG, + // the same channel carries discrete events (BUTTON_DOWN/UP, key events, + // resize) and application-level RPC (tensorboard/update_geometry etc.) + // that must not be lost or reordered. Mouse coalescing (rAF + server- + // side replace_or_merge_mouse) already prevents event backlog, so the + // extra reliability is free in practice on a local network. try { - this.dataChannel = pc.createDataChannel( - 'ClientDataChannel', - {ordered: false, maxRetransmits: 0}); + this.dataChannel = pc.createDataChannel('ClientDataChannel'); var dataChannel = this.dataChannel; dataChannel.onopen = function() { console.log('local datachannel open'); From e98fba4ad63af7314b23d84a400c4fb80b4d7e62 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 16 Jun 2026 00:11:51 -0700 Subject: [PATCH 07/11] Upgrade webrtc library to M149 (latest). WIP --- .github/workflows/webrtc.yml | 169 +++++---- .github/workflows/windows.yml | 5 +- 3rdparty/find_dependencies.cmake | 17 +- ...uild-enable-rtc_use_cxx11_abi-option.patch | 15 +- .../0002-build-enable_safe_libstdcxx.patch | 11 + ...003-src-fix-nullptr_t-with-libstdcxx.patch | 12 + ...-gcc-suppress-port-interface-network.patch | 17 + ...ll-payload_type_picker-gcc-flat_tree.patch | 12 + .../webrtc/0006-build-win-dynamic-crt.patch | 35 ++ 3rdparty/webrtc/CMakeLists.txt | 15 +- 3rdparty/webrtc/README.md | 26 +- 3rdparty/webrtc/apply_webrtc_patches.sh | 48 +++ 3rdparty/webrtc/webrtc_build.sh | 201 ++++++----- 3rdparty/webrtc/webrtc_common.cmake | 74 ++-- 3rdparty/webrtc/webrtc_download.cmake | 97 +++-- cpp/open3d/t/geometry/kernel/MinimumOBE.cpp | 4 +- .../webrtc_server/BitmapTrackSource.cpp | 6 +- .../webrtc_server/BitmapTrackSource.h | 12 +- .../webrtc_server/CMakeLists.txt | 18 + .../webrtc_server/ImageCapturer.cpp | 18 +- .../webrtc_server/ImageCapturer.h | 24 +- .../webrtc_server/PeerConnectionManager.cpp | 334 ++++++++++-------- .../webrtc_server/PeerConnectionManager.h | 94 +++-- .../visualization/webrtc_server/VideoFilter.h | 12 +- .../visualization/webrtc_server/VideoScaler.h | 21 +- .../webrtc_server/WebRTCWindowSystem.cpp | 45 ++- .../webrtc_server/html/index.html | 4 +- .../webrtc_server/html/open3d_logo.ico | Bin 34494 -> 0 bytes .../webrtc_server/html/webrtcstreamer.js | 46 +-- examples/cpp/CMakeLists.txt | 9 + 30 files changed, 856 insertions(+), 545 deletions(-) create mode 100644 3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch create mode 100644 3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch create mode 100644 3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch create mode 100644 3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch create mode 100644 3rdparty/webrtc/0006-build-win-dynamic-crt.patch create mode 100755 3rdparty/webrtc/apply_webrtc_patches.sh delete mode 100644 cpp/open3d/visualization/webrtc_server/html/open3d_logo.ico diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 45b27363ebc..481671a1e6c 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -5,13 +5,13 @@ on: workflow_dispatch: inputs: webrtc_commit: - description: 'Specify WebRTC commit to build.' + description: 'WebRTC src commit (full or short).' required: false - default: '60e674842ebae283cc6b2627f4b6f2f8186f3317' # Date: Wed Apr 7 19:12:13 2021 +0200 + default: 'e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e' depot_tools_commit: - description: 'Specify Depot Tools commit to to use for the build.' + description: 'depot_tools commit (empty = HEAD).' required: false - default: 'e1a98941d3ab10549be6d82d0686bb0fb91ec903' # Date: Wed Apr 7 21:35:29 2021 +0000 + default: '' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,29 +20,34 @@ concurrency: env: WEBRTC_COMMIT: ${{ github.event.inputs.webrtc_commit }} DEPOT_TOOLS_COMMIT: ${{ github.event.inputs.depot_tools_commit }} + NPROC: 4 jobs: Unix: permissions: - contents: write # upload + contents: write runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-13] + include: + - os: ubuntu-22.04 + package_suffix: linux_cxx-abi-1 + - os: macos-14 + package_suffix: macos_arm64 steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Set up Python version + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' - - name: Install dependencies - if: ${{ matrix.os == 'ubuntu-22.04' }} + - name: Install dependencies (Ubuntu) + if: matrix.os == 'ubuntu-22.04' run: | source 3rdparty/webrtc/webrtc_build.sh install_dependencies_ubuntu @@ -60,21 +65,36 @@ jobs: - name: Upload WebRTC uses: actions/upload-artifact@v4 with: - name: webrtc_release_${{ matrix.os }} + name: webrtc_${{ matrix.package_suffix }} path: | - webrtc_*.tar.gz - checksum_*.txt + webrtc_*.tar.gz + checksum_webrtc_*.tar.gz if-no-files-found: error Windows: permissions: - contents: write # upload - # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/windows_build_instructions.md + contents: write runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + include: + - config: Release + static_runtime: ON + tag: Release_mt + - config: Release + static_runtime: OFF + tag: Release_md + - config: Debug + static_runtime: ON + tag: Debug_mt + - config: Debug + static_runtime: OFF + tag: Debug_md env: - WORK_DIR: "C:\\WebRTC" # Not enough space in D: - OPEN3D_DIR: "D:\\a\\open3d\\open3d" - DEPOT_TOOLS_UPDATE: 1 # Fix cannot find python3_bin_reldir.txt + WORK_DIR: 'C:\WebRTC' + OPEN3D_DIR: ${{ github.workspace }} + DEPOT_TOOLS_UPDATE: 1 DEPOT_TOOLS_WIN_TOOLCHAIN: 0 NPROC: 2 @@ -82,17 +102,18 @@ jobs: - name: Checkout source code uses: actions/checkout@v4 - - name: Set up Python version + - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.10' - - name: Disk space + - name: Prepare work directory + shell: pwsh run: | Get-PSDrive - mkdir "$env:WORK_DIR" + New-Item -ItemType Directory -Force -Path $env:WORK_DIR - - name: Setup PATH for Visual Studio # Required for Ninja + - name: Setup PATH for Visual Studio uses: ilammy/msvc-dev-cmd@v1 with: arch: x64 @@ -102,74 +123,70 @@ jobs: working-directory: ${{ env.WORK_DIR }} run: | $ErrorActionPreference = 'Stop' - echo "Get depot_tools" - # Checkout to a specific version - # Ref: https://chromium.googlesource.com/chromium/src/+/main/docs/building_old_revisions.md - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - git -C depot_tools checkout $env:DEPOT_TOOLS_COMMIT - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - - echo "Get WebRTC" - mkdir webrtc - cd webrtc - fetch webrtc - - git -C src checkout $env:WEBRTC_COMMIT - git -C src submodule update --init --recursive - echo "gclient sync" + if (-not (Test-Path depot_tools)) { + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + } + if ($env:DEPOT_TOOLS_COMMIT) { + git -C depot_tools checkout $env:DEPOT_TOOLS_COMMIT + } + $env:Path = (Get-Item depot_tools).FullName + ';' + $env:Path + if (-not (Test-Path webrtc/src)) { + New-Item -ItemType Directory -Force -Path webrtc | Out-Null + Push-Location webrtc + fetch webrtc + Pop-Location + } + git -C webrtc/src checkout $env:WEBRTC_COMMIT + git -C webrtc/src submodule update --init --recursive + Push-Location webrtc gclient sync -D --force --reset - cd .. - echo "random.org" - curl "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" -o skipcache + Pop-Location - name: Patch WebRTC + shell: pwsh working-directory: ${{ env.WORK_DIR }} run: | $ErrorActionPreference = 'Stop' - cp "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/ - cp "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/ - - - name: Build WebRTC (Release) - working-directory: ${{ env.WORK_DIR }} - run: | - $ErrorActionPreference = 'Stop' - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - mkdir webrtc/build - cd webrtc/build - cmake -G Ninja -D CMAKE_BUILD_TYPE=Release ` - -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Release ` - .. - ninja install - echo "Cleanup build folder for next config build" - cd .. - rm -r build + Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/ + Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/ + bash "$env:OPEN3D_DIR/3rdparty/webrtc/apply_webrtc_patches.sh" ` + "$env:OPEN3D_DIR" "$env:WORK_DIR/webrtc/src" - - name: Build WebRTC (Debug) + - name: Build and package WebRTC + shell: pwsh working-directory: ${{ env.WORK_DIR }} + env: + BUILD_CONFIG: ${{ matrix.config }} + STATIC_RT: ${{ matrix.static_runtime }} + WIN_TAG: ${{ matrix.tag }} run: | $ErrorActionPreference = 'Stop' - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - mkdir webrtc/build - cd webrtc/build - cmake -G Ninja -D CMAKE_BUILD_TYPE=Debug ` - -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Debug ` - .. + $env:Path = (Get-Item depot_tools).FullName + ';' + $env:Path + $installRoot = Join-Path $env:WORK_DIR "webrtc_pkg" + if (Test-Path $installRoot) { Remove-Item -Recurse -Force $installRoot } + New-Item -ItemType Directory -Force -Path webrtc/build | Out-Null + Push-Location webrtc/build + $debugFlag = if ($env:BUILD_CONFIG -eq 'Debug') { 'ON' } else { 'OFF' } + cmake -G Ninja ` + -D CMAKE_BUILD_TYPE=$env:BUILD_CONFIG ` + -D WEBRTC_IS_DEBUG=$debugFlag ` + -D WEBRTC_STATIC_MSVC_RUNTIME=$env:STATIC_RT ` + -D CMAKE_INSTALL_PREFIX=$installRoot ` + .. ninja install - - - name: Package WebRTC - working-directory: ${{ env.WORK_DIR }} - run: | - $ErrorActionPreference = 'Stop' - $env:WEBRTC_COMMIT_SHORT = (git -C webrtc/src rev-parse --short=7 HEAD) - cmake -E tar cv webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip ` - --format=zip -- webrtc_release - cmake -E sha256sum webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip | Tee-Object -FilePath checksum_win.txt + Pop-Location + $short = (git -C webrtc/src rev-parse --short=7 HEAD) + $zip = "webrtc_${short}_win_$env:WIN_TAG.zip" + Push-Location $installRoot + cmake -E tar cvf (Join-Path $env:OPEN3D_DIR $zip) --format=zip . + Pop-Location + cmake -E sha256sum (Join-Path $env:OPEN3D_DIR $zip) | Tee-Object -FilePath (Join-Path $env:OPEN3D_DIR "checksum_$zip") - name: Upload WebRTC uses: actions/upload-artifact@v4 with: - name: webrtc_release_windows + name: webrtc_win_${{ matrix.tag }} path: | - ${{ env.WORK_DIR }}/webrtc_*.zip - ${{ env.WORK_DIR }}/checksum_*.txt + webrtc_*_win_*.zip + checksum_webrtc_*_win_*.zip if-no-files-found: error diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d571f97e295..93b303e02e3 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -52,7 +52,10 @@ jobs: - BUILD_CUDA_MODULE: ON # FIXME CONFIG: Debug env: - BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} + # WebRTC prebuilt ships static libs for both MSVC runtimes (/MT, /MD). + # Enable it for static-lib + static-runtime (/MT) and shared-lib + + # dynamic-runtime (/MD). Shared libs always use the dynamic runtime. + BUILD_WEBRTC: ${{ ( ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) || ( matrix.BUILD_SHARED_LIBS == 'ON' && matrix.STATIC_RUNTIME == 'OFF' ) ) && 'ON' || 'OFF' }} BUILD_PYTORCH_OPS: ${{ ( matrix.BUILD_CUDA_MODULE == 'ON' || matrix.CONFIG == 'Debug' ) && 'OFF' || 'ON' }} # FIXME steps: diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index fb926d36ebb..3c7456fa4d8 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -1988,11 +1988,24 @@ if(BUILD_WEBRTC) open3d_import_3rdparty_library(3rdparty_webrtc HIDDEN INCLUDE_DIRS ${WEBRTC_INCLUDE_DIRS} - LIB_DIR ${WEBRTC_LIB_DIR} - LIBRARIES ${WEBRTC_LIBRARIES} DEPENDS ext_webrtc_all ) + if(UNIX AND NOT APPLE) + target_link_libraries(3rdparty_webrtc INTERFACE + "-Wl,--whole-archive" + "$" + "-Wl,--no-whole-archive" + "$") + else() + target_link_libraries(3rdparty_webrtc INTERFACE + "$" + "$") + endif() target_link_libraries(3rdparty_webrtc INTERFACE Open3D::3rdparty_threads ${CMAKE_DL_LIBS}) + # libwebrtc.a and libturbojpeg.a both export jpeg_* symbols (WebRTC bundles libjpeg). + if(UNIX AND NOT APPLE) + target_link_options(3rdparty_webrtc INTERFACE "LINKER:--allow-multiple-definition") + endif() if (MSVC) # https://github.com/iimachines/webrtc-build/issues/2#issuecomment-503535704 target_link_libraries(3rdparty_webrtc INTERFACE secur32 winmm dmoguids wmcodecdspuuid msdmo strmiids) endif() diff --git a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch index 5d1897193c9..1c87f7ffc0d 100644 --- a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch +++ b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch @@ -8,22 +8,17 @@ Subject: [PATCH] build: enable rtc_use_cxx11_abi option 1 file changed, 6 insertions(+) diff --git a/config/BUILDCONFIG.gn b/config/BUILDCONFIG.gn -index 0ef73ab2b..5ab677e27 100644 --- a/config/BUILDCONFIG.gn +++ b/config/BUILDCONFIG.gn -@@ -163,6 +163,12 @@ declare_args() { - is_component_build = is_debug && current_os != "ios" +@@ -171,6 +171,12 @@ declare_args() { + is_debug && current_os != "ios" && current_os != "watchos" } +declare_args() { -+ # Set to false to define "_GLIBCXX_USE_CXX11_ABI=0". If set to true, the -+ # default will be used, which corresponds to the new CXX11 ABI. ++ # Open3D: GCC libstdc++ ABI selection on Linux. + rtc_use_cxx11_abi = true +} + assert(!(is_debug && is_official_build), "Can't do official debug builds") - - # ============================================================================== --- -2.17.1 - + assert(!(current_os == "ios" && is_component_build), + "Can't use component build on iOS") diff --git a/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch b/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch new file mode 100644 index 00000000000..29df04e0a43 --- /dev/null +++ b/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch @@ -0,0 +1,11 @@ +diff --git a/build.gni b/build.gni +--- a/build.gni ++++ b/build.gni +@@ -10,6 +10,7 @@ enable_java_templates = true + + # Enables assertions on safety checks in libc++. + enable_safe_libcxx = true ++enable_safe_libstdcxx = true + + # Don't set this variable to true when building standalone WebRTC, it is + # only needed to support both WebRTC standalone and Chromium builds. diff --git a/3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch b/3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch new file mode 100644 index 00000000000..396a02f6aaf --- /dev/null +++ b/3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch @@ -0,0 +1,12 @@ +diff --git a/rtc_base/ssl_stream_adapter.h b/rtc_base/ssl_stream_adapter.h +--- a/rtc_base/ssl_stream_adapter.h ++++ b/rtc_base/ssl_stream_adapter.h +@@ -129,7 +129,7 @@ class SSLStreamAdapter : public StreamInterface { + static std::unique_ptr Create( + std::unique_ptr stream, + absl::AnyInvocable handshake_error, +- nullptr_t /*field_trials*/) { ++ std::nullptr_t /*field_trials*/) { + return Create(std::move(stream), std::move(handshake_error)); + } + diff --git a/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch b/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch new file mode 100644 index 00000000000..ed0a638cd7f --- /dev/null +++ b/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch @@ -0,0 +1,17 @@ +diff --git a/p2p/base/port_interface.h b/p2p/base/port_interface.h +--- a/p2p/base/port_interface.h ++++ b/p2p/base/port_interface.h +@@ -52,7 +52,11 @@ class PortInterface { + virtual ~PortInterface(); + + virtual IceCandidateType Type() const = 0; ++#if defined(__GNUC__) && !defined(__clang__) ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wchanges-meaning" ++#endif + virtual const Network* Network() const = 0; ++#if defined(__GNUC__) && !defined(__clang__) ++#pragma GCC diagnostic pop ++#endif + + // Methods to set/get ICE role and tiebreaker values. diff --git a/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch b/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch new file mode 100644 index 00000000000..8dc679d130f --- /dev/null +++ b/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch @@ -0,0 +1,12 @@ +diff --git a/call/payload_type_picker.cc b/call/payload_type_picker.cc +--- a/call/payload_type_picker.cc ++++ b/call/payload_type_picker.cc +@@ -354,7 +354,7 @@ RTCErrorOr RtpHeaderExtensionRecorder::LookupId(absl::string_view uri, + + RTCErrorOr RtpHeaderExtensionRecorder::LookupId(absl::string_view uri, + bool encrypt) const { +- auto it = uri_to_id_.find(std::pair{uri, encrypt}); ++ auto it = uri_to_id_.find(std::pair{std::string(uri), encrypt}); + if (it == uri_to_id_.end()) { + return RTCError(RTCErrorType::INVALID_PARAMETER, + "No ID found for extension"); diff --git a/3rdparty/webrtc/0006-build-win-dynamic-crt.patch b/3rdparty/webrtc/0006-build-win-dynamic-crt.patch new file mode 100644 index 00000000000..c5f7277dabf --- /dev/null +++ b/3rdparty/webrtc/0006-build-win-dynamic-crt.patch @@ -0,0 +1,35 @@ +Subject: [PATCH] build: allow dynamic CRT for non-component desktop Windows + +Open3D ships a static libwebrtc but needs to support both the static +(/MT[d]) and dynamic (/MD[d]) MSVC runtimes. Upstream ties the CRT choice +to is_component_build, which would force a component (shared) build to get +/MD. Add an rtc_win_dynamic_crt gn arg so a non-component static build can +still select the dynamic CRT, matching Open3D STATIC_WINDOWS_RUNTIME=OFF. + +diff --git a/config/win/BUILD.gn b/config/win/BUILD.gn +--- a/config/win/BUILD.gn ++++ b/config/win/BUILD.gn +@@ -513,6 +513,13 @@ if (build_with_chromium && current_cpu == target_cpu && host_os == "win") { + # Configures how the runtime library (CRT) is going to be used. + # See https://msdn.microsoft.com/en-us/library/2kzt1wy3.aspx for a reference of + # what each value does. ++declare_args() { ++ # Open3D: use the dynamic CRT (/MD[d]) for non-component desktop Windows ++ # builds so we can ship a static libwebrtc that links against the dynamic ++ # MSVC runtime (matches Open3D STATIC_WINDOWS_RUNTIME=OFF). ++ rtc_win_dynamic_crt = false ++} ++ + config("default_crt") { + if (is_component_build) { + # Component mode: dynamic CRT. Since the library is shared, it requires +@@ -525,6 +532,9 @@ config("default_crt") { + # contains a details explanation of what is happening with the Windows + # CRT in Visual Studio releases related to Windows store applications. + configs = [ ":dynamic_crt" ] ++ } else if (rtc_win_dynamic_crt) { ++ # Open3D: static library with the dynamic CRT (/MD[d]). ++ configs = [ ":dynamic_crt" ] + } else { + # Desktop Windows: static CRT. + configs = [ ":static_crt" ] diff --git a/3rdparty/webrtc/CMakeLists.txt b/3rdparty/webrtc/CMakeLists.txt index 34d052ae4df..6876f6d340a 100644 --- a/3rdparty/webrtc/CMakeLists.txt +++ b/3rdparty/webrtc/CMakeLists.txt @@ -31,6 +31,9 @@ cmake_dependent_option(WEBRTC_IS_DEBUG "WebRTC Debug build. Use ON for Win32 Open3D Debug." OFF "NOT CMAKE_BUILD_TYPE STREQUAL Debug OR NOT WIN32" ON) option(GLIBCXX_USE_CXX11_ABI "Set -D_GLIBCXX_USE_CXX11_ABI=1" ON) +if(MSVC) + option(WEBRTC_STATIC_MSVC_RUNTIME "Use /MT /MTd for WebRTC (Windows)" ON) +endif() # Set paths set(WEBRTC_ROOT ${PROJECT_SOURCE_DIR}) @@ -55,12 +58,10 @@ set(WEBRTC_NINJA_ROOT ${WEBRTC_ROOT}/src/out/${WEBRTC_BUILD}) # Common configs for WebRTC include(${PROJECT_SOURCE_DIR}/webrtc_common.cmake) -# Generate build/args.gn -if(NOT EXISTS ${WEBRTC_NINJA_ROOT}/args.gn) - get_webrtc_args(WEBRTC_ARGS) - file(WRITE ${WEBRTC_NINJA_ROOT}/args.gn ${WEBRTC_ARGS}) - message(STATUS "Configs written to ${WEBRTC_NINJA_ROOT}/args.gn") -endif() +get_webrtc_args(WEBRTC_ARGS) +file(MAKE_DIRECTORY "${WEBRTC_NINJA_ROOT}") +file(WRITE "${WEBRTC_NINJA_ROOT}/args.gn" "${WEBRTC_ARGS}") +message(STATUS "WebRTC args.gn -> ${WEBRTC_NINJA_ROOT}/args.gn") # libwebrtc.a add_custom_target(webrtc @@ -95,6 +96,8 @@ set_target_properties(webrtc_extra PROPERTIES # `-- libwebrtc_extra.a file(GLOB_RECURSE WEBRTC_INCLUDES RELATIVE ${WEBRTC_ROOT}/src ${WEBRTC_ROOT}/src/*.h + ${WEBRTC_ROOT}/src/*.hpp + ${WEBRTC_ROOT}/src/*.inc ) foreach(header ${WEBRTC_INCLUDES}) get_filename_component(dir ${header} DIRECTORY) diff --git a/3rdparty/webrtc/README.md b/3rdparty/webrtc/README.md index eb54b078089..c00b8d09a8a 100644 --- a/3rdparty/webrtc/README.md +++ b/3rdparty/webrtc/README.md @@ -13,13 +13,35 @@ webrtc_download.cmake # Used by Open3D CMake. Consume pre-compiled WebRTC. (Meth webrtc_build.cmake # Used by Open3D CMake. Build and consume WebRTC. (Method 2) # Other files -0001-xxx.patch x3 # Git patch for -DBUILD_WEBRTC_FROM_SOURCE=ON. (Method 1 Prepare-Phase & Method 2) +000*.patch # Git patches applied before building WebRTC. (Method 1 Prepare-Phase & Method 2) +apply_webrtc_patches.sh # Applies the patches to the WebRTC checkout. (Method 1 Prepare-Phase & Method 2) CMakeLists.txt # Used by `webrtc_build.sh` to compile WebRTC. (Method 1 Prepare-Phase) Dockerfile.webrtc # Calls `webrtc_build.sh` to compile WebRTC. (Method 1 Prepare-Phase) webrtc_build.sh # Used by `Dockerfile.webrtc`. (Method 1 Prepare-Phase) -webrtc_common.cmake # Specifies Common WebRTC targets. (Method 1 Prepare-Phase) +webrtc_common.cmake # Specifies Common WebRTC targets and gn args. (Method 1 Prepare-Phase) ``` +## Patches + +Applied by `apply_webrtc_patches.sh` (each is skipped if it does not apply +cleanly to the pinned WebRTC commit): + +``` +0001-src-enable-rtc_use_cxx11_abi-option.patch # -> src +0001-build-enable-rtc_use_cxx11_abi-option.patch # -> src/build +0001-third_party-enable-rtc_use_cxx11_abi-option.patch # -> src/third_party +0002-build-enable_safe_libstdcxx.patch # -> src/build_overrides +0003-src-fix-nullptr_t-with-libstdcxx.patch # -> src (GCC) +0004-src-gcc-suppress-port-interface-network.patch # -> src (GCC) +0005-call-payload_type_picker-gcc-flat_tree.patch # -> src (GCC) +0006-build-win-dynamic-crt.patch # -> src/build (Windows /MD) +``` + +`0006-build-win-dynamic-crt.patch` adds a `rtc_win_dynamic_crt` gn arg so a +non-component (static) Windows build can use the dynamic MSVC runtime +(`/MD[d]`). This lets Open3D ship a static `libwebrtc` for both +`STATIC_WINDOWS_RUNTIME=ON` (`/MT[d]`) and `OFF` (`/MD[d]`). + ## Method 1 The pre-compiled WebRTC package used in Method 1 is generated by diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh new file mode 100755 index 00000000000..94e89a911a5 --- /dev/null +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Apply Open3D WebRTC patches under WEBRTC_SRC (webrtc checkout src/). +set -euo pipefail + +OPEN3D_DIR="${1:?Open3D repo path}" +WEBRTC_SRC="${2:?WebRTC src path}" + +# Apply a patch, hard-failing if a required patch cannot be applied. +# +# Args: [required|optional] (default: required) +# +# A patch is considered "already applied" when it applies in reverse; in that +# case it is skipped without error so the script is safe to re-run and tolerant +# of fixes that have landed upstream. A required patch that neither applies nor +# is already applied aborts the build: these patches add gn args / ABI defines +# or fix compile errors, so silently skipping them produces broken or +# confusing artifacts (e.g. an undeclared gn arg in args.gn, a C++11/C++17 ABI +# mismatch, or a hard compile error) rather than a clear failure here. +apply_one() { + local patch="$1" + local dir="$2" + local required="${3:-required}" + local name + name="$(basename "$patch")" + if git -C "$dir" apply --check "$patch" 2>/dev/null; then + git -C "$dir" apply "$patch" + echo "Applied $name in $dir" + elif git -C "$dir" apply --reverse --check "$patch" 2>/dev/null; then + echo "Skip $name (already applied) in $dir" + elif [[ "$required" == "optional" ]]; then + echo "Skip $name (does not apply; optional) in $dir" + else + echo "ERROR: required patch $name does not apply in $dir." >&2 + echo " Refresh the patch for the pinned WebRTC commit." >&2 + exit 1 + fi +} + +PATCH_DIR="$OPEN3D_DIR/3rdparty/webrtc" +# Required: declare gn args consumed by args.gn and fix GCC compile errors. +apply_one "$PATCH_DIR/0001-src-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0001-build-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/build" +apply_one "$PATCH_DIR/0001-third_party-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/third_party" +apply_one "$PATCH_DIR/0002-build-enable_safe_libstdcxx.patch" "$WEBRTC_SRC/build_overrides" +apply_one "$PATCH_DIR/0003-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0004-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0005-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0006-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index b4e48f126b2..809cb804274 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -1,78 +1,75 @@ #!/usr/bin/env bash set -euox pipefail -# This script builds WebRTC for Open3D for Ubuntu and macOS. For Windows, see -# .github/workflows/webrtc.yml +# Builds WebRTC static libraries for Open3D (Ubuntu/macOS). Windows: webrtc.yml # -# Usage: -# $ bash # Start a new shell -# Specify custom configuration by exporting environment variables -# GLIBCXX_USE_CXX11_ABI, WEBRTC_COMMIT and DEPOT_TOOLS_COMMIT, if required. -# $ source 3rdparty/webrtc/webrtc_build.sh -# $ install_dependencies_ubuntu # Ubuntu only -# $ download_webrtc_sources -# $ build_webrtc -# A webrtc__platform.tar.gz file will be created that can be used to -# build Open3D with WebRTC support. -# -# Procedure: -# -# 1) Download depot_tools, webrtc to following directories: -# ├── Oepn3D -# ├── depot_tools -# └── webrtc -#    ├── .gclient -#    └── src +# Layout (default: repo parent holds depot_tools + webrtc): +# / +# ├── Open3D/ # this repository +# ├── depot_tools/ +# └── webrtc/ +# └── src/ # -# 2) depot_tools and webrtc have compatible versions, see: -# https://chromium.googlesource.com/chromium/src/+/master/docs/building_old_revisions.md -# -# 3) Apply the following patch to enable GLIBCXX_USE_CXX11_ABI selection: -# - 0001-build-enable-rtc_use_cxx11_abi-option.patch # apply to webrtc/src -# - 0001-src-enable-rtc_use_cxx11_abi-option.patch # apply to webrtc/src/build -# - 0001-third_party-enable-rtc_use_cxx11_abi-option.patch # apply to webrtc/src/third_party -# Note that these patches may or may not be compatible with your custom -# WebRTC commits. You may have to patch them manually. +# Usage: +# cd /path/to/Open3D +# export WEBRTC_COMMIT=... # optional +# source 3rdparty/webrtc/webrtc_build.sh +# install_dependencies_ubuntu # optional on Ubuntu +# download_webrtc_sources +# build_webrtc -# Date: Wed Apr 7 19:12:13 2021 +0200 -WEBRTC_COMMIT=${WEBRTC_COMMIT:-60e674842ebae283cc6b2627f4b6f2f8186f3317} -# Date: Wed Apr 7 21:35:29 2021 +0000 -DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-e1a98941d3ab10549be6d82d0686bb0fb91ec903} +# libwebrtc-bin M149 / Open3D target milestone +WEBRTC_COMMIT=${WEBRTC_COMMIT:-e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e} +# Optional pin; unset uses depot_tools HEAD. +DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-} -GLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI:-0} -NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN)} # POSIX: MacOS + Linux -SUDO=${SUDO:-sudo} # Set to command if running inside docker -export PATH="$PWD/../depot_tools":${PATH} # $(basename $PWD) == Open3D -export DEPOT_TOOLS_UPDATE=0 +GLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI:-1} +NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN)} +SUDO=${SUDO:-sudo} + +webrtc_work_root() { + if [[ -n "${WEBRTC_WORK_ROOT:-}" ]]; then + echo "$WEBRTC_WORK_ROOT" + else + dirname "$PWD" + fi +} + +webrtc_setup_path() { + export PATH="$(webrtc_work_root)/depot_tools:${PATH}" + export DEPOT_TOOLS_UPDATE=0 +} install_dependencies_ubuntu() { options="$(echo "$@" | tr ' ' '|')" - # Dependencies - # python* : resolve ImportError: No module named pkg_resources - # libglib2.0-dev: resolve pkg_config("glib") $SUDO apt-get update $SUDO apt-get install -y \ apt-transport-https \ build-essential \ ca-certificates \ + clang \ git \ gnupg \ libglib2.0-dev \ - python \ - python-pip \ - python-setuptools \ - python-wheel \ + libnss3-dev \ + libgtk-3-dev \ + ninja-build \ + python3 \ + python3-pip \ + python3-setuptools \ + pkg-config \ software-properties-common \ tree \ curl - curl https://apt.kitware.com/keys/kitware-archive-latest.asc \ - 2>/dev/null | gpg --dearmor - | - $SUDO sed -n 'w /etc/apt/trusted.gpg.d/kitware.gpg' # Write to file, no stdout - source <(grep VERSION_CODENAME /etc/os-release) - $SUDO apt-add-repository --yes "deb https://apt.kitware.com/ubuntu/ $VERSION_CODENAME main" - $SUDO apt-get update - $SUDO apt-get --yes install cmake - cmake --version >/dev/null + if ! command -v cmake >/dev/null 2>&1 || [[ $(cmake --version | head -1 | grep -oE '[0-9]+\.[0-9]+') < "3.18" ]]; then + curl https://apt.kitware.com/keys/kitware-archive-latest.asc \ + 2>/dev/null | gpg --dearmor - | + $SUDO tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null + source <(grep VERSION_CODENAME /etc/os-release) + $SUDO apt-add-repository --yes "deb https://apt.kitware.com/ubuntu/ $VERSION_CODENAME main" + $SUDO apt-get update + $SUDO apt-get --yes install cmake + fi if [[ "purge-cache" =~ ^($options)$ ]]; then $SUDO apt-get clean $SUDO rm -rf /var/lib/apt/lists/* @@ -80,67 +77,69 @@ install_dependencies_ubuntu() { } download_webrtc_sources() { - # PWD=Open3D - pushd .. - echo Get depot_tools - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - git -C depot_tools checkout $DEPOT_TOOLS_COMMIT + local root + root="$(webrtc_work_root)" + pushd "$root" + if [[ ! -d depot_tools ]]; then + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + fi + if [[ -n "$DEPOT_TOOLS_COMMIT" ]]; then + git -C depot_tools checkout "$DEPOT_TOOLS_COMMIT" + fi + webrtc_setup_path command -V fetch - echo Get WebRTC - mkdir webrtc - cd webrtc - fetch webrtc + if [[ ! -d webrtc/src ]]; then + mkdir -p webrtc + pushd webrtc + fetch --nohooks webrtc + popd + fi - # Checkout to a specific version - # Ref: https://chromium.googlesource.com/chromium/src/+/master/docs/building_old_revisions.md - git -C src checkout $WEBRTC_COMMIT - git -C src submodule update --init --recursive - echo gclient sync - gclient sync -D --force --reset - cd .. - echo random.org - curl "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" -o skipcache + git -C webrtc/src checkout "$WEBRTC_COMMIT" + git -C webrtc/src submodule update --init --recursive + pushd webrtc + gclient sync -D --force --reset --no-history + popd popd } build_webrtc() { - # PWD=Open3D - OPEN3D_DIR="$PWD" - echo Apply patches - cp 3rdparty/webrtc/{CMakeLists.txt,webrtc_common.cmake} ../webrtc - git -C ../webrtc/src apply \ - "$OPEN3D_DIR"/3rdparty/webrtc/0001-src-enable-rtc_use_cxx11_abi-option.patch - git -C ../webrtc/src/build apply \ - "$OPEN3D_DIR"/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch - git -C ../webrtc/src/third_party apply \ - "$OPEN3D_DIR"/3rdparty/webrtc/0001-third_party-enable-rtc_use_cxx11_abi-option.patch - WEBRTC_COMMIT_SHORT=$(git -C ../webrtc/src rev-parse --short=7 HEAD) + local root open3d_dir + open3d_dir="$PWD" + root="$(webrtc_work_root)" + webrtc_setup_path - echo Build WebRTC - mkdir ../webrtc/build - pushd ../webrtc/build - cmake -DCMAKE_INSTALL_PREFIX=../../webrtc_release \ - -DGLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI} \ + cp "$open3d_dir"/3rdparty/webrtc/{CMakeLists.txt,webrtc_common.cmake} "$root/webrtc/" + bash "$open3d_dir"/3rdparty/webrtc/apply_webrtc_patches.sh \ + "$open3d_dir" "$root/webrtc/src" + + WEBRTC_COMMIT_SHORT=$(git -C "$root/webrtc/src" rev-parse --short=7 HEAD) + + mkdir -p "$root/webrtc/build" + pushd "$root/webrtc/build" + cmake -G Ninja \ + -DCMAKE_INSTALL_PREFIX="$root/webrtc_release" \ + -DGLIBCXX_USE_CXX11_ABI="${GLIBCXX_USE_CXX11_ABI}" \ .. - make -j$NPROC - make install - popd # PWD=Open3D - pushd .. - tree -L 2 webrtc_release || ls webrtc_release/* + ninja -j"${NPROC}" + ninja install + popd - echo Package WebRTC + pushd "$root" + tree -L 2 webrtc_release || ls -la webrtc_release if [[ $(uname -s) == 'Linux' ]]; then tar -czf \ - "$OPEN3D_DIR/webrtc_${WEBRTC_COMMIT_SHORT}_linux_cxx-abi-${GLIBCXX_USE_CXX11_ABI}.tar.gz" \ - webrtc_release + "$open3d_dir/webrtc_${WEBRTC_COMMIT_SHORT}_linux_cxx-abi-${GLIBCXX_USE_CXX11_ABI}.tar.gz" \ + -C "$root/webrtc_release" . elif [[ $(uname -s) == 'Darwin' ]]; then tar -czf \ - "$OPEN3D_DIR/webrtc_${WEBRTC_COMMIT_SHORT}_macos.tar.gz" \ - webrtc_release + "$open3d_dir/webrtc_${WEBRTC_COMMIT_SHORT}_macos_arm64.tar.gz" \ + -C "$root/webrtc_release" . fi - popd # PWD=Open3D - webrtc_package=$(ls webrtc_*.tar.gz) - cmake -E sha256sum "$webrtc_package" | tee "checksum_${webrtc_package%%.*}.txt" + popd + + webrtc_package=$(ls "$open3d_dir"/webrtc_*.tar.gz | tail -1) + cmake -E sha256sum "$webrtc_package" | tee "$open3d_dir/checksum_${webrtc_package##*/}" | sed 's|.*/||' ls -alh "$webrtc_package" } diff --git a/3rdparty/webrtc/webrtc_common.cmake b/3rdparty/webrtc/webrtc_common.cmake index 98c19336c6a..70708eb27b7 100644 --- a/3rdparty/webrtc/webrtc_common.cmake +++ b/3rdparty/webrtc/webrtc_common.cmake @@ -10,7 +10,7 @@ function(get_webrtc_args WEBRTC_ARGS) set(WEBRTC_ARGS "") if(NOT MSVC) - # ABI selection + # ABI selection (Linux only; Open3D Ubuntu 22.04 uses cxx11 ABI=1). if(GLIBCXX_USE_CXX11_ABI) set(WEBRTC_ARGS rtc_use_cxx11_abi=true\n${WEBRTC_ARGS}) else() @@ -18,57 +18,65 @@ function(get_webrtc_args WEBRTC_ARGS) endif() endif() - if (APPLE) # WebRTC default + if(APPLE) set(WEBRTC_ARGS is_clang=true\n${WEBRTC_ARGS}) - else() - # Do not use Google clang for compilation due to LTO error when Open3D - # is built with gcc on Ubuntu 20.04. + if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(WEBRTC_ARGS target_cpu=\"arm64\"\n${WEBRTC_ARGS}) + endif() + elseif(UNIX) set(WEBRTC_ARGS is_clang=false\n${WEBRTC_ARGS}) + set(WEBRTC_ARGS extra_cxxflags=[\"-Wno-changes-meaning\"]\n${WEBRTC_ARGS}) endif() - # Don't use libc++ (Clang), use libstdc++ (GNU) - # https://stackoverflow.com/a/47384787/1255535 set(WEBRTC_ARGS use_custom_libcxx=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS use_custom_libcxx_for_host=false\n${WEBRTC_ARGS}) - # Debug/Release if(WEBRTC_IS_DEBUG) set(WEBRTC_ARGS is_debug=true\n${WEBRTC_ARGS}) - if (MSVC) - # WebRTC default is false in Debug due to a performance penalty, but this would disable - # iterator debugging for Open3D and any user code as well with MSVC. + if(MSVC) set(WEBRTC_ARGS enable_iterator_debugging=true\n${WEBRTC_ARGS}) endif() else() set(WEBRTC_ARGS is_debug=false\n${WEBRTC_ARGS}) endif() - # H264 support - set(WEBRTC_ARGS is_chrome_branded=true\n${WEBRTC_ARGS}) + # H.264 (replaces deprecated is_chrome_branded on recent milestones). + set(WEBRTC_ARGS rtc_use_h264=true\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_include_tests=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_enable_protobuf=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_build_examples=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_build_tools=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS treat_warnings_as_errors=false\n${WEBRTC_ARGS}) - set(WEBRTC_ARGS rtc_enable_libevent=false\n${WEBRTC_ARGS}) - set(WEBRTC_ARGS rtc_build_libevent=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS use_sysroot=false\n${WEBRTC_ARGS}) + set(WEBRTC_ARGS rtc_use_perfetto=false\n${WEBRTC_ARGS}) - # Disable screen capturing set(WEBRTC_ARGS rtc_use_x11=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_use_pipewire=false\n${WEBRTC_ARGS}) - # Disable sound support set(WEBRTC_ARGS rtc_include_pulse_audio=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_include_internal_audio_device=false\n${WEBRTC_ARGS}) - # Use ccache if available, not recommended inside Docker + if(MSVC) + # Always build a static (non-component) libwebrtc that uses the MSVC STL + # (use_custom_libcxx=false, set above) so its ABI matches Open3D. Force + # is_component_build=false because it defaults to ON in Debug, which would + # produce shared component DLLs instead of a static lib. The MSVC runtime + # (/MT[d] vs /MD[d]) is selected independently via the rtc_win_dynamic_crt + # gn arg added by 0006-build-win-dynamic-crt.patch. + set(WEBRTC_ARGS is_component_build=false\n${WEBRTC_ARGS}) + if(WEBRTC_STATIC_MSVC_RUNTIME) + set(WEBRTC_ARGS rtc_win_dynamic_crt=false\n${WEBRTC_ARGS}) + else() + set(WEBRTC_ARGS rtc_win_dynamic_crt=true\n${WEBRTC_ARGS}) + endif() + endif() + find_program(CCACHE_BIN "ccache") if(CCACHE_BIN) - set(WEBRTC_ARGS cc_wrapper="ccache"\n${WEBRTC_ARGS}) + set(WEBRTC_ARGS cc_wrapper=\"ccache\"\n${WEBRTC_ARGS}) endif() - set(WEBRTC_ARGS ${WEBRTC_ARGS} PARENT_SCOPE) + set(WEBRTC_ARGS ${WEBRTC_ARGS} PARENT_SCOPE) endfunction() # webrtc -> libwebrtc.a @@ -79,13 +87,17 @@ set(NINJA_TARGETS jsoncpp builtin_video_decoder_factory builtin_video_encoder_factory - peerconnection + peer_connection p2p_server_utils task_queue default_task_queue_factory + # M149 modular PeerConnectionFactory (not all pulled into libwebrtc.a). + field_trials + enable_media_with_defaults + create_modular_peer_connection_factory + environment_factory ) -# Byproducts for ninja build, later packaged by CMake into libwebrtc_extra.a if(NOT WEBRTC_NINJA_ROOT) message(FATAL_ERROR "Please define WEBRTC_NINJA_ROOT before including webrtc_common.cmake") endif() @@ -96,4 +108,20 @@ set(EXTRA_WEBRTC_OBJS ${WEBRTC_NINJA_ROOT}/obj/p2p/p2p_server_utils/stun_server${CMAKE_CXX_OUTPUT_EXTENSION} ${WEBRTC_NINJA_ROOT}/obj/p2p/p2p_server_utils/turn_server${CMAKE_CXX_OUTPUT_EXTENSION} ${WEBRTC_NINJA_ROOT}/obj/rtc_base/rtc_json/json${CMAKE_CXX_OUTPUT_EXTENSION} - ) + ${WEBRTC_NINJA_ROOT}/obj/api/field_trials/field_trials${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/field_trials_registry/field_trials_registry${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/enable_media/enable_media${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/enable_media_with_defaults/enable_media_with_defaults${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/create_modular_peer_connection_factory/create_modular_peer_connection_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/environment/environment_factory/environment_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/environment/deprecated_global_field_trials/deprecated_global_field_trials${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/audio_codecs/builtin_audio_encoder_factory/builtin_audio_encoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/audio_codecs/builtin_audio_decoder_factory/builtin_audio_decoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/builtin_video_encoder_factory/builtin_video_encoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/builtin_video_decoder_factory/builtin_video_decoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/media/rtc_simulcast_encoder_adapter/simulcast_encoder_adapter${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/media/rtc_internal_video_codecs/internal_encoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/media/rtc_internal_video_codecs/internal_decoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/rtc_software_fallback_wrappers/video_encoder_software_fallback_wrapper${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/rtc_software_fallback_wrappers/video_decoder_software_fallback_wrapper${CMAKE_CXX_OUTPUT_EXTENSION} +) diff --git a/3rdparty/webrtc/webrtc_download.cmake b/3rdparty/webrtc/webrtc_download.cmake index 8ac5a0b6ef7..075c4ac00d8 100644 --- a/3rdparty/webrtc/webrtc_download.cmake +++ b/3rdparty/webrtc/webrtc_download.cmake @@ -4,43 +4,74 @@ include(ExternalProject) -set(WEBRTC_VER 60e6748) +set(WEBRTC_VER e8b4d4c) +if(DEFINED ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE} AND NOT "$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}" STREQUAL "") + set(WEBRTC_URL "file://$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}") +endif() if (APPLE) - set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc/webrtc_${WEBRTC_VER}_macos_10.14.tar.gz - ) - set(WEBRTC_SHA256 e9d1f4e4fefb2e28ef4f16cf4a4f0008baf4fe638ca3ad329e82e7fd0ce87f56) + if(NOT WEBRTC_URL) + set(WEBRTC_URL + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_macos_arm64.tar.gz + ) + endif() + # Update after publishing M149 macOS arm64 artifact from webrtc.yml. + set(WEBRTC_SHA256 PLACEHOLDER_MACOS_ARM64_SHA256) elseif (WIN32) - if (BUILD_SHARED_LIBS OR NOT STATIC_WINDOWS_RUNTIME) - message(FATAL_ERROR "Pre-built WebRTC binaries are not available for " - "BUILD_SHARED_LIBS=ON or STATIC_WINDOWS_RUNTIME=OFF. Please use " - "(a) BUILD_WEBRTC=OFF or " - "(b) BUILD_SHARED_LIBS=OFF and STATIC_WINDOWS_RUNTIME=ON or " - "(c) BUILD_WEBRTC_FROM_SOURCE=ON") + # Prebuilt WebRTC is a static lib for both MSVC runtimes. A shared Open3D + # build links it into open3d.dll and always uses the dynamic runtime, so + # only BUILD_SHARED_LIBS=ON + STATIC_WINDOWS_RUNTIME=ON is unsupported. + if (BUILD_SHARED_LIBS AND STATIC_WINDOWS_RUNTIME) + message(FATAL_ERROR "Pre-built WebRTC does not support " + "BUILD_SHARED_LIBS=ON with STATIC_WINDOWS_RUNTIME=ON. Use " + "STATIC_WINDOWS_RUNTIME=OFF or BUILD_WEBRTC_FROM_SOURCE=ON.") endif() - set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc/webrtc_${WEBRTC_VER}_win.zip - ) - set(WEBRTC_SHA256 f4686d0028ef5c36c5d7158a638fa834b63183b522f0b63932f7f70ebffeea22) -else() # Linux - if(GLIBCXX_USE_CXX11_ABI) + if(STATIC_WINDOWS_RUNTIME) + set(WEBRTC_RUNTIME_TAG mt) + else() + set(WEBRTC_RUNTIME_TAG md) + endif() + if(CMAKE_BUILD_TYPE STREQUAL Debug) + set(WEBRTC_CONFIG_TAG Debug) + else() + set(WEBRTC_CONFIG_TAG Release) + endif() + if(NOT WEBRTC_URL) set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v3/webrtc_${WEBRTC_VER}_cxx-abi-1.tar.gz + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_win_${WEBRTC_CONFIG_TAG}_${WEBRTC_RUNTIME_TAG}.zip ) - set(WEBRTC_SHA256 0d98ddbc4164b9e7bfc50b7d4eaa912a753dabde0847d85a64f93a062ae4c335) - else() + endif() + # Update after publishing four Windows artifacts from webrtc.yml. + set(WEBRTC_SHA256 PLACEHOLDER_WIN_SHA256) +else() # Linux + if(NOT WEBRTC_URL) + if(NOT GLIBCXX_USE_CXX11_ABI) + message(FATAL_ERROR "Pre-built WebRTC with GLIBCXX_USE_CXX11_ABI=OFF is " + "no longer provided. Use GLIBCXX_USE_CXX11_ABI=ON or " + "BUILD_WEBRTC_FROM_SOURCE=ON.") + endif() set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v3/webrtc_${WEBRTC_VER}_cxx-abi-0.tar.gz + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_linux_cxx-abi-1.tar.gz ) - set(WEBRTC_SHA256 2a3714713908f84079f1fbce8594c9b7010846b5db74b086f7bf30f22f1f5835) endif() + set(WEBRTC_SHA256 1b529bf448d5abd07ec1f8d310ee5c94bd79e84fe563ae1562420f8e478cc202) +endif() + +if(WEBRTC_SHA256 MATCHES "^PLACEHOLDER") + message(WARNING "WebRTC prebuilt SHA256 not set for this platform (${WEBRTC_SHA256}). " + "Set OPEN3D_WEBRTC_PREBUILT_ARCHIVE or update webrtc_download.cmake after CI publish.") + unset(WEBRTC_SHA256) +endif() + +set(_webrtc_url_hash "") +if(WEBRTC_SHA256) + set(_webrtc_url_hash URL_HASH SHA256=${WEBRTC_SHA256}) endif() ExternalProject_Add( ext_webrtc PREFIX webrtc URL ${WEBRTC_URL} - URL_HASH SHA256=${WEBRTC_SHA256} + ${_webrtc_url_hash} DOWNLOAD_DIR "${OPEN3D_THIRD_PARTY_DOWNLOAD_DIR}/webrtc" UPDATE_COMMAND "" CONFIGURE_COMMAND "" @@ -50,22 +81,18 @@ ExternalProject_Add( ) ExternalProject_Get_Property(ext_webrtc SOURCE_DIR) -if (WIN32) - set(SOURCE_DIR "${SOURCE_DIR}/$,Debug,Release>") -endif() -set(LIBPNG_INCLUDE_DIRS ${INSTALL_DIR}/include/) # "/" is critical. -set(LIBPNG_LIB_DIR ${INSTALL_DIR}/${Open3D_INSTALL_LIB_DIR}) -set(LIBPNG_LIBRARIES ${lib_name}$<$:d>) +# Prebuilt layout: flat include/ and lib/ at archive root (M149 packages). +set(WEBRTC_PREBUILT_ROOT ${SOURCE_DIR}) # Variables consumed by find_dependencies.cmake set(WEBRTC_INCLUDE_DIRS - ${SOURCE_DIR}/include/ - ${SOURCE_DIR}/include/third_party/abseil-cpp/ - ${SOURCE_DIR}/include/third_party/jsoncpp/source/include/ - ${SOURCE_DIR}/include/third_party/jsoncpp/generated/ - ${SOURCE_DIR}/include/third_party/libyuv/include/ + ${WEBRTC_PREBUILT_ROOT}/include/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/abseil-cpp/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/jsoncpp/source/include/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/jsoncpp/generated/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/libyuv/include/ ) -set(WEBRTC_LIB_DIR ${SOURCE_DIR}/lib) +set(WEBRTC_LIB_DIR ${WEBRTC_PREBUILT_ROOT}/lib) set(WEBRTC_LIBRARIES webrtc webrtc_extra diff --git a/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp b/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp index cf4844c1064..c16315568cd 100644 --- a/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp +++ b/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp @@ -52,8 +52,8 @@ void MapOBEToClosestIdentity(EigenOBE& obe) { Eigen::Vector3d& radii = obe.radii_; Eigen::Vector3d col[3] = {R.col(0), R.col(1), R.col(2)}; double best_score = -1e9; - Eigen::Matrix3d best_R; - Eigen::Vector3d best_radii; + Eigen::Matrix3d best_R = Eigen::Matrix3d::Identity(); + Eigen::Vector3d best_radii = Eigen::Vector3d::Zero(); // Hard-coded permutations of indices [0,1,2] static const std::array, 6> permutations = { diff --git a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp index 6fd753baf43..5a164646eb5 100644 --- a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp +++ b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp @@ -39,14 +39,14 @@ void BitmapTrackSource::SetState( } void BitmapTrackSource::AddOrUpdateSink( - rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) { + webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) { RTC_DCHECK(worker_thread_checker_.IsCurrent()); source()->AddOrUpdateSink(sink, wants); } void BitmapTrackSource::RemoveSink( - rtc::VideoSinkInterface* sink) { + webrtc::VideoSinkInterface* sink) { RTC_DCHECK(worker_thread_checker_.IsCurrent()); source()->RemoveSink(sink); } diff --git a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h index 81ea8243392..5dd350a95b8 100644 --- a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h +++ b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h @@ -71,15 +71,15 @@ class BitmapTrackSource : public webrtc::Notifier { return absl::nullopt; } bool GetStats(Stats* stats) override { return false; } - void AddOrUpdateSink(rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) override; - void RemoveSink(rtc::VideoSinkInterface* sink) override; + void AddOrUpdateSink(webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) override; + void RemoveSink(webrtc::VideoSinkInterface* sink) override; bool SupportsEncodedOutput() const override { return false; } void GenerateKeyFrame() override {} - void AddEncodedSink(rtc::VideoSinkInterface* + void AddEncodedSink(webrtc::VideoSinkInterface* sink) override {} void RemoveEncodedSink( - rtc::VideoSinkInterface* sink) + webrtc::VideoSinkInterface* sink) override {} virtual void OnFrame(const std::shared_ptr& frame) override { @@ -88,7 +88,7 @@ class BitmapTrackSource : public webrtc::Notifier { } protected: - virtual rtc::VideoSourceInterface* source() = 0; + virtual webrtc::VideoSourceInterface* source() = 0; private: webrtc::SequenceChecker worker_thread_checker_; diff --git a/cpp/open3d/visualization/webrtc_server/CMakeLists.txt b/cpp/open3d/visualization/webrtc_server/CMakeLists.txt index f5977e72e52..97124c25de7 100644 --- a/cpp/open3d/visualization/webrtc_server/CMakeLists.txt +++ b/cpp/open3d/visualization/webrtc_server/CMakeLists.txt @@ -15,6 +15,12 @@ target_compile_definitions(webrtc_server PRIVATE _FILE_OFFSET_BITS=64 # for civetweb _LARGEFILE_SOURCE=1 # for civetweb ) +# Prebuilt WebRTC is compiled in Release (RTC_DCHECK_IS_ON=0). Match that when +# Open3D is Debug so inlined WebRTC headers do not reference missing DCHECK +# symbols (e.g. SequenceCheckerImpl::ExpectationToString). +if(NOT BUILD_WEBRTC_FROM_SOURCE) + target_compile_definitions(webrtc_server PRIVATE NDEBUG) +endif() add_dependencies(webrtc_server copy_html_dir) open3d_show_and_abort_on_warning(webrtc_server) @@ -22,6 +28,8 @@ open3d_set_global_properties(webrtc_server) open3d_set_open3d_lib_properties(webrtc_server) open3d_link_3rdparty_libraries(webrtc_server) +set_target_properties(webrtc_server PROPERTIES CXX_STANDARD 20) + if (NOT GUI_RESOURCE_DIR) message(FATAL_ERROR "GUI_RESOURCE_DIR is not defined.") @@ -29,6 +37,12 @@ endif() message(STATUS "Copying ${CMAKE_CURRENT_SOURCE_DIR}/html to ${GUI_RESOURCE_DIR}.") file(MAKE_DIRECTORY ${GUI_RESOURCE_DIR}) +# Favicon is shared with Sphinx docs (single source of truth). +set(OPEN3D_ICON "${CMAKE_SOURCE_DIR}/docs/_static/open3d_logo.ico") +if(NOT EXISTS "${OPEN3D_ICON}") + message(FATAL_ERROR "Open3D WebRTC favicon not found: ${OPEN3D_ICON}") +endif() + # Force update ${GUI_RESOURCE_DIR}/html every time. add_custom_target(copy_html_dir ALL COMMAND ${CMAKE_COMMAND} -E rm -rf @@ -36,4 +50,8 @@ add_custom_target(copy_html_dir ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/html ${GUI_RESOURCE_DIR}/html + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${OPEN3D_ICON} + ${GUI_RESOURCE_DIR}/html/open3d_logo.ico + DEPENDS ${OPEN3D_ICON} ) diff --git a/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp b/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp index a7ad0109607..4d23c2d933c 100644 --- a/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp +++ b/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp @@ -7,11 +7,13 @@ #include "open3d/visualization/webrtc_server/ImageCapturer.h" +#include #include #include #include #include #include +#include #include @@ -50,7 +52,7 @@ void ImageCapturer::OnCaptureResult( int height = (int)frame->GetShape(0); int width = (int)frame->GetShape(1); - rtc::scoped_refptr i420_buffer = + webrtc::scoped_refptr i420_buffer = webrtc::I420Buffer::Create(width, height); // frame->data() @@ -64,7 +66,7 @@ void ImageCapturer::OnCaptureResult( if (conversion_result >= 0) { webrtc::VideoFrame video_frame(i420_buffer, webrtc::VideoRotation::kVideoRotation_0, - rtc::TimeMicros()); + webrtc::TimeMicros()); if ((height_ == 0) && (width_ == 0)) { broadcaster_.OnFrame(video_frame); } else { @@ -77,13 +79,13 @@ void ImageCapturer::OnCaptureResult( } int stride_y = width; int stride_uv = (width + 1) / 2; - rtc::scoped_refptr scaled_buffer = + webrtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(width, height, stride_y, stride_uv, stride_uv); scaled_buffer->ScaleFrom( *video_frame.video_frame_buffer()->ToI420()); webrtc::VideoFrame frame = webrtc::VideoFrame( - scaled_buffer, webrtc::kVideoRotation_0, rtc::TimeMicros()); + scaled_buffer, webrtc::kVideoRotation_0, webrtc::TimeMicros()); broadcaster_.OnFrame(frame); } @@ -93,15 +95,15 @@ void ImageCapturer::OnCaptureResult( } } -// Override rtc::VideoSourceInterface. +// Override webrtc::VideoSourceInterface. void ImageCapturer::AddOrUpdateSink( - rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) { + webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) { broadcaster_.AddOrUpdateSink(sink, wants); } void ImageCapturer::RemoveSink( - rtc::VideoSinkInterface* sink) { + webrtc::VideoSinkInterface* sink) { broadcaster_.RemoveSink(sink); } diff --git a/cpp/open3d/visualization/webrtc_server/ImageCapturer.h b/cpp/open3d/visualization/webrtc_server/ImageCapturer.h index f1b094ba3ca..b8fc188cc9c 100644 --- a/cpp/open3d/visualization/webrtc_server/ImageCapturer.h +++ b/cpp/open3d/visualization/webrtc_server/ImageCapturer.h @@ -10,7 +10,8 @@ #pragma once -#include +#include +#include #include #include #include @@ -26,7 +27,7 @@ namespace open3d { namespace visualization { namespace webrtc_server { -class ImageCapturer : public rtc::VideoSourceInterface { +class ImageCapturer : public webrtc::VideoSourceInterface { public: ImageCapturer(const std::string& url_, const std::map& opts); @@ -39,23 +40,23 @@ class ImageCapturer : public rtc::VideoSourceInterface { ImageCapturer(const std::map& opts); virtual void AddOrUpdateSink( - rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) override; + webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) override; virtual void RemoveSink( - rtc::VideoSinkInterface* sink) override; + webrtc::VideoSinkInterface* sink) override; void OnCaptureResult(const std::shared_ptr& frame); protected: int width_; int height_; - rtc::VideoBroadcaster broadcaster_; + webrtc::VideoBroadcaster broadcaster_; }; class ImageTrackSource : public BitmapTrackSource { public: - static rtc::scoped_refptr Create( + static webrtc::scoped_refptr Create( const std::string& window_uid, const std::map& opts) { std::unique_ptr capturer = @@ -63,10 +64,9 @@ class ImageTrackSource : public BitmapTrackSource { if (!capturer) { return nullptr; } - rtc::scoped_refptr video_source = - new rtc::RefCountedObject( - std::move(capturer)); - return video_source; + return webrtc::scoped_refptr( + new webrtc::RefCountedObject( + std::move(capturer))); } void OnFrame(const std::shared_ptr& frame) final override { @@ -78,7 +78,7 @@ class ImageTrackSource : public BitmapTrackSource { : BitmapTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {} private: - rtc::VideoSourceInterface* source() override { + webrtc::VideoSourceInterface* source() override { return capturer_.get(); } std::unique_ptr capturer_; diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index 68ff8256e29..9e1ccaca0bd 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -15,6 +15,11 @@ #include "open3d/visualization/webrtc_server/PeerConnectionManager.h" +#include +#include +#include +#include +#include #include #include #include @@ -24,10 +29,11 @@ #include #include #include -#include +#include #include #include +#include #include #include "open3d/utility/IJsonConvertible.h" @@ -86,52 +92,41 @@ static IceServer GetIceServerFromUrl(const std::string &url) { return srv; } -static webrtc::PeerConnectionFactoryDependencies -CreatePeerConnectionFactoryDependencies() { - // Disable WebRTC's pacing delay and allow the pacer to drain its queue - // immediately. This reduces the added latency from packet smoothing, which - // is unnecessary for a local interactive streaming use case. - // Force playout delay to zero in the RTP header extension so the receiver - // renders frames immediately rather than buffering for jitter smoothing. - // Disable automatic resolution reduction so quality is only degraded if the - // encoder is genuinely overloaded (content hint kFluid still allows it). - webrtc::field_trial::InitFieldTrialsFromString( - "WebRTC-Pacer-DrainQueue/Enabled/" - "WebRTC-ForceSendPlayoutDelay/min_ms:0,max_ms:0/" - "WebRTC-Video-DisableAutomaticResize/Enabled/"); +static bool PeerConnectionHasStreamForWindow( + webrtc::PeerConnectionInterface* peer_connection, + const std::string& window_uid) { + if (!peer_connection) { + return false; + } + for (const auto& sender : peer_connection->GetSenders()) { + if (!sender) { + continue; + } + for (const std::string& stream_id : sender->stream_ids()) { + if (stream_id == window_uid) { + return true; + } + } + } + return false; +} +static webrtc::PeerConnectionFactoryDependencies +CreatePeerConnectionFactoryDependencies( + webrtc::FieldTrials* field_trials) { + (void)field_trials; webrtc::PeerConnectionFactoryDependencies dependencies; + dependencies.worker_thread = webrtc::Thread::Current(); dependencies.network_thread = nullptr; - dependencies.worker_thread = rtc::Thread::Current(); dependencies.signaling_thread = nullptr; - dependencies.call_factory = webrtc::CreateCallFactory(); - dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); - dependencies.event_log_factory = - absl::make_unique( - dependencies.task_queue_factory.get()); - - cricket::MediaEngineDependencies media_dependencies; - media_dependencies.task_queue_factory = - dependencies.task_queue_factory.get(); - - // Dummy audio factory. - rtc::scoped_refptr audio_device_module( + + webrtc::EnvironmentFactory env_factory; + env_factory.Set(field_trials); + dependencies.env = env_factory.Create(); + + dependencies.adm = webrtc::scoped_refptr( new webrtc::FakeAudioDeviceModule()); - media_dependencies.adm = std::move(audio_device_module); - media_dependencies.audio_encoder_factory = - webrtc::CreateBuiltinAudioEncoderFactory(); - media_dependencies.audio_decoder_factory = - webrtc::CreateBuiltinAudioDecoderFactory(); - media_dependencies.audio_processing = - webrtc::AudioProcessingBuilder().Create(); - - media_dependencies.video_encoder_factory = - webrtc::CreateBuiltinVideoEncoderFactory(); - media_dependencies.video_decoder_factory = - webrtc::CreateBuiltinVideoDecoderFactory(); - - dependencies.media_engine = - cricket::CreateMediaEngine(std::move(media_dependencies)); + webrtc::EnableMediaWithDefaults(dependencies); return dependencies; } @@ -141,12 +136,16 @@ PeerConnectionManager::PeerConnectionManager( const Json::Value &config, const std::string &publish_filter, const std::string &webrtc_udp_port_range) - : task_queue_factory_(webrtc::CreateDefaultTaskQueueFactory()), + : field_trials_(webrtc::FieldTrials::Create( + "WebRTC-Pacer-DrainQueue/Enabled/" + "WebRTC-ForceSendPlayoutDelay/min_ms:0,max_ms:0/" + "WebRTC-Video-DisableAutomaticResize/Enabled/")), peer_connection_factory_(webrtc::CreateModularPeerConnectionFactory( - CreatePeerConnectionFactoryDependencies())), + CreatePeerConnectionFactoryDependencies(field_trials_.get()))), ice_server_list_(ice_server_list), config_(config), publish_filter_(publish_filter) { + webrtc_worker_thread_ = webrtc::Thread::Current(); // Set the webrtc port range. webrtc_port_range_ = webrtc_udp_port_range; @@ -260,9 +259,9 @@ const Json::Value PeerConnectionManager::GetIceServers() { } // Get PeerConnection associated with peerid. -rtc::scoped_refptr +webrtc::scoped_refptr PeerConnectionManager::GetPeerConnection(const std::string &peerid) { - rtc::scoped_refptr peer_connection; + webrtc::scoped_refptr peer_connection; auto it = peerid_to_connection_.find(peerid); if (it != peerid_to_connection_.end()) { peer_connection = it->second->GetPeerConnection(); @@ -277,12 +276,12 @@ const Json::Value PeerConnectionManager::AddIceCandidate( std::string sdp_mid; int sdp_mlineindex = 0; std::string sdp; - if (!rtc::GetStringFromJsonObject(json_message, k_candidate_sdp_mid_name, + if (!webrtc::GetStringFromJsonObject(json_message, k_candidate_sdp_mid_name, &sdp_mid) || - !rtc::GetIntFromJsonObject(json_message, + !webrtc::GetIntFromJsonObject(json_message, k_candidate_sdp_mline_index_name, &sdp_mlineindex) || - !rtc::GetStringFromJsonObject(json_message, k_candidate_sdp_name, + !webrtc::GetStringFromJsonObject(json_message, k_candidate_sdp_name, &sdp)) { utility::LogWarning("Can't parse received message."); } else { @@ -304,7 +303,7 @@ const Json::Value PeerConnectionManager::AddIceCandidate( } else { std::lock_guard mutex_lock( peerid_to_connection_mutex_); - rtc::scoped_refptr + webrtc::scoped_refptr peer_connection = this->GetPeerConnection(peerid); if (peer_connection) { if (!peer_connection->AddIceCandidate(candidate.get())) { @@ -334,9 +333,9 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, std::string type; std::string sdp; - if (!rtc::GetStringFromJsonObject(json_message, + if (!webrtc::GetStringFromJsonObject(json_message, k_session_description_type_name, &type) || - !rtc::GetStringFromJsonObject(json_message, + !webrtc::GetStringFromJsonObject(json_message, k_session_description_sdp_name, &sdp)) { utility::LogWarning("Can't parse received message."); } else { @@ -348,12 +347,11 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, utility::LogError("Failed to initialize PeerConnection"); delete peer_connection_observer; } else { - rtc::scoped_refptr - peer_connection = - peer_connection_observer->GetPeerConnection(); - utility::LogDebug("nbStreams local: {}, remote: {}", - peer_connection->local_streams()->count(), - peer_connection->remote_streams()->count()); + webrtc::PeerConnectionInterface* peer_connection_ptr = + peer_connection_observer->GetPeerConnection().get(); + utility::LogDebug("nbSenders: {}, nbReceivers: {}", + peer_connection_ptr->GetSenders().size(), + peer_connection_ptr->GetReceivers().size()); // Register peerid. { @@ -371,19 +369,27 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, } // Set remote offer. - webrtc::SessionDescriptionInterface *session_description( - webrtc::CreateSessionDescription(type, sdp, nullptr)); + std::optional sdp_type = + webrtc::SdpTypeFromString(type); + std::unique_ptr + session_description; + if (!sdp_type) { + utility::LogError("Unknown session description type: {}.", type); + } else { + session_description = + webrtc::CreateSessionDescription(*sdp_type, sdp); + } if (!session_description) { utility::LogError( "Can't parse received session description message. " "Cannot create session description."); } else { - std::promise + std::promise remote_promise; - peer_connection->SetRemoteDescription( - SetSessionDescriptionObserver::Create(peer_connection, - remote_promise), - session_description); + peer_connection_ptr->SetRemoteDescription( + SetSessionDescriptionObserver::Create( + peer_connection_ptr, remote_promise), + session_description.release()); // Waiting for remote description. std::future remote_future = remote_promise.get_future(); @@ -398,7 +404,7 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, } // Add local stream. - if (!this->AddStreams(peer_connection, window_uid, options)) { + if (!this->AddStreams(peer_connection_ptr, window_uid, options)) { utility::LogError("Can't add stream {}, {}.", window_uid, options); } @@ -407,9 +413,9 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions rtc_options; std::promise local_promise; - peer_connection->CreateAnswer( - CreateSessionDescriptionObserver::Create(peer_connection, - local_promise), + peer_connection_ptr->CreateAnswer( + CreateSessionDescriptionObserver::Create( + peer_connection_ptr, local_promise), rtc_options); // Waiting for answer. @@ -438,26 +444,20 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, } bool PeerConnectionManager::WindowStillUsed(const std::string &window_uid) { - bool still_used = false; for (auto it : peerid_to_connection_) { - rtc::scoped_refptr peer_connection = - it.second->GetPeerConnection(); - rtc::scoped_refptr local_streams( - peer_connection->local_streams()); - for (unsigned int i = 0; i < local_streams->count(); i++) { - if (local_streams->at(i)->id() == window_uid) { - still_used = true; - break; - } + if (PeerConnectionHasStreamForWindow( + it.second->GetPeerConnection().get(), window_uid)) { + return true; } } - return still_used; + return false; } // Hangup a call. const Json::Value PeerConnectionManager::HangUp(const std::string &peerid) { bool result = false; PeerConnectionObserver *pc_observer = nullptr; + std::string hangup_window_uid; { std::lock_guard mutex_lock(peerid_to_connection_mutex_); auto it = peerid_to_connection_.find(peerid); @@ -469,37 +469,27 @@ const Json::Value PeerConnectionManager::HangUp(const std::string &peerid) { if (peerid_to_window_uid_.count(peerid) != 0) { std::lock_guard mutex_lock( window_uid_to_peerids_mutex_); - const std::string window_uid = peerid_to_window_uid_.at(peerid); + hangup_window_uid = peerid_to_window_uid_.at(peerid); peerid_to_window_uid_.erase(peerid); // After window_uid_to_peerids_[window_uid] becomes empty, we don't // remove the window_uid from the map here. We remove window_uid // from window_uid_to_peerids_ when the Window is closed. - window_uid_to_peerids_[window_uid].erase(peerid); + window_uid_to_peerids_[hangup_window_uid].erase(peerid); } if (pc_observer) { - rtc::scoped_refptr - peer_connection = pc_observer->GetPeerConnection(); - - rtc::scoped_refptr local_streams( - peer_connection->local_streams()); - for (unsigned int i = 0; i < local_streams->count(); i++) { - auto stream = local_streams->at(i); - - std::string window_uid = stream->id(); - bool still_used = this->WindowStillUsed(window_uid); - if (!still_used) { - std::lock_guard mlock( - window_uid_to_track_source_mutex_); - auto it = window_uid_to_track_source_.find(window_uid); - if (it != window_uid_to_track_source_.end()) { - window_uid_to_track_source_.erase(it); - } - utility::LogDebug("HangUp stream closed {}.", window_uid); + if (!hangup_window_uid.empty() && + !this->WindowStillUsed(hangup_window_uid)) { + std::lock_guard mlock( + window_uid_to_track_source_mutex_); + auto track_it = + window_uid_to_track_source_.find(hangup_window_uid); + if (track_it != window_uid_to_track_source_.end()) { + window_uid_to_track_source_.erase(track_it); } - - peer_connection->RemoveStream(stream); + utility::LogDebug("HangUp stream closed {}.", + hangup_window_uid); } delete pc_observer; @@ -540,9 +530,35 @@ bool PeerConnectionManager::InitializePeerConnection() { return (peer_connection_factory_.get() != nullptr); } +PeerConnectionManager::PeerConnectionObserver::PeerConnectionObserver( + PeerConnectionManager* peer_connection_manager, + const std::string& peerid) + : peer_connection_manager_(peer_connection_manager), + peerid_(peerid), + local_channel_(nullptr), + remote_channel_(nullptr), + ice_candidate_list_(Json::arrayValue), + deleting_(false) { + stats_callback_ = + new webrtc::RefCountedObject(); +} + +void PeerConnectionManager::PeerConnectionObserver::Initialize( + webrtc::scoped_refptr peer_connection) { + pc_ = peer_connection; + if (pc_.get()) { + auto channel_result = + pc_->CreateDataChannelOrError("ServerDataChannel", nullptr); + if (channel_result.ok()) { + local_channel_ = new DataChannelObserver( + peer_connection_manager_, channel_result.value(), peerid_); + } + } +} + // Create a new PeerConnection. -PeerConnectionManager::PeerConnectionObserver * -PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { +PeerConnectionManager::PeerConnectionObserver* +PeerConnectionManager::CreatePeerConnection(const std::string& peerid) { webrtc::PeerConnectionInterface::RTCConfiguration config; // Max bundle multiplexes all media and data channels on a single transport, // eliminating separate ICE/DTLS handshakes per track and reducing latency. @@ -569,24 +585,30 @@ PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { max_port = std::stoi(port); } } - std::unique_ptr port_allocator( - new cricket::BasicPortAllocator(new rtc::BasicNetworkManager())); - port_allocator->SetPortRange(min_port, max_port); + config.set_min_port(min_port); + config.set_max_port(max_port); utility::LogDebug("CreatePeerConnection webrtcPortRange: {}:{}.", min_port, max_port); utility::LogDebug("CreatePeerConnection peerid: {}.", peerid); - PeerConnectionObserver *obs = new PeerConnectionObserver( - this, peerid, config, std::move(port_allocator)); - if (!obs) { - utility::LogError("CreatePeerConnection failed."); - } else { - utility::LogDebug("CreatePeerConnection success!"); + + PeerConnectionObserver* obs = + new PeerConnectionObserver(this, peerid); + webrtc::PeerConnectionDependencies dependencies(obs); + auto pc_result = peer_connection_factory_->CreatePeerConnectionOrError( + config, std::move(dependencies)); + if (!pc_result.ok()) { + utility::LogError("CreatePeerConnection failed: {}.", + pc_result.error().message()); + delete obs; + return nullptr; } + obs->Initialize(pc_result.MoveValue()); + utility::LogDebug("CreatePeerConnection success!"); return obs; } // Get the capturer from its URL. -rtc::scoped_refptr +webrtc::scoped_refptr PeerConnectionManager::CreateVideoSource( const std::string &window_uid, const std::map &opts) { @@ -652,52 +674,42 @@ bool PeerConnectionManager::AddStreams( if (!existing_stream) { // Create a new stream and add to window_uid_to_track_source_. - rtc::scoped_refptr video_source( + webrtc::scoped_refptr video_source( this->CreateVideoSource(video, opts)); std::lock_guard mlock(window_uid_to_track_source_mutex_); window_uid_to_track_source_[window_uid] = video_source; } - // AddTrack and AddStream to peer_connection. + // Add local video track (Unified Plan). { std::lock_guard mlock(window_uid_to_track_source_mutex_); auto it = window_uid_to_track_source_.find(window_uid); if (it != window_uid_to_track_source_.end()) { - rtc::scoped_refptr stream = - peer_connection_factory_->CreateLocalMediaStream( - window_uid); - if (!stream.get()) { - utility::LogError("Cannot create stream."); + webrtc::scoped_refptr video_source = + it->second; + webrtc::scoped_refptr video_track; + if (!video_source) { + utility::LogError("Cannot create capturer video: {}.", + window_uid); } else { - rtc::scoped_refptr video_source = - it->second; - rtc::scoped_refptr video_track; - if (!video_source) { - utility::LogError("Cannot create capturer video: {}.", - window_uid); - } else { - rtc::scoped_refptr videoScaled = - VideoFilter::Create(video_source, - opts); - video_track = peer_connection_factory_->CreateVideoTrack( - window_uid + "_video", videoScaled); - // Prefer framerate over resolution when the encoder - // is under pressure (bandwidth or CPU constrained). - // For interactive 3D rendering, motion smoothness - // matters more than pixel-perfect resolution. - video_track->set_content_hint( - webrtc::VideoTrackInterface::ContentHint::kFluid); - } - - if ((video_track) && (!stream->AddTrack(video_track))) { - utility::LogError( - "Adding VideoTrack to MediaStream failed."); - } + webrtc::scoped_refptr videoScaled = + VideoFilter::Create(video_source, opts); + video_track = peer_connection_factory_->CreateVideoTrack( + videoScaled, window_uid + "_video"); + video_track->set_content_hint( + webrtc::VideoTrackInterface::ContentHint::kFluid); + } - if (!peer_connection->AddStream(stream)) { - utility::LogError("Adding stream to PeerConnection failed"); + if (video_track) { + webrtc::RTCErrorOr> + add_result = peer_connection->AddTrack( + video_track, {window_uid}); + if (!add_result.ok()) { + utility::LogError("Adding track to PeerConnection failed: {}", + add_result.error().message()); } else { - utility::LogDebug("Stream added to PeerConnection."); + utility::LogDebug("Track added to PeerConnection."); ret = true; } } @@ -725,7 +737,7 @@ void PeerConnectionManager::PeerConnectionObserver::OnIceCandidate( } } -rtc::scoped_refptr +webrtc::scoped_refptr PeerConnectionManager::GetVideoTrackSource(const std::string &window_uid) { { std::lock_guard mlock(window_uid_to_track_source_mutex_); @@ -763,8 +775,8 @@ void PeerConnectionManager::CloseWindowConnections( } // Encoder thread: wakes on each new frame, drains the per-window latest-frame -// map, and calls video_track_source->OnFrame() (libyuv + WebRTC encode) -// off the render thread so frame delivery never blocks GUI redraws. +// map, and posts OnFrame to the WebRTC worker thread. libyuv conversion and +// VideoBroadcaster must run on the worker thread (same as PCM creation). void PeerConnectionManager::EncoderThreadLoop() { while (encoder_running_) { std::unordered_map> snapshot; @@ -779,11 +791,23 @@ void PeerConnectionManager::EncoderThreadLoop() { // encode only the latest per window (implicit frame coalescing). snapshot = std::move(pending_frames_); } - for (const auto &kv : snapshot) { - auto video_track_source = GetVideoTrackSource(kv.first); - if (video_track_source && kv.second) { - video_track_source->OnFrame(kv.second); + webrtc::Thread* worker = webrtc_worker_thread_; + if (!worker) { + continue; + } + for (const auto& kv : snapshot) { + const std::shared_ptr& frame = kv.second; + if (!frame) { + continue; } + webrtc::scoped_refptr track_source = + GetVideoTrackSource(kv.first); + if (!track_source) { + continue; + } + worker->PostTask([track_source, frame]() { + track_source->OnFrame(frame); + }); } } } diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h index 72a005e96c4..2c85a41a40e 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h @@ -18,7 +18,11 @@ #pragma once +#include +#include #include +#include +#include #include #include @@ -34,6 +38,10 @@ #include "open3d/visualization/webrtc_server/HttpServerRequestHandler.h" #include "open3d/visualization/webrtc_server/WebRTCWindowSystem.h" +namespace webrtc { +class Thread; +} + namespace open3d { namespace visualization { namespace webrtc_server { @@ -73,23 +81,23 @@ namespace webrtc_server { /// /// TODO (yixing): Use PImpl. class PeerConnectionManager { - class VideoSink : public rtc::VideoSinkInterface { + class VideoSink : public webrtc::VideoSinkInterface { public: VideoSink(webrtc::VideoTrackInterface* track) : track_(track) { - track_->AddOrUpdateSink(this, rtc::VideoSinkWants()); + track_->AddOrUpdateSink(this, webrtc::VideoSinkWants()); } virtual ~VideoSink() { track_->RemoveSink(this); } // VideoSinkInterface implementation virtual void OnFrame(const webrtc::VideoFrame& video_frame) { - rtc::scoped_refptr buffer( + webrtc::scoped_refptr buffer( video_frame.video_frame_buffer()->ToI420()); utility::LogDebug("[{}] frame: {}x{}", OPEN3D_FUNCTION, buffer->height(), buffer->width()); } protected: - rtc::scoped_refptr track_; + webrtc::scoped_refptr track_; }; class SetSessionDescriptionObserver @@ -99,7 +107,7 @@ class PeerConnectionManager { webrtc::PeerConnectionInterface* pc, std::promise& promise) { - return new rtc::RefCountedObject( + return new webrtc::RefCountedObject( pc, promise); } virtual void OnSuccess() { @@ -136,7 +144,7 @@ class PeerConnectionManager { webrtc::PeerConnectionInterface* pc, std::promise& promise) { - return new rtc::RefCountedObject( + return new webrtc::RefCountedObject( pc, promise); } virtual void OnSuccess(webrtc::SessionDescriptionInterface* desc) { @@ -171,13 +179,12 @@ class PeerConnectionManager { protected: virtual void OnStatsDelivered( - const rtc::scoped_refptr& + const webrtc::scoped_refptr& report) { for (const webrtc::RTCStats& stats : *report) { Json::Value stats_members; - for (const webrtc::RTCStatsMemberInterface* member : - stats.Members()) { - stats_members[member->name()] = member->ValueToString(); + for (const webrtc::Attribute& attribute : stats.Attributes()) { + stats_members[attribute.name()] = attribute.ToString(); } report_[stats.id()] = stats_members; } @@ -190,7 +197,7 @@ class PeerConnectionManager { public: DataChannelObserver( PeerConnectionManager* peer_connection_manager, - rtc::scoped_refptr data_channel, + webrtc::scoped_refptr data_channel, const std::string& peerid) : peer_connection_manager_(peer_connection_manager), data_channel_(data_channel), @@ -251,38 +258,18 @@ class PeerConnectionManager { protected: PeerConnectionManager* peer_connection_manager_; - rtc::scoped_refptr data_channel_; + webrtc::scoped_refptr data_channel_; const std::string peerid_; }; class PeerConnectionObserver : public webrtc::PeerConnectionObserver { public: - PeerConnectionObserver( - PeerConnectionManager* peer_connection_manager, - const std::string& peerid, - const webrtc::PeerConnectionInterface::RTCConfiguration& config, - std::unique_ptr port_allocator) - : peer_connection_manager_(peer_connection_manager), - peerid_(peerid), - local_channel_(nullptr), - remote_channel_(nullptr), - ice_candidate_list_(Json::arrayValue), - deleting_(false) { - pc_ = peer_connection_manager_->peer_connection_factory_ - ->CreatePeerConnection(config, - std::move(port_allocator), - nullptr, this); + PeerConnectionObserver(PeerConnectionManager* peer_connection_manager, + const std::string& peerid); - if (pc_.get()) { - rtc::scoped_refptr channel = - pc_->CreateDataChannel("ServerDataChannel", nullptr); - local_channel_ = new DataChannelObserver( - peer_connection_manager_, channel, peerid_); - } - - stats_callback_ = new rtc::RefCountedObject< - PeerConnectionStatsCollectorCallback>(); - }; + void Initialize( + webrtc::scoped_refptr + peer_connection); virtual ~PeerConnectionObserver() { delete local_channel_; @@ -298,7 +285,7 @@ class PeerConnectionManager { Json::Value GetStats() { stats_callback_->clearReport(); - pc_->GetStats(stats_callback_); + pc_->GetStats(stats_callback_.get()); int count = 10; while ((stats_callback_->getReport().empty()) && (--count > 0)) { std::this_thread::sleep_for(std::chrono::milliseconds(1000)); @@ -306,27 +293,27 @@ class PeerConnectionManager { return Json::Value(stats_callback_->getReport()); }; - rtc::scoped_refptr + webrtc::scoped_refptr GetPeerConnection() { return pc_; }; // PeerConnectionObserver interface virtual void OnAddStream( - rtc::scoped_refptr stream) { + webrtc::scoped_refptr stream) { utility::LogDebug("[{}] GetVideoTracks().size(): {}.", OPEN3D_FUNCTION, stream->GetVideoTracks().size()); webrtc::VideoTrackVector videoTracks = stream->GetVideoTracks(); if (videoTracks.size() > 0) { - video_sink_.reset(new VideoSink(videoTracks.at(0))); + video_sink_.reset(new VideoSink(videoTracks.at(0).get())); } } virtual void OnRemoveStream( - rtc::scoped_refptr stream) { + webrtc::scoped_refptr stream) { video_sink_.reset(); } virtual void OnDataChannel( - rtc::scoped_refptr channel) { + webrtc::scoped_refptr channel) { utility::LogDebug( "PeerConnectionObserver::OnDataChannel peerid: {}", peerid_); @@ -369,11 +356,11 @@ class PeerConnectionManager { private: PeerConnectionManager* peer_connection_manager_; const std::string peerid_; - rtc::scoped_refptr pc_; + webrtc::scoped_refptr pc_; DataChannelObserver* local_channel_; DataChannelObserver* remote_channel_; Json::Value ice_candidate_list_; - rtc::scoped_refptr + webrtc::scoped_refptr stats_callback_; std::unique_ptr video_sink_; bool deleting_; @@ -409,22 +396,22 @@ class PeerConnectionManager { const std::shared_ptr& im); protected: - rtc::scoped_refptr GetVideoTrackSource( + webrtc::scoped_refptr GetVideoTrackSource( const std::string& window_uid); PeerConnectionObserver* CreatePeerConnection(const std::string& peerid); bool AddStreams(webrtc::PeerConnectionInterface* peer_connection, const std::string& window_uid, const std::string& options); - rtc::scoped_refptr CreateVideoSource( + webrtc::scoped_refptr CreateVideoSource( const std::string& window_uid, const std::map& opts); bool WindowStillUsed(const std::string& window_uid); - rtc::scoped_refptr GetPeerConnection( + webrtc::scoped_refptr GetPeerConnection( const std::string& peerid); protected: - std::unique_ptr task_queue_factory_; - rtc::scoped_refptr + std::unique_ptr field_trials_; + webrtc::scoped_refptr peer_connection_factory_; // Each peer has exactly one connection. @@ -437,7 +424,7 @@ class PeerConnectionManager { // Each Window has exactly one TrackSource. std::unordered_map> + webrtc::scoped_refptr> window_uid_to_track_source_; std::mutex window_uid_to_track_source_mutex_; @@ -456,13 +443,16 @@ class PeerConnectionManager { // Async encoder thread: decouples the render thread from the blocking // libyuv + WebRTC encode path. OnFrame() posts the latest frame per window; - // the thread drains the map and calls video_track_source->OnFrame(). + // the thread drains the map and posts video_track_source->OnFrame() to the + // WebRTC worker thread. std::unordered_map> pending_frames_; std::mutex pending_frames_mutex_; std::condition_variable pending_frames_cv_; std::atomic encoder_running_{false}; std::thread encoder_thread_; + // WebRTC worker thread (PeerConnectionFactoryDependencies::worker_thread). + webrtc::Thread* webrtc_worker_thread_ = nullptr; void EncoderThreadLoop(); }; diff --git a/cpp/open3d/visualization/webrtc_server/VideoFilter.h b/cpp/open3d/visualization/webrtc_server/VideoFilter.h index 16588c96a2a..94a95c0099a 100644 --- a/cpp/open3d/visualization/webrtc_server/VideoFilter.h +++ b/cpp/open3d/visualization/webrtc_server/VideoFilter.h @@ -18,7 +18,8 @@ #pragma once -#include +#include +#include #include "open3d/visualization/webrtc_server/BitmapTrackSource.h" @@ -34,14 +35,15 @@ namespace webrtc_server { template class VideoFilter : public BitmapTrackSource { public: - static rtc::scoped_refptr Create( - rtc::scoped_refptr video_source, + static webrtc::scoped_refptr Create( + webrtc::scoped_refptr video_source, const std::map& opts) { std::unique_ptr source = absl::WrapUnique(new T(video_source, opts)); if (!source) { return nullptr; } - return new rtc::RefCountedObject(std::move(source)); + return webrtc::scoped_refptr( + new webrtc::RefCountedObject(std::move(source))); } protected: @@ -61,7 +63,7 @@ class VideoFilter : public BitmapTrackSource { } private: - rtc::VideoSourceInterface* source() override { + webrtc::VideoSourceInterface* source() override { return source_.get(); } std::unique_ptr source_; diff --git a/cpp/open3d/visualization/webrtc_server/VideoScaler.h b/cpp/open3d/visualization/webrtc_server/VideoScaler.h index cd57e3dc898..fcebdda0bbc 100644 --- a/cpp/open3d/visualization/webrtc_server/VideoScaler.h +++ b/cpp/open3d/visualization/webrtc_server/VideoScaler.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "open3d/visualization/webrtc_server/BitmapTrackSource.h" @@ -27,10 +28,10 @@ namespace open3d { namespace visualization { namespace webrtc_server { -class VideoScaler : public rtc::VideoSinkInterface, - public rtc::VideoSourceInterface { +class VideoScaler : public webrtc::VideoSinkInterface, + public webrtc::VideoSourceInterface { public: - VideoScaler(rtc::scoped_refptr video_source, + VideoScaler(webrtc::scoped_refptr video_source, const std::map &opts) : video_source_(video_source), width_(0), @@ -148,7 +149,7 @@ class VideoScaler : public rtc::VideoSinkInterface, } else if (width == 0) { width = (roi_width_ * height) / roi_height_; } - rtc::scoped_refptr scaled_buffer = + webrtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(width, height); if (roi_width_ != frame.width() || roi_height_ != frame.height()) { scaled_buffer->CropAndScaleFrom( @@ -158,22 +159,22 @@ class VideoScaler : public rtc::VideoSinkInterface, scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); } webrtc::VideoFrame scaledFrame = - webrtc::VideoFrame(scaled_buffer, frame.timestamp(), + webrtc::VideoFrame(scaled_buffer, frame.timestamp_us(), frame.render_time_ms(), rotation_); broadcaster_.OnFrame(scaledFrame); } } - void AddOrUpdateSink(rtc::VideoSinkInterface *sink, - const rtc::VideoSinkWants &wants) override { + void AddOrUpdateSink(webrtc::VideoSinkInterface *sink, + const webrtc::VideoSinkWants &wants) override { video_source_->AddOrUpdateSink(this, wants); broadcaster_.AddOrUpdateSink(sink, wants); } void RemoveSink( - rtc::VideoSinkInterface *sink) override { + webrtc::VideoSinkInterface *sink) override { video_source_->RemoveSink(this); broadcaster_.RemoveSink(sink); @@ -183,8 +184,8 @@ class VideoScaler : public rtc::VideoSinkInterface, int height() { return roi_height_; } private: - rtc::scoped_refptr video_source_; - rtc::VideoBroadcaster broadcaster_; + webrtc::scoped_refptr video_source_; + webrtc::VideoBroadcaster broadcaster_; int width_; int height_; diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index c703ac5f66a..e2d87b60169 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -7,13 +7,12 @@ #include "open3d/visualization/webrtc_server/WebRTCWindowSystem.h" -#include -#include -#include #include +#include #include #include +#include #include #include #include @@ -103,6 +102,8 @@ struct WebRTCWindowSystem::Impl { std::thread webrtc_thread_; bool sever_started_ = false; + // Set while the WebRTC std::thread is inside Run(); used for shutdown. + std::atomic webrtc_message_thread_{nullptr}; std::unordered_map> data_channel_message_callbacks_; @@ -197,8 +198,14 @@ WebRTCWindowSystem::WebRTCWindowSystem() } WebRTCWindowSystem::~WebRTCWindowSystem() { + if (impl_->sever_started_ && impl_->webrtc_thread_.joinable()) { + webrtc::Thread* message_thread = impl_->webrtc_message_thread_.load(); + if (message_thread) { + message_thread->Quit(); + } + impl_->webrtc_thread_.join(); + } impl_->peer_connection_manager_ = nullptr; - rtc::Thread::Current()->Quit(); } WebRTCWindowSystem::OSWindow WebRTCWindowSystem::CreateOSWindow( @@ -262,16 +269,25 @@ void WebRTCWindowSystem::StartWebRTCServer() { gui::Application::GetInstance().GetResourcePath()); impl_->web_root_ = resource_path + "/html"; - // Logging settings. - // src/rtc_base/logging.h: LS_VERBOSE, LS_ERROR - rtc::LogMessage::LogToDebug((rtc::LoggingSeverity)rtc::LS_ERROR); - - rtc::LogMessage::LogTimestamps(); - rtc::LogMessage::LogThreads(); + // Logging settings (M149: rtc_base/logging.h). + webrtc::LoggingConfig log_config; + log_config.set_debug_severity(webrtc::LS_ERROR); + log_config.set_log_thread(true); + log_config.set_log_timestamp(true); + webrtc::InitializeLogging(std::move(log_config)); + + // Associate this std::thread with WebRTC's message loop (required + // before Thread::Current()->Run() and PeerConnectionFactory). + webrtc::ThreadManager::Instance()->WrapCurrentThread(); + struct WebRtcThreadScope { + ~WebRtcThreadScope() { + webrtc::ThreadManager::Instance()->UnwrapCurrentThread(); + } + } webrtc_thread_scope; + webrtc::Thread* thread = webrtc::Thread::Current(); + impl_->webrtc_message_thread_.store(thread); - // PeerConnectionManager manages all WebRTC connections. - rtc::Thread *thread = rtc::Thread::Current(); - rtc::InitializeSSL(); + webrtc::InitializeSSL(); Json::Value config; std::list ice_servers; ice_servers.insert(ice_servers.end(), s_public_ice_servers.begin(), @@ -345,7 +361,8 @@ void WebRTCWindowSystem::StartWebRTCServer() { utility::LogInfo("WebRTC Jupyter handshake mode enabled."); thread->Run(); } - rtc::CleanupSSL(); + impl_->webrtc_message_thread_.store(nullptr); + webrtc::CleanupSSL(); }; impl_->webrtc_thread_ = std::thread(start_webrtc_thread); impl_->sever_started_ = true; diff --git a/cpp/open3d/visualization/webrtc_server/html/index.html b/cpp/open3d/visualization/webrtc_server/html/index.html index 995ce95dea5..b318932f9f8 100644 --- a/cpp/open3d/visualization/webrtc_server/html/index.html +++ b/cpp/open3d/visualization/webrtc_server/html/index.html @@ -1,4 +1,5 @@ - + + Open3D WebVisualizer @@ -136,6 +137,7 @@ videoElt.muted = true; videoElt.controls = false; videoElt.playsinline = true; + videoElt.preload = "none"; videoElt.innerText = "Your browser does not support HTML5 video."; divElt.appendChild(videoElt); diff --git a/cpp/open3d/visualization/webrtc_server/html/open3d_logo.ico b/cpp/open3d/visualization/webrtc_server/html/open3d_logo.ico deleted file mode 100644 index ce428261c26acc89e930a65e01f8370b3fee5344..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34494 zcmd6Q2bdJa)^^YAE-WmYW|mFBGUNY5{B&grU{!xaOH zpnw6CpnwR7C}sr}t_cHB-}_c|byxRv&kV@_KewK(t*(5|sZ;6HsT#|QSUFZvkp(wl zos?}^hggO96bJG%PRgDdDbGFb`9)u*x?bYLRDv%Q@_q|D=S)3hPbyaD<sptZWibEQsBe2kd0TPI&3B78-&`o>&wpDy z@x*iD!3Q1{!-tO%EF1a%f;}l>Rm%Libz4}ubt~GVo==cpoPYjhV#kgh!Zf$vcApq~ z_z9vJ;9(l%UkMnRMXf5UL;Jotb#lsX1>T~%dXV`1^DoV;;*(E46>VDg6ipg;5QiUj zyvWHZMScm#l@Y5jT3C?SKN>0d0WgS%{{6ToFX1@v+)HJhtatAb2aBFP_JqHPFnkR= z2>ITnzk9bm#DWFyh-J%Gh-FKci}&AODgN`Hmqnw>_ICc5C@M^d@#9Vv4H`5-SPV2R znFG9EV*YMjYs6b`EfOnMtP;zYe<)V2{7Ag?(rnSBNhc?Nthl)tKOXt>nn)aS>sBrR zE`3qf7j^5Di~H|;!~;J)7hQCftc&m;F!Bh|qx%r}i$qqm;djMF&Gw2|`4QlO#w-$@ zI`$Wze)_qmj13#U5FI-770sJfi=&P_N#tZVB<`Y4PWiOny@q5(Eb?8fDZ*B;j&r75 zEVggo?kVH``)7y~jz8ULKj6Kle!WJm^6IydJP@(U$-5Jf|F6>0*5cZ0Zx(asE)cV4 zzajqp;A7&RyB`uGMv#AG+iwFuPO_}1!+%*;55iE4GG3GXTdz`_0RC;#n6#607Lf?s z95xbv131JxHv7WKj}OqGj4+G<9hUa#Gwj1LV~$yc?~f3+25qz!Fs}oTuY$X8)aG60 zGb}vpn}wxg$DUBpy?cK!cwr;tX@a(?0=|j-22HB+>R09%c_ImT63A5pbZBekNPxfq z!#JIec@jv&aRx(`h2hBJ_$3X-tQ6x6YtUAu-}(PKf2bR^nxT?7#;*98Xt#4OzUbX9e=bFL$i>t1h zE^>1#WScWQ!0tfUR2$vU@}pK`j%Ndrb|cFm4`pS=#MG%*i2e6J9R31XC;N=Day-%X zZ2%5kqE>>{1l3^y#s=W2XLQ+l^&6pob(Q1GBab{Ox>gSsEn9RKty=aFty}dJu~>5n z1Jm2~Uho>`3Tb?G6k0jirDKt9tK@gUJ_UX9>#x6+_Sav36W@QgQGE5)H{zRbz7rpQ zxLRC!#SNl9U?84$`CH(QV84WM%5T$Rf|cK(suHw$m+6Rmb(evHV?>y_r1n~I;)FBB zfuoL;>Fx4a&#QosQzVV7Lx|rv7hHh29fYNB-E#2&#*Q$vV)-gjh4Har!&c&|E2leS ztz8G_8Ss6_2&-jIo3516T^h$dO~k&p-bXSp46A|1Bn-e71y{c<#6FVdB(N z&auZ4ryW+Hjs)kjVSA0P3mV?dx;aiYENLm`&wD$tYz8u}B;Hb{C9d_rw-;Z0wdmHZ zM%HQLKssH{d8)ju?LpaDr9UayRh#<9SzXz0zx^)8j2Y)Chd6ZYI#|?Re2u7w@zH4O zuTaOpy!wroD%fL!^USTc-YaIkJV(6v;%wW#DqV(ge9p^jY?fj3O-!6|`gvlXeGc{1 z9gUW}3VwgiEdS0s9}us<{-&5SXTHZWjO&Mj{Hmn#QHD7CtP92PVF!82$GG|y%I*Q$ ze5&YEEKUa;+qP}fB=NX^{PBl4^2ifCWsp7?_d%blMK1V`>vN{33Dap~s0+PX&KUxCwiVu&*sEZPU8Ew5tT)o@d9Sjz1Y+ z2Wug=Lzr2+W*x?%ZlW>PPB%`!Q*doZd`K6x#kP96jZSC5FtYzvL}M2RZTjJ!h)uN-5EQvwkmnJL0+R=yF6$7)PAnNQFb-#=emxZ>=JQF z?KNVATqkpWA>U~0mW}jZfi^v&k*Xlrjk>ZytN6vL{30=Q=z-$$%dZ!8>Xg|$;*4Rk zJ(-Rcw7IMxzrrX-wXKF-yIgyNPNm3m6yQAJl1r|gFmmMKCt+@!h_F*p&S_X_ z6YKM{F|Pj$_6&Si=I3=w1G`fv)9tJYguSlVVCBjkSrPu{S_y2%Xltyfw6(Iz{B z#5kQ!ubMSIm>^umt=yuo09j4Q96t*I^oFmG7VZ*JXvdEIlaBibi5NXj5zs zV~t|57L~8hT~Ik_;9l?-lp|bQ>=W7U(l&%kTVmTH%vD(-;vPRgSB0VZ>-zY-ZE1-^6HG|dXK#PCamlS zaz5xiOuRB{u4sWVP>*{$K81^UF4llVhH>yq^9|aUa6P=uEf;+rdzMQtxmM1%fY0u? zp0U8wckR4f_Z^|vJ!$HVWJjBFz1R-@bsh5A<#Bz)wQWAe{&wPp7hVyA2JH)f1K2{J zm3v`799SR5DzmJf>1be;#9L%zoVeA*o49hkI&s2eaolmI$$j+Px$~t=g8MpcPdf1| zu}95F?&pm8VOU9hJ8>@#;w>z7C9x$Mg?zm{vH_U-zB zCP{So3@~&~!HZVIl2!%a^A|MkoI|+g4cf=+IsOqL_C5ZKhnqq{Uqk ztBCuA0Q{pC_N3#1`yVFmBljOGIESV)@4WM#;9i~U8t(7a{yd2WoTGTMsTK^LEmMy!Br=EOXv}w~T zh!;Nq3{4~0*J2-54>GR@T^O-W#5(Yf+wTvo&mYMB7-dZwc3lqJ;lvXs%dsk3%82w> z;`Gf+F*igY!yAis-=gy~ANLDewgj(p&HA~A?b$=guXMfIWxGS3B_4X{2{Clf1A}Pr z1=98}$Ztwo0Djf>9BYm}b`tvB3~|pr4~5$MAY0?O!?xD2yXC0zI5v)ejPPN=KpCB% z7g5(N@Z95WKJX6e=Q#G%Q_qXXAAdH~vTe_tJe6zul=Y9oKAZg!eLJas(C2N`T@CAu z^`t-P_2Z8}nWPT<-nj7xF>v5ML9|6!u^bO5?;SY6p9bSvo$`&4XY3ud+MpB0&jAg$ zD>}rae0ll}w+Ghg4`lh~T3(f*_S|QmeUX%9kfvrEv7Z+gH^1rPi>}JC(;6GxJC1BZQHdOBFBJ;)sQs6 z938`4t@w~-oP5$*;;+B{l67*<37dCbe|>?7$6CmG^qg_~?f1#~k>%(z>0({64RYtn ztQba)!P(Yue8ln*{~wi>^i%!kl~?8_;p`9i>#x7j*Uv;ACI1?Az>(syN1u`MZ9mVW z-4~$GmtHb;iBmt}h*+hh!5*-fdv*UQ5|E>apMKixuQJL1!3Q5o*?0-$EYCjsqS$Nb zXbF!li!N~g1>^i^tYbWN+jKzvMOg2~Z*%j=J+rtCvX69zZ3CI6ly^^`euoGe^DxHG zXwtYNq_NUC^PgQmXdw529Z^5z7McddqFvkGV$JH0r5v4dC|cI{_ut#(H~rhTZ5Nw1 z{U8?1e+T=~AZ9mZ{2=@(#*ac;CH`ybC$nV!*Nl;asb|54#-nT><`DiE(xnaQz7Kh}GP$ z!7hh?XjUXO&*?gC7mhUO4~0?God;a=AwMm^`C(zto&&IpE-XSg&Ua$5?5vVF*6bya z`ITZ^E5rVwysWf!1>`f0aMo7du%tOoM$)%=NE!vnKl>lr*H~vn2zhdfj#0xS99l>c zc>G)ro0~u)*z6j0z!j}kxFFPPYT<>A)F2&g0(odz1|<_LQpe4d=PAe`Tw;Esy9`bhDWENUw@k}0*j^G27Qy`D1;vc6*zA#|Miz& z%Lfl0hK)yl8Tt+_WE@gnW6Ese>{rSb(`7$nf1@q5Z3CPWn`60#MY|?E{0)+-N?mF&A(hQ6_G=iv)TD8E5t$h}H#^49OI zk8^%J-a_g^a1BKL5&!x+RsMiDw(}&exia9F6zx%FVQsqy)&ZM9576FTd)O{vL?5Yyo_Fcvlr8DY z^LCzF(Y9~bSDwdnU$k=NYB6ZwzF434apHA4<@OWU>&QKQG7fGXsP}s8Jup3|9CwHx zjcy11ZDDPD3}|nxRTZ5`cb;8RUy6H!?{Lm9EojekZtfwt&v(LgJY4DoY}~j>a9>s3 zWsqz;mp*1Y$nWnEHy~n_XqYsN+6@eERyE2hE^6Kc>qU&!njYF!YySQBJ3J#MF+B z{rPf#sbSk0m+~v<5D1+0i|enyRm_8a1LeakH>f^44DzU0MXq&cx;C)7Ru8HNnbw0U zAKOn`#W$3}uU-4e&VcNsWnZBTS&do6qB&$Z??aD|x@=w^a(UH;pXW8uX_6)d@6?5? zzx7Da{=arUgC^=6J`#{pfJS+nMtAZ7ebe5l`~&Y!%zL40_I^~oon ziT(CF%!8i`hlWSWz=rzzvemxb0IPGS0j)uM&UpsUFrQlAoCKNSpMNTuVa9;l38`z! z`6{@NDp}aASo?8IrJOA|or42p7qLRs1gahGyYJy_=)g?F8dJBEt2u!+ti==Ffcuqd!0y{0^{Irxfa6TPp?};OY1ud=WE`tX`7yA3;^h#zuZQ5;V zK@Z}2W}oy}j zV$ioqh?z5I?F#5J59P&L2Rj&kU7w;W>9sG;sHjh*^m8~b2l5c{-T>PN`V-VqaOj}F z2|wCJo@<{n@f>l=#3|BFgd1cT?+HJ73iPr2rR5u4A6?c-y`O(Wzn$yYAl*7MJ?5#I zSYJ|?(LSSb@nCyGA5NTi-eu4ynkFv4?0Rt-EZt0PzrMrM!0*Mm$ znYiw{TT-5Xa_tAbxZ7@=en%a3&L;JOxMxN^UL2y)xYTV48z0l*#XA<^a&b*l>i);Y zWtU!uea%X_t_vS;C10%T+h=%(p56DA>j=9IMu85LU3h54a#;W3#qXyBH86h1_8nsC z1y>k!@s_V>51mw;uW;O%09}}H*rmSYHs}?cgUZbH4;DkQdMN7=Km59U-s$-L_um6y zOczMFcxNOCF7`9-2cCT5KjP(=)5|V!PWSe}f%{g>pSxAk9{(gBb{q5o9^bfpr29d! zz&QZMh37q>lBZ0oBQ>vSm*K&(EhWDg3mp^QC*nHJ+gH5&4B6PPpgpgR(Rk>u0t$Io z;hL~E7=S$=hVO9#*B_27};cNrQ?c;Cg{HDu#k304>LEZLUFHtC;=_Jm)?Q)UbLk4D~K{AIE23k#cyna}Uqx{i7i93y#8kn&#M z*GT3Y7Z>bF{=z!oWDOf!lV1YA>sXTse|^CIm>+)nSs(QgcXeIO@${2*pNf0so>$AS zT$s%|!S}PV&WTlbK3LK=1pcHrTYtVE%KFwUPk(c);Cha-6!t$_$`QWDI76L-&p!KH z`gNS?GSAy@y(_S14WPX{zk&9fvCi4Y3YQ;h`X|A!0*W}EUZ-I;T{Z7C!aAgbv|OvU z!W}BE+1`D3spyFLigz+~yw~k0N*lI@42!S^)17sJM)zXOsT)Fb4ST3-k43Fe7SFp- zm*1*=AIA=y;)%zhw`Gj+I?YbHJFqvI+PPD=jNvrk_TU?}TCgt%p4}<>vHfx9 z4{Nk9zW7S6!_;`9!d}2>*3C`0ap+%sp6eDyDyJHJ=iCEd5Ey` z9^q%W&wB;-Y3X=cMhI(t_?<8rU^HOh=>r+I7N4D7}|F)cv>#Kl?E6pmV>v zbm<48X;o**@9AOGZKLsUUCg!5LXIdE!F(`z&P082NiZAn5)WH%rWas)2KHsxS)lzZ zeftidjd@_s*=Juk7k=f`v)-01Tk7@eH>f_w zAhc18wgD{klw$eNcPqe}t+1%DX))FbvGTGuag3`a*b|jvEmey9t7W*mTG6g;uS)2% zl_i>XO1s7ZP^s`B%sSo!e?Yj7-<__mdw05okLA(ma>Kqejo0b^nH!}6C{2af0%uVI z0PcYgxIDPDYLqg8I(+A?VwkimU2!~fS^!fV19_4&;>^INC$=Qq=2@~!jx zUFQA&ZG!6bDnptSyYold(tPmAkTS?S%l`cH z&$4~?IiM8tA@<#--dwx{ep;!c9GdXu^YtN^}xe*KExzuJlW91H=yAj;yRxzY z@#o$RXA2G-9vCT`;(I5QLE;`R?j@#tS0xSp!FAYmZ-P6NcVc!DH;s?RPrD2^@ixP8 z-?cOJ1{=0RogsT~1uJ=gcl;METq1bq&i|%G7~OU;d?xZ0RixKN^5Bw!USa7hyi1US zwx6zel6#a!vf zSL3DKF#L3Ukjp&OSv?wc{OQLEt=K}Lv;}#UAg7)b5O`4$XId>cNdFqz1aNLK?0s4~_js@N1 znb#wDZt9GgaY5f4GoOR5Nzb0nn}G)H zHU3HN7QnQ>tqbPhVB=!qCvmgGnU2OE7HLqwgN5~MH?%!@%c$S`&2f$TNU7d5+Szj@ zte@q5R^Dq)t#6Y=gV~?~*4An7YiOMPbaDL9G0VmM+xt{IYn-(!`AzaY`}1k1o#))Y z-C10N(my)mX>l{&vf}yR;fI}oF(bXch22JcGvMTGYp}*oyXnS82XDm-S=4pkA8o(c z#b4vDU3G3n{aK#n(3t1W{5Ji5Jk2=sn`dx*Ppet8YQb@X`XkgM@y?$~?QZZV-}sE< zj3nc?Tujet;C*4Ny(GV- zg2|5Ke0X)lD#%oB>cO8316})5(DvXZ-PYQ5X+fUkoonhq?=Hjfk@q23HtAbb*i7DI z<$EbRITn5m8ua5?jpFB|Il;&8OlC#SnY_?y-mF_|wEc$)cj9eY)$iFh)Z^kE+udch zjAzZf6Dy4$-ypwr%+V)jqwTL# zyk?f^{U-jr$N$G4e|pf@{aCx6NW&8pFZ+4YeP!qpV-0cBO?P1|{6*d!CrxZVPv=ti zYK$QxAR&VAw!=y+ytOb8^S!Mvq}tkBm(uUyn-A0t-d%>e7u3NZUA^_GF_yY4Yw)HX z-_EJD-v#!k3-R;p2y!YpA1Y1=purQ5F1AJ=cyvDA`FK*5qx-w&FW1FBNL+v2tq$rz zZRtcBA50gPmg^m*7nzJd`I&U#8--i9Zk6vm2kFgw;da^%XUwrPZN%BmG(kMTwjYc2 zs_#5h(?Rh+d5?GNgLJM8RGrT-e4PaN;QKWP;2jK?uf25=FXA7^o1a``Q9p=pKD+6? zeN)lF6Sf9n?Ts~&qJsT@@W2DC;lsu>!o3N|;gajt<#9Zwo(*-Ncc+=6{PcWnPqb6ZVJZdsdncEZ`Es8@lH~wu3u4gu#SE`xOt48+Quk zc0Fyc>9P?t*uyx3m-r7F zaIiIa(0*O8&i^E+PTdaVKfdX{bjb(1gnL*fj_d0FQ6S8|woqeXC+G;R$K4pdUs7(o zE1C>{QkjrnIQLBOq=B#=d-UnLiKf+eVXc!?m%?4OKlkx`1Ji6n^IPL*hVk3<>onS> zzE;N$Y42F+_Oo3D58txoTV{NpS+|kb4djL2kmjhU)lRm3-JDj|zybU2otM|-r|`a~ z<~z!9c2#Fw<7>KngZ6?8E)Su3QoDI@B&@s#F(2)}9(wVe&3ADFM?#n*!dy`ZnR3a(A(aF=fx z4B6uVsh>$5&3|h9_uD)0jkj=E9JoEO5?`J*aV_#1e(!*9*eRNZ-q^@N6n3E2E6;J&495PDY zAARQOnHkH8D5q>%)dk}~ECI{$z@{tdtZA*jSqIk~yra4u_b&K$rfawRzO~=)x_P+A z<9Pso8^qk_+3jM~uVJFAVRFOR&c(&e#EW=OlKQo{|6ucfa$2`+_U9P&#(x;L*Fgty z3`E>9u)M#BdXg>Eb8NZ$u7B^&;ZNRES9j4e-phI5BmR8jmHPp{vxT)=Af5a;;e49E zi*VPIPdt}vzukrN_V^2m#+r7j?Nwei|8b7tyWqRi;I5OD^8~f8A8yl6+*`F$_g^U= zorEs^`yL?I0DOl5WeR?@URi_^F_KkqZp z=7Clm2cU}rbZ^U&YMsr;k^O~n3Pzy#`Anp~e-KijAZcZ_&;mCmH=|2qFKcyEorc}6({ z$Gmsm=9y1B$q&J>cxi&Y{pT3-d*;>aByk}=A=V$S{nEAXh#iMI{|K&E(SbDJTL669 zCCr#}U0513d?9`Vi@%A~6Z=4T#H!FbeH2yL9E(-5H?`|_=2F`pvzn5mVwfU`JLEA4uzpqME4GzM~k7pSE zPCEOJ+CISF)&mKq19^b!!kcftOYluGz9}y)-V@~8Jv`T^;qR2*cH6yfyiS+o-Y;&v z@y;}3o)>?#{Z`x!9%opKh^SzirU-*4&P_*L!=J)qD4#Oy>aq`%QLC+YZ|0VqvQCdSm`bJ zjXm>xGhMrDT-|0)9tU+AyyL;&Y~$}$%6F5~;!iyJcGv7z-;gr8!|}F4_`20>ON~?J zF4}%A_IMp~>$D**nd0r>hCKS4`T4Dl4wtSP3u6;NgMj(aY%lVDv3#40@-m(s@vRxP zrq$^*&OvyZapt!_t&IB{vd^!}Z{|am<53rr7>&>?x zsLR){-w3zNUw}K#<@tF%9XK_-JK=`qO$w(D1#sUF_9rh56b@`-jKfks z$uaMWD{e4k(_#4gX=$blqK%XOXYBC~#vSVwX;r5}-2X3vZe1~cmnH^1&^X=* zE5YwXH-x@ZDb`M9_zkaeykA!dTiUThpG@^z{dlGG2hqk1%Ot;A3$jSHnq{#)MCWxMBWyrqyBEHN(uZ%x^QT8SekiG#&HF zw7E4!(Ym!;I!a$Ho(p&aqt@p9@e}2ck`s}$hgB2d+Xj}Ea33z<4V&bL7jkov1SZ!S zM<@GotA?(X_@b6h)*ch3iy=dr2nv>_mXj2X?bgV55|B?;gXauvw=tf2peeE4ybaQ8 zMQW6rW4rv_8yf9ud{tXO%1vljO5iN3 zCZWRBFN)!LhHGnOK_j`cJwj@r Date: Tue, 16 Jun 2026 11:45:28 -0700 Subject: [PATCH 08/11] Bug fixes, strip webrtc binaries, download webrtc source archive for faster download. General cleanup. --- .github/workflows/webrtc.yml | 44 +++------ ...uild-enable-rtc_use_cxx11_abi-option.patch | 11 +-- 3rdparty/webrtc/CMakeLists.txt | 71 ++++++++------ 3rdparty/webrtc/webrtc_build.sh | 94 +++++++++++++------ 3rdparty/webrtc/webrtc_common.cmake | 15 ++- 5 files changed, 132 insertions(+), 103 deletions(-) diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 481671a1e6c..2532b21077d 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -9,9 +9,9 @@ on: required: false default: 'e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e' depot_tools_commit: - description: 'depot_tools commit (empty = HEAD).' + description: 'depot_tools commit (override pin in webrtc_build.sh).' required: false - default: '' + default: '10eda50a3fd9c34ad8d31ec74e5f4eb5823d60f6' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,13 +20,14 @@ concurrency: env: WEBRTC_COMMIT: ${{ github.event.inputs.webrtc_commit }} DEPOT_TOOLS_COMMIT: ${{ github.event.inputs.depot_tools_commit }} - NPROC: 4 + WEBRTC_WORK_ROOT: ${{ github.workspace }}/.. + GCLIENT_JOBS: 8 jobs: Unix: permissions: - contents: write + contents: read runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -73,7 +74,7 @@ jobs: Windows: permissions: - contents: write + contents: read runs-on: windows-2022 strategy: fail-fast: false @@ -94,9 +95,9 @@ jobs: env: WORK_DIR: 'C:\WebRTC' OPEN3D_DIR: ${{ github.workspace }} - DEPOT_TOOLS_UPDATE: 1 - DEPOT_TOOLS_WIN_TOOLCHAIN: 0 - NPROC: 2 + WEBRTC_WORK_ROOT: 'C:\WebRTC' + DEPOT_TOOLS_UPDATE: 0 # belt-and-suspenders; also set by webrtc_setup_path + DEPOT_TOOLS_WIN_TOOLCHAIN: 0 # use locally installed VS, not the Chromium toolchain steps: - name: Checkout source code @@ -119,28 +120,13 @@ jobs: arch: x64 - name: Download WebRTC sources - shell: pwsh - working-directory: ${{ env.WORK_DIR }} + # shell: bash uses Git Bash on Windows, which transparently converts + # Windows-style env paths (e.g. OPEN3D_DIR, WEBRTC_WORK_ROOT) so they + # work in bash string and pushd contexts. + shell: bash run: | - $ErrorActionPreference = 'Stop' - if (-not (Test-Path depot_tools)) { - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - } - if ($env:DEPOT_TOOLS_COMMIT) { - git -C depot_tools checkout $env:DEPOT_TOOLS_COMMIT - } - $env:Path = (Get-Item depot_tools).FullName + ';' + $env:Path - if (-not (Test-Path webrtc/src)) { - New-Item -ItemType Directory -Force -Path webrtc | Out-Null - Push-Location webrtc - fetch webrtc - Pop-Location - } - git -C webrtc/src checkout $env:WEBRTC_COMMIT - git -C webrtc/src submodule update --init --recursive - Push-Location webrtc - gclient sync -D --force --reset - Pop-Location + source "$OPEN3D_DIR/3rdparty/webrtc/webrtc_build.sh" + download_webrtc_sources - name: Patch WebRTC shell: pwsh diff --git a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch index 1c87f7ffc0d..11e5cab1ff6 100644 --- a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch +++ b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch @@ -1,16 +1,7 @@ -From c47a1b6c0faa2206395647cb83cb1a0542101847 Mon Sep 17 00:00:00 2001 -From: Yixing Lao -Date: Wed, 7 Apr 2021 16:17:39 -0700 -Subject: [PATCH] build: enable rtc_use_cxx11_abi option - ---- - config/BUILDCONFIG.gn | 6 ++++++ - 1 file changed, 6 insertions(+) - diff --git a/config/BUILDCONFIG.gn b/config/BUILDCONFIG.gn --- a/config/BUILDCONFIG.gn +++ b/config/BUILDCONFIG.gn -@@ -171,6 +171,12 @@ declare_args() { +@@ -171,6 +171,11 @@ declare_args() { is_debug && current_os != "ios" && current_os != "watchos" } diff --git a/3rdparty/webrtc/CMakeLists.txt b/3rdparty/webrtc/CMakeLists.txt index 6876f6d340a..16bc558d0ae 100644 --- a/3rdparty/webrtc/CMakeLists.txt +++ b/3rdparty/webrtc/CMakeLists.txt @@ -1,25 +1,25 @@ -# This CMake file is intended to be used inside Dockerfile.webrtc. +# CMake driver for building the WebRTC prebuilt static libraries. +# Invoked by webrtc_build.sh (Unix CI) and webrtc.yml (Windows CI). +# Copied to /webrtc/ alongside webrtc_common.cmake before cmake is +# configured from /webrtc/build/. # -# 1) We assume the following directory structure: -# / -# ├── depot_tools # ${DEPOT_TOOLS_ROOT}, should be added to PATH -# └── webrtc # ${WEBRTC_ROOT} -#    ├── CMakeLists.txt # This CMakeLists.txt itself (copied to container) -#    ├── webrtc_common.cmake # Common configs for WebRTC (copied to container) -#    ├── .gclient -#    └── src # The actual git directory +# Expected directory layout: +# / +# ├── depot_tools/ # ${DEPOT_TOOLS_ROOT}, must be on PATH +# └── webrtc/ # ${WEBRTC_ROOT} = ${PROJECT_SOURCE_DIR} +# ├── CMakeLists.txt # this file +# ├── webrtc_common.cmake +# ├── .gclient +# └── src/ # WebRTC source tree # -# 2) CMake will compile two libraries libwebrtc.a and libwebrtc_extra.a. -# - libwebrtc.a compilation is driven by Ninja, the output will be in: -# ${WEBRTC_ROOT}/src/out/Release/obj/libwebrtc.a -# - libwebrtc_extra.a compilation is driven by Ninja but CMake packages the -# object files into a static library, the output will be in: -# ${WEBRTC_ROOT}/src/out/Release/obj/libwebrtc_extra.a +# Outputs (relative to CMAKE_INSTALL_PREFIX): +# lib/libwebrtc.a - main WebRTC static lib (built by gn/ninja) +# lib/libwebrtc_extra.a - supplementary objects packaged by CMake +# include/ - headers from webrtc/src/ # -# 3) Finally, `make install` will install headers and binaries to -# - build/lib -# - build/include - +# Build: +# cmake -G Ninja -DCMAKE_INSTALL_PREFIX= /webrtc +# ninja -j$(nproc) install cmake_minimum_required(VERSION 3.18) project(webrtc CXX) @@ -83,17 +83,9 @@ set_target_properties(webrtc_extra PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${WEBRTC_NINJA_ROOT}/obj ) -# Install headers and binaries -# /webrtc_install -# |-- include -# | |-- api -# | |-- audio -# ... -# | |-- tools_webrtc -# | `-- video -# `-- lib -# |-- libwebrtc.a -# `-- libwebrtc_extra.a +# Install headers and libs into CMAKE_INSTALL_PREFIX: +# include/ - all .h/.hpp/.inc headers mirroring webrtc/src/ structure +# lib/ - libwebrtc.a libwebrtc_extra.a (or .lib on Windows) file(GLOB_RECURSE WEBRTC_INCLUDES RELATIVE ${WEBRTC_ROOT}/src ${WEBRTC_ROOT}/src/*.h ${WEBRTC_ROOT}/src/*.hpp @@ -108,3 +100,22 @@ install(FILES ${WEBRTC_NINJA_ROOT}/obj/${CMAKE_STATIC_LIBRARY_PREFIX}webrtc_extra${CMAKE_STATIC_LIBRARY_SUFFIX} DESTINATION lib ) + +# Release prebuilts: strip debug sections from installed static libraries (Unix/macOS). +# MSVC .lib files are kept small via GN symbol_level=0; COFF static libs have no +# equivalent strip tool in the MSVC toolchain. +if(NOT WEBRTC_IS_DEBUG AND UNIX AND CMAKE_STRIP) + # macOS strip uses -S (debug symbols only); GNU strip uses --strip-debug. + if(APPLE) + set(_webrtc_strip_flags -S) + else() + set(_webrtc_strip_flags --strip-debug) + endif() + install(CODE " + file(GLOB _webrtc_libs \"\${CMAKE_INSTALL_PREFIX}/lib/*${CMAKE_STATIC_LIBRARY_SUFFIX}\") + foreach(_lib \${_webrtc_libs}) + execute_process(COMMAND \"${CMAKE_STRIP}\" ${_webrtc_strip_flags} \"\${_lib}\" + ERROR_QUIET) + endforeach() + ") +endif() diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 809cb804274..80f437038c0 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -1,37 +1,48 @@ #!/usr/bin/env bash -set -euox pipefail - -# Builds WebRTC static libraries for Open3D (Ubuntu/macOS). Windows: webrtc.yml +# Build WebRTC static libraries for Open3D (Ubuntu/macOS). +# Windows uses download_webrtc_sources() from this file via Git Bash; +# the cmake/ninja build itself is driven by the webrtc.yml PowerShell steps. +# +# This file is sourced (not executed) by CI steps so that functions are +# available as shell commands. Sourcing applies `set -euo pipefail` to the +# calling shell for strict error checking across the entire CI step. # -# Layout (default: repo parent holds depot_tools + webrtc): +# Expected directory layout ( = parent of the Open3D checkout, or +# $WEBRTC_WORK_ROOT if set): # / # ├── Open3D/ # this repository -# ├── depot_tools/ +# ├── depot_tools/ # fetched by clone_depot_tools() # └── webrtc/ -# └── src/ +# ├── .gclient # created by `fetch --nohooks --no-history webrtc` +# └── src/ # WebRTC source tree, pinned to WEBRTC_COMMIT # -# Usage: -# cd /path/to/Open3D -# export WEBRTC_COMMIT=... # optional +# Usage (Unix): # source 3rdparty/webrtc/webrtc_build.sh -# install_dependencies_ubuntu # optional on Ubuntu -# download_webrtc_sources -# build_webrtc +# install_dependencies_ubuntu # Ubuntu only +# download_webrtc_sources # fetches depot_tools + runs gclient sync +# build_webrtc # cmake/ninja build, installs, packages tar.gz + +set -euo pipefail # libwebrtc-bin M149 / Open3D target milestone WEBRTC_COMMIT=${WEBRTC_COMMIT:-e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e} -# Optional pin; unset uses depot_tools HEAD. -DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-} +# Pinned depot_tools (update intentionally when refreshing the WebRTC toolchain). +DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-10eda50a3fd9c34ad8d31ec74e5f4eb5823d60f6} +DEPOT_TOOLS_URL="https://chromium.googlesource.com/chromium/tools/depot_tools" GLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI:-1} -NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN)} +NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)} SUDO=${SUDO:-sudo} +# Parallel gclient git operations (speeds DEPS fetch on CI). +GCLIENT_JOBS=${GCLIENT_JOBS:-${NPROC}} + +_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" webrtc_work_root() { if [[ -n "${WEBRTC_WORK_ROOT:-}" ]]; then echo "$WEBRTC_WORK_ROOT" else - dirname "$PWD" + dirname "$_OPEN3D_ROOT" fi } @@ -40,6 +51,35 @@ webrtc_setup_path() { export DEPOT_TOOLS_UPDATE=0 } +# Fetch a pinned depot_tools tree via Gitiles tarball. +clone_depot_tools() { + local root="$1" + local dest="$root/depot_tools" + local commit="$DEPOT_TOOLS_COMMIT" + local stamp="$dest/.open3d_pinned_commit" + + if [[ -f "$stamp" && "$(cat "$stamp")" == "$commit" && -x "$dest/fetch" ]]; then + return 0 + fi + + local tmp archive + tmp="$(mktemp -d)" + archive="$tmp/depot_tools.tar.gz" + curl -fL --retry 3 --retry-delay 5 \ + -o "$archive" "${DEPOT_TOOLS_URL}/+archive/${commit}.tar.gz" + rm -rf "$dest" + mkdir -p "$dest" + # Gitiles +archive tarballs unpack flat (fetch at archive root, not in a subdir). + tar -xzf "$archive" -C "$dest" + rm -rf "$tmp" + + if [[ ! -x "$dest/fetch" ]]; then + echo "ERROR: depot_tools archive at ${commit} is missing fetch" >&2 + exit 1 + fi + echo "$commit" > "$stamp" +} + install_dependencies_ubuntu() { options="$(echo "$@" | tr ' ' '|')" $SUDO apt-get update @@ -79,34 +119,31 @@ install_dependencies_ubuntu() { download_webrtc_sources() { local root root="$(webrtc_work_root)" + pushd "$root" - if [[ ! -d depot_tools ]]; then - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - fi - if [[ -n "$DEPOT_TOOLS_COMMIT" ]]; then - git -C depot_tools checkout "$DEPOT_TOOLS_COMMIT" - fi + clone_depot_tools "$root" webrtc_setup_path + # Verify fetch is on PATH (exits non-zero under set -e if not found). command -V fetch if [[ ! -d webrtc/src ]]; then mkdir -p webrtc pushd webrtc - fetch --nohooks webrtc + fetch --nohooks --no-history webrtc popd fi - git -C webrtc/src checkout "$WEBRTC_COMMIT" - git -C webrtc/src submodule update --init --recursive pushd webrtc - gclient sync -D --force --reset --no-history + gclient sync -D --force --reset --no-history \ + --jobs="${GCLIENT_JOBS}" \ + --revision "src@${WEBRTC_COMMIT}" popd popd } build_webrtc() { local root open3d_dir - open3d_dir="$PWD" + open3d_dir="$_OPEN3D_ROOT" root="$(webrtc_work_root)" webrtc_setup_path @@ -122,8 +159,7 @@ build_webrtc() { -DCMAKE_INSTALL_PREFIX="$root/webrtc_release" \ -DGLIBCXX_USE_CXX11_ABI="${GLIBCXX_USE_CXX11_ABI}" \ .. - ninja -j"${NPROC}" - ninja install + ninja -j"${NPROC}" install popd pushd "$root" diff --git a/3rdparty/webrtc/webrtc_common.cmake b/3rdparty/webrtc/webrtc_common.cmake index 70708eb27b7..712595aaf30 100644 --- a/3rdparty/webrtc/webrtc_common.cmake +++ b/3rdparty/webrtc/webrtc_common.cmake @@ -1,10 +1,13 @@ -# Common configs for building WebRTC from source. Used in both native build -# and building inside docker. +# Common GN args and ninja target lists for building WebRTC from source. +# Included by CMakeLists.txt (which is driven by webrtc_build.sh on Unix CI and +# by webrtc.yml PowerShell steps on Windows CI). +# +# Callers must set WEBRTC_NINJA_ROOT before including this file. # # Exports: -# - get_webrtc_args(WEBRTC_ARGS) function -# - NINJA_TARGETS -# - EXTRA_WEBRTC_OBJS # You have to define WEBRTC_NINJA_ROOT before including this file +# get_webrtc_args(OUT_VAR) - function: returns a newline-separated args.gn string +# NINJA_TARGETS - list of gn targets to build +# EXTRA_WEBRTC_OBJS - object files not in libwebrtc.a, packed into libwebrtc_extra.a function(get_webrtc_args WEBRTC_ARGS) set(WEBRTC_ARGS "") @@ -38,6 +41,8 @@ function(get_webrtc_args WEBRTC_ARGS) endif() else() set(WEBRTC_ARGS is_debug=false\n${WEBRTC_ARGS}) + # Smaller static libs for prebuilt packages (no need for debug symbols). + set(WEBRTC_ARGS symbol_level=0\n${WEBRTC_ARGS}) endif() # H.264 (replaces deprecated is_chrome_branded on recent milestones). From ecc935b850ded29f2c5ca1514dc205aabfcbc03e Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 16 Jun 2026 13:26:46 -0700 Subject: [PATCH 09/11] fixes. --- .../webrtc/0002-build-enable_safe_libstdcxx.patch | 11 ----------- ...ch => 0002-src-fix-nullptr_t-with-libstdcxx.patch} | 0 ...003-src-gcc-suppress-port-interface-network.patch} | 3 ++- ...0004-call-payload_type_picker-gcc-flat_tree.patch} | 11 ++++++++++- ...mic-crt.patch => 0005-build-win-dynamic-crt.patch} | 0 3rdparty/webrtc/apply_webrtc_patches.sh | 9 ++++----- 3rdparty/webrtc/webrtc_build.sh | 7 ++++++- 7 files changed, 22 insertions(+), 19 deletions(-) delete mode 100644 3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch rename 3rdparty/webrtc/{0003-src-fix-nullptr_t-with-libstdcxx.patch => 0002-src-fix-nullptr_t-with-libstdcxx.patch} (100%) rename 3rdparty/webrtc/{0004-src-gcc-suppress-port-interface-network.patch => 0003-src-gcc-suppress-port-interface-network.patch} (85%) rename 3rdparty/webrtc/{0005-call-payload_type_picker-gcc-flat_tree.patch => 0004-call-payload_type_picker-gcc-flat_tree.patch} (52%) rename 3rdparty/webrtc/{0006-build-win-dynamic-crt.patch => 0005-build-win-dynamic-crt.patch} (100%) diff --git a/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch b/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch deleted file mode 100644 index 29df04e0a43..00000000000 --- a/3rdparty/webrtc/0002-build-enable_safe_libstdcxx.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff --git a/build.gni b/build.gni ---- a/build.gni -+++ b/build.gni -@@ -10,6 +10,7 @@ enable_java_templates = true - - # Enables assertions on safety checks in libc++. - enable_safe_libcxx = true -+enable_safe_libstdcxx = true - - # Don't set this variable to true when building standalone WebRTC, it is - # only needed to support both WebRTC standalone and Chromium builds. diff --git a/3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch b/3rdparty/webrtc/0002-src-fix-nullptr_t-with-libstdcxx.patch similarity index 100% rename from 3rdparty/webrtc/0003-src-fix-nullptr_t-with-libstdcxx.patch rename to 3rdparty/webrtc/0002-src-fix-nullptr_t-with-libstdcxx.patch diff --git a/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch b/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch similarity index 85% rename from 3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch rename to 3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch index ed0a638cd7f..5db2a598bb9 100644 --- a/3rdparty/webrtc/0004-src-gcc-suppress-port-interface-network.patch +++ b/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch @@ -1,7 +1,7 @@ diff --git a/p2p/base/port_interface.h b/p2p/base/port_interface.h --- a/p2p/base/port_interface.h +++ b/p2p/base/port_interface.h -@@ -52,7 +52,11 @@ class PortInterface { +@@ -52,7 +52,14 @@ class PortInterface { virtual ~PortInterface(); virtual IceCandidateType Type() const = 0; @@ -15,3 +15,4 @@ diff --git a/p2p/base/port_interface.h b/p2p/base/port_interface.h +#endif // Methods to set/get ICE role and tiebreaker values. + virtual void SetIceRole(IceRole role) = 0; diff --git a/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch b/3rdparty/webrtc/0004-call-payload_type_picker-gcc-flat_tree.patch similarity index 52% rename from 3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch rename to 3rdparty/webrtc/0004-call-payload_type_picker-gcc-flat_tree.patch index 8dc679d130f..584f3e81914 100644 --- a/3rdparty/webrtc/0005-call-payload_type_picker-gcc-flat_tree.patch +++ b/3rdparty/webrtc/0004-call-payload_type_picker-gcc-flat_tree.patch @@ -1,7 +1,16 @@ diff --git a/call/payload_type_picker.cc b/call/payload_type_picker.cc --- a/call/payload_type_picker.cc +++ b/call/payload_type_picker.cc -@@ -354,7 +354,7 @@ RTCErrorOr RtpHeaderExtensionRecorder::LookupId(absl::string_view uri, +@@ -338,7 +338,7 @@ + RTCError RtpHeaderExtensionRecorder::AddMapping(int id, + absl::string_view uri, + bool encrypt) { +- auto it = uri_to_id_.find(std::pair{uri, encrypt}); ++ auto it = uri_to_id_.find(std::pair{std::string(uri), encrypt}); + if (it != uri_to_id_.end()) { + if (it->second != id) { + // TODO: https://issues.webrtc.org/41480892 - This will return an error in +@@ -354,7 +354,7 @@ RTCErrorOr RtpHeaderExtensionRecorder::LookupId(absl::string_view uri, bool encrypt) const { diff --git a/3rdparty/webrtc/0006-build-win-dynamic-crt.patch b/3rdparty/webrtc/0005-build-win-dynamic-crt.patch similarity index 100% rename from 3rdparty/webrtc/0006-build-win-dynamic-crt.patch rename to 3rdparty/webrtc/0005-build-win-dynamic-crt.patch diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index 94e89a911a5..fcab2ca64b0 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -41,8 +41,7 @@ PATCH_DIR="$OPEN3D_DIR/3rdparty/webrtc" apply_one "$PATCH_DIR/0001-src-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0001-build-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/build" apply_one "$PATCH_DIR/0001-third_party-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/third_party" -apply_one "$PATCH_DIR/0002-build-enable_safe_libstdcxx.patch" "$WEBRTC_SRC/build_overrides" -apply_one "$PATCH_DIR/0003-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" -apply_one "$PATCH_DIR/0004-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" -apply_one "$PATCH_DIR/0005-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" -apply_one "$PATCH_DIR/0006-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" +apply_one "$PATCH_DIR/0002-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0003-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0004-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0005-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 80f437038c0..7e12bda80bd 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -70,7 +70,12 @@ clone_depot_tools() { rm -rf "$dest" mkdir -p "$dest" # Gitiles +archive tarballs unpack flat (fetch at archive root, not in a subdir). - tar -xzf "$archive" -C "$dest" + # On Windows (Git Bash), symlinks in the archive fail to extract because + # symlink creation requires elevated privileges. Those symlinks are + # Linux-only helper scripts (cbuildbot, luci-auth-fido2-plugin, etc.) and + # are not needed for WebRTC builds. The 'fetch' check below validates the + # critical tools were extracted. + tar -xzf "$archive" -C "$dest" || true rm -rf "$tmp" if [[ ! -x "$dest/fetch" ]]; then From 96b874808de3a9e75e4a248391c2423003f9b872 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 16 Jun 2026 22:20:01 -0700 Subject: [PATCH 10/11] fix2 --- 3rdparty/webrtc/webrtc_build.sh | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 7e12bda80bd..0bd8ab50317 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -47,7 +47,18 @@ webrtc_work_root() { } webrtc_setup_path() { - export PATH="$(webrtc_work_root)/depot_tools:${PATH}" + local root + root="$(webrtc_work_root)" + # On Windows the work root is a native path like 'C:\WebRTC'. The colon in + # the drive letter is bash's PATH separator, which would split + # 'C:\WebRTC/depot_tools' into the two entries 'C' and '\WebRTC/depot_tools'. + # '\WebRTC/...' then causes sha256sum to prefix its output with '\' (GNU + # coreutils escapes paths that contain backslashes), making the CIPD hash + # check fail. cygpath is available in Git Bash / MSYS2; convert to POSIX. + if command -v cygpath >/dev/null 2>&1; then + root="$(cygpath -u "$root")" + fi + export PATH="${root}/depot_tools:${PATH}" export DEPOT_TOOLS_UPDATE=0 } @@ -82,6 +93,16 @@ clone_depot_tools() { echo "ERROR: depot_tools archive at ${commit} is missing fetch" >&2 exit 1 fi + + # Bootstrap the depot_tools Python runtime. This creates + # python3_bin_reldir.txt which is required by the gn and ninja wrappers. + # On Unix, ensure_bootstrap calls bootstrap_python3 which downloads a + # hermetic Python 3 via CIPD and writes python3_bin_reldir.txt. + # On Windows (Git Bash/MINGW), win_tools.bat handles the Python bootstrap + # when gn.bat / ninja.bat are first invoked, so ensure_bootstrap skips it. + # DEPOT_TOOLS_DIR must be set so ensure_bootstrap resolves scripts correctly. + DEPOT_TOOLS_DIR="$dest" "$dest/ensure_bootstrap" + echo "$commit" > "$stamp" } From 4dcdc1dac05a2f2b51ba0e3842b740f4d6e02e98 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 17 Jun 2026 13:56:56 -0700 Subject: [PATCH 11/11] Fix WebRTC CI bootstrap and macOS compilation - Prepend depot_tools to PATH and run win_tools.bat on Windows - Disable protobuf constinit on Apple to fix Xcode 15.4 build --- ...-protobuf-disable-constinit-on-apple.patch | 26 ++++++++++++++++ 3rdparty/webrtc/apply_webrtc_patches.sh | 1 + 3rdparty/webrtc/webrtc_build.sh | 30 ++++++++++++++----- 3 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch diff --git a/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch b/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch new file mode 100644 index 00000000000..35fe389efbe --- /dev/null +++ b/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch @@ -0,0 +1,26 @@ +diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc +index 16423927bd33..edef8148783b 100644 +--- a/src/google/protobuf/port_def.inc ++++ b/src/google/protobuf/port_def.inc +@@ -469,7 +469,7 @@ + # define PROTOBUF_CONSTEXPR constexpr + # endif + #else +-# if defined(__cpp_constinit) && !defined(__CYGWIN__) ++# if defined(__cpp_constinit) && !defined(__CYGWIN__) && !defined(__APPLE__) + # define PROTOBUF_CONSTINIT constinit + # define PROTOBUF_CONSTEXPR constexpr + # define PROTOBUF_CONSTINIT_DEFAULT_INSTANCES +@@ -477,10 +477,9 @@ + // constant-initializing weak default instance pointers. Versions 12.0 and + // higher seem to work, except that XCode 12.5.1 shows the error even though it + // uses Clang 12.0.5. +-#elif !defined(__CYGWIN__) && !defined(__MINGW32__) && \ ++# elif !defined(__CYGWIN__) && !defined(__MINGW32__) && !defined(__APPLE__) && \ + ABSL_HAVE_CPP_ATTRIBUTE(clang::require_constant_initialization) && \ +- ((defined(__APPLE__) && PROTOBUF_CLANG_MIN(13, 0)) || \ +- (!defined(__APPLE__) && PROTOBUF_CLANG_MIN(12, 0))) ++ PROTOBUF_CLANG_MIN(12, 0) + # define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]] + # define PROTOBUF_CONSTEXPR constexpr + # define PROTOBUF_CONSTINIT_DEFAULT_INSTANCES diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh index fcab2ca64b0..287877e882e 100755 --- a/3rdparty/webrtc/apply_webrtc_patches.sh +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -45,3 +45,4 @@ apply_one "$PATCH_DIR/0002-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0003-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0004-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" apply_one "$PATCH_DIR/0005-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" +apply_one "$PATCH_DIR/0006-third_party-protobuf-disable-constinit-on-apple.patch" "$WEBRTC_SRC/third_party/protobuf" diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index 0bd8ab50317..37011ad0d14 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -94,14 +94,28 @@ clone_depot_tools() { exit 1 fi - # Bootstrap the depot_tools Python runtime. This creates - # python3_bin_reldir.txt which is required by the gn and ninja wrappers. - # On Unix, ensure_bootstrap calls bootstrap_python3 which downloads a - # hermetic Python 3 via CIPD and writes python3_bin_reldir.txt. - # On Windows (Git Bash/MINGW), win_tools.bat handles the Python bootstrap - # when gn.bat / ninja.bat are first invoked, so ensure_bootstrap skips it. - # DEPOT_TOOLS_DIR must be set so ensure_bootstrap resolves scripts correctly. - DEPOT_TOOLS_DIR="$dest" "$dest/ensure_bootstrap" + # Bootstrap depot_tools. + # We must temporarily prepend depot_tools to PATH so that python scripts + # and subprocesses launched during bootstrap (like gsutil.py calling luci-auth) + # can find the depot_tools executables on both Unix and Windows. + local old_path="$PATH" + export PATH="${dest}:${PATH}" + + if [[ "$(uname -s)" == *"MINGW"* || "$(uname -s)" == *"MSYS"* || "$(uname -s)" == *"CYGWIN"* ]]; then + # On Windows, bootstrap Python and Git via the batch files. + # This creates git.bat, python3.bat, and downloads cipd tools. + # We must run them using cmd.exe inside the depot_tools directory. + pushd "$dest" + cmd.exe //c "cipd_bin_setup.bat" + cmd.exe //c "bootstrap\\win_tools.bat" + popd + else + # On Unix (Ubuntu/macOS), ensure_bootstrap downloads Python 3 via CIPD and writes python3_bin_reldir.txt. + # DEPOT_TOOLS_DIR must be set so ensure_bootstrap resolves scripts correctly. + DEPOT_TOOLS_DIR="$dest" "$dest/ensure_bootstrap" + fi + + export PATH="$old_path" echo "$commit" > "$stamp" }