diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 45b27363ebc..2532b21077d 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -5,13 +5,13 @@ on: workflow_dispatch: inputs: webrtc_commit: - description: 'Specify WebRTC commit to build.' + description: 'WebRTC src commit (full or short).' required: false - default: '60e674842ebae283cc6b2627f4b6f2f8186f3317' # Date: Wed Apr 7 19:12:13 2021 +0200 + default: 'e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e' depot_tools_commit: - description: 'Specify Depot Tools commit to to use for the build.' + description: 'depot_tools commit (override pin in webrtc_build.sh).' required: false - default: 'e1a98941d3ab10549be6d82d0686bb0fb91ec903' # Date: Wed Apr 7 21:35:29 2021 +0000 + default: '10eda50a3fd9c34ad8d31ec74e5f4eb5823d60f6' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,29 +20,35 @@ concurrency: env: WEBRTC_COMMIT: ${{ github.event.inputs.webrtc_commit }} DEPOT_TOOLS_COMMIT: ${{ github.event.inputs.depot_tools_commit }} + WEBRTC_WORK_ROOT: ${{ github.workspace }}/.. + GCLIENT_JOBS: 8 jobs: Unix: permissions: - contents: write # upload + contents: read runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-13] + include: + - os: ubuntu-22.04 + package_suffix: linux_cxx-abi-1 + - os: macos-14 + package_suffix: macos_arm64 steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Set up Python version + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' - - name: Install dependencies - if: ${{ matrix.os == 'ubuntu-22.04' }} + - name: Install dependencies (Ubuntu) + if: matrix.os == 'ubuntu-22.04' run: | source 3rdparty/webrtc/webrtc_build.sh install_dependencies_ubuntu @@ -60,116 +66,113 @@ jobs: - name: Upload WebRTC uses: actions/upload-artifact@v4 with: - name: webrtc_release_${{ matrix.os }} + name: webrtc_${{ matrix.package_suffix }} path: | - webrtc_*.tar.gz - checksum_*.txt + webrtc_*.tar.gz + checksum_webrtc_*.tar.gz if-no-files-found: error Windows: permissions: - contents: write # upload - # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/windows_build_instructions.md + contents: read runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + include: + - config: Release + static_runtime: ON + tag: Release_mt + - config: Release + static_runtime: OFF + tag: Release_md + - config: Debug + static_runtime: ON + tag: Debug_mt + - config: Debug + static_runtime: OFF + tag: Debug_md env: - WORK_DIR: "C:\\WebRTC" # Not enough space in D: - OPEN3D_DIR: "D:\\a\\open3d\\open3d" - DEPOT_TOOLS_UPDATE: 1 # Fix cannot find python3_bin_reldir.txt - DEPOT_TOOLS_WIN_TOOLCHAIN: 0 - NPROC: 2 + WORK_DIR: 'C:\WebRTC' + OPEN3D_DIR: ${{ github.workspace }} + WEBRTC_WORK_ROOT: 'C:\WebRTC' + DEPOT_TOOLS_UPDATE: 0 # belt-and-suspenders; also set by webrtc_setup_path + DEPOT_TOOLS_WIN_TOOLCHAIN: 0 # use locally installed VS, not the Chromium toolchain steps: - name: Checkout source code uses: actions/checkout@v4 - - name: Set up Python version + - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.10' - - name: Disk space + - name: Prepare work directory + shell: pwsh run: | Get-PSDrive - mkdir "$env:WORK_DIR" + New-Item -ItemType Directory -Force -Path $env:WORK_DIR - - name: Setup PATH for Visual Studio # Required for Ninja + - name: Setup PATH for Visual Studio uses: ilammy/msvc-dev-cmd@v1 with: arch: x64 - name: Download WebRTC sources - shell: pwsh - working-directory: ${{ env.WORK_DIR }} + # shell: bash uses Git Bash on Windows, which transparently converts + # Windows-style env paths (e.g. OPEN3D_DIR, WEBRTC_WORK_ROOT) so they + # work in bash string and pushd contexts. + shell: bash run: | - $ErrorActionPreference = 'Stop' - echo "Get depot_tools" - # Checkout to a specific version - # Ref: https://chromium.googlesource.com/chromium/src/+/main/docs/building_old_revisions.md - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - git -C depot_tools checkout $env:DEPOT_TOOLS_COMMIT - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - - echo "Get WebRTC" - mkdir webrtc - cd webrtc - fetch webrtc - - git -C src checkout $env:WEBRTC_COMMIT - git -C src submodule update --init --recursive - echo "gclient sync" - gclient sync -D --force --reset - cd .. - echo "random.org" - curl "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" -o skipcache + source "$OPEN3D_DIR/3rdparty/webrtc/webrtc_build.sh" + download_webrtc_sources - name: Patch WebRTC + shell: pwsh working-directory: ${{ env.WORK_DIR }} run: | $ErrorActionPreference = 'Stop' - cp "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/ - cp "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/ - - - name: Build WebRTC (Release) - working-directory: ${{ env.WORK_DIR }} - run: | - $ErrorActionPreference = 'Stop' - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - mkdir webrtc/build - cd webrtc/build - cmake -G Ninja -D CMAKE_BUILD_TYPE=Release ` - -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Release ` - .. - ninja install - echo "Cleanup build folder for next config build" - cd .. - rm -r build + Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/CMakeLists.txt" webrtc/ + Copy-Item "$env:OPEN3D_DIR/3rdparty/webrtc/webrtc_common.cmake" webrtc/ + bash "$env:OPEN3D_DIR/3rdparty/webrtc/apply_webrtc_patches.sh" ` + "$env:OPEN3D_DIR" "$env:WORK_DIR/webrtc/src" - - name: Build WebRTC (Debug) + - name: Build and package WebRTC + shell: pwsh working-directory: ${{ env.WORK_DIR }} + env: + BUILD_CONFIG: ${{ matrix.config }} + STATIC_RT: ${{ matrix.static_runtime }} + WIN_TAG: ${{ matrix.tag }} run: | $ErrorActionPreference = 'Stop' - $env:Path = (Get-Item depot_tools).FullName + ";" + $env:Path - mkdir webrtc/build - cd webrtc/build - cmake -G Ninja -D CMAKE_BUILD_TYPE=Debug ` - -D CMAKE_INSTALL_PREFIX=${{ env.WORK_DIR }}/webrtc_release/Debug ` - .. + $env:Path = (Get-Item depot_tools).FullName + ';' + $env:Path + $installRoot = Join-Path $env:WORK_DIR "webrtc_pkg" + if (Test-Path $installRoot) { Remove-Item -Recurse -Force $installRoot } + New-Item -ItemType Directory -Force -Path webrtc/build | Out-Null + Push-Location webrtc/build + $debugFlag = if ($env:BUILD_CONFIG -eq 'Debug') { 'ON' } else { 'OFF' } + cmake -G Ninja ` + -D CMAKE_BUILD_TYPE=$env:BUILD_CONFIG ` + -D WEBRTC_IS_DEBUG=$debugFlag ` + -D WEBRTC_STATIC_MSVC_RUNTIME=$env:STATIC_RT ` + -D CMAKE_INSTALL_PREFIX=$installRoot ` + .. ninja install - - - name: Package WebRTC - working-directory: ${{ env.WORK_DIR }} - run: | - $ErrorActionPreference = 'Stop' - $env:WEBRTC_COMMIT_SHORT = (git -C webrtc/src rev-parse --short=7 HEAD) - cmake -E tar cv webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip ` - --format=zip -- webrtc_release - cmake -E sha256sum webrtc_${env:WEBRTC_COMMIT_SHORT}_win.zip | Tee-Object -FilePath checksum_win.txt + Pop-Location + $short = (git -C webrtc/src rev-parse --short=7 HEAD) + $zip = "webrtc_${short}_win_$env:WIN_TAG.zip" + Push-Location $installRoot + cmake -E tar cvf (Join-Path $env:OPEN3D_DIR $zip) --format=zip . + Pop-Location + cmake -E sha256sum (Join-Path $env:OPEN3D_DIR $zip) | Tee-Object -FilePath (Join-Path $env:OPEN3D_DIR "checksum_$zip") - name: Upload WebRTC uses: actions/upload-artifact@v4 with: - name: webrtc_release_windows + name: webrtc_win_${{ matrix.tag }} path: | - ${{ env.WORK_DIR }}/webrtc_*.zip - ${{ env.WORK_DIR }}/checksum_*.txt + webrtc_*_win_*.zip + checksum_webrtc_*_win_*.zip if-no-files-found: error diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d571f97e295..93b303e02e3 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -52,7 +52,10 @@ jobs: - BUILD_CUDA_MODULE: ON # FIXME CONFIG: Debug env: - BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} + # WebRTC prebuilt ships static libs for both MSVC runtimes (/MT, /MD). + # Enable it for static-lib + static-runtime (/MT) and shared-lib + + # dynamic-runtime (/MD). Shared libs always use the dynamic runtime. + BUILD_WEBRTC: ${{ ( ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) || ( matrix.BUILD_SHARED_LIBS == 'ON' && matrix.STATIC_RUNTIME == 'OFF' ) ) && 'ON' || 'OFF' }} BUILD_PYTORCH_OPS: ${{ ( matrix.BUILD_CUDA_MODULE == 'ON' || matrix.CONFIG == 'Debug' ) && 'OFF' || 'ON' }} # FIXME steps: diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index fb926d36ebb..3c7456fa4d8 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -1988,11 +1988,24 @@ if(BUILD_WEBRTC) open3d_import_3rdparty_library(3rdparty_webrtc HIDDEN INCLUDE_DIRS ${WEBRTC_INCLUDE_DIRS} - LIB_DIR ${WEBRTC_LIB_DIR} - LIBRARIES ${WEBRTC_LIBRARIES} DEPENDS ext_webrtc_all ) + if(UNIX AND NOT APPLE) + target_link_libraries(3rdparty_webrtc INTERFACE + "-Wl,--whole-archive" + "$" + "-Wl,--no-whole-archive" + "$") + else() + target_link_libraries(3rdparty_webrtc INTERFACE + "$" + "$") + endif() target_link_libraries(3rdparty_webrtc INTERFACE Open3D::3rdparty_threads ${CMAKE_DL_LIBS}) + # libwebrtc.a and libturbojpeg.a both export jpeg_* symbols (WebRTC bundles libjpeg). + if(UNIX AND NOT APPLE) + target_link_options(3rdparty_webrtc INTERFACE "LINKER:--allow-multiple-definition") + endif() if (MSVC) # https://github.com/iimachines/webrtc-build/issues/2#issuecomment-503535704 target_link_libraries(3rdparty_webrtc INTERFACE secur32 winmm dmoguids wmcodecdspuuid msdmo strmiids) endif() diff --git a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch index 5d1897193c9..11e5cab1ff6 100644 --- a/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch +++ b/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch @@ -1,29 +1,15 @@ -From c47a1b6c0faa2206395647cb83cb1a0542101847 Mon Sep 17 00:00:00 2001 -From: Yixing Lao -Date: Wed, 7 Apr 2021 16:17:39 -0700 -Subject: [PATCH] build: enable rtc_use_cxx11_abi option - ---- - config/BUILDCONFIG.gn | 6 ++++++ - 1 file changed, 6 insertions(+) - diff --git a/config/BUILDCONFIG.gn b/config/BUILDCONFIG.gn -index 0ef73ab2b..5ab677e27 100644 --- a/config/BUILDCONFIG.gn +++ b/config/BUILDCONFIG.gn -@@ -163,6 +163,12 @@ declare_args() { - is_component_build = is_debug && current_os != "ios" +@@ -171,6 +171,11 @@ declare_args() { + is_debug && current_os != "ios" && current_os != "watchos" } +declare_args() { -+ # Set to false to define "_GLIBCXX_USE_CXX11_ABI=0". If set to true, the -+ # default will be used, which corresponds to the new CXX11 ABI. ++ # Open3D: GCC libstdc++ ABI selection on Linux. + rtc_use_cxx11_abi = true +} + assert(!(is_debug && is_official_build), "Can't do official debug builds") - - # ============================================================================== --- -2.17.1 - + assert(!(current_os == "ios" && is_component_build), + "Can't use component build on iOS") diff --git a/3rdparty/webrtc/0002-src-fix-nullptr_t-with-libstdcxx.patch b/3rdparty/webrtc/0002-src-fix-nullptr_t-with-libstdcxx.patch new file mode 100644 index 00000000000..396a02f6aaf --- /dev/null +++ b/3rdparty/webrtc/0002-src-fix-nullptr_t-with-libstdcxx.patch @@ -0,0 +1,12 @@ +diff --git a/rtc_base/ssl_stream_adapter.h b/rtc_base/ssl_stream_adapter.h +--- a/rtc_base/ssl_stream_adapter.h ++++ b/rtc_base/ssl_stream_adapter.h +@@ -129,7 +129,7 @@ class SSLStreamAdapter : public StreamInterface { + static std::unique_ptr Create( + std::unique_ptr stream, + absl::AnyInvocable handshake_error, +- nullptr_t /*field_trials*/) { ++ std::nullptr_t /*field_trials*/) { + return Create(std::move(stream), std::move(handshake_error)); + } + diff --git a/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch b/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch new file mode 100644 index 00000000000..5db2a598bb9 --- /dev/null +++ b/3rdparty/webrtc/0003-src-gcc-suppress-port-interface-network.patch @@ -0,0 +1,18 @@ +diff --git a/p2p/base/port_interface.h b/p2p/base/port_interface.h +--- a/p2p/base/port_interface.h ++++ b/p2p/base/port_interface.h +@@ -52,7 +52,14 @@ class PortInterface { + virtual ~PortInterface(); + + virtual IceCandidateType Type() const = 0; ++#if defined(__GNUC__) && !defined(__clang__) ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wchanges-meaning" ++#endif + virtual const Network* Network() const = 0; ++#if defined(__GNUC__) && !defined(__clang__) ++#pragma GCC diagnostic pop ++#endif + + // Methods to set/get ICE role and tiebreaker values. + virtual void SetIceRole(IceRole role) = 0; diff --git a/3rdparty/webrtc/0004-call-payload_type_picker-gcc-flat_tree.patch b/3rdparty/webrtc/0004-call-payload_type_picker-gcc-flat_tree.patch new file mode 100644 index 00000000000..584f3e81914 --- /dev/null +++ b/3rdparty/webrtc/0004-call-payload_type_picker-gcc-flat_tree.patch @@ -0,0 +1,21 @@ +diff --git a/call/payload_type_picker.cc b/call/payload_type_picker.cc +--- a/call/payload_type_picker.cc ++++ b/call/payload_type_picker.cc +@@ -338,7 +338,7 @@ + RTCError RtpHeaderExtensionRecorder::AddMapping(int id, + absl::string_view uri, + bool encrypt) { +- auto it = uri_to_id_.find(std::pair{uri, encrypt}); ++ auto it = uri_to_id_.find(std::pair{std::string(uri), encrypt}); + if (it != uri_to_id_.end()) { + if (it->second != id) { + // TODO: https://issues.webrtc.org/41480892 - This will return an error in +@@ -354,7 +354,7 @@ + + RTCErrorOr RtpHeaderExtensionRecorder::LookupId(absl::string_view uri, + bool encrypt) const { +- auto it = uri_to_id_.find(std::pair{uri, encrypt}); ++ auto it = uri_to_id_.find(std::pair{std::string(uri), encrypt}); + if (it == uri_to_id_.end()) { + return RTCError(RTCErrorType::INVALID_PARAMETER, + "No ID found for extension"); diff --git a/3rdparty/webrtc/0005-build-win-dynamic-crt.patch b/3rdparty/webrtc/0005-build-win-dynamic-crt.patch new file mode 100644 index 00000000000..c5f7277dabf --- /dev/null +++ b/3rdparty/webrtc/0005-build-win-dynamic-crt.patch @@ -0,0 +1,35 @@ +Subject: [PATCH] build: allow dynamic CRT for non-component desktop Windows + +Open3D ships a static libwebrtc but needs to support both the static +(/MT[d]) and dynamic (/MD[d]) MSVC runtimes. Upstream ties the CRT choice +to is_component_build, which would force a component (shared) build to get +/MD. Add an rtc_win_dynamic_crt gn arg so a non-component static build can +still select the dynamic CRT, matching Open3D STATIC_WINDOWS_RUNTIME=OFF. + +diff --git a/config/win/BUILD.gn b/config/win/BUILD.gn +--- a/config/win/BUILD.gn ++++ b/config/win/BUILD.gn +@@ -513,6 +513,13 @@ if (build_with_chromium && current_cpu == target_cpu && host_os == "win") { + # Configures how the runtime library (CRT) is going to be used. + # See https://msdn.microsoft.com/en-us/library/2kzt1wy3.aspx for a reference of + # what each value does. ++declare_args() { ++ # Open3D: use the dynamic CRT (/MD[d]) for non-component desktop Windows ++ # builds so we can ship a static libwebrtc that links against the dynamic ++ # MSVC runtime (matches Open3D STATIC_WINDOWS_RUNTIME=OFF). ++ rtc_win_dynamic_crt = false ++} ++ + config("default_crt") { + if (is_component_build) { + # Component mode: dynamic CRT. Since the library is shared, it requires +@@ -525,6 +532,9 @@ config("default_crt") { + # contains a details explanation of what is happening with the Windows + # CRT in Visual Studio releases related to Windows store applications. + configs = [ ":dynamic_crt" ] ++ } else if (rtc_win_dynamic_crt) { ++ # Open3D: static library with the dynamic CRT (/MD[d]). ++ configs = [ ":dynamic_crt" ] + } else { + # Desktop Windows: static CRT. + configs = [ ":static_crt" ] diff --git a/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch b/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch new file mode 100644 index 00000000000..35fe389efbe --- /dev/null +++ b/3rdparty/webrtc/0006-third_party-protobuf-disable-constinit-on-apple.patch @@ -0,0 +1,26 @@ +diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc +index 16423927bd33..edef8148783b 100644 +--- a/src/google/protobuf/port_def.inc ++++ b/src/google/protobuf/port_def.inc +@@ -469,7 +469,7 @@ + # define PROTOBUF_CONSTEXPR constexpr + # endif + #else +-# if defined(__cpp_constinit) && !defined(__CYGWIN__) ++# if defined(__cpp_constinit) && !defined(__CYGWIN__) && !defined(__APPLE__) + # define PROTOBUF_CONSTINIT constinit + # define PROTOBUF_CONSTEXPR constexpr + # define PROTOBUF_CONSTINIT_DEFAULT_INSTANCES +@@ -477,10 +477,9 @@ + // constant-initializing weak default instance pointers. Versions 12.0 and + // higher seem to work, except that XCode 12.5.1 shows the error even though it + // uses Clang 12.0.5. +-#elif !defined(__CYGWIN__) && !defined(__MINGW32__) && \ ++# elif !defined(__CYGWIN__) && !defined(__MINGW32__) && !defined(__APPLE__) && \ + ABSL_HAVE_CPP_ATTRIBUTE(clang::require_constant_initialization) && \ +- ((defined(__APPLE__) && PROTOBUF_CLANG_MIN(13, 0)) || \ +- (!defined(__APPLE__) && PROTOBUF_CLANG_MIN(12, 0))) ++ PROTOBUF_CLANG_MIN(12, 0) + # define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]] + # define PROTOBUF_CONSTEXPR constexpr + # define PROTOBUF_CONSTINIT_DEFAULT_INSTANCES diff --git a/3rdparty/webrtc/CMakeLists.txt b/3rdparty/webrtc/CMakeLists.txt index 34d052ae4df..16bc558d0ae 100644 --- a/3rdparty/webrtc/CMakeLists.txt +++ b/3rdparty/webrtc/CMakeLists.txt @@ -1,25 +1,25 @@ -# This CMake file is intended to be used inside Dockerfile.webrtc. +# CMake driver for building the WebRTC prebuilt static libraries. +# Invoked by webrtc_build.sh (Unix CI) and webrtc.yml (Windows CI). +# Copied to /webrtc/ alongside webrtc_common.cmake before cmake is +# configured from /webrtc/build/. # -# 1) We assume the following directory structure: -# / -# ├── depot_tools # ${DEPOT_TOOLS_ROOT}, should be added to PATH -# └── webrtc # ${WEBRTC_ROOT} -#    ├── CMakeLists.txt # This CMakeLists.txt itself (copied to container) -#    ├── webrtc_common.cmake # Common configs for WebRTC (copied to container) -#    ├── .gclient -#    └── src # The actual git directory +# Expected directory layout: +# / +# ├── depot_tools/ # ${DEPOT_TOOLS_ROOT}, must be on PATH +# └── webrtc/ # ${WEBRTC_ROOT} = ${PROJECT_SOURCE_DIR} +# ├── CMakeLists.txt # this file +# ├── webrtc_common.cmake +# ├── .gclient +# └── src/ # WebRTC source tree # -# 2) CMake will compile two libraries libwebrtc.a and libwebrtc_extra.a. -# - libwebrtc.a compilation is driven by Ninja, the output will be in: -# ${WEBRTC_ROOT}/src/out/Release/obj/libwebrtc.a -# - libwebrtc_extra.a compilation is driven by Ninja but CMake packages the -# object files into a static library, the output will be in: -# ${WEBRTC_ROOT}/src/out/Release/obj/libwebrtc_extra.a +# Outputs (relative to CMAKE_INSTALL_PREFIX): +# lib/libwebrtc.a - main WebRTC static lib (built by gn/ninja) +# lib/libwebrtc_extra.a - supplementary objects packaged by CMake +# include/ - headers from webrtc/src/ # -# 3) Finally, `make install` will install headers and binaries to -# - build/lib -# - build/include - +# Build: +# cmake -G Ninja -DCMAKE_INSTALL_PREFIX= /webrtc +# ninja -j$(nproc) install cmake_minimum_required(VERSION 3.18) project(webrtc CXX) @@ -31,6 +31,9 @@ cmake_dependent_option(WEBRTC_IS_DEBUG "WebRTC Debug build. Use ON for Win32 Open3D Debug." OFF "NOT CMAKE_BUILD_TYPE STREQUAL Debug OR NOT WIN32" ON) option(GLIBCXX_USE_CXX11_ABI "Set -D_GLIBCXX_USE_CXX11_ABI=1" ON) +if(MSVC) + option(WEBRTC_STATIC_MSVC_RUNTIME "Use /MT /MTd for WebRTC (Windows)" ON) +endif() # Set paths set(WEBRTC_ROOT ${PROJECT_SOURCE_DIR}) @@ -55,12 +58,10 @@ set(WEBRTC_NINJA_ROOT ${WEBRTC_ROOT}/src/out/${WEBRTC_BUILD}) # Common configs for WebRTC include(${PROJECT_SOURCE_DIR}/webrtc_common.cmake) -# Generate build/args.gn -if(NOT EXISTS ${WEBRTC_NINJA_ROOT}/args.gn) - get_webrtc_args(WEBRTC_ARGS) - file(WRITE ${WEBRTC_NINJA_ROOT}/args.gn ${WEBRTC_ARGS}) - message(STATUS "Configs written to ${WEBRTC_NINJA_ROOT}/args.gn") -endif() +get_webrtc_args(WEBRTC_ARGS) +file(MAKE_DIRECTORY "${WEBRTC_NINJA_ROOT}") +file(WRITE "${WEBRTC_NINJA_ROOT}/args.gn" "${WEBRTC_ARGS}") +message(STATUS "WebRTC args.gn -> ${WEBRTC_NINJA_ROOT}/args.gn") # libwebrtc.a add_custom_target(webrtc @@ -82,19 +83,13 @@ set_target_properties(webrtc_extra PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${WEBRTC_NINJA_ROOT}/obj ) -# Install headers and binaries -# /webrtc_install -# |-- include -# | |-- api -# | |-- audio -# ... -# | |-- tools_webrtc -# | `-- video -# `-- lib -# |-- libwebrtc.a -# `-- libwebrtc_extra.a +# Install headers and libs into CMAKE_INSTALL_PREFIX: +# include/ - all .h/.hpp/.inc headers mirroring webrtc/src/ structure +# lib/ - libwebrtc.a libwebrtc_extra.a (or .lib on Windows) file(GLOB_RECURSE WEBRTC_INCLUDES RELATIVE ${WEBRTC_ROOT}/src ${WEBRTC_ROOT}/src/*.h + ${WEBRTC_ROOT}/src/*.hpp + ${WEBRTC_ROOT}/src/*.inc ) foreach(header ${WEBRTC_INCLUDES}) get_filename_component(dir ${header} DIRECTORY) @@ -105,3 +100,22 @@ install(FILES ${WEBRTC_NINJA_ROOT}/obj/${CMAKE_STATIC_LIBRARY_PREFIX}webrtc_extra${CMAKE_STATIC_LIBRARY_SUFFIX} DESTINATION lib ) + +# Release prebuilts: strip debug sections from installed static libraries (Unix/macOS). +# MSVC .lib files are kept small via GN symbol_level=0; COFF static libs have no +# equivalent strip tool in the MSVC toolchain. +if(NOT WEBRTC_IS_DEBUG AND UNIX AND CMAKE_STRIP) + # macOS strip uses -S (debug symbols only); GNU strip uses --strip-debug. + if(APPLE) + set(_webrtc_strip_flags -S) + else() + set(_webrtc_strip_flags --strip-debug) + endif() + install(CODE " + file(GLOB _webrtc_libs \"\${CMAKE_INSTALL_PREFIX}/lib/*${CMAKE_STATIC_LIBRARY_SUFFIX}\") + foreach(_lib \${_webrtc_libs}) + execute_process(COMMAND \"${CMAKE_STRIP}\" ${_webrtc_strip_flags} \"\${_lib}\" + ERROR_QUIET) + endforeach() + ") +endif() diff --git a/3rdparty/webrtc/README.md b/3rdparty/webrtc/README.md index eb54b078089..c00b8d09a8a 100644 --- a/3rdparty/webrtc/README.md +++ b/3rdparty/webrtc/README.md @@ -13,13 +13,35 @@ webrtc_download.cmake # Used by Open3D CMake. Consume pre-compiled WebRTC. (Meth webrtc_build.cmake # Used by Open3D CMake. Build and consume WebRTC. (Method 2) # Other files -0001-xxx.patch x3 # Git patch for -DBUILD_WEBRTC_FROM_SOURCE=ON. (Method 1 Prepare-Phase & Method 2) +000*.patch # Git patches applied before building WebRTC. (Method 1 Prepare-Phase & Method 2) +apply_webrtc_patches.sh # Applies the patches to the WebRTC checkout. (Method 1 Prepare-Phase & Method 2) CMakeLists.txt # Used by `webrtc_build.sh` to compile WebRTC. (Method 1 Prepare-Phase) Dockerfile.webrtc # Calls `webrtc_build.sh` to compile WebRTC. (Method 1 Prepare-Phase) webrtc_build.sh # Used by `Dockerfile.webrtc`. (Method 1 Prepare-Phase) -webrtc_common.cmake # Specifies Common WebRTC targets. (Method 1 Prepare-Phase) +webrtc_common.cmake # Specifies Common WebRTC targets and gn args. (Method 1 Prepare-Phase) ``` +## Patches + +Applied by `apply_webrtc_patches.sh` (each is skipped if it does not apply +cleanly to the pinned WebRTC commit): + +``` +0001-src-enable-rtc_use_cxx11_abi-option.patch # -> src +0001-build-enable-rtc_use_cxx11_abi-option.patch # -> src/build +0001-third_party-enable-rtc_use_cxx11_abi-option.patch # -> src/third_party +0002-build-enable_safe_libstdcxx.patch # -> src/build_overrides +0003-src-fix-nullptr_t-with-libstdcxx.patch # -> src (GCC) +0004-src-gcc-suppress-port-interface-network.patch # -> src (GCC) +0005-call-payload_type_picker-gcc-flat_tree.patch # -> src (GCC) +0006-build-win-dynamic-crt.patch # -> src/build (Windows /MD) +``` + +`0006-build-win-dynamic-crt.patch` adds a `rtc_win_dynamic_crt` gn arg so a +non-component (static) Windows build can use the dynamic MSVC runtime +(`/MD[d]`). This lets Open3D ship a static `libwebrtc` for both +`STATIC_WINDOWS_RUNTIME=ON` (`/MT[d]`) and `OFF` (`/MD[d]`). + ## Method 1 The pre-compiled WebRTC package used in Method 1 is generated by diff --git a/3rdparty/webrtc/apply_webrtc_patches.sh b/3rdparty/webrtc/apply_webrtc_patches.sh new file mode 100755 index 00000000000..287877e882e --- /dev/null +++ b/3rdparty/webrtc/apply_webrtc_patches.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Apply Open3D WebRTC patches under WEBRTC_SRC (webrtc checkout src/). +set -euo pipefail + +OPEN3D_DIR="${1:?Open3D repo path}" +WEBRTC_SRC="${2:?WebRTC src path}" + +# Apply a patch, hard-failing if a required patch cannot be applied. +# +# Args: [required|optional] (default: required) +# +# A patch is considered "already applied" when it applies in reverse; in that +# case it is skipped without error so the script is safe to re-run and tolerant +# of fixes that have landed upstream. A required patch that neither applies nor +# is already applied aborts the build: these patches add gn args / ABI defines +# or fix compile errors, so silently skipping them produces broken or +# confusing artifacts (e.g. an undeclared gn arg in args.gn, a C++11/C++17 ABI +# mismatch, or a hard compile error) rather than a clear failure here. +apply_one() { + local patch="$1" + local dir="$2" + local required="${3:-required}" + local name + name="$(basename "$patch")" + if git -C "$dir" apply --check "$patch" 2>/dev/null; then + git -C "$dir" apply "$patch" + echo "Applied $name in $dir" + elif git -C "$dir" apply --reverse --check "$patch" 2>/dev/null; then + echo "Skip $name (already applied) in $dir" + elif [[ "$required" == "optional" ]]; then + echo "Skip $name (does not apply; optional) in $dir" + else + echo "ERROR: required patch $name does not apply in $dir." >&2 + echo " Refresh the patch for the pinned WebRTC commit." >&2 + exit 1 + fi +} + +PATCH_DIR="$OPEN3D_DIR/3rdparty/webrtc" +# Required: declare gn args consumed by args.gn and fix GCC compile errors. +apply_one "$PATCH_DIR/0001-src-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0001-build-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/build" +apply_one "$PATCH_DIR/0001-third_party-enable-rtc_use_cxx11_abi-option.patch" "$WEBRTC_SRC/third_party" +apply_one "$PATCH_DIR/0002-src-fix-nullptr_t-with-libstdcxx.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0003-src-gcc-suppress-port-interface-network.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0004-call-payload_type_picker-gcc-flat_tree.patch" "$WEBRTC_SRC" +apply_one "$PATCH_DIR/0005-build-win-dynamic-crt.patch" "$WEBRTC_SRC/build" +apply_one "$PATCH_DIR/0006-third_party-protobuf-disable-constinit-on-apple.patch" "$WEBRTC_SRC/third_party/protobuf" diff --git a/3rdparty/webrtc/webrtc_build.sh b/3rdparty/webrtc/webrtc_build.sh index b4e48f126b2..37011ad0d14 100755 --- a/3rdparty/webrtc/webrtc_build.sh +++ b/3rdparty/webrtc/webrtc_build.sh @@ -1,78 +1,155 @@ #!/usr/bin/env bash -set -euox pipefail - -# This script builds WebRTC for Open3D for Ubuntu and macOS. For Windows, see -# .github/workflows/webrtc.yml -# -# Usage: -# $ bash # Start a new shell -# Specify custom configuration by exporting environment variables -# GLIBCXX_USE_CXX11_ABI, WEBRTC_COMMIT and DEPOT_TOOLS_COMMIT, if required. -# $ source 3rdparty/webrtc/webrtc_build.sh -# $ install_dependencies_ubuntu # Ubuntu only -# $ download_webrtc_sources -# $ build_webrtc -# A webrtc__platform.tar.gz file will be created that can be used to -# build Open3D with WebRTC support. -# -# Procedure: +# Build WebRTC static libraries for Open3D (Ubuntu/macOS). +# Windows uses download_webrtc_sources() from this file via Git Bash; +# the cmake/ninja build itself is driven by the webrtc.yml PowerShell steps. # -# 1) Download depot_tools, webrtc to following directories: -# ├── Oepn3D -# ├── depot_tools -# └── webrtc -#    ├── .gclient -#    └── src +# This file is sourced (not executed) by CI steps so that functions are +# available as shell commands. Sourcing applies `set -euo pipefail` to the +# calling shell for strict error checking across the entire CI step. # -# 2) depot_tools and webrtc have compatible versions, see: -# https://chromium.googlesource.com/chromium/src/+/master/docs/building_old_revisions.md +# Expected directory layout ( = parent of the Open3D checkout, or +# $WEBRTC_WORK_ROOT if set): +# / +# ├── Open3D/ # this repository +# ├── depot_tools/ # fetched by clone_depot_tools() +# └── webrtc/ +# ├── .gclient # created by `fetch --nohooks --no-history webrtc` +# └── src/ # WebRTC source tree, pinned to WEBRTC_COMMIT # -# 3) Apply the following patch to enable GLIBCXX_USE_CXX11_ABI selection: -# - 0001-build-enable-rtc_use_cxx11_abi-option.patch # apply to webrtc/src -# - 0001-src-enable-rtc_use_cxx11_abi-option.patch # apply to webrtc/src/build -# - 0001-third_party-enable-rtc_use_cxx11_abi-option.patch # apply to webrtc/src/third_party -# Note that these patches may or may not be compatible with your custom -# WebRTC commits. You may have to patch them manually. - -# Date: Wed Apr 7 19:12:13 2021 +0200 -WEBRTC_COMMIT=${WEBRTC_COMMIT:-60e674842ebae283cc6b2627f4b6f2f8186f3317} -# Date: Wed Apr 7 21:35:29 2021 +0000 -DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-e1a98941d3ab10549be6d82d0686bb0fb91ec903} - -GLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI:-0} -NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN)} # POSIX: MacOS + Linux -SUDO=${SUDO:-sudo} # Set to command if running inside docker -export PATH="$PWD/../depot_tools":${PATH} # $(basename $PWD) == Open3D -export DEPOT_TOOLS_UPDATE=0 +# Usage (Unix): +# source 3rdparty/webrtc/webrtc_build.sh +# install_dependencies_ubuntu # Ubuntu only +# download_webrtc_sources # fetches depot_tools + runs gclient sync +# build_webrtc # cmake/ninja build, installs, packages tar.gz + +set -euo pipefail + +# libwebrtc-bin M149 / Open3D target milestone +WEBRTC_COMMIT=${WEBRTC_COMMIT:-e8b4d4c5952a8fb7b35c2a6cba4e8c3de2ea2e1e} +# Pinned depot_tools (update intentionally when refreshing the WebRTC toolchain). +DEPOT_TOOLS_COMMIT=${DEPOT_TOOLS_COMMIT:-10eda50a3fd9c34ad8d31ec74e5f4eb5823d60f6} +DEPOT_TOOLS_URL="https://chromium.googlesource.com/chromium/tools/depot_tools" + +GLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI:-1} +NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)} +SUDO=${SUDO:-sudo} +# Parallel gclient git operations (speeds DEPS fetch on CI). +GCLIENT_JOBS=${GCLIENT_JOBS:-${NPROC}} + +_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +webrtc_work_root() { + if [[ -n "${WEBRTC_WORK_ROOT:-}" ]]; then + echo "$WEBRTC_WORK_ROOT" + else + dirname "$_OPEN3D_ROOT" + fi +} + +webrtc_setup_path() { + local root + root="$(webrtc_work_root)" + # On Windows the work root is a native path like 'C:\WebRTC'. The colon in + # the drive letter is bash's PATH separator, which would split + # 'C:\WebRTC/depot_tools' into the two entries 'C' and '\WebRTC/depot_tools'. + # '\WebRTC/...' then causes sha256sum to prefix its output with '\' (GNU + # coreutils escapes paths that contain backslashes), making the CIPD hash + # check fail. cygpath is available in Git Bash / MSYS2; convert to POSIX. + if command -v cygpath >/dev/null 2>&1; then + root="$(cygpath -u "$root")" + fi + export PATH="${root}/depot_tools:${PATH}" + export DEPOT_TOOLS_UPDATE=0 +} + +# Fetch a pinned depot_tools tree via Gitiles tarball. +clone_depot_tools() { + local root="$1" + local dest="$root/depot_tools" + local commit="$DEPOT_TOOLS_COMMIT" + local stamp="$dest/.open3d_pinned_commit" + + if [[ -f "$stamp" && "$(cat "$stamp")" == "$commit" && -x "$dest/fetch" ]]; then + return 0 + fi + + local tmp archive + tmp="$(mktemp -d)" + archive="$tmp/depot_tools.tar.gz" + curl -fL --retry 3 --retry-delay 5 \ + -o "$archive" "${DEPOT_TOOLS_URL}/+archive/${commit}.tar.gz" + rm -rf "$dest" + mkdir -p "$dest" + # Gitiles +archive tarballs unpack flat (fetch at archive root, not in a subdir). + # On Windows (Git Bash), symlinks in the archive fail to extract because + # symlink creation requires elevated privileges. Those symlinks are + # Linux-only helper scripts (cbuildbot, luci-auth-fido2-plugin, etc.) and + # are not needed for WebRTC builds. The 'fetch' check below validates the + # critical tools were extracted. + tar -xzf "$archive" -C "$dest" || true + rm -rf "$tmp" + + if [[ ! -x "$dest/fetch" ]]; then + echo "ERROR: depot_tools archive at ${commit} is missing fetch" >&2 + exit 1 + fi + + # Bootstrap depot_tools. + # We must temporarily prepend depot_tools to PATH so that python scripts + # and subprocesses launched during bootstrap (like gsutil.py calling luci-auth) + # can find the depot_tools executables on both Unix and Windows. + local old_path="$PATH" + export PATH="${dest}:${PATH}" + + if [[ "$(uname -s)" == *"MINGW"* || "$(uname -s)" == *"MSYS"* || "$(uname -s)" == *"CYGWIN"* ]]; then + # On Windows, bootstrap Python and Git via the batch files. + # This creates git.bat, python3.bat, and downloads cipd tools. + # We must run them using cmd.exe inside the depot_tools directory. + pushd "$dest" + cmd.exe //c "cipd_bin_setup.bat" + cmd.exe //c "bootstrap\\win_tools.bat" + popd + else + # On Unix (Ubuntu/macOS), ensure_bootstrap downloads Python 3 via CIPD and writes python3_bin_reldir.txt. + # DEPOT_TOOLS_DIR must be set so ensure_bootstrap resolves scripts correctly. + DEPOT_TOOLS_DIR="$dest" "$dest/ensure_bootstrap" + fi + + export PATH="$old_path" + + echo "$commit" > "$stamp" +} install_dependencies_ubuntu() { options="$(echo "$@" | tr ' ' '|')" - # Dependencies - # python* : resolve ImportError: No module named pkg_resources - # libglib2.0-dev: resolve pkg_config("glib") $SUDO apt-get update $SUDO apt-get install -y \ apt-transport-https \ build-essential \ ca-certificates \ + clang \ git \ gnupg \ libglib2.0-dev \ - python \ - python-pip \ - python-setuptools \ - python-wheel \ + libnss3-dev \ + libgtk-3-dev \ + ninja-build \ + python3 \ + python3-pip \ + python3-setuptools \ + pkg-config \ software-properties-common \ tree \ curl - curl https://apt.kitware.com/keys/kitware-archive-latest.asc \ - 2>/dev/null | gpg --dearmor - | - $SUDO sed -n 'w /etc/apt/trusted.gpg.d/kitware.gpg' # Write to file, no stdout - source <(grep VERSION_CODENAME /etc/os-release) - $SUDO apt-add-repository --yes "deb https://apt.kitware.com/ubuntu/ $VERSION_CODENAME main" - $SUDO apt-get update - $SUDO apt-get --yes install cmake - cmake --version >/dev/null + if ! command -v cmake >/dev/null 2>&1 || [[ $(cmake --version | head -1 | grep -oE '[0-9]+\.[0-9]+') < "3.18" ]]; then + curl https://apt.kitware.com/keys/kitware-archive-latest.asc \ + 2>/dev/null | gpg --dearmor - | + $SUDO tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null + source <(grep VERSION_CODENAME /etc/os-release) + $SUDO apt-add-repository --yes "deb https://apt.kitware.com/ubuntu/ $VERSION_CODENAME main" + $SUDO apt-get update + $SUDO apt-get --yes install cmake + fi if [[ "purge-cache" =~ ^($options)$ ]]; then $SUDO apt-get clean $SUDO rm -rf /var/lib/apt/lists/* @@ -80,67 +157,65 @@ install_dependencies_ubuntu() { } download_webrtc_sources() { - # PWD=Open3D - pushd .. - echo Get depot_tools - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - git -C depot_tools checkout $DEPOT_TOOLS_COMMIT + local root + root="$(webrtc_work_root)" + + pushd "$root" + clone_depot_tools "$root" + webrtc_setup_path + # Verify fetch is on PATH (exits non-zero under set -e if not found). command -V fetch - echo Get WebRTC - mkdir webrtc - cd webrtc - fetch webrtc - - # Checkout to a specific version - # Ref: https://chromium.googlesource.com/chromium/src/+/master/docs/building_old_revisions.md - git -C src checkout $WEBRTC_COMMIT - git -C src submodule update --init --recursive - echo gclient sync - gclient sync -D --force --reset - cd .. - echo random.org - curl "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" -o skipcache + if [[ ! -d webrtc/src ]]; then + mkdir -p webrtc + pushd webrtc + fetch --nohooks --no-history webrtc + popd + fi + + pushd webrtc + gclient sync -D --force --reset --no-history \ + --jobs="${GCLIENT_JOBS}" \ + --revision "src@${WEBRTC_COMMIT}" + popd popd } build_webrtc() { - # PWD=Open3D - OPEN3D_DIR="$PWD" - echo Apply patches - cp 3rdparty/webrtc/{CMakeLists.txt,webrtc_common.cmake} ../webrtc - git -C ../webrtc/src apply \ - "$OPEN3D_DIR"/3rdparty/webrtc/0001-src-enable-rtc_use_cxx11_abi-option.patch - git -C ../webrtc/src/build apply \ - "$OPEN3D_DIR"/3rdparty/webrtc/0001-build-enable-rtc_use_cxx11_abi-option.patch - git -C ../webrtc/src/third_party apply \ - "$OPEN3D_DIR"/3rdparty/webrtc/0001-third_party-enable-rtc_use_cxx11_abi-option.patch - WEBRTC_COMMIT_SHORT=$(git -C ../webrtc/src rev-parse --short=7 HEAD) - - echo Build WebRTC - mkdir ../webrtc/build - pushd ../webrtc/build - cmake -DCMAKE_INSTALL_PREFIX=../../webrtc_release \ - -DGLIBCXX_USE_CXX11_ABI=${GLIBCXX_USE_CXX11_ABI} \ + local root open3d_dir + open3d_dir="$_OPEN3D_ROOT" + root="$(webrtc_work_root)" + webrtc_setup_path + + cp "$open3d_dir"/3rdparty/webrtc/{CMakeLists.txt,webrtc_common.cmake} "$root/webrtc/" + bash "$open3d_dir"/3rdparty/webrtc/apply_webrtc_patches.sh \ + "$open3d_dir" "$root/webrtc/src" + + WEBRTC_COMMIT_SHORT=$(git -C "$root/webrtc/src" rev-parse --short=7 HEAD) + + mkdir -p "$root/webrtc/build" + pushd "$root/webrtc/build" + cmake -G Ninja \ + -DCMAKE_INSTALL_PREFIX="$root/webrtc_release" \ + -DGLIBCXX_USE_CXX11_ABI="${GLIBCXX_USE_CXX11_ABI}" \ .. - make -j$NPROC - make install - popd # PWD=Open3D - pushd .. - tree -L 2 webrtc_release || ls webrtc_release/* + ninja -j"${NPROC}" install + popd - echo Package WebRTC + pushd "$root" + tree -L 2 webrtc_release || ls -la webrtc_release if [[ $(uname -s) == 'Linux' ]]; then tar -czf \ - "$OPEN3D_DIR/webrtc_${WEBRTC_COMMIT_SHORT}_linux_cxx-abi-${GLIBCXX_USE_CXX11_ABI}.tar.gz" \ - webrtc_release + "$open3d_dir/webrtc_${WEBRTC_COMMIT_SHORT}_linux_cxx-abi-${GLIBCXX_USE_CXX11_ABI}.tar.gz" \ + -C "$root/webrtc_release" . elif [[ $(uname -s) == 'Darwin' ]]; then tar -czf \ - "$OPEN3D_DIR/webrtc_${WEBRTC_COMMIT_SHORT}_macos.tar.gz" \ - webrtc_release + "$open3d_dir/webrtc_${WEBRTC_COMMIT_SHORT}_macos_arm64.tar.gz" \ + -C "$root/webrtc_release" . fi - popd # PWD=Open3D - webrtc_package=$(ls webrtc_*.tar.gz) - cmake -E sha256sum "$webrtc_package" | tee "checksum_${webrtc_package%%.*}.txt" + popd + + webrtc_package=$(ls "$open3d_dir"/webrtc_*.tar.gz | tail -1) + cmake -E sha256sum "$webrtc_package" | tee "$open3d_dir/checksum_${webrtc_package##*/}" | sed 's|.*/||' ls -alh "$webrtc_package" } diff --git a/3rdparty/webrtc/webrtc_common.cmake b/3rdparty/webrtc/webrtc_common.cmake index 98c19336c6a..712595aaf30 100644 --- a/3rdparty/webrtc/webrtc_common.cmake +++ b/3rdparty/webrtc/webrtc_common.cmake @@ -1,16 +1,19 @@ -# Common configs for building WebRTC from source. Used in both native build -# and building inside docker. +# Common GN args and ninja target lists for building WebRTC from source. +# Included by CMakeLists.txt (which is driven by webrtc_build.sh on Unix CI and +# by webrtc.yml PowerShell steps on Windows CI). +# +# Callers must set WEBRTC_NINJA_ROOT before including this file. # # Exports: -# - get_webrtc_args(WEBRTC_ARGS) function -# - NINJA_TARGETS -# - EXTRA_WEBRTC_OBJS # You have to define WEBRTC_NINJA_ROOT before including this file +# get_webrtc_args(OUT_VAR) - function: returns a newline-separated args.gn string +# NINJA_TARGETS - list of gn targets to build +# EXTRA_WEBRTC_OBJS - object files not in libwebrtc.a, packed into libwebrtc_extra.a function(get_webrtc_args WEBRTC_ARGS) set(WEBRTC_ARGS "") if(NOT MSVC) - # ABI selection + # ABI selection (Linux only; Open3D Ubuntu 22.04 uses cxx11 ABI=1). if(GLIBCXX_USE_CXX11_ABI) set(WEBRTC_ARGS rtc_use_cxx11_abi=true\n${WEBRTC_ARGS}) else() @@ -18,57 +21,67 @@ function(get_webrtc_args WEBRTC_ARGS) endif() endif() - if (APPLE) # WebRTC default + if(APPLE) set(WEBRTC_ARGS is_clang=true\n${WEBRTC_ARGS}) - else() - # Do not use Google clang for compilation due to LTO error when Open3D - # is built with gcc on Ubuntu 20.04. + if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(WEBRTC_ARGS target_cpu=\"arm64\"\n${WEBRTC_ARGS}) + endif() + elseif(UNIX) set(WEBRTC_ARGS is_clang=false\n${WEBRTC_ARGS}) + set(WEBRTC_ARGS extra_cxxflags=[\"-Wno-changes-meaning\"]\n${WEBRTC_ARGS}) endif() - # Don't use libc++ (Clang), use libstdc++ (GNU) - # https://stackoverflow.com/a/47384787/1255535 set(WEBRTC_ARGS use_custom_libcxx=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS use_custom_libcxx_for_host=false\n${WEBRTC_ARGS}) - # Debug/Release if(WEBRTC_IS_DEBUG) set(WEBRTC_ARGS is_debug=true\n${WEBRTC_ARGS}) - if (MSVC) - # WebRTC default is false in Debug due to a performance penalty, but this would disable - # iterator debugging for Open3D and any user code as well with MSVC. + if(MSVC) set(WEBRTC_ARGS enable_iterator_debugging=true\n${WEBRTC_ARGS}) endif() else() set(WEBRTC_ARGS is_debug=false\n${WEBRTC_ARGS}) + # Smaller static libs for prebuilt packages (no need for debug symbols). + set(WEBRTC_ARGS symbol_level=0\n${WEBRTC_ARGS}) endif() - # H264 support - set(WEBRTC_ARGS is_chrome_branded=true\n${WEBRTC_ARGS}) + # H.264 (replaces deprecated is_chrome_branded on recent milestones). + set(WEBRTC_ARGS rtc_use_h264=true\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_include_tests=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_enable_protobuf=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_build_examples=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_build_tools=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS treat_warnings_as_errors=false\n${WEBRTC_ARGS}) - set(WEBRTC_ARGS rtc_enable_libevent=false\n${WEBRTC_ARGS}) - set(WEBRTC_ARGS rtc_build_libevent=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS use_sysroot=false\n${WEBRTC_ARGS}) + set(WEBRTC_ARGS rtc_use_perfetto=false\n${WEBRTC_ARGS}) - # Disable screen capturing set(WEBRTC_ARGS rtc_use_x11=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_use_pipewire=false\n${WEBRTC_ARGS}) - # Disable sound support set(WEBRTC_ARGS rtc_include_pulse_audio=false\n${WEBRTC_ARGS}) set(WEBRTC_ARGS rtc_include_internal_audio_device=false\n${WEBRTC_ARGS}) - # Use ccache if available, not recommended inside Docker + if(MSVC) + # Always build a static (non-component) libwebrtc that uses the MSVC STL + # (use_custom_libcxx=false, set above) so its ABI matches Open3D. Force + # is_component_build=false because it defaults to ON in Debug, which would + # produce shared component DLLs instead of a static lib. The MSVC runtime + # (/MT[d] vs /MD[d]) is selected independently via the rtc_win_dynamic_crt + # gn arg added by 0006-build-win-dynamic-crt.patch. + set(WEBRTC_ARGS is_component_build=false\n${WEBRTC_ARGS}) + if(WEBRTC_STATIC_MSVC_RUNTIME) + set(WEBRTC_ARGS rtc_win_dynamic_crt=false\n${WEBRTC_ARGS}) + else() + set(WEBRTC_ARGS rtc_win_dynamic_crt=true\n${WEBRTC_ARGS}) + endif() + endif() + find_program(CCACHE_BIN "ccache") if(CCACHE_BIN) - set(WEBRTC_ARGS cc_wrapper="ccache"\n${WEBRTC_ARGS}) + set(WEBRTC_ARGS cc_wrapper=\"ccache\"\n${WEBRTC_ARGS}) endif() - set(WEBRTC_ARGS ${WEBRTC_ARGS} PARENT_SCOPE) + set(WEBRTC_ARGS ${WEBRTC_ARGS} PARENT_SCOPE) endfunction() # webrtc -> libwebrtc.a @@ -79,13 +92,17 @@ set(NINJA_TARGETS jsoncpp builtin_video_decoder_factory builtin_video_encoder_factory - peerconnection + peer_connection p2p_server_utils task_queue default_task_queue_factory + # M149 modular PeerConnectionFactory (not all pulled into libwebrtc.a). + field_trials + enable_media_with_defaults + create_modular_peer_connection_factory + environment_factory ) -# Byproducts for ninja build, later packaged by CMake into libwebrtc_extra.a if(NOT WEBRTC_NINJA_ROOT) message(FATAL_ERROR "Please define WEBRTC_NINJA_ROOT before including webrtc_common.cmake") endif() @@ -96,4 +113,20 @@ set(EXTRA_WEBRTC_OBJS ${WEBRTC_NINJA_ROOT}/obj/p2p/p2p_server_utils/stun_server${CMAKE_CXX_OUTPUT_EXTENSION} ${WEBRTC_NINJA_ROOT}/obj/p2p/p2p_server_utils/turn_server${CMAKE_CXX_OUTPUT_EXTENSION} ${WEBRTC_NINJA_ROOT}/obj/rtc_base/rtc_json/json${CMAKE_CXX_OUTPUT_EXTENSION} - ) + ${WEBRTC_NINJA_ROOT}/obj/api/field_trials/field_trials${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/field_trials_registry/field_trials_registry${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/enable_media/enable_media${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/enable_media_with_defaults/enable_media_with_defaults${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/create_modular_peer_connection_factory/create_modular_peer_connection_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/environment/environment_factory/environment_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/environment/deprecated_global_field_trials/deprecated_global_field_trials${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/audio_codecs/builtin_audio_encoder_factory/builtin_audio_encoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/audio_codecs/builtin_audio_decoder_factory/builtin_audio_decoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/builtin_video_encoder_factory/builtin_video_encoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/builtin_video_decoder_factory/builtin_video_decoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/media/rtc_simulcast_encoder_adapter/simulcast_encoder_adapter${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/media/rtc_internal_video_codecs/internal_encoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/media/rtc_internal_video_codecs/internal_decoder_factory${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/rtc_software_fallback_wrappers/video_encoder_software_fallback_wrapper${CMAKE_CXX_OUTPUT_EXTENSION} + ${WEBRTC_NINJA_ROOT}/obj/api/video_codecs/rtc_software_fallback_wrappers/video_decoder_software_fallback_wrapper${CMAKE_CXX_OUTPUT_EXTENSION} +) diff --git a/3rdparty/webrtc/webrtc_download.cmake b/3rdparty/webrtc/webrtc_download.cmake index 8ac5a0b6ef7..075c4ac00d8 100644 --- a/3rdparty/webrtc/webrtc_download.cmake +++ b/3rdparty/webrtc/webrtc_download.cmake @@ -4,43 +4,74 @@ include(ExternalProject) -set(WEBRTC_VER 60e6748) +set(WEBRTC_VER e8b4d4c) +if(DEFINED ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE} AND NOT "$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}" STREQUAL "") + set(WEBRTC_URL "file://$ENV{OPEN3D_WEBRTC_PREBUILT_ARCHIVE}") +endif() if (APPLE) - set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc/webrtc_${WEBRTC_VER}_macos_10.14.tar.gz - ) - set(WEBRTC_SHA256 e9d1f4e4fefb2e28ef4f16cf4a4f0008baf4fe638ca3ad329e82e7fd0ce87f56) + if(NOT WEBRTC_URL) + set(WEBRTC_URL + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_macos_arm64.tar.gz + ) + endif() + # Update after publishing M149 macOS arm64 artifact from webrtc.yml. + set(WEBRTC_SHA256 PLACEHOLDER_MACOS_ARM64_SHA256) elseif (WIN32) - if (BUILD_SHARED_LIBS OR NOT STATIC_WINDOWS_RUNTIME) - message(FATAL_ERROR "Pre-built WebRTC binaries are not available for " - "BUILD_SHARED_LIBS=ON or STATIC_WINDOWS_RUNTIME=OFF. Please use " - "(a) BUILD_WEBRTC=OFF or " - "(b) BUILD_SHARED_LIBS=OFF and STATIC_WINDOWS_RUNTIME=ON or " - "(c) BUILD_WEBRTC_FROM_SOURCE=ON") + # Prebuilt WebRTC is a static lib for both MSVC runtimes. A shared Open3D + # build links it into open3d.dll and always uses the dynamic runtime, so + # only BUILD_SHARED_LIBS=ON + STATIC_WINDOWS_RUNTIME=ON is unsupported. + if (BUILD_SHARED_LIBS AND STATIC_WINDOWS_RUNTIME) + message(FATAL_ERROR "Pre-built WebRTC does not support " + "BUILD_SHARED_LIBS=ON with STATIC_WINDOWS_RUNTIME=ON. Use " + "STATIC_WINDOWS_RUNTIME=OFF or BUILD_WEBRTC_FROM_SOURCE=ON.") endif() - set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc/webrtc_${WEBRTC_VER}_win.zip - ) - set(WEBRTC_SHA256 f4686d0028ef5c36c5d7158a638fa834b63183b522f0b63932f7f70ebffeea22) -else() # Linux - if(GLIBCXX_USE_CXX11_ABI) + if(STATIC_WINDOWS_RUNTIME) + set(WEBRTC_RUNTIME_TAG mt) + else() + set(WEBRTC_RUNTIME_TAG md) + endif() + if(CMAKE_BUILD_TYPE STREQUAL Debug) + set(WEBRTC_CONFIG_TAG Debug) + else() + set(WEBRTC_CONFIG_TAG Release) + endif() + if(NOT WEBRTC_URL) set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v3/webrtc_${WEBRTC_VER}_cxx-abi-1.tar.gz + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_win_${WEBRTC_CONFIG_TAG}_${WEBRTC_RUNTIME_TAG}.zip ) - set(WEBRTC_SHA256 0d98ddbc4164b9e7bfc50b7d4eaa912a753dabde0847d85a64f93a062ae4c335) - else() + endif() + # Update after publishing four Windows artifacts from webrtc.yml. + set(WEBRTC_SHA256 PLACEHOLDER_WIN_SHA256) +else() # Linux + if(NOT WEBRTC_URL) + if(NOT GLIBCXX_USE_CXX11_ABI) + message(FATAL_ERROR "Pre-built WebRTC with GLIBCXX_USE_CXX11_ABI=OFF is " + "no longer provided. Use GLIBCXX_USE_CXX11_ABI=ON or " + "BUILD_WEBRTC_FROM_SOURCE=ON.") + endif() set(WEBRTC_URL - https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v3/webrtc_${WEBRTC_VER}_cxx-abi-0.tar.gz + https://github.com/isl-org/open3d_downloads/releases/download/webrtc-v4/webrtc_${WEBRTC_VER}_linux_cxx-abi-1.tar.gz ) - set(WEBRTC_SHA256 2a3714713908f84079f1fbce8594c9b7010846b5db74b086f7bf30f22f1f5835) endif() + set(WEBRTC_SHA256 1b529bf448d5abd07ec1f8d310ee5c94bd79e84fe563ae1562420f8e478cc202) +endif() + +if(WEBRTC_SHA256 MATCHES "^PLACEHOLDER") + message(WARNING "WebRTC prebuilt SHA256 not set for this platform (${WEBRTC_SHA256}). " + "Set OPEN3D_WEBRTC_PREBUILT_ARCHIVE or update webrtc_download.cmake after CI publish.") + unset(WEBRTC_SHA256) +endif() + +set(_webrtc_url_hash "") +if(WEBRTC_SHA256) + set(_webrtc_url_hash URL_HASH SHA256=${WEBRTC_SHA256}) endif() ExternalProject_Add( ext_webrtc PREFIX webrtc URL ${WEBRTC_URL} - URL_HASH SHA256=${WEBRTC_SHA256} + ${_webrtc_url_hash} DOWNLOAD_DIR "${OPEN3D_THIRD_PARTY_DOWNLOAD_DIR}/webrtc" UPDATE_COMMAND "" CONFIGURE_COMMAND "" @@ -50,22 +81,18 @@ ExternalProject_Add( ) ExternalProject_Get_Property(ext_webrtc SOURCE_DIR) -if (WIN32) - set(SOURCE_DIR "${SOURCE_DIR}/$,Debug,Release>") -endif() -set(LIBPNG_INCLUDE_DIRS ${INSTALL_DIR}/include/) # "/" is critical. -set(LIBPNG_LIB_DIR ${INSTALL_DIR}/${Open3D_INSTALL_LIB_DIR}) -set(LIBPNG_LIBRARIES ${lib_name}$<$:d>) +# Prebuilt layout: flat include/ and lib/ at archive root (M149 packages). +set(WEBRTC_PREBUILT_ROOT ${SOURCE_DIR}) # Variables consumed by find_dependencies.cmake set(WEBRTC_INCLUDE_DIRS - ${SOURCE_DIR}/include/ - ${SOURCE_DIR}/include/third_party/abseil-cpp/ - ${SOURCE_DIR}/include/third_party/jsoncpp/source/include/ - ${SOURCE_DIR}/include/third_party/jsoncpp/generated/ - ${SOURCE_DIR}/include/third_party/libyuv/include/ + ${WEBRTC_PREBUILT_ROOT}/include/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/abseil-cpp/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/jsoncpp/source/include/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/jsoncpp/generated/ + ${WEBRTC_PREBUILT_ROOT}/include/third_party/libyuv/include/ ) -set(WEBRTC_LIB_DIR ${SOURCE_DIR}/lib) +set(WEBRTC_LIB_DIR ${WEBRTC_PREBUILT_ROOT}/lib) set(WEBRTC_LIBRARIES webrtc webrtc_extra diff --git a/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp b/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp index cf4844c1064..c16315568cd 100644 --- a/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp +++ b/cpp/open3d/t/geometry/kernel/MinimumOBE.cpp @@ -52,8 +52,8 @@ void MapOBEToClosestIdentity(EigenOBE& obe) { Eigen::Vector3d& radii = obe.radii_; Eigen::Vector3d col[3] = {R.col(0), R.col(1), R.col(2)}; double best_score = -1e9; - Eigen::Matrix3d best_R; - Eigen::Vector3d best_radii; + Eigen::Matrix3d best_R = Eigen::Matrix3d::Identity(); + Eigen::Vector3d best_radii = Eigen::Vector3d::Zero(); // Hard-coded permutations of indices [0,1,2] static const std::array, 6> permutations = { diff --git a/cpp/open3d/t/io/file_format/FileASSIMP.cpp b/cpp/open3d/t/io/file_format/FileASSIMP.cpp index 383483a1874..9cae51c804f 100644 --- a/cpp/open3d/t/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/t/io/file_format/FileASSIMP.cpp @@ -16,6 +16,7 @@ #include #include +#include "open3d/core/Dtype.h" #include "open3d/core/ParallelFor.h" #include "open3d/core/TensorFunction.h" #include "open3d/t/io/ImageIO.h" @@ -177,11 +178,31 @@ bool ReadTriangleMeshUsingASSIMP( return true; } -static void SetTextureMaterialProperty(aiMaterial* mat, - aiScene* scene, - int texture_idx, - aiTextureType tt, - t::geometry::Image& img) { +namespace { + +// Checks whether a texture map is present, not empty and uint8 or uint16 +// datatype. +bool HasValidTexture(const visualization::rendering::Material& material, + const std::string& key) { + if (!material.HasTextureMap(key) || material.GetTextureMap(key).IsEmpty()) { + return false; + } + const auto& dt = material.GetTextureMap(key).GetDtype(); + if (dt != core::UInt8 && dt != core::UInt16) { + utility::LogWarning( + "Skipping texture map '{}' with unsupported data type '{}'. " + "Only uint8 and uint16 are supported.", + key, dt.ToString()); + return false; + } + return true; +} + +void SetTextureMaterialProperty(aiMaterial* mat, + aiScene* scene, + int texture_idx, + aiTextureType tt, + t::geometry::Image& img) { // Encode image as PNG std::vector img_buffer; WriteImageToPNGInMemory(img_buffer, img, 6); @@ -208,7 +229,6 @@ static void SetTextureMaterialProperty(aiMaterial* mat, mat->AddProperty(&mode, 1, AI_MATKEY_MAPPINGMODE_V(tt, 0)); } -namespace { // Add hash function for tuple key struct TupleHash { size_t operator()(const std::tuple& t) const { @@ -546,96 +566,97 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, ai_mat->AddProperty(&ac, 1, AI_MATKEY_COLOR_EMISSIVE); } - // Count texture maps... + // Build a list of texture-embed actions in a single pass. Each action + // is a lambda that writes one texture slot into the assimp scene; the + // slot index is passed at call time. . + // // NOTE: GLTF2 expects a single combined roughness/metal map. If the // model has one we just export it, otherwise if both roughness and // metal maps are available we combine them, otherwise if only one or // the other is available we just export the one map. - int n_textures = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) ++n_textures; - if (w_mesh.GetMaterial().HasNormalMap()) ++n_textures; - if (w_mesh.GetMaterial().HasAOMap()) ++n_textures; - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { - ++n_textures; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { - ++n_textures; + const auto& material = w_mesh.GetMaterial(); + using TextureAction = std::function; + std::vector texture_actions; + + if (HasValidTexture(material, "albedo")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAlbedoMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_DIFFUSE, img); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_BASE_COLOR, img); + }); + } + if (HasValidTexture(material, "ambient_occlusion")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAOMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_LIGHTMAP, img); + }); + } + if (HasValidTexture(material, "ao_rough_metal")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAORoughnessMetalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } else if (HasValidTexture(material, "roughness") && + HasValidTexture(material, "metallic")) { + texture_actions.push_back([&](int idx) { + auto rough = material.GetRoughnessMap().AsTensor(); + auto metal = material.GetMetallicMap().AsTensor(); + if (rough.GetShape() != metal.GetShape()) { + utility::LogError( + "RoughnessMap (shape={}) and MetallicMap " + "(shape={}) must have the same shape.", + rough.GetShape(), metal.GetShape()); + } + auto rows = rough.GetShape(0); + auto cols = rough.GetShape(1); + auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, + core::Dtype::UInt8); + rough_metal.Slice(2, 2, 3) = + metal.Slice(2, 0, 1); // blue channel is metal + rough_metal.Slice(2, 1, 2) = + rough.Slice(2, 0, 1); // green channel is roughness + geometry::Image rough_metal_img(rough_metal); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, + rough_metal_img); + }); } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) ++n_textures; - if (w_mesh.GetMaterial().HasMetallicMap()) ++n_textures; + if (HasValidTexture(material, "roughness")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetRoughnessMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } + if (HasValidTexture(material, "metallic")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetMetallicMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } + } + if (HasValidTexture(material, "normal")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetNormalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_NORMALS, img); + }); } + + int n_textures = static_cast(texture_actions.size()); if (n_textures > 0) { ai_scene->mTextures = new aiTexture*[n_textures]; for (int i = 0; i < n_textures; ++i) { ai_scene->mTextures[i] = new aiTexture(); + texture_actions[i](i); } ai_scene->mNumTextures = n_textures; } - - // Now embed the textures that are available... - int current_idx = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) { - auto img = w_mesh.GetMaterial().GetAlbedoMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_DIFFUSE, img); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_BASE_COLOR, img); - ++current_idx; - } - if (w_mesh.GetMaterial().HasAOMap()) { - auto img = w_mesh.GetMaterial().GetAOMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_LIGHTMAP, img); - ++current_idx; - } - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { - auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { - auto rough = w_mesh.GetMaterial().GetRoughnessMap().AsTensor(); - auto metal = w_mesh.GetMaterial().GetMetallicMap().AsTensor(); - if (rough.GetShape() != metal.GetShape()) { - utility::LogError( - "RoughnessMap (shape={}) and MetallicMap (shape={}) " - "must have the same shape.", - rough.GetShape(), metal.GetShape()); - } - auto rows = rough.GetShape(0); - auto cols = rough.GetShape(1); - auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, - core::Dtype::UInt8); - rough_metal.Slice(2, 2, 3) = - metal.Slice(2, 0, 1); // blue channel is metal - rough_metal.Slice(2, 1, 2) = - rough.Slice(2, 0, 1); // green channel is roughness - - geometry::Image rough_metal_img(rough_metal); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, rough_metal_img); - ++current_idx; - } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) { - auto img = w_mesh.GetMaterial().GetRoughnessMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - if (w_mesh.GetMaterial().HasMetallicMap()) { - auto img = w_mesh.GetMaterial().GetMetallicMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - } - if (w_mesh.GetMaterial().HasNormalMap()) { - auto img = w_mesh.GetMaterial().GetNormalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_NORMALS, img); - ++current_idx; - } } ai_scene->mMaterials[0] = ai_mat; diff --git a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp index 3db9d0cefef..b73d680b381 100644 --- a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp +++ b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "open3d/geometry/Image.h" #include "open3d/utility/Logging.h" @@ -44,10 +45,21 @@ struct BitmapEvent { virtual void Execute() = 0; }; +// Forward declaration: BitmapDrawEvent needs a pointer to BitmapEventQueue +// to clear the per-window pending-draw flag before calling OnDraw(). +struct BitmapEventQueue; + struct BitmapDrawEvent : public BitmapEvent { - BitmapDrawEvent(BitmapWindow *target) : BitmapEvent(target) {} + // queue_ is used to clear the pending-draw flag before OnDraw() so that + // a PostRedraw() called *inside* OnDraw() is not suppressed. + BitmapEventQueue *queue_; + + BitmapDrawEvent(BitmapWindow *target, BitmapEventQueue *queue) + : BitmapEvent(target), queue_(queue) {} - void Execute() override { event_target->o3d_window->OnDraw(); } + // Execute() is defined after BitmapEventQueue (below) because it calls + // clear_pending_draw(), which requires the full BitmapEventQueue type. + void Execute() override; }; struct BitmapResizeEvent : public BitmapEvent { @@ -93,15 +105,23 @@ struct BitmapTextInputEvent : public BitmapEvent { } }; -/// Thread safe event queue (multiple producers and consumers). pop_front() and -/// push() are protected by a mutex. push() may fail if the mutex cannot be -/// acquired immediately. empty() is not protected and is not reliable. +/// Thread safe event queue (multiple producers and consumers). +/// pop_front() and push() are protected by a mutex. +/// push() may fail if the mutex cannot be acquired immediately. +/// empty() is not protected and is not reliable. +/// +/// Extended to support: +/// - Draw coalescing: at most one pending draw per window (push_draw). +/// - Input coalescing: MOVE/DRAG replace latest, WHEEL accumulates +/// dx/dy (replace_or_merge_mouse). Old mouse positions are stale; +/// processing them forces stale frames to be encoded and sent. struct BitmapEventQueue : public std::queue> { using value_t = std::shared_ptr; using super = std::queue; using super::empty; // not reliable using super::super; + // pop + front needs to be atomic for thread safety. This is exception safe // since shared_ptr copy ctor is noexcept, when it is returned by value. value_t pop_front() { @@ -110,6 +130,7 @@ struct BitmapEventQueue : public std::queue> { super::pop(); return evt; } + void push(const value_t &event) { if (evt_q_mutex_.try_lock()) { super::push(event); @@ -117,10 +138,87 @@ struct BitmapEventQueue : public std::queue> { } } + // Push a draw event only if no draw is already pending for this window. + // Returns true if pushed. The caller must pass the shared_ptr to the draw + // event; this function inserts the window into pending_draw_windows_ so + // that BitmapDrawEvent::Execute() can clear it on completion. + bool push_draw(BitmapWindow *window, const value_t &event) { + if (evt_q_mutex_.try_lock()) { + bool pushed = false; + if (pending_draw_windows_.find(window) == + pending_draw_windows_.end()) { + pending_draw_windows_.insert(window); + super::push(event); + pushed = true; + } + evt_q_mutex_.unlock(); + return pushed; + } + return false; + } + + // Called by BitmapDrawEvent::Execute() just before OnDraw() so that a + // redraw posted *during* drawing is not suppressed. + void clear_pending_draw(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // Remove all pending state for a window that is being destroyed. + void remove_window(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // For MOVE/DRAG: replace the last queued event of the same (target, type) + // with the new event (latest absolute position wins; camera controllers + // derive delta from absolute coords so intermediate positions are useless). + // For WHEEL: accumulate wheel.dx/dy into the last queued event of the same + // (target, type) so that the total scroll amount is preserved even when + // multiple notches fire faster than the render loop. + // Falls back to a normal push when no matching event is at the back. + void replace_or_merge_mouse(const value_t &event) { + std::lock_guard lock(evt_q_mutex_); + auto *new_evt = static_cast(event.get()); + if (!super::c.empty()) { + auto *back_mouse = + dynamic_cast(super::c.back().get()); + if (back_mouse && + back_mouse->event_target == new_evt->event_target && + back_mouse->event.type == new_evt->event.type) { + if (new_evt->event.type == MouseEvent::WHEEL) { + // Accumulate scroll deltas; update cursor position and + // other fields to the latest event values. + back_mouse->event.wheel.dx += new_evt->event.wheel.dx; + back_mouse->event.wheel.dy += new_evt->event.wheel.dy; + back_mouse->event.x = new_evt->event.x; + back_mouse->event.y = new_evt->event.y; + back_mouse->event.modifiers = new_evt->event.modifiers; + back_mouse->event.wheel.isTrackpad = + new_evt->event.wheel.isTrackpad; + } else { + // Replace: only the latest absolute position matters. + back_mouse->event = new_evt->event; + } + return; + } + } + super::push(event); + } + private: std::mutex evt_q_mutex_; + // Windows with a draw event currently in the queue (not yet executed). + std::unordered_set pending_draw_windows_; }; +// Out-of-class definition: BitmapEventQueue is now fully defined so +// clear_pending_draw() can be called. +void BitmapDrawEvent::Execute() { + queue_->clear_pending_draw(event_target); + event_target->o3d_window->OnDraw(); +} + } // namespace struct BitmapWindowSystem::Impl { @@ -187,18 +285,32 @@ void BitmapWindowSystem::DestroyWindow(OSWindow w) { while (!filtered_reversed.empty()) { impl_->event_queue_.push(filtered_reversed.pop_front()); } + // Clear any pending-draw entry for this window so the coalescing set + // does not hold a dangling pointer. + impl_->event_queue_.remove_window(the_deceased); // Requiem aeternam dona ei. Requiscat in pace. delete (BitmapWindow *)w; } void BitmapWindowSystem::PostRedrawEvent(OSWindow w) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw)); + // push_draw is a no-op when a draw event is already queued for this + // window, preventing redundant renders from piling up. + impl_->event_queue_.push_draw( + hw, std::make_shared(hw, &impl_->event_queue_)); } void BitmapWindowSystem::PostMouseEvent(OSWindow w, const MouseEvent &e) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw, e)); + if (e.type == MouseEvent::MOVE || e.type == MouseEvent::DRAG || + e.type == MouseEvent::WHEEL) { + // Coalesce: MOVE/DRAG replace latest (absolute position); WHEEL + // accumulates dx/dy. Only the most recent state matters for rendering. + impl_->event_queue_.replace_or_merge_mouse( + std::make_shared(hw, e)); + } else { + impl_->event_queue_.push(std::make_shared(hw, e)); + } } void BitmapWindowSystem::PostKeyEvent(OSWindow w, const KeyEvent &e) { diff --git a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp index 6fd753baf43..5a164646eb5 100644 --- a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp +++ b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.cpp @@ -39,14 +39,14 @@ void BitmapTrackSource::SetState( } void BitmapTrackSource::AddOrUpdateSink( - rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) { + webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) { RTC_DCHECK(worker_thread_checker_.IsCurrent()); source()->AddOrUpdateSink(sink, wants); } void BitmapTrackSource::RemoveSink( - rtc::VideoSinkInterface* sink) { + webrtc::VideoSinkInterface* sink) { RTC_DCHECK(worker_thread_checker_.IsCurrent()); source()->RemoveSink(sink); } diff --git a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h index 81ea8243392..5dd350a95b8 100644 --- a/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h +++ b/cpp/open3d/visualization/webrtc_server/BitmapTrackSource.h @@ -71,15 +71,15 @@ class BitmapTrackSource : public webrtc::Notifier { return absl::nullopt; } bool GetStats(Stats* stats) override { return false; } - void AddOrUpdateSink(rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) override; - void RemoveSink(rtc::VideoSinkInterface* sink) override; + void AddOrUpdateSink(webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) override; + void RemoveSink(webrtc::VideoSinkInterface* sink) override; bool SupportsEncodedOutput() const override { return false; } void GenerateKeyFrame() override {} - void AddEncodedSink(rtc::VideoSinkInterface* + void AddEncodedSink(webrtc::VideoSinkInterface* sink) override {} void RemoveEncodedSink( - rtc::VideoSinkInterface* sink) + webrtc::VideoSinkInterface* sink) override {} virtual void OnFrame(const std::shared_ptr& frame) override { @@ -88,7 +88,7 @@ class BitmapTrackSource : public webrtc::Notifier { } protected: - virtual rtc::VideoSourceInterface* source() = 0; + virtual webrtc::VideoSourceInterface* source() = 0; private: webrtc::SequenceChecker worker_thread_checker_; diff --git a/cpp/open3d/visualization/webrtc_server/CMakeLists.txt b/cpp/open3d/visualization/webrtc_server/CMakeLists.txt index f5977e72e52..97124c25de7 100644 --- a/cpp/open3d/visualization/webrtc_server/CMakeLists.txt +++ b/cpp/open3d/visualization/webrtc_server/CMakeLists.txt @@ -15,6 +15,12 @@ target_compile_definitions(webrtc_server PRIVATE _FILE_OFFSET_BITS=64 # for civetweb _LARGEFILE_SOURCE=1 # for civetweb ) +# Prebuilt WebRTC is compiled in Release (RTC_DCHECK_IS_ON=0). Match that when +# Open3D is Debug so inlined WebRTC headers do not reference missing DCHECK +# symbols (e.g. SequenceCheckerImpl::ExpectationToString). +if(NOT BUILD_WEBRTC_FROM_SOURCE) + target_compile_definitions(webrtc_server PRIVATE NDEBUG) +endif() add_dependencies(webrtc_server copy_html_dir) open3d_show_and_abort_on_warning(webrtc_server) @@ -22,6 +28,8 @@ open3d_set_global_properties(webrtc_server) open3d_set_open3d_lib_properties(webrtc_server) open3d_link_3rdparty_libraries(webrtc_server) +set_target_properties(webrtc_server PROPERTIES CXX_STANDARD 20) + if (NOT GUI_RESOURCE_DIR) message(FATAL_ERROR "GUI_RESOURCE_DIR is not defined.") @@ -29,6 +37,12 @@ endif() message(STATUS "Copying ${CMAKE_CURRENT_SOURCE_DIR}/html to ${GUI_RESOURCE_DIR}.") file(MAKE_DIRECTORY ${GUI_RESOURCE_DIR}) +# Favicon is shared with Sphinx docs (single source of truth). +set(OPEN3D_ICON "${CMAKE_SOURCE_DIR}/docs/_static/open3d_logo.ico") +if(NOT EXISTS "${OPEN3D_ICON}") + message(FATAL_ERROR "Open3D WebRTC favicon not found: ${OPEN3D_ICON}") +endif() + # Force update ${GUI_RESOURCE_DIR}/html every time. add_custom_target(copy_html_dir ALL COMMAND ${CMAKE_COMMAND} -E rm -rf @@ -36,4 +50,8 @@ add_custom_target(copy_html_dir ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/html ${GUI_RESOURCE_DIR}/html + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${OPEN3D_ICON} + ${GUI_RESOURCE_DIR}/html/open3d_logo.ico + DEPENDS ${OPEN3D_ICON} ) diff --git a/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp b/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp index a7ad0109607..4d23c2d933c 100644 --- a/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp +++ b/cpp/open3d/visualization/webrtc_server/ImageCapturer.cpp @@ -7,11 +7,13 @@ #include "open3d/visualization/webrtc_server/ImageCapturer.h" +#include #include #include #include #include #include +#include #include @@ -50,7 +52,7 @@ void ImageCapturer::OnCaptureResult( int height = (int)frame->GetShape(0); int width = (int)frame->GetShape(1); - rtc::scoped_refptr i420_buffer = + webrtc::scoped_refptr i420_buffer = webrtc::I420Buffer::Create(width, height); // frame->data() @@ -64,7 +66,7 @@ void ImageCapturer::OnCaptureResult( if (conversion_result >= 0) { webrtc::VideoFrame video_frame(i420_buffer, webrtc::VideoRotation::kVideoRotation_0, - rtc::TimeMicros()); + webrtc::TimeMicros()); if ((height_ == 0) && (width_ == 0)) { broadcaster_.OnFrame(video_frame); } else { @@ -77,13 +79,13 @@ void ImageCapturer::OnCaptureResult( } int stride_y = width; int stride_uv = (width + 1) / 2; - rtc::scoped_refptr scaled_buffer = + webrtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(width, height, stride_y, stride_uv, stride_uv); scaled_buffer->ScaleFrom( *video_frame.video_frame_buffer()->ToI420()); webrtc::VideoFrame frame = webrtc::VideoFrame( - scaled_buffer, webrtc::kVideoRotation_0, rtc::TimeMicros()); + scaled_buffer, webrtc::kVideoRotation_0, webrtc::TimeMicros()); broadcaster_.OnFrame(frame); } @@ -93,15 +95,15 @@ void ImageCapturer::OnCaptureResult( } } -// Override rtc::VideoSourceInterface. +// Override webrtc::VideoSourceInterface. void ImageCapturer::AddOrUpdateSink( - rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) { + webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) { broadcaster_.AddOrUpdateSink(sink, wants); } void ImageCapturer::RemoveSink( - rtc::VideoSinkInterface* sink) { + webrtc::VideoSinkInterface* sink) { broadcaster_.RemoveSink(sink); } diff --git a/cpp/open3d/visualization/webrtc_server/ImageCapturer.h b/cpp/open3d/visualization/webrtc_server/ImageCapturer.h index f1b094ba3ca..b8fc188cc9c 100644 --- a/cpp/open3d/visualization/webrtc_server/ImageCapturer.h +++ b/cpp/open3d/visualization/webrtc_server/ImageCapturer.h @@ -10,7 +10,8 @@ #pragma once -#include +#include +#include #include #include #include @@ -26,7 +27,7 @@ namespace open3d { namespace visualization { namespace webrtc_server { -class ImageCapturer : public rtc::VideoSourceInterface { +class ImageCapturer : public webrtc::VideoSourceInterface { public: ImageCapturer(const std::string& url_, const std::map& opts); @@ -39,23 +40,23 @@ class ImageCapturer : public rtc::VideoSourceInterface { ImageCapturer(const std::map& opts); virtual void AddOrUpdateSink( - rtc::VideoSinkInterface* sink, - const rtc::VideoSinkWants& wants) override; + webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) override; virtual void RemoveSink( - rtc::VideoSinkInterface* sink) override; + webrtc::VideoSinkInterface* sink) override; void OnCaptureResult(const std::shared_ptr& frame); protected: int width_; int height_; - rtc::VideoBroadcaster broadcaster_; + webrtc::VideoBroadcaster broadcaster_; }; class ImageTrackSource : public BitmapTrackSource { public: - static rtc::scoped_refptr Create( + static webrtc::scoped_refptr Create( const std::string& window_uid, const std::map& opts) { std::unique_ptr capturer = @@ -63,10 +64,9 @@ class ImageTrackSource : public BitmapTrackSource { if (!capturer) { return nullptr; } - rtc::scoped_refptr video_source = - new rtc::RefCountedObject( - std::move(capturer)); - return video_source; + return webrtc::scoped_refptr( + new webrtc::RefCountedObject( + std::move(capturer))); } void OnFrame(const std::shared_ptr& frame) final override { @@ -78,7 +78,7 @@ class ImageTrackSource : public BitmapTrackSource { : BitmapTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {} private: - rtc::VideoSourceInterface* source() override { + webrtc::VideoSourceInterface* source() override { return capturer_.get(); } std::unique_ptr capturer_; diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index c29696c39e2..9e1ccaca0bd 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -15,6 +15,11 @@ #include "open3d/visualization/webrtc_server/PeerConnectionManager.h" +#include +#include +#include +#include +#include #include #include #include @@ -24,9 +29,11 @@ #include #include #include +#include #include #include +#include #include #include "open3d/utility/IJsonConvertible.h" @@ -85,40 +92,41 @@ static IceServer GetIceServerFromUrl(const std::string &url) { return srv; } +static bool PeerConnectionHasStreamForWindow( + webrtc::PeerConnectionInterface* peer_connection, + const std::string& window_uid) { + if (!peer_connection) { + return false; + } + for (const auto& sender : peer_connection->GetSenders()) { + if (!sender) { + continue; + } + for (const std::string& stream_id : sender->stream_ids()) { + if (stream_id == window_uid) { + return true; + } + } + } + return false; +} + static webrtc::PeerConnectionFactoryDependencies -CreatePeerConnectionFactoryDependencies() { +CreatePeerConnectionFactoryDependencies( + webrtc::FieldTrials* field_trials) { + (void)field_trials; webrtc::PeerConnectionFactoryDependencies dependencies; + dependencies.worker_thread = webrtc::Thread::Current(); dependencies.network_thread = nullptr; - dependencies.worker_thread = rtc::Thread::Current(); dependencies.signaling_thread = nullptr; - dependencies.call_factory = webrtc::CreateCallFactory(); - dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(); - dependencies.event_log_factory = - absl::make_unique( - dependencies.task_queue_factory.get()); - - cricket::MediaEngineDependencies media_dependencies; - media_dependencies.task_queue_factory = - dependencies.task_queue_factory.get(); - - // Dummy audio factory. - rtc::scoped_refptr audio_device_module( + + webrtc::EnvironmentFactory env_factory; + env_factory.Set(field_trials); + dependencies.env = env_factory.Create(); + + dependencies.adm = webrtc::scoped_refptr( new webrtc::FakeAudioDeviceModule()); - media_dependencies.adm = std::move(audio_device_module); - media_dependencies.audio_encoder_factory = - webrtc::CreateBuiltinAudioEncoderFactory(); - media_dependencies.audio_decoder_factory = - webrtc::CreateBuiltinAudioDecoderFactory(); - media_dependencies.audio_processing = - webrtc::AudioProcessingBuilder().Create(); - - media_dependencies.video_encoder_factory = - webrtc::CreateBuiltinVideoEncoderFactory(); - media_dependencies.video_decoder_factory = - webrtc::CreateBuiltinVideoDecoderFactory(); - - dependencies.media_engine = - cricket::CreateMediaEngine(std::move(media_dependencies)); + webrtc::EnableMediaWithDefaults(dependencies); return dependencies; } @@ -128,12 +136,16 @@ PeerConnectionManager::PeerConnectionManager( const Json::Value &config, const std::string &publish_filter, const std::string &webrtc_udp_port_range) - : task_queue_factory_(webrtc::CreateDefaultTaskQueueFactory()), + : field_trials_(webrtc::FieldTrials::Create( + "WebRTC-Pacer-DrainQueue/Enabled/" + "WebRTC-ForceSendPlayoutDelay/min_ms:0,max_ms:0/" + "WebRTC-Video-DisableAutomaticResize/Enabled/")), peer_connection_factory_(webrtc::CreateModularPeerConnectionFactory( - CreatePeerConnectionFactoryDependencies())), + CreatePeerConnectionFactoryDependencies(field_trials_.get()))), ice_server_list_(ice_server_list), config_(config), publish_filter_(publish_filter) { + webrtc_worker_thread_ = webrtc::Thread::Current(); // Set the webrtc port range. webrtc_port_range_ = webrtc_udp_port_range; @@ -195,9 +207,19 @@ PeerConnectionManager::PeerConnectionManager( } return this->HangUp(peerid); }; + + // Start async encoder thread. + encoder_running_ = true; + encoder_thread_ = + std::thread(&PeerConnectionManager::EncoderThreadLoop, this); } -PeerConnectionManager::~PeerConnectionManager() {} +PeerConnectionManager::~PeerConnectionManager() { + // Stop async encoder thread before WebRTC resources are torn down. + encoder_running_ = false; + pending_frames_cv_.notify_all(); + encoder_thread_.join(); +} // Return deviceList as JSON vector. const Json::Value PeerConnectionManager::GetMediaList() { @@ -237,9 +259,9 @@ const Json::Value PeerConnectionManager::GetIceServers() { } // Get PeerConnection associated with peerid. -rtc::scoped_refptr +webrtc::scoped_refptr PeerConnectionManager::GetPeerConnection(const std::string &peerid) { - rtc::scoped_refptr peer_connection; + webrtc::scoped_refptr peer_connection; auto it = peerid_to_connection_.find(peerid); if (it != peerid_to_connection_.end()) { peer_connection = it->second->GetPeerConnection(); @@ -254,12 +276,12 @@ const Json::Value PeerConnectionManager::AddIceCandidate( std::string sdp_mid; int sdp_mlineindex = 0; std::string sdp; - if (!rtc::GetStringFromJsonObject(json_message, k_candidate_sdp_mid_name, + if (!webrtc::GetStringFromJsonObject(json_message, k_candidate_sdp_mid_name, &sdp_mid) || - !rtc::GetIntFromJsonObject(json_message, + !webrtc::GetIntFromJsonObject(json_message, k_candidate_sdp_mline_index_name, &sdp_mlineindex) || - !rtc::GetStringFromJsonObject(json_message, k_candidate_sdp_name, + !webrtc::GetStringFromJsonObject(json_message, k_candidate_sdp_name, &sdp)) { utility::LogWarning("Can't parse received message."); } else { @@ -281,7 +303,7 @@ const Json::Value PeerConnectionManager::AddIceCandidate( } else { std::lock_guard mutex_lock( peerid_to_connection_mutex_); - rtc::scoped_refptr + webrtc::scoped_refptr peer_connection = this->GetPeerConnection(peerid); if (peer_connection) { if (!peer_connection->AddIceCandidate(candidate.get())) { @@ -311,9 +333,9 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, std::string type; std::string sdp; - if (!rtc::GetStringFromJsonObject(json_message, + if (!webrtc::GetStringFromJsonObject(json_message, k_session_description_type_name, &type) || - !rtc::GetStringFromJsonObject(json_message, + !webrtc::GetStringFromJsonObject(json_message, k_session_description_sdp_name, &sdp)) { utility::LogWarning("Can't parse received message."); } else { @@ -325,12 +347,11 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, utility::LogError("Failed to initialize PeerConnection"); delete peer_connection_observer; } else { - rtc::scoped_refptr - peer_connection = - peer_connection_observer->GetPeerConnection(); - utility::LogDebug("nbStreams local: {}, remote: {}", - peer_connection->local_streams()->count(), - peer_connection->remote_streams()->count()); + webrtc::PeerConnectionInterface* peer_connection_ptr = + peer_connection_observer->GetPeerConnection().get(); + utility::LogDebug("nbSenders: {}, nbReceivers: {}", + peer_connection_ptr->GetSenders().size(), + peer_connection_ptr->GetReceivers().size()); // Register peerid. { @@ -348,19 +369,27 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, } // Set remote offer. - webrtc::SessionDescriptionInterface *session_description( - webrtc::CreateSessionDescription(type, sdp, nullptr)); + std::optional sdp_type = + webrtc::SdpTypeFromString(type); + std::unique_ptr + session_description; + if (!sdp_type) { + utility::LogError("Unknown session description type: {}.", type); + } else { + session_description = + webrtc::CreateSessionDescription(*sdp_type, sdp); + } if (!session_description) { utility::LogError( "Can't parse received session description message. " "Cannot create session description."); } else { - std::promise + std::promise remote_promise; - peer_connection->SetRemoteDescription( - SetSessionDescriptionObserver::Create(peer_connection, - remote_promise), - session_description); + peer_connection_ptr->SetRemoteDescription( + SetSessionDescriptionObserver::Create( + peer_connection_ptr, remote_promise), + session_description.release()); // Waiting for remote description. std::future remote_future = remote_promise.get_future(); @@ -375,7 +404,7 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, } // Add local stream. - if (!this->AddStreams(peer_connection, window_uid, options)) { + if (!this->AddStreams(peer_connection_ptr, window_uid, options)) { utility::LogError("Can't add stream {}, {}.", window_uid, options); } @@ -384,9 +413,9 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions rtc_options; std::promise local_promise; - peer_connection->CreateAnswer( - CreateSessionDescriptionObserver::Create(peer_connection, - local_promise), + peer_connection_ptr->CreateAnswer( + CreateSessionDescriptionObserver::Create( + peer_connection_ptr, local_promise), rtc_options); // Waiting for answer. @@ -415,26 +444,20 @@ const Json::Value PeerConnectionManager::Call(const std::string &peerid, } bool PeerConnectionManager::WindowStillUsed(const std::string &window_uid) { - bool still_used = false; for (auto it : peerid_to_connection_) { - rtc::scoped_refptr peer_connection = - it.second->GetPeerConnection(); - rtc::scoped_refptr local_streams( - peer_connection->local_streams()); - for (unsigned int i = 0; i < local_streams->count(); i++) { - if (local_streams->at(i)->id() == window_uid) { - still_used = true; - break; - } + if (PeerConnectionHasStreamForWindow( + it.second->GetPeerConnection().get(), window_uid)) { + return true; } } - return still_used; + return false; } // Hangup a call. const Json::Value PeerConnectionManager::HangUp(const std::string &peerid) { bool result = false; PeerConnectionObserver *pc_observer = nullptr; + std::string hangup_window_uid; { std::lock_guard mutex_lock(peerid_to_connection_mutex_); auto it = peerid_to_connection_.find(peerid); @@ -446,37 +469,27 @@ const Json::Value PeerConnectionManager::HangUp(const std::string &peerid) { if (peerid_to_window_uid_.count(peerid) != 0) { std::lock_guard mutex_lock( window_uid_to_peerids_mutex_); - const std::string window_uid = peerid_to_window_uid_.at(peerid); + hangup_window_uid = peerid_to_window_uid_.at(peerid); peerid_to_window_uid_.erase(peerid); // After window_uid_to_peerids_[window_uid] becomes empty, we don't // remove the window_uid from the map here. We remove window_uid // from window_uid_to_peerids_ when the Window is closed. - window_uid_to_peerids_[window_uid].erase(peerid); + window_uid_to_peerids_[hangup_window_uid].erase(peerid); } if (pc_observer) { - rtc::scoped_refptr - peer_connection = pc_observer->GetPeerConnection(); - - rtc::scoped_refptr local_streams( - peer_connection->local_streams()); - for (unsigned int i = 0; i < local_streams->count(); i++) { - auto stream = local_streams->at(i); - - std::string window_uid = stream->id(); - bool still_used = this->WindowStillUsed(window_uid); - if (!still_used) { - std::lock_guard mlock( - window_uid_to_track_source_mutex_); - auto it = window_uid_to_track_source_.find(window_uid); - if (it != window_uid_to_track_source_.end()) { - window_uid_to_track_source_.erase(it); - } - utility::LogDebug("HangUp stream closed {}.", window_uid); + if (!hangup_window_uid.empty() && + !this->WindowStillUsed(hangup_window_uid)) { + std::lock_guard mlock( + window_uid_to_track_source_mutex_); + auto track_it = + window_uid_to_track_source_.find(hangup_window_uid); + if (track_it != window_uid_to_track_source_.end()) { + window_uid_to_track_source_.erase(track_it); } - - peer_connection->RemoveStream(stream); + utility::LogDebug("HangUp stream closed {}.", + hangup_window_uid); } delete pc_observer; @@ -517,10 +530,40 @@ bool PeerConnectionManager::InitializePeerConnection() { return (peer_connection_factory_.get() != nullptr); } +PeerConnectionManager::PeerConnectionObserver::PeerConnectionObserver( + PeerConnectionManager* peer_connection_manager, + const std::string& peerid) + : peer_connection_manager_(peer_connection_manager), + peerid_(peerid), + local_channel_(nullptr), + remote_channel_(nullptr), + ice_candidate_list_(Json::arrayValue), + deleting_(false) { + stats_callback_ = + new webrtc::RefCountedObject(); +} + +void PeerConnectionManager::PeerConnectionObserver::Initialize( + webrtc::scoped_refptr peer_connection) { + pc_ = peer_connection; + if (pc_.get()) { + auto channel_result = + pc_->CreateDataChannelOrError("ServerDataChannel", nullptr); + if (channel_result.ok()) { + local_channel_ = new DataChannelObserver( + peer_connection_manager_, channel_result.value(), peerid_); + } + } +} + // Create a new PeerConnection. -PeerConnectionManager::PeerConnectionObserver * -PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { +PeerConnectionManager::PeerConnectionObserver* +PeerConnectionManager::CreatePeerConnection(const std::string& peerid) { webrtc::PeerConnectionInterface::RTCConfiguration config; + // Max bundle multiplexes all media and data channels on a single transport, + // eliminating separate ICE/DTLS handshakes per track and reducing latency. + config.bundle_policy = + webrtc::PeerConnectionInterface::kBundlePolicyMaxBundle; for (auto ice_server : ice_server_list_) { webrtc::PeerConnectionInterface::IceServer server; IceServer srv = GetIceServerFromUrl(ice_server); @@ -542,24 +585,30 @@ PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { max_port = std::stoi(port); } } - std::unique_ptr port_allocator( - new cricket::BasicPortAllocator(new rtc::BasicNetworkManager())); - port_allocator->SetPortRange(min_port, max_port); + config.set_min_port(min_port); + config.set_max_port(max_port); utility::LogDebug("CreatePeerConnection webrtcPortRange: {}:{}.", min_port, max_port); utility::LogDebug("CreatePeerConnection peerid: {}.", peerid); - PeerConnectionObserver *obs = new PeerConnectionObserver( - this, peerid, config, std::move(port_allocator)); - if (!obs) { - utility::LogError("CreatePeerConnection failed."); - } else { - utility::LogDebug("CreatePeerConnection success!"); + + PeerConnectionObserver* obs = + new PeerConnectionObserver(this, peerid); + webrtc::PeerConnectionDependencies dependencies(obs); + auto pc_result = peer_connection_factory_->CreatePeerConnectionOrError( + config, std::move(dependencies)); + if (!pc_result.ok()) { + utility::LogError("CreatePeerConnection failed: {}.", + pc_result.error().message()); + delete obs; + return nullptr; } + obs->Initialize(pc_result.MoveValue()); + utility::LogDebug("CreatePeerConnection success!"); return obs; } // Get the capturer from its URL. -rtc::scoped_refptr +webrtc::scoped_refptr PeerConnectionManager::CreateVideoSource( const std::string &window_uid, const std::map &opts) { @@ -625,46 +674,42 @@ bool PeerConnectionManager::AddStreams( if (!existing_stream) { // Create a new stream and add to window_uid_to_track_source_. - rtc::scoped_refptr video_source( + webrtc::scoped_refptr video_source( this->CreateVideoSource(video, opts)); std::lock_guard mlock(window_uid_to_track_source_mutex_); window_uid_to_track_source_[window_uid] = video_source; } - // AddTrack and AddStream to peer_connection. + // Add local video track (Unified Plan). { std::lock_guard mlock(window_uid_to_track_source_mutex_); auto it = window_uid_to_track_source_.find(window_uid); if (it != window_uid_to_track_source_.end()) { - rtc::scoped_refptr stream = - peer_connection_factory_->CreateLocalMediaStream( - window_uid); - if (!stream.get()) { - utility::LogError("Cannot create stream."); + webrtc::scoped_refptr video_source = + it->second; + webrtc::scoped_refptr video_track; + if (!video_source) { + utility::LogError("Cannot create capturer video: {}.", + window_uid); } else { - rtc::scoped_refptr video_source = - it->second; - rtc::scoped_refptr video_track; - if (!video_source) { - utility::LogError("Cannot create capturer video: {}.", - window_uid); - } else { - rtc::scoped_refptr videoScaled = - VideoFilter::Create(video_source, - opts); - video_track = peer_connection_factory_->CreateVideoTrack( - window_uid + "_video", videoScaled); - } - - if ((video_track) && (!stream->AddTrack(video_track))) { - utility::LogError( - "Adding VideoTrack to MediaStream failed."); - } + webrtc::scoped_refptr videoScaled = + VideoFilter::Create(video_source, opts); + video_track = peer_connection_factory_->CreateVideoTrack( + videoScaled, window_uid + "_video"); + video_track->set_content_hint( + webrtc::VideoTrackInterface::ContentHint::kFluid); + } - if (!peer_connection->AddStream(stream)) { - utility::LogError("Adding stream to PeerConnection failed"); + if (video_track) { + webrtc::RTCErrorOr> + add_result = peer_connection->AddTrack( + video_track, {window_uid}); + if (!add_result.ok()) { + utility::LogError("Adding track to PeerConnection failed: {}", + add_result.error().message()); } else { - utility::LogDebug("Stream added to PeerConnection."); + utility::LogDebug("Track added to PeerConnection."); ret = true; } } @@ -692,7 +737,7 @@ void PeerConnectionManager::PeerConnectionObserver::OnIceCandidate( } } -rtc::scoped_refptr +webrtc::scoped_refptr PeerConnectionManager::GetVideoTrackSource(const std::string &window_uid) { { std::lock_guard mlock(window_uid_to_track_source_mutex_); @@ -729,19 +774,56 @@ void PeerConnectionManager::CloseWindowConnections( } } +// Encoder thread: wakes on each new frame, drains the per-window latest-frame +// map, and posts OnFrame to the WebRTC worker thread. libyuv conversion and +// VideoBroadcaster must run on the worker thread (same as PCM creation). +void PeerConnectionManager::EncoderThreadLoop() { + while (encoder_running_) { + std::unordered_map> snapshot; + { + std::unique_lock lock(pending_frames_mutex_); + pending_frames_cv_.wait(lock, [this] { + return !pending_frames_.empty() || !encoder_running_; + }); + if (!encoder_running_) break; + // Drain: take all pending frames in one batch; late-arriving frames + // for the same window have already overwritten earlier ones, so we + // encode only the latest per window (implicit frame coalescing). + snapshot = std::move(pending_frames_); + } + webrtc::Thread* worker = webrtc_worker_thread_; + if (!worker) { + continue; + } + for (const auto& kv : snapshot) { + const std::shared_ptr& frame = kv.second; + if (!frame) { + continue; + } + webrtc::scoped_refptr track_source = + GetVideoTrackSource(kv.first); + if (!track_source) { + continue; + } + worker->PostTask([track_source, frame]() { + track_source->OnFrame(frame); + }); + } + } +} + void PeerConnectionManager::OnFrame(const std::string &window_uid, const std::shared_ptr &im) { - // Get the WebRTC stream that corresponds to the window_uid. - // video_track_source is nullptr if the server is running but no client is - // connected. - rtc::scoped_refptr video_track_source = - GetVideoTrackSource(window_uid); - if (video_track_source) { - // TODO: this OnFrame(im); is a blocking call. Do we need to handle - // OnFrame in a separate thread? e.g. attach to a queue of frames, even - // if the queue size is just 1. - video_track_source->OnFrame(im); + // Skip if no peer is connected for this window. + if (!GetVideoTrackSource(window_uid)) return; + // Post the latest frame; overwrites any pending unencoded frame for this + // window (frame coalescing) so the encoder thread always sees the freshest + // content without blocking the render thread. + { + std::lock_guard lock(pending_frames_mutex_); + pending_frames_[window_uid] = im; } + pending_frames_cv_.notify_one(); } } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h index 0120e20b4fe..2c85a41a40e 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h @@ -18,9 +18,15 @@ #pragma once +#include +#include #include +#include +#include #include +#include +#include #include #include #include @@ -32,6 +38,10 @@ #include "open3d/visualization/webrtc_server/HttpServerRequestHandler.h" #include "open3d/visualization/webrtc_server/WebRTCWindowSystem.h" +namespace webrtc { +class Thread; +} + namespace open3d { namespace visualization { namespace webrtc_server { @@ -71,23 +81,23 @@ namespace webrtc_server { /// /// TODO (yixing): Use PImpl. class PeerConnectionManager { - class VideoSink : public rtc::VideoSinkInterface { + class VideoSink : public webrtc::VideoSinkInterface { public: VideoSink(webrtc::VideoTrackInterface* track) : track_(track) { - track_->AddOrUpdateSink(this, rtc::VideoSinkWants()); + track_->AddOrUpdateSink(this, webrtc::VideoSinkWants()); } virtual ~VideoSink() { track_->RemoveSink(this); } // VideoSinkInterface implementation virtual void OnFrame(const webrtc::VideoFrame& video_frame) { - rtc::scoped_refptr buffer( + webrtc::scoped_refptr buffer( video_frame.video_frame_buffer()->ToI420()); utility::LogDebug("[{}] frame: {}x{}", OPEN3D_FUNCTION, buffer->height(), buffer->width()); } protected: - rtc::scoped_refptr track_; + webrtc::scoped_refptr track_; }; class SetSessionDescriptionObserver @@ -97,7 +107,7 @@ class PeerConnectionManager { webrtc::PeerConnectionInterface* pc, std::promise& promise) { - return new rtc::RefCountedObject( + return new webrtc::RefCountedObject( pc, promise); } virtual void OnSuccess() { @@ -134,7 +144,7 @@ class PeerConnectionManager { webrtc::PeerConnectionInterface* pc, std::promise& promise) { - return new rtc::RefCountedObject( + return new webrtc::RefCountedObject( pc, promise); } virtual void OnSuccess(webrtc::SessionDescriptionInterface* desc) { @@ -169,13 +179,12 @@ class PeerConnectionManager { protected: virtual void OnStatsDelivered( - const rtc::scoped_refptr& + const webrtc::scoped_refptr& report) { for (const webrtc::RTCStats& stats : *report) { Json::Value stats_members; - for (const webrtc::RTCStatsMemberInterface* member : - stats.Members()) { - stats_members[member->name()] = member->ValueToString(); + for (const webrtc::Attribute& attribute : stats.Attributes()) { + stats_members[attribute.name()] = attribute.ToString(); } report_[stats.id()] = stats_members; } @@ -188,7 +197,7 @@ class PeerConnectionManager { public: DataChannelObserver( PeerConnectionManager* peer_connection_manager, - rtc::scoped_refptr data_channel, + webrtc::scoped_refptr data_channel, const std::string& peerid) : peer_connection_manager_(peer_connection_manager), data_channel_(data_channel), @@ -249,38 +258,18 @@ class PeerConnectionManager { protected: PeerConnectionManager* peer_connection_manager_; - rtc::scoped_refptr data_channel_; + webrtc::scoped_refptr data_channel_; const std::string peerid_; }; class PeerConnectionObserver : public webrtc::PeerConnectionObserver { public: - PeerConnectionObserver( - PeerConnectionManager* peer_connection_manager, - const std::string& peerid, - const webrtc::PeerConnectionInterface::RTCConfiguration& config, - std::unique_ptr port_allocator) - : peer_connection_manager_(peer_connection_manager), - peerid_(peerid), - local_channel_(nullptr), - remote_channel_(nullptr), - ice_candidate_list_(Json::arrayValue), - deleting_(false) { - pc_ = peer_connection_manager_->peer_connection_factory_ - ->CreatePeerConnection(config, - std::move(port_allocator), - nullptr, this); - - if (pc_.get()) { - rtc::scoped_refptr channel = - pc_->CreateDataChannel("ServerDataChannel", nullptr); - local_channel_ = new DataChannelObserver( - peer_connection_manager_, channel, peerid_); - } + PeerConnectionObserver(PeerConnectionManager* peer_connection_manager, + const std::string& peerid); - stats_callback_ = new rtc::RefCountedObject< - PeerConnectionStatsCollectorCallback>(); - }; + void Initialize( + webrtc::scoped_refptr + peer_connection); virtual ~PeerConnectionObserver() { delete local_channel_; @@ -296,7 +285,7 @@ class PeerConnectionManager { Json::Value GetStats() { stats_callback_->clearReport(); - pc_->GetStats(stats_callback_); + pc_->GetStats(stats_callback_.get()); int count = 10; while ((stats_callback_->getReport().empty()) && (--count > 0)) { std::this_thread::sleep_for(std::chrono::milliseconds(1000)); @@ -304,27 +293,27 @@ class PeerConnectionManager { return Json::Value(stats_callback_->getReport()); }; - rtc::scoped_refptr + webrtc::scoped_refptr GetPeerConnection() { return pc_; }; // PeerConnectionObserver interface virtual void OnAddStream( - rtc::scoped_refptr stream) { + webrtc::scoped_refptr stream) { utility::LogDebug("[{}] GetVideoTracks().size(): {}.", OPEN3D_FUNCTION, stream->GetVideoTracks().size()); webrtc::VideoTrackVector videoTracks = stream->GetVideoTracks(); if (videoTracks.size() > 0) { - video_sink_.reset(new VideoSink(videoTracks.at(0))); + video_sink_.reset(new VideoSink(videoTracks.at(0).get())); } } virtual void OnRemoveStream( - rtc::scoped_refptr stream) { + webrtc::scoped_refptr stream) { video_sink_.reset(); } virtual void OnDataChannel( - rtc::scoped_refptr channel) { + webrtc::scoped_refptr channel) { utility::LogDebug( "PeerConnectionObserver::OnDataChannel peerid: {}", peerid_); @@ -367,11 +356,11 @@ class PeerConnectionManager { private: PeerConnectionManager* peer_connection_manager_; const std::string peerid_; - rtc::scoped_refptr pc_; + webrtc::scoped_refptr pc_; DataChannelObserver* local_channel_; DataChannelObserver* remote_channel_; Json::Value ice_candidate_list_; - rtc::scoped_refptr + webrtc::scoped_refptr stats_callback_; std::unique_ptr video_sink_; bool deleting_; @@ -407,22 +396,22 @@ class PeerConnectionManager { const std::shared_ptr& im); protected: - rtc::scoped_refptr GetVideoTrackSource( + webrtc::scoped_refptr GetVideoTrackSource( const std::string& window_uid); PeerConnectionObserver* CreatePeerConnection(const std::string& peerid); bool AddStreams(webrtc::PeerConnectionInterface* peer_connection, const std::string& window_uid, const std::string& options); - rtc::scoped_refptr CreateVideoSource( + webrtc::scoped_refptr CreateVideoSource( const std::string& window_uid, const std::map& opts); bool WindowStillUsed(const std::string& window_uid); - rtc::scoped_refptr GetPeerConnection( + webrtc::scoped_refptr GetPeerConnection( const std::string& peerid); protected: - std::unique_ptr task_queue_factory_; - rtc::scoped_refptr + std::unique_ptr field_trials_; + webrtc::scoped_refptr peer_connection_factory_; // Each peer has exactly one connection. @@ -435,7 +424,7 @@ class PeerConnectionManager { // Each Window has exactly one TrackSource. std::unordered_map> + webrtc::scoped_refptr> window_uid_to_track_source_; std::mutex window_uid_to_track_source_mutex_; @@ -451,6 +440,21 @@ class PeerConnectionManager { const std::regex publish_filter_; std::map func_; std::string webrtc_port_range_; + + // Async encoder thread: decouples the render thread from the blocking + // libyuv + WebRTC encode path. OnFrame() posts the latest frame per window; + // the thread drains the map and posts video_track_source->OnFrame() to the + // WebRTC worker thread. + std::unordered_map> + pending_frames_; + std::mutex pending_frames_mutex_; + std::condition_variable pending_frames_cv_; + std::atomic encoder_running_{false}; + std::thread encoder_thread_; + // WebRTC worker thread (PeerConnectionFactoryDependencies::worker_thread). + webrtc::Thread* webrtc_worker_thread_ = nullptr; + + void EncoderThreadLoop(); }; } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/VideoFilter.h b/cpp/open3d/visualization/webrtc_server/VideoFilter.h index 16588c96a2a..94a95c0099a 100644 --- a/cpp/open3d/visualization/webrtc_server/VideoFilter.h +++ b/cpp/open3d/visualization/webrtc_server/VideoFilter.h @@ -18,7 +18,8 @@ #pragma once -#include +#include +#include #include "open3d/visualization/webrtc_server/BitmapTrackSource.h" @@ -34,14 +35,15 @@ namespace webrtc_server { template class VideoFilter : public BitmapTrackSource { public: - static rtc::scoped_refptr Create( - rtc::scoped_refptr video_source, + static webrtc::scoped_refptr Create( + webrtc::scoped_refptr video_source, const std::map& opts) { std::unique_ptr source = absl::WrapUnique(new T(video_source, opts)); if (!source) { return nullptr; } - return new rtc::RefCountedObject(std::move(source)); + return webrtc::scoped_refptr( + new webrtc::RefCountedObject(std::move(source))); } protected: @@ -61,7 +63,7 @@ class VideoFilter : public BitmapTrackSource { } private: - rtc::VideoSourceInterface* source() override { + webrtc::VideoSourceInterface* source() override { return source_.get(); } std::unique_ptr source_; diff --git a/cpp/open3d/visualization/webrtc_server/VideoScaler.h b/cpp/open3d/visualization/webrtc_server/VideoScaler.h index cd57e3dc898..fcebdda0bbc 100644 --- a/cpp/open3d/visualization/webrtc_server/VideoScaler.h +++ b/cpp/open3d/visualization/webrtc_server/VideoScaler.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "open3d/visualization/webrtc_server/BitmapTrackSource.h" @@ -27,10 +28,10 @@ namespace open3d { namespace visualization { namespace webrtc_server { -class VideoScaler : public rtc::VideoSinkInterface, - public rtc::VideoSourceInterface { +class VideoScaler : public webrtc::VideoSinkInterface, + public webrtc::VideoSourceInterface { public: - VideoScaler(rtc::scoped_refptr video_source, + VideoScaler(webrtc::scoped_refptr video_source, const std::map &opts) : video_source_(video_source), width_(0), @@ -148,7 +149,7 @@ class VideoScaler : public rtc::VideoSinkInterface, } else if (width == 0) { width = (roi_width_ * height) / roi_height_; } - rtc::scoped_refptr scaled_buffer = + webrtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(width, height); if (roi_width_ != frame.width() || roi_height_ != frame.height()) { scaled_buffer->CropAndScaleFrom( @@ -158,22 +159,22 @@ class VideoScaler : public rtc::VideoSinkInterface, scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); } webrtc::VideoFrame scaledFrame = - webrtc::VideoFrame(scaled_buffer, frame.timestamp(), + webrtc::VideoFrame(scaled_buffer, frame.timestamp_us(), frame.render_time_ms(), rotation_); broadcaster_.OnFrame(scaledFrame); } } - void AddOrUpdateSink(rtc::VideoSinkInterface *sink, - const rtc::VideoSinkWants &wants) override { + void AddOrUpdateSink(webrtc::VideoSinkInterface *sink, + const webrtc::VideoSinkWants &wants) override { video_source_->AddOrUpdateSink(this, wants); broadcaster_.AddOrUpdateSink(sink, wants); } void RemoveSink( - rtc::VideoSinkInterface *sink) override { + webrtc::VideoSinkInterface *sink) override { video_source_->RemoveSink(this); broadcaster_.RemoveSink(sink); @@ -183,8 +184,8 @@ class VideoScaler : public rtc::VideoSinkInterface, int height() { return roi_height_; } private: - rtc::scoped_refptr video_source_; - rtc::VideoBroadcaster broadcaster_; + webrtc::scoped_refptr video_source_; + webrtc::VideoBroadcaster broadcaster_; int width_; int height_; diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index 667a7c88be4..e2d87b60169 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -7,13 +7,12 @@ #include "open3d/visualization/webrtc_server/WebRTCWindowSystem.h" -#include -#include -#include #include +#include #include #include +#include #include #include #include @@ -103,6 +102,8 @@ struct WebRTCWindowSystem::Impl { std::thread webrtc_thread_; bool sever_started_ = false; + // Set while the WebRTC std::thread is inside Run(); used for shutdown. + std::atomic webrtc_message_thread_{nullptr}; std::unordered_map> data_channel_message_callbacks_; @@ -197,8 +198,14 @@ WebRTCWindowSystem::WebRTCWindowSystem() } WebRTCWindowSystem::~WebRTCWindowSystem() { + if (impl_->sever_started_ && impl_->webrtc_thread_.joinable()) { + webrtc::Thread* message_thread = impl_->webrtc_message_thread_.load(); + if (message_thread) { + message_thread->Quit(); + } + impl_->webrtc_thread_.join(); + } impl_->peer_connection_manager_ = nullptr; - rtc::Thread::Current()->Quit(); } WebRTCWindowSystem::OSWindow WebRTCWindowSystem::CreateOSWindow( @@ -262,16 +269,25 @@ void WebRTCWindowSystem::StartWebRTCServer() { gui::Application::GetInstance().GetResourcePath()); impl_->web_root_ = resource_path + "/html"; - // Logging settings. - // src/rtc_base/logging.h: LS_VERBOSE, LS_ERROR - rtc::LogMessage::LogToDebug((rtc::LoggingSeverity)rtc::LS_ERROR); - - rtc::LogMessage::LogTimestamps(); - rtc::LogMessage::LogThreads(); + // Logging settings (M149: rtc_base/logging.h). + webrtc::LoggingConfig log_config; + log_config.set_debug_severity(webrtc::LS_ERROR); + log_config.set_log_thread(true); + log_config.set_log_timestamp(true); + webrtc::InitializeLogging(std::move(log_config)); + + // Associate this std::thread with WebRTC's message loop (required + // before Thread::Current()->Run() and PeerConnectionFactory). + webrtc::ThreadManager::Instance()->WrapCurrentThread(); + struct WebRtcThreadScope { + ~WebRtcThreadScope() { + webrtc::ThreadManager::Instance()->UnwrapCurrentThread(); + } + } webrtc_thread_scope; + webrtc::Thread* thread = webrtc::Thread::Current(); + impl_->webrtc_message_thread_.store(thread); - // PeerConnectionManager manages all WebRTC connections. - rtc::Thread *thread = rtc::Thread::Current(); - rtc::InitializeSSL(); + webrtc::InitializeSSL(); Json::Value config; std::list ice_servers; ice_servers.insert(ice_servers.end(), s_public_ice_servers.begin(), @@ -345,7 +361,8 @@ void WebRTCWindowSystem::StartWebRTCServer() { utility::LogInfo("WebRTC Jupyter handshake mode enabled."); thread->Run(); } - rtc::CleanupSSL(); + impl_->webrtc_message_thread_.store(nullptr); + webrtc::CleanupSSL(); }; impl_->webrtc_thread_ = std::thread(start_webrtc_thread); impl_->sever_started_ = true; @@ -364,8 +381,8 @@ std::string WebRTCWindowSystem::OnDataChannelMessage( if (impl_->data_channel_message_callbacks_.count(class_name) != 0) { reply = impl_->data_channel_message_callbacks_.at(class_name)( message); - const auto os_window = GetOSWindowByUID(window_uid); - if (os_window) PostRedrawEvent(os_window); + // Custom callbacks that mutate GUI state (e.g. add/remove geometry) + // must call window->PostRedraw() or post_redraw() themselves. return reply; } else { reply = fmt::format( @@ -405,7 +422,7 @@ void WebRTCWindowSystem::OnFrame(const std::string &window_uid, void WebRTCWindowSystem::SendInitFrames(const std::string &window_uid) { utility::LogInfo("Sending init frames to {}.", window_uid); static const int s_max_initial_frames = 5; - static const int s_sleep_between_frames_ms = 100; + static const int s_sleep_between_frames_ms = 50; const auto os_window = GetOSWindowByUID(window_uid); if (!os_window) return; for (int i = 0; os_window != nullptr && i < s_max_initial_frames; ++i) { diff --git a/cpp/open3d/visualization/webrtc_server/html/index.html b/cpp/open3d/visualization/webrtc_server/html/index.html index 995ce95dea5..b318932f9f8 100644 --- a/cpp/open3d/visualization/webrtc_server/html/index.html +++ b/cpp/open3d/visualization/webrtc_server/html/index.html @@ -1,4 +1,5 @@ - + + Open3D WebVisualizer @@ -136,6 +137,7 @@ videoElt.muted = true; videoElt.controls = false; videoElt.playsinline = true; + videoElt.preload = "none"; videoElt.innerText = "Your browser does not support HTML5 video."; divElt.appendChild(videoElt); diff --git a/cpp/open3d/visualization/webrtc_server/html/open3d_logo.ico b/cpp/open3d/visualization/webrtc_server/html/open3d_logo.ico deleted file mode 100644 index ce428261c26..00000000000 Binary files a/cpp/open3d/visualization/webrtc_server/html/open3d_logo.ico and /dev/null differ diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index d7bcd8f6c13..7c625e67c1e 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -12,18 +12,6 @@ // any purpose. // ---------------------------------------------------------------------------- -(function() { -const enableLogging = false; -if (enableLogging === false) { - if (typeof window.console === 'undefined') { - window.console = {}; - } - window.console.log = window.console.info = window.console.debug = - window.console.warning = window.console.assert = - window.console.error = function() {}; -} -}()); - let WebRtcStreamer = (function() { // Immediately-executing anonymous functions to enforce variable scope. @@ -58,10 +46,18 @@ let WebRtcStreamer = (function() { this.iceServers = null; this.earlyCandidates = []; + this.remoteStream = null; // Open3D-specific functions. this.onClose = onClose; this.commsFetch = commsFetch; + + // Pending coalesced pointer/wheel events. A single requestAnimationFrame + // flushes both at most once per browser frame (~60 Hz), preventing a + // backlog of stale events from queuing up on the server. + this.pendingPointerEvent = null; // MOVE or DRAG (latest wins) + this.pendingWheelEvent = null; // WHEEL (dx/dy accumulated) + this.rafPending = false; } const logAndReturn = function(value) { @@ -182,6 +178,26 @@ let WebRtcStreamer = (function() { } }; + // Schedule a requestAnimationFrame flush of pending coalesced events. + // Only one rAF is scheduled at a time; calling again while one is already + // pending is a no-op. When the frame fires, the latest pointer event and + // the accumulated wheel event (if any) are sent and cleared. + WebRtcStreamer.prototype._scheduleRafFlush = function() { + if (this.rafPending) return; + this.rafPending = true; + requestAnimationFrame(() => { + this.rafPending = false; + if (this.pendingPointerEvent !== null) { + this.sendJsonData(this.pendingPointerEvent); + this.pendingPointerEvent = null; + } + if (this.pendingWheelEvent !== null) { + this.sendJsonData(this.pendingWheelEvent); + this.pendingWheelEvent = null; + } + }); + }; + WebRtcStreamer.prototype.addEventListeners = function(windowUID) { if (this.videoElt) { var parentDivElt = this.videoElt.parentElement; @@ -333,7 +349,9 @@ let WebRtcStreamer = (function() { // - Open3D: L=1, M=2, R=4 // - JavaScript: L=1, R=2, M=4 event.preventDefault(); - var open3dMouseEvent = { + // Throttle to one event per animation frame. Only the latest + // absolute position matters; intermediate positions are stale. + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: event.buttons === 0 ? 'MOVE' : 'DRAG', @@ -344,7 +362,7 @@ let WebRtcStreamer = (function() { buttons: event.buttons, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('touchmove', (event) => { // TODO: Known differences. Currently only left-key drag works. @@ -352,7 +370,8 @@ let WebRtcStreamer = (function() { // - JavaScript: L=1, R=2, M=4 event.preventDefault(); var rect = event.target.getBoundingClientRect(); - var open3dMouseEvent = { + // Throttle to one event per animation frame (latest wins). + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: 'DRAG', @@ -363,7 +382,7 @@ let WebRtcStreamer = (function() { buttons: 1, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('mouseleave', (event) => { var open3dMouseEvent = { @@ -397,20 +416,36 @@ let WebRtcStreamer = (function() { dx = dx === 0 ? dx : (-dx / Math.abs(dx)) * 1; dy = dy === 0 ? dy : (-dy / Math.abs(dy)) * 1; - var open3dMouseEvent = { - window_uid: windowUID, - class_name: 'MouseEvent', - type: 'WHEEL', - x: event.offsetX, - y: event.offsetY, - modifiers: WebRtcStreamer._getModifiers(event), - wheel: { - dx: dx, - dy: dy, - isTrackpad: isTrackpad ? 1 : 0, - }, - }; - this.sendJsonData(open3dMouseEvent); + if (this.pendingWheelEvent === null) { + // First wheel event in this frame: create a new pending + // event and schedule a flush. + this.pendingWheelEvent = { + window_uid: windowUID, + class_name: 'MouseEvent', + type: 'WHEEL', + x: event.offsetX, + y: event.offsetY, + modifiers: WebRtcStreamer._getModifiers(event), + wheel: { + dx: dx, + dy: dy, + isTrackpad: isTrackpad ? 1 : 0, + }, + }; + this._scheduleRafFlush(); + } else { + // Subsequent wheel events in the same frame: accumulate + // dx/dy so the total scroll amount is preserved, and + // update cursor position to the latest coordinates. + this.pendingWheelEvent.wheel.dx += dx; + this.pendingWheelEvent.wheel.dy += dy; + this.pendingWheelEvent.x = event.offsetX; + this.pendingWheelEvent.y = event.offsetY; + this.pendingWheelEvent.modifiers = + WebRtcStreamer._getModifiers(event); + this.pendingWheelEvent.wheel.isTrackpad = + isTrackpad ? 1 : 0; + } }, {passive: false}); } }; @@ -420,8 +455,9 @@ let WebRtcStreamer = (function() { */ WebRtcStreamer.prototype.disconnect = function() { if (this.videoElt) { - this.videoElt.src = ''; + this.videoElt.srcObject = null; } + this.remoteStream = null; if (this.pc) { WebRtcStreamer .remoteCall( @@ -464,6 +500,48 @@ let WebRtcStreamer = (function() { this.pc.addStream(stream); } + // Prefer VP9 via the standard setCodecPreferences() API. + // Must be called on the transceiver BEFORE createOffer(). + // For receive-only mode (no local stream), we create the video + // transceiver explicitly — offerToReceiveVideo creates it lazily + // inside createOffer(), which is too late to set preferences. + // Falls back silently to VP8 if the browser does not support the + // API or if VP9 is not in the browser's capabilities. + var pc = this.pc; + if (typeof RTCRtpReceiver !== 'undefined' && + RTCRtpReceiver.getCapabilities && pc.addTransceiver) { + var videoCaps = RTCRtpReceiver.getCapabilities('video'); + if (videoCaps) { + var vp9Codecs = videoCaps.codecs.filter(function(c) { + return c.mimeType === 'video/VP9'; + }); + if (vp9Codecs.length) { + var preferredCodecs = vp9Codecs.concat( + videoCaps.codecs.filter(function(c) { + return c.mimeType !== 'video/VP9'; + })); + if (stream) { + // Local video being sent: find the transceiver + // addStream() created and apply preferences. + pc.getTransceivers().forEach(function(t) { + if (t.sender && t.sender.track && + t.sender.track.kind === 'video' && + t.setCodecPreferences) { + t.setCodecPreferences(preferredCodecs); + } + }); + } else { + // Receive-only: create transceiver explicitly. + var vt = pc.addTransceiver( + 'video', {direction: 'recvonly'}); + if (vt.setCodecPreferences) { + vt.setCodecPreferences(preferredCodecs); + } + } + } + } + } + // clear early candidates this.earlyCandidates.length = 0; @@ -538,7 +616,7 @@ let WebRtcStreamer = (function() { 'createPeerConnection config: ' + JSON.stringify(this.pcConfig) + ' option:' + JSON.stringify(this.pcOptions)); - this.pc = new RTCPeerConnection(this.pcConfig, this.pcOptions); + this.pc = new RTCPeerConnection(this.pcConfig); var pc = this.pc; pc.peerid = Math.random(); @@ -546,9 +624,8 @@ let WebRtcStreamer = (function() { pc.onicecandidate = function(evt) { bind.onIceCandidate.call(bind, evt); }; - pc.onaddstream = function( - evt) { // TODO: Deprecated. Switch to ontrack. - bind.onAddStream.call(bind, evt); + pc.ontrack = function(evt) { + bind.onTrack.call(bind, evt); }; pc.oniceconnectionstatechange = function(evt) { console.log( @@ -589,17 +666,35 @@ let WebRtcStreamer = (function() { const recvs = pc.getReceivers(); recvs.forEach((recv) => { - if (recv.track && recv.track.kind === 'video' && - typeof recv.getParameters != 'undefined') { - console.log( - 'codecs:' + - JSON.stringify(recv.getParameters().codecs)); + if (recv.track && recv.track.kind === 'video') { + // Minimize browser jitter buffer to reduce playout + // latency. The server already sends RTP playout-delay + // header extensions with min=max=0 via the + // WebRTC-ForceSendPlayoutDelay field trial, but set + // jitterBufferTarget as well for browsers that honour + // the JS API over the in-band RTP extension. + if (typeof recv.jitterBufferTarget !== 'undefined') { + recv.jitterBufferTarget = 0; + } + if (typeof recv.getParameters != 'undefined') { + console.log( + 'codecs:' + + JSON.stringify( + recv.getParameters().codecs)); + } } }); } }; - // Local datachannel sends data + // Local datachannel sends data. + // Use reliable ordered delivery (the default). While unordered + + // unreliable would reduce head-of-line blocking for mouse MOVE/DRAG, + // the same channel carries discrete events (BUTTON_DOWN/UP, key events, + // resize) and application-level RPC (tensorboard/update_geometry etc.) + // that must not be lost or reordered. Mouse coalescing (rAF + server- + // side replace_or_merge_mouse) already prevents event backlog, so the + // extra reliability is free in practice on a local network. try { this.dataChannel = pc.createDataChannel('ClientDataChannel'); var dataChannel = this.dataChannel; @@ -662,12 +757,27 @@ let WebRtcStreamer = (function() { }; /* - * RTCPeerConnection AddTrack callback + * RTCPeerConnection ontrack callback (Unified Plan). */ - WebRtcStreamer.prototype.onAddStream = function(event) { - console.log('Remote track added:' + JSON.stringify(event)); + WebRtcStreamer.prototype.onTrack = function(event) { + console.log('Remote track added: ' + event.track.kind); + + if (event.track.kind !== 'video') { + return; + } + + var stream = event.streams && event.streams[0]; + if (!stream) { + if (!this.remoteStream) { + this.remoteStream = new MediaStream(); + } + this.remoteStream.addTrack(event.track); + stream = this.remoteStream; + } else { + this.remoteStream = stream; + } - this.videoElt.srcObject = event.stream; + this.videoElt.srcObject = stream; var promise = this.videoElt.play(); if (typeof promise !== 'undefined') { var bind = this; diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index ee6d747977b..a690bf2eda7 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -91,6 +91,15 @@ endif() if(BUILD_WEBRTC) open3d_add_example(DrawWebRTC) + # Static MKL + monolithic libwebrtc.a need a late --start-group pass on Linux. + if(UNIX AND NOT APPLE AND NOT USE_BLAS) + target_link_libraries(DrawWebRTC PRIVATE + "-Wl,--start-group" + "${CMAKE_BINARY_DIR}/mkl_install/lib/${CMAKE_STATIC_LIBRARY_PREFIX}mkl_intel_ilp64${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${CMAKE_BINARY_DIR}/mkl_install/lib/${CMAKE_STATIC_LIBRARY_PREFIX}mkl_tbb_thread${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${CMAKE_BINARY_DIR}/mkl_install/lib/${CMAKE_STATIC_LIBRARY_PREFIX}mkl_core${CMAKE_STATIC_LIBRARY_SUFFIX}" + "-Wl,--end-group") + endif() endif() if (BUILD_LIBREALSENSE) diff --git a/python/js/package.json b/python/js/package.json index d364052ec22..0e2f7d78069 100644 --- a/python/js/package.json +++ b/python/js/package.json @@ -54,4 +54,4 @@ } } } -} \ No newline at end of file +}