Adding TBB to sysdeps

BUILD_TOPOLOGY.toml

@@ -400,6 +400,13 @@ artifact_deps = []
 feature_group = "CORE"
 disable_platforms = ["windows"]
 
+[artifacts.sysdeps-tbb]
+artifact_group = "third-party-sysdeps"
+type = "target-neutral"
+artifact_deps = []
+feature_group = "CORE"
+disable_platforms = ["windows"]
+
 [artifacts.sysdeps-hwloc]
 artifact_group = "third-party-sysdeps"
 type = "target-neutral"
@@ -728,7 +735,7 @@ disable_platforms = ["windows"]
 [artifacts.rocprofiler-systems]
 artifact_group = "profiler-apps"
 type = "target-neutral"
-artifact_deps = ["amd-llvm", "core-hip", "rocprofiler-sdk", "core-amdsmi", "spdlog", "openmpi"]
+artifact_deps = ["amd-llvm", "core-hip", "rocprofiler-sdk", "core-amdsmi", "spdlog", "openmpi", "sysdeps-tbb"]
 feature_name = "ROCPROFSYS"
 feature_group = "PROFILER"
 disable_platforms = ["windows"]

CMakeLists.txt

@@ -391,6 +391,7 @@ set(THEROCK_BUNDLED_MPFR)
 set(THEROCK_BUNDLED_NCURSES)
 set(THEROCK_BUNDLED_NUMACTL)
 set(THEROCK_BUNDLED_SQLITE3)
+set(THEROCK_BUNDLED_TBB)
 set(THEROCK_BUNDLED_ZLIB)
 set(THEROCK_BUNDLED_ZSTD)
 set(THEROCK_BUNDLED_AMDMESA)
@@ -432,6 +433,9 @@ if(THEROCK_BUNDLE_SYSDEPS)
     if(THEROCK_ENABLE_SYSDEPS_NCURSES)
       set(THEROCK_BUNDLED_NCURSES therock-ncurses)
     endif()
+    if(THEROCK_ENABLE_SYSDEPS_TBB)
+      set(THEROCK_BUNDLED_TBB therock-tbb)
+    endif()
     if(THEROCK_ENABLE_SYSDEPS_AMD_MESA)
       set(THEROCK_BUNDLED_AMDMESA therock-amd-mesa)
     endif()

build_tools/install_rocm_from_artifacts.py

@@ -453,6 +453,8 @@ def retrieve_artifacts_by_run_id(args):
             extra_artifacts.append("rocprofiler-systems")
             # Contains executables (rocprof-sys-run, rocprof-sys-instrument, etc.)
             argv.append("rocprofiler-systems_run")
+            # rocprofiler-systems depends on sysdeps-tbb (Dyninst links against it)
+            extra_artifacts.append("sysdeps-tbb")
         if args.rocrtst:
             extra_artifacts.append("rocrtst")
             # rocrtst depends on sysdeps-hwloc (which depends on sysdeps-libpciaccess)

docs/development/dependencies.md

@@ -54,6 +54,7 @@ project wide:
   - `THEROCK_BUNDLED_LIBNL`
   - `THEROCK_BUNDLED_NUMACTL`
   - `THEROCK_BUNDLED_SQLITE3`
+  - `THEROCK_BUNDLED_TBB`
   - `THEROCK_BUNDLED_ZLIB`
   - `THEROCK_BUNDLED_ZSTD`
 - Sub-projects must arrange for any libraries that depend on these to add the
@@ -201,6 +202,17 @@ SIMDe (SIMD Everywhere) is a header-only portability library for SIMD intrinsics
 - Import library: `SQLite::SQLite3`
 - Alternatives: none
 
+## TBB
+
+oneAPI Threading Building Blocks. Optional sysdep; only bundled when
+`THEROCK_ENABLE_SYSDEPS_TBB=ON`.
+
+Supported sub-libraries: `tbb`, `tbbmalloc`, `tbbmalloc_proxy`.
+
+- Canonical method: `find_package(TBB CONFIG)`
+- Import libraries: `TBB::tbb`, `TBB::tbbmalloc`, `TBB::tbbmalloc_proxy`
+- Alternatives: `pkg_check_modules(TBB tbb)`
+
 ## zlib
 
 - Canonical method: `find_package(ZLIB)`

profiler/CMakeLists.txt

@@ -309,7 +309,6 @@ if (THEROCK_ENABLE_ROCPROFSYS AND THEROCK_SANITIZER STREQUAL "")
     BACKGROUND_BUILD
     CMAKE_ARGS
       -DHIP_PLATFORM=amd
-      -DROCPROFSYS_BUILD_TBB=ON
       -DROCPROFSYS_BUILD_LIBIBERTY=ON
       -DROCPROFSYS_BUILD_BOOST=ON
       -DROCPROFSYS_BUILD_DYNINST=ON
@@ -342,6 +341,7 @@ if (THEROCK_ENABLE_ROCPROFSYS AND THEROCK_SANITIZER STREQUAL "")
       ${THEROCK_BUNDLED_ELFUTILS}
       ${THEROCK_BUNDLED_LIBDRM}
       ${THEROCK_BUNDLED_SQLITE3}
+      ${THEROCK_BUNDLED_TBB}
       ${_openmpi_optional_dep}
   )
   therock_cmake_subproject_glob_c_sources(rocprofiler-systems

profiler/post_hook_rocprofiler-systems.cmake

@@ -0,0 +1,29 @@
+# Copyright Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+
+# Dyninst and rocprofiler-systems internal libraries live in lib/rocprofiler-systems.
+# Set the RPATH origin per-target so that (for example) lib/rocm_sysdeps/lib resolves as
+# $ORIGIN/../rocm_sysdeps/lib rather than $ORIGIN/rocm_sysdeps/lib.
+set(_rocprofsys_lib_targets
+  common
+  dynElf
+  dynDwarf
+  dyninstAPI
+  dyninstAPI_RT
+  dynC_API
+  instructionAPI
+  parseAPI
+  patchAPI
+  pcontrol
+  stackwalk
+  symtabAPI
+  gotcha
+)
+
+foreach(_target ${_rocprofsys_lib_targets})
+  if(TARGET "${_target}")
+    set_target_properties("${_target}" PROPERTIES
+      THEROCK_INSTALL_RPATH_ORIGIN lib/rocprofiler-systems
+    )
+  endif()
+endforeach()

third-party/sysdeps/linux/CMakeLists.txt

@@ -131,6 +131,20 @@ if(THEROCK_ENABLE_SYSDEPS_LIBPCIACCESS)
   )
 endif()
 
+# Optional sysdep: TBB (Threading Building Blocks)
+if(THEROCK_ENABLE_SYSDEPS_TBB)
+  add_subdirectory(tbb)
+  therock_provide_artifact(sysdeps-tbb
+    TARGET_NEUTRAL
+    DESCRIPTOR artifact-tbb.toml
+    COMPONENTS
+      dev
+      lib
+    SUBPROJECT_DEPS
+      therock-tbb
+  )
+endif()
+
 # Optional sysdep: hwloc (depends on numactl and libpciaccess)
 if(THEROCK_ENABLE_SYSDEPS_HWLOC)
   add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../common/hwloc" "${CMAKE_CURRENT_BINARY_DIR}/hwloc")

third-party/sysdeps/linux/artifact-tbb.toml

@@ -0,0 +1,3 @@
+# tbb
+[components.lib."third-party/sysdeps/linux/tbb/build/stage"]
+[components.dev."third-party/sysdeps/linux/tbb/build/stage"]

third-party/sysdeps/linux/tbb/CMakeLists.txt

@@ -0,0 +1,93 @@
+# Copyright Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+
+if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+  # Super-project block: download sources and declare the sub-project.
+  set(_source_dir "${CMAKE_CURRENT_BINARY_DIR}/source")
+  set(_download_stamp "${_source_dir}/download.stamp")
+
+  therock_subproject_fetch(therock-tbb-sources
+    SOURCE_DIR "${_source_dir}"
+    # Originally mirrored from: https://github.com/uxlfoundation/oneTBB/archive/refs/tags/v2022.3.0.tar.gz
+    URL https://rocm-third-party-deps.s3.us-east-2.amazonaws.com/oneTBB-2022.3.0.tar.gz
+    URL_HASH SHA256=01598a46c1162c27253a0de0236f520fd8ee8166e9ebb84a4243574f88e6e50a
+    TOUCH "${_download_stamp}"
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+  )
+
+  therock_cmake_subproject_declare(therock-tbb
+    USE_DIST_AMDGPU_TARGETS
+    BACKGROUND_BUILD
+    EXCLUDE_FROM_ALL
+    NO_MERGE_COMPILE_COMMANDS
+    OUTPUT_ON_FAILURE
+    EXTERNAL_SOURCE_DIR .
+    FPRINT_SOURCE_DIR "${_source_dir}"
+    FPRINT_FILE_GLOBS "${CMAKE_CURRENT_LIST_DIR}/*"
+    BINARY_DIR build
+    CMAKE_ARGS
+      "-DSOURCE_DIR=${_source_dir}"
+      "-DPATCHELF=${PATCHELF}"
+    INSTALL_DESTINATION
+      lib/rocm_sysdeps
+    INTERFACE_LINK_DIRS
+      lib/rocm_sysdeps/lib
+    INTERFACE_INSTALL_RPATH_DIRS
+      lib/rocm_sysdeps/lib
+    INTERFACE_PKG_CONFIG_DIRS
+      lib/rocm_sysdeps/lib/pkgconfig
+    EXTRA_DEPENDS
+      "${_download_stamp}"
+  )
+
+  therock_cmake_subproject_provide_package(therock-tbb TBB lib/rocm_sysdeps/lib/cmake/TBB)
+  therock_cmake_subproject_activate(therock-tbb)
+
+  therock_test_validate_shared_lib(
+    PATH build/dist/lib/rocm_sysdeps/lib
+    LIB_NAMES librocm_sysdeps_tbb.so
+  )
+
+  return()
+endif()
+
+# Sub-project block: invoked by TheRock's cmake sub-project machinery.
+cmake_minimum_required(VERSION 3.25)
+project(TBB_BUILD)
+
+if(NOT PATCHELF)
+  message(FATAL_ERROR "Missing PATCHELF from super-project")
+endif()
+
+add_custom_target(build ALL
+  WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  # Clean any previous install and create a fresh working copy of the source.
+  COMMAND
+    "${CMAKE_COMMAND}" -E rm -rf -- "${CMAKE_INSTALL_PREFIX}" "${CMAKE_CURRENT_BINARY_DIR}/s"
+  COMMAND
+    "${CMAKE_COMMAND}" -E copy_directory "${SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/s"
+  # Patch source: rename library output names to rocm_sysdeps_ prefix.
+  COMMAND
+    bash "${CMAKE_CURRENT_SOURCE_DIR}/patch_source.sh" "${CMAKE_CURRENT_BINARY_DIR}/s"
+  # Configure oneTBB.
+  COMMAND
+    "${CMAKE_COMMAND}"
+      -S "${CMAKE_CURRENT_BINARY_DIR}/s"
+      -B "${CMAKE_CURRENT_BINARY_DIR}/tbb-build"
+      -G Ninja
+      "-DCMAKE_BUILD_TYPE=Release"
+      "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
+      "-DCMAKE_INSTALL_LIBDIR=lib"
+      "-DTBB_TEST=OFF"
+  # Build.
+  COMMAND
+    "${CMAKE_COMMAND}" --build "${CMAKE_CURRENT_BINARY_DIR}/tbb-build" --verbose
+  # Install.
+  COMMAND
+    "${CMAKE_COMMAND}" --install "${CMAKE_CURRENT_BINARY_DIR}/tbb-build"
+  # Patch install: fix RPATH, relocate .pc, remove upstream cmake config.
+  COMMAND
+    "${CMAKE_COMMAND}" -E env
+      "PATCHELF=${PATCHELF}" --
+    bash "${CMAKE_CURRENT_SOURCE_DIR}/patch_install.sh" "${CMAKE_INSTALL_PREFIX}"
+)

third-party/sysdeps/linux/tbb/patch_install.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/bash
+# Copyright Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+
+# Patches installed TBB libraries for portable sysdeps distribution:
+#   - Sets RPATH to $ORIGIN on all real (non-symlink) shared libraries.
+#   - Makes tbb.pc relocatable by replacing the absolute prefix.
+#   - Removes the upstream cmake config (TheRock provides its own wrapper).
+#
+# Args: install_dir
+# Env:  PATCHELF
+
+set -e
+
+PREFIX="${1:?Expected install prefix argument}"
+PATCHELF="${PATCHELF:?PATCHELF env var required}"
+
+LIB_DIR="$PREFIX/lib"
+
+echo "Patching TBB install..."
+
+shopt -s nullglob
+
+# Set RPATH and create unprefixed symlinks (e.g. libtbb.so -> librocm_sysdeps_tbb.so.12).
+for lib in "$LIB_DIR"/librocm_sysdeps_tbb*.so.*; do
+    [ -L "$lib" ] && continue
+    "$PATCHELF" --set-rpath '$ORIGIN' "$lib"
+    base=$(basename "$lib")                    # librocm_sysdeps_tbb.so.12.17
+    soname="${base%.*}"                        # librocm_sysdeps_tbb.so.12
+    unprefixed="${soname/rocm_sysdeps_/}"      # libtbb.so.12
+    symlink="${unprefixed%.so.*}.so"           # libtbb.so
+    ln -sf "$soname" "$LIB_DIR/$symlink"
+done
+
+# Make tbb.pc relocatable: replace the absolute build-time prefix with
+# a pcfiledir-relative path. .pc files live at $PREFIX/lib/pkgconfig,
+# so ${pcfiledir}/../.. resolves back to $PREFIX at runtime.
+PC="$LIB_DIR/pkgconfig/tbb.pc"
+if [ -f "$PC" ]; then
+    abs_prefix=$(grep '^prefix=' "$PC" | cut -d= -f2-)
+    sed -i "s|prefix=${abs_prefix}|prefix=\${pcfiledir}/../..|" "$PC"
+    sed -i "s|${abs_prefix}/|\${prefix}/|g" "$PC"
+fi
+
+# Remove the upstream cmake config. It references the original (unprefixed)
+# library names and would conflict with TheRock's wrapper config provided via
+# therock_cmake_subproject_provide_package.
+rm -rf "$PREFIX/lib/cmake/TBB"
+
+echo "Done patching TBB install."

third-party/sysdeps/linux/tbb/patch_source.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/bash
+# Copyright Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+
+# Patches TBB CMakeLists files to rename library output names with the
+# rocm_sysdeps_ prefix, so that installed libraries do not conflict with
+# system-installed TBB. Must be run before the cmake configure step.
+#
+# The sed patterns below are pinned to oneTBB 2022.3.0 and must be
+# revalidated when bumping the version.
+#
+# Args: source_dir
+
+set -e
+
+SOURCE_DIR="${1:?Source directory must be given}"
+
+verify_patch() {
+    local file="$1" name="$2"
+    if ! grep -q "OUTPUT_NAME.*rocm_sysdeps_${name}" "$file"; then
+        echo "ERROR: OUTPUT_NAME patch for ${name} did not apply to ${file}" >&2
+        echo "The sed patterns may need updating for this oneTBB version." >&2
+        exit 1
+    fi
+}
+
+echo "Patching TBB sources..."
+
+# tbb: insert OUTPUT_NAME after the SOVERSION line in set_target_properties.
+sed -i 's/    SOVERSION \${TBB_BINARY_VERSION}$/    SOVERSION ${TBB_BINARY_VERSION}\n    OUTPUT_NAME "rocm_sysdeps_tbb"/' \
+    "$SOURCE_DIR/src/tbb/CMakeLists.txt"
+verify_patch "$SOURCE_DIR/src/tbb/CMakeLists.txt" "tbb"
+
+# tbbmalloc: same.
+sed -i 's/    SOVERSION \${TBBMALLOC_BINARY_VERSION}$/    SOVERSION ${TBBMALLOC_BINARY_VERSION}\n    OUTPUT_NAME "rocm_sysdeps_tbbmalloc"/' \
+    "$SOURCE_DIR/src/tbbmalloc/CMakeLists.txt"
+verify_patch "$SOURCE_DIR/src/tbbmalloc/CMakeLists.txt" "tbbmalloc"
+
+# tbbmalloc_proxy: SOVERSION is followed by a closing paren on the same line.
+sed -i 's/    SOVERSION \${TBBMALLOC_BINARY_VERSION})$/    SOVERSION ${TBBMALLOC_BINARY_VERSION}\n    OUTPUT_NAME "rocm_sysdeps_tbbmalloc_proxy")/' \
+    "$SOURCE_DIR/src/tbbmalloc_proxy/CMakeLists.txt"
+verify_patch "$SOURCE_DIR/src/tbbmalloc_proxy/CMakeLists.txt" "tbbmalloc_proxy"
+
+echo "Done patching TBB sources."