diff --git a/BUILD_TOPOLOGY.toml b/BUILD_TOPOLOGY.toml index b4d39932eb4..58d94f0c25f 100644 --- a/BUILD_TOPOLOGY.toml +++ b/BUILD_TOPOLOGY.toml @@ -400,6 +400,13 @@ artifact_deps = [] feature_group = "CORE" disable_platforms = ["windows"] +[artifacts.sysdeps-tbb] +artifact_group = "third-party-sysdeps" +type = "target-neutral" +artifact_deps = [] +feature_group = "CORE" +disable_platforms = ["windows"] + [artifacts.sysdeps-hwloc] artifact_group = "third-party-sysdeps" type = "target-neutral" @@ -728,7 +735,7 @@ disable_platforms = ["windows"] [artifacts.rocprofiler-systems] artifact_group = "profiler-apps" type = "target-neutral" -artifact_deps = ["amd-llvm", "core-hip", "rocprofiler-sdk", "core-amdsmi", "spdlog", "openmpi"] +artifact_deps = ["amd-llvm", "core-hip", "rocprofiler-sdk", "core-amdsmi", "spdlog", "openmpi", "sysdeps-tbb"] feature_name = "ROCPROFSYS" feature_group = "PROFILER" disable_platforms = ["windows"] diff --git a/CMakeLists.txt b/CMakeLists.txt index cb0d0919cf8..e7e196841f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -391,6 +391,7 @@ set(THEROCK_BUNDLED_MPFR) set(THEROCK_BUNDLED_NCURSES) set(THEROCK_BUNDLED_NUMACTL) set(THEROCK_BUNDLED_SQLITE3) +set(THEROCK_BUNDLED_TBB) set(THEROCK_BUNDLED_ZLIB) set(THEROCK_BUNDLED_ZSTD) set(THEROCK_BUNDLED_AMDMESA) @@ -432,6 +433,9 @@ if(THEROCK_BUNDLE_SYSDEPS) if(THEROCK_ENABLE_SYSDEPS_NCURSES) set(THEROCK_BUNDLED_NCURSES therock-ncurses) endif() + if(THEROCK_ENABLE_SYSDEPS_TBB) + set(THEROCK_BUNDLED_TBB therock-tbb) + endif() if(THEROCK_ENABLE_SYSDEPS_AMD_MESA) set(THEROCK_BUNDLED_AMDMESA therock-amd-mesa) endif() diff --git a/build_tools/install_rocm_from_artifacts.py b/build_tools/install_rocm_from_artifacts.py index 26172a68853..edabba870e2 100644 --- a/build_tools/install_rocm_from_artifacts.py +++ b/build_tools/install_rocm_from_artifacts.py @@ -453,6 +453,8 @@ def retrieve_artifacts_by_run_id(args): extra_artifacts.append("rocprofiler-systems") # Contains executables (rocprof-sys-run, rocprof-sys-instrument, etc.) argv.append("rocprofiler-systems_run") + # rocprofiler-systems depends on sysdeps-tbb (Dyninst links against it) + extra_artifacts.append("sysdeps-tbb") if args.rocrtst: extra_artifacts.append("rocrtst") # rocrtst depends on sysdeps-hwloc (which depends on sysdeps-libpciaccess) diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md index 1560896b7cb..3fb3b32018a 100644 --- a/docs/development/dependencies.md +++ b/docs/development/dependencies.md @@ -54,6 +54,7 @@ project wide: - `THEROCK_BUNDLED_LIBNL` - `THEROCK_BUNDLED_NUMACTL` - `THEROCK_BUNDLED_SQLITE3` + - `THEROCK_BUNDLED_TBB` - `THEROCK_BUNDLED_ZLIB` - `THEROCK_BUNDLED_ZSTD` - Sub-projects must arrange for any libraries that depend on these to add the @@ -201,6 +202,17 @@ SIMDe (SIMD Everywhere) is a header-only portability library for SIMD intrinsics - Import library: `SQLite::SQLite3` - Alternatives: none +## TBB + +oneAPI Threading Building Blocks. Optional sysdep; only bundled when +`THEROCK_ENABLE_SYSDEPS_TBB=ON`. + +Supported sub-libraries: `tbb`, `tbbmalloc`, `tbbmalloc_proxy`. + +- Canonical method: `find_package(TBB CONFIG)` +- Import libraries: `TBB::tbb`, `TBB::tbbmalloc`, `TBB::tbbmalloc_proxy` +- Alternatives: `pkg_check_modules(TBB tbb)` + ## zlib - Canonical method: `find_package(ZLIB)` diff --git a/profiler/CMakeLists.txt b/profiler/CMakeLists.txt index 241f02f380a..81071386388 100644 --- a/profiler/CMakeLists.txt +++ b/profiler/CMakeLists.txt @@ -309,7 +309,6 @@ if (THEROCK_ENABLE_ROCPROFSYS AND THEROCK_SANITIZER STREQUAL "") BACKGROUND_BUILD CMAKE_ARGS -DHIP_PLATFORM=amd - -DROCPROFSYS_BUILD_TBB=ON -DROCPROFSYS_BUILD_LIBIBERTY=ON -DROCPROFSYS_BUILD_BOOST=ON -DROCPROFSYS_BUILD_DYNINST=ON @@ -342,6 +341,7 @@ if (THEROCK_ENABLE_ROCPROFSYS AND THEROCK_SANITIZER STREQUAL "") ${THEROCK_BUNDLED_ELFUTILS} ${THEROCK_BUNDLED_LIBDRM} ${THEROCK_BUNDLED_SQLITE3} + ${THEROCK_BUNDLED_TBB} ${_openmpi_optional_dep} ) therock_cmake_subproject_glob_c_sources(rocprofiler-systems diff --git a/profiler/post_hook_rocprofiler-systems.cmake b/profiler/post_hook_rocprofiler-systems.cmake new file mode 100644 index 00000000000..e8461cd4bca --- /dev/null +++ b/profiler/post_hook_rocprofiler-systems.cmake @@ -0,0 +1,29 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Dyninst and rocprofiler-systems internal libraries live in lib/rocprofiler-systems. +# Set the RPATH origin per-target so that (for example) lib/rocm_sysdeps/lib resolves as +# $ORIGIN/../rocm_sysdeps/lib rather than $ORIGIN/rocm_sysdeps/lib. +set(_rocprofsys_lib_targets + common + dynElf + dynDwarf + dyninstAPI + dyninstAPI_RT + dynC_API + instructionAPI + parseAPI + patchAPI + pcontrol + stackwalk + symtabAPI + gotcha +) + +foreach(_target ${_rocprofsys_lib_targets}) + if(TARGET "${_target}") + set_target_properties("${_target}" PROPERTIES + THEROCK_INSTALL_RPATH_ORIGIN lib/rocprofiler-systems + ) + endif() +endforeach() diff --git a/third-party/sysdeps/linux/CMakeLists.txt b/third-party/sysdeps/linux/CMakeLists.txt index 5e00d04e43c..af2458fa9ae 100644 --- a/third-party/sysdeps/linux/CMakeLists.txt +++ b/third-party/sysdeps/linux/CMakeLists.txt @@ -131,6 +131,20 @@ if(THEROCK_ENABLE_SYSDEPS_LIBPCIACCESS) ) endif() +# Optional sysdep: TBB (Threading Building Blocks) +if(THEROCK_ENABLE_SYSDEPS_TBB) + add_subdirectory(tbb) + therock_provide_artifact(sysdeps-tbb + TARGET_NEUTRAL + DESCRIPTOR artifact-tbb.toml + COMPONENTS + dev + lib + SUBPROJECT_DEPS + therock-tbb + ) +endif() + # Optional sysdep: hwloc (depends on numactl and libpciaccess) if(THEROCK_ENABLE_SYSDEPS_HWLOC) add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../common/hwloc" "${CMAKE_CURRENT_BINARY_DIR}/hwloc") diff --git a/third-party/sysdeps/linux/artifact-tbb.toml b/third-party/sysdeps/linux/artifact-tbb.toml new file mode 100644 index 00000000000..36c5d5f29ba --- /dev/null +++ b/third-party/sysdeps/linux/artifact-tbb.toml @@ -0,0 +1,3 @@ +# tbb +[components.lib."third-party/sysdeps/linux/tbb/build/stage"] +[components.dev."third-party/sysdeps/linux/tbb/build/stage"] diff --git a/third-party/sysdeps/linux/tbb/CMakeLists.txt b/third-party/sysdeps/linux/tbb/CMakeLists.txt new file mode 100644 index 00000000000..6a631b2304b --- /dev/null +++ b/third-party/sysdeps/linux/tbb/CMakeLists.txt @@ -0,0 +1,93 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + # Super-project block: download sources and declare the sub-project. + set(_source_dir "${CMAKE_CURRENT_BINARY_DIR}/source") + set(_download_stamp "${_source_dir}/download.stamp") + + therock_subproject_fetch(therock-tbb-sources + SOURCE_DIR "${_source_dir}" + # Originally mirrored from: https://github.com/uxlfoundation/oneTBB/archive/refs/tags/v2022.3.0.tar.gz + URL https://rocm-third-party-deps.s3.us-east-2.amazonaws.com/oneTBB-2022.3.0.tar.gz + URL_HASH SHA256=01598a46c1162c27253a0de0236f520fd8ee8166e9ebb84a4243574f88e6e50a + TOUCH "${_download_stamp}" + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + ) + + therock_cmake_subproject_declare(therock-tbb + USE_DIST_AMDGPU_TARGETS + BACKGROUND_BUILD + EXCLUDE_FROM_ALL + NO_MERGE_COMPILE_COMMANDS + OUTPUT_ON_FAILURE + EXTERNAL_SOURCE_DIR . + FPRINT_SOURCE_DIR "${_source_dir}" + FPRINT_FILE_GLOBS "${CMAKE_CURRENT_LIST_DIR}/*" + BINARY_DIR build + CMAKE_ARGS + "-DSOURCE_DIR=${_source_dir}" + "-DPATCHELF=${PATCHELF}" + INSTALL_DESTINATION + lib/rocm_sysdeps + INTERFACE_LINK_DIRS + lib/rocm_sysdeps/lib + INTERFACE_INSTALL_RPATH_DIRS + lib/rocm_sysdeps/lib + INTERFACE_PKG_CONFIG_DIRS + lib/rocm_sysdeps/lib/pkgconfig + EXTRA_DEPENDS + "${_download_stamp}" + ) + + therock_cmake_subproject_provide_package(therock-tbb TBB lib/rocm_sysdeps/lib/cmake/TBB) + therock_cmake_subproject_activate(therock-tbb) + + therock_test_validate_shared_lib( + PATH build/dist/lib/rocm_sysdeps/lib + LIB_NAMES librocm_sysdeps_tbb.so + ) + + return() +endif() + +# Sub-project block: invoked by TheRock's cmake sub-project machinery. +cmake_minimum_required(VERSION 3.25) +project(TBB_BUILD) + +if(NOT PATCHELF) + message(FATAL_ERROR "Missing PATCHELF from super-project") +endif() + +add_custom_target(build ALL + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + # Clean any previous install and create a fresh working copy of the source. + COMMAND + "${CMAKE_COMMAND}" -E rm -rf -- "${CMAKE_INSTALL_PREFIX}" "${CMAKE_CURRENT_BINARY_DIR}/s" + COMMAND + "${CMAKE_COMMAND}" -E copy_directory "${SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/s" + # Patch source: rename library output names to rocm_sysdeps_ prefix. + COMMAND + bash "${CMAKE_CURRENT_SOURCE_DIR}/patch_source.sh" "${CMAKE_CURRENT_BINARY_DIR}/s" + # Configure oneTBB. + COMMAND + "${CMAKE_COMMAND}" + -S "${CMAKE_CURRENT_BINARY_DIR}/s" + -B "${CMAKE_CURRENT_BINARY_DIR}/tbb-build" + -G Ninja + "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" + "-DCMAKE_INSTALL_LIBDIR=lib" + "-DTBB_TEST=OFF" + # Build. + COMMAND + "${CMAKE_COMMAND}" --build "${CMAKE_CURRENT_BINARY_DIR}/tbb-build" --verbose + # Install. + COMMAND + "${CMAKE_COMMAND}" --install "${CMAKE_CURRENT_BINARY_DIR}/tbb-build" + # Patch install: fix RPATH, relocate .pc, remove upstream cmake config. + COMMAND + "${CMAKE_COMMAND}" -E env + "PATCHELF=${PATCHELF}" -- + bash "${CMAKE_CURRENT_SOURCE_DIR}/patch_install.sh" "${CMAKE_INSTALL_PREFIX}" +) diff --git a/third-party/sysdeps/linux/tbb/patch_install.sh b/third-party/sysdeps/linux/tbb/patch_install.sh new file mode 100644 index 00000000000..b14e397a8cd --- /dev/null +++ b/third-party/sysdeps/linux/tbb/patch_install.sh @@ -0,0 +1,50 @@ +#!/usr/bin/bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Patches installed TBB libraries for portable sysdeps distribution: +# - Sets RPATH to $ORIGIN on all real (non-symlink) shared libraries. +# - Makes tbb.pc relocatable by replacing the absolute prefix. +# - Removes the upstream cmake config (TheRock provides its own wrapper). +# +# Args: install_dir +# Env: PATCHELF + +set -e + +PREFIX="${1:?Expected install prefix argument}" +PATCHELF="${PATCHELF:?PATCHELF env var required}" + +LIB_DIR="$PREFIX/lib" + +echo "Patching TBB install..." + +shopt -s nullglob + +# Set RPATH and create unprefixed symlinks (e.g. libtbb.so -> librocm_sysdeps_tbb.so.12). +for lib in "$LIB_DIR"/librocm_sysdeps_tbb*.so.*; do + [ -L "$lib" ] && continue + "$PATCHELF" --set-rpath '$ORIGIN' "$lib" + base=$(basename "$lib") # librocm_sysdeps_tbb.so.12.17 + soname="${base%.*}" # librocm_sysdeps_tbb.so.12 + unprefixed="${soname/rocm_sysdeps_/}" # libtbb.so.12 + symlink="${unprefixed%.so.*}.so" # libtbb.so + ln -sf "$soname" "$LIB_DIR/$symlink" +done + +# Make tbb.pc relocatable: replace the absolute build-time prefix with +# a pcfiledir-relative path. .pc files live at $PREFIX/lib/pkgconfig, +# so ${pcfiledir}/../.. resolves back to $PREFIX at runtime. +PC="$LIB_DIR/pkgconfig/tbb.pc" +if [ -f "$PC" ]; then + abs_prefix=$(grep '^prefix=' "$PC" | cut -d= -f2-) + sed -i "s|prefix=${abs_prefix}|prefix=\${pcfiledir}/../..|" "$PC" + sed -i "s|${abs_prefix}/|\${prefix}/|g" "$PC" +fi + +# Remove the upstream cmake config. It references the original (unprefixed) +# library names and would conflict with TheRock's wrapper config provided via +# therock_cmake_subproject_provide_package. +rm -rf "$PREFIX/lib/cmake/TBB" + +echo "Done patching TBB install." diff --git a/third-party/sysdeps/linux/tbb/patch_source.sh b/third-party/sysdeps/linux/tbb/patch_source.sh new file mode 100644 index 00000000000..03da9957e0d --- /dev/null +++ b/third-party/sysdeps/linux/tbb/patch_source.sh @@ -0,0 +1,44 @@ +#!/usr/bin/bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Patches TBB CMakeLists files to rename library output names with the +# rocm_sysdeps_ prefix, so that installed libraries do not conflict with +# system-installed TBB. Must be run before the cmake configure step. +# +# The sed patterns below are pinned to oneTBB 2022.3.0 and must be +# revalidated when bumping the version. +# +# Args: source_dir + +set -e + +SOURCE_DIR="${1:?Source directory must be given}" + +verify_patch() { + local file="$1" name="$2" + if ! grep -q "OUTPUT_NAME.*rocm_sysdeps_${name}" "$file"; then + echo "ERROR: OUTPUT_NAME patch for ${name} did not apply to ${file}" >&2 + echo "The sed patterns may need updating for this oneTBB version." >&2 + exit 1 + fi +} + +echo "Patching TBB sources..." + +# tbb: insert OUTPUT_NAME after the SOVERSION line in set_target_properties. +sed -i 's/ SOVERSION \${TBB_BINARY_VERSION}$/ SOVERSION ${TBB_BINARY_VERSION}\n OUTPUT_NAME "rocm_sysdeps_tbb"/' \ + "$SOURCE_DIR/src/tbb/CMakeLists.txt" +verify_patch "$SOURCE_DIR/src/tbb/CMakeLists.txt" "tbb" + +# tbbmalloc: same. +sed -i 's/ SOVERSION \${TBBMALLOC_BINARY_VERSION}$/ SOVERSION ${TBBMALLOC_BINARY_VERSION}\n OUTPUT_NAME "rocm_sysdeps_tbbmalloc"/' \ + "$SOURCE_DIR/src/tbbmalloc/CMakeLists.txt" +verify_patch "$SOURCE_DIR/src/tbbmalloc/CMakeLists.txt" "tbbmalloc" + +# tbbmalloc_proxy: SOVERSION is followed by a closing paren on the same line. +sed -i 's/ SOVERSION \${TBBMALLOC_BINARY_VERSION})$/ SOVERSION ${TBBMALLOC_BINARY_VERSION}\n OUTPUT_NAME "rocm_sysdeps_tbbmalloc_proxy")/' \ + "$SOURCE_DIR/src/tbbmalloc_proxy/CMakeLists.txt" +verify_patch "$SOURCE_DIR/src/tbbmalloc_proxy/CMakeLists.txt" "tbbmalloc_proxy" + +echo "Done patching TBB sources."