diff --git a/config.yaml b/config.yaml index ab4cb09ca..0d73b77c2 100644 --- a/config.yaml +++ b/config.yaml @@ -378,6 +378,14 @@ uenvs: daint: [gh200] santis: [gh200] eiger: [zen2] + prgenv-dpcpp: + "25.11": + recipes: + gh200: 25.11/gh200 + deploy: + clariden: [gh200] + daint: [gh200] + santis: [gh200] prgenv-nvfortran: "24.11": recipes: diff --git a/recipes/prgenv-dpcpp/25.11/gh200/README.md b/recipes/prgenv-dpcpp/25.11/gh200/README.md new file mode 100644 index 000000000..ca876df2c --- /dev/null +++ b/recipes/prgenv-dpcpp/25.11/gh200/README.md @@ -0,0 +1,160 @@ +# prgenv-dpcpp + +> **Status: Experimental** + +GNU compiler toolchain with Intel DPC++ SYCL compiler and CUDA backend for +GH200 on Alps aarch64 systems. + +## Overview + +This recipe extends `prgenv-gnu/25.11` with Intel DPC++ built from source. +The Intel oneAPI binary distribution is x86_64 only, so a custom Spack package +(`llvmdpcpp`) builds the compiler from the +[intel/llvm](https://github.com/intel/llvm) `sycl` branch directly on aarch64. + +The environment includes everything from `prgenv-gnu/25.11` (cray-mpich, Boost, +HDF5, NetCDF, Kokkos, FFTW, etc.) plus: + +- Intel DPC++ compiler (`clang++ -fsycl`) with CUDA backend targeting sm_90 +- GPU-aware MPI (cray-mpich + GTL CUDA) compatible with SYCL device pointers + +## Changes from prgenv-gnu/25.11 + +### config.yaml +- Spack pinned to `v1.1.1` (stable release) +- spack-packages updated to `7178c89` (2026-03-09) +- Added `default-view: default` + +### environments.yaml +- `unify: when_possible` (required for DPC++ concretization) +- `cuda@12` pinned +- Added `llvmdpcpp@sycl +clang +cuda cuda_arch=90` + +### Custom Spack package: llvmdpcpp +Located at `repo/packages/llvmdpcpp/package.py`. Key details: + +- Fetches `https://github.com/intel/llvm.git` branch `sycl` +- CMakeLists.txt is in the `llvm/` subdirectory +- Enables SYCL runtime, llvm-spirv, xpti, xptifw, libdevice, sycl-jit, libclc +- CUDA backend via `SYCL_ENABLE_BACKENDS=opencl;level_zero;cuda` +- `compiler-rt` excluded (system GCC too old for sanitizer code on aarch64) +- XPTI tracing disabled (avoids CUPTI path issues in uenv) +- Uses Spack 1.0+ API (`spack_repo.builtin.build_systems` imports) + +## Build + +Build on a compute node using `/dev/shm` for performance: + +```bash +salloc -N 1 --time=240 -A + +mkdir -p /dev/shm/$USER +git clone https://github.com/eth-cscs/stackinator.git /dev/shm/$USER/stackinator +uv tool install --editable /dev/shm/$USER/stackinator +git clone git@github.com:eth-cscs/alps-cluster-config.git /dev/shm/$USER/alps-cluster-config + +stack-config \ + --build /dev/shm/$USER/dpc \ + --recipe /path/to/recipes/prgenv-dpcpp/25.11/gh200 \ + --system /dev/shm/$USER/alps-cluster-config/ + +# Workaround: pre-create CXI staging directories (Spack bug with / in version strings) +mkdir -p /dev/shm/$USER/dpc/tmp/$USER/spack-stage/spack-stage-{cxi-driver,cassini-headers,libcxi}-git.release + +cd /dev/shm/$USER/dpc +env --ignore-environment PATH=/usr/bin:/bin:$(pwd -P)/spack/bin HOME=$HOME make store.squashfs -j144 + +# Copy to persistent storage +cp store.squashfs /path/to/persistent/dpcpp-prgenv.squashfs +``` + +Build time: ~2-3 hours on a single GH200 node. + +## Usage + +### Setup + +```bash +uenv start --view=default /path/to/dpcpp-prgenv.squashfs +export CUDA_PATH=$(ls -d /user-environment/linux-neoverse_v2/cuda-*) +``` + +### Compile + +```bash +# Non-MPI +clang++ -std=c++17 -O3 -fsycl \ + -fsycl-targets=nvptx64-nvidia-cuda \ + -Xsycl-target-backend --cuda-gpu-arch=sm_90 \ + --cuda-path=$CUDA_PATH \ + source.cpp -o binary + +# MPI (MPICH_CXX tells mpicxx to use DPC++ instead of g++) +MPICH_CXX=clang++ mpicxx -std=c++17 -O3 -fsycl \ + -fsycl-targets=nvptx64-nvidia-cuda \ + -Xsycl-target-backend --cuda-gpu-arch=sm_90 \ + --cuda-path=$CUDA_PATH \ + source.cpp -o binary +``` + +### Run + +```bash +# Single GPU +srun -n 1 -A \ + --uenv=/path/to/dpcpp-prgenv.squashfs \ + /path/to/bind_numa.sh ./binary + +# Multi-node MPI +srun -n 8 -N 2 -A --mpi=cray_shasta \ + --uenv=/path/to/dpcpp-prgenv.squashfs \ + --export=ALL,MPICH_GPU_SUPPORT_ENABLED=1 \ + /path/to/bind_numa.sh ./binary +``` + +### GPU-aware MPI + +SYCL device pointers (`sycl::malloc_device`) can be passed directly to MPI +calls without host staging. Requires `MPICH_GPU_SUPPORT_ENABLED=1`, which can +be set via `--export` on the `srun` command line or in the `bind_numa.sh` +wrapper script. + +### oneAPI DPL + +If the application uses `#include ` headers, oneDPL must be +provided separately (it is not included in the uenv): + +```bash +git clone https://github.com/oneapi-src/oneDPL.git /path/to/oneDPL +# Add to compile command: -I/path/to/oneDPL/include +``` + +## Known issues + +1. **CXI staging directories**: Spack has a bug with `/` in version strings + for CXI network packages (`cxi-driver`, `cassini-headers`, `libcxi`). + Staging directories must be pre-created before building (see build + instructions above). + +2. **UR/CUDA cleanup errors at exit**: The DPC++ runtime may report + `UR_RESULT_ERROR_UNKNOWN` errors during program exit. These are caused by + SYCL device memory not being explicitly freed before `MPI_Finalize()` and + do not affect simulation results. + +3. **`--mpi=cray_shasta`**: Required on Clariden for MPI to initialize + correctly. Not required on Santis. + +4. **`--cuda-path`**: Required at compile time. The DPC++ compiler needs this + flag to locate CUDA libdevice files. + +5. **`MPICH_CXX=clang++`**: Required when compiling with the `mpicxx` wrapper. + Without it, `mpicxx` uses `g++` which does not support `-fsycl`. + +6. **oneAPI libraries**: Only the DPC++ compiler is included. Libraries such + as oneDPL, oneMKL, and oneTBB are not part of this uenv and must be + provided separately if needed. + +## Tested on + +- **Santis** (GH200, aarch64): single-node and multi-node MPI +- **Clariden** (GH200, aarch64): up to 2262 nodes (9048 GPUs), production runs diff --git a/recipes/prgenv-dpcpp/25.11/gh200/compilers.yaml b/recipes/prgenv-dpcpp/25.11/gh200/compilers.yaml new file mode 100644 index 000000000..65120c9f4 --- /dev/null +++ b/recipes/prgenv-dpcpp/25.11/gh200/compilers.yaml @@ -0,0 +1,2 @@ +gcc: + version: "14" diff --git a/recipes/prgenv-dpcpp/25.11/gh200/config.yaml b/recipes/prgenv-dpcpp/25.11/gh200/config.yaml new file mode 100644 index 000000000..6839b2d13 --- /dev/null +++ b/recipes/prgenv-dpcpp/25.11/gh200/config.yaml @@ -0,0 +1,11 @@ +name: prgenv-dpcpp +spack: + repo: https://github.com/spack/spack.git + commit: v1.1.1 + packages: + repo: https://github.com/spack/spack-packages.git + commit: 7178c89f9bb9bb96f692325509e9348f243b9a0a # develop on 2026-03-09 +store: /user-environment +description: GNU Compiler toolchain with Intel DPC++ SYCL, cray-mpich, Python, CMake and other development tools. +default-view: default +version: 2 diff --git a/recipes/prgenv-dpcpp/25.11/gh200/environments.yaml b/recipes/prgenv-dpcpp/25.11/gh200/environments.yaml new file mode 100644 index 000000000..dc0a4fb17 --- /dev/null +++ b/recipes/prgenv-dpcpp/25.11/gh200/environments.yaml @@ -0,0 +1,49 @@ +gcc-env: + compiler: [gcc] + network: + mpi: cray-mpich@9 +cuda + unify: when_possible + specs: + - llvmdpcpp@sycl +clang +cuda cuda_arch=90 + - boost +chrono +filesystem +iostreams +mpi +python +regex +serialization +shared +system +timer + - cmake + - fftw + - fmt + - gmp + - gsl + - hdf5+cxx+hl+fortran + - kokkos +aggressive_vectorization ~alloc_async cuda_arch=90 +cuda_constexpr +cuda_lambda ~cuda_relocatable_device_code ~cuda_uvm cxxstd=17 +openmp +pic +serial +shared +tuning +wrapper + - kokkos-kernels +blas +cublas +cusparse +cusolver +execspace_cuda +execspace_openmp +execspace_serial +lapack +memspace_cudaspace +openmp scalars=float,double,complex_float,complex_double +serial +shared +superlu + - kokkos-tools +mpi +papi + - netlib-scalapack + - lua + - libtree + - lz4 + - meson + - netcdf-c + - netcdf-cxx + - netcdf-cxx4 + - netcdf-fortran + - ninja + - openblas threads=openmp + - osu-micro-benchmarks + - papi + - python + - zlib-ng + - nccl + - nccl-tests + - cuda + - xcb-util-cursor + - aws-ofi-nccl + - superlu + variants: + - +mpi + - +cuda + - cuda_arch=90a + views: + default: + link: roots + uenv: + add_compilers: true + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] diff --git a/recipes/prgenv-dpcpp/25.11/gh200/extra/reframe.yaml b/recipes/prgenv-dpcpp/25.11/gh200/extra/reframe.yaml new file mode 100644 index 000000000..f6eb64bab --- /dev/null +++ b/recipes/prgenv-dpcpp/25.11/gh200/extra/reframe.yaml @@ -0,0 +1,17 @@ +default: + features: + - cuda + - mpi + - cray-mpich + - nccl + - nccl-tests + - openmp + - osu-micro-benchmarks + - prgenv + - serial + - sycl-dpcpp + cc: mpicc + cxx: mpic++ + ftn: mpifort + views: + - default diff --git a/recipes/prgenv-dpcpp/25.11/gh200/modules.yaml b/recipes/prgenv-dpcpp/25.11/gh200/modules.yaml new file mode 100644 index 000000000..7b94eed4c --- /dev/null +++ b/recipes/prgenv-dpcpp/25.11/gh200/modules.yaml @@ -0,0 +1,23 @@ +modules: + # Paths to check when creating modules for all module sets + prefix_inspections: + bin: + - PATH + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + + default: + arch_folder: false + # Where to install modules + roots: + tcl: /user-environment/modules + tcl: + all: + autoload: none + hash_length: 0 + exclude_implicits: true + exclude: [] + projections: + all: '{name}/{version}' diff --git a/recipes/prgenv-dpcpp/25.11/gh200/repo/packages/llvmdpcpp/package.py b/recipes/prgenv-dpcpp/25.11/gh200/repo/packages/llvmdpcpp/package.py new file mode 100644 index 000000000..b737b763a --- /dev/null +++ b/recipes/prgenv-dpcpp/25.11/gh200/repo/packages/llvmdpcpp/package.py @@ -0,0 +1,113 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +""" +Custom Spack package for Intel DPC++ (intel/llvm sycl branch). + +This builds the full DPC++ SYCL compiler from the intel/llvm repository, +replicating what `configure.py --cuda` does but through Spack. +""" + +import os + +from spack_repo.builtin.build_systems.cmake import CMakePackage, generator +from spack_repo.builtin.build_systems.cuda import CudaPackage + +from spack.package import * + + +class Llvmdpcpp(CMakePackage, CudaPackage): + """Intel DPC++ SYCL compiler built from the intel/llvm open-source repository.""" + + homepage = "https://github.com/intel/llvm" + git = "https://github.com/intel/llvm.git" + + maintainers("gandanie") + + version("sycl", branch="sycl", preferred=True) + version("sycl-2025.1", tag="sycl/unified/2025.1") + + variant("clang", default=True, description="Build clang (required for DPC++)") + + depends_on("cmake@3.20:", type="build") + depends_on("ninja", type="build") + depends_on("python@3.6:", type="build") + depends_on("zstd", type="link") + + # CUDA dependencies + depends_on("cuda@12:", when="+cuda") + + # Force clang on + conflicts("~clang", msg="DPC++ requires clang") + + generator("ninja") + + @property + def root_cmakelists_dir(self): + """The CMakeLists.txt is in the llvm subdirectory.""" + return "llvm" + + def cmake_args(self): + spec = self.spec + src = self.stage.source_path + + # Paths to DPC++-specific sub-projects within intel/llvm + sycl_dir = os.path.join(src, "sycl") + spirv_dir = os.path.join(src, "llvm-spirv") + xpti_dir = os.path.join(src, "xpti") + xptifw_dir = os.path.join(src, "xptifw") + libdevice_dir = os.path.join(src, "libdevice") + jit_dir = os.path.join(src, "sycl-jit") + + # LLVM targets + llvm_targets = "AArch64;ARM;X86;SPIRV" + + # SYCL backends + sycl_backends = ["opencl", "level_zero"] + + # LibCLC targets + libclc_targets = "" + + if spec.satisfies("+cuda"): + llvm_targets += ";NVPTX" + sycl_backends.append("cuda") + libclc_targets = "nvptx64-nvidia-cuda" + + args = [ + self.define("CMAKE_BUILD_TYPE", "Release"), + self.define("LLVM_ENABLE_ASSERTIONS", False), + self.define("LLVM_TARGETS_TO_BUILD", llvm_targets), + self.define("LLVM_EXTERNAL_PROJECTS", + "sycl;llvm-spirv;opencl;xpti;xptifw;libdevice;sycl-jit"), + self.define("LLVM_EXTERNAL_SYCL_SOURCE_DIR", sycl_dir), + self.define("LLVM_EXTERNAL_LLVM_SPIRV_SOURCE_DIR", spirv_dir), + self.define("LLVM_EXTERNAL_XPTI_SOURCE_DIR", xpti_dir), + self.define("XPTI_SOURCE_DIR", xpti_dir), + self.define("LLVM_EXTERNAL_XPTIFW_SOURCE_DIR", xptifw_dir), + self.define("LLVM_EXTERNAL_LIBDEVICE_SOURCE_DIR", libdevice_dir), + self.define("LLVM_EXTERNAL_SYCL_JIT_SOURCE_DIR", jit_dir), + self.define("LLVM_ENABLE_PROJECTS", + "clang;llvm-spirv;opencl;xpti;xptifw;" + "libdevice;sycl;sycl-jit;libclc"), + self.define("LLVM_BUILD_TOOLS", True), + self.define("LLVM_ENABLE_ZSTD", True), + self.define("LLVM_USE_STATIC_ZSTD", True), + self.define("SYCL_ENABLE_WERROR", False), + self.define("SYCL_INCLUDE_TESTS", False), + self.define("BUILD_SHARED_LIBS", False), + self.define("SYCL_ENABLE_XPTI_TRACING", False), + self.define("SYCL_ENABLE_BACKENDS", ";".join(sycl_backends)), + self.define("SYCL_ENABLE_EXTENSION_JIT", True), + self.define("SYCL_ENABLE_MAJOR_RELEASE_PREVIEW_LIB", True), + self.define("BUG_REPORT_URL", "https://github.com/intel/llvm/issues"), + ] + + if spec.satisfies("+cuda"): + args.extend([ + self.define("CUDA_TOOLKIT_ROOT_DIR", spec["cuda"].prefix), + self.define("LIBCLC_TARGETS_TO_BUILD", libclc_targets), + self.define("LIBCLC_GENERATE_REMANGLED_VARIANTS", True), + ]) + + return args diff --git a/recipes/prgenv-dpcpp/25.11/gh200/repo/repo.yaml b/recipes/prgenv-dpcpp/25.11/gh200/repo/repo.yaml new file mode 100644 index 000000000..9a8f0ae84 --- /dev/null +++ b/recipes/prgenv-dpcpp/25.11/gh200/repo/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: dpcpp