Trying to build Intel DPCPP with ROCm 7.2.0

config.yaml

@@ -351,6 +351,10 @@ uenvs:
         santis: [gh200]
         bristen: [a100]
         eiger: [zen2]
+  prgenv-gnu-dpcpp:
+    "25.11":
+      recipes:
+        mi200: 25.11/amdgpu
   prgenv-gnu-openmpi:
     "25.12":
       recipes:

recipes/prgenv-gnu-dpcpp/25.11/amdgpu/README.md

@@ -0,0 +1,103 @@
+# prgenv-gnu-dpcpp (AMD GPU / HIP)
+
+> **Status: Experimental**
+
+GNU compiler toolchain with Intel DPC++ SYCL compiler and HIP backend for
+AMD GPU systems on Alps (gfx90a / MI250X, gfx942 / MI300A & MI300X).
+
+## Overview
+
+This recipe combines:
+- The AMD GPU toolchain from `prgenv-gnu/7.2.0/amdgpu` (ROCm 7.2.0, HIP, RCCL, …)
+- The Intel DPC++ SYCL compiler approach from `prgenv-gnu-dpcpp/25.11/gh200`
+
+The Intel oneAPI binary distribution is x86_64 only and does not expose the
+full HIP backend build path, so a custom Spack package (`llvmdpcpp`) builds
+the compiler from the [intel/llvm](https://github.com/intel/llvm) `sycl`
+branch directly, configured with `SYCL_ENABLE_BACKENDS=opencl;level_zero;hip`
+and `LLVM_TARGETS_TO_BUILD=X86;SPIRV;AMDGPU`.
+
+The environment provides:
+
+- Intel DPC++ compiler (`clang++ -fsycl`) with HIP backend targeting gfx942, gfx90a
+- Full ROCm 7.2.0 stack (rocBLAS, rocFFT, rocSOLVER, rocSPARSE, rocProfiler, …)
+- HIP libraries (hipBLAS, hipBLASLt, hipFFT, hipSPARSE, hipRAND, hipDNN)
+- GPU-aware MPI (cray-mpich@9.1.0 + ROCm GTL)
+- RCCL + aws-ofi-rccl for multi-node GPU collectives
+- Common development tools: Boost, CMake, FFTW, HDF5, NetCDF, OpenBLAS, Python, …
+
+## Custom Spack package: llvmdpcpp
+
+Located at `repo/packages/llvmdpcpp/package.py`. The build mirrors the
+upstream `configure.py --hip` path from the
+[GetStartedGuide](https://github.com/intel/llvm/blob/sycl/sycl/doc/GetStartedGuide.md#build-dpc-toolchain-with-support-for-hip-amd).
+Key differences from the gh200 (CUDA) variant:
+
+- `+hip` instead of `+cuda cuda_arch=90` — no `amdgpu_target` variant needed;
+  architecture is chosen at application compile time via `--offload-arch=`
+- `SYCL_ENABLE_BACKENDS` includes `hip` instead of `cuda`
+- `LLVM_TARGETS_TO_BUILD` includes `AMDGPU` instead of `NVPTX`
+- `lld` added to `LLVM_ENABLE_PROJECTS` (required for AMDGPU ELF code object linking)
+- `SYCL_BUILD_UR_HIP_PLATFORM=AMD` (Unified Runtime HIP adapter platform)
+- `UR_HIP_INCLUDE_DIR` / `UR_HIP_HSA_INCLUDE_DIR` / `UR_HIP_LIB_DIR` point to
+  Spack's `hip` and `hsa-rocr-dev` prefixes (replaces the monolithic `UR_HIP_ROCM_DIR`
+  that upstream assumes at `/opt/rocm`)
+- libclc built as a **runtime target** using `LLVM_RUNTIME_TARGETS=default;amdgcn-amd-amdhsa-llvm`
+  and `RUNTIMES_amdgcn-amd-amdhsa-llvm_LLVM_ENABLE_RUNTIMES=libclc` — not via
+  per-arch `LIBCLC_TARGETS_TO_BUILD` entries
+
+## Build
+
+```bash
+salloc -N 1 --time=240 -A <ACCOUNT>
+
+mkdir -p /dev/shm/$USER
+git clone https://github.com/eth-cscs/stackinator.git /dev/shm/$USER/stackinator
+uv tool install --editable /dev/shm/$USER/stackinator
+git clone git@github.com:eth-cscs/alps-cluster-config.git /dev/shm/$USER/alps-cluster-config
+
+stack-config \
+    --build /dev/shm/$USER/dpcpp-amdgpu \
+    --recipe /path/to/recipes/prgenv-gnu-dpcpp/25.11/amdgpu \
+    --system /dev/shm/$USER/alps-cluster-config/<cluster>
+
+cd /dev/shm/$USER/dpcpp-amdgpu
+env --ignore-environment PATH=/usr/bin:/bin:$(pwd -P)/spack/bin HOME=$HOME make store.squashfs -j144
+
+cp store.squashfs /path/to/persistent/dpcpp-amdgpu.squashfs
+```
+
+## Usage
+
+### Setup
+
+```bash
+uenv start --view=default /path/to/dpcpp-amdgpu.squashfs
+export ROCM_PATH=$(ls -d /user-environment/linux-x86_64/llvm-amdgpu-*)
+```
+
+### Compile
+
+```bash
+# Non-MPI SYCL targeting AMD GPU
+clang++ -std=c++17 -O3 -fsycl \
+    -fsycl-targets=amdgcn-amd-amdhsa \
+    -Xsycl-target-backend --offload-arch=gfx942 \
+    source.cpp -o binary
+
+# MPI (MPICH_CXX tells mpicxx to use DPC++ instead of g++)
+MPICH_CXX=clang++ mpicxx -std=c++17 -O3 -fsycl \
+    -fsycl-targets=amdgcn-amd-amdhsa \
+    -Xsycl-target-backend --offload-arch=gfx942 \
+    source.cpp -o binary
+```
+
+### Run
+
+```bash
+# single-node
+./binary
+
+# multi-node MPI
+srun -n <procs> ./binary
+```

recipes/prgenv-gnu-dpcpp/25.11/amdgpu/compilers.yaml

@@ -0,0 +1,4 @@
+gcc:
+  version: "13"
+llvm-amdgpu:
+  version: "7.2.0"

recipes/prgenv-gnu-dpcpp/25.11/amdgpu/config.yaml

@@ -0,0 +1,11 @@
+name: prgenv-gnu-dpcpp
+spack:
+  repo: https://github.com/spack/spack.git
+  commit: releases/v1.1
+  packages:
+    repo: https://github.com/spack/spack-packages.git
+    commit: 876d656ad17366f92e2233d843ba285d655150dd # develop on 2026-03-09
+store: /user-environment
+description: GNU Compiler toolchain with Intel DPC++ SYCL (HIP backend), cray-mpich, Python, CMake and other development tools for gfx90a, gfx942 (MI300A, MI250X)
+default-view: default
+version: 2

recipes/prgenv-gnu-dpcpp/25.11/amdgpu/environments.yaml

@@ -0,0 +1,94 @@
+gcc-env:
+  compiler: [gcc, llvm-amdgpu]
+  network:
+      mpi: cray-mpich@9.1.0+rocm
+  unify: when_possible
+  duplicates:
+    strategy: full
+  specs:
+  # Intel DPC++ SYCL compiler with HIP/ROCm backend
+  - llvmdpcpp@sycl +clang +hip
+  # Base development tools (from prgenv-gnu-dpcpp/25.11)
+  - boost@1.88.0 +chrono +filesystem +iostreams +mpi +python +regex +serialization +shared +system +timer
+  - cmake
+  - fftw
+  - fmt
+  - gmp
+  - gsl
+  - hdf5+cxx+hl+fortran
+  - netlib-scalapack
+  - lua
+  - libtree
+  - lz4
+  - meson
+  - netcdf-c
+  - netcdf-cxx
+  - netcdf-cxx4
+  - netcdf-fortran
+  - ninja
+  - openblas threads=openmp
+  - osu-micro-benchmarks ~xccl
+  - papi
+  - python@3
+  - zlib-ng
+  # AMD GPU packages (from prgenv-gnu/7.2.0/amdgpu)
+  - hip@7.2.0 ^mesa
+  - hdf5+mpi
+  - hipcub
+  - llvm-amdgpu
+  # rocm packages
+  - rocblas ~tensile ^py-msgpack %cxx=gcc
+  - rocfft
+  - rocm-cmake %cxx=gcc
+  - rocm-openmp-extras
+  - rocm-core %cxx=gcc
+  - rocminfo
+  - rocprim
+  - rocprofiler-dev
+  - rocprofiler-sdk
+  - rocprofiler-systems +internal-dyninst +internal-boost
+  - rocprofiler-compute
+  - rocrand
+  - rocsolver
+  - rocsparse
+  - rocthrust
+  - roctracer-dev-api %cxx=gcc
+  - roctracer-dev
+  - rocm-gdb
+  - rocm-debug-agent
+  - hsa-rocr-dev
+  - comgr
+  # hip packages
+  - hipblas
+  - hipblaslt
+  - hiprand
+  - hipsparse
+  - hipfft
+  - hipblas-common
+  # hipdnn excluded: MiopenLegacyPlugin.hpp uses #pragma once which is treated
+  # as an error by amdclang++ (-Werror=portability-avoid-pragma-once).
+  # miopen-hip is similarly excluded in prgenv-gnu/7.2.0/amdgpu.
+  # - hipdnn
+  # networking
+  - rccl
+  # commits pinned for reproducibility, not bugs; update when updating recipe
+  - aws-ofi-rccl@git.6dae1b0d588c27689ba6daa32710359ea96de78c=cxi
+  - rccl-tests@git.6405c76e6826663bbb67bd40aeee8c70aa5d3094=develop %cxx=llvm-amdgpu
+  variants:
+  - +mpi
+  - +rocm
+  - amdgpu_target=gfx942
+  - amdgpu_target_sram_ecc=gfx942
+  - ^python@3.12.12
+  views:
+    default:
+      link: roots
+      exclude: ["llvm"]
+      uenv:
+        add_compilers: true
+        prefix_paths:
+          LD_LIBRARY_PATH: [lib, lib64]
+        env_vars:
+          set:
+          - FI_PROVIDER: "cxi"
+          - MPIR_CVAR_CH4_OFI_ENABLE_HMEM: "1"

recipes/prgenv-gnu-dpcpp/25.11/amdgpu/extra/reframe.yaml

@@ -0,0 +1,17 @@
+default:
+  features:
+    - rocm
+    - mpi
+    - rccl
+    - rccl-tests
+    - openmp
+    - prgenv
+    - serial
+    - mpich
+    - osu-micro-benchmarks
+    - sycl-dpcpp
+  cc: mpicc
+  cxx: mpic++
+  ftn: mpifort
+  views:
+    - default

recipes/prgenv-gnu-dpcpp/25.11/amdgpu/modules.yaml

@@ -0,0 +1,23 @@
+modules:
+  # Paths to check when creating modules for all module sets
+  prefix_inspections:
+    bin:
+      - PATH
+    lib:
+      - LD_LIBRARY_PATH
+    lib64:
+      - LD_LIBRARY_PATH
+
+  default:
+    arch_folder: false
+    # Where to install modules
+    roots:
+      tcl: /user-environment/modules
+    tcl:
+      all:
+        autoload: none
+      hash_length: 0
+      exclude_implicits: true
+      exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0', 'llvm-amdgpu', 'rccl-tests']
+      projections:
+        all: '{name}/{version}'

recipes/prgenv-gnu-dpcpp/25.11/amdgpu/post-install

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+# remove offending environment variables
+jq '.views["default"].env.values.scalar["HIPCC_LINK_FLAGS_APPEND"]|="" | .views["default"].env.values.scalar["HIPCC_COMPILE_FLAGS_APPEND"]|="" ' /user-environment/meta/env.json > /tmp/env.json
+# copy file back to destination
+cp /tmp/env.json /user-environment/meta/env.json