prgenv-gnu with ROCm 7

config.yaml

@@ -68,7 +68,7 @@ clusters:
           F7T_CLIENT_SECRET: $F7T_TDS_CLIENT_SECRET
           F7T_URL: "https://api.tds.cscs.ch/stp/firecrest/v2"
           F7T_TOKEN_URL: "https://auth-tds.cscs.ch/auth/realms/firecrest-clients/protocol/openid-connect/token"
-          SLURM_TIMELIMIT: '500'
+          SLURM_TIMELIMIT: '720'
           SLURM_CPUS_PER_TASK: '64'
       -
         uarch: 'mi300'
@@ -349,6 +349,11 @@ uenvs:
         santis: [gh200]
         bristen: [a100]
         eiger: [zen2]
+    "7.1.0":
+      recipes:
+        mi200: 7.1.0/amdgpu
+      deploy:
+        beverin: [mi200]
   prgenv-gnu-openmpi:
     "25.12":
       recipes:

recipes/prgenv-gnu/7.1.0/amdgpu/compilers.yaml

@@ -0,0 +1,4 @@
+gcc:
+  version: "13"
+llvm-amdgpu:
+  version: "7.1.0"

recipes/prgenv-gnu/7.1.0/amdgpu/config.yaml

@@ -0,0 +1,9 @@
+name: prgenv-gnu
+version: 2
+spack:
+  commit: releases/v1.0
+  repo: https://github.com/spack/spack.git
+  packages:
+    repo: https://github.com/msimberg/spack-packages.git
+    commit: aws-ofi-nccl-with-rocm-old-develop # develop on 2025-12-19 + https://github.com/spack/spack-packages/compare/develop...msimberg:spack-packages:aws-ofi-nccl-with-rocm-old-develop
+description: GNU Compiler toolchain with cray-mpich, Python, CMake and other development tools for gfx90a, gfx942 (MI300A, MI250X)

recipes/prgenv-gnu/7.1.0/amdgpu/environments.yaml

@@ -0,0 +1,86 @@
+gcc-env:
+  compiler: [gcc, llvm-amdgpu]
+  network:
+      mpi: cray-mpich@9.1 +rocm
+  unify: when_possible
+  duplicates:
+    strategy: full
+  specs:
+  - boost +chrono +filesystem +iostreams +mpi +python +regex +serialization +shared +system +timer
+  - cmake
+  - fftw
+  - fmt
+  - gsl
+  - hdf5+cxx+hl+fortran+mpi
+  - netlib-scalapack
+  - lua
+  - libtree
+  - lz4
+  - meson
+  - ninja
+  - openblas threads=openmp
+  - osu-micro-benchmarks ~xccl
+  - python@3
+  - zlib-ng
+  - netcdf-c
+  - netcdf-cxx
+  - netcdf-cxx4
+  - netcdf-fortran
+  # add GPU-specific packages here, for easier comparison with mc version
+  - hip@7.1.0 ^mesa@23.3.6
+  - hipcub
+  - llvm-amdgpu
+  # rocm packages
+  - rocblas ^py-msgpack %cxx=gcc
+  - rocfft
+  - rocm-cmake %cxx=gcc
+  - rocm-openmp-extras
+  - rocm-core %cxx=gcc
+  - rocminfo
+  - rocprim
+  - rocprofiler-dev
+  - rocprofiler-sdk
+  - rocprof-trace-decoder
+  - rocprofiler-systems +internal-dyninst+internal-boost+trace-decoder
+  - rocprofiler-compute
+  - rocrand
+  - rocsolver
+  - rocsparse
+  - rocthrust
+  - roctracer-dev-api %cxx=gcc
+  - roctracer-dev
+  - rocm-gdb
+  - rocm-debug-agent
+  - hsa-rocr-dev
+  - comgr
+  # hip packages
+  - hipblas
+  - hipblaslt
+  - hiprand
+  - hipsparse~shared-libs
+  - hipfft
+  - hipblas-common
+  # - miopen-hip  # TODO(iomaganaris): enable for julia AMD gpu build
+  # networking
+  - rccl
+  # commits pinned for reproducibility, not bugs; update when updating recipe
+  - aws-ofi-nccl@1.18.0
+  - rccl-tests@git.6405c76e6826663bbb67bd40aeee8c70aa5d3094=develop %cxx=llvm-amdgpu ^cray-mpich +rocm %c,cxx,fortran=gcc
+  variants:
+  - +mpi
+  - +rocm
+  - amdgpu_target=gfx942,gfx90a
+  - amdgpu_target_sram_ecc=gfx942,gfx90a
+  - ^python@3.12.12
+  views:
+    default:
+      link: roots
+      exclude: ["llvm"]
+      uenv:
+        add_compilers: true
+        prefix_paths:
+          LD_LIBRARY_PATH: [lib, lib64]
+        env_vars:
+          set:
+          - NCCL_NET_PLUGIN: "ofi"
+          - HSA_NO_SCRATCH_RECLAIM: "1"

recipes/prgenv-gnu/7.1.0/amdgpu/extra/reframe.yaml

@@ -0,0 +1,16 @@
+default:
+  features:
+    - rocm
+    - mpi
+    - rccl
+    - rccl-tests
+    - openmp
+    - prgenv
+    - serial
+    - cray-mpich
+    - osu-micro-benchmarks
+  cc: mpicc
+  cxx: mpic++
+  ftn: mpifort
+  views:
+    - default

recipes/prgenv-gnu/7.1.0/amdgpu/modules.yaml

@@ -0,0 +1,23 @@
+modules:
+  # Paths to check when creating modules for all module sets
+  prefix_inspections:
+    bin:
+      - PATH
+    lib:
+      - LD_LIBRARY_PATH
+    lib64:
+      - LD_LIBRARY_PATH
+
+  default:
+    arch_folder: false
+    # Where to install modules
+    roots:
+      tcl: /user-environment/modules
+    tcl:
+      all:
+        autoload: none
+      hash_length: 0
+      exclude_implicits: true
+      exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0']
+      projections:
+        all: '{name}/{version}'

recipes/prgenv-gnu/7.1.0/amdgpu/post-install

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+# remove offending environment variables
+jq '.views["default"].env.values.scalar["HIPCC_LINK_FLAGS_APPEND"]|="" | .views["default"].env.values.scalar["HIPCC_COMPILE_FLAGS_APPEND"]|="" ' /user-environment/meta/env.json > /tmp/env.json
+# copy file back to destination
+cp /tmp/env.json /user-environment/meta/env.json

recipes/prgenv-gnu/7.1.0/amdgpu/repo/packages/hipblaslt/0001-Set-LLVM-Path-6.3.Patch

@@ -0,0 +1,53 @@
+diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt
+index 865dcd9..0ca1815 100644
+--- a/library/CMakeLists.txt
++++ b/library/CMakeLists.txt
+@@ -72,6 +72,7 @@ include(src/CMakeLists.txt)
+ # Create hipBLASLt library
+ add_library(hipblaslt ${hipblaslt_source} ${hipblaslt_headers_public})
+ add_library(roc::hipblaslt ALIAS hipblaslt)
++target_include_directories( hipblaslt PRIVATE ${MSGPACK_DIR}/include )
+
+ # Target compile definitions
+ if(NOT BUILD_CUDA)
+@@ -154,7 +155,7 @@ if( LEGACY_HIPBLAS_DIRECT )
+   )
+ else()
+   target_include_directories(hipblaslt
+-                             PUBLIC  $<BUILD_INTERFACE:${HIPBLAS-COMMON_INCLUDE_DIRS}>
++                             PUBLIC  ${HIPBLAS-COMMON_INCLUDE_DIRS}
+   )
+ endif()
+
+diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py
+index 6b5cdc8..9f28e46 100644
+--- a/tensilelite/Tensile/Common.py
++++ b/tensilelite/Tensile/Common.py
+@@ -277,6 +277,8 @@ globalParameters["LazyLibraryLoading"] = False # Load library and code object fi
+ globalParameters["UseUserArgs"] = False
+
+ globalParameters["RotatingBufferSize"] = 0 # Size in MB
++globalParameters["LLVMPath"] = os.environ.get("LLVM_PATH")
++globalParameters["ROCmSMIPath"] = os.environ.get("ROCM_SMI_PATH")
+ globalParameters["RotatingMode"] = 0 # Default is 0, allocated in order A0B0C0D0..ANBNCNDN. 1 is in order A0 pad B0 pad .... AN pad BN pad.
+                                      # Mode 0 requires memcpy everytime when the problem changes to reset the data, but mode 1 doesn't.
+
+@@ -1576,7 +1578,7 @@ def assignGlobalParameters( config ):
+   if os.name == "nt":
+     globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe")
+   else:
+-    globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm_agent_enumerator")
++    globalParameters["ROCmAgentEnumeratorPath"] = os.environ.get("ROCM_AGENT_ENUMERATOR_PATH")
+
+   if "CxxCompiler" in config:
+     globalParameters["CxxCompiler"] = config["CxxCompiler"]
+@@ -1602,8 +1604,7 @@ def assignGlobalParameters( config ):
+       compiler = "clang++" if globalParameters["CxxCompiler"] == "hipcc" else "amdclang++"
+       globalParameters["AssemblerPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], bin_path), compiler)
+
+-  globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi")
+-  globalParameters["ROCmLdPath"]  = locateExe(os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "ld.lld")
++  globalParameters["ROCmLdPath"]  = locateExe(os.path.join(globalParameters["LLVMPath"], "bin"), "ld.lld")
+
+   globalParameters["ExtractKernelPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel")
+

recipes/prgenv-gnu/7.1.0/amdgpu/repo/packages/hipblaslt/0001-Set-LLVM_Path-Add-Hiblas-Include-to-CmakeLists-6.1.Patch

@@ -0,0 +1,88 @@
+From 085d965e11cda1830cf325e0d12db3faf61a94d0 Mon Sep 17 00:00:00 2001
+From: sreenivasa murthy kolam <sreenivasamurthy.kolam@amd.com>
+Date: Thu, 23 May 2024 05:49:34 +0000
+Subject: [PATCH] Add hipblas include dir in CMakeLists.txt and Modify the LLVM
+ Path in the Tensile code for Spack
+
+---
+ clients/gtest/CMakeLists.txt            | 1 +
+ library/CMakeLists.txt                  | 2 ++
+ tensilelite/Tensile/Common.py           | 7 ++++---
+ tensilelite/Tensile/Ops/gen_assembly.sh | 2 +-
+ 4 files changed, 8 insertions(+), 4 deletions(-)
+
+diff --git a/clients/gtest/CMakeLists.txt b/clients/gtest/CMakeLists.txt
+index 825bdca..f817e12 100644
+--- a/clients/gtest/CMakeLists.txt
++++ b/clients/gtest/CMakeLists.txt
+@@ -53,6 +53,7 @@ target_include_directories( hipblaslt-test
+     $<BUILD_INTERFACE:${BLAS_INCLUDE_DIR}>
+     $<BUILD_INTERFACE:${BLIS_INCLUDE_DIR}> # may be blank if not used
+     $<BUILD_INTERFACE:${GTEST_INCLUDE_DIRS}>
++    $<BUILD_INTERFACE:${HIPBLAS_INCLUDE_DIRS}>
+ )
+ message("BLIS_INCLUDE_DIR=" ${BLIS_INCLUDE_DIR})
+ target_link_libraries( hipblaslt-test PRIVATE ${BLAS_LIBRARY} ${GTEST_BOTH_LIBRARIES} roc::hipblaslt )
+diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt
+index 3252da0..1b8d628 100644
+--- a/library/CMakeLists.txt
++++ b/library/CMakeLists.txt
+@@ -72,6 +72,8 @@ include(src/CMakeLists.txt)
+ # Create hipBLASLt library
+ add_library(hipblaslt ${hipblaslt_source} ${hipblaslt_headers_public})
+ add_library(roc::hipblaslt ALIAS hipblaslt)
++target_include_directories( hipblaslt PRIVATE ${HIPBLAS_INCLUDE_DIRS} )
++target_include_directories( hipblaslt PRIVATE ${MSGPACK_DIR}/include )
+
+ # Target compile definitions
+ if(NOT BUILD_CUDA)
+diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py
+index 8ee6373..52d6a97 100644
+--- a/tensilelite/Tensile/Common.py
++++ b/tensilelite/Tensile/Common.py
+@@ -273,6 +273,7 @@ globalParameters["LazyLibraryLoading"] = False # Load library and code object fi
+ globalParameters["UseUserArgs"] = False
+
+ globalParameters["RotatingBufferSize"] = 0 # Size in MB
++globalParameters["LLVMPath"] = os.environ.get("LLVM_PATH")
+
+ # Save a copy - since pytest doesn't re-run this initialization code and YAML files can override global settings - odd things can happen
+ defaultGlobalParameters = deepcopy(globalParameters)
+@@ -1488,10 +1489,10 @@ def assignGlobalParameters( config ):
+     if os.name == "nt":
+       globalParameters["AssemblerPath"] = locateExe(globalParameters["ROCmBinPath"], "clang++.exe")
+     else:
+-      globalParameters["AssemblerPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "clang++")
++      globalParameters["AssemblerPath"] = locateExe(os.path.join(globalParameters["LLVMPath"], "bin"), "clang++")
+
+   globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi")
+-  globalParameters["ROCmLdPath"]  = locateExe(os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "ld.lld")
++  globalParameters["ROCmLdPath"]  = locateExe(os.path.join(globalParameters["LLVMPath"], "bin"), "ld.lld")
+
+   globalParameters["ExtractKernelPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel")
+
+@@ -1501,7 +1502,7 @@ def assignGlobalParameters( config ):
+     if os.name == "nt":
+       globalParameters["ClangOffloadBundlerPath"] = locateExe(globalParameters["ROCmBinPath"], "clang-offload-bundler.exe")
+     else:
+-      globalParameters["ClangOffloadBundlerPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "clang-offload-bundler")
++      globalParameters["ClangOffloadBundlerPath"] = locateExe(os.path.join(globalParameters["LLVMPath"], "bin"), "clang-offload-bundler")
+
+   if "ROCmAgentEnumeratorPath" in config:
+     globalParameters["ROCmAgentEnumeratorPath"] = config["ROCmAgentEnumeratorPath"]
+diff --git a/tensilelite/Tensile/Ops/gen_assembly.sh b/tensilelite/Tensile/Ops/gen_assembly.sh
+index 230c446..9f7dc6d 100644
+--- a/tensilelite/Tensile/Ops/gen_assembly.sh
++++ b/tensilelite/Tensile/Ops/gen_assembly.sh
+@@ -32,7 +32,7 @@ if ! [ -z ${ROCM_PATH+x} ]; then
+     rocm_path=${ROCM_PATH}
+ fi
+
+-toolchain=${rocm_path}/llvm/bin/clang++
++toolchain=${LLVM_PATH}/bin/clang++
+
+ . ${venv}/bin/activate
+
+-- 
+2.39.3
+

recipes/prgenv-gnu/7.1.0/amdgpu/repo/packages/hipblaslt/0004-Set-rocm-smi-ld-path-7.0.patch

@@ -0,0 +1,22 @@
+diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py
+index 4cbf586..e2602b9 100644
+--- a/tensilelite/Tensile/Common/GlobalParameters.py
++++ b/tensilelite/Tensile/Common/GlobalParameters.py
+@@ -552,7 +552,7 @@ def assignGlobalParameters(config, isaInfoMap: Dict[IsaVersion, IsaInfo]):
+
+     globalParameters["ROCmBinPath"] = os.path.join(globalParameters["ROCmPath"], "bin")
+     try:
+-        globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi")
++        globalParameters["ROCmSMIPath"] = os.environ.get("ROCM_SMI_PATH")
+     except OSError:
+         if os.name == "nt":
+             # rocm-smi is not presently supported on Windows so do not require it.
+@@ -560,7 +560,7 @@ def assignGlobalParameters(config, isaInfoMap: Dict[IsaVersion, IsaInfo]):
+         else:
+             raise
+     globalParameters["ROCmLdPath"] = locateExe(
+-        os.path.join(globalParameters["ROCmPath"], "lib/llvm/bin"),
++        os.environ.get("ROCM_LD_PATH"),
+         "ld.lld" if os.name != "nt" else "ld.lld.exe"
+     )
+