Merge pull request #24972 from easybuilders/5.2.x

release EasyBuild v5.2.0

Author: boegel

.github/workflows/linting.yml

@@ -22,7 +22,7 @@ jobs:
     - name: install Python packages
       run: |
         pip install --upgrade pip
-        pip install --upgrade flake8
+        pip install --upgrade flake8 flake8-comprehensions
 
     - name: Run flake8 to verify PEP8-compliance of Python code
       run: flake8

.github/workflows/tagbot.py

@@ -2,11 +2,12 @@
 # It should not use any untrusted third party code, or any code checked into the repository itself
 # as that could indirectly grant PRs the ability to edit labels and comments on PRs.
 
+import json
 import os
+from pathlib import Path
+
 import git
 import requests
-import json
-from pathlib import Path
 
 
 def get_first_commit_date(repo, file_path):
@@ -23,8 +24,8 @@ def sort_by_added_date(repo, file_paths):
     return [file for date, file in sorted_files]
 
 
-def similar_easyconfigs(repo, new_file):
-    possible_neighbours = [x for x in new_file.parent.glob('*.eb') if x != new_file]
+def similar_easyconfigs(repo, new_file, new_ecs):
+    possible_neighbours = [x for x in new_file.parent.glob('*.eb') if x not in new_ecs]
     return sort_by_added_date(repo, possible_neighbours)
 
 
@@ -74,11 +75,16 @@ def pr_ecs(pr_diff):
 print("Newly added ECs:", ', '.join(str(p) for p in new_ecs))
 print("Modified workflow:", modified_workflow)
 
+
+# Check for new and updated software.
+# First, try to determine if the software exists. If so,
+# we'll generate a comment diffing against existing EasyConfigs
+# to make review easier.
 new_software = 0
 updated_software = 0
-to_diff = dict()
+to_diff = {}
 for new_file in new_ecs:
-    neighbours = similar_easyconfigs(gitrepo, new_file)
+    neighbours = similar_easyconfigs(gitrepo, new_file, new_ecs)
     print(f"Found {len(neighbours)} neighbours for {new_file}")
     if neighbours:
         updated_software += 1
@@ -114,13 +120,88 @@ def pr_ecs(pr_diff):
             comment += gitrepo.git.diff(f'HEAD:{neighbour}', f'HEAD:{new_file}')
             comment += '\n```\n</details>\n\n'
 
+# After that, try to add additional labels based on the PR contents.
+# Add manual_label if download_instructions is present. This reads the file.
+manual_download = False
+for file in new_ecs + changed_ecs:
+    if file.is_file():
+        with file.open() as f:
+            content = f.read()
+        if 'download_instructions' in content:
+            manual_download = True
+            break
+
+# Add toolchain labels based on matching new added / changed filenames againt our toolchain policy
+# This doesn't include LLVM or NVHPC yet, and needs to be adapted together with the test suite
+# when adding new toolchains and their respective labels.
+# We are only checking the file names here and will not read the actual file.
+gcc_tc_gen_map = {
+    '10.2': '2020b',
+    '10.3': '2021a',
+    '11.2': '2021b',
+    '11.3': '2022a',
+    '12.2': '2022b',
+    '12.3': '2023a',
+    '13.2': '2023b',
+    '13.3': '2024a',
+    '14.2': '2025a',
+    '14.3': '2025b',
+}
+
+ic_tc_gen_map = {
+    '2021.2.0': '2021a',
+    '2021.4.0': '2021b',
+    '2022.1.0': '2022a',
+    '2023.1.0': '2023a',
+    '2023.2.1': '2023b',
+    '2024.2.0': '2024a',
+    '2025.1.1': '2025a',
+    '2025.2.0': '2025b',
+}
+
+llvm_tc_gen_map = {
+    '20.1.5': '2023b',
+    '20.1.8': '2025b',
+}
+
+toolchain_names = ['foss', 'gompi', 'gfbf', 'iimpi', 'iimkl', 'intel', 'llvm-compilers', 'lfbf',
+                   'lompi', 'lmpich', 'lfoss', 'lmpflf']
+toolchain_present = {}
+for toolchain_ver in gcc_tc_gen_map.values():
+    toolchain_present[toolchain_ver] = False
+
+for file in new_ecs + changed_ecs:
+    file_path = str(file)
+    # Check for GCCcore / GCC
+    for gcc_version, toolchain_version in gcc_tc_gen_map.items():
+        if f"-GCCcore-{gcc_version}" in file_path or f"-GCC-{gcc_version}" in file_path:
+            toolchain_present[toolchain_version] = True
+            continue
+    # Check for intel-compilers
+    for intel_version, toolchain_version in ic_tc_gen_map.items():
+        if f"-intel-compilers-{intel_version}" in file_path:
+            toolchain_present[toolchain_version] = True
+            continue
+    for llvm_version, toolchain_version in llvm_tc_gen_map.items():
+        if f"-llvm-compilers-{llvm_version}" in file_path:
+            toolchain_present[toolchain_version] = True
+            continue
+    # Check for common toolchains with our toolchain naming
+    for toolchain_version in gcc_tc_gen_map.values():
+        if any(f"-{toolchain_name}-{toolchain_version}" in file_path for toolchain_name in toolchain_names):
+            toolchain_present[toolchain_version] = True
+            continue
+
 print("Adjusting labels")
 current_labels = [label['name'] for label in data['pull_request']['labels']]
 
 label_checks = [(changed_ecs, 'change'),
                 (new_software, 'new'),
                 (updated_software, 'update'),
-                (modified_workflow, 'workflow')]
+                (modified_workflow, 'workflow'),
+                (manual_download, 'manual_download')]
+# This covers all the toolchain labels we may be able to add
+label_checks += [(val, key) for key, val in toolchain_present.items()]
 
 labels_add = []
 labels_del = []
@@ -164,6 +245,11 @@ def pr_ecs(pr_diff):
         if existing_comment["user"]["login"] == "github-actions[bot]":  # Bot username in GitHub Actions
             comment_id = existing_comment["id"]
 
+    if len(comment) >= 65536:
+        # Comment is too long to post, so post a message saying that
+        comment = "Diff of new easyconfig(s) against existing ones is too long for a GitHub comment. "
+        comment += "Use `--review-pr` (and `--review-pr-filter` / `--review-pr-max`) locally."
+
     if comment_id:
         # Update existing comment
         url = f"{GITHUB_API_URL}/repos/{repo}/issues/comments/{comment_id}"

.github/workflows/unit_tests.yml

@@ -120,9 +120,18 @@ jobs:
           unset PYTHONPATH
 
           # install easyconfigs via distribution package
-          python setup.py sdist > /dev/null
+          echo "::group::Create source distribution"
+          python setup.py sdist
           ls dist
-          pip install dist/easybuild-easyconfigs*tar.gz > /dev/null
+          echo "::endgroup::"
+          echo "::group::Install from source distribution"
+          # Depending on the version of setuptools/distutils the dash will be replaced by underscore
+          if [ -f dist/easybuild-easyconfigs*tar.gz ]; then
+            pip install dist/easybuild-easyconfigs*tar.gz
+          else
+            pip install dist/easybuild_easyconfigs*tar.gz
+          fi
+          echo "::endgroup::"
 
           # robot-paths value should not be empty, but have an entry that includes easybuild/easyconfigs subdir
           echo "eb --show-config"
@@ -161,8 +170,14 @@ jobs:
     - name: Inspect files included in source distribution
       working-directory: dist
       run: |
-          tar xfz easybuild-easyconfigs*tar.gz
-          cd easybuild-easyconfigs-*/
+          ls
+          if [ -f easybuild-easyconfigs*tar.gz ]; then
+            tar xzf easybuild-easyconfigs*tar.gz
+            cd easybuild-easyconfigs-*/
+          else
+            tar xzf easybuild_easyconfigs*tar.gz
+            cd easybuild_easyconfigs-*/
+          fi
 
           # .git folder should not be there in source tarball
           dot_git_files=$(find . -name .git)