Run regression on CI

.github/workflows/_regression-job.yml

@@ -0,0 +1,126 @@
+name: Regression Job (Reusable)
+
+on:
+  workflow_call:
+    inputs:
+      platform:
+        required: true
+        type: string
+      language:
+        required: true
+        type: string
+      version:
+        required: true
+        type: string
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    env:
+      RESOURCE_PREFIX: sebs-ci
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Restore SeBS cache
+        uses: actions/cache/restore@v4
+        with:
+          path: regression-cache/
+          key: sebs-cache-${{ github.ref_name }}-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          restore-keys: |
+            sebs-cache-${{ github.ref_name }}-
+
+      - name: Setup GCP credentials
+        if: inputs.platform == 'gcp'
+        run: |
+          echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" > /tmp/gcp-credentials.json
+          echo "GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json" >> $GITHUB_ENV
+
+      - name: Setup Azure credentials
+        if: inputs.platform == 'azure'
+        run: |
+          echo "AZURE_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> $GITHUB_ENV
+          echo "AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}" >> $GITHUB_ENV
+          echo "AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}" >> $GITHUB_ENV
+          echo "AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}" >> $GITHUB_ENV
+
+      - name: Setup AWS credentials
+        if: inputs.platform == 'aws'
+        run: |
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> $GITHUB_ENV
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
+          echo "AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION || 'us-east-1' }}" >> $GITHUB_ENV
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install SeBS
+        run: uv pip install .
+
+      - name: Run regression tests
+        timeout-minutes: 5
+        run: |
+          sebs benchmark regression test \
+            --config configs/example.json \
+            --deployment ${{ inputs.platform }} \
+            --language ${{ inputs.language }} \
+            --language-version ${{ inputs.version }} \
+            --architecture x64 --selected-architecture \
+            --resource-prefix sebs-ci
+
+      - name: Generate test summary
+        if: always()
+        run: |
+          echo "Regression Test Summary" > test-summary.txt
+          echo "======================" >> test-summary.txt
+          echo "Platform: ${{ inputs.platform }}" >> test-summary.txt
+          echo "Language: ${{ inputs.language }}" >> test-summary.txt
+          echo "Version: ${{ inputs.version }}" >> test-summary.txt
+          echo "" >> test-summary.txt
+          if ls regression_*.json 1> /dev/null 2>&1; then
+            ls -1 regression_*.json | wc -l | xargs echo "Benchmarks tested:" >> test-summary.txt
+            echo "" >> test-summary.txt
+            echo "Results saved to artifacts/results/" >> test-summary.txt
+          else
+            echo "No benchmark results found" >> test-summary.txt
+          fi
+
+      - name: Upload test summary
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-summary-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          path: test-summary.txt
+
+      - name: Collect and upload regression results
+        if: always()
+        run: |
+          mkdir -p results
+          if ls regression_*.json 1> /dev/null 2>&1; then
+            mv regression_*.json results/ || true
+          fi
+
+      - name: Upload regression results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: results-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          path: results/
+          if-no-files-found: ignore
+
+      - name: Upload cache snapshot
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cache-snapshot-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          path: cache/
+          if-no-files-found: ignore
+
+      - name: Save SeBS cache
+        if: success()
+        uses: actions/cache/save@v4
+        with:
+          path: regression-cache/
+          key: sebs-cache-${{ github.ref_name }}-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}

.github/workflows/lint.yml

@@ -0,0 +1,55 @@
+name: Lint
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  linting:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install system dependencies
+        run: sudo apt update && sudo apt install -y libcurl4-openssl-dev
+
+      - name: Cache uv dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: uv-${{ runner.os }}-${{ hashFiles('requirements.txt', 'pyproject.toml') }}
+          restore-keys: |
+            uv-${{ runner.os }}-
+
+      - name: Install SeBS with dev dependencies
+        run: uv sync --extra dev
+
+      - name: Python code formatting with black
+        run: uv run black sebs --check --config .black.toml
+
+      - name: Python code lint with flake8
+        run: uv run flake8 sebs --config=.flake8.cfg --tee --output-file flake-reports
+
+      - name: Python static code verification with mypy
+        run: uv run mypy sebs --config-file=.mypy.ini
+
+      - name: Check for Python documentation coverage
+        run: uv run interrogate -v --fail-under 100 sebs
+
+      - name: Upload flake8 reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: flake-reports
+          path: flake-reports

.github/workflows/regression.yml

@@ -0,0 +1,25 @@
+name: Regression Tests
+
+on:
+  push:
+    branches:
+      - master
+      - 'feature/**'
+  workflow_dispatch:
+
+jobs:
+  regression:
+    strategy:
+      matrix:
+        include:
+          - platform: aws
+            language: python
+            version: "3.11"
+      fail-fast: false
+
+    uses: ./.github/workflows/_regression-job.yml
+    with:
+      platform: ${{ matrix.platform }}
+      language: ${{ matrix.language }}
+      version: ${{ matrix.version }}
+    secrets: inherit

benchmarks/wrappers/aws/python/setup.py

@@ -1,14 +1,9 @@
 # Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved.
 from distutils.core import setup
 from glob import glob
-from pkg_resources import parse_requirements
-
-with open('requirements.txt') as f:
-    requirements = [str(r) for r in parse_requirements(f)]
 
 setup(
     name='function',
-    install_requires=requirements,
     packages=['function'],
     package_dir={'function': '.'},
     package_data={'function': glob('**', recursive=True)},

sebs/cli.py

@@ -11,6 +11,7 @@
 import logging
 import functools
 import os
+import sys
 import traceback
 from typing import cast, List, Optional
 
@@ -455,33 +456,43 @@ def package(
     multiple=True,
     help="JSON configuration of deployed storage.",
 )
-@common_params
-@click.option(
-    "--cache",
-    default=os.path.join(os.path.curdir, "regression-cache"),
-    help="Location of experiments cache.",
-)
 @click.option(
-    "--output-dir",
-    default=os.path.join(os.path.curdir, "regression-output"),
-    help="Output directory for results.",
+    "--selected-architecture/--all-architectures",
+    type=bool,
+    default=False,
+    help="Skip non-selected CPU architectures.",
 )
-def regression(benchmark_input_size, benchmark_name, storage_configuration, **kwargs):
+@common_params
+def regression(
+    benchmark_input_size, benchmark_name, storage_configuration, selected_architecture, **kwargs
+):
     """Run regression test suite across benchmarks."""
+
     # for regression, deployment client is initialized locally
     # disable default initialization
+
+    from pathlib import Path
+
+    if Path(kwargs["cache"]) == Path("cache"):
+        kwargs["cache"] = os.path.join(os.path.curdir, "regression-cache")
+
     (config, output_dir, logging_filename, sebs_client, _) = parse_common_params(
         initialize_deployment=False,
         storage_configuration=storage_configuration,
         **kwargs,
     )
-    regression_suite(
+    architecture = config["experiments"]["architecture"] if selected_architecture else None
+    has_failures = regression_suite(
         sebs_client,
         config["experiments"],
         set((config["deployment"]["name"],)),
         config,
+        kwargs["resource_prefix"],
         benchmark_name,
+        architecture,
     )
+    # Exit with non-zero code if any tests failed
+    sys.exit(1 if has_failures else 0)
 
 
 @cli.group()