Run regression on CI

.github/workflows/_regression-job.yml

@@ -0,0 +1,126 @@
+name: Regression Job (Reusable)
+
+on:
+  workflow_call:
+    inputs:
+      platform:
+        required: true
+        type: string
+      language:
+        required: true
+        type: string
+      version:
+        required: true
+        type: string
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    env:
+      RESOURCE_PREFIX: sebs-ci
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Restore SeBS cache
+        uses: actions/cache/restore@v4
+        with:
+          path: regression-cache/
+          key: sebs-cache-${{ github.ref_name }}-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          restore-keys: |
+            sebs-cache-${{ github.ref_name }}-
+
+      - name: Setup GCP credentials
+        if: inputs.platform == 'gcp'
+        run: |
+          echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" > /tmp/gcp-credentials.json
+          echo "GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json" >> $GITHUB_ENV
+
+      - name: Setup Azure credentials
+        if: inputs.platform == 'azure'
+        run: |
+          echo "AZURE_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> $GITHUB_ENV
+          echo "AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}" >> $GITHUB_ENV
+          echo "AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}" >> $GITHUB_ENV
+          echo "AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}" >> $GITHUB_ENV
+
+      - name: Setup AWS credentials
+        if: inputs.platform == 'aws'
+        run: |
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> $GITHUB_ENV
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
+          echo "AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION || 'us-east-1' }}" >> $GITHUB_ENV
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install SeBS
+        run: uv pip install --system .
+
+      - name: Run regression tests
+        timeout-minutes: 5
+        run: |
+          sebs benchmark regression test \
+            --config configs/example.json \
+            --deployment ${{ inputs.platform }} \
+            --language ${{ inputs.language }} \
+            --language-version ${{ inputs.version }} \
+            --architecture x64 --selected-architecture \
+            --resource-prefix sebs-ci
+
+      - name: Generate test summary
+        if: always()
+        run: |
+          echo "Regression Test Summary" > test-summary.txt
+          echo "======================" >> test-summary.txt
+          echo "Platform: ${{ inputs.platform }}" >> test-summary.txt
+          echo "Language: ${{ inputs.language }}" >> test-summary.txt
+          echo "Version: ${{ inputs.version }}" >> test-summary.txt
+          echo "" >> test-summary.txt
+          if ls regression_*.json 1> /dev/null 2>&1; then
+            ls -1 regression_*.json | wc -l | xargs echo "Benchmarks tested:" >> test-summary.txt
+            echo "" >> test-summary.txt
+            echo "Results saved to artifacts/results/" >> test-summary.txt
+          else
+            echo "No benchmark results found" >> test-summary.txt
+          fi
+
+      - name: Upload test summary
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-summary-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          path: test-summary.txt
+
+      - name: Collect and upload regression results
+        if: always()
+        run: |
+          mkdir -p results
+          if ls regression_*.json 1> /dev/null 2>&1; then
+            mv regression_*.json results/ || true
+          fi
+
+      - name: Upload regression results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: results-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          path: results/
+          if-no-files-found: ignore
+
+      - name: Upload cache snapshot
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cache-snapshot-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          path: cache/
+          if-no-files-found: ignore
+
+      - name: Save SeBS cache
+        if: success()
+        uses: actions/cache/save@v4
+        with:
+          path: regression-cache/
+          key: sebs-cache-${{ github.ref_name }}-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}

.github/workflows/lint.yml

@@ -0,0 +1,55 @@
+name: Lint
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  linting:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install system dependencies
+        run: sudo apt update && sudo apt install -y libcurl4-openssl-dev
+
+      - name: Cache uv dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: uv-${{ runner.os }}-${{ hashFiles('requirements.txt', 'pyproject.toml') }}
+          restore-keys: |
+            uv-${{ runner.os }}-
+
+      - name: Install SeBS with dev dependencies
+        run: uv sync --extra dev
+
+      - name: Python code formatting with black
+        run: uv run black sebs --check --config .black.toml
+
+      - name: Python code lint with flake8
+        run: uv run flake8 sebs --config=.flake8.cfg --tee --output-file flake-reports
+
+      - name: Python static code verification with mypy
+        run: uv run mypy sebs --config-file=.mypy.ini
+
+      - name: Check for Python documentation coverage
+        run: uv run interrogate -v --fail-under 100 sebs
+
+      - name: Upload flake8 reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: flake-reports
+          path: flake-reports

.github/workflows/regression.yml

@@ -0,0 +1,25 @@
+name: Regression Tests
+
+on:
+  push:
+    branches:
+      - master
+      - 'feature/**'
+  workflow_dispatch:
+
+jobs:
+  regression:
+    strategy:
+      matrix:
+        include:
+          - platform: aws
+            language: python
+            version: "3.11"
+      fail-fast: false
+
+    uses: ./.github/workflows/_regression-job.yml
+    with:
+      platform: ${{ matrix.platform }}
+      language: ${{ matrix.language }}
+      version: ${{ matrix.version }}
+    secrets: inherit

benchmarks/wrappers/aws/python/setup.py

@@ -1,14 +1,9 @@
 # Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved.
 from distutils.core import setup
 from glob import glob
-from pkg_resources import parse_requirements
-
-with open('requirements.txt') as f:
-    requirements = [str(r) for r in parse_requirements(f)]
 
 setup(
     name='function',
-    install_requires=requirements,
     packages=['function'],
     package_dir={'function': '.'},
     package_data={'function': glob('**', recursive=True)},

sebs/azure/cli.py

@@ -129,6 +129,8 @@ def execute(self, cmd: str) -> bytes:
             RuntimeError: If command execution fails.
         """
         exit_code, out = self.docker_instance.exec_run(cmd, user="docker_user")
+        # exec_run without stream=True always returns bytes
+        assert isinstance(out, bytes)
         if exit_code != 0:
             raise RuntimeError(
                 "Command {} failed at Azure CLI docker!\n Output {}".format(

sebs/benchmark.py

@@ -1338,14 +1338,14 @@ def ensure_image(name: str) -> None:
                         with open(tar_archive, "rb") as data:
                             container.put_archive("/mnt/function", data.read())
                         # do the build step
-                        exit_code, stdout = container.exec_run(
+                        exit_code, stdout = container.exec_run(  # type: ignore[assignment]
                             cmd="/bin/bash /sebs/installer.sh",
                             user="docker_user",
                             stdout=True,
                             stderr=True,
                         )
                         # copy updated code with package
-                        data, stat = container.get_archive("/mnt/function")
+                        data, stat = container.get_archive("/mnt/function")  # type: ignore[assignment]
                         with open(tar_archive, "wb") as output_filef:
                             for chunk in data:
                                 output_filef.write(chunk)