Repository navigation
Add downstream performance workflow #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Downstream Performance' | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| suites: | |
| description: 'Downstream suites to run' | |
| required: true | |
| default: 'all' | |
| type: choice | |
| options: | |
| - all | |
| - kevm | |
| - kontrol | |
| pr_number: | |
| description: 'PR number to comment on when running manually' | |
| required: false | |
| type: string | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| - labeled | |
| - unlabeled | |
| - ready_for_review | |
| permissions: | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| select: | |
| name: 'Select downstream perf suites' | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| should_run: ${{ steps.decide.outputs.should_run }} | |
| run_kevm: ${{ steps.decide.outputs.run_kevm }} | |
| run_kontrol: ${{ steps.decide.outputs.run_kontrol }} | |
| reason: ${{ steps.decide.outputs.reason }} | |
| target_pr: ${{ steps.decide.outputs.target_pr }} | |
| steps: | |
| - name: 'Check out code' | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| fetch-depth: 0 | |
| - name: 'Detect relevant changes' | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| filters: | | |
| downstream_perf: | |
| - 'booster/**' | |
| - 'kore/**' | |
| - 'kore-rpc-types/**' | |
| - 'flake.nix' | |
| - 'deps/**' | |
| - 'scripts/performance-tests-kevm.sh' | |
| - 'scripts/performance-tests-kontrol.sh' | |
| - 'scripts/compare.py' | |
| - '.github/workflows/downstream-perf.yml' | |
| - name: 'Decide suites' | |
| id: decide | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| REQUESTED_SUITES: ${{ inputs.suites || 'all' }} | |
| REQUESTED_PR: ${{ inputs.pr_number || '' }} | |
| PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || '' }} | |
| HAS_PERF_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'perf') }} | |
| RELEVANT_CHANGES: ${{ steps.changes.outputs.downstream_perf || 'false' }} | |
| run: | | |
| should_run=false | |
| run_kevm=false | |
| run_kontrol=false | |
| reason='not-requested' | |
| target_pr="$PULL_REQUEST_NUMBER" | |
| if [ -z "$target_pr" ] && [ -n "$REQUESTED_PR" ]; then | |
| target_pr="$REQUESTED_PR" | |
| fi | |
| if [ "$EVENT_NAME" = "workflow_dispatch" ]; then | |
| should_run=true | |
| reason='manual-dispatch' | |
| elif [ "$HAS_PERF_LABEL" = "true" ]; then | |
| should_run=true | |
| reason='perf-label' | |
| elif [ "$RELEVANT_CHANGES" = "true" ]; then | |
| should_run=true | |
| reason='relevant-path-change' | |
| fi | |
| if [ "$should_run" = "true" ]; then | |
| case "$REQUESTED_SUITES" in | |
| all) | |
| run_kevm=true | |
| run_kontrol=true | |
| ;; | |
| kevm) | |
| run_kevm=true | |
| ;; | |
| kontrol) | |
| run_kontrol=true | |
| ;; | |
| *) | |
| echo "Unknown suites selection: $REQUESTED_SUITES" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| fi | |
| { | |
| echo "should_run=$should_run" | |
| echo "run_kevm=$run_kevm" | |
| echo "run_kontrol=$run_kontrol" | |
| echo "reason=$reason" | |
| echo "target_pr=$target_pr" | |
| } >> "$GITHUB_OUTPUT" | |
| { | |
| echo "## Downstream perf selection" | |
| echo | |
| echo "- Event: $EVENT_NAME" | |
| echo "- Reason: $reason" | |
| echo "- KEVM: $run_kevm" | |
| echo "- Kontrol: $run_kontrol" | |
| echo "- Target PR: ${target_pr:-none}" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| kevm: | |
| name: 'KEVM downstream performance' | |
| needs: select | |
| if: ${{ needs.select.outputs.should_run == 'true' && needs.select.outputs.run_kevm == 'true' }} | |
| runs-on: [self-hosted, linux, normal] | |
| timeout-minutes: 180 | |
| steps: | |
| - name: 'Check out code' | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: 'Install Nix' | |
| uses: cachix/install-nix-action@v26 | |
| with: | |
| install_url: https://releases.nixos.org/nix/nix-2.30.1/install | |
| extra_nix_config: | | |
| access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} | |
| substituters = http://cache.nixos.org https://cache.iog.io | |
| trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= | |
| - name: 'Install Cachix' | |
| uses: cachix/cachix-action@v14 | |
| with: | |
| name: k-framework | |
| skipPush: true | |
| - name: 'Install uv' | |
| run: | | |
| python3 -m pip install --user uv | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: 'Run KEVM performance test' | |
| env: | |
| FEATURE_BRANCH_NAME: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| run: | | |
| scripts/performance-tests-kevm.sh | |
| - name: 'Collect KEVM results' | |
| env: | |
| FEATURE_BRANCH_NAME: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| REASON: ${{ needs.select.outputs.reason }} | |
| run: | | |
| set -euo pipefail | |
| branch_slug="${FEATURE_BRANCH_NAME//\//-}" | |
| mkdir -p downstream-perf/kevm | |
| compare_file="$(find scripts/logs -maxdepth 1 -type f -name "kevm-*-$branch_slug-compare" -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-)" | |
| feature_log="$(find scripts/logs -maxdepth 1 -type f -name "kevm-*-$branch_slug.log" -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-)" | |
| master_log="$(find scripts/logs -maxdepth 1 -type f -name 'kevm-*-master-*.log' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-)" | |
| summary_file="downstream-perf/kevm/summary.md" | |
| cp "$compare_file" downstream-perf/kevm/ | |
| cp "$feature_log" downstream-perf/kevm/ | |
| cp "$master_log" downstream-perf/kevm/ | |
| { | |
| echo "## KEVM downstream performance" | |
| echo | |
| echo "- Trigger: $REASON" | |
| echo "- Feature branch: $FEATURE_BRANCH_NAME" | |
| echo "- Compare file: $(basename "$compare_file")" | |
| echo | |
| if [ -s "$compare_file" ]; then | |
| cat "$compare_file" | |
| else | |
| echo "No significant performance deltas above the current compare thresholds." | |
| fi | |
| } | tee "$summary_file" >> "$GITHUB_STEP_SUMMARY" | |
| - name: 'Upload KEVM results' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: kevm-downstream-perf-${{ github.run_id }} | |
| path: downstream-perf/kevm | |
| if-no-files-found: error | |
| kontrol: | |
| name: 'Kontrol downstream performance' | |
| needs: select | |
| if: ${{ needs.select.outputs.should_run == 'true' && needs.select.outputs.run_kontrol == 'true' }} | |
| runs-on: [self-hosted, linux, normal] | |
| timeout-minutes: 180 | |
| steps: | |
| - name: 'Check out code' | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: 'Install Nix' | |
| uses: cachix/install-nix-action@v26 | |
| with: | |
| install_url: https://releases.nixos.org/nix/nix-2.30.1/install | |
| extra_nix_config: | | |
| access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} | |
| substituters = http://cache.nixos.org https://cache.iog.io | |
| trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= | |
| - name: 'Install Cachix' | |
| uses: cachix/cachix-action@v14 | |
| with: | |
| name: k-framework | |
| skipPush: true | |
| - name: 'Install uv' | |
| run: | | |
| python3 -m pip install --user uv | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: 'Run Kontrol performance test' | |
| env: | |
| FEATURE_BRANCH_NAME: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| run: | | |
| scripts/performance-tests-kontrol.sh | |
| - name: 'Collect Kontrol results' | |
| env: | |
| FEATURE_BRANCH_NAME: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| REASON: ${{ needs.select.outputs.reason }} | |
| run: | | |
| set -euo pipefail | |
| branch_slug="${FEATURE_BRANCH_NAME//\//-}" | |
| mkdir -p downstream-perf/kontrol | |
| compare_file="$(find scripts/logs -maxdepth 1 -type f -name "kontrol-*-$branch_slug-compare" -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-)" | |
| feature_log="$(find scripts/logs -maxdepth 1 -type f -name "kontrol-*-$branch_slug.log" -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-)" | |
| master_log="$(find scripts/logs -maxdepth 1 -type f -name 'kontrol-*-master-*.log' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-)" | |
| summary_file="downstream-perf/kontrol/summary.md" | |
| cp "$compare_file" downstream-perf/kontrol/ | |
| cp "$feature_log" downstream-perf/kontrol/ | |
| cp "$master_log" downstream-perf/kontrol/ | |
| { | |
| echo "## Kontrol downstream performance" | |
| echo | |
| echo "- Trigger: $REASON" | |
| echo "- Feature branch: $FEATURE_BRANCH_NAME" | |
| echo "- Compare file: $(basename "$compare_file")" | |
| echo | |
| if [ -s "$compare_file" ]; then | |
| cat "$compare_file" | |
| else | |
| echo "No significant performance deltas above the current compare thresholds." | |
| fi | |
| } | tee "$summary_file" >> "$GITHUB_STEP_SUMMARY" | |
| - name: 'Upload Kontrol results' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: kontrol-downstream-perf-${{ github.run_id }} | |
| path: downstream-perf/kontrol | |
| if-no-files-found: error | |
| report: | |
| name: 'Publish downstream perf PR comment' | |
| needs: | |
| - select | |
| - kevm | |
| - kontrol | |
| if: ${{ always() && needs.select.outputs.should_run == 'true' && needs.select.outputs.target_pr != '' && (needs.select.outputs.run_kevm != 'true' || needs.kevm.result != 'cancelled') && (needs.select.outputs.run_kontrol != 'true' || needs.kontrol.result != 'cancelled') }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: 'Download KEVM results' | |
| if: ${{ needs.select.outputs.run_kevm == 'true' && needs.kevm.result == 'success' }} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: kevm-downstream-perf-${{ github.run_id }} | |
| path: downstream-perf/kevm | |
| - name: 'Download Kontrol results' | |
| if: ${{ needs.select.outputs.run_kontrol == 'true' && needs.kontrol.result == 'success' }} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: kontrol-downstream-perf-${{ github.run_id }} | |
| path: downstream-perf/kontrol | |
| - name: 'Build PR comment body' | |
| env: | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| RUN_KEVM: ${{ needs.select.outputs.run_kevm }} | |
| RUN_KONTROL: ${{ needs.select.outputs.run_kontrol }} | |
| KEVM_RESULT: ${{ needs.kevm.result }} | |
| KONTROL_RESULT: ${{ needs.kontrol.result }} | |
| TRIGGER_REASON: ${{ needs.select.outputs.reason }} | |
| TARGET_PR: ${{ needs.select.outputs.target_pr }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p downstream-perf/report | |
| comment_file="downstream-perf/report/comment.md" | |
| { | |
| echo '<!-- downstream-perf-report -->' | |
| echo '## Downstream Performance' | |
| echo | |
| echo "- Trigger: $TRIGGER_REASON" | |
| echo "- Target PR: #$TARGET_PR" | |
| echo "- Workflow run: $RUN_URL" | |
| echo | |
| if [ "$RUN_KEVM" = "true" ]; then | |
| echo "### KEVM" | |
| echo | |
| echo "- Job result: $KEVM_RESULT" | |
| echo | |
| if [ -f downstream-perf/kevm/summary.md ]; then | |
| sed '1,/^$/d' downstream-perf/kevm/summary.md | |
| else | |
| echo 'No KEVM summary artifact was produced.' | |
| fi | |
| echo | |
| fi | |
| if [ "$RUN_KONTROL" = "true" ]; then | |
| echo "### Kontrol" | |
| echo | |
| echo "- Job result: $KONTROL_RESULT" | |
| echo | |
| if [ -f downstream-perf/kontrol/summary.md ]; then | |
| sed '1,/^$/d' downstream-perf/kontrol/summary.md | |
| else | |
| echo 'No Kontrol summary artifact was produced.' | |
| fi | |
| echo | |
| fi | |
| } > "$comment_file" | |
| - name: 'Upsert PR comment' | |
| uses: actions/github-script@v7 | |
| env: | |
| COMMENT_PATH: downstream-perf/report/comment.md | |
| TARGET_PR: ${{ needs.select.outputs.target_pr }} | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const marker = '<!-- downstream-perf-report -->'; | |
| const body = fs.readFileSync(process.env.COMMENT_PATH, 'utf8'); | |
| const issue_number = Number(process.env.TARGET_PR); | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner, | |
| repo, | |
| issue_number, | |
| per_page: 100, | |
| }); | |
| const existing = comments.find((comment) => comment.body && comment.body.includes(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner, | |
| repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number, | |
| body, | |
| }); | |
| } |