benchmarks.yml

name: Performance Benchmarks

# Benchmarks only run on main to track performance over time.
# They compare each commit against its parent.

on:
  push:
    branches:
      - main
    paths:
      - "crates/**"
      - "Cargo.toml"
      - "Cargo.lock"
      - "rust-toolchain.toml"
      - ".github/workflows/benchmarks.yml"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  RUSTFLAGS: "-Dwarnings"
  BENCHMARK_SIMPLE_BATCH_SIZE: 5000
  BENCHMARK_FULL_BATCH_SIZE: 1000
  REGRESSION_THRESHOLD: -20

jobs:
  benchmarks:
    name: Performance Benchmarks
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - name: Cache Cargo dependencies
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: benchmark-${{ hashFiles('**/Cargo.lock') }}

      - name: Build benchmark tool
        run: cargo build --package synddb-benchmark --release

      - name: Create benchmark helper functions
        run: |
          cat > /tmp/benchmark-helpers.sh << 'EOF'
          # Extract peak and sustained rates from benchmark log file
          extract_rates() {
            local log_file=$1
            local peak=$(grep "Peak achieved rate:" "$log_file" | tail -1 | sed -E 's/.*Peak achieved rate: ([0-9]+).*/\1/')
            local sustained=$(grep "Sustained stable rate:" "$log_file" | tail -1 | sed -E 's/.*Sustained stable rate: ([0-9]+).*/\1/')
            echo "$peak $sustained"
          }

          # Run a benchmark and handle errors
          run_benchmark() {
            local db_name=$1
            local log_file=$2
            local mode_label=$3
            local mode_flags=$4
            local batch_size=$5

            echo "Starting $mode_label benchmark..."
            cargo run --package synddb-benchmark --release -- init --db "$db_name"

            if ! cargo run --package synddb-benchmark --release -- run --db "$db_name" --clean --rate 0 $mode_flags --batch-size $batch_size 2>&1 | tee "$log_file"; then
              echo "::error::$mode_label benchmark failed!"
              cat "$log_file"
              exit 1
            fi
            echo "::notice::$mode_label benchmark completed"
          }

          # Extract rates and output to GitHub Actions
          output_rates() {
            local log_file=$1
            local output_prefix=$2
            local display_label=$3

            read peak sustained <<< $(extract_rates "$log_file")
            echo "${output_prefix}_peak=$peak" >> $GITHUB_OUTPUT
            echo "${output_prefix}_sustained=$sustained" >> $GITHUB_OUTPUT
            echo "$display_label - Peak: $peak ops/sec, Sustained: $sustained ops/sec"
          }
          EOF
          chmod +x /tmp/benchmark-helpers.sh

      - name: Get baseline commit SHA
        id: baseline-sha
        run: |
          BASELINE_SHA=$(git rev-parse HEAD~1)
          echo "Baseline: previous commit ($BASELINE_SHA)"
          echo "sha=$BASELINE_SHA" >> $GITHUB_OUTPUT

      - name: Restore cached baseline benchmarks
        id: cache-baseline
        uses: actions/cache@v5
        with:
          path: |
            bench-baseline-simple.log
            bench-baseline-full.log
          key: benchmark-baseline-${{ steps.baseline-sha.outputs.sha }}

      - name: Run benchmarks on current commit
        id: current-bench
        run: |
          source /tmp/benchmark-helpers.sh

          run_benchmark bench-current-simple.db bench-current-simple.log "Simple" "--simple" ${{ env.BENCHMARK_SIMPLE_BATCH_SIZE }}
          output_rates bench-current-simple.log simple "Current simple"

          run_benchmark bench-current-full.db bench-current-full.log "Full" "" ${{ env.BENCHMARK_FULL_BATCH_SIZE }}
          output_rates bench-current-full.log full "Current full"

      - name: Run baseline benchmarks
        if: steps.cache-baseline.outputs.cache-hit != 'true'
        run: |
          source /tmp/benchmark-helpers.sh

          BASELINE_SHA=${{ steps.baseline-sha.outputs.sha }}
          git fetch origin
          git checkout $BASELINE_SHA
          cargo build --package synddb-benchmark --release

          run_benchmark bench-baseline-simple.db bench-baseline-simple.log "Baseline simple" "--simple" ${{ env.BENCHMARK_SIMPLE_BATCH_SIZE }}
          run_benchmark bench-baseline-full.db bench-baseline-full.log "Baseline full" "" ${{ env.BENCHMARK_FULL_BATCH_SIZE }}

          git checkout -

      - name: Extract baseline rates
        id: baseline-rates
        run: |
          source /tmp/benchmark-helpers.sh

          if [ "${{ steps.cache-baseline.outputs.cache-hit }}" == "true" ]; then
            echo "Using cached baseline"
          else
            echo "Using fresh baseline"
          fi

          output_rates bench-baseline-simple.log simple "Baseline simple"
          output_rates bench-baseline-full.log full "Baseline full"

      - name: Compare performance and check for regressions
        run: |
          BASELINE_SIMPLE_PEAK=${{ steps.baseline-rates.outputs.simple_peak }}
          BASELINE_SIMPLE_SUSTAINED=${{ steps.baseline-rates.outputs.simple_sustained }}
          BASELINE_FULL_PEAK=${{ steps.baseline-rates.outputs.full_peak }}
          BASELINE_FULL_SUSTAINED=${{ steps.baseline-rates.outputs.full_sustained }}

          CURRENT_SIMPLE_PEAK=${{ steps.current-bench.outputs.simple_peak }}
          CURRENT_SIMPLE_SUSTAINED=${{ steps.current-bench.outputs.simple_sustained }}
          CURRENT_FULL_PEAK=${{ steps.current-bench.outputs.full_peak }}
          CURRENT_FULL_SUSTAINED=${{ steps.current-bench.outputs.full_sustained }}

          echo "=== Performance Comparison ==="
          echo "Simple mode:"
          echo "  Previous commit - Peak: $BASELINE_SIMPLE_PEAK ops/sec, Sustained: $BASELINE_SIMPLE_SUSTAINED ops/sec"
          echo "  Current commit - Peak: $CURRENT_SIMPLE_PEAK ops/sec, Sustained: $CURRENT_SIMPLE_SUSTAINED ops/sec"
          echo "Full mode:"
          echo "  Previous commit - Peak: $BASELINE_FULL_PEAK ops/sec, Sustained: $BASELINE_FULL_SUSTAINED ops/sec"
          echo "  Current commit - Peak: $CURRENT_FULL_PEAK ops/sec, Sustained: $CURRENT_FULL_SUSTAINED ops/sec"

          CHANGE_SIMPLE=$(echo "scale=2; (($CURRENT_SIMPLE_SUSTAINED - $BASELINE_SIMPLE_SUSTAINED) / $BASELINE_SIMPLE_SUSTAINED) * 100" | bc)
          CHANGE_FULL=$(echo "scale=2; (($CURRENT_FULL_SUSTAINED - $BASELINE_FULL_SUSTAINED) / $BASELINE_FULL_SUSTAINED) * 100" | bc)
          echo "Simple mode sustained change: $CHANGE_SIMPLE%"
          echo "Full mode sustained change: $CHANGE_FULL%"

          # Write summary
          echo "## Benchmark Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Simple Mode (Insert-Only)" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Metric | Previous | Current | Change |" >> $GITHUB_STEP_SUMMARY
          echo "|--------|----------|---------|--------|" >> $GITHUB_STEP_SUMMARY
          echo "| Peak achieved | $BASELINE_SIMPLE_PEAK ops/sec | $CURRENT_SIMPLE_PEAK ops/sec | - |" >> $GITHUB_STEP_SUMMARY
          echo "| Sustained stable | $BASELINE_SIMPLE_SUSTAINED ops/sec | $CURRENT_SIMPLE_SUSTAINED ops/sec | $CHANGE_SIMPLE% |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Full Mode (Mixed Operations)" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Metric | Previous | Current | Change |" >> $GITHUB_STEP_SUMMARY
          echo "|--------|----------|---------|--------|" >> $GITHUB_STEP_SUMMARY
          echo "| Peak achieved | $BASELINE_FULL_PEAK ops/sec | $CURRENT_FULL_PEAK ops/sec | - |" >> $GITHUB_STEP_SUMMARY
          echo "| Sustained stable | $BASELINE_FULL_SUSTAINED ops/sec | $CURRENT_FULL_SUSTAINED ops/sec | $CHANGE_FULL% |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Check for regression
          THRESHOLD=${{ env.REGRESSION_THRESHOLD }}
          IS_REGRESSION_SIMPLE=$(echo "$CHANGE_SIMPLE < $THRESHOLD" | bc)
          IS_REGRESSION_FULL=$(echo "$CHANGE_FULL < $THRESHOLD" | bc)

          if [ "$IS_REGRESSION_SIMPLE" -eq 1 ] || [ "$IS_REGRESSION_FULL" -eq 1 ]; then
            echo "::error::PERFORMANCE REGRESSION DETECTED!"
            if [ "$IS_REGRESSION_SIMPLE" -eq 1 ]; then
              echo "Simple mode: ${CHANGE_SIMPLE}% (exceeds ${THRESHOLD}% threshold)"
            fi
            if [ "$IS_REGRESSION_FULL" -eq 1 ]; then
              echo "Full mode: ${CHANGE_FULL}% (exceeds ${THRESHOLD}% threshold)"
            fi
            echo "### Performance Regression Detected" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            if [ "$IS_REGRESSION_SIMPLE" -eq 1 ]; then
              echo "**Simple mode** sustained rate decreased by **${CHANGE_SIMPLE}%** (threshold: ${THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY
            fi
            if [ "$IS_REGRESSION_FULL" -eq 1 ]; then
              echo "**Full mode** sustained rate decreased by **${CHANGE_FULL}%** (threshold: ${THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY
            fi
            exit 1
          elif [ $(echo "$CHANGE_SIMPLE < 0" | bc) -eq 1 ] || [ $(echo "$CHANGE_FULL < 0" | bc) -eq 1 ]; then
            echo "Minor performance decrease detected"
            echo "This is within acceptable range (< ${THRESHOLD}%)."
            echo "### Minor Performance Decrease" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "Changes are within acceptable range." >> $GITHUB_STEP_SUMMARY
          else
            echo "Performance improved or neutral!"
            echo "### Performance Improvement" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "Both benchmarks show improvement or neutral performance!" >> $GITHUB_STEP_SUMMARY
          fi

      - name: Upload benchmark results
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: benchmark-results
          path: |
            bench-current-*.log
            bench-current-*.db
            bench-baseline-*.log
            bench-baseline-*.db