Repository navigation
feat: production infrastructure, TEE attestation, example apps, and comprehensive tooling #532
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Benchmarks | |
| # COST OPTIMIZATION: | |
| # Benchmarks are expensive (~30 min). On PRs, they only run when the | |
| # 'run-benchmarks' label is added. On main pushes, they always run. | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review, labeled] | |
| paths: | |
| - "crates/**" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "rust-toolchain.toml" | |
| - ".github/workflows/benchmarks.yml" | |
| # Run on main pushes | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| # Make sure CI fails on all warnings, including Clippy lints | |
| RUSTFLAGS: "-Dwarnings" | |
| # Benchmark configuration | |
| BENCHMARK_SIMPLE_BATCH_SIZE: 5000 | |
| BENCHMARK_FULL_BATCH_SIZE: 1000 | |
| REGRESSION_THRESHOLD: -20 | |
| jobs: | |
| benchmarks: | |
| # On PRs: only run if 'run-benchmarks' label is present | |
| # On main pushes: always run | |
| if: | | |
| github.event_name == 'push' || | |
| (github.event_name == 'pull_request' && | |
| github.event.pull_request.draft == false && | |
| contains(github.event.pull_request.labels.*.name, 'run-benchmarks')) | |
| name: Performance Benchmarks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache Cargo dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: benchmark-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Build benchmark tool | |
| run: cargo build --package synddb-benchmark --release | |
| - name: Create benchmark helper functions | |
| run: | | |
| cat > /tmp/benchmark-helpers.sh << 'EOF' | |
| # Extract peak and sustained rates from benchmark log file | |
| extract_rates() { | |
| local log_file=$1 | |
| local peak=$(grep "Peak achieved rate:" "$log_file" | tail -1 | sed -E 's/.*Peak achieved rate: ([0-9]+).*/\1/') | |
| local sustained=$(grep "Sustained stable rate:" "$log_file" | tail -1 | sed -E 's/.*Sustained stable rate: ([0-9]+).*/\1/') | |
| echo "$peak $sustained" | |
| } | |
| # Run a benchmark and handle errors | |
| run_benchmark() { | |
| local db_name=$1 | |
| local log_file=$2 | |
| local mode_label=$3 | |
| local mode_flags=$4 | |
| local batch_size=$5 | |
| echo "Starting $mode_label benchmark..." | |
| cargo run --package synddb-benchmark --release -- init --db "$db_name" | |
| if ! cargo run --package synddb-benchmark --release -- run --db "$db_name" --clean --rate 0 $mode_flags --batch-size $batch_size 2>&1 | tee "$log_file"; then | |
| echo "::error::$mode_label benchmark failed!" | |
| cat "$log_file" | |
| exit 1 | |
| fi | |
| echo "::notice::$mode_label benchmark completed" | |
| } | |
| # Extract rates and output to GitHub Actions | |
| output_rates() { | |
| local log_file=$1 | |
| local output_prefix=$2 | |
| local display_label=$3 | |
| read peak sustained <<< $(extract_rates "$log_file") | |
| echo "${output_prefix}_peak=$peak" >> $GITHUB_OUTPUT | |
| echo "${output_prefix}_sustained=$sustained" >> $GITHUB_OUTPUT | |
| echo "$display_label - Peak: $peak ops/sec, Sustained: $sustained ops/sec" | |
| } | |
| EOF | |
| chmod +x /tmp/benchmark-helpers.sh | |
| - name: Get baseline commit SHA | |
| id: baseline-sha | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| BASELINE_SHA=$(git rev-parse origin/main) | |
| echo "Baseline: main branch ($BASELINE_SHA)" | |
| else | |
| BASELINE_SHA=$(git rev-parse HEAD~1) | |
| echo "Baseline: previous commit ($BASELINE_SHA)" | |
| fi | |
| echo "sha=$BASELINE_SHA" >> $GITHUB_OUTPUT | |
| - name: Restore cached baseline benchmarks | |
| id: cache-baseline | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| bench-baseline-simple.log | |
| bench-baseline-full.log | |
| key: benchmark-baseline-${{ steps.baseline-sha.outputs.sha }} | |
| - name: Run benchmarks on current commit | |
| id: current-bench | |
| run: | | |
| source /tmp/benchmark-helpers.sh | |
| run_benchmark bench-current-simple.db bench-current-simple.log "Simple" "--simple" ${{ env.BENCHMARK_SIMPLE_BATCH_SIZE }} | |
| output_rates bench-current-simple.log simple "Current simple" | |
| run_benchmark bench-current-full.db bench-current-full.log "Full" "" ${{ env.BENCHMARK_FULL_BATCH_SIZE }} | |
| output_rates bench-current-full.log full "Current full" | |
| - name: Run baseline benchmarks | |
| if: steps.cache-baseline.outputs.cache-hit != 'true' | |
| run: | | |
| source /tmp/benchmark-helpers.sh | |
| BASELINE_SHA=${{ steps.baseline-sha.outputs.sha }} | |
| git fetch origin | |
| git checkout $BASELINE_SHA | |
| cargo build --package synddb-benchmark --release | |
| run_benchmark bench-baseline-simple.db bench-baseline-simple.log "Baseline simple" "--simple" ${{ env.BENCHMARK_SIMPLE_BATCH_SIZE }} | |
| run_benchmark bench-baseline-full.db bench-baseline-full.log "Baseline full" "" ${{ env.BENCHMARK_FULL_BATCH_SIZE }} | |
| git checkout - | |
| - name: Extract baseline rates | |
| id: baseline-rates | |
| run: | | |
| source /tmp/benchmark-helpers.sh | |
| if [ "${{ steps.cache-baseline.outputs.cache-hit }}" == "true" ]; then | |
| echo "Using cached baseline" | |
| else | |
| echo "Using fresh baseline" | |
| fi | |
| output_rates bench-baseline-simple.log simple "Baseline simple" | |
| output_rates bench-baseline-full.log full "Baseline full" | |
| - name: Compare performance and check for regressions | |
| run: | | |
| BASELINE_SIMPLE_PEAK=${{ steps.baseline-rates.outputs.simple_peak }} | |
| BASELINE_SIMPLE_SUSTAINED=${{ steps.baseline-rates.outputs.simple_sustained }} | |
| BASELINE_FULL_PEAK=${{ steps.baseline-rates.outputs.full_peak }} | |
| BASELINE_FULL_SUSTAINED=${{ steps.baseline-rates.outputs.full_sustained }} | |
| CURRENT_SIMPLE_PEAK=${{ steps.current-bench.outputs.simple_peak }} | |
| CURRENT_SIMPLE_SUSTAINED=${{ steps.current-bench.outputs.simple_sustained }} | |
| CURRENT_FULL_PEAK=${{ steps.current-bench.outputs.full_peak }} | |
| CURRENT_FULL_SUSTAINED=${{ steps.current-bench.outputs.full_sustained }} | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| BASELINE_LABEL="Baseline (main)" | |
| CURRENT_LABEL="Current (PR)" | |
| else | |
| BASELINE_LABEL="Previous commit" | |
| CURRENT_LABEL="Current commit" | |
| fi | |
| echo "=== Performance Comparison ===" | |
| echo "Simple mode:" | |
| echo " $BASELINE_LABEL - Peak: $BASELINE_SIMPLE_PEAK ops/sec, Sustained: $BASELINE_SIMPLE_SUSTAINED ops/sec" | |
| echo " $CURRENT_LABEL - Peak: $CURRENT_SIMPLE_PEAK ops/sec, Sustained: $CURRENT_SIMPLE_SUSTAINED ops/sec" | |
| echo "Full mode:" | |
| echo " $BASELINE_LABEL - Peak: $BASELINE_FULL_PEAK ops/sec, Sustained: $BASELINE_FULL_SUSTAINED ops/sec" | |
| echo " $CURRENT_LABEL - Peak: $CURRENT_FULL_PEAK ops/sec, Sustained: $CURRENT_FULL_SUSTAINED ops/sec" | |
| CHANGE_SIMPLE=$(echo "scale=2; (($CURRENT_SIMPLE_SUSTAINED - $BASELINE_SIMPLE_SUSTAINED) / $BASELINE_SIMPLE_SUSTAINED) * 100" | bc) | |
| CHANGE_FULL=$(echo "scale=2; (($CURRENT_FULL_SUSTAINED - $BASELINE_FULL_SUSTAINED) / $BASELINE_FULL_SUSTAINED) * 100" | bc) | |
| echo "Simple mode sustained change: $CHANGE_SIMPLE%" | |
| echo "Full mode sustained change: $CHANGE_FULL%" | |
| # Write summary | |
| echo "## Benchmark Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Simple Mode (Insert-Only)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Metric | $BASELINE_LABEL | $CURRENT_LABEL | Change |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|------------------|----------------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Peak achieved | $BASELINE_SIMPLE_PEAK ops/sec | $CURRENT_SIMPLE_PEAK ops/sec | - |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Sustained stable | $BASELINE_SIMPLE_SUSTAINED ops/sec | $CURRENT_SIMPLE_SUSTAINED ops/sec | $CHANGE_SIMPLE% |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Full Mode (Mixed Operations)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Metric | $BASELINE_LABEL | $CURRENT_LABEL | Change |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|------------------|----------------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Peak achieved | $BASELINE_FULL_PEAK ops/sec | $CURRENT_FULL_PEAK ops/sec | - |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Sustained stable | $BASELINE_FULL_SUSTAINED ops/sec | $CURRENT_FULL_SUSTAINED ops/sec | $CHANGE_FULL% |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Check for regression | |
| THRESHOLD=${{ env.REGRESSION_THRESHOLD }} | |
| IS_REGRESSION_SIMPLE=$(echo "$CHANGE_SIMPLE < $THRESHOLD" | bc) | |
| IS_REGRESSION_FULL=$(echo "$CHANGE_FULL < $THRESHOLD" | bc) | |
| if [ "$IS_REGRESSION_SIMPLE" -eq 1 ] || [ "$IS_REGRESSION_FULL" -eq 1 ]; then | |
| echo "::error::PERFORMANCE REGRESSION DETECTED!" | |
| if [ "$IS_REGRESSION_SIMPLE" -eq 1 ]; then | |
| echo "Simple mode: ${CHANGE_SIMPLE}% (exceeds ${THRESHOLD}% threshold)" | |
| fi | |
| if [ "$IS_REGRESSION_FULL" -eq 1 ]; then | |
| echo "Full mode: ${CHANGE_FULL}% (exceeds ${THRESHOLD}% threshold)" | |
| fi | |
| echo "### Performance Regression Detected" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ "$IS_REGRESSION_SIMPLE" -eq 1 ]; then | |
| echo "**Simple mode** sustained rate decreased by **${CHANGE_SIMPLE}%** (threshold: ${THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ "$IS_REGRESSION_FULL" -eq 1 ]; then | |
| echo "**Full mode** sustained rate decreased by **${CHANGE_FULL}%** (threshold: ${THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| exit 1 | |
| elif [ $(echo "$CHANGE_SIMPLE < 0" | bc) -eq 1 ] || [ $(echo "$CHANGE_FULL < 0" | bc) -eq 1 ]; then | |
| echo "Minor performance decrease detected" | |
| echo "This is within acceptable range (< ${THRESHOLD}%)." | |
| echo "### Minor Performance Decrease" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Changes are within acceptable range." >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "Performance improved or neutral!" | |
| echo "### Performance Improvement" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Both benchmarks show improvement or neutral performance!" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| bench-current-*.log | |
| bench-current-*.db | |
| bench-baseline-*.log | |
| bench-baseline-*.db | |
| # Provide instructions when benchmarks are skipped on PRs | |
| benchmark-skip-notice: | |
| if: | | |
| github.event_name == 'pull_request' && | |
| !contains(github.event.pull_request.labels.*.name, 'run-benchmarks') | |
| name: Benchmarks Skipped | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Notice | |
| run: | | |
| echo "## Benchmarks Skipped" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Performance benchmarks are opt-in for PRs to save CI resources." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "To run benchmarks, add the \`run-benchmarks\` label to this PR." >> $GITHUB_STEP_SUMMARY |