Repository navigation
chore(deps): bump chrono from 0.4.42 to 0.4.43 (#77) #555
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Benchmarks | |
| # Benchmarks only run on main to track performance over time. | |
| # They compare each commit against its parent. | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "crates/**" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "rust-toolchain.toml" | |
| - ".github/workflows/benchmarks.yml" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUSTFLAGS: "-Dwarnings" | |
| BENCHMARK_SIMPLE_BATCH_SIZE: 5000 | |
| BENCHMARK_FULL_BATCH_SIZE: 1000 | |
| REGRESSION_THRESHOLD: -20 | |
| jobs: | |
| benchmarks: | |
| name: Performance Benchmarks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache Cargo dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: benchmark-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Build benchmark tool | |
| run: cargo build --package synddb-benchmark --release | |
| - name: Create benchmark helper functions | |
| run: | | |
| cat > /tmp/benchmark-helpers.sh << 'EOF' | |
| # Extract peak and sustained rates from benchmark log file | |
| extract_rates() { | |
| local log_file=$1 | |
| local peak=$(grep "Peak achieved rate:" "$log_file" | tail -1 | sed -E 's/.*Peak achieved rate: ([0-9]+).*/\1/') | |
| local sustained=$(grep "Sustained stable rate:" "$log_file" | tail -1 | sed -E 's/.*Sustained stable rate: ([0-9]+).*/\1/') | |
| echo "$peak $sustained" | |
| } | |
| # Run a benchmark and handle errors | |
| run_benchmark() { | |
| local db_name=$1 | |
| local log_file=$2 | |
| local mode_label=$3 | |
| local mode_flags=$4 | |
| local batch_size=$5 | |
| echo "Starting $mode_label benchmark..." | |
| cargo run --package synddb-benchmark --release -- init --db "$db_name" | |
| if ! cargo run --package synddb-benchmark --release -- run --db "$db_name" --clean --rate 0 $mode_flags --batch-size $batch_size 2>&1 | tee "$log_file"; then | |
| echo "::error::$mode_label benchmark failed!" | |
| cat "$log_file" | |
| exit 1 | |
| fi | |
| echo "::notice::$mode_label benchmark completed" | |
| } | |
| # Extract rates and output to GitHub Actions | |
| output_rates() { | |
| local log_file=$1 | |
| local output_prefix=$2 | |
| local display_label=$3 | |
| read peak sustained <<< $(extract_rates "$log_file") | |
| echo "${output_prefix}_peak=$peak" >> $GITHUB_OUTPUT | |
| echo "${output_prefix}_sustained=$sustained" >> $GITHUB_OUTPUT | |
| echo "$display_label - Peak: $peak ops/sec, Sustained: $sustained ops/sec" | |
| } | |
| EOF | |
| chmod +x /tmp/benchmark-helpers.sh | |
| - name: Get baseline commit SHA | |
| id: baseline-sha | |
| run: | | |
| BASELINE_SHA=$(git rev-parse HEAD~1) | |
| echo "Baseline: previous commit ($BASELINE_SHA)" | |
| echo "sha=$BASELINE_SHA" >> $GITHUB_OUTPUT | |
| - name: Restore cached baseline benchmarks | |
| id: cache-baseline | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| bench-baseline-simple.log | |
| bench-baseline-full.log | |
| key: benchmark-baseline-${{ steps.baseline-sha.outputs.sha }} | |
| - name: Run benchmarks on current commit | |
| id: current-bench | |
| run: | | |
| source /tmp/benchmark-helpers.sh | |
| run_benchmark bench-current-simple.db bench-current-simple.log "Simple" "--simple" ${{ env.BENCHMARK_SIMPLE_BATCH_SIZE }} | |
| output_rates bench-current-simple.log simple "Current simple" | |
| run_benchmark bench-current-full.db bench-current-full.log "Full" "" ${{ env.BENCHMARK_FULL_BATCH_SIZE }} | |
| output_rates bench-current-full.log full "Current full" | |
| - name: Run baseline benchmarks | |
| if: steps.cache-baseline.outputs.cache-hit != 'true' | |
| run: | | |
| source /tmp/benchmark-helpers.sh | |
| BASELINE_SHA=${{ steps.baseline-sha.outputs.sha }} | |
| git fetch origin | |
| git checkout $BASELINE_SHA | |
| cargo build --package synddb-benchmark --release | |
| run_benchmark bench-baseline-simple.db bench-baseline-simple.log "Baseline simple" "--simple" ${{ env.BENCHMARK_SIMPLE_BATCH_SIZE }} | |
| run_benchmark bench-baseline-full.db bench-baseline-full.log "Baseline full" "" ${{ env.BENCHMARK_FULL_BATCH_SIZE }} | |
| git checkout - | |
| - name: Extract baseline rates | |
| id: baseline-rates | |
| run: | | |
| source /tmp/benchmark-helpers.sh | |
| if [ "${{ steps.cache-baseline.outputs.cache-hit }}" == "true" ]; then | |
| echo "Using cached baseline" | |
| else | |
| echo "Using fresh baseline" | |
| fi | |
| output_rates bench-baseline-simple.log simple "Baseline simple" | |
| output_rates bench-baseline-full.log full "Baseline full" | |
| - name: Compare performance and check for regressions | |
| run: | | |
| BASELINE_SIMPLE_PEAK=${{ steps.baseline-rates.outputs.simple_peak }} | |
| BASELINE_SIMPLE_SUSTAINED=${{ steps.baseline-rates.outputs.simple_sustained }} | |
| BASELINE_FULL_PEAK=${{ steps.baseline-rates.outputs.full_peak }} | |
| BASELINE_FULL_SUSTAINED=${{ steps.baseline-rates.outputs.full_sustained }} | |
| CURRENT_SIMPLE_PEAK=${{ steps.current-bench.outputs.simple_peak }} | |
| CURRENT_SIMPLE_SUSTAINED=${{ steps.current-bench.outputs.simple_sustained }} | |
| CURRENT_FULL_PEAK=${{ steps.current-bench.outputs.full_peak }} | |
| CURRENT_FULL_SUSTAINED=${{ steps.current-bench.outputs.full_sustained }} | |
| echo "=== Performance Comparison ===" | |
| echo "Simple mode:" | |
| echo " Previous commit - Peak: $BASELINE_SIMPLE_PEAK ops/sec, Sustained: $BASELINE_SIMPLE_SUSTAINED ops/sec" | |
| echo " Current commit - Peak: $CURRENT_SIMPLE_PEAK ops/sec, Sustained: $CURRENT_SIMPLE_SUSTAINED ops/sec" | |
| echo "Full mode:" | |
| echo " Previous commit - Peak: $BASELINE_FULL_PEAK ops/sec, Sustained: $BASELINE_FULL_SUSTAINED ops/sec" | |
| echo " Current commit - Peak: $CURRENT_FULL_PEAK ops/sec, Sustained: $CURRENT_FULL_SUSTAINED ops/sec" | |
| CHANGE_SIMPLE=$(echo "scale=2; (($CURRENT_SIMPLE_SUSTAINED - $BASELINE_SIMPLE_SUSTAINED) / $BASELINE_SIMPLE_SUSTAINED) * 100" | bc) | |
| CHANGE_FULL=$(echo "scale=2; (($CURRENT_FULL_SUSTAINED - $BASELINE_FULL_SUSTAINED) / $BASELINE_FULL_SUSTAINED) * 100" | bc) | |
| echo "Simple mode sustained change: $CHANGE_SIMPLE%" | |
| echo "Full mode sustained change: $CHANGE_FULL%" | |
| # Write summary | |
| echo "## Benchmark Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Simple Mode (Insert-Only)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Metric | Previous | Current | Change |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|----------|---------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Peak achieved | $BASELINE_SIMPLE_PEAK ops/sec | $CURRENT_SIMPLE_PEAK ops/sec | - |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Sustained stable | $BASELINE_SIMPLE_SUSTAINED ops/sec | $CURRENT_SIMPLE_SUSTAINED ops/sec | $CHANGE_SIMPLE% |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Full Mode (Mixed Operations)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Metric | Previous | Current | Change |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|----------|---------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Peak achieved | $BASELINE_FULL_PEAK ops/sec | $CURRENT_FULL_PEAK ops/sec | - |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Sustained stable | $BASELINE_FULL_SUSTAINED ops/sec | $CURRENT_FULL_SUSTAINED ops/sec | $CHANGE_FULL% |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Check for regression | |
| THRESHOLD=${{ env.REGRESSION_THRESHOLD }} | |
| IS_REGRESSION_SIMPLE=$(echo "$CHANGE_SIMPLE < $THRESHOLD" | bc) | |
| IS_REGRESSION_FULL=$(echo "$CHANGE_FULL < $THRESHOLD" | bc) | |
| if [ "$IS_REGRESSION_SIMPLE" -eq 1 ] || [ "$IS_REGRESSION_FULL" -eq 1 ]; then | |
| echo "::error::PERFORMANCE REGRESSION DETECTED!" | |
| if [ "$IS_REGRESSION_SIMPLE" -eq 1 ]; then | |
| echo "Simple mode: ${CHANGE_SIMPLE}% (exceeds ${THRESHOLD}% threshold)" | |
| fi | |
| if [ "$IS_REGRESSION_FULL" -eq 1 ]; then | |
| echo "Full mode: ${CHANGE_FULL}% (exceeds ${THRESHOLD}% threshold)" | |
| fi | |
| echo "### Performance Regression Detected" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ "$IS_REGRESSION_SIMPLE" -eq 1 ]; then | |
| echo "**Simple mode** sustained rate decreased by **${CHANGE_SIMPLE}%** (threshold: ${THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ "$IS_REGRESSION_FULL" -eq 1 ]; then | |
| echo "**Full mode** sustained rate decreased by **${CHANGE_FULL}%** (threshold: ${THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| exit 1 | |
| elif [ $(echo "$CHANGE_SIMPLE < 0" | bc) -eq 1 ] || [ $(echo "$CHANGE_FULL < 0" | bc) -eq 1 ]; then | |
| echo "Minor performance decrease detected" | |
| echo "This is within acceptable range (< ${THRESHOLD}%)." | |
| echo "### Minor Performance Decrease" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Changes are within acceptable range." >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "Performance improved or neutral!" | |
| echo "### Performance Improvement" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Both benchmarks show improvement or neutral performance!" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| bench-current-*.log | |
| bench-current-*.db | |
| bench-baseline-*.log | |
| bench-baseline-*.db |