diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index e272643e100..c3d3092f5b5 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -6,6 +6,7 @@ on: push: branches: - main + - perf_experiments workflow_dispatch: permissions: read-all @@ -52,11 +53,11 @@ jobs: mkdir build cd build cmake -GNinja -DWORKER_THREADS=2 .. - ninja + ninja basic submit # Microbenchmarks - ./tests.sh -VV -L benchmark + # ./tests.sh -VV -L benchmark # End to end performance tests - ./tests.sh -VV -L perf -C perf + ./tests.sh -VV -L perf -C perf -R pi_basic_mt # Convert microbenchmark output to bencher json source env/bin/activate PYTHONPATH=../tests python convert_pico_to_bencher.py @@ -70,7 +71,6 @@ jobs: build/workspace/*/*.config.json build/workspace/*/out build/workspace/*/err - build/workspace/*/*.ledger/* build/workspace/*/stack_trace if-no-files-found: ignore if: success() || failure() @@ -112,11 +112,11 @@ jobs: mkdir build cd build cmake -GNinja -DWORKER_THREADS=2 .. - ninja + ninja basic submit # Microbenchmarks - ./tests.sh -VV -L benchmark -E task_bench + # ./tests.sh -VV -L benchmark -E task_bench # End to end performance tests - ./tests.sh -VV -L perf -C perf + ./tests.sh -VV -L perf -C perf -R pi_basic_mt # Convert microbenchmark output to bencher json source env/bin/activate PYTHONPATH=../tests python convert_pico_to_bencher.py @@ -131,8 +131,8 @@ jobs: build/workspace/*/*.config.json build/workspace/*/out build/workspace/*/err - build/workspace/*/*.ledger/* build/workspace/*/stack_trace + build/bencher.json if-no-files-found: ignore if: success() || failure() @@ -147,3 +147,60 @@ jobs: --adapter json \ --err \ --file build/bencher.json + if: github.ref == 'refs/heads/main' + + benchmark_aci: + name: Benchmark ACI + runs-on: + [ + self-hosted, + 1ES.Pool=gha-aci-ci, + "JobId=bench_aci-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}", + ] + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Build and run ACI perf tests + run: | + git config --global --add safe.directory /__w/CCF/CCF + mkdir build + cd build + cmake -GNinja -DWORKER_THREADS=2 .. + ninja basic submit + # Microbenchmarks + # ./tests.sh -VV -L benchmark -E task_bench + # End to end performance tests + ./tests.sh -VV -L perf -C perf -R pi_basic_mt + # Convert microbenchmark output to bencher json + source env/bin/activate + PYTHONPATH=../tests python convert_pico_to_bencher.py + shell: bash + + - name: "Upload logs" + uses: actions/upload-artifact@v7 + with: + name: logs-bench-aci + path: | + dmesg.log + build/workspace/*/*.config.json + build/workspace/*/out + build/workspace/*/err + build/workspace/*/stack_trace + build/bencher.json + if-no-files-found: ignore + if: success() || failure() + + - uses: bencherdev/bencher@main + - name: Track base branch benchmarks with Bencher + run: | + bencher run \ + --project ccf \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch main \ + --testbed gha-c-aci-ci \ + --adapter json \ + --err \ + --file build/bencher.json + if: github.ref == 'refs/heads/main' \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index f87ea9aafb6..a7b7dae6726 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1345,7 +1345,7 @@ if(BUILD_TESTS) --package "samples/apps/basic/basic" --client-def - "${WORKER_THREADS},write,2000000,primary" + "${WORKER_THREADS},write,4000000,primary" ) endif() diff --git a/tests/infra/basicperf.py b/tests/infra/basicperf.py index b9514767736..e422ed860dd 100644 --- a/tests/infra/basicperf.py +++ b/tests/infra/basicperf.py @@ -261,7 +261,7 @@ def replace_primary(network, host, old_primary, snapshots_dir, statistics): def run(args): - hosts = args.nodes or infra.e2e_args.nodes(args, 1) + hosts = args.nodes or infra.e2e_args.nodes(args, 3) if args.stop_primary_after_s: assert ( @@ -822,7 +822,7 @@ def cli_args(): parser.add_argument( "--client-timeout-s", help="Number of seconds after which unresponsive clients are shut down", - default=300, + default=600, type=float, ) parser.add_argument(