diff --git a/.github/workflows/run-benchmark-v2.yml b/.github/workflows/run-benchmark-v2.yml index 2cc06f36f..e830df09f 100644 --- a/.github/workflows/run-benchmark-v2.yml +++ b/.github/workflows/run-benchmark-v2.yml @@ -100,6 +100,7 @@ jobs: echo "Generating e2e proof for block $block_number..." export JEMALLOC_SYS_WITH_MALLOC_CONF="retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:10000,muzzy_decay_ms:10000,abort_conf:true" set +e + RUST_MIN_STACK=16777216 \ CENO_GPU_CACHE_LEVEL=0 RUSTFLAGS="-C target-feature=+avx2" \ RUST_LOG=info,openvm_stark_*=warn,openvm_cuda_common=warn \ cargo run --features "jemalloc,gpu" --config net.git-fetch-with-cli=true \ @@ -206,6 +207,97 @@ jobs: - name: Path to result run: | echo "https://github.com/${{ github.repository }}/blob/gh-pages/${{ env.GH_PAGES_PATH }}/${{ env.LOG_NAME }}_summary.md" + + check-gpu-mem-estimation: + runs-on: [self-hosted, x64, linux, gpu] + needs: benchmark + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: nightly-2025-11-20 # keep it same with ceno repo + components: rust-src + cache: false + + - name: Give Github Action access to ceno-gpu + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: | + ${{ secrets.SECRET_FOR_CENO_GPU }} + + - name: Run GPU memory tracking benchmark + id: run_gpu_mem_verify + run: | + block_number="${{ inputs.block_number }}" + # Install Ceno CLI + echo "Installing Ceno CLI..." + CENO_VERSION="${{ inputs.ceno_version }}" + CENO_REF_FLAG="--rev ${CENO_VERSION}" + if git ls-remote --heads https://github.com/scroll-tech/ceno.git "${CENO_VERSION}" | grep -q "${CENO_VERSION}"; then + echo "Detected branch '${CENO_VERSION}', installing via --branch" + CENO_REF_FLAG="--branch ${CENO_VERSION}" + else + echo "Installing via --rev ${CENO_VERSION}" + fi + cargo install --git https://github.com/scroll-tech/ceno.git \ + ${CENO_REF_FLAG} \ + --features jemalloc \ + --features nightly-features \ + --locked \ + --force \ + cargo-ceno + + RPC_1=${{ secrets.RPC_URL_1 }} + + # Build client binary + cd bin/ceno-client-eth + cargo ceno build --release + + cd ../.. + + # Create necessary directories + mkdir -p output + mkdir -p rpc-cache + + # LOG_NAME with Block number and Timestamp(UTC+8) + TIMESTAMP=$(date -u -d '+8 hours' +'%Y%m%d-%H%M%S') + LOG_NAME="gpu-mem-verify-mainnet${block_number}-${TIMESTAMP}" + LOG_FILE="${LOG_NAME}.log" + echo "LOG_NAME=${LOG_NAME}" >> $GITHUB_ENV + echo "LOG_FILE=${LOG_FILE}" >> $GITHUB_ENV + + echo "Running GPU memory tracking benchmark for block $block_number..." + export JEMALLOC_SYS_WITH_MALLOC_CONF="retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:10000,muzzy_decay_ms:10000,abort_conf:true" + set +e + + # Note: `CENO_GPU_MEM_TRACKING=1 CENO_CONCURRENT_CHIP_PROVING=0` to check GPU memory estimation + CENO_GPU_MEM_TRACKING=1 \ + CENO_CONCURRENT_CHIP_PROVING=0 \ + RUST_MIN_STACK=16777216 \ + CENO_GPU_CACHE_LEVEL=0 RUSTFLAGS="-C target-feature=+avx2" \ + RUST_LOG=info,openvm_stark_*=warn,openvm_cuda_common=warn \ + cargo run --features "jemalloc,gpu" --config net.git-fetch-with-cli=true \ + --release --bin ceno-reth-benchmark-bin -- \ + --mode prove-stark \ + --block-number $block_number \ + --rpc-url ${{ secrets.RPC_URL_1 }} \ + --output-dir output \ + --cache-dir rpc-cache > "$LOG_FILE" 2>&1 + benchmark_status=$? + set -e + grep -v '\[ceno-gpu\]' "$LOG_FILE" # filter out ceno-gpu logs + if [ $benchmark_status -ne 0 ]; then + echo "GPU memory tracking benchmark failed with status $benchmark_status" + if [ "${{ inputs.ignore_failed }}" != "true" ]; then + exit $benchmark_status + fi + echo "Continuing despite failure because ignore_failed=true" + else + echo "GPU memory tracking benchmark completed successfully" + fi # fi # # # Verify input file exists diff --git a/Cargo.lock b/Cargo.lock index c92834459..49c1db640 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1663,7 +1663,7 @@ dependencies = [ [[package]] name = "cargo-ceno" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "anyhow", "bincode 1.3.3", @@ -1818,7 +1818,7 @@ dependencies = [ [[package]] name = "ceno-examples" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "glob", ] @@ -1838,7 +1838,7 @@ dependencies = [ [[package]] name = "ceno_crypto_primitives" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno-patch?branch=main#b79232a6fc80c799f584380273fd6d3055b36808" +source = "git+https://github.com/scroll-tech/ceno-patch.git?branch=main#b79232a6fc80c799f584380273fd6d3055b36808" dependencies = [ "ceno_syscall", "elliptic-curve", @@ -1847,7 +1847,7 @@ dependencies = [ [[package]] name = "ceno_emul" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "anyhow", "ceno_rt", @@ -1880,7 +1880,7 @@ dependencies = [ [[package]] name = "ceno_host" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "anyhow", "ceno_emul", @@ -1893,7 +1893,7 @@ dependencies = [ [[package]] name = "ceno_recursion" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "bincode 1.3.3", "ceno-examples", @@ -1936,7 +1936,7 @@ dependencies = [ [[package]] name = "ceno_rt" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "ceno_serde", "getrandom 0.2.16", @@ -1947,7 +1947,7 @@ dependencies = [ [[package]] name = "ceno_serde" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "bytemuck", "serde", @@ -1956,12 +1956,12 @@ dependencies = [ [[package]] name = "ceno_syscall" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno-patch?branch=main#b79232a6fc80c799f584380273fd6d3055b36808" +source = "git+https://github.com/scroll-tech/ceno-patch.git?branch=main#b79232a6fc80c799f584380273fd6d3055b36808" [[package]] name = "ceno_zkvm" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "arrayref", "base64 0.22.1", @@ -2429,7 +2429,7 @@ dependencies = [ [[package]] name = "cuda_hal" version = "0.1.0" -source = "git+ssh://git@github.com/scroll-tech/ceno-gpu.git?branch=main#ef9b61c587fa5c964e1c40d0ebf58d0abed7b07e" +source = "git+ssh://git@github.com/scroll-tech/ceno-gpu.git?branch=feat%2Fcleanup-0227#342ac395cd2a5d4e32cadfee8605d841dad79bb7" dependencies = [ "anyhow", "cuda-runtime-sys", @@ -2660,7 +2660,7 @@ dependencies = [ [[package]] name = "derive" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "quote", "syn 1.0.109", @@ -3090,7 +3090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3299,7 +3299,7 @@ dependencies = [ [[package]] name = "ff_ext" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "once_cell", "p3", @@ -3644,7 +3644,7 @@ dependencies = [ [[package]] name = "gkr_iop" version = "0.1.0" -source = "git+https://github.com/scroll-tech/ceno?branch=master#67092deca647f5220d4482383e9e7635bdac0997" +source = "git+https://github.com/scroll-tech/ceno?branch=feat%2Fconcurrent-chip-proving#9cbc15c131d620fc5071928e9233685858112040" dependencies = [ "bincode 1.3.3", "cuda_hal", @@ -4432,7 +4432,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4963,7 +4963,7 @@ dependencies = [ [[package]] name = "mpcs" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "bincode 1.3.3", "clap", @@ -4987,7 +4987,7 @@ dependencies = [ [[package]] name = "multilinear_extensions" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "either", "ff_ext", @@ -5035,7 +5035,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6619,7 +6619,7 @@ dependencies = [ [[package]] name = "p3" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "p3-air", "p3-baby-bear", @@ -7360,7 +7360,7 @@ dependencies = [ [[package]] name = "poseidon" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "ff_ext", "p3", @@ -7622,7 +7622,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -8901,7 +8901,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9542,7 +9542,7 @@ dependencies = [ [[package]] name = "sp1-curves" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "cfg-if", "dashu", @@ -9580,7 +9580,7 @@ dependencies = [ [[package]] name = "sppark" version = "0.1.11" -source = "git+ssh://git@github.com/scroll-tech/ceno-gpu.git?branch=main#ef9b61c587fa5c964e1c40d0ebf58d0abed7b07e" +source = "git+ssh://git@github.com/scroll-tech/ceno-gpu.git?branch=feat%2Fcleanup-0227#342ac395cd2a5d4e32cadfee8605d841dad79bb7" dependencies = [ "cc", "which", @@ -9589,7 +9589,7 @@ dependencies = [ [[package]] name = "sppark_plug" version = "0.1.0" -source = "git+ssh://git@github.com/scroll-tech/ceno-gpu.git?branch=main#ef9b61c587fa5c964e1c40d0ebf58d0abed7b07e" +source = "git+ssh://git@github.com/scroll-tech/ceno-gpu.git?branch=feat%2Fcleanup-0227#342ac395cd2a5d4e32cadfee8605d841dad79bb7" dependencies = [ "cc", "ff_ext", @@ -9699,7 +9699,7 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "sumcheck" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "either", "ff_ext", @@ -9717,7 +9717,7 @@ dependencies = [ [[package]] name = "sumcheck_macro" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "itertools 0.13.0", "p3", @@ -9869,7 +9869,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10409,7 +10409,7 @@ dependencies = [ [[package]] name = "transcript" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "ff_ext", "itertools 0.13.0", @@ -10785,7 +10785,7 @@ dependencies = [ [[package]] name = "whir" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "bincode 1.3.3", "clap", @@ -10827,7 +10827,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -11163,7 +11163,7 @@ checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "witness" version = "0.1.0" -source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.20#4f5a2b99af8edc9e8d808a6ab7c73e4088d61e39" +source = "git+https://github.com/scroll-tech/gkr-backend.git?tag=v1.0.0-alpha.22#a140b93cf80109ef86c2327b9a940d4cace83628" dependencies = [ "ff_ext", "multilinear_extensions", diff --git a/Cargo.toml b/Cargo.toml index 6406278f0..c54ec42e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,14 +60,14 @@ openvm-primitives = { path = "./crates/primitives" } openvm-reth-benchmark = { path = "./crates/host-bench", default-features = false } openvm-mpt = { path = "./crates/mpt" } # ceno -ceno_emul = { git = "https://github.com/scroll-tech/ceno", branch = "master" } -ceno_host = { git = "https://github.com/scroll-tech/ceno", branch = "master" } -ceno_zkvm = { git = "https://github.com/scroll-tech/ceno", branch = "master" } -ceno_cli = { git = "https://github.com/scroll-tech/ceno", package = "cargo-ceno", branch = "master" } -ceno_recursion = { git = "https://github.com/scroll-tech/ceno", branch = "master" } -gkr_iop = { git = "https://github.com/scroll-tech/ceno", branch = "master" } -ff_ext = { git = "https://github.com/scroll-tech/gkr-backend.git", tag = "v1.0.0-alpha.20" } -mpcs = { git = "https://github.com/scroll-tech/gkr-backend.git", tag = "v1.0.0-alpha.20" } +ceno_emul = { git = "https://github.com/scroll-tech/ceno", branch = "feat/concurrent-chip-proving" } +ceno_host = { git = "https://github.com/scroll-tech/ceno", branch = "feat/concurrent-chip-proving" } +ceno_zkvm = { git = "https://github.com/scroll-tech/ceno", branch = "feat/concurrent-chip-proving" } +ceno_cli = { git = "https://github.com/scroll-tech/ceno", package = "cargo-ceno", branch = "feat/concurrent-chip-proving" } +ceno_recursion = { git = "https://github.com/scroll-tech/ceno", branch = "feat/concurrent-chip-proving" } +gkr_iop = { git = "https://github.com/scroll-tech/ceno", branch = "feat/concurrent-chip-proving" } +ff_ext = { git = "https://github.com/scroll-tech/gkr-backend.git", tag = "v1.0.0-alpha.22" } +mpcs = { git = "https://github.com/scroll-tech/gkr-backend.git", tag = "v1.0.0-alpha.22" } # reth reth-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v1.9.3", default-features = false } @@ -166,7 +166,7 @@ openvm-k256 = { git = "https://github.com/scroll-tech/openvm.git", package = "k2 openvm-p256 = { git = "https://github.com/scroll-tech/openvm.git", package = "p256", branch = "feat/v1.4.1-scroll-ext" } [patch."https://github.com/scroll-tech/ceno-gpu-mock.git"] -ceno_gpu = { git = "ssh://git@github.com/scroll-tech/ceno-gpu.git", package = "cuda_hal", branch = "main", default-features = false, features = ["bb31"] } +ceno_gpu = { git = "ssh://git@github.com/scroll-tech/ceno-gpu.git", package = "cuda_hal", branch = "feat/cleanup-0227", default-features = false, features = ["bb31"] } # ceno_gpu = { path = "../ceno-gpu/cuda_hal", package = "cuda_hal", default-feature = false, features = ["bb31"] } [patch."https://github.com/axiom-crypto/openvm-kzg.git"] @@ -206,10 +206,10 @@ opt-level = 3 inherits = "release" debug = 1 -#[patch."https://github.com/scroll-tech/ceno.git"] -#ceno_emul = { path = "../ceno/ceno_emul" } -#ceno_host = { path = "../ceno/ceno_host" } -#ceno_zkvm = { path = "../ceno/ceno_zkvm" } -#gkr_iop = { path = "../ceno/gkr_iop" } -#ceno_cli = { path = "../ceno/ceno_cli", package = "cargo-ceno" } -#ceno_recursion = { path = "../ceno/ceno_recursion" } +# [patch."https://github.com/scroll-tech/ceno.git"] +# ceno_emul = { path = "../ceno/ceno_emul" } +# ceno_host = { path = "../ceno/ceno_host" } +# ceno_zkvm = { path = "../ceno/ceno_zkvm" } +# gkr_iop = { path = "../ceno/gkr_iop" } +# ceno_cli = { path = "../ceno/ceno_cli", package = "cargo-ceno" } +# ceno_recursion = { path = "../ceno/ceno_recursion" } diff --git a/server/prove_block.sh b/server/prove_block.sh index 74e2e4824..d9c99c983 100755 --- a/server/prove_block.sh +++ b/server/prove_block.sh @@ -204,7 +204,7 @@ OUTPUT_PATH="$job_dir/metrics.json" export CENO_GPU_CACHE_LEVEL -"$BIN_PATH" \ +RUST_MIN_STACK=16777216 "$BIN_PATH" \ --mode "$MODE" \ --block-number "$BLOCK_NUMBER" \ --input-path "$INPUT_PATH" \