From 864cf248b81632a891f324c4c1ca1b2fdc578309 Mon Sep 17 00:00:00 2001 From: Ben Marx Date: Tue, 1 Sep 2026 14:26:39 -0700 Subject: [PATCH 1/5] ci: fold the imported workflows into this repo's set Step 5 of the monorepo migration. The 11 workflows that came with doublezero-offchain and doublezero-solana are folded in and the imported `.github` directories are deleted. GitHub reads workflows only at the repository root, so nothing in those directories ever ran here. Five release workflows come across, one per component, matching the shape this repo already uses. They run goreleaser from the repository root rather than from `offchain/`: the merged workspace writes to the root `target/`, which is where goreleaser's rust builder looks for the binary it packages. The five goreleaser configs get their relative paths rewritten to match, and `release.github.name` moves from `doublezero-offchain` to `doublezero` so the releases land here. This repo already holds the three secrets they need. Two workflows are new rather than ported. `solana` runs that tree's own lint, library tests, docs and SBF tests plus the checksum gate, because the tree is excluded from the root workspace and `rust.yml` never reaches it, so it would otherwise have no CI here at all. `elixir` runs the scheduler's format check, compile, credo and tests on the pull request that changes it; upstream those ran on a release tag alone, so a pull request that broke the scheduler passed. Offchain's `ci.yml` is dropped, since `make rust-build`, `make rust-lint` and `make rust-test` now cover those crates. Two things it carried are kept elsewhere: `rust-cli-static` grows from the client alone to all five released CLIs, and the per-crate changelog check moves into this repo's `changelog-reminder`. One thing is not kept: the `just ci` coverage floor, whose workspace no longer exists. `offchain/Justfile` keeps its Elixir recipes and loses its Rust ones, which would otherwise run cargo over the whole workspace with a different fmt and clippy line than CI uses. `offchain/scripts/release-rc.sh` resolves the repository root two levels up, so local release candidates build against the merged workspace. --- .github/workflows/changelog-reminder.yml | 43 ++++- .github/workflows/elixir.yml | 41 +++++ .../workflows/offchain.local-validator.yml | 79 +++++++++ .../workflows/release.contributor-rewards.yml | 8 +- .../release.doublezero-solana-cli.yml | 11 +- .../workflows/release.offchain-scheduler.yml | 45 +++--- .../workflows/release.offchain-sentinel.yml | 10 +- .../release.solana-validator-debt.yml | 8 +- .github/workflows/rust.yml | 35 ++-- .github/workflows/solana.yml | 79 +++++++++ CHANGELOG.md | 9 ++ offchain/.github/pull_request_template.md | 9 -- .../.github/workflows/changelog-reminder.yml | 79 --------- offchain/.github/workflows/ci.yml | 87 ---------- .../.github/workflows/local-validator.yml | 153 ------------------ offchain/Justfile | 79 ++------- .../.goreleaser.contributor-rewards.yaml | 6 +- ...eleaser.doublezero-offchain-scheduler.yaml | 14 +- .../.goreleaser.doublezero-solana-cli.yaml | 2 +- ...aser.doublezero-solana-validator-debt.yaml | 6 +- offchain/release/.goreleaser.sentinel.yaml | 6 +- offchain/scripts/release-rc.sh | 17 +- solana/.github/workflows/local-validator.yml | 59 ------- solana/.github/workflows/rust.yml | 58 ------- solana/.github/workflows/verify-build.yml | 14 -- 25 files changed, 357 insertions(+), 600 deletions(-) create mode 100644 .github/workflows/elixir.yml create mode 100644 .github/workflows/offchain.local-validator.yml rename {offchain/.github => .github}/workflows/release.contributor-rewards.yml (75%) rename {offchain/.github => .github}/workflows/release.doublezero-solana-cli.yml (71%) rename offchain/.github/workflows/release.scheduler.yml => .github/workflows/release.offchain-scheduler.yml (57%) rename offchain/.github/workflows/release.sentinel.yml => .github/workflows/release.offchain-sentinel.yml (72%) rename {offchain/.github => .github}/workflows/release.solana-validator-debt.yml (74%) create mode 100644 .github/workflows/solana.yml delete mode 100644 offchain/.github/pull_request_template.md delete mode 100644 offchain/.github/workflows/changelog-reminder.yml delete mode 100644 offchain/.github/workflows/ci.yml delete mode 100644 offchain/.github/workflows/local-validator.yml delete mode 100644 solana/.github/workflows/local-validator.yml delete mode 100644 solana/.github/workflows/rust.yml delete mode 100644 solana/.github/workflows/verify-build.yml diff --git a/.github/workflows/changelog-reminder.yml b/.github/workflows/changelog-reminder.yml index ffe0eff600..512ae32302 100644 --- a/.github/workflows/changelog-reminder.yml +++ b/.github/workflows/changelog-reminder.yml @@ -36,10 +36,49 @@ jobs: changed_files=$(git diff --name-only "$base_ref"..."$head_ref") - if echo "$changed_files" | grep -q "^CHANGELOG.md$"; then + if ! echo "$changed_files" | grep -q "^CHANGELOG.md$"; then + echo "CHANGELOG.md was not updated." + echo "If this change intentionally does not require a changelog, add the 'skip-changelog' label to the PR." + exit 1 + fi + + # The offchain crates each keep their own changelog as well, which is + # what their releases read. This list comes from the changelog-reminder + # that doublezero-offchain ran before the merge, with its paths moved + # under offchain/. + projects=( + "offchain/crates/contributor-rewards" + "offchain/crates/scheduled-command" + "offchain/crates/sentinel" + "offchain/crates/slack-notifier" + "offchain/crates/solana-admin-cli/passport" + "offchain/crates/solana-admin-cli/revenue-distribution" + "offchain/crates/solana-admin-cli/sol-conversion" + "offchain/crates/solana-cli" + "offchain/crates/solana-client-tools" + "offchain/crates/solana-fork" + "offchain/crates/solana-interface/sol-conversion" + "offchain/crates/validator-debt" + "offchain/scheduler" + ) + + missing=() + for subdir in "${projects[@]}"; do + if echo "$changed_files" | grep -q "^${subdir}/"; then + if ! echo "$changed_files" | grep -q "^${subdir}/CHANGELOG.md$"; then + missing+=("$subdir") + fi + fi + done + + if [ ${#missing[@]} -eq 0 ]; then exit 0 fi - echo "CHANGELOG.md was not updated." + echo "These subprojects changed but their own changelog was not updated:" + for m in "${missing[@]}"; do + echo " - $m (expected $m/CHANGELOG.md)" + done + echo echo "If this change intentionally does not require a changelog, add the 'skip-changelog' label to the PR." exit 1 diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml new file mode 100644 index 0000000000..4e87c7562a --- /dev/null +++ b/.github/workflows/elixir.yml @@ -0,0 +1,41 @@ +# The offchain scheduler is the only Elixir component. Its checks ran on a release +# tag alone before the merge; here they run on the pull request that changes it. +name: elixir + +on: + push: + branches: [main, 'hotfix/**'] + pull_request: + paths: + - 'offchain/scheduler/**' + - '.github/workflows/elixir.yml' + +defaults: + run: + working-directory: offchain/scheduler + +jobs: + elixir-scheduler: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Elixir and Erlang + uses: erlef/setup-beam@v1 + with: + elixir-version: "1.19.3" + otp-version: "28.1" + # mix compile builds the Rust NIF, which is a root workspace member. + - uses: dtolnay/rust-toolchain@1.97.1 + - uses: Swatinem/rust-cache@v2 + - name: Cache deps + uses: actions/cache@v4 + with: + path: offchain/scheduler/deps + key: mix-deps-${{ hashFiles('offchain/scheduler/mix.lock') }} + restore-keys: | + mix-deps- + - run: mix deps.get + - run: mix format --check-formatted + - run: mix compile --warnings-as-errors + - run: mix credo --strict + - run: mix test --cover diff --git a/.github/workflows/offchain.local-validator.yml b/.github/workflows/offchain.local-validator.yml new file mode 100644 index 0000000000..b731a67b95 --- /dev/null +++ b/.github/workflows/offchain.local-validator.yml @@ -0,0 +1,79 @@ +# Fork tests for the offchain Solana tooling: they start a mainnet-beta fork with +# doublezero-solana-fork and drive the CLIs against it. +# +# Every step runs from the repository root, not from offchain/, because the shell +# scripts resolve their binaries as target/debug/ and the merged workspace +# writes to the root target/ directory. The fork and the keypairs the scripts read +# also land in the working directory, so all of it has to agree. +name: offchain-local-validator + +on: + push: + branches: [main, 'hotfix/**'] + pull_request: + paths: + - 'offchain/**' + - 'solana/**' + - 'Cargo.lock' + - 'Cargo.toml' + - '.github/workflows/offchain.local-validator.yml' + +env: + SOLANA_CLI: v3.0.12 + +jobs: + test-doublezero-solana-fork: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Free disk space + run: | + df -h + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo docker builder prune -a --force + df -h + - uses: dtolnay/rust-toolchain@1.97.1 + - uses: Swatinem/rust-cache@v2 + - name: Solana toolchain + run: | + sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" + echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH + - name: Generate ~/.config/solana/id.json + run: solana-keygen new --silent --no-bip39-passphrase + - name: Generate manager keypair for synthetic ValidatorClientRewards + run: solana-keygen new --silent --no-bip39-passphrase -o manager_keypair.json + - name: Start Solana mainnet-beta fork in background + run: | + MANAGER_PUBKEY=$(solana address -k manager_keypair.json) + cargo run --bin doublezero-solana-fork -- -um --reset --synthetic-validator-client-rewards-manager "$MANAGER_PUBKEY" > /dev/null 2>&1 & + - name: Build `doublezero-solana` + run: cargo build --bin doublezero-solana + - name: Run `doublezero-solana` tests + run: bash offchain/sh/test_doublezero_solana_fork.sh + + test-doublezero-solana-validator-debt-fork-current: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Free disk space + run: | + df -h + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo docker builder prune -a --force + df -h + - uses: dtolnay/rust-toolchain@1.97.1 + - uses: Swatinem/rust-cache@v2 + - name: Solana toolchain + run: | + sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" + echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH + - name: Generate ~/.config/solana/id.json + run: solana-keygen new --silent --no-bip39-passphrase + - name: Start Solana mainnet-beta fork in background + run: cargo run --bin doublezero-solana-fork -- -um --reset --god-mode > /dev/null 2>&1 & + - name: Build the admin and validator-debt CLIs + run: cargo build --bin doublezero-revenue-distribution-admin --bin doublezero-solana-validator-debt --bin doublezero-solana + - name: Run `doublezero-solana-validator-debt` tests + run: bash offchain/sh/test_validator_debt_fork.sh diff --git a/offchain/.github/workflows/release.contributor-rewards.yml b/.github/workflows/release.contributor-rewards.yml similarity index 75% rename from offchain/.github/workflows/release.contributor-rewards.yml rename to .github/workflows/release.contributor-rewards.yml index 81665783be..defc25b55d 100644 --- a/offchain/.github/workflows/release.contributor-rewards.yml +++ b/.github/workflows/release.contributor-rewards.yml @@ -20,19 +20,21 @@ jobs: cache-targets: | target target/x86_64-unknown-linux-musl/release - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@1.97.1 with: - toolchain: 1.92.0 targets: x86_64-unknown-linux-musl - name: Install dependencies for rpm packaging and musl static build run: | sudo apt update sudo apt install squashfs-tools rpm musl-tools cmake -y + # goreleaser runs from the repository root, not from offchain/: the merged + # workspace writes to the root target/ directory, which is where the rust + # builder looks for the binary it packages. - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - args: release -f release/.goreleaser.contributor-rewards.yaml --clean + args: release -f offchain/release/.goreleaser.contributor-rewards.yaml --clean env: SERVICEABILITY_PROGRAM_ID: devnet SLACK_WEBHOOK: ${{ secrets.SLACK_BOTS_WEBHOOK }} diff --git a/offchain/.github/workflows/release.doublezero-solana-cli.yml b/.github/workflows/release.doublezero-solana-cli.yml similarity index 71% rename from offchain/.github/workflows/release.doublezero-solana-cli.yml rename to .github/workflows/release.doublezero-solana-cli.yml index daa545177b..c08d54ccad 100644 --- a/offchain/.github/workflows/release.doublezero-solana-cli.yml +++ b/.github/workflows/release.doublezero-solana-cli.yml @@ -1,4 +1,4 @@ -name: releaser.doublezero-solana +name: releaser.doublezero-solana-cli on: push: @@ -20,21 +20,22 @@ jobs: cache-targets: | target target/x86_64-unknown-linux-musl/release - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@1.97.1 with: - toolchain: 1.92.0 targets: x86_64-unknown-linux-musl - name: Install dependencies for rpm packaging and musl static build run: | sudo apt update sudo apt install squashfs-tools rpm musl-tools cmake -y + # goreleaser runs from the repository root, not from offchain/: the merged + # workspace writes to the root target/ directory, which is where the rust + # builder looks for the binary it packages. - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - args: release -f release/.goreleaser.doublezero-solana-cli.yaml --clean + args: release -f offchain/release/.goreleaser.doublezero-solana-cli.yaml --clean env: - SERVICEABILITY_PROGRAM_ID: devnet SLACK_WEBHOOK: ${{ secrets.SLACK_BOTS_WEBHOOK }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/offchain/.github/workflows/release.scheduler.yml b/.github/workflows/release.offchain-scheduler.yml similarity index 57% rename from offchain/.github/workflows/release.scheduler.yml rename to .github/workflows/release.offchain-scheduler.yml index ecd9b1cfdd..07e4ff0986 100644 --- a/offchain/.github/workflows/release.scheduler.yml +++ b/.github/workflows/release.offchain-scheduler.yml @@ -11,12 +11,7 @@ permissions: jobs: goreleaser: runs-on: ubuntu-latest - strategy: - matrix: - platform: [linux/amd64] steps: - - name: Set platform - run: echo "Running on ${{ matrix.platform }}" - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -32,50 +27,50 @@ jobs: elixir-version: "1.19.3" otp-version: "28.1" + # The Rust NIF the release builds is a member of the root workspace, so it + # needs this repository's toolchain rather than whatever rustup defaults to. + - uses: dtolnay/rust-toolchain@1.97.1 + - uses: Swatinem/rust-cache@v2 + - name: Cache deps - id: cache-deps - uses: actions/cache@v3 - env: - cache-name: cache-elixir-deps + uses: actions/cache@v4 with: - path: deps - key: mix-${{ env.cache-name }}-${{ hashFiles('**/mix.lock') }} + path: offchain/scheduler/deps + key: mix-deps-${{ hashFiles('offchain/scheduler/mix.lock') }} restore-keys: | - mix-${{ env.cache-name }}- + mix-deps- - name: Cache compiled build - id: cache-build - uses: actions/cache@v3 - env: - cache-name: cache-compiled-build + uses: actions/cache@v4 with: - path: _build - key: mix-${{ env.cache-name }}-${{ hashFiles('**/mix.lock') }} + path: offchain/scheduler/_build + key: mix-build-${{ hashFiles('offchain/scheduler/mix.lock') }} restore-keys: | - mix-${{ env.cache-name }}- - ${{ runner.os }}-mix- + mix-build- - name: Install deps - working-directory: scheduler + working-directory: offchain/scheduler run: mix deps.get - name: Compile (warnings as errors) - working-directory: scheduler + working-directory: offchain/scheduler run: mix compile --warnings-as-errors - name: Credo - working-directory: scheduler + working-directory: offchain/scheduler run: mix credo --strict - name: Tests - working-directory: scheduler + working-directory: offchain/scheduler run: mix test --cover + # goreleaser runs from the repository root and its `mix release` hooks name + # offchain/scheduler as their directory. - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - args: release -f release/.goreleaser.doublezero-offchain-scheduler.yaml --clean --verbose + args: release -f offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml --clean --verbose env: SLACK_WEBHOOK: ${{ secrets.SLACK_BOTS_WEBHOOK }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/offchain/.github/workflows/release.sentinel.yml b/.github/workflows/release.offchain-sentinel.yml similarity index 72% rename from offchain/.github/workflows/release.sentinel.yml rename to .github/workflows/release.offchain-sentinel.yml index 7e4b8e11f9..4a66802fe9 100644 --- a/offchain/.github/workflows/release.sentinel.yml +++ b/.github/workflows/release.offchain-sentinel.yml @@ -1,4 +1,4 @@ -name: releaser.sentinel +name: releaser.offchain-sentinel on: push: @@ -20,19 +20,21 @@ jobs: cache-targets: | target target/x86_64-unknown-linux-musl/release - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@1.97.1 with: - toolchain: 1.92.0 targets: x86_64-unknown-linux-musl - name: Install dependencies for rpm packaging and musl static build run: | sudo apt update sudo apt install squashfs-tools rpm musl-tools cmake -y + # goreleaser runs from the repository root, not from offchain/: the merged + # workspace writes to the root target/ directory, which is where the rust + # builder looks for the binary it packages. - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - args: release -f release/.goreleaser.sentinel.yaml --clean + args: release -f offchain/release/.goreleaser.sentinel.yaml --clean env: SLACK_WEBHOOK: ${{ secrets.SLACK_BOTS_WEBHOOK }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/offchain/.github/workflows/release.solana-validator-debt.yml b/.github/workflows/release.solana-validator-debt.yml similarity index 74% rename from offchain/.github/workflows/release.solana-validator-debt.yml rename to .github/workflows/release.solana-validator-debt.yml index ef56333e8d..25d9a33c3d 100644 --- a/offchain/.github/workflows/release.solana-validator-debt.yml +++ b/.github/workflows/release.solana-validator-debt.yml @@ -20,19 +20,21 @@ jobs: cache-targets: | target target/x86_64-unknown-linux-musl/release - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@1.97.1 with: - toolchain: 1.92.0 targets: x86_64-unknown-linux-musl - name: Install dependencies for rpm packaging and musl static build run: | sudo apt update sudo apt install squashfs-tools rpm musl-tools cmake -y + # goreleaser runs from the repository root, not from offchain/: the merged + # workspace writes to the root target/ directory, which is where the rust + # builder looks for the binary it packages. - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser-pro - args: release -f release/.goreleaser.doublezero-solana-validator-debt.yaml --clean + args: release -f offchain/release/.goreleaser.doublezero-solana-validator-debt.yaml --clean env: SLACK_WEBHOOK: ${{ secrets.SLACK_BOTS_WEBHOOK }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e635bb832f..aa663a4f1e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -47,9 +47,10 @@ jobs: sh -c "$(curl -sSfL https://release.anza.xyz/v3.0.4/install)" echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - run: make rust-validator-test - # The doublezero CLI ships as a static musl binary so it loads on any Linux - # regardless of the host glibc version. Build it for the musl target and assert - # the result is fully static. + # Every released CLI ships as a static musl binary so it loads on any Linux + # regardless of the host glibc version. Build each package on its own, the way + # goreleaser does so feature unification matches the release, and assert the + # result is fully static. rust-cli-static: runs-on: ubuntu-24.04 steps: @@ -59,21 +60,29 @@ jobs: targets: x86_64-unknown-linux-musl - uses: Swatinem/rust-cache@v2 - name: Install musl toolchain - run: sudo apt-get update && sudo apt-get install -y musl-tools - - name: Build CLI for musl and assert it is statically linked + run: sudo apt-get update && sudo apt-get install -y musl-tools cmake + - name: Build the release CLIs for musl and assert they are statically linked env: CC_x86_64_unknown_linux_musl: musl-gcc CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: musl-gcc run: | - cargo build --release -p doublezero --target x86_64-unknown-linux-musl - bin=target/x86_64-unknown-linux-musl/release/doublezero - file "$bin" - if file "$bin" | grep -qE "statically linked|static-pie linked"; then + for entry in \ + doublezero:doublezero \ + doublezero-solana-cli:doublezero-solana \ + doublezero-contributor-rewards:doublezero-contributor-rewards \ + doublezero-ledger-sentinel:doublezero-sentinel \ + doublezero-solana-validator-debt:doublezero-solana-validator-debt + do + package="${entry%%:*}" + bin="target/x86_64-unknown-linux-musl/release/${entry##*:}" + cargo build --release --package "$package" --target x86_64-unknown-linux-musl + file "$bin" + if ! file "$bin" | grep -qE "statically linked|static-pie linked"; then + echo "::error::$bin is not statically linked" + exit 1 + fi echo "OK: $bin is statically linked" - else - echo "::error::doublezero CLI is not statically linked" - exit 1 - fi + done # The release bump runs `cargo update --workspace`, which can silently rebind # members onto a different locked minor. release-bump-dry-run: diff --git a/.github/workflows/solana.yml b/.github/workflows/solana.yml new file mode 100644 index 0000000000..e5adbf08cd --- /dev/null +++ b/.github/workflows/solana.yml @@ -0,0 +1,79 @@ +# CI for the solana/ tree, which is a nested workspace excluded from the root one +# (D2 of docs/superpowers/specs/2026-08-27-monorepo-migration-design.md). Nothing +# in rust.yml reaches it, so it needs its own jobs. Every step runs from solana/, +# where its own Cargo.lock and rust-toolchain.toml apply. +name: solana + +on: + push: + branches: [main, 'hotfix/**'] + pull_request: + paths: + - 'solana/**' + - '.github/workflows/solana.yml' + +env: + SOLANA_CLI: v3.0.12 + +defaults: + run: + working-directory: solana + +jobs: + solana-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: solana + - run: make lint + + solana-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: solana + - run: make test-lib + + solana-doc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: solana + - run: make doc + + solana-test-sbf: + runs-on: ubuntu-latest + strategy: + matrix: + network: [mainnet-beta, development] + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: solana + - name: Solana toolchain + run: | + sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" + echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH + - run: NETWORK=${{ matrix.network }} make test-sbf + + # The gate that makes holding this tree out of the root workspace worth it: the + # deployed artifacts still hash to what programs/sha256sums_*.txt records. + solana-verify-build: + runs-on: ubuntu-latest + strategy: + matrix: + network: [mainnet-beta, development] + steps: + - uses: actions/checkout@v4 + - run: NETWORK=${{ matrix.network }} make build-checked-artifacts diff --git a/CHANGELOG.md b/CHANGELOG.md index 54cdf66ab7..0c7ea54705 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,15 @@ All notable changes to this project will be documented in this file. - shreds-e2e pins one heavy test to its own shard instead of three. `TestE2E_MultiUserInstantAllocationAndWithdrawal` and `TestE2E_DeviceScale` no longer exist in doublezero-shreds, so the pin validation failed every run and the matrix was never built. Only `TestE2E_FeedSubscriptionOracleExpiryTeardown` stays pinned, leaving 1 pinned + 2 round-robin shards. Dropping `shard-e2e (shard 4)` and `shard-e2e (shard 5)` from the required status checks in the main ruleset is a separate, manual step — until it happens those contexts are required but never reported. - `.cursor/BUGBOT.md` and `.github/copilot-instructions.md` now tell Bugbot and Copilot to read the nearest sibling, flag a path that skips a zero or a duplicate, and assert a specific error and the exact log line at the expected index. Onchain checks apply only when the repository has onchain code. The eight path-scoped files under `.github/instructions/` are removed so Copilot reads only the repo-wide file. (#4247) - The e2e shard step passes `GITHUB_TOKEN` to `go test`. `TestE2E_BackwardCompatibility` asks the GitHub API which client releases exist, and unauthenticated that is 60 requests an hour shared across every runner on the same address, so shard 1 failed on a 403 rate limit before reaching a devnet. The test already read the variable; nothing was setting it. + - The 11 workflows imported with `doublezero-offchain` and `doublezero-solana` are folded into this repo's set, and the imported `.github` directories are deleted. GitHub only reads workflows at the repository root, so those files did nothing where they sat. + - Five release workflows come across, one per component: contributor-rewards, doublezero-solana-cli, the offchain sentinel, solana-validator-debt and the offchain scheduler. They run goreleaser from the repository root rather than from `offchain/`, because the merged workspace writes to the root `target/` directory, which is where goreleaser's rust builder looks for the binary it packages. Every path inside the five goreleaser configs is rewritten to match, and `release.github.name` moves from `doublezero-offchain` to `doublezero`, so the releases land on this repository. The three secrets they need are already configured here. + - New `solana` workflow. The `solana/` tree is excluded from the root workspace, so `rust.yml` never reaches it and it would otherwise have no CI here at all. It runs that tree's own lint, library tests, docs and SBF tests, plus the checksum gate that rebuilds both networks' artifacts and verifies them against `solana/programs/sha256sums_*.txt`, path-scoped to `solana/**`. + - New `elixir` workflow for the offchain scheduler, path-scoped to `offchain/scheduler/**`: format check, compile with warnings as errors, credo and tests. Those checks only ran on a release tag before, so a pull request that broke the scheduler passed. + - New `offchain-local-validator` workflow carrying the two live fork tests. It runs from the repository root, since the shell scripts resolve their binaries as `target/debug/`. It is path-scoped to the trees that can affect it rather than running on every pull request as it did before. The two jobs that were disabled or commented out upstream are not carried over. + - `rust-cli-static` now asserts static musl linkage for all five released CLIs instead of the client alone, which is what offchain's `rust-musl-static` job did for its four. Each package is built on its own so feature unification matches the release. + - `changelog-reminder` keeps the per-crate changelog check that offchain enforced, with paths moved under `offchain/`. A change to one of those 13 subprojects needs both the root `CHANGELOG.md` and that subproject's own. + - Offchain's `ci.yml` is dropped: `make rust-build`, `make rust-lint` and `make rust-test` cover those crates now that they are workspace members. Its `just ci` coverage floor (`cargo llvm-cov --fail-under-lines 25`) is not carried over, since the workspace it measured no longer exists. `offchain/Justfile` keeps its Elixir recipes and loses the Rust ones, which would otherwise act on the whole workspace with a different fmt and clippy line than CI uses. + - `offchain/scripts/release-rc.sh` resolves the repository root two levels up rather than one, so local release candidates build against the merged workspace. - E2E/QA - New e2e coverage for RFC-27 proof enforcement with `require-ip-ownership-proof` set: the working path still reaches BGP, a client with no verifier to reach is rejected with `IpOwnershipProofRequired`, a wildcard (`0.0.0.0`) access pass binds `client_ip` only when a proof is attached, the sentinel authority stays exempt so the oracle path keeps working, and `connect` refuses a proof whose address disagrees with the one it provisions. (#4243) - Remove `TestQA_MulticastSettlement`. It funded a seat through `doublezero-solana shreds pay`, which is going away. The agent seat-pay RPC now returns Unimplemented if something still calls it. Unused settlement helpers go with the test. (#4248) diff --git a/offchain/.github/pull_request_template.md b/offchain/.github/pull_request_template.md deleted file mode 100644 index ded953873b..0000000000 --- a/offchain/.github/pull_request_template.md +++ /dev/null @@ -1,9 +0,0 @@ -## Summary of Changes -* Describe what changed in the PR -* Explain why the change is necessary -* Note any metrics that were exposed in this PR -* Is there supporting documentation or external resources that explain the change? -* Is a CHANGELOG.md update needed? - -## Testing Verification -* Show evidence of testing the change diff --git a/offchain/.github/workflows/changelog-reminder.yml b/offchain/.github/workflows/changelog-reminder.yml deleted file mode 100644 index 7d0518e31d..0000000000 --- a/offchain/.github/workflows/changelog-reminder.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: changelog-reminder - -on: - pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] - -jobs: - changelog: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: check-changelogs - run: | - set -euo pipefail - - if echo "${{ toJson(github.event.pull_request.labels.*.name) }}" | grep -q "skip-changelog"; then - echo "skip-changelog label present; passing without requiring changelog updates." - exit 0 - fi - - base_branch="${GITHUB_BASE_REF:-}" - - if [ -z "$base_branch" ]; then - default_remote_head=$(git symbolic-ref --short refs/remotes/origin/HEAD) - base_branch="${default_remote_head#origin/}" - fi - - git fetch origin "$base_branch" - - base_ref="origin/$base_branch" - head_ref="HEAD" - - changed_files=$(git diff --name-only "$base_ref"..."$head_ref") - - projects=( - "crates/contributor-rewards::crates/contributor-rewards/CHANGELOG.md" - "crates/scheduled-command::crates/scheduled-command/CHANGELOG.md" - "crates/sentinel::crates/sentinel/CHANGELOG.md" - "crates/slack-notifier::crates/slack-notifier/CHANGELOG.md" - "crates/solana-admin-cli/passport::crates/solana-admin-cli/passport/CHANGELOG.md" - "crates/solana-admin-cli/revenue-distribution::crates/solana-admin-cli/revenue-distribution/CHANGELOG.md" - "crates/solana-admin-cli/sol-conversion::crates/solana-admin-cli/sol-conversion/CHANGELOG.md" - "crates/solana-cli::crates/solana-cli/CHANGELOG.md" - "crates/solana-client-tools::crates/solana-client-tools/CHANGELOG.md" - "crates/solana-fork::crates/solana-fork/CHANGELOG.md" - "crates/solana-interface/sol-conversion::crates/solana-interface/sol-conversion/CHANGELOG.md" - "crates/validator-debt::crates/validator-debt/CHANGELOG.md" - "scheduler::scheduler/CHANGELOG.md" - ) - - missing=() - - for entry in "${projects[@]}"; do - subdir=${entry%%::*} - changelog=${entry##*::} - - if echo "$changed_files" | grep -q "^${subdir}/"; then - if ! echo "$changed_files" | grep -q "^${changelog}$"; then - missing+=("$subdir (expected ${changelog})") - fi - fi - done - - if [ ${#missing[@]} -eq 0 ]; then - exit 0 - fi - - echo "The following subprojects changed but their changelog was not updated:" - for m in "${missing[@]}"; do - echo " - $m" - done - - echo - echo "If this change intentionally does not require a changelog, add the 'skip-changelog' label to the PR." - exit 1 diff --git a/offchain/.github/workflows/ci.yml b/offchain/.github/workflows/ci.yml deleted file mode 100644 index 592fdebe8b..0000000000 --- a/offchain/.github/workflows/ci.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: CI - -on: - pull_request: - branches: [main] - push: - branches: [main] - tags: ["*"] - -jobs: - rust: - runs-on: ubuntu-latest - steps: - - name: Setup | Cancel previous runs - uses: styfle/cancel-workflow-action@0.12.1 - - - name: Setup | Checkout - uses: actions/checkout@v4 - - - name: Check disk space before cleanup - run: df -h - - - name: Free disk space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force - sudo docker builder prune -a --force - - - name: Check disk space after cleanup - run: df -h - - - name: Setup | Apt packages - run: sudo apt-get update - - - name: Setup | Rust toolchain from rust-toolchain.toml - run: | - rustup toolchain install - rustup component add llvm-tools-preview - - - name: Setup | Rust cache - uses: Swatinem/rust-cache@v2 - - - name: Setup | Install cargo-nextest - uses: taiki-e/install-action@nextest - - - name: Setup | Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - - name: Setup | Just - uses: taiki-e/install-action@just - - - name: Test | CI Pipeline - run: just ci - - # All four release components ship as static musl binaries so they load on - # any Linux regardless of the host glibc version. Build each package - # individually (as goreleaser does, so feature unification matches the - # release) and assert every binary is fully static. - rust-musl-static: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup | Rust toolchain from rust-toolchain.toml - run: rustup toolchain install - - uses: Swatinem/rust-cache@v2 - - name: Install musl toolchain - run: sudo apt-get update && sudo apt-get install -y musl-tools cmake - - name: Build release packages for musl and assert static linkage - env: - CC_x86_64_unknown_linux_musl: musl-gcc - CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: musl-gcc - run: | - for entry in \ - doublezero-solana-cli:doublezero-solana \ - doublezero-contributor-rewards:doublezero-contributor-rewards \ - doublezero-ledger-sentinel:doublezero-sentinel \ - doublezero-solana-validator-debt:doublezero-solana-validator-debt - do - package="${entry%%:*}" - binary="target/x86_64-unknown-linux-musl/release/${entry##*:}" - cargo build --release --package "$package" --target x86_64-unknown-linux-musl - file "$binary" - if ! file "$binary" | grep -qE "statically linked|static-pie linked"; then - echo "::error::$binary is not statically linked" - exit 1 - fi - done diff --git a/offchain/.github/workflows/local-validator.yml b/offchain/.github/workflows/local-validator.yml deleted file mode 100644 index a023377241..0000000000 --- a/offchain/.github/workflows/local-validator.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: local-validator -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - SOLANA_CLI: v3.0.12 - -jobs: - test-doublezero-solana-fork: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Check disk space before cleanup - run: df -h - - - name: Free disk space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force - sudo docker builder prune -a --force - - - name: Check disk space after cleanup - run: df -h - - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Solana toolchain - run: | - sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" - echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - - name: Generate ~/.config/solana/id.json - run: solana-keygen new --silent --no-bip39-passphrase - - name: Generate manager keypair for synthetic ValidatorClientRewards - run: solana-keygen new --silent --no-bip39-passphrase -o manager_keypair.json - - name: Start Solana mainnet-beta fork in background - run: | - MANAGER_PUBKEY=$(solana address -k manager_keypair.json) - cargo run --bin doublezero-solana-fork -- -um --reset --synthetic-validator-client-rewards-manager "$MANAGER_PUBKEY" > /dev/null 2>&1 & - - name: Build `doublezero-solana` - run: cargo build --bin doublezero-solana - - name: Run `doublezero-solana` tests - run: bash sh/test_doublezero_solana_fork.sh - - test-doublezero-solana-validator-debt-fork-current: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Check disk space before cleanup - run: df -h - - - name: Free disk space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force - sudo docker builder prune -a --force - - - name: Check disk space after cleanup - run: df -h - - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Solana toolchain - run: | - sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" - echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - - name: Generate ~/.config/solana/id.json - run: solana-keygen new --silent --no-bip39-passphrase - - name: Start Solana mainnet-beta fork in background - run: cargo run --bin doublezero-solana-fork -- -um --reset --god-mode > /dev/null 2>&1 & - - name: Build `doublezero-revenue-distribution-admin` and `doublezero-solana-validator-debt` - run: cargo build --bin doublezero-revenue-distribution-admin --bin doublezero-solana-validator-debt --bin doublezero-solana - - name: Run `doublezero-solana-validator-debt` tests - run: bash sh/test_validator_debt_fork.sh - -# Commenting out as we may bring this back in the future - # test-full-debt-flow: - # runs-on: ubuntu-latest - # needs: [test-doublezero-solana-validator-debt-fork-current] - # steps: - # - uses: actions/checkout@v4 - - # - name: Check disk space before cleanup - # run: df -h - - # - name: Free disk space - # run: | - # sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - # sudo docker image prune --all --force - # sudo docker builder prune -a --force - - # - name: Check disk space after cleanup - # run: df -h - - # - uses: dtolnay/rust-toolchain@stable - # - uses: Swatinem/rust-cache@v2 - # - name: Solana toolchain - # run: | - # sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" - # echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - # - name: Generate ~/.config/solana/id.json - # run: solana-keygen new --silent --no-bip39-passphrase - # - name: Build binaries - # run: | - # cargo build --bin doublezero-solana \ - # --bin doublezero-solana-validator-debt \ - # --bin doublezero-revenue-distribution-admin \ - # --bin doublezero-solana-fork - # - name: Start Solana mainnet-beta fork in background - # run: cargo run --bin doublezero-solana-fork -- -um --reset --god-mode > /dev/null 2>&1 & - # - name: Run full debt flow test - # run: CI=1 SKIP_FORK_START=1 bash sh/test_full_debt_flow.sh - # timeout-minutes: 10 - # - name: Run Rust integration tests - # run: cargo test --features integration -p doublezero-solana-validator-debt - # timeout-minutes: 5 - - test-doublezero-solana-validator-debt-fork-override-76: - if: false - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Check disk space before cleanup - run: df -h - - - name: Free disk space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force - sudo docker builder prune -a --force - - - name: Check disk space after cleanup - run: df -h - - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Solana toolchain - run: | - sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" - echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - - name: Generate ~/.config/solana/id.json - run: solana-keygen new --silent --no-bip39-passphrase - - name: Start Solana mainnet-beta fork in background - run: cargo run --bin doublezero-solana-fork -- -um --reset --god-mode --next-completed-dz-epoch-override 76 > /dev/null 2>&1 & - - name: Build `doublezero-revenue-distribution-admin` and `doublezero-solana-validator-debt` - run: cargo build --bin doublezero-revenue-distribution-admin --bin doublezero-solana-validator-debt - - name: Run `doublezero-solana-validator-debt` tests - run: bash sh/test_validator_debt_fork.sh diff --git a/offchain/Justfile b/offchain/Justfile index 50c369114b..c3066c865a 100644 --- a/offchain/Justfile +++ b/offchain/Justfile @@ -1,47 +1,13 @@ -# Export required env -export SERVICEABILITY_PROGRAM_ID := "devnet" - -# Fail on warnings -export RUSTFLAGS := "-Dwarnings" +# Elixir recipes for the offchain scheduler. The Rust recipes that used to live +# here are gone: these crates are members of the root Cargo workspace now, so +# `make rust-build`, `make rust-lint`, `make rust-test` and `make rust-fmt` at the +# repository root cover them, and running cargo from here would act on the whole +# workspace with a different fmt and clippy line than CI uses. # Default (list of commands) default: just -l -# Run fmt -fmt: - @rustup component add rustfmt - @cargo fmt --all -- --config imports_granularity=Crate,group_imports=StdExternalCrate - -# Check fmt -fmt-check: - @rustup component add rustfmt - @cargo fmt --all -- --check --config imports_granularity=Crate,group_imports=StdExternalCrate || (echo "Formatting check failed. Please run 'just fmt' to fix formatting issues." && exit 1) - -# Build (release) -build: - cargo build --release - -# Run clippy -clippy: - cargo clippy --all-features --all-targets -- -Dclippy::all - -# Run tests -test: - cargo nextest run - -# Clean -clean: - cargo clean - -# Coverage -cov: - cargo llvm-cov nextest --lcov --output-path lcov.info - -# Coverage check (fail if below threshold) -cov-check: - cargo llvm-cov nextest --fail-under-lines 25 - # Check Elixir formatting elixir-fmt-check: cd scheduler && mix format --check-formatted @@ -62,28 +28,17 @@ elixir-credo: elixir-test: cd scheduler && mix test -# Run CI pipeline -ci: - @just fmt-check - @just clippy - @just test - @just cov-check - -# Run unit tests only (fast, no external dependencies) -test-unit: - cargo nextest run - cd scheduler && mix test --cover - -# Run integration tests (requires local validator) -test-integration: - cargo test --features integration -p doublezero-solana-validator-debt +# Everything the elixir workflow runs +elixir-ci: + @just elixir-fmt-check + @just elixir-compile + @just elixir-credo + @just elixir-test -# Run end-to-end tests (starts fork, runs full lifecycle) -test-e2e: - bash sh/test_full_debt_flow.sh +# Start a Solana mainnet-beta fork and drive the CLIs against it +test-fork: + cd .. && bash offchain/sh/test_doublezero_solana_fork.sh -# Run all tests (unit + integration + e2e) -test-all: - @just test-unit - @just test-integration - @just test-e2e +# Run the full debt flow against a running fork +test-debt-flow: + cd .. && bash offchain/sh/test_full_debt_flow.sh diff --git a/offchain/release/.goreleaser.contributor-rewards.yaml b/offchain/release/.goreleaser.contributor-rewards.yaml index 68230df70a..2dc73d94ed 100644 --- a/offchain/release/.goreleaser.contributor-rewards.yaml +++ b/offchain/release/.goreleaser.contributor-rewards.yaml @@ -55,13 +55,13 @@ nfpms: release: "1" section: default contents: - - src: release/packaging/systemd/doublezero-contributor-rewards.service + - src: offchain/release/packaging/systemd/doublezero-contributor-rewards.service dst: /lib/systemd/system/doublezero-contributor-rewards.service type: config overrides: rpm: contents: - - src: release/packaging/systemd/doublezero-contributor-rewards.service + - src: offchain/release/packaging/systemd/doublezero-contributor-rewards.service dst: /usr/lib/systemd/system/doublezero-contributor-rewards.service type: config @@ -75,7 +75,7 @@ changelog: release: github: owner: malbeclabs - name: doublezero-offchain + name: doublezero draft: false replace_existing_artifacts: true diff --git a/offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml b/offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml index e3ccac9d21..36e51e9244 100644 --- a/offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml +++ b/offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml @@ -23,13 +23,13 @@ builds: pre: - cmd: echo $MIX_ENV - cmd: mix deps.clean --all - dir: scheduler + dir: offchain/scheduler - cmd: mix deps.get --only prod - dir: scheduler + dir: offchain/scheduler - cmd: mix compile --warnings-as-errors --verbose - dir: scheduler + dir: offchain/scheduler - cmd: mix release - dir: scheduler + dir: offchain/scheduler prebuilt: path: ./scheduler/_build/prod/rel/scheduler/bin/scheduler @@ -61,10 +61,10 @@ nfpms: release: 1 section: default contents: - - src: release/packaging/systemd/doublezero-offchain-scheduler.service + - src: offchain/release/packaging/systemd/doublezero-offchain-scheduler.service dst: /lib/systemd/system/doublezero-offchain-scheduler.service type: config - - src: scheduler/_build/prod/rel/scheduler + - src: offchain/scheduler/_build/prod/rel/scheduler dst: /opt/doublezero-offchain-scheduler type: tree - src: /opt/doublezero-offchain-scheduler/bin/scheduler @@ -81,7 +81,7 @@ changelog: release: github: owner: malbeclabs - name: doublezero-offchain + name: doublezero draft: false replace_existing_artifacts: true diff --git a/offchain/release/.goreleaser.doublezero-solana-cli.yaml b/offchain/release/.goreleaser.doublezero-solana-cli.yaml index 44b6fc7c91..8fccb50969 100644 --- a/offchain/release/.goreleaser.doublezero-solana-cli.yaml +++ b/offchain/release/.goreleaser.doublezero-solana-cli.yaml @@ -67,7 +67,7 @@ changelog: release: github: owner: malbeclabs - name: doublezero-offchain + name: doublezero draft: false replace_existing_artifacts: true diff --git a/offchain/release/.goreleaser.doublezero-solana-validator-debt.yaml b/offchain/release/.goreleaser.doublezero-solana-validator-debt.yaml index 125278f9ed..8bf6039fa5 100644 --- a/offchain/release/.goreleaser.doublezero-solana-validator-debt.yaml +++ b/offchain/release/.goreleaser.doublezero-solana-validator-debt.yaml @@ -55,13 +55,13 @@ nfpms: release: 1 section: default contents: - - src: release/packaging/systemd/doublezero-solana-validator-debt.service + - src: offchain/release/packaging/systemd/doublezero-solana-validator-debt.service dst: /lib/systemd/system/doublezero-solana-validator-debt.service type: config overrides: rpm: contents: - - src: release/packaging/systemd/doublezero-solana-validator-debt.service + - src: offchain/release/packaging/systemd/doublezero-solana-validator-debt.service dst: /usr/lib/systemd/system/doublezero-solana-validator-debt.service type: config @@ -75,7 +75,7 @@ changelog: release: github: owner: malbeclabs - name: doublezero-offchain + name: doublezero draft: false replace_existing_artifacts: true diff --git a/offchain/release/.goreleaser.sentinel.yaml b/offchain/release/.goreleaser.sentinel.yaml index d07fa68902..15f8019fa8 100644 --- a/offchain/release/.goreleaser.sentinel.yaml +++ b/offchain/release/.goreleaser.sentinel.yaml @@ -55,13 +55,13 @@ nfpms: release: 1 section: default contents: - - src: release/packaging/systemd/doublezero-sentinel.service + - src: offchain/release/packaging/systemd/doublezero-sentinel.service dst: /lib/systemd/system/doublezero-sentinel.service type: config overrides: rpm: contents: - - src: release/packaging/systemd/doublezero-sentinel.service + - src: offchain/release/packaging/systemd/doublezero-sentinel.service dst: /usr/lib/systemd/system/doublezero-sentinel.service type: config @@ -75,7 +75,7 @@ changelog: release: github: owner: malbeclabs - name: doublezero-offchain + name: doublezero draft: false replace_existing_artifacts: true diff --git a/offchain/scripts/release-rc.sh b/offchain/scripts/release-rc.sh index 366e3f0f29..8bede14a33 100755 --- a/offchain/scripts/release-rc.sh +++ b/offchain/scripts/release-rc.sh @@ -10,7 +10,7 @@ # ./scripts/release-rc.sh sentinel --version 0.2.6 --dry-run # # The corresponds to a goreleaser config file at: -# release/.goreleaser..yaml +# offchain/release/.goreleaser..yaml # # Requirements: docker, gh (GitHub CLI, authenticated) # Environment: GORELEASER_KEY (goreleaser pro license key) @@ -24,7 +24,10 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +# The repository root, two levels up: the offchain crates are members of the root +# Cargo workspace, so the build and its target/ directory live there, and every +# path in the goreleaser configs is relative to it. +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" # --- Formatting --- @@ -53,10 +56,10 @@ while [[ $# -gt 0 ]]; do echo "" echo "Arguments:" echo " Name of the goreleaser config (e.g. doublezero-solana-cli)" - echo " Must match: release/.goreleaser..yaml" + echo " Must match: offchain/release/.goreleaser..yaml" echo "" echo "Available configs:" - ls release/.goreleaser.*.yaml 2>/dev/null | sed 's|.*/\.goreleaser\.||; s|\.yaml$||; s|^| |' + ls offchain/release/.goreleaser.*.yaml 2>/dev/null | sed 's|.*/\.goreleaser\.||; s|\.yaml$||; s|^| |' echo "" echo "Flags:" echo " --version Base version for the RC (e.g. 0.4.2); defaults to latest RC series" @@ -86,7 +89,7 @@ command -v gh >/dev/null 2>&1 || die "gh (GitHub CLI) is required" cd "$REPO_ROOT" -GORELEASER_CONFIG="release/.goreleaser.${CONFIG_NAME}.yaml" +GORELEASER_CONFIG="offchain/release/.goreleaser.${CONFIG_NAME}.yaml" [[ -f "$GORELEASER_CONFIG" ]] || die "Config not found: $GORELEASER_CONFIG" # --- Parse goreleaser config --- @@ -154,8 +157,8 @@ if ! docker image inspect "$RELEASE_IMAGE" >/dev/null 2>&1; then echo "" docker build --platform linux/amd64 \ -t "$RELEASE_IMAGE" \ - -f release/Dockerfile.release \ - release/ + -f offchain/release/Dockerfile.release \ + offchain/release/ echo "" fi diff --git a/solana/.github/workflows/local-validator.yml b/solana/.github/workflows/local-validator.yml deleted file mode 100644 index 61f728b0c1..0000000000 --- a/solana/.github/workflows/local-validator.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: local-validator -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - SOLANA_CLI: v3.0.12 - DOUBLEZERO_OFFCHAIN_GIT_INSTALL: --git https://github.com/malbeclabs/doublezero-offchain.git --locked - -jobs: - mainnet-fork-upgrade: - # Disabled: the migrate-program-accounts instruction now takes a journal - # plus a list of distribution accounts, so the admin CLI invocation below - # no longer matches. Needs a rethink before this can run again. - if: false - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Solana toolchain - run: | - sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" - echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - - name: Generate ~/.config/solana/id.json - run: solana-keygen new --silent --no-bip39-passphrase - - name: Install `doublezero-solana-fork` CLI - run: cargo install doublezero-solana-fork-cli $DOUBLEZERO_OFFCHAIN_GIT_INSTALL - - name: Start Solana mainnet-beta fork in background - run: doublezero-solana-fork -um --reset --god-mode > /dev/null 2>&1 & - - name: Install `doublezero-revenue-distribution-admin` CLI - run: cargo install doublezero-revenue-distribution-admin-cli $DOUBLEZERO_OFFCHAIN_GIT_INSTALL - - name: Build Revenue Distribution program - run: make build-artifacts - - name: Wait for local validator to be ready - run: | - echo "Waiting for solana-test-validator to start..." - for i in {1..30}; do - if solana cluster-version -ul > /dev/null 2>&1; then - solana cluster-version -ul - break - fi - echo "Attempt $i: solana-test-validator not ready yet, waiting..." - sleep 5 - done - if ! solana cluster-version -ul > /dev/null 2>&1; then - echo "solana-test-validator failed to start within timeout" - exit 1 - fi - - name: Upgrade Revenue Distribution program - run: solana program deploy -ul --program-id dzrevZC94tBLwuHw1dyynZxaXTWyp7yocsinyEVPtt4 artifacts-mainnet-beta/doublezero_revenue_distribution.so - - name: Wait for upgrade to finalize - run: sleep 15 - - name: Perform Revenue Distribution program migration - run: doublezero-revenue-distribution-admin migrate-program-accounts -ul -v - - name: Perform Revenue Distribution program migration again - run: doublezero-revenue-distribution-admin migrate-program-accounts -ul -v diff --git a/solana/.github/workflows/rust.yml b/solana/.github/workflows/rust.yml deleted file mode 100644 index 24c42b3cef..0000000000 --- a/solana/.github/workflows/rust.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: rust -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -env: - SOLANA_CLI: v3.0.12 - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - run: make lint - - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - run: make test-lib - - doc: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - run: make doc - - test-sbf: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Solana toolchain - run: | - sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" - echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - - run: make test-sbf - - test-sbf-development: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Solana toolchain - run: | - sh -c "$(curl -sSfL https://release.anza.xyz/$SOLANA_CLI/install)" - echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH - - run: NETWORK=development make test-sbf diff --git a/solana/.github/workflows/verify-build.yml b/solana/.github/workflows/verify-build.yml deleted file mode 100644 index c5809624c6..0000000000 --- a/solana/.github/workflows/verify-build.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: verify-build -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - sha256sums: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: NETWORK=mainnet-beta make build-checked-artifacts - - run: NETWORK=development make build-checked-artifacts From ab0c61e6de651afe15f76aba8f2cce82a20a6f2a Mon Sep 17 00:00:00 2001 From: Ben Marx Date: Tue, 1 Sep 2026 17:32:37 -0700 Subject: [PATCH 2/5] release: point the scheduler prebuilt path at offchain/scheduler goreleaser runs from the repository root now, and the four mix hooks and the package contents were repointed, but the prebuilt builder's own path was not. It uses a different key and a ./ prefix, so it slipped through the rewrite. A tagged scheduler release would have run the hooks, built the release into offchain/scheduler/_build, then failed to find the binary. Audited every path-like value across the five configs. This was the only one left. The remaining relative path, /opt/doublezero-offchain-scheduler/bin/scheduler, is where the package installs on the target machine, not a build path. --- offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml b/offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml index 36e51e9244..eb7838bf76 100644 --- a/offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml +++ b/offchain/release/.goreleaser.doublezero-offchain-scheduler.yaml @@ -31,7 +31,7 @@ builds: - cmd: mix release dir: offchain/scheduler prebuilt: - path: ./scheduler/_build/prod/rel/scheduler/bin/scheduler + path: ./offchain/scheduler/_build/prod/rel/scheduler/bin/scheduler archives: - id: doublezero_offchain_scheduler_archive From af9b2cb8c85b0522393703b4135c0ed8fbc1762c Mon Sep 17 00:00:00 2001 From: Ben Marx Date: Tue, 1 Sep 2026 17:42:03 -0700 Subject: [PATCH 3/5] ci: name the package in the offchain fork-test cargo calls The ported workflow ran `cargo build --bin doublezero-solana` from the repository root, which fails here: error: manifest path `/home/runner/work/doublezero/doublezero` contains no package: The manifest is virtual, and the workspace has no members. The root workspace sets `default-members = []`, so a bare cargo command selects nothing. doublezero-offchain's workspace had no such restriction, so these commands worked there unchanged. Each call now names its package. Checked locally: all four binaries build and land in target/debug at the repository root, which is where the fork scripts look for them. --- .github/workflows/offchain.local-validator.yml | 17 +++++++++++++---- CHANGELOG.md | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/offchain.local-validator.yml b/.github/workflows/offchain.local-validator.yml index b731a67b95..bfa3789bfb 100644 --- a/.github/workflows/offchain.local-validator.yml +++ b/.github/workflows/offchain.local-validator.yml @@ -21,6 +21,11 @@ on: env: SOLANA_CLI: v3.0.12 +# Every cargo invocation names its package. The root workspace sets +# `default-members = []`, so a bare `cargo build --bin X` selects nothing and +# fails with "the workspace has no members". These commands came from a +# repository whose workspace had no such restriction. + jobs: test-doublezero-solana-fork: runs-on: ubuntu-latest @@ -46,9 +51,9 @@ jobs: - name: Start Solana mainnet-beta fork in background run: | MANAGER_PUBKEY=$(solana address -k manager_keypair.json) - cargo run --bin doublezero-solana-fork -- -um --reset --synthetic-validator-client-rewards-manager "$MANAGER_PUBKEY" > /dev/null 2>&1 & + cargo run -p doublezero-solana-fork-cli --bin doublezero-solana-fork -- -um --reset --synthetic-validator-client-rewards-manager "$MANAGER_PUBKEY" > /dev/null 2>&1 & - name: Build `doublezero-solana` - run: cargo build --bin doublezero-solana + run: cargo build -p doublezero-solana-cli --bin doublezero-solana - name: Run `doublezero-solana` tests run: bash offchain/sh/test_doublezero_solana_fork.sh @@ -72,8 +77,12 @@ jobs: - name: Generate ~/.config/solana/id.json run: solana-keygen new --silent --no-bip39-passphrase - name: Start Solana mainnet-beta fork in background - run: cargo run --bin doublezero-solana-fork -- -um --reset --god-mode > /dev/null 2>&1 & + run: cargo run -p doublezero-solana-fork-cli --bin doublezero-solana-fork -- -um --reset --god-mode > /dev/null 2>&1 & - name: Build the admin and validator-debt CLIs - run: cargo build --bin doublezero-revenue-distribution-admin --bin doublezero-solana-validator-debt --bin doublezero-solana + run: | + cargo build \ + -p doublezero-revenue-distribution-admin-cli --bin doublezero-revenue-distribution-admin \ + -p doublezero-solana-validator-debt --bin doublezero-solana-validator-debt \ + -p doublezero-solana-cli --bin doublezero-solana - name: Run `doublezero-solana-validator-debt` tests run: bash offchain/sh/test_validator_debt_fork.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c7ea54705..b647843ee4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ All notable changes to this project will be documented in this file. - Five release workflows come across, one per component: contributor-rewards, doublezero-solana-cli, the offchain sentinel, solana-validator-debt and the offchain scheduler. They run goreleaser from the repository root rather than from `offchain/`, because the merged workspace writes to the root `target/` directory, which is where goreleaser's rust builder looks for the binary it packages. Every path inside the five goreleaser configs is rewritten to match, and `release.github.name` moves from `doublezero-offchain` to `doublezero`, so the releases land on this repository. The three secrets they need are already configured here. - New `solana` workflow. The `solana/` tree is excluded from the root workspace, so `rust.yml` never reaches it and it would otherwise have no CI here at all. It runs that tree's own lint, library tests, docs and SBF tests, plus the checksum gate that rebuilds both networks' artifacts and verifies them against `solana/programs/sha256sums_*.txt`, path-scoped to `solana/**`. - New `elixir` workflow for the offchain scheduler, path-scoped to `offchain/scheduler/**`: format check, compile with warnings as errors, credo and tests. Those checks only ran on a release tag before, so a pull request that broke the scheduler passed. - - New `offchain-local-validator` workflow carrying the two live fork tests. It runs from the repository root, since the shell scripts resolve their binaries as `target/debug/`. It is path-scoped to the trees that can affect it rather than running on every pull request as it did before. The two jobs that were disabled or commented out upstream are not carried over. + - New `offchain-local-validator` workflow carrying the two live fork tests. It runs from the repository root, since the shell scripts resolve their binaries as `target/debug/`. Every cargo call names its package, because the root workspace sets `default-members = []` and a bare `cargo build --bin` selects nothing. It is path-scoped to the trees that can affect it rather than running on every pull request as it did before. The two jobs that were disabled or commented out upstream are not carried over. - `rust-cli-static` now asserts static musl linkage for all five released CLIs instead of the client alone, which is what offchain's `rust-musl-static` job did for its four. Each package is built on its own so feature unification matches the release. - `changelog-reminder` keeps the per-crate changelog check that offchain enforced, with paths moved under `offchain/`. A change to one of those 13 subprojects needs both the root `CHANGELOG.md` and that subproject's own. - Offchain's `ci.yml` is dropped: `make rust-build`, `make rust-lint` and `make rust-test` cover those crates now that they are workspace members. Its `just ci` coverage floor (`cargo llvm-cov --fail-under-lines 25`) is not carried over, since the workspace it measured no longer exists. `offchain/Justfile` keeps its Elixir recipes and loses the Rust ones, which would otherwise act on the whole workspace with a different fmt and clippy line than CI uses. From 0aadf18109cf28ccfac7c207957846b2c4df395f Mon Sep 17 00:00:00 2001 From: Ben Marx Date: Tue, 1 Sep 2026 17:57:54 -0700 Subject: [PATCH 4/5] ci: give the new workflows the same paths filter on push The three workflows added here filtered by path on pull_request but not on push, so every push to main or a hotfix branch started them whether or not it touched their trees. That is two mainnet-beta fork jobs, six solana jobs including two Docker artifact builds, and the Elixir suite, on pushes that cannot affect any of them. Each push trigger now carries the same paths list as its pull_request trigger. --- .github/workflows/elixir.yml | 3 +++ .github/workflows/offchain.local-validator.yml | 6 ++++++ .github/workflows/solana.yml | 3 +++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 4e87c7562a..09b5decaba 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -5,6 +5,9 @@ name: elixir on: push: branches: [main, 'hotfix/**'] + paths: + - 'offchain/scheduler/**' + - '.github/workflows/elixir.yml' pull_request: paths: - 'offchain/scheduler/**' diff --git a/.github/workflows/offchain.local-validator.yml b/.github/workflows/offchain.local-validator.yml index bfa3789bfb..498e3c6a89 100644 --- a/.github/workflows/offchain.local-validator.yml +++ b/.github/workflows/offchain.local-validator.yml @@ -10,6 +10,12 @@ name: offchain-local-validator on: push: branches: [main, 'hotfix/**'] + paths: + - 'offchain/**' + - 'solana/**' + - 'Cargo.lock' + - 'Cargo.toml' + - '.github/workflows/offchain.local-validator.yml' pull_request: paths: - 'offchain/**' diff --git a/.github/workflows/solana.yml b/.github/workflows/solana.yml index e5adbf08cd..90745505c5 100644 --- a/.github/workflows/solana.yml +++ b/.github/workflows/solana.yml @@ -7,6 +7,9 @@ name: solana on: push: branches: [main, 'hotfix/**'] + paths: + - 'solana/**' + - '.github/workflows/solana.yml' pull_request: paths: - 'solana/**' From a2b7d8466375cfb24d95deaa5524c31da3bbcc9b Mon Sep 17 00:00:00 2001 From: Ben Marx Date: Tue, 1 Sep 2026 18:04:59 -0700 Subject: [PATCH 5/5] ci: install the toolchain solana pins, not a toolchain named in the workflow The solana jobs used dtolnay/rust-toolchain@stable, so the toolchain came from whatever rustup resolved at the time rather than from solana/rust-toolchain.toml, which pins 1.91 with rustfmt and clippy. The root rust-toolchain.toml does not apply to that tree, and naming a version in the workflow as well would give two places to drift. Each job now runs `rustup toolchain install` with no arguments, from solana/, which installs exactly what that tree's toolchain file names, components included. This is how both imported repositories set their toolchain up. --- .github/workflows/solana.yml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/solana.yml b/.github/workflows/solana.yml index 90745505c5..0e8e7c70c5 100644 --- a/.github/workflows/solana.yml +++ b/.github/workflows/solana.yml @@ -27,7 +27,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + # Installs what solana/rust-toolchain.toml pins, 1.91 with rustfmt and + # clippy, rather than a toolchain named here. Pinning the version in two + # places would let them drift, and the root toolchain file does not apply + # to this tree. + - name: Rust toolchain from solana/rust-toolchain.toml + run: rustup toolchain install - uses: Swatinem/rust-cache@v2 with: workspaces: solana @@ -37,7 +42,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + # Installs what solana/rust-toolchain.toml pins, 1.91 with rustfmt and + # clippy, rather than a toolchain named here. Pinning the version in two + # places would let them drift, and the root toolchain file does not apply + # to this tree. + - name: Rust toolchain from solana/rust-toolchain.toml + run: rustup toolchain install - uses: Swatinem/rust-cache@v2 with: workspaces: solana @@ -47,7 +57,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + # Installs what solana/rust-toolchain.toml pins, 1.91 with rustfmt and + # clippy, rather than a toolchain named here. Pinning the version in two + # places would let them drift, and the root toolchain file does not apply + # to this tree. + - name: Rust toolchain from solana/rust-toolchain.toml + run: rustup toolchain install - uses: Swatinem/rust-cache@v2 with: workspaces: solana @@ -60,7 +75,12 @@ jobs: network: [mainnet-beta, development] steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + # Installs what solana/rust-toolchain.toml pins, 1.91 with rustfmt and + # clippy, rather than a toolchain named here. Pinning the version in two + # places would let them drift, and the root toolchain file does not apply + # to this tree. + - name: Rust toolchain from solana/rust-toolchain.toml + run: rustup toolchain install - uses: Swatinem/rust-cache@v2 with: workspaces: solana